OpenBLAS/kernel/ia64/gemv_t.S

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#define P	4096
#define SP	r12

#define M	r32
#define N	r33
#define A	r36
#define LDA	r37
#define X	r38
#define INCX	r39
#define Y	r34
#define INCY	r35
#define BUFFER	r11

#define MIN_M	r14
#define I	r15
#define J	r16
#define IS	r17
#define AO1	r18
#define AO2	r19
#define AO3	r20
#define AO4	r21
#define AO5	r22
#define AO6	r23
#define AO7	r24
#define AO8	r25
#define BO	r26
#define LDAP	r27

#define RPRE1	loc0
#define RPRE2	loc1
#define RPRE3	loc2
#define RPRE4	loc3
#define RPRE5	loc4
#define RPRE6	loc5
#define RPRE7	loc6
#define RPRE8	loc7

#define AO21	loc8
#define AO41	loc9
#define AO61	loc10
#define AO81	loc11

#define PREB	r8
#define WPRE	r9
#define OFFSET	PREB
#define CO	r10

#define ARLC	r29
#define PR	r30
#define ARPFS	r31

#ifdef DOUBLE
#define RPREFETCH	(16 * 3 +  8)
#else
#define RPREFETCH	(16 * 3 + 16)
#endif
#define PREFETCH	lfetch.nt1

#define ALPHA	f6

	PROLOGUE
	.prologue
	PROFCODE
	{ .mmi
	.save	ar.pfs, ARPFS
	alloc	ARPFS = ar.pfs, 8, 16, 8, 0
	setf.sig f11 = LDA
	mov	ARLC  = ar.lc
	}
	{ .mmi
	adds	r15 = 24, SP
	adds	r16 = 32, SP
	adds	r14 = 16, SP
	}
	;;
	{ .mmi
	setf.sig f10 = N
	ld8	Y      = [r14]
	mov	PR = pr
	}
	{ .mmi
	ld8	INCY   = [r15]
	adds	r8 = -8 * 16, SP
	adds	r9 = -7 * 16, SP
	}
	;;
	{ .mmi
	stf.spill  [r8] = f16, 32
	stf.spill  [r9] = f17, 32
	adds	SP = -8 * 16, SP
	}
	;;
	{ .mmf
	stf.spill  [r8] = f18, 32
	stf.spill  [r9] = f19, 32
	mov	ALPHA = f8
	}
	;;
	{ .mmi
	stf.spill  [r8] = f20, 32
	stf.spill  [r9] = f21, 32
	mov	IS = 0
	}
	;;
	{ .mmf
	stf.spill  [r8] = f22
	stf.spill  [r9] = f23
	xmpy.l f10 = f10, f11
	}
	.body
	;;
	;;
	{ .mmi
	ld8	BUFFER = [r16]
	cmp.ge	p7, p0 = r0, M
	cmp.ge	p6, p0 = r0, N
	}
	;;
	{ .mmi
	shladd	INCX = INCX, BASE_SHIFT, r0
	shladd	LDA  = LDA, BASE_SHIFT, r0
	shladd	INCY = INCY, BASE_SHIFT, r0
	}
	;;
	{ .mmi
	getf.sig LDAP = f10
	mov	r2 = P
	tbit.nz	p8, p0 = A,   BASE_SHIFT
	}
	{ .mmi
	nop	__LINE__
	nop	__LINE__
	tbit.nz	p9, p0 = LDA, BASE_SHIFT
	}
	;;
	{ .mbb
	sub	LDAP = r2, LDAP
	(p7) br.cond.dpnt .L999
	(p6) br.cond.dpnt .L999
	}
	.align 16
	;;

.LIs_loop:
	{ .mmi
	sub	MIN_M = M, IS
	(p8) LDFD f32 = [X],  INCX
	mov	pr.rot= 0
	}
	{ .mmi
	mov	AO1 = BUFFER
	adds	AO2 = 4 * SIZE, BUFFER
	}
	;;
	cmp.le	p6, p0 = r2, MIN_M
	;;
	(p6) mov MIN_M = P
	;;
	(p8) adds MIN_M = -1, MIN_M
	;;
	{ .mmi
	shladd	OFFSET = INCX, 2, INCX
	shladd	BO  = INCX, 2, X
	shr	I = MIN_M, 3
	}
	;;
	{ .mmi
	adds I = -1, I
	cmp.eq	p16, p0 = r0, r0
	mov	ar.ec= 5
	}
	;;
	{ .mmi
	(p8) STFD [AO1] = f32, 2 * SIZE
	(p8) adds	AO2 = 6 * SIZE, BUFFER
	mov	ar.lc = I
	}
	{ .mib
	cmp.gt	p6, p0 = 0, I
	tbit.nz	p13, p0 = MIN_M, 2
	(p6) br.cond.dpnt .L05
	}
	;;
	.align 16

.L01:
	(p20) STFD [AO1] = f36,  SIZE
	(p20) STFD [AO2] = f56,  SIZE
	(p16) LDFD f32 = [X],  INCX
	(p16) LDFD f52 = [BO], INCX
	;;
	(p20) STFD [AO1] = f41,  SIZE
	(p20) STFD [AO2] = f61,  SIZE
	(p16) LDFD f37 = [X],  INCX
	(p16) LDFD f57 = [BO], INCX
	;;
	(p20) STFD [AO1] = f46,  SIZE
	(p20) STFD [AO2] = f66,  SIZE
	(p16) LDFD f42 = [X],  INCX
	(p16) LDFD f62 = [BO], INCX
	;;
	(p20) STFD [AO1] = f51,  5 * SIZE
	(p20) STFD [AO2] = f71,  5 * SIZE
	(p16) LDFD f47 = [X],  OFFSET
	(p16) LDFD f67 = [BO], OFFSET
	br.ctop.sptk.few .L01
	;;
	.align 16

.L05:
	(p13) LDFD f32 = [X],  INCX
	tbit.nz	p14, p0 = MIN_M, 1
	;;
	(p13) LDFD f33 = [X],  INCX
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p13) LDFD f34 = [X],  INCX
	;;
	(p13) LDFD f35 = [X],  INCX
	;;
	(p14) LDFD f36 = [X],  INCX
	;;
	(p13) STFD [AO1] = f32, SIZE
	(p14) LDFD f37 = [X],  INCX
	;;
	(p13) STFD [AO1] = f33, SIZE
	(p15) LDFD f38 = [X],  INCX
	;;
	(p13) STFD [AO1] = f34, SIZE
	;;
	(p13) STFD [AO1] = f35, SIZE
	;;
	(p14) STFD [AO1] = f36, SIZE
	;;
	(p14) STFD [AO1] = f37, SIZE
	;;
	(p15) STFD [AO1] = f38, SIZE
	(p9) br.cond.dpnt .L100
	;;
	.align 16

.L10:
	{ .mmi
	mov	CO  = Y
	nop	__LINE__
	shr	J   = N, 3
	}
	;;
	{ .mib
	nop	__LINE__
	cmp.eq	p6, p0 = r0, J
	(p6) br.cond.dpnt .L20
	}
	;;
	.align 16

.L11:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	shr	I = MIN_M, 4
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f14 = f0
	}
	;;
	{ .mmf
	shladd	AO5 = LDA, 1, AO3
	shladd	AO6 = LDA, 1, AO4
	mov	f16 = f0
	}
	{ .mmf
	(p8) LDFD f34 = [AO3], SIZE
	(p8) LDFD f35 = [AO4], SIZE
	mov	f18 = f0
	}
	;;
	{ .mmf
	shladd	AO7 = LDA, 1, AO5
	shladd	AO8 = LDA, 1, AO6
	mov	f20 = f0
	}
	{ .mmf
	(p8) LDFD f36 = [AO5], SIZE
	(p8) LDFD f37 = [AO6], SIZE
	mov	f22 = f0
	}
	;;
	{ .mfi
	(p8) LDFD f38 = [AO7], SIZE
	mov	f9  = f0
	mov	ar.ec= 2
	}
	{ .mmf
	(p8) LDFD f39 = [AO8], SIZE
	mov	BO  = BUFFER
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	shladd	A   = LDA, 3, A
	cmp.eq	p16, p0 = r0, r0
	mov	f15 = f0
	}
	;;
	{ .mmf
	add	I = I, I
	nop	__LINE__
	mov	f17 = f0
	}
	{ .mmf
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	mov	f19 = f0
	}
	;;
	{ .mmf
	adds	I = -1, I
	nop	__LINE__
	mov	f21 = f0
	}
	{ .mmf
	adds	RPRE3  = RPREFETCH * SIZE, AO3
	adds	RPRE4  = (RPREFETCH + 8) * SIZE, AO4
	mov	f23 = f0
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p8) FMPY	f8  = f40, f32
	}
	{ .mmf
	adds	RPRE5  = RPREFETCH * SIZE, AO5
	adds	RPRE6  = (RPREFETCH + 8) * SIZE, AO6
	(p8) FMPY	f10 = f40, f33
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p8) FMPY	f12 = f40, f34
	}
	{ .mmf
	adds	RPRE7  = RPREFETCH * SIZE, AO7
	adds	RPRE8  = (RPREFETCH + 8) * SIZE, AO8
	(p8) FMPY	f14 = f40, f35
	}
	;;
	{ .mfi
	nop	__LINE__
	(p8) FMPY	f16 = f40, f36
	mov	ar.lc = I
	}
	{ .mmf
	adds	WPRE = 8 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f18 = f40, f37
	}
	;;
	{ .mmf
	lfetch.excl.nt1	[WPRE]
	nop	__LINE__
	(p8) FMPY	f20 = f40, f38
	}
	{ .mfb
	nop	__LINE__
	(p8) FMPY	f22 = f40, f39
	(p6) br.cond.dpnt .L15
	}
	;;
	.align 16

.L12:
	{ .mfi
	(p17) LDFPD	f95, f96 = [AO8], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	{ .mfi
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f9  = f105, f34, f9
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	nop	__LINE__
	}
	{ .mfi
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p17) FMA	f11 = f105, f36, f11
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f34, f35 = [AO2], 2 * SIZE
	(p17) FMA	f12 = f104, f37, f12
	nop	__LINE__
	}
	{ .mfi
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p17) FMA	f13 = f105, f38, f13
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f36, f37 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f104, f39, f14
	nop	__LINE__
	}
	{ .mfi
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p17) FMA	f15 = f105, f40, f15
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f38, f39 = [AO4], 2 * SIZE
	(p17) FMA	f16 = f104, f41, f16
	nop	__LINE__
	}
	{ .mfi
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p17) FMA	f17 = f105, f42, f17
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f40, f41 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f104, f43, f18
	nop	__LINE__
	}
	{ .mfi
	(p14) PREFETCH [RPRE5], 16 * SIZE
	(p17) FMA	f19 = f105, f44, f19
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f42, f43 = [AO6], 2 * SIZE
	(p17) FMA	f20 = f104, f45, f20
	nop	__LINE__
	}
	{ .mfi
	(p15) PREFETCH [RPRE6], 16 * SIZE
	(p17) FMA	f21 = f105, f46, f21
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f44, f45 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f104, f47, f22
	nop	__LINE__
	}
	{ .mfi
	(p14) PREFETCH [RPRE7], 16 * SIZE
	(p17) FMA	f23 = f105, f48, f23
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f46, f47 = [AO8], 2 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	nop	__LINE__
	}
	{ .mfi
	(p15) PREFETCH [RPRE8], 16 * SIZE
	(p17) FMA	f9  = f107, f50, f9
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	nop	__LINE__
	}
	{ .mfi
	(p14) PREFETCH [PREB], 16 * SIZE
	(p17) FMA	f11 = f107, f52, f11
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f50, f51 = [AO2], 2 * SIZE
	(p17) FMA	f12 = f106, f53, f12
	nop	__LINE__
	}
	{ .mfi
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f13 = f107, f54, f13
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f52, f53 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f106, f55, f14
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f15 = f107, f56, f15
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f54, f55 = [AO4], 2 * SIZE
	(p17) FMA	f16 = f106, f57, f16
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f17 = f107, f58, f17
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f56, f57 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f106, f59, f18
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f19 = f107, f60, f19
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f58, f59 = [AO6], 2 * SIZE
	(p17) FMA	f20 = f106, f61, f20
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f21 = f107, f62, f21
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f60, f61 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f106, f63, f22
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f23 = f107, f64, f23
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f62, f63 = [AO8], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	nop	__LINE__
	}
	{ .mfi
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p17) FMA	f9  = f109, f66, f9
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f108, f67, f10
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f66, f67 = [AO2], 2 * SIZE
	(p17) FMA	f12 = f108, f69, f12
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f13 = f109, f70, f13
 	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f68, f69 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f108, f71, f14
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f15 = f109, f72, f15
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f70, f71 = [AO4], 2 * SIZE
	(p17) FMA	f16 = f108, f73, f16
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f17 = f109, f74, f17
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f72, f73 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f108, f75, f18
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f19 = f109, f76, f19
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f74, f75 = [AO6], 2 * SIZE
	(p17) FMA	f20 = f108, f77, f20
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f21 = f109, f78, f21
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f76, f77 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f108, f79, f22
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f23 = f109, f80, f23
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	nop	__LINE__
	}
	{ .mfi
	(p16) LDFPD	f78, f79 = [AO8], 2 * SIZE
	(p17) FMA	f9  = f111, f82, f9
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f110, f83, f10
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f11 = f111, f84, f11
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f82, f83 = [AO2], 2 * SIZE
	(p17) FMA	f12 = f110, f85, f12
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f13 = f111, f86, f13
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f84, f85 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f110, f87, f14
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f15 = f111, f88, f15
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f86, f87 = [AO4], 2 * SIZE
	(p17) FMA	f16 = f110, f89, f16
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f17 = f111, f90, f17
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f88, f89 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f110, f91, f18
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f19 = f111, f92, f19
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f90, f91 = [AO6], 2 * SIZE
	(p17) FMA	f20 = f110, f93, f20
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f21 = f111, f94, f21
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f92, f93 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f110, f95, f22
	nop	__LINE__
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f23 = f111, f96, f23
	br.ctop.sptk.few .L12
	}
	;;
	.align 16

.L15:
	and	I = 15, MIN_M
	mov	pr.rot= 0
	;;
	cmp.eq	p6,  p0 = 0, I
	cmp.eq	p16, p15 = r0, r0
	;;
	adds	I = 1, I
	;;
	shr	I = I, 1
	;;
	adds	I = -1, I
	;;
	mov	ar.lc = I
	mov	ar.ec= 3
	and	I = 15, MIN_M
	(p6) br.cond.dpnt .L18
	;;
	.align 16

.L16:
	{ .mfi
	(p16) LDFPD	f104, f107 = [BO], 2 * SIZE
	(p18) FMA	f8  = f106, f34, f8
	nop	__LINE__
	}
	{ .mfi
	(p16) LDFPD	f32,  f35  = [AO1], 2 * SIZE
	(p15) FMA	f9  = f109, f37, f9
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f38,  f41  = [AO2], 2 * SIZE
	(p18) FMA	f10 = f106, f40, f10
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f11 = f109, f43, f11
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f44,  f47  = [AO3], 2 * SIZE
	(p18) FMA	f12 = f106, f46, f12
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f13 = f109, f49, f13
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f50,  f53  = [AO4], 2 * SIZE
	(p18) FMA	f14 = f106, f52, f14
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f15 = f109, f55, f15
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f56,  f59  = [AO5], 2 * SIZE
	(p18) FMA	f16 = f106, f58, f16
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f17 = f109, f61, f17
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f62,  f65  = [AO6], 2 * SIZE
	(p18) FMA	f18 = f106, f64, f18
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f19 = f109, f67, f19
	(p17) adds	I = -2, I
	}
	;;
	{ .mfi
	(p16) LDFPD	f68,  f71  = [AO7], 2 * SIZE
	(p18) FMA	f20 = f106, f70, f20
	nop	__LINE__
	}
	{ .mfi
	nop	__LINE__
	(p15) FMA	f21 = f109, f73, f21
	nop	__LINE__
	}
	;;
	{ .mfi
	(p16) LDFPD	f74,  f77  = [AO8], 2 * SIZE
	(p15) FMA	f23 = f109, f79, f23
	(p17) cmp.ne.unc p15, p0 = -1, I
	}
	{ .mfb
	nop	__LINE__
	(p18) FMA	f22 = f106, f76, f22
	br.ctop.sptk.few .L16
	}
	;;

.L18:
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	FADD	f8  = f8,  f9
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	LDFD	f34 = [CO], INCY
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	;;
	{ .mmf
	LDFD	f35 = [CO], INCY
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	LDFD	f36 = [CO], INCY
	nop	__LINE__
	FADD	f16 = f16, f17
	}
	;;
	{ .mmf
	LDFD	f37 = [CO], INCY
	nop	__LINE__
	FADD	f18 = f18, f19
	}
	;;
	{ .mmf
	LDFD	f38 = [CO], INCY
	nop	__LINE__
	FADD	f20 = f20, f21
	}
	;;
	{ .mmf
	LDFD	f39 = [CO], INCY
	nop	__LINE__
	FADD	f22 = f22, f23
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f34 = ALPHA, f12, f34
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f35 = ALPHA, f14, f35
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	FMA	f36 = ALPHA, f16, f36
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	add	AO1 = AO1, INCY
	FMA	f37 = ALPHA, f18, f37
	}
	;;
	{ .mmf
	STFD [AO1] = f34
	add	AO1 = AO1, INCY
	FMA	f38 = ALPHA, f20, f38
	}
	;;
	{ .mmf
	STFD [AO1] = f35
	add	AO1 = AO1, INCY
	FMA	f39 = ALPHA, f22, f39
	}
	;;
	{ .mmi
	STFD [AO1] = f36
	add	AO1 = AO1, INCY
	adds J = -1, J
	}
	;;
	{ .mmi
	STFD [AO1] = f37
	add	AO1 = AO1, INCY
	nop	__LINE__
	}
	;;
	{ .mmi
	STFD [AO1] = f38
	add	AO1 = AO1, INCY
	cmp4.lt p6, p0 = 0, J
	}
	;;
	{ .mib
	STFD [AO1] = f39
	add	AO1 = AO1, INCY
	(p6) br.cond.dptk .L11
	}
	;;
	.align 16

.L20:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 2
	}
	;;
	{ .mfi
	shladd	AO3 = LDA, 1, A
	mov	f12 = f0
	shr	I = MIN_M, 4
	}
	{ .mfb
	shladd	AO4 = LDA, 1, AO2
	mov	f14 = f0
	(p6) br.cond.dpnt .L30
	}
	;;
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f9  = f0
	}
	{ .mmf
	mov	BO  = BUFFER
	shladd	A   = LDA, 2, A
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	(p8) LDFD f34 = [AO3], SIZE
	(p8) LDFD f35 = [AO4], SIZE
	mov	f15 = f0
	}
	;;
	{ .mmi
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	mov	ar.ec= 2
	}
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	add	I = I, I
	}
	;;
	{ .mmf
	adds	WPRE =  4 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f8  = f40, f32
	}
	{ .mmf
	adds	RPRE3  = RPREFETCH * SIZE, AO3
	adds	I = -1, I
	(p8) FMPY	f10 = f40, f33
	}
	;;
	{ .mfi
	lfetch.excl.nt1	[WPRE]
	(p8) FMPY	f12 = f40, f34
	mov	ar.lc = I
	}
	{ .mfb
	adds	RPRE4  = (RPREFETCH + 8) * SIZE, AO4
	(p8) FMPY	f14 = f40, f35
	(p6) br.cond.dpnt .L25
	}
	;;
	.align 16

.L22:
	{ .mmf
	(p17) LDFPD	f87, f88 = [AO4], 2 * SIZE
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFPD	f34, f35 = [AO2], 2 * SIZE
	(p17) FMA	f12 = f104, f37, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f105, f38, f13
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFPD	f36, f37 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f104, f39, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f105, f40, f15
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFPD	f38, f39 = [AO4], 2 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f50, f51 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f12 = f106, f53, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f107, f54, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f52, f53 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f106, f55, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f107, f56, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f54, f55 = [AO4], 2 * SIZE
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f66, f67 = [AO2], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f12 = f108, f69, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f109, f70, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f68, f69 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f108, f71, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f109, f72, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f70, f71 = [AO4], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f111, f84, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f82, f83 = [AO2], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f12 = f110, f85, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f111, f86, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f84, f85 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f110, f87, f14
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f15 = f111, f88, f15
	br.ctop.sptk.few .L22
	}
	;;
	.align 16

.L25:
	and	I = 15, MIN_M
	mov	pr.rot= 0
	;;
	cmp.eq	p6,  p0 = 0, I
	cmp.eq	p16, p15 = r0, r0
	;;
	adds	I = 1, I
	;;
	shr	I = I, 1
	;;
	adds	I = -1, I
	;;
	mov	ar.lc = I
	mov	ar.ec= 3
	and	I = 15, MIN_M
	(p6) br.cond.dpnt .L28
	;;
	.align 16

.L26:
	{ .mmf
	(p16) LDFPD	f104, f107 = [BO], 2 * SIZE
	(p16) LDFPD	f32,  f35  = [AO1], 2 * SIZE
	(p18) FMA	f8  = f106, f34, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f9  = f109, f37, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f38,  f41  = [AO2], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f10 = f106, f40, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f11 = f109, f43, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f44,  f47  = [AO3], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f12 = f106, f46, f12
	}
	{ .mmf
	nop	__LINE__
	(p17) adds	I = -2, I
	(p15) FMA	f13 = f109, f49, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f50,  f53  = [AO4], 2 * SIZE
	nop	__LINE__
	(p15) FMA	f15 = f109, f55, f15
	}
	{ .mfb
	(p17) cmp.ne.unc p15, p0 = -1, I
	(p18) FMA	f14 = f106, f52, f14
	br.ctop.sptk.few .L26
	}
	;;

.L28:
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	FADD	f8  = f8,  f9
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	LDFD	f34 = [CO], INCY
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	;;
	{ .mmf
	LDFD	f35 = [CO], INCY
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f34 = ALPHA, f12, f34
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f35 = ALPHA, f14, f35
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f34
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f35
	add	AO1 = AO1, INCY
	}
	;;
	.align 16

.L30:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 1
	}
	;;
	{ .mfi
	mov	BO  = BUFFER
	mov	f12 = f0
	shr	I = MIN_M, 4
	}
	{ .mfb
	adds	WPRE =  4 * SIZE, CO
	mov	f14 = f0
	(p6) br.cond.dpnt .L40
	}
	;;
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f9  = f0
	}
	{ .mfi
	shladd	A   = LDA, 1, A
	mov	f11 = f0
	mov	ar.ec= 2
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	add	I = I, I
	mov	f15 = f0
	}
	;;
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	adds	I = -1, I
	}
	;;
	{ .mfi
	lfetch.excl.nt1	[WPRE]
	(p8) FMPY	f8  = f40, f32
	mov	ar.lc = I
	}
	{ .mfb
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f10 = f40, f33
	(p6) br.cond.dpnt .L35
	}
	;;
	.align 16

.L32:
	{ .mmf
	(p17) LDFPD	f83, f84 = [AO2], 2 * SIZE
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFPD	f34, f35 = [AO2], 2 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f50, f51 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f66, f67 = [AO2], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f11 = f111, f84, f11
	br.ctop.sptk.few .L32
	}
	;;
	.align 16

.L35:
	and	I = 15, MIN_M
	;;
	cmp.eq	p6,  p0 = 0, I
	(p6) br.cond.dpnt .L38
	;;
	tbit.nz	p12, p0 = MIN_M, 3
	tbit.nz	p13, p0 = MIN_M, 2
	tbit.nz	p14, p0 = MIN_M, 1
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p12) LDFPD	f32,  f33  = [AO1], 2 * SIZE
	(p12) LDFPD	f34,  f35  = [AO2], 2 * SIZE
	(p12) LDFPD	f100, f101 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f36,  f37  = [AO1], 2 * SIZE
	(p12) LDFPD	f38,  f39  = [AO2], 2 * SIZE
	(p12) LDFPD	f102, f103 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f40,  f41  = [AO1], 2 * SIZE
	(p12) LDFPD	f42,  f43  = [AO2], 2 * SIZE
	(p12) LDFPD	f104, f105 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f44,  f45  = [AO1], 2 * SIZE
	(p12) LDFPD	f46,  f47  = [AO2], 2 * SIZE
	(p12) LDFPD	f106, f107 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f48,  f49  = [AO1], 2 * SIZE
	(p13) LDFPD	f50,  f51  = [AO2], 2 * SIZE
	(p13) LDFPD	f108, f109 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f52,  f53  = [AO1], 2 * SIZE
	(p13) LDFPD	f54,  f55  = [AO2], 2 * SIZE
	(p13) LDFPD	f110, f111 = [BO], 2 * SIZE
	;;
	(p14) LDFPD	f56,  f57  = [AO1], 2 * SIZE
	(p14) LDFPD	f58,  f59  = [AO2], 2 * SIZE
	(p14) LDFPD	f112, f113 = [BO], 2 * SIZE
	;;
	(p15) LDFD	f60        = [AO1]
	(p15) LDFD	f61        = [AO2]
	(p15) LDFD	f114       = [BO]
	;;
	(p12) FMA	f8  = f100, f32, f8
	(p12) FMA	f9  = f101, f33, f9
	(p12) FMA	f10 = f100, f34, f10
	(p12) FMA	f11 = f101, f35, f11
	;;
	(p12) FMA	f12 = f102, f36, f12
	(p12) FMA	f13 = f103, f37, f13
	(p12) FMA	f14 = f102, f38, f14
	(p12) FMA	f15 = f103, f39, f15
	;;
	(p12) FMA	f8  = f104, f40, f8
	(p12) FMA	f9  = f105, f41, f9
	(p12) FMA	f10 = f104, f42, f10
	(p12) FMA	f11 = f105, f43, f11
	;;
	(p12) FMA	f12 = f106, f44, f12
	(p12) FMA	f13 = f107, f45, f13
	(p12) FMA	f14 = f106, f46, f14
	(p12) FMA	f15 = f107, f47, f15
	;;
	(p13) FMA	f8  = f108, f48, f8
	(p13) FMA	f9  = f109, f49, f9
	(p13) FMA	f10 = f108, f50, f10
	(p13) FMA	f11 = f109, f51, f11
	;;
	(p13) FMA	f12 = f110, f52, f12
	(p13) FMA	f13 = f111, f53, f13
	(p13) FMA	f14 = f110, f54, f14
	(p13) FMA	f15 = f111, f55, f15
	;;
	(p14) FMA	f8  = f112, f56, f8
	(p14) FMA	f9  = f113, f57, f9
	(p14) FMA	f10 = f112, f58, f10
	(p14) FMA	f11 = f113, f59, f11
	;;
	(p15) FMA	f12 = f114, f60, f12
	(p15) FMA	f14 = f114, f61, f14
	;;
.L38:
	FADD	f8  = f8,  f9
	FADD	f10 = f10, f11
	FADD	f12 = f12, f13
	FADD	f14 = f14, f15
	;;
	FADD	f8  = f8,  f12
	FADD	f10 = f10, f14
	;;
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	}
	;;
	.align 16

.L40:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	shr	I = MIN_M, 4
	}
	{ .mfi
	mov	BO  = BUFFER
	mov	f10 = f0
	tbit.z	p7, p0  = N, 0
	}
	;;
	{ .mfi
	cmp.eq	p6, p0 = 0, I
	mov	f12 = f0
	mov	pr.rot= 0
	}
	{ .mfb
	add	I = I, I
	mov	f14 = f0
	(p7) br.cond.dpnt .L99
	}
	;;
	{ .mfi
	(p8) LDFD f32 = [AO1], SIZE
	mov	f9  = f0
	mov	ar.ec= 2
	}
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	add	A   = A, LDA
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	WPRE =  1 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	mov	f13 = f0
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	adds	I = -1, I
	mov	f15 = f0
	}
	;;
	{ .mfi
	lfetch.excl.nt1	[WPRE]
	(p8) FMPY	f8  = f40, f32
	mov	ar.lc = I
	}
	{ .mmb
	nop	__LINE__
	nop	__LINE__
	(p6) br.cond.dpnt .L45
	}
	;;
	.align 16

.L42:
	{ .mmf
	(p17) LDFPD	f81, f82   = [AO1], 2 * SIZE
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p16) LDFPD	f32, f33   = [AO1], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65  = [AO1], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f9  = f111, f82, f9
	br.ctop.sptk.few .L42
	}
	;;
	.align 16

.L45:
	and	I = 15, MIN_M
	;;
	cmp.eq	p6,  p0 = 0, I
	(p6) br.cond.dpnt .L48
	;;
	tbit.nz	p12, p0 = MIN_M, 3
	tbit.nz	p13, p0 = MIN_M, 2
	tbit.nz	p14, p0 = MIN_M, 1
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p12) LDFPD	f32,  f33  = [AO1], 2 * SIZE
	(p12) LDFPD	f100, f101 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f36,  f37  = [AO1], 2 * SIZE
	(p12) LDFPD	f102, f103 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f40,  f41  = [AO1], 2 * SIZE
	(p12) LDFPD	f104, f105 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f44,  f45  = [AO1], 2 * SIZE
	(p12) LDFPD	f106, f107 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f48,  f49  = [AO1], 2 * SIZE
	(p13) LDFPD	f108, f109 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f52,  f53  = [AO1], 2 * SIZE
	(p13) LDFPD	f110, f111 = [BO], 2 * SIZE
	;;
	(p14) LDFPD	f56,  f57  = [AO1], 2 * SIZE
	(p14) LDFPD	f112, f113 = [BO], 2 * SIZE
	;;
	(p15) LDFD	f60        = [AO1]
	(p15) LDFD	f114       = [BO]
	;;
	(p12) FMA	f8  = f100, f32, f8
	(p12) FMA	f9  = f101, f33, f9
	(p12) FMA	f10 = f102, f36, f10
	(p12) FMA	f11 = f103, f37, f11
	(p12) FMA	f12 = f104, f40, f12
	(p12) FMA	f13 = f105, f41, f13
	(p12) FMA	f14 = f106, f44, f14
	(p12) FMA	f15 = f107, f45, f15
	;;
	(p13) FMA	f8  = f108, f48, f8
	(p13) FMA	f9  = f109, f49, f9
	(p13) FMA	f10 = f110, f52, f10
	(p13) FMA	f11 = f111, f53, f11
	(p14) FMA	f12 = f112, f56, f12
	(p14) FMA	f13 = f113, f57, f13
	(p15) FMA	f14 = f114, f60, f14
	;;
.L48:
	{ .mmf
	LDFD	f32 = [CO]
	nop	__LINE__
	FADD	f8  = f8,  f9
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f14
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f10
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	;;
	{ .mmf
	STFD [CO] = f32
	}
	;;
	.align 16

.L99:
	adds	IS = P, IS
	shladd	A  = LDAP, BASE_SHIFT, A
	;;
	cmp.gt	p6, p0 = M, IS
	(p6) br.cond.dptk .LIs_loop
	br   .L999
	.align 4
	;;

.L100:
	shr	J   = N, 3
	mov	CO  = Y
	;;
	cmp.eq	p6, p0 = r0, J
	(p6) br.cond.dpnt .L120
	;;
	.align 16

.L111:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	shr	I = MIN_M, 4
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f14 = f0
	}
	;;
	{ .mmf
	shladd	AO5 = LDA, 1, AO3
	shladd	AO6 = LDA, 1, AO4
	mov	f16 = f0
	}
	{ .mmf
	(p8) LDFD f34 = [AO3], SIZE
	(p8) LDFD f35 = [AO4], SIZE
	mov	f18 = f0
	}
	;;
	{ .mmf
	shladd	AO7 = LDA, 1, AO5
	shladd	AO8 = LDA, 1, AO6
	mov	f20 = f0
	}
	{ .mmf
	(p8) LDFD f36 = [AO5], SIZE
	(p8) LDFD f37 = [AO6], SIZE
	mov	f22 = f0
	}
	;;
	{ .mfi
	(p8) LDFD f38 = [AO7], SIZE
	mov	f9  = f0
	mov	ar.ec= 2
	}
	{ .mmf
	(p8) LDFD f39 = [AO8], SIZE
	mov	BO  = BUFFER
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	shladd	A   = LDA, 3, A
	cmp.eq	p16, p0 = r0, r0
	mov	f15 = f0
	}
	;;
	{ .mmf
	add	I = I, I
	nop	__LINE__
	mov	f17 = f0
	}
	{ .mmf
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	mov	f19 = f0
	}
	;;
	{ .mmf
	adds	I = -1, I
	nop	__LINE__
	mov	f21 = f0
	}
	{ .mmf
	adds	RPRE3  = RPREFETCH * SIZE, AO3
	adds	RPRE4  = (RPREFETCH + 8) * SIZE, AO4
	mov	f23 = f0
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p8) FMPY	f8  = f40, f32
	}
	{ .mmf
	adds	RPRE5  = RPREFETCH * SIZE, AO5
	adds	RPRE6  = (RPREFETCH + 8) * SIZE, AO6
	(p8) FMPY	f10 = f40, f33
	}
	;;
	{ .mmf
	adds	AO21 = 7 * SIZE, AO2
	adds	AO41 = 7 * SIZE, AO4
	(p8) FMPY	f12 = f40, f34
	}
	{ .mmf
	adds	RPRE7  = RPREFETCH * SIZE, AO7
	adds	RPRE8  = (RPREFETCH + 8) * SIZE, AO8
	(p8) FMPY	f14 = f40, f35
	}
	;;
	{ .mfi
	nop	__LINE__
	(p8) FMPY	f16 = f40, f36
	mov	ar.lc = I
	}
	{ .mmf
	adds	WPRE = 8 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f18 = f40, f37
	}
	;;
	{ .mmf
	lfetch.excl.nt1	[WPRE]
	adds	AO61 = 7 * SIZE, AO6
	(p8) FMPY	f20 = f40, f38
	}
	{ .mfb
	adds	AO81 = 7 * SIZE, AO8
	(p8) FMPY	f22 = f40, f39
	(p6) br.cond.dpnt .L115
	}
	;;
	.align 16

.L112:
	{ .mmf
	(p17) LDFPD	f80, f95 = [AO8]
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	(p17) adds	AO8 = 3 * SIZE, AO8
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f34      = [AO2], 1 * SIZE
	(p17) FMA	f12 = f104, f37, f12
	}
	{ .mmf
	(p17) LDFD	f84      = [AO21], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f13 = f105, f38, f13
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFPD	f36, f37 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f104, f39, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f105, f40, f15
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFD	f38      = [AO4], 1 * SIZE
	(p17) FMA	f16 = f104, f41, f16
	}
	{ .mmf
	(p17) LDFD	f88      = [AO41], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f17 = f105, f42, f17
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE5], 16 * SIZE
	(p16) LDFPD	f40, f41 = [AO5], 2 * SIZE
	(p17) FMA	f18 = f104, f43, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f105, f44, f19
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE6], 16 * SIZE
	(p16) LDFD	f42      = [AO6], 1 * SIZE
	(p17) FMA	f20 = f104, f45, f20
	}
	{ .mmf
	(p17) LDFD	f92      = [AO61], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f21 = f105, f46, f21
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE7], 16 * SIZE
	(p16) LDFPD	f44, f45 = [AO7], 2 * SIZE
	(p17) FMA	f22 = f104, f47, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f105, f48, f23
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE8], 16 * SIZE
	(p16) LDFD	f46      = [AO8], 1 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	(p17) LDFD	f96      = [AO81], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f35, f50 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f12 = f106, f53, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f107, f54, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f52, f53 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f106, f55, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f107, f56, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f39, f54 = [AO4], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f16 = f106, f57, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f17 = f107, f58, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f56, f57 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f106, f59, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f107, f60, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f43, f58 = [AO6], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f20 = f106, f61, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f21 = f107, f62, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f60, f61 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f106, f63, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f107, f64, f23
	}
	;;
	{ .mmf
	(p16) LDFPD	f47, f62 = [AO8], 2 * SIZE
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f51, f66 = [AO2], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f12 = f108, f69, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f109, f70, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f68, f69 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f108, f71, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f109, f72, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f55, f70 = [AO4], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f16 = f108, f73, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f17 = f109, f74, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f72, f73 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f108, f75, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f109, f76, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f59, f74 = [AO6], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f20 = f108, f77, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f21 = f109, f78, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f76, f77 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f108, f79, f22
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f23 = f109, f80, f23
	}
	;;
	{ .mmf
	(p16) LDFPD	f63, f78 = [AO8], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f111, f84, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f67, f82 = [AO2]
	nop	__LINE__
	(p17) FMA	f12 = f110, f85, f12
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO2 = 3 * SIZE, AO2
	(p17) FMA	f13 = f111, f86, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f84, f85 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f110, f87, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f111, f88, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f71, f86 = [AO4]
	nop	__LINE__
	(p17) FMA	f16 = f110, f89, f16
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO4 = 3 * SIZE, AO4
	(p17) FMA	f17 = f111, f90, f17
	}
	;;
	{ .mmf
	(p16) LDFPD	f88, f89 = [AO5], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f18 = f110, f91, f18
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f19 = f111, f92, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f75, f90 = [AO6]
	nop	__LINE__
	(p17) FMA	f20 = f110, f93, f20
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO6 = 3 * SIZE, AO6
	(p17) FMA	f21 = f111, f94, f21
	}
	;;
	{ .mmf
	(p16) LDFPD	f92, f93 = [AO7], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f22 = f110, f95, f22
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f23 = f111, f96, f23
	br.ctop.sptk.few .L112
	}
	;;
	.align 16

.L115:
	and	I = 15, MIN_M
	mov	pr.rot= 0
	;;
	cmp.eq	p6,  p0 = 0, I
	cmp.eq	p16, p15 = r0, r0
	;;
	adds	I = 1, I
	;;
	shr	I = I, 1
	;;
	adds	I = -1, I
	adds	AO21 = 1 * SIZE, AO2
	adds	AO41 = 1 * SIZE, AO4
	adds	AO61 = 1 * SIZE, AO6
	adds	AO81 = 1 * SIZE, AO8
	;;
	mov	ar.lc = I
	mov	ar.ec= 3
	and	I = 15, MIN_M
	(p6) br.cond.dpnt .L118
	;;
	.align 16

.L116:
	{ .mmf
	(p16) LDFPD	f104, f107 = [BO], 2 * SIZE
	(p16) LDFPD	f32,  f35  = [AO1], 2 * SIZE
	(p18) FMA	f8  = f106, f34, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f9  = f109, f37, f9
	}
	;;
	{ .mmf
	(p16) LDFD	f38  = [AO2], 2 * SIZE
	(p17) LDFD	f42  = [AO21], 2 * SIZE
	(p18) FMA	f10 = f106, f40, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f11 = f109, f43, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f44,  f47  = [AO3], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f12 = f106, f46, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f13 = f109, f49, f13
	}
	;;
	{ .mmf
	(p16) LDFD	f50  = [AO4], 2 * SIZE
	(p17) LDFD	f54  = [AO41], 2 * SIZE
	(p18) FMA	f14 = f106, f52, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f15 = f109, f55, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f56,  f59  = [AO5], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f16 = f106, f58, f16
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f17 = f109, f61, f17
	}
	;;
	{ .mmf
	(p16) LDFD	f62  = [AO6], 2 * SIZE
	(p17) LDFD	f66  = [AO61], 2 * SIZE
	(p18) FMA	f18 = f106, f64, f18
	}
	{ .mmf
	nop	__LINE__
	(p17) adds	I = -2, I
	(p15) FMA	f19 = f109, f67, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f68,  f71  = [AO7], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f20 = f106, f70, f20
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f21 = f109, f73, f21
	}
	;;
	{ .mmf
	(p16) LDFD	f74  = [AO8], 2 * SIZE
	(p17) LDFD	f78  = [AO81], 2 * SIZE
	(p15) FMA	f23 = f109, f79, f23
	}
	{ .mfb
	(p17) cmp.ne.unc p15, p0 = -1, I
	(p18) FMA	f22 = f106, f76, f22
	br.ctop.sptk.few .L116
	}
	;;

.L118:
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	FADD	f8  = f8,  f9
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	LDFD	f34 = [CO], INCY
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	;;
	{ .mmf
	LDFD	f35 = [CO], INCY
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	LDFD	f36 = [CO], INCY
	nop	__LINE__
	FADD	f16 = f16, f17
	}
	;;
	{ .mmf
	LDFD	f37 = [CO], INCY
	nop	__LINE__
	FADD	f18 = f18, f19
	}
	;;
	{ .mmf
	LDFD	f38 = [CO], INCY
	nop	__LINE__
	FADD	f20 = f20, f21
	}
	;;
	{ .mmf
	LDFD	f39 = [CO], INCY
	nop	__LINE__
	FADD	f22 = f22, f23
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f34 = ALPHA, f12, f34
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f35 = ALPHA, f14, f35
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	FMA	f36 = ALPHA, f16, f36
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	add	AO1 = AO1, INCY
	FMA	f37 = ALPHA, f18, f37
	}
	;;
	{ .mmf
	STFD [AO1] = f34
	add	AO1 = AO1, INCY
	FMA	f38 = ALPHA, f20, f38
	}
	;;
	{ .mmf
	STFD [AO1] = f35
	add	AO1 = AO1, INCY
	FMA	f39 = ALPHA, f22, f39
	}
	;;
	{ .mmi
	STFD [AO1] = f36
	add	AO1 = AO1, INCY
	adds J = -1, J
	}
	;;
	{ .mmi
	STFD [AO1] = f37
	add	AO1 = AO1, INCY
	nop	__LINE__
	}
	;;
	{ .mmi
	STFD [AO1] = f38
	add	AO1 = AO1, INCY
	cmp4.lt p6, p0 = 0, J
	}
	;;
	{ .mib
	STFD [AO1] = f39
	add	AO1 = AO1, INCY
	(p6) br.cond.dptk .L111
	}
	;;
	.align 16

.L120:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 2
	}
	;;
	{ .mfi
	shladd	AO3 = LDA, 1, A
	mov	f12 = f0
	shr	I = MIN_M, 4
	}
	{ .mfb
	shladd	AO4 = LDA, 1, AO2
	mov	f14 = f0
	(p6) br.cond.dpnt .L130
	}
	;;
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f9  = f0
	}
	{ .mmf
	mov	BO  = BUFFER
	shladd	A   = LDA, 2, A
	mov	f11 = f0
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	(p8) LDFD f34 = [AO3], SIZE
	(p8) LDFD f35 = [AO4], SIZE
	mov	f15 = f0
	}
	;;
	{ .mmi
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	mov	ar.ec= 2
	}
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	add	I = I, I
	adds	AO21 = 7 * SIZE, AO2
	}
	;;
	{ .mmf
	adds	WPRE =  4 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f8  = f40, f32
	}
	{ .mmf
	adds	RPRE3  = RPREFETCH * SIZE, AO3
	adds	I = -1, I
	(p8) FMPY	f10 = f40, f33
	}
	;;
	{ .mfi
	adds	AO41 = 7 * SIZE, AO4
	(p8) FMPY	f12 = f40, f34
	mov	ar.lc = I
	}
	{ .mfb
	adds	RPRE4  = (RPREFETCH + 8) * SIZE, AO4
	(p8) FMPY	f14 = f40, f35
	(p6) br.cond.dpnt .L125
	}
	;;
	.align 16

.L122:
	{ .mmf
	(p17) LDFPD	f72, f87 = [AO4]
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	(p17) adds	AO4 = 3 * SIZE, AO4
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f34      = [AO2], 1 * SIZE
	(p17) FMA	f12 = f104, f37, f12
	}
	{ .mmf
	(p17) LDFD	f84      = [AO21], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f13 = f105, f38, f13
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFPD	f36, f37 = [AO3], 2 * SIZE
	(p17) FMA	f14 = f104, f39, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f105, f40, f15
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFD	f38      = [AO4], 1 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	(p17) LDFD	f88      = [AO41], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f35, f50 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f12 = f106, f53, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f107, f54, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f52, f53 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f106, f55, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f107, f56, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f39, f54 = [AO4], 2 * SIZE
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f51, f66 = [AO2], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f12 = f108, f69, f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f13 = f109, f70, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f68, f69 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f108, f71, f14
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f15 = f109, f72, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f55, f70 = [AO4], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f111, f84, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f67, f82 = [AO2]
	nop	__LINE__
	(p17) FMA	f12 = f110, f85, f12
	}
	{ .mmf
	nop	__LINE__
	(p16) adds	AO2 = 3 * SIZE, AO2
	(p17) FMA	f13 = f111, f86, f13
	}
	;;
	{ .mmf
	(p16) LDFPD	f84, f85 = [AO3], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f14 = f110, f87, f14
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f15 = f111, f88, f15
	br.ctop.sptk.few .L122
	}
	;;
	.align 16

.L125:
	and	I = 15, MIN_M
	mov	pr.rot= 0
	;;
	cmp.eq	p6,  p0 = 0, I
	cmp.eq	p16, p15 = r0, r0
	;;
	adds	I = 1, I
	adds	AO21 = 1 * SIZE, AO2
	adds	AO41 = 1 * SIZE, AO4
	;;
	shr	I = I, 1
	;;
	adds	I = -1, I
	;;
	mov	ar.lc = I
	mov	ar.ec= 3
	and	I = 15, MIN_M
	(p6) br.cond.dpnt .L128
	;;
	.align 16

.L126:
	{ .mmf
	(p16) LDFPD	f104, f107 = [BO], 2 * SIZE
	(p16) LDFPD	f32,  f35  = [AO1], 2 * SIZE
	(p18) FMA	f8  = f106, f34, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f9  = f109, f37, f9
	}
	;;
	{ .mmf
	(p17) LDFD	f42        = [AO21], 2 * SIZE
	(p16) LDFD	f38        = [AO2], 2 * SIZE
	(p18) FMA	f10 = f106, f40, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p15) FMA	f11 = f109, f43, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f44,  f47  = [AO3], 2 * SIZE
	nop	__LINE__
	(p18) FMA	f12 = f106, f46, f12
	}
	{ .mmf
	nop	__LINE__
	(p17) adds	I = -2, I
	(p15) FMA	f13 = f109, f49, f13
	}
	;;
	{ .mmf
	(p17) LDFD	f54        = [AO41], 2 * SIZE
	(p16) LDFD	f50        = [AO4], 2 * SIZE
	(p15) FMA	f15 = f109, f55, f15
	}
	{ .mfb
	(p17) cmp.ne.unc p15, p0 = -1, I
	(p18) FMA	f14 = f106, f52, f14
	br.ctop.sptk.few .L126
	}
	;;

.L128:
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	FADD	f8  = f8,  f9
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	LDFD	f34 = [CO], INCY
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	;;
	{ .mmf
	LDFD	f35 = [CO], INCY
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f34 = ALPHA, f12, f34
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f35 = ALPHA, f14, f35
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f34
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f35
	add	AO1 = AO1, INCY
	}
	;;
	.align 16

.L130:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 1
	}
	;;
	{ .mfi
	mov	BO  = BUFFER
	mov	f12 = f0
	shr	I = MIN_M, 4
	}
	{ .mfb
	adds	WPRE =  4 * SIZE, CO
	mov	f14 = f0
	(p6) br.cond.dpnt .L140
	}
	;;
	{ .mmf
	(p8) LDFD f32 = [AO1], SIZE
	(p8) LDFD f33 = [AO2], SIZE
	mov	f9  = f0
	}
	{ .mfi
	shladd	A   = LDA, 1, A
	mov	f11 = f0
	mov	ar.ec= 2
	}
	;;
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	cmp.eq	p6, p0 = 0, I
	mov	f13 = f0
	}
	{ .mmf
	adds	RPRE1  = RPREFETCH * SIZE, AO1
	add	I = I, I
	mov	f15 = f0
	}
	;;
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	adds	RPRE2  = (RPREFETCH + 8) * SIZE, AO2
	adds	I = -1, I
	}
	;;
	{ .mfi
	adds	AO21 = 7 * SIZE, AO2
	(p8) FMPY	f8  = f40, f32
	mov	ar.lc = I
	}
	{ .mfb
	adds	PREB  = RPREFETCH * SIZE, BO
	(p8) FMPY	f10 = f40, f33
	(p6) br.cond.dpnt .L135
	}
	;;
	.align 16

.L132:
	{ .mmf
	(p17) LDFPD	f68, f83 = [AO2]
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	(p17) adds	AO2 = 3 * SIZE, AO2
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32, f33 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f104, f35, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f105, f36, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f34      = [AO2], 1 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	(p17) LDFD	f84      = [AO21], 8 * SIZE
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f106, f51, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f107, f52, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f35, f50 = [AO2], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p16) LDFPD	f64, f65 = [AO1], 2 * SIZE
	(p17) FMA	f10 = f108, f67, f10
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f11 = f109, f68, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f51, f66 = [AO2], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f111, f82, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f80, f81 = [AO1], 2 * SIZE
	nop	__LINE__
	(p17) FMA	f10 = f110, f83, f10
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f11 = f111, f84, f11
	br.ctop.sptk.few .L132
	}
	;;
	.align 16

.L135:
	and	I = 15, MIN_M
	;;
	cmp.eq	p6,  p0 = 0, I
	(p6) br.cond.dpnt .L138
	;;
	tbit.nz	p12, p0 = MIN_M, 3
	tbit.nz	p13, p0 = MIN_M, 2
	tbit.nz	p14, p0 = MIN_M, 1
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p12) LDFPD	f100, f101 = [BO], 2 * SIZE
	(p12) LDFPD	f32,  f33  = [AO1], 2 * SIZE
	(p12) LDFD	f34        = [AO2], 1 * SIZE
	;;
	(p12) LDFPD	f36,  f37  = [AO1], 2 * SIZE
	(p12) LDFPD	f35,  f38  = [AO2], 2 * SIZE
	;;
	(p12) LDFPD	f102, f103 = [BO],  2 * SIZE
	(p12) LDFPD	f39,  f42  = [AO2], 2 * SIZE
	;;
	(p12) LDFPD	f40,  f41  = [AO1], 2 * SIZE
	(p12) LDFPD	f43,  f46  = [AO2], 2 * SIZE
	;;
	(p12) LDFPD	f104, f105 = [BO], 2 * SIZE
	(p12) LDFPD	f44,  f45  = [AO1], 2 * SIZE
	(p12) LDFD	f47        = [AO2], 1 * SIZE
	;;
	(p12) LDFPD	f106, f107 = [BO], 2 * SIZE
	(p13) LDFD	f50        = [AO2], 1 * SIZE
	(p13) LDFPD	f48,  f49  = [AO1], 2 * SIZE
	;;
	(p13) LDFPD	f108, f109 = [BO], 2 * SIZE
	(p13) LDFPD	f51,  f54  = [AO2], 2 * SIZE
	;;
	(p13) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p13) LDFPD	f52,  f53  = [AO1], 2 * SIZE
	(p13) LDFD	f55        = [AO2], 1 * SIZE
	;;
	(p14) LDFPD	f56,  f57  = [AO1], 2 * SIZE
	(p14) LDFD	f58        = [AO2], 1 * SIZE
	;;
	(p14) LDFPD	f112, f113 = [BO], 2 * SIZE
	(p15) LDFD	f60        = [AO1]
	(p14) LDFD	f59        = [AO2], 1 * SIZE
	;;
	(p15) LDFD	f61        = [AO2]
	(p15) LDFD	f114       = [BO]
	;;
	(p12) FMA	f8  = f100, f32, f8
	(p12) FMA	f9  = f101, f33, f9
	(p12) FMA	f10 = f100, f34, f10
	(p12) FMA	f11 = f101, f35, f11
	;;
	(p12) FMA	f12 = f102, f36, f12
	(p12) FMA	f13 = f103, f37, f13
	(p12) FMA	f14 = f102, f38, f14
	(p12) FMA	f15 = f103, f39, f15
	;;
	(p12) FMA	f8  = f104, f40, f8
	(p12) FMA	f9  = f105, f41, f9
	(p12) FMA	f10 = f104, f42, f10
	(p12) FMA	f11 = f105, f43, f11
	;;
	(p12) FMA	f12 = f106, f44, f12
	(p12) FMA	f13 = f107, f45, f13
	(p12) FMA	f14 = f106, f46, f14
	(p12) FMA	f15 = f107, f47, f15
	;;
	(p13) FMA	f8  = f108, f48, f8
	(p13) FMA	f9  = f109, f49, f9
	(p13) FMA	f10 = f108, f50, f10
	(p13) FMA	f11 = f109, f51, f11
	;;
	(p13) FMA	f12 = f110, f52, f12
	(p13) FMA	f13 = f111, f53, f13
	(p13) FMA	f14 = f110, f54, f14
	(p13) FMA	f15 = f111, f55, f15
	;;
	(p14) FMA	f8  = f112, f56, f8
	(p14) FMA	f9  = f113, f57, f9
	(p14) FMA	f10 = f112, f58, f10
	(p14) FMA	f11 = f113, f59, f11
	;;
	(p15) FMA	f12 = f114, f60, f12
	(p15) FMA	f14 = f114, f61, f14
	;;
.L138:
	FADD	f8  = f8,  f9
	FADD	f10 = f10, f11
	FADD	f12 = f12, f13
	FADD	f14 = f14, f15
	;;
	FADD	f8  = f8,  f12
	FADD	f10 = f10, f14
	;;
	{ .mmf
	mov	AO1 = CO
	LDFD	f32 = [CO], INCY
	}
	;;
	{ .mmf
	LDFD	f33 = [CO], INCY
	nop	__LINE__
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f33 = ALPHA, f10, f33
	}
	;;
	{ .mmf
	STFD [AO1] = f32
	add	AO1 = AO1, INCY
	}
	;;
	{ .mmf
	STFD [AO1] = f33
	}
	;;
	.align 16

.L140:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	shr	I = MIN_M, 4
	}
	{ .mfi
	mov	BO  = BUFFER
	mov	f10 = f0
	tbit.z	p7, p0  = N, 0
	}
	;;
	{ .mfi
	cmp.eq	p6, p0 = 0, I
	mov	f12 = f0
	mov	pr.rot= 0
	}
	{ .mfb
	add	I = I, I
	mov	f14 = f0
	(p7) br.cond.dpnt .L199
	}
	;;
	{ .mfi
	(p8) LDFD f32 = [AO1], SIZE
	mov	f9  = f0
	mov	ar.ec= 2
	}
	{ .mmf
	(p8) LDFD f40 = [BO], 2 * SIZE
	add	A   = A, LDA
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	WPRE =  1 * SIZE, CO
	adds	PREB  = RPREFETCH * SIZE, BO
	mov	f13 = f0
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	adds	I = -1, I
	mov	f15 = f0
	}
	;;
	{ .mfi
	lfetch.excl.nt1	[WPRE]
	(p8) FMPY	f8  = f40, f32
	mov	ar.lc = I
	}
	{ .mmb
	nop	__LINE__
	nop	__LINE__
	(p6) br.cond.dpnt .L145
	}
	;;
	.align 16

.L142:
	{ .mmf
	(p17) LDFPD	f81, f82   = [AO1], 2 * SIZE
	(p17) LDFPD	f110, f111 = [BO], 2 * SIZE
	(p17) FMA	f8  = f104, f33, f8
	}
	{ .mfi
	nop	__LINE__
	(p17) FMA	f9  = f105, f34, f9
	(p16) tbit.nz.unc p14, p15 = I, 0
	}
	;;
	{ .mmf
	(p16) LDFPD	f32, f33   = [AO1], 2 * SIZE
	(p16) LDFPD	f103, f104 = [BO], 2 * SIZE
	(p17) FMA	f8  = f106, f49, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f107, f50, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f105, f106 = [BO], 2 * SIZE
	(p16) LDFPD	f48, f49 = [AO1], 2 * SIZE
	(p17) FMA	f8  = f108, f65, f8
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	(p17) FMA	f9  = f109, f66, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f64, f65  = [AO1], 2 * SIZE
	(p16) LDFPD	f107, f108 = [BO], 2 * SIZE
	(p17) FMA	f8  = f110, f81, f8
	}
	{ .mfb
	adds	I = -1, I
	(p17) FMA	f9  = f111, f82, f9
	br.ctop.sptk.few .L142
	}
	;;
	.align 16

.L145:
	and	I = 15, MIN_M
	;;
	cmp.eq	p6,  p0 = 0, I
	(p6) br.cond.dpnt .L148
	;;
	tbit.nz	p12, p0 = MIN_M, 3
	tbit.nz	p13, p0 = MIN_M, 2
	tbit.nz	p14, p0 = MIN_M, 1
	tbit.nz	p15, p0 = MIN_M, 0
	;;
	(p12) LDFPD	f32,  f33  = [AO1], 2 * SIZE
	(p12) LDFPD	f100, f101 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f36,  f37  = [AO1], 2 * SIZE
	(p12) LDFPD	f102, f103 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f40,  f41  = [AO1], 2 * SIZE
	(p12) LDFPD	f104, f105 = [BO], 2 * SIZE
	;;
	(p12) LDFPD	f44,  f45  = [AO1], 2 * SIZE
	(p12) LDFPD	f106, f107 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f48,  f49  = [AO1], 2 * SIZE
	(p13) LDFPD	f108, f109 = [BO], 2 * SIZE
	;;
	(p13) LDFPD	f52,  f53  = [AO1], 2 * SIZE
	(p13) LDFPD	f110, f111 = [BO], 2 * SIZE
	;;
	(p14) LDFPD	f56,  f57  = [AO1], 2 * SIZE
	(p14) LDFPD	f112, f113 = [BO], 2 * SIZE
	;;
	(p15) LDFD	f60        = [AO1]
	(p15) LDFD	f114       = [BO]
	;;
	(p12) FMA	f8  = f100, f32, f8
	(p12) FMA	f9  = f101, f33, f9
	(p12) FMA	f10 = f102, f36, f10
	(p12) FMA	f11 = f103, f37, f11
	(p12) FMA	f12 = f104, f40, f12
	(p12) FMA	f13 = f105, f41, f13
	(p12) FMA	f14 = f106, f44, f14
	(p12) FMA	f15 = f107, f45, f15
	;;
	(p13) FMA	f8  = f108, f48, f8
	(p13) FMA	f9  = f109, f49, f9
	(p13) FMA	f10 = f110, f52, f10
	(p13) FMA	f11 = f111, f53, f11
	(p14) FMA	f12 = f112, f56, f12
	(p14) FMA	f13 = f113, f57, f13
	(p15) FMA	f14 = f114, f60, f14
	;;
.L148:
	{ .mmf
	LDFD	f32 = [CO]
	nop	__LINE__
	FADD	f8  = f8,  f9
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f11
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f12 = f12, f13
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f14 = f14, f15
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f12
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f10 = f10, f14
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FADD	f8  = f8,  f10
	}
	;;
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FMA	f32 = ALPHA, f8,  f32
	}
	;;
	{ .mmf
	STFD [CO] = f32
	nop	__LINE__
	nop	__LINE__
	}
	;;
	.align 16

.L199:
	adds	IS = P, IS
	shladd	A  = LDAP, BASE_SHIFT, A
	;;
	cmp.gt	p6, p0 = M, IS
	(p6) br.cond.dptk .LIs_loop
	.align 4
	;;

.L999:
	mov	r8 = r0
	adds	r9 = 1 * 16, SP
	;;
	ldf.fill  f16 = [SP], 32
	ldf.fill  f17 = [r9], 32
	mov	 ar.lc = ARLC
	;;
	ldf.fill  f18 = [SP], 32
	ldf.fill  f19 = [r9], 32
	mov pr    = PR, -1
	;;
	ldf.fill  f20 = [SP], 32
	ldf.fill  f21 = [r9], 32
	mov	ar.pfs = ARPFS
	;;
	ldf.fill  f22 = [SP], 32
	ldf.fill  f23 = [r9]
	br.ret.sptk.many b0
	;;
	EPILOGUE