OpenBLAS/kernel/ia64/zgemv_t.S

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#define SP	r12

#define M	r32
#define N	r33
#define A	r37
#define LDA	r38
#define X	r39
#define INCX	r34
#define Y	r35
#define INCY	r36
#define BUFFER	r11

#define I	r15
#define J	r16
#define AO1	r18
#define AO2	r19
#define AO3	r20
#define AO4	r21
#define AO5	r22
#define AO6	r23
#define AO7	r24
#define AO8	r25
#define BO	r26
#define INCYM1	r28

#define RPRE1	loc0
#define RPRE2	loc1
#define RPRE3	loc2
#define RPRE4	loc3
#define RPRE5	loc4
#define RPRE6	loc5
#define RPRE7	loc6
#define RPRE8	loc7

#define AO21	loc8
#define AO41	loc9
#define AO61	loc10
#define AO81	loc11
#define CLD1	loc12
#define CLD2	loc13
#define CST1	loc14
#define CST2	loc15

#define PREB	r8
#define WPRE	r9
#define OFFSET	PREB
#define INCX3M1	WPRE
#define INCY3M1	r10

#define ARLC	r29
#define PR	r30
#define ARPFS	r31

#ifdef DOUBLE
#define RPREFETCH	(16 * 2 +  8)
#else
#define RPREFETCH	(16 * 2 + 16)
#endif
#define PREFETCH	lfetch.nt1

#define ALPHA_R	f6
#define ALPHA_I	f7

#if   !defined(CONJ) && !defined(XCONJ)
#define ADD1	     FMA
#define ADD2	     FMA
#define ADD3	     FNMA
#define ADD4	     FMA
#elif  defined(CONJ) && !defined(XCONJ)
#define ADD1	     FMA
#define ADD2	     FMA
#define ADD3	     FMA
#define ADD4	     FNMA
#elif !defined(CONJ) &&  defined(XCONJ)
#define ADD1	     FMA
#define ADD2	     FNMA
#define ADD3	     FMA
#define ADD4	     FMA
#else
#define ADD1	     FMA
#define ADD2	     FNMA
#define ADD3	     FNMA
#define ADD4	     FNMA
#endif

	PROLOGUE
	PROFCODE
	.prologue

	{ .mmi
	.save	ar.pfs, ARPFS
	alloc	ARPFS = ar.pfs, 8, 16, 0, 0
	adds	r14 = 16, SP
	mov	ARLC  = ar.lc
	}
	{ .mmi
	adds	r8 = -8 * 16, SP
	adds	r9 = -7 * 16, SP
	adds	SP = -8 * 16, SP
	}
	;;
	{ .mmi
	stf.spill  [r8] = f16, 32
	stf.spill  [r9] = f17, 32
	mov	PR = pr
	}
	;;
	{ .mmi
	stf.spill  [r8] = f18, 32
	stf.spill  [r9] = f19, 32
	adds	r15 = 152, SP
	}
	;;
	{ .mmi
	stf.spill  [r8] = f20, 32
	stf.spill  [r9] = f21, 32
	adds	r16 = 160, SP
	}
	;;
	{ .mmi
	stf.spill  [r8] = f22
	stf.spill  [r9] = f23
	adds	r17 = 168, SP
	}
	.body
	;;
	{ .mmf
	ld8	INCX   = [r14]
	ld8	Y      = [r15]
	mov	ALPHA_R = f8
	}
	{ .mmf
	ld8	INCY   = [r16]
	ld8	BUFFER = [r17]
	mov	ALPHA_I = f9
	}
	;;
	{ .mmi
	shladd	INCX = INCX, ZBASE_SHIFT, r0
	shladd	LDA  = LDA,  ZBASE_SHIFT, r0
	mov	pr.rot= 0
	}
	{ .mmi
	cmp.ge	p7, p0 = 0, M
	cmp.ge	p6, p0 = 0, N
	shladd	INCY = INCY, ZBASE_SHIFT, r0
	}
	;;
	{ .mmi
	mov	AO1 = BUFFER
	adds	OFFSET = -SIZE, INCX
	shr	I = M, 3
	}
	{ .mib
	adds	INCYM1 = - SIZE, INCY
	shladd	INCX3M1  = INCX, 1, INCX
	(p7) br.cond.dpnt .L999
	}
	;;
	{ .mmi
	shladd	BO  = INCX, 1, X
	adds	AO2 = 4 * SIZE, BUFFER
	mov	ar.ec= 5
	}
	{ .mmb
	shladd	INCY3M1  = INCY, 1, INCYM1
	adds I = -1, I
	(p6) br.cond.dpnt .L999
	}
	;;
	{ .mmi
	adds	INCX3M1 = -SIZE, INCX3M1
	cmp.eq	p16, p0 = r0, r0
	tbit.nz	p13, p0 = M, 2
	}
	{ .mib
	cmp.gt	p6, p0 = 0, I
	mov	ar.lc = I
	(p6) br.cond.dpnt .L05
	}
	;;
	.align 16

.L01:
	(p20) STFD	[AO1] = f36, SIZE
	(p20) STFD	[AO2] = f56, SIZE
	(p16) LDFD	f32 = [X], SIZE
	(p16) LDFD	f52 = [BO], SIZE
	;;
	(p20) STFD	[AO1] = f41, SIZE
	(p20) STFD	[AO2] = f61, SIZE
	(p16) LDFD	f37 = [X], OFFSET
	(p16) LDFD	f57 = [BO], OFFSET
	;;
	(p20) STFD	[AO1] = f46, SIZE
	(p20) STFD	[AO2] = f66, SIZE
	(p16) LDFD	f42 = [X], SIZE
	(p16) LDFD	f62 = [BO], SIZE
	;;
	(p20) STFD	[AO1] = f51, 5 * SIZE
	(p20) STFD	[AO2] = f71, 5 * SIZE
	(p16) LDFD	f47 = [X], INCX3M1
	(p16) LDFD	f67 = [BO], INCX3M1
	;;
	(p20) STFD	[AO1] = f76, SIZE
	(p20) STFD	[AO2] = f96, SIZE
	(p16) LDFD	f72 = [X], SIZE
	(p16) LDFD	f92 = [BO], SIZE
	;;
	(p20) STFD	[AO1] = f81, SIZE
	(p20) STFD	[AO2] = f101, SIZE
	(p16) LDFD	f77 = [X], OFFSET
	(p16) LDFD	f97 = [BO], OFFSET
	;;
	(p20) STFD	[AO1] = f86, SIZE
	(p20) STFD	[AO2] = f106, SIZE
	(p16) LDFD	f82 = [X], SIZE
	(p16) LDFD	f102 = [BO], SIZE
	;;
	(p20) STFD	[AO1] = f91, 5 * SIZE
	(p20) STFD	[AO2] = f111, 5 * SIZE
	(p16) LDFD	f87 = [X], INCX3M1
	(p16) LDFD	f107 = [BO], INCX3M1
	br.ctop.sptk.few .L01
	;;
	.align 16

.L05:
	{ .mmi
	(p13) LDFD f32 = [X],  SIZE
	(p13) LDFD f36 = [BO],  SIZE
	tbit.nz	p14, p0 = M, 1
	}
	;;
	{ .mmi
	(p13) LDFD f33 = [X],  OFFSET
	(p13) LDFD f37 = [BO],  OFFSET
	tbit.nz	p15, p0 = M, 0
	}
	;;
	{ .mmb
	(p13) LDFD f34 = [X],  SIZE
	(p13) LDFD f38 = [BO],  SIZE
	}
	;;
	{ .mmi
	(p13) LDFD f35 = [X],  INCX3M1
	(p13) LDFD f39 = [BO],  INCX3M1
	}
	;;
	{ .mmi
	(p14) LDFD f40 = [X],  SIZE
	}
	;;
	(p14) LDFD f41 = [X],  OFFSET
	(p13) STFD [AO1] = f32, SIZE
	tbit.nz	p8, p0 = A,   BASE_SHIFT
	;;
	(p14) LDFD f42 = [X],  SIZE
	(p13) STFD [AO2] = f36, SIZE
	;;
	(p14) LDFD f43 = [X],  OFFSET
	(p13) STFD [AO1] = f33, SIZE
	;;
	(p15) LDFD f44 = [X],  SIZE
	(p13) STFD [AO2] = f37, SIZE
	;;
	(p15) LDFD f45 = [X],  OFFSET
	(p13) STFD [AO1] = f34, SIZE
	(p13) STFD [AO2] = f38, SIZE
	;;
	(p13) STFD [AO1] = f35, 5 * SIZE
	(p13) STFD [AO2] = f39, 5 * SIZE
	;;
	(p14) STFD [AO1] = f40, SIZE
	;;
	(p14) STFD [AO1] = f41, SIZE
	;;
	(p14) STFD [AO1] = f42, SIZE
	;;
	(p14) STFD [AO1] = f43, SIZE
	;;
	(p15) STFD [AO1] = f44, SIZE
	;;
	(p15) STFD [AO1] = f45, SIZE
	(p8) br.cond.dpnt .L100
	;;
	.align 16

.L10:
	{ .mmi
	mov	CLD1  = Y
	shladd	CLD2  = INCY, 1, Y
	shr	J   = N, 3
	}
	;;
	{ .mmb
	mov	CST1  = Y
	cmp.eq	p6, p0 = r0, J
	(p6) br.cond.dpnt .L20
	}
	;;
	.align 16

.L11:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	mov	BO  = BUFFER
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mmf
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	mov	f14 = f0
	}
	;;
	{ .mmf
	shladd	AO5 = LDA, 1, AO3
	shladd	AO6 = LDA, 1, AO4
	mov	f16 = f0
	}
	{ .mmf
	adds	RPRE3  = (RPREFETCH +  4) * SIZE, AO3
	adds	RPRE4  = (RPREFETCH +  6) * SIZE, AO4
	mov	f18 = f0
	}
	;;
	{ .mmf
	shladd	AO7 = LDA, 1, AO5
	shladd	AO8 = LDA, 1, AO6
	mov	f20 = f0
	}
	{ .mmf
	adds	RPRE5  = (RPREFETCH +  8) * SIZE, AO5
	adds	RPRE6  = (RPREFETCH + 10) * SIZE, AO6
	mov	f22 = f0
	}
	;;
	{ .mfi
	shladd	A   = LDA, 3, A
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	adds	RPRE7  = (RPREFETCH + 12) * SIZE, AO7
	adds	RPRE8  = (RPREFETCH + 14) * SIZE, AO8
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	WPRE = 16 * SIZE, CLD1
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	f13 = f0
	}
	{ .mmf
	adds	I = -1, M
	cmp.eq	p16, p0 = r0, r0
	mov	f15 = f0
	}
	;;
	{ .mfi
	cmp.eq  p12, p0 = r0, r0
	mov	f17 = f0
	mov	ar.lc = I
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	mov	f19 = f0
	}
	;;
	{ .mmf
	lfetch.excl.nt1	[WPRE]
	nop	__LINE__
	mov	f21 = f0
	}
	{ .mmf
	mov	I = 0
	nop	__LINE__
	mov	f23 = f0
	}
	;;
	.align 16

.L16:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32,  f37  = [AO1], 2 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 1, I
	nop   __LINE__
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p13) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) cmp.eq.unc p14, p0 = 2, I
	(p16) cmp.eq.unc p15, p0 = 3, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f42,  f47  = [AO2], 2 * SIZE
	nop   __LINE__
	(p20) ADD1	f12 = f116, f56, f12
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f13 = f121, f56, f13
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	nop   __LINE__
	(p20) ADD1	f14 = f116, f66, f14
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f15 = f121, f66, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f52,  f57  = [AO3], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f8  = f121, f41, f8
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f9  = f116, f41, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f10 = f121, f51, f10
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f11 = f116, f51, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f62,  f67  = [AO4], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f12 = f121, f61, f12
	}
	{ .mmf
	(p16) cmp.eq.unc p12, p0 = 4, I
	(p16) cmp.eq.unc p13, p0 = 5, I
	(p20) ADD4	f13 = f116, f61, f13
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f14 = f121, f71, f14
	}
	{ .mmf
	(p16) cmp.eq.unc p14, p0 = 6, I
	(p16) cmp.eq.unc p15, p0 = 7, I
	(p20) ADD4	f15 = f116, f71, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f72,  f77  = [AO5], 2 * SIZE
	nop   __LINE__
	(p20) ADD1	f16 = f116, f76, f16
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f17 = f121, f76, f17
	}
	;;
	{ .mmf
	(p12) PREFETCH [RPRE5], 16 * SIZE
	nop   __LINE__
	(p20) ADD1	f18 = f116, f86, f18
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f19 = f121, f86, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f82,  f87  = [AO6], 2 * SIZE
	nop   __LINE__
	(p20) ADD1	f20 = f116, f96, f20
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f21 = f121, f96, f21
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE6], 16 * SIZE
	nop   __LINE__
	(p20) ADD1	f22 = f116, f106, f22
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f23 = f121, f106, f23
	}
	;;
	{ .mmf
	(p16) LDFPD	f92,  f97  = [AO7], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f16 = f121, f81, f16
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f17 = f116, f81, f17
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE7], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f18 = f121, f91, f18
	}
	{ .mmf
	nop   __LINE__
	(p16) adds I = 1, I
	(p20) ADD4	f19 = f116, f91, f19
	}
	;;
	{ .mmf
	(p16) LDFPD	f102, f107 = [AO8], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f20 = f121, f101, f20
	}
	{ .mmf
	(p15) mov I = 0
	nop   __LINE__
	(p20) ADD4	f21 = f116, f101, f21
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE8], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f22 = f121, f111, f22
	}
	{ .mfb
	(p16) cmp.eq.unc p12, p0 = 0, I
	(p20) ADD4	f23 = f116, f111, f23
	br.ctop.sptk.few .L16
	}
	;;

.L18:
	LDFD	f32 = [CLD1], SIZE
	LDFD	f36 = [CLD2], SIZE
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	LDFD	f37 = [CLD2], INCYM1
	;;
	LDFD	f34 = [CLD1], SIZE
	LDFD	f38 = [CLD2], SIZE
	;;
	LDFD	f35 = [CLD1], INCY3M1
	LDFD	f39 = [CLD2], INCY3M1
	;;
	LDFD	f40 = [CLD1], SIZE
	LDFD	f44 = [CLD2], SIZE
	;;
	LDFD	f41 = [CLD1], INCYM1
	LDFD	f45 = [CLD2], INCYM1
	;;
	LDFD	f42 = [CLD1], SIZE
	LDFD	f46 = [CLD2], SIZE
	;;
	LDFD	f43 = [CLD1], INCY3M1
	LDFD	f47 = [CLD2], INCY3M1
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f36 = ALPHA_R, f12, f36
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f37 = ALPHA_I, f12, f37
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f38 = ALPHA_R, f14, f38
	FMA	f35 = ALPHA_I, f10, f35
	FMA	f39 = ALPHA_I, f14, f39
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FNMA	f36 = ALPHA_I, f13, f36
	FMA	f33 = ALPHA_R, f9,  f33
	FMA	f37 = ALPHA_R, f13, f37
	FNMA	f34 = ALPHA_I, f11, f34
	FNMA	f38 = ALPHA_I, f15, f38
	FMA	f35 = ALPHA_R, f11, f35
	FMA	f39 = ALPHA_R, f15, f39
	;;
	FMA	f40 = ALPHA_R, f16, f40
	FMA	f44 = ALPHA_R, f20, f44
	FMA	f41 = ALPHA_I, f16, f41
	FMA	f45 = ALPHA_I, f20, f45
	FMA	f42 = ALPHA_R, f18, f42
	FMA	f46 = ALPHA_R, f22, f46
	FMA	f43 = ALPHA_I, f18, f43
	FMA	f47 = ALPHA_I, f22, f47
	;;
	{ .mmf
	STFD [CST1] = f32, SIZE
	STFD [CST2] = f36, SIZE
	FNMA	f40 = ALPHA_I, f17, f40
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FNMA	f44 = ALPHA_I, f21, f44
	}
	;;
	{ .mmf
	STFD [CST1] = f33
	STFD [CST2] = f37
	FMA	f41 = ALPHA_R, f17, f41
	}
	{ .mmf
	add  CST1 = CST1, INCYM1
	add  CST2 = CST2, INCYM1
	FMA	f45 = ALPHA_R, f21, f45
	}
	;;
	{ .mmf
	STFD [CST1] = f34, SIZE
	STFD [CST2] = f38, SIZE
	FNMA	f42 = ALPHA_I, f19, f42
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FNMA	f46 = ALPHA_I, f23, f46
	}
	;;
	{ .mmf
	STFD [CST1] = f35
	STFD [CST2] = f39
	FMA	f43 = ALPHA_R, f19, f43
	}
	{ .mmf
	add  CST1 = CST1, INCY3M1
	add  CST2 = CST2, INCY3M1
	FMA	f47 = ALPHA_R, f23, f47
	}
	;;
	{ .mmi
	STFD [CST1] = f40, SIZE
	STFD [CST2] = f44, SIZE
	adds J = -1, J
	}
	;;
	{ .mmi
	STFD [CST1] = f41
	STFD [CST2] = f45
	add  CST1 = CST1, INCYM1
	}
	{ .mmi
	nop  __LINE__
	nop  __LINE__
	add  CST2 = CST2, INCYM1
	}
	;;
	{ .mmi
	STFD [CST1] = f42, SIZE
	STFD [CST2] = f46, SIZE
	cmp.lt p6, p0 = 0, J
	}
	;;
	{ .mmi
	STFD [CST1] = f43
	STFD [CST2] = f47
	add  CST1 = CST1, INCY3M1
	}
	{ .mmb
	add  CST2 = CST2, INCY3M1
	(p6) br.cond.dptk .L11
	}
	;;
	.align 16

.L20:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 2
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mfb
	mov	BO  = BUFFER
	mov	f14 = f0
	(p6) br.cond.dpnt .L30
	}
	;;
	{ .mfi
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	adds	I = -1, M
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	RPRE3  = (RPREFETCH +  4) * SIZE, AO3
	adds	RPRE4  = (RPREFETCH +  6) * SIZE, AO4
	mov	f13 = f0
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	shladd	A   = LDA, 2, A
	mov	f15 = f0
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	ar.lc = I
	}
	{ .mmi
	adds	WPRE = 16 * SIZE, CLD1
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L26:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32,  f37  = [AO1], 2 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 2, I
	nop   __LINE__
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p12) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) cmp.eq.unc p14, p0 = 4, I
	(p16) cmp.eq.unc p15, p0 = 6, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f42,  f47  = [AO2], 2 * SIZE
	nop   __LINE__
	(p20) ADD1	f12 = f116, f56, f12
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f13 = f121, f56, f13
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	nop   __LINE__
	(p20) ADD1	f14 = f116, f66, f14
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f15 = f121, f66, f15
	}
	;;
	{ .mmf
	(p16) LDFPD	f52,  f57  = [AO3], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f8  = f121, f41, f8
	}
	{ .mmf
	(p16) adds I = 1, I
	nop   __LINE__
	(p20) ADD4	f9  = f116, f41, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f10 = f121, f51, f10
	}
	{ .mmf
	(p16) cmp.eq.unc p15, p0 = 8, I
	nop   __LINE__
	(p20) ADD4	f11 = f116, f51, f11
	}
	;;
	{ .mmf
	(p16) LDFPD	f62,  f67  = [AO4], 2 * SIZE
	nop   __LINE__
	(p20) ADD3	f12 = f121, f61, f12
	}
	{ .mmf
	(p15) mov I = 0
	nop   __LINE__
	(p20) ADD4	f13 = f116, f61, f13
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	nop   __LINE__
	(p20) ADD3	f14 = f121, f71, f14
	}
	{ .mfb
	(p16) cmp.eq.unc p12, p0 = 0, I
	(p20) ADD4	f15 = f116, f71, f15
	br.ctop.sptk.few .L26
	}
	;;
.L28:
	LDFD	f32 = [CLD1], SIZE
	LDFD	f36 = [CLD2], SIZE
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	LDFD	f37 = [CLD2], INCYM1
	;;
	LDFD	f34 = [CLD1], SIZE
	LDFD	f38 = [CLD2], SIZE
	;;
	LDFD	f35 = [CLD1], INCY3M1
	LDFD	f39 = [CLD2], INCY3M1
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f36 = ALPHA_R, f12, f36
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f37 = ALPHA_I, f12, f37
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f38 = ALPHA_R, f14, f38
	FMA	f35 = ALPHA_I, f10, f35
	FMA	f39 = ALPHA_I, f14, f39
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FNMA	f36 = ALPHA_I, f13, f36
	FMA	f33 = ALPHA_R, f9,  f33
	FMA	f37 = ALPHA_R, f13, f37
	FNMA	f34 = ALPHA_I, f11, f34
	FNMA	f38 = ALPHA_I, f15, f38
	FMA	f35 = ALPHA_R, f11, f35
	FMA	f39 = ALPHA_R, f15, f39
	;;
	STFD [CST1] = f32, SIZE
	STFD [CST2] = f36, SIZE
	;;
	STFD [CST1] = f33
	STFD [CST2] = f37
	add  CST1 = CST1, INCYM1
	add  CST2 = CST2, INCYM1
	;;
	STFD [CST1] = f34, SIZE
	STFD [CST2] = f38, SIZE
	;;
	STFD [CST1] = f35
	STFD [CST2] = f39
	add  CST1 = CST1, INCY3M1
	add  CST2 = CST2, INCY3M1
	;;
	.align 16

.L30:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 1
	}
	;;
	{ .mmf
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	mov	f12 = f0
	}
	{ .mfb
	adds	I = -1, M
	mov	f14 = f0
	(p6) br.cond.dpnt .L40
	}
	;;
	{ .mfi
	mov	BO  = BUFFER
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	shladd	A   = LDA, 1, A
	mov	f11 = f0
	}
	;;
	{ .mfi
	adds	WPRE = 16 * SIZE, CLD1
	mov	f13 = f0
	mov	ar.lc = I
	}
	{ .mmf
	adds	PREB   = RPREFETCH * SIZE, BO
	nop	__LINE__
	mov	f15 = f0
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L36:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32,  f37  = [AO1], 2 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 4, I
	(p16) adds I = 1, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p12) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) cmp.eq.unc p12, p0 = 8, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFPD	f42,  f47  = [AO2], 2 * SIZE
	(p20) ADD3	f12 = f121, f41, f12
	}
	{ .mmf
	(p12) mov I = 0
	(p20) ADD4	f13 = f116, f41, f13
	}
	;;
	{ .mmf
	(p20) ADD3	f14 = f121, f51, f14
	}
	{ .mfb
	nop	__LINE__
	(p20) ADD4	f15 = f116, f51, f15
	br.ctop.sptk.few .L36
	}
	;;

.L38:
	LDFD	f32 = [CLD1], SIZE
	FADD	f8  = f8,  f12
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	FADD	f10 = f10, f14
	;;
	LDFD	f34 = [CLD1], SIZE
	FADD	f9  = f9,  f13
	;;
	LDFD	f35 = [CLD1], INCYM1
	FADD	f11 = f11, f15
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f35 = ALPHA_I, f10, f35
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FMA	f33 = ALPHA_R, f9,  f33
	FNMA	f34 = ALPHA_I, f11, f34
	FMA	f35 = ALPHA_R, f11, f35
	;;
	STFD [CST1] = f32, SIZE
	;;
	STFD [CST1] = f33
	add  CST1 = CST1, INCYM1
	;;
	STFD [CST1] = f34, SIZE
	;;
	STFD [CST1] = f35
	add  CST1 = CST1, INCYM1
	;;
	.align 16


.L40:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	mov	f9  = f0
	tbit.z	p6, p0  = N, 0
	}
	;;
	{ .mfi
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	mov	f10 = f0
	mov	ar.ec= 5
	}
	{ .mfb
	adds	I = -1, M
	mov	f11 = f0
	(p6) br.cond.dpnt .L999
	}
	;;
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	add	A   = LDA, A
	mov	ar.lc = I
	}
	{ .mmi
	adds	WPRE = 16 * SIZE, CLD1
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	BO  = BUFFER
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L46:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFPD	f32,  f37  = [AO1], 2 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p12, p0 = 7, I
	(p16) adds I = 1, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD3	f10  = f121, f41, f10
	}
	{ .mfb
	(p12) mov I = 0
	(p20) ADD4	f11  = f116, f41, f11
	br.ctop.sptk.few .L46
	}
	;;

.L48:
	LDFD	f32 = [CLD1], SIZE
	FADD	f8  = f8,  f10
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	FADD	f9  = f9,  f11
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f33 = ALPHA_I, f8,  f33
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FMA	f33 = ALPHA_R, f9,  f33
	;;
	STFD [CST1] = f32, SIZE
	;;
	STFD [CST1] = f33
	add  CST1 = CST1, INCYM1
	br   .L999
	.align 16
	;;

.L100:
	{ .mmi
	mov	CLD1  = Y
	shladd	CLD2  = INCY, 1, Y
	shr	J   = N, 3
	}
	;;
	{ .mmb
	mov	CST1  = Y
	cmp.eq	p6, p0 = r0, J
	(p6) br.cond.dpnt .L120
	}
	;;
	.align 16

.L111:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	mov	BO  = BUFFER
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mmf
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	mov	f14 = f0
	}
	;;
	{ .mmf
	shladd	AO5 = LDA, 1, AO3
	shladd	AO6 = LDA, 1, AO4
	mov	f16 = f0
	}
	{ .mmf
	adds	RPRE3  = (RPREFETCH +  4) * SIZE, AO3
	adds	RPRE4  = (RPREFETCH +  6) * SIZE, AO4
	mov	f18 = f0
	}
	;;
	{ .mmf
	shladd	AO7 = LDA, 1, AO5
	shladd	AO8 = LDA, 1, AO6
	mov	f20 = f0
	}
	{ .mmf
	adds	RPRE5  = (RPREFETCH +  8) * SIZE, AO5
	adds	RPRE6  = (RPREFETCH + 10) * SIZE, AO6
	mov	f22 = f0
	}
	;;
	{ .mfi
	shladd	A   = LDA, 3, A
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	adds	RPRE7  = (RPREFETCH + 12) * SIZE, AO7
	adds	RPRE8  = (RPREFETCH + 14) * SIZE, AO8
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	WPRE = 16 * SIZE, CLD1
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	f13 = f0
	}
	{ .mmf
	adds	I = -1, M
	cmp.eq	p16, p0 = r0, r0
	mov	f15 = f0
	}
	;;
	{ .mfi
	cmp.eq  p12, p0 = r0, r0
	mov	f17 = f0
	mov	ar.lc = I
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	mov	f19 = f0
	}
	;;
	{ .mmf
	lfetch.excl.nt1	[WPRE]
	nop	__LINE__
	mov	f21 = f0
	}
	{ .mmf
	mov	I = 0
	nop	__LINE__
	mov	f23 = f0
	}
	;;
	.align 16

.L116:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFD	f32 = [AO1], 1 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 1, I
	(p16) cmp.eq.unc p14, p0 = 2, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p13) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) LDFD	f37 = [AO1], 1 * SIZE
	(p16) cmp.eq.unc p15, p0 = 3, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f42 = [AO2], 1 * SIZE
	(p20) ADD1	f12 = f116, f56, f12
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f13 = f121, f56, f13
	}
	;;
	{ .mmf
	(p16) LDFD	f47 = [AO2], 1 * SIZE
	nop   __LINE__
	(p20) ADD1	f14 = f116, f66, f14
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f15 = f121, f66, f15
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFD	f52 = [AO3], 1 * SIZE
	(p20) ADD3	f8  = f121, f41, f8
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f9  = f116, f41, f9
	}
	;;
	{ .mmf
	(p16) LDFD	f57 = [AO3], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f10 = f121, f51, f10
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f11 = f116, f51, f11
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFD	f62 = [AO4], 1 * SIZE
	(p20) ADD3	f12 = f121, f61, f12
	}
	{ .mmf
	(p16) cmp.eq.unc p12, p0 = 4, I
	(p16) cmp.eq.unc p13, p0 = 5, I
	(p20) ADD4	f13 = f116, f61, f13
	}
	;;
	{ .mmf
	(p16) LDFD	f67 = [AO4], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f14 = f121, f71, f14
	}
	{ .mmf
	(p16) cmp.eq.unc p14, p0 = 6, I
	(p16) cmp.eq.unc p15, p0 = 7, I
	(p20) ADD4	f15 = f116, f71, f15
	}
	;;
	{ .mmf
	(p12) PREFETCH [RPRE5], 16 * SIZE
	(p16) LDFD	f72 = [AO5], 1 * SIZE
	(p20) ADD1	f16 = f116, f76, f16
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f17 = f121, f76, f17
	}
	;;
	{ .mmf
	(p16) LDFD	f77 = [AO5], 1 * SIZE
	nop   __LINE__
	(p20) ADD1	f18 = f116, f86, f18
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f19 = f121, f86, f19
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE6], 16 * SIZE
	(p16) LDFD	f82 = [AO6], 1 * SIZE
	(p20) ADD1	f20 = f116, f96, f20
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f21 = f121, f96, f21
	}
	;;
	{ .mmf
	(p16) LDFD	f87 = [AO6], 1 * SIZE
	nop   __LINE__
	(p20) ADD1	f22 = f116, f106, f22
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f23 = f121, f106, f23
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE7], 16 * SIZE
	(p16) LDFD	f92 = [AO7], 1 * SIZE
	(p20) ADD3	f16 = f121, f81, f16
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD4	f17 = f116, f81, f17
	}
	;;
	{ .mmf
	(p16) LDFD	f97 = [AO7], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f18 = f121, f91, f18
	}
	{ .mmf
	nop   __LINE__
	(p16) adds I = 1, I
	(p20) ADD4	f19 = f116, f91, f19
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE8], 16 * SIZE
	(p16) LDFD	f102 = [AO8], 1 * SIZE
	(p20) ADD3	f20 = f121, f101, f20
	}
	{ .mmf
	(p15) mov I = 0
	nop   __LINE__
	(p20) ADD4	f21 = f116, f101, f21
	}
	;;
	{ .mmf
	(p16) LDFD	f107 = [AO8], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f22 = f121, f111, f22
	}
	{ .mfb
	(p16) cmp.eq.unc p12, p0 = 0, I
	(p20) ADD4	f23 = f116, f111, f23
	br.ctop.sptk.few .L116
	}
	;;

.L118:
	LDFD	f32 = [CLD1], SIZE
	LDFD	f36 = [CLD2], SIZE
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	LDFD	f37 = [CLD2], INCYM1
	;;
	LDFD	f34 = [CLD1], SIZE
	LDFD	f38 = [CLD2], SIZE
	;;
	LDFD	f35 = [CLD1], INCY3M1
	LDFD	f39 = [CLD2], INCY3M1
	;;
	LDFD	f40 = [CLD1], SIZE
	LDFD	f44 = [CLD2], SIZE
	;;
	LDFD	f41 = [CLD1], INCYM1
	LDFD	f45 = [CLD2], INCYM1
	;;
	LDFD	f42 = [CLD1], SIZE
	LDFD	f46 = [CLD2], SIZE
	;;
	LDFD	f43 = [CLD1], INCY3M1
	LDFD	f47 = [CLD2], INCY3M1
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f36 = ALPHA_R, f12, f36
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f37 = ALPHA_I, f12, f37
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f38 = ALPHA_R, f14, f38
	FMA	f35 = ALPHA_I, f10, f35
	FMA	f39 = ALPHA_I, f14, f39
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FNMA	f36 = ALPHA_I, f13, f36
	FMA	f33 = ALPHA_R, f9,  f33
	FMA	f37 = ALPHA_R, f13, f37
	FNMA	f34 = ALPHA_I, f11, f34
	FNMA	f38 = ALPHA_I, f15, f38
	FMA	f35 = ALPHA_R, f11, f35
	FMA	f39 = ALPHA_R, f15, f39
	;;
	FMA	f40 = ALPHA_R, f16, f40
	FMA	f44 = ALPHA_R, f20, f44
	FMA	f41 = ALPHA_I, f16, f41
	FMA	f45 = ALPHA_I, f20, f45
	FMA	f42 = ALPHA_R, f18, f42
	FMA	f46 = ALPHA_R, f22, f46
	FMA	f43 = ALPHA_I, f18, f43
	FMA	f47 = ALPHA_I, f22, f47
	;;
	{ .mmf
	STFD [CST1] = f32, SIZE
	STFD [CST2] = f36, SIZE
	FNMA	f40 = ALPHA_I, f17, f40
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FNMA	f44 = ALPHA_I, f21, f44
	}
	;;
	{ .mmf
	STFD [CST1] = f33
	STFD [CST2] = f37
	FMA	f41 = ALPHA_R, f17, f41
	}
	{ .mmf
	add  CST1 = CST1, INCYM1
	add  CST2 = CST2, INCYM1
	FMA	f45 = ALPHA_R, f21, f45
	}
	;;
	{ .mmf
	STFD [CST1] = f34, SIZE
	STFD [CST2] = f38, SIZE
	FNMA	f42 = ALPHA_I, f19, f42
	}
	{ .mmf
	nop	__LINE__
	nop	__LINE__
	FNMA	f46 = ALPHA_I, f23, f46
	}
	;;
	{ .mmf
	STFD [CST1] = f35
	STFD [CST2] = f39
	FMA	f43 = ALPHA_R, f19, f43
	}
	{ .mmf
	add  CST1 = CST1, INCY3M1
	add  CST2 = CST2, INCY3M1
	FMA	f47 = ALPHA_R, f23, f47
	}
	;;
	{ .mmi
	STFD [CST1] = f40, SIZE
	STFD [CST2] = f44, SIZE
	adds J = -1, J
	}
	;;
	{ .mmi
	STFD [CST1] = f41
	STFD [CST2] = f45
	add  CST1 = CST1, INCYM1
	}
	{ .mmi
	nop  __LINE__
	nop  __LINE__
	add  CST2 = CST2, INCYM1
	}
	;;
	{ .mmi
	STFD [CST1] = f42, SIZE
	STFD [CST2] = f46, SIZE
	cmp.lt p6, p0 = 0, J
	}
	;;
	{ .mmi
	STFD [CST1] = f43
	STFD [CST2] = f47
	add  CST1 = CST1, INCY3M1
	}
	{ .mmb
	add  CST2 = CST2, INCY3M1
	(p6) br.cond.dptk .L111
	}
	;;
	.align 16

.L120:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 2
	}
	;;
	{ .mmf
	shladd	AO3 = LDA, 1, A
	shladd	AO4 = LDA, 1, AO2
	mov	f12 = f0
	}
	{ .mfb
	mov	BO  = BUFFER
	mov	f14 = f0
	(p6) br.cond.dpnt .L130
	}
	;;
	{ .mfi
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	adds	I = -1, M
	mov	f11 = f0
	}
	;;
	{ .mmf
	adds	RPRE3  = (RPREFETCH +  4) * SIZE, AO3
	adds	RPRE4  = (RPREFETCH +  6) * SIZE, AO4
	mov	f13 = f0
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	shladd	A   = LDA, 2, A
	mov	f15 = f0
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	ar.lc = I
	}
	{ .mmi
	adds	WPRE = 16 * SIZE, CLD1
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L126:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFD	f32 = [AO1], 1 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 2, I
	(p16) cmp.eq.unc p14, p0 = 4, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p12) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) LDFD	f37 = [AO1], 1 * SIZE
	(p16) cmp.eq.unc p15, p0 = 6, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p16) LDFD	f42 = [AO2], 1 * SIZE
	nop   __LINE__
	(p20) ADD1	f12 = f116, f56, f12
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f13 = f121, f56, f13
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f47 = [AO2], 1 * SIZE
	(p20) ADD1	f14 = f116, f66, f14
	}
	{ .mmf
	nop   __LINE__
	nop   __LINE__
	(p20) ADD2	f15 = f121, f66, f15
	}
	;;
	{ .mmf
	(p16) LDFD	f52 = [AO3], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f8  = f121, f41, f8
	}
	{ .mmf
	nop   __LINE__
	(p16) adds I = 1, I
	(p20) ADD4	f9  = f116, f41, f9
	}
	;;
	{ .mmf
	(p14) PREFETCH [RPRE3], 16 * SIZE
	(p16) LDFD	f57 = [AO3], 1 * SIZE
	(p20) ADD3	f10 = f121, f51, f10
	}
	{ .mmf
	nop   __LINE__
	(p16) cmp.eq.unc p15, p0 = 8, I
	(p20) ADD4	f11 = f116, f51, f11
	}
	;;
	{ .mmf
	(p16) LDFD	f62 = [AO4], 1 * SIZE
	nop   __LINE__
	(p20) ADD3	f12 = f121, f61, f12
	}
	{ .mmf
	(p15) mov I = 0
	nop   __LINE__
	(p20) ADD4	f13 = f116, f61, f13
	}
	;;
	{ .mmf
	(p15) PREFETCH [RPRE4], 16 * SIZE
	(p16) LDFD	f67 = [AO4], 1 * SIZE
	(p20) ADD3	f14 = f121, f71, f14
	}
	{ .mfb
	(p16) cmp.eq.unc p12, p0 = 0, I
	(p20) ADD4	f15 = f116, f71, f15
	br.ctop.sptk.few .L126
	}
	;;
.L128:
	LDFD	f32 = [CLD1], SIZE
	LDFD	f36 = [CLD2], SIZE
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	LDFD	f37 = [CLD2], INCYM1
	;;
	LDFD	f34 = [CLD1], SIZE
	LDFD	f38 = [CLD2], SIZE
	;;
	LDFD	f35 = [CLD1], INCY3M1
	LDFD	f39 = [CLD2], INCY3M1
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f36 = ALPHA_R, f12, f36
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f37 = ALPHA_I, f12, f37
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f38 = ALPHA_R, f14, f38
	FMA	f35 = ALPHA_I, f10, f35
	FMA	f39 = ALPHA_I, f14, f39
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FNMA	f36 = ALPHA_I, f13, f36
	FMA	f33 = ALPHA_R, f9,  f33
	FMA	f37 = ALPHA_R, f13, f37
	FNMA	f34 = ALPHA_I, f11, f34
	FNMA	f38 = ALPHA_I, f15, f38
	FMA	f35 = ALPHA_R, f11, f35
	FMA	f39 = ALPHA_R, f15, f39
	;;
	STFD [CST1] = f32, SIZE
	STFD [CST2] = f36, SIZE
	;;
	STFD [CST1] = f33
	STFD [CST2] = f37
	add  CST1 = CST1, INCYM1
	add  CST2 = CST2, INCYM1
	;;
	STFD [CST1] = f34, SIZE
	STFD [CST2] = f38, SIZE
	;;
	STFD [CST1] = f35
	STFD [CST2] = f39
	add  CST1 = CST1, INCY3M1
	add  CST2 = CST2, INCY3M1
	;;
	.align 16

.L130:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	add	AO2 = LDA, A
	mov	f10 = f0
	tbit.z	p6, p0  = N, 1
	}
	;;
	{ .mmf
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	adds	RPRE2  = (RPREFETCH +  2) * SIZE, AO2
	mov	f12 = f0
	}
	{ .mfb
	adds	I = -1, M
	mov	f14 = f0
	(p6) br.cond.dpnt .L140
	}
	;;
	{ .mfi
	mov	BO  = BUFFER
	mov	f9  = f0
	mov	ar.ec= 5
	}
	{ .mmf
	cmp.eq	p16, p0 = r0, r0
	shladd	A   = LDA, 1, A
	mov	f11 = f0
	}
	;;
	{ .mfi
	adds	WPRE = 16 * SIZE, CLD1
	mov	f13 = f0
	mov	ar.lc = I
	}
	{ .mmf
	adds	PREB   = RPREFETCH * SIZE, BO
	nop	__LINE__
	mov	f15 = f0
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L136:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFD	f32 = [AO1], 1 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p13, p0 = 4, I
	(p16) adds I = 1, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p12) PREFETCH [PREB], 16 * SIZE
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p20) ADD1	f10 = f116, f46, f10
	}
	{ .mmf
	(p16) LDFD	f37 = [AO1], 1 * SIZE
	(p16) cmp.eq.unc p12, p0 = 8, I
	(p20) ADD2	f11 = f121, f46, f11
	}
	;;
	{ .mmf
	(p13) PREFETCH [RPRE2], 16 * SIZE
	(p16) LDFD	f42 = [AO2], 1 * SIZE
	(p20) ADD3	f12 = f121, f41, f12
	}
	{ .mmf
	(p12) mov I = 0
	nop	__LINE__
	(p20) ADD4	f13 = f116, f41, f13
	}
	;;
	{ .mmf
	(p16) LDFD	f47 = [AO2], 1 * SIZE
	nop	__LINE__
	(p20) ADD3	f14 = f121, f51, f14
	}
	{ .mfb
	nop	__LINE__
	(p20) ADD4	f15 = f116, f51, f15
	br.ctop.sptk.few .L136
	}
	;;

.L138:
	LDFD	f32 = [CLD1], SIZE
	FADD	f8  = f8,  f12
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	FADD	f10 = f10, f14
	;;
	LDFD	f34 = [CLD1], SIZE
	FADD	f9  = f9,  f13
	;;
	LDFD	f35 = [CLD1], INCYM1
	FADD	f11 = f11, f15
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f33 = ALPHA_I, f8,  f33
	FMA	f34 = ALPHA_R, f10, f34
	FMA	f35 = ALPHA_I, f10, f35
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FMA	f33 = ALPHA_R, f9,  f33
	FNMA	f34 = ALPHA_I, f11, f34
	FMA	f35 = ALPHA_R, f11, f35
	;;
	STFD [CST1] = f32, SIZE
	;;
	STFD [CST1] = f33
	add  CST1 = CST1, INCYM1
	;;
	STFD [CST1] = f34, SIZE
	;;
	STFD [CST1] = f35
	add  CST1 = CST1, INCYM1
	;;
	.align 16


.L140:
	{ .mfi
	mov	AO1 = A
	mov	f8  = f0
	mov	pr.rot= 0
	}
	{ .mfi
	mov	f9  = f0
	tbit.z	p6, p0  = N, 0
	}
	;;
	{ .mfi
	adds	RPRE1  = (RPREFETCH +  0) * SIZE, AO1
	mov	f10 = f0
	mov	ar.ec= 5
	}
	{ .mfb
	adds	I = -1, M
	mov	f11 = f0
	(p6) br.cond.dpnt .L999
	}
	;;
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	shladd	A   = LDA, 1, A
	mov	ar.lc = I
	}
	{ .mmi
	adds	WPRE = 16 * SIZE, CLD1
	adds	PREB   = RPREFETCH * SIZE, BO
	mov	BO  = BUFFER
	}
	;;
	{ .mmi
	lfetch.excl.nt1	[WPRE]
	cmp.eq  p12, p0 = r0, r0
	mov	I = 0
	}
	;;
	.align 16

.L146:
	{ .mmf
	(p12) PREFETCH [RPRE1], 16 * SIZE
	(p16) LDFD	f32  = [AO1], 1 * SIZE
	(p20) ADD1	f8  = f116, f36, f8
	}
	{ .mmf
	(p16) cmp.eq.unc p12, p0 = 7, I
	(p16) adds I = 1, I
	(p20) ADD2	f9  = f121, f36, f9
	}
	;;
	{ .mmf
	(p16) LDFPD	f112, f117 = [BO], 2 * SIZE
	(p16) LDFD	f37  = [AO1], 1 * SIZE
	(p20) ADD3	f10  = f121, f41, f10
	}
	{ .mfb
	(p12) mov I = 0
	(p20) ADD4	f11  = f116, f41, f11
	br.ctop.sptk.few .L146
	}
	;;

.L148:
	LDFD	f32 = [CLD1], SIZE
	FADD	f8  = f8,  f10
	shladd	CST2  = INCY, 1, CST1
	;;
	LDFD	f33 = [CLD1], INCYM1
	FADD	f9  = f9,  f11
	;;
	FMA	f32 = ALPHA_R, f8,  f32
	FMA	f33 = ALPHA_I, f8,  f33
	;;
	FNMA	f32 = ALPHA_I, f9,  f32
	FMA	f33 = ALPHA_R, f9,  f33
	;;
	STFD [CST1] = f32, SIZE
	;;
	STFD [CST1] = f33
	add  CST1 = CST1, INCYM1
	;;
	.align 16

.L999:
	mov	r8 = r0
	adds	r9 = 1 * 16, SP
	;;
	ldf.fill  f16 = [SP], 32
	ldf.fill  f17 = [r9], 32
	mov	 ar.lc = ARLC
	;;
	ldf.fill  f18 = [SP], 32
	ldf.fill  f19 = [r9], 32
	mov pr    = PR, -1
	;;
	ldf.fill  f20 = [SP], 32
	ldf.fill  f21 = [r9], 32
	mov	ar.pfs = ARPFS
	;;
	ldf.fill  f22 = [SP], 32
	ldf.fill  f23 = [r9]
	br.ret.sptk.many b0
	;;
	EPILOGUE