OpenBLAS/kernel/ia64/gemm_ncopy.S

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#define PREFETCHSIZE   64
#define WPREFETCHSIZE  32

#ifndef XDOUBLE
#define LD	LDF8
#define ST	STF8_NTA
#else
#define LD	LDFD
#define ST	STFD_NTA
#endif

#define J	r15
#define PREB	r17
#define PREA	r18

#define A1	r19
#define A2	r20
#define A3	r21
#define A4	r22
#define A5	r23
#define A6	r24
#define A7	r25
#define A8	r26
#define B1	r27
#define B2	r28

#define COUNT	r9
#define I	r10
#define II	r11

#define ARLC	r29
#define PR	r30

#define M	r32
#define N	r33
#define A	r34
#define LDA	r35
#define B	r36

	PROLOGUE
	.prologue
	PROFCODE

	.body
	{ .mii
	shladd	LDA = LDA, BASE_SHIFT, r0
	mov	PR = pr
	shr	J = N, 3
	}
	;;
	{ .mib
	cmp.eq	p8,  p0 = 0, J
	mov	ARLC  = ar.lc
	(p8)  br.cond.dpnt .L20
	}
	;;
	.align 32

.L11:
	{ .mmi
	mov	A1 = A
	add	A2 = A, LDA
	mov	pr.rot = 0
	}
	{ .mmi
	shladd	A3 = LDA, 1, A
	shladd	A5 = LDA, 2, A
	adds	I = 1, M
	}
	;;
	{ .mmi
	shladd	A4 = LDA, 1, A2
	shladd	A6 = LDA, 2, A2
	mov	ar.ec  = 6
	}
	{ .mmi
	cmp.eq	p16, p0 = r0, r0
	shladd	A7 = LDA, 2, A3
	shr	I = I, 1
	}
	;;
	{ .mmi
	adds	B1 = 8 * SIZE, B
	shladd	A8 = LDA, 2, A4
	shladd	A = LDA, 3, A
	}
	{ .mmi
	adds	I = -1, I
	mov	COUNT = 0
	adds	J = -1, J
	}
	;;
	{ .mmi
	adds	PREA =  PREFETCHSIZE * SIZE, A
	adds	PREB = WPREFETCHSIZE * SIZE, B
	mov	ar.lc = I
	}
	{ .mmi
	mov	I  = M
	mov	II = M
	cmp.ne p14, p0 = r0, r0
	}
	;;
	.align 32

.L12:
	{ .mmi
	(p21) ST	[B ] = f37,  1 * SIZE
	(p14) ST	[B1] = f49,  1 * SIZE
	(p16) cmp.ne.unc p13, p0 = 1, I
	}
	{ .mmi
	lfetch.nt1	[PREA], LDA
	lfetch.excl.nt1 [PREB]
	adds	PREB = 16 * SIZE, PREB
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f43,  1 * SIZE
	(p14) ST	[B1] = f55,  1 * SIZE
	cmp.eq	p9, p0 = 8, COUNT
	}
	{ .mmi
	(p16) LD	f32  = [A1], SIZE
	(p16) LD	f38  = [A2], SIZE
	(p16) adds	I  = -2, I
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f61,  1 * SIZE
	(p14) ST	[B1] = f73,  1 * SIZE
	(p9) mov COUNT = 0
	}
	{ .mmi
	(p13) LD	f44  = [A1], SIZE
	(p13) LD	f50  = [A2], SIZE
	(p21) adds	II = -2, II
	}
	;;
	{ .mmb
	(p21) ST	[B ] = f67,  1 * SIZE
	(p14) ST	[B1] = f79,  1 * SIZE
	nop   __LINE__
	}
	{ .mmb
	(p16) LD	f56  = [A3], SIZE
	(p16) LD	f62  = [A4], SIZE
	nop   __LINE__
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f85,  1 * SIZE
	(p14) ST	[B1] = f97,  1 * SIZE
	(p9) adds PREA =  (PREFETCHSIZE - 2)* SIZE, A1
	}
	{ .mmb
	(p13) LD	f68  = [A3], SIZE
	(p13) LD	f74  = [A4], SIZE
	nop   __LINE__
	}
	;;
	{ .mmb
	(p21) ST	[B ] = f91,  1 * SIZE
	(p14) ST	[B1] = f103, 1 * SIZE
	nop   __LINE__
	}
	{ .mmb
	(p16) LD	f80  = [A5], SIZE
	(p16) LD	f86  = [A6], SIZE
	nop   __LINE__
	}
	;;
	{ .mmb
	(p21) ST	[B ] = f109, 1 * SIZE
	(p14) ST	[B1] = f121, 1 * SIZE
	nop   __LINE__
	}
	{ .mmb
	(p13) LD	f92  = [A5], SIZE
	(p13) LD	f98  = [A6], SIZE
	nop   __LINE__
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f115, 1 * SIZE
	(p14) ST	[B1] = f127, 9 * SIZE
	(p16) adds	COUNT = 1, COUNT
	}
	{ .mmb
	(p16) LD	f104 = [A7], SIZE
	(p16) LD	f110 = [A8], SIZE
	nop   __LINE__
	}
	;;
	{ .mmi
	(p13) LD	f116 = [A7], SIZE
	(p13) LD	f122 = [A8], SIZE
	(p14) adds	B = 8 * SIZE, B
	}
	{ .mmb
	(p20) cmp.ne.unc p14, p0 = 1, II
	nop   __LINE__
	br.ctop.sptk.few .L12
	}
	;;
	{ .mmb
	cmp.ne	p6, p0 = 0, J
	nop   __LINE__
	(p6) br.cond.dptk .L11
	}
	;;
	.align 32

.L20:
	{ .mmi
	adds	I = 1, M
	mov	A1 = A
	mov	pr.rot = 0
	}
	{ .mmi
	add	A2 = A, LDA
	shladd	A3 = LDA, 1, A
	tbit.z	p6, p0 = N, 2
	}
	;;
	{ .mmi
	shladd	A4 = LDA, 1, A2
	adds	B1 = 4 * SIZE, B
	mov	ar.ec  = 6
	}
	{ .mib
	cmp.eq	p16, p0 = r0, r0
	shr	I = I, 1
	(p6)  br.cond.dpnt .L30
	}
	;;
	{ .mmi
	shladd	A = LDA, 2, A
	nop	__LINE__
	nop	__LINE__
	}
	{ .mmi
	adds	I = -1, I
	mov	COUNT = 0
	adds	J = -1, J
	}
	;;
	{ .mmi
	adds	PREA =  PREFETCHSIZE * SIZE, A
	adds	PREB = WPREFETCHSIZE * SIZE, B
	mov	ar.lc = I
	}
	{ .mmi
	mov	I  = M
	mov	II = M
	cmp.ne p14, p0 = r0, r0
	}
	;;
	.align 32

.L22:
	{ .mmi
	(p21) ST	[B ] = f37,  1 * SIZE
	(p14) ST	[B1] = f49,  1 * SIZE
	(p16) cmp.ne.unc p13, p0 = 1, I
	}
	{ .mmi
	lfetch.nt1	[PREA], LDA
	lfetch.excl.nt1 [PREB], 8 * SIZE
	cmp.eq	p9, p0 = 4, COUNT
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f43,  1 * SIZE
	(p14) ST	[B1] = f55,  1 * SIZE
	(p16) adds	I  = -2, I
	}
	{ .mmi
	(p16) LD	f32  = [A1], SIZE
	(p16) LD	f38  = [A2], SIZE
	(p21) adds	II = -2, II
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f61,  1 * SIZE
	(p14) ST	[B1] = f73,  1 * SIZE
	(p9) mov COUNT = 0
	}
	{ .mmi
	(p13) LD	f44  = [A1], SIZE
	(p13) LD	f50  = [A2], SIZE
	nop   __LINE__
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f67,  1 * SIZE
	(p14) ST	[B1] = f79,  5 * SIZE
	(p9) adds PREA =  PREFETCHSIZE * SIZE, A1
	}
	{ .mmb
	(p16) LD	f56  = [A3], SIZE
	(p16) LD	f62  = [A4], SIZE
	nop   __LINE__
	}
	;;
	{ .mmi
	(p13) LD	f68  = [A3], SIZE
	(p13) LD	f74  = [A4], SIZE
	(p16) adds	COUNT = 1, COUNT
	}
	{ .mmb
	(p14) adds	B = 4 * SIZE, B
	(p20) cmp.ne.unc p14, p0 = 1, II
	br.ctop.sptk.few .L22
	}
	;;
	.align 32

.L30:
	{ .mmi
	adds	I = 1, M
	mov	A1 = A
	mov	pr.rot = 0
	}
	{ .mmi
	add	A2 = A, LDA
	adds	B1 = 2 * SIZE, B
	tbit.z	p6, p0 = N, 1
	}
	;;
	{ .mmi
	nop	__LINE__
	nop	__LINE__
	mov	ar.ec  = 6
	}
	{ .mib
	cmp.eq	p16, p0 = r0, r0
	shr	I = I, 1
	(p6)  br.cond.dpnt .L40
	}
	;;
	{ .mmi
	adds	I = -1, I
	;;
	shladd	A = LDA, 1, A
	mov	ar.lc = I
	}
	{ .mmi
	mov	I  = M
	mov	II = M
	cmp.ne p14, p0 = r0, r0
	}
	;;
	.align 32

.L32:
	{ .mmi
	(p21) ST	[B ] = f37,  1 * SIZE
	(p14) ST	[B1] = f49,  1 * SIZE
	(p16) cmp.ne.unc p13, p0 = 1, I
	}
	{ .mmi
	nop	__LINE__
	nop	__LINE__
	(p21) adds	II = -2, II
	}
	;;
	{ .mmi
	(p21) ST	[B ] = f43,  1 * SIZE
	(p14) ST	[B1] = f55,  3 * SIZE
	nop	__LINE__
	}
	{ .mmi
	(p16) LD	f32  = [A1], SIZE
	(p16) LD	f38  = [A2], SIZE
	nop	__LINE__
	}
	;;
	{ .mmi
	(p13) LD	f44  = [A1], SIZE
	(p13) LD	f50  = [A2], SIZE
	(p16) adds	I  = -2, I
	}
	{ .mmb
	(p14) adds	B = 2 * SIZE, B
	(p20) cmp.ne.unc p14, p0 = 1, II
	br.ctop.sptk.few .L32
	}
	;;
	.align 32

.L40:
	{ .mmi
	adds	I = 1, M
	mov	A1 = A
	mov	pr.rot = 0
	}
	{ .mmi
	tbit.z	p6, p0 = N, 0
	}
	;;
	{ .mmi
	nop	__LINE__
	nop	__LINE__
	mov	ar.ec  = 6
	}
	{ .mib
	cmp.eq	p16, p0 = r0, r0
	shr	I = I, 1
	(p6)  br.cond.dpnt .L999
	}
	;;
	{ .mmi
	adds	I = -1, I
	;;
	mov	ar.lc = I
	}
	{ .mmi
	mov	I  = M
	mov	II = M
	cmp.ne p14, p0 = r0, r0
	}
	;;
	.align 32

.L42:
	{ .mmi
	(p21) ST	[B ] = f37,  1 * SIZE
	(p16) cmp.ne.unc p13, p0 = 1, I
	(p21) adds	II = -2, II
	}
	;;
	{ .mmi
	(p14) ST	[B ] = f49,  1 * SIZE
	(p16) LD	f32  = [A1], SIZE
	(p16) adds	I  = -2, I
	}
	;;
	{ .mmb
	(p13) LD	f44  = [A1], SIZE
	(p20) cmp.ne.unc p14, p0 = 1, II
	br.ctop.sptk.few .L42
	}
	;;
	.align 32

.L999:
	mov pr    = PR, -1
	mov	 ar.lc = ARLC
	br.ret.sptk.many b0
	EPILOGUE