OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

loongarch64: Add zgemm and cgemm optimization

tags/v0.3.26
pengxu Shiyou Yin 1 year ago
parent
546f13558c
commit
a5d0d21378
14 changed files with 4621 additions and 5 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +4
    -0
      common_loongarch64.h
  2. +12
    -0
      kernel/loongarch64/KERNEL.LOONGSON2K1000
  3. +23
    -1
      kernel/loongarch64/KERNEL.LOONGSON3R5
  4. +857
    -0
      kernel/loongarch64/cgemm_kernel_2x2_lasx.S
  5. +812
    -0
      kernel/loongarch64/cgemm_kernel_2x2_lsx.S
  6. +193
    -0
      kernel/loongarch64/cgemm_ncopy_2_lasx.S
  7. +202
    -0
      kernel/loongarch64/cgemm_ncopy_2_lsx.S
  8. +218
    -0
      kernel/loongarch64/cgemm_tcopy_2_lasx.S
  9. +218
    -0
      kernel/loongarch64/cgemm_tcopy_2_lsx.S
  10. +848
    -0
      kernel/loongarch64/zgemm_kernel_2x2.S
  11. +822
    -0
      kernel/loongarch64/zgemm_kernel_2x2_lasx.S
  12. +196
    -0
      kernel/loongarch64/zgemm_ncopy_2_lasx.S
  13. +212
    -0
      kernel/loongarch64/zgemm_tcopy_2_lasx.S
  14. +4
    -4
      param.h

+ 4
- 0
common_loongarch64.h View File

@@ -144,6 +144,7 @@ static inline int WhereAmI(void){
#define XVCMPLT xvfcmp.clt.d
#define XVMUL xvfmul.d
#define XVMSUB xvfmsub.d
#define XVNMSUB xvfnmsub.d

#define VFSUB vfsub.d
#define VFADD vfadd.d
@@ -158,6 +159,7 @@ static inline int WhereAmI(void){
#define VCMPLT vfcmp.clt.d
#define VMUL vfmul.d
#define VMSUB vfmsub.d
#define VNMSUB vfnmsub.d

#else

@@ -198,6 +200,7 @@ static inline int WhereAmI(void){
#define XVCMPLT xvfcmp.clt.s
#define XVMUL xvfmul.s
#define XVMSUB xvfmsub.s
#define XVNMSUB xvfnmsub.s

#define VFSUB vfsub.s
#define VFADD vfadd.s
@@ -212,6 +215,7 @@ static inline int WhereAmI(void){
#define VCMPLT vfcmp.clt.s
#define VMUL vfmul.s
#define VMSUB vfmsub.s
#define VNMSUB vfnmsub.s

#endif /* defined(DOUBLE) */



+ 12
- 0
kernel/loongarch64/KERNEL.LOONGSON2K1000 View File

@@ -95,4 +95,16 @@ DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
DTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
DTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c

CGEMMKERNEL = cgemm_kernel_2x2_lsx.S
CGEMMONCOPY = cgemm_ncopy_2_lsx.S
CGEMMOTCOPY = cgemm_tcopy_2_lsx.S
CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX)
CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)

CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c

endif

+ 23
- 1
kernel/loongarch64/KERNEL.LOONGSON3R5 View File

@@ -107,13 +107,35 @@ SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX)
SGEMVNKERNEL = sgemv_n_8_lasx.S
SGEMVTKERNEL = sgemv_t_8_lasx.S

CGEMMKERNEL = cgemm_kernel_2x2_lsx.S
CGEMMONCOPY = cgemm_ncopy_2_lsx.S
CGEMMOTCOPY = cgemm_tcopy_2_lsx.S
CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX)
CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX)

CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
CTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
CTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c

ZGEMMKERNEL = zgemm_kernel_2x2_lasx.S
ZGEMMONCOPY = zgemm_ncopy_2_lasx.S
ZGEMMOTCOPY = zgemm_tcopy_2_lasx.S
ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX)
ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX)

ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
ZTRSMKERNEL_RT = ../generic/trsm_kernel_RT.c

DTRSMKERNEL_LN = dtrsm_kernel_LN_16x4_lasx.S
DTRSMKERNEL_LT = dtrsm_kernel_LT_16x4_lasx.S
DTRSMKERNEL_RN = dtrsm_kernel_RN_16x4_lasx.S
DTRSMKERNEL_RT = dtrsm_kernel_RT_16x4_lasx.S
endif

STRSMKERNEL_LN = ../generic/trsm_kernel_LN.c
STRSMKERNEL_LT = ../generic/trsm_kernel_LT.c
STRSMKERNEL_RN = ../generic/trsm_kernel_RN.c
STRSMKERNEL_RT = ../generic/trsm_kernel_RT.c
endif

+ 857
- 0
kernel/loongarch64/cgemm_kernel_2x2_lasx.S View File

@@ -0,0 +1,857 @@
/*******************************************************************************
Copyright (c) 2023, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"


/* Function parameters */
#define M $r4 // param 1: bm
#define N $r5 // param 2: bn
#define K $r6 // param 3: bk
#define ALPHA_R $f0 // param 4: alphar
#define ALPHA_I $f1 // param 5: alphai
#define A $r7 // param 6: ba
#define B $r8 // param 7: bb
#define C $r9 // param 8: bc
#define LDC $r10 // param 9: ldc

#if defined (TRMMKERNEL)
#define OFFSET $r11 // param 10: offset
#endif
#define OFF $r26

#define I $r12
#define J $r13
#define L $r14
#define TL $r15
#define A0 $r16
#define B0 $r17
#define C0 $r18
#define C1 $r19
#define C2 $r20
#define C3 $r23
#define T0 $r24
#define T1 $r25

#define a1 $f2
#define a2 $f3
#define a3 $f4
#define a4 $f5
#define a5 $f6
#define a6 $f7
#define a7 $f8
#define a8 $f9
#define b1 $f10
#define b2 $f11
#define b3 $f12
#define b4 $f13
#define b5 $f14
#define b6 $f15
#define b7 $f16
#define b8 $f17
#define c11 $f18
#define c12 $f19
#define c21 $f20
#define c22 $f21
#define c31 $f22
#define c32 $f23
#define c41 $f24
#define c42 $f25

/* LASX vectors */
#define U0 $xr30
#define U1 $xr31
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define U8 $xr8
#define U9 $xr9
#define U10 $xr10
#define U11 $xr11
#define U12 $xr12
#define U13 $xr13
#define U14 $xr14
#define U15 $xr15
#define D0 $xr16
#define D1 $xr17
#define D2 $xr18
#define D3 $xr19
#define D4 $xr20
#define D5 $xr21
#define D6 $xr22
#define D7 $xr23
#define D8 $xr24
#define D9 $xr25
#define D10 $xr26
#define D11 $xr27
#define VALPHAR $xr28
#define VALPHAI $xr29


#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define XVMADD1 XVFMADD
#define XVMADD2 XVFMADD
#define XVMADD3 XVNMSUB
#define XVMADD4 XVFMADD

#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VNMSUB
#define VMADD4 VFMADD

#define XVFADD1 XVFADD
#define XVFADD2 XVFADD
#define XVFADD3 XVFSUB
#define XVFADD4 XVFADD

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 NMSUB
#define MADD4 MADD
#endif

#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define XVMADD1 XVFMADD
#define XVMADD2 XVFMADD
#define XVMADD3 XVFMADD
#define XVMADD4 XVNMSUB

#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VFMADD
#define VMADD4 VNMSUB

#define XVFADD1 XVFADD
#define XVFADD2 XVFADD
#define XVFADD3 XVFADD
#define XVFADD4 XVFSUB

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 MADD
#define MADD4 NMSUB
#endif

#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define XVMADD1 XVFMADD
#define XVMADD2 XVNMSUB
#define XVMADD3 XVFMADD
#define XVMADD4 XVFMADD

#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VFMADD
#define VMADD4 VFMADD

#define XVFADD1 XVFADD
#define XVFADD2 XVFSUB
#define XVFADD3 XVFADD
#define XVFADD4 XVFADD

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 MADD
#define MADD4 MADD
#endif

#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define XVMADD1 XVFMADD
#define XVMADD2 XVNMSUB
#define XVMADD3 XVNMSUB
#define XVMADD4 XVNMSUB

#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VNMSUB
#define VMADD4 VNMSUB

#define XVFADD1 XVFADD
#define XVFADD2 XVFSUB
#define XVFADD3 XVFSUB
#define XVFADD4 XVFSUB

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 NMSUB
#define MADD4 NMSUB
#endif

PROLOGUE

addi.d $sp, $sp, -128
SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
SDARG $r26, $sp, 24
SDARG $r27, $sp, 32
ST $f23, $sp, 40
ST $f24, $sp, 48
ST $f25, $sp, 56
ST $f26, $sp, 64
ST $f27, $sp, 72
ST $f28, $sp, 80
ST $f29, $sp, 88
ST $f30, $sp, 96
ST $f31, $sp, 104
ST ALPHA_R,$sp, 112
ST ALPHA_I,$sp, 120

xvldrepl.w VALPHAR, $sp, 112
xvldrepl.w VALPHAI, $sp, 120

#if defined (TRMMKERNEL) && !defined(LEFT)
sub.d OFF, $r0, OFFSET
#else
xor OFF, OFF, OFF
#endif

slli.d LDC, LDC, 2

move J, $r0
srai.d T0, N, 1
beq J, T0, .L19

.L10: /* for(j=0; j<bn/2; j+=1) */
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
move A0, A //ptrba

#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move I, $r0
srai.d T0, M, 1
beq I, T0, .L150

.L11: /* for(i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF //temp
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

xvxor.v U0, U0, U0
xvxor.v U1, U1, U1

move L, $r0 //cycle param k
srai.d C2, TL, 2
beq L, C2, .L130
blt C2, L, .L130

.L12: /* for(k=0; k<bk/4; k+=1) */
xvld D0, A0, 0x00 //a 0-7
xvld D1, A0, 0x20 //a 8-15
xvld D2, B0, 0x00 //b 0-7
xvld D3, B0, 0x20 //b 8-15

xvand.v D4, D0, D0
xvpermi.q D4, D1, 0x02 //a 0 1 2 3 8 9 10 11
xvand.v D5, D4, D4
xvshuf4i.w D4, D4, 0x88 //a 0 2 0 2 8 10 8 10
xvshuf4i.w D5, D5, 0xdd //a 1 3 1 3 9 11 9 11

xvand.v D6, D1, D1
xvpermi.q D6, D0, 0x31 //a 4 5 6 7 12 13 14 15
xvand.v D7, D6, D6
xvshuf4i.w D6, D6, 0x88 //a 4 6 4 6 12 14 12 14
xvshuf4i.w D7, D7, 0xdd //a 5 7 5 7 13 15 13 15

xvand.v D8, D2, D2
xvpermi.q D8, D3, 0x02 //b 0 1 2 3 8 9 10 11
xvand.v D9, D8, D8
xvshuf4i.w D8, D8, 0xa0 //b 0 0 2 2 8 8 10 10
xvshuf4i.w D9, D9, 0xf5 //a 1 1 3 3 9 9 11 11

xvand.v D10, D3, D3
xvpermi.q D10, D2, 0x31 //b 4 5 6 7 12 13 14 15
xvand.v D11, D10, D10
xvshuf4i.w D10, D10, 0xa0 //b 4 4 6 6 12 12 14 14
xvshuf4i.w D11, D11, 0xf5 //a 5 5 7 7 13 13 15 15

XVMADD1 U0, D4, D8, U0 //res0 2 4 6 0 2 4 6
XVMADD2 U1, D5, D8, U1 //res1 3 4 7 1 3 4 7

xvpermi.q U0, U0, 0x01
xvpermi.q U1, U1, 0x01
XVMADD1 U0, D4, D8, U0
XVMADD2 U1, D5, D8, U1

XVMADD3 U0, D5, D9, U0
XVMADD4 U1, D4, D9, U1

xvpermi.q U0, U0, 0x01
xvpermi.q U1, U1, 0x01
XVMADD3 U0, D5, D9, U0
XVMADD4 U1, D4, D9, U1

XVMADD1 U0, D6, D10, U0 //res0 2 4 6 0 2 4 6
XVMADD2 U1, D7, D10, U1 //res1 3 4 7 1 3 4 7

xvpermi.q U0, U0, 0x01
xvpermi.q U1, U1, 0x01
XVMADD1 U0, D6, D10, U0
XVMADD2 U1, D7, D10, U1

XVMADD3 U0, D7, D11, U0
XVMADD4 U1, D6, D11, U1

xvpermi.q U0, U0, 0x01
xvpermi.q U1, U1, 0x01
XVMADD3 U0, D7, D11, U0
XVMADD4 U1, D6, D11, U1

addi.d A0, A0, 0x40
addi.d B0, B0, 0x40

addi.d L, L, 1
blt L, C2, .L12

.L130:
move L, $r0
andi C2, TL, 3
beq L, C2, .L14

.L13: /* for(k=0; k<(bk&3); k+=1) */
vld $vr16, A0, 0x00 //a0 a1 a2 a3
vld $vr17, B0, 0x00 //b0 b1 b2 b3

vshuf4i.w $vr20, $vr17, 0xa0 //b0 b0 b2 b2
vshuf4i.w $vr21, $vr17, 0xf5 //b1 b1 b3 b3

vshuf4i.w $vr18, $vr16, 0x88 //a0 a2 a0 a2
vshuf4i.w $vr19, $vr16, 0xdd //a1 a3 a1 a3

VMADD1 $vr30, $vr18, $vr20, $vr30 //res0 2 4 6
VMADD2 $vr31, $vr19, $vr20, $vr31 //res1 3 5 7
VMADD3 $vr30, $vr19, $vr21, $vr30
VMADD4 $vr31, $vr18, $vr21, $vr31

addi.d A0, A0, 0x10
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, C2, .L13

.L14:
#if defined(TRMMKERNEL)
vld $vr8, C0, 0x00 //0 1 2 3
vld $vr9, C1, 0x00 //4 5 6 7

vpackev.w $vr10, $vr9, $vr8 //0 4 2 6
vpermi.w $vr10, $vr10, 0xd8 //0 2 4 6

vpackod.w $vr11, $vr9, $vr8 //1 5 3 7
vpermi.w $vr11, $vr11, 0xd8 //1 3 5 7

vfmul.s $vr10, $vr30, $vr28
vfmul.s $vr11, $vr31, $vr28
VNMSUB $vr10, $vr31, $vr29, $vr10
VFMADD $vr11, $vr30, $vr29, $vr11

vilvl.w $vr8, $vr11, $vr10 //0 1 2 3

vilvh.w $vr9, $vr11, $vr10 //4 5 6 7

vst $vr8, C0, 0x00
vst $vr9, C1, 0x00
#else
vld $vr8, C0, 0x00 //0 1 2 3
vld $vr9, C1, 0x00 //4 5 6 7

vpackev.w $vr10, $vr9, $vr8 //0 4 2 6
vpermi.w $vr10, $vr10, 0xd8 //0 2 4 6

vpackod.w $vr11, $vr9, $vr8 //1 5 3 7
vpermi.w $vr11, $vr11, 0xd8 //1 3 5 7

VFMADD $vr10, $vr30, $vr28, $vr10
VFMADD $vr11, $vr31, $vr28, $vr11
VNMSUB $vr10, $vr31, $vr29, $vr10
VFMADD $vr11, $vr30, $vr29, $vr11

vilvl.w $vr8, $vr11, $vr10 //0 1 2 3

vilvh.w $vr9, $vr11, $vr10 //4 5 6 7

vst $vr8, C0, 0x00
vst $vr9, C1, 0x00
#endif

#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10
addi.d C1, C1, 0x10

addi.d I, I, 1
blt I, T0, .L11

.L150:
move I, $r0
andi T0, M, 1
beq I, T0, .L18

.L15: /* for(i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x03
add.d A0, A0, C3
slli.d C3, OFF, 0x04
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L17
blt TL, L, .L17

.L16: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD b3, B0, 0x08 //load4
MADD1 c21, a1, b3, c21 //res2
MADD2 c22, a2, b3, c22 //res3
LD b4, B0, 0x0c //load5
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22

addi.d A0, A0, 0x08
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L16

.L17:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04

MUL b5, c21, ALPHA_R
MUL b6, c22, ALPHA_I
SUB b5, b5, b6
ST b5, C1, 0x00

MUL b5, c22, ALPHA_R
MUL b6, c21, ALPHA_I
ADD b6, b5, b6
ST b6, C1, 0x04
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD b5, C1, 0x00 //C1[0]
LD b6, C1, 0x04 //C1[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x04

MADD b5, c21, ALPHA_R, b5
MADD b6, c22, ALPHA_R, b6
NMSUB b5, c22, ALPHA_I, b5
MADD b6, c21, ALPHA_I, b6
ST b5, C1, 0x00
ST b6, C1, 0x04
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x03
add.d A0, A0, C3
slli.d C3, TL, 0x04
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x08
addi.d C1, C1, 0x08

addi.d I, I, 1
blt I, T0, .L15

.L18:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 2
#endif

slli.d L, K, 0x04
add.d B, B, L

slli.d I, LDC, 0x02
add.d C, C, I

addi.d J, J, 1
srai.d T0, N, 1
blt J, T0, .L10

.L19:
move J, $r0
andi T0, N, 1
beq J, T0, .L30

.L20: /* for (j=0; j<(bn&1); j+=1) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move C0, C
move A0, A //ptrba

move I, $r0
srai.d T0, M, 1
beq I, T0, .L24

.L21: /* for (i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
slli.d C3, OFF, 0x03
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L23
blt TL, L, .L23

.L22: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x08 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x0c //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22

addi.d A0, A0, 0x10
addi.d B0, B0, 0x08

addi.d L, L, 1
blt L, TL, .L22

.L23:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04

MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_I
SUB a7, a7, a8
ST a7, C0, 0x08

MUL a7, c22, ALPHA_R
MUL a8, c21, ALPHA_I
ADD a8, a7, a8
ST a8, C0, 0x0c
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C0, 0x08 //C1[2]
LD a8, C0, 0x0c //C1[3]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8

ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C0, 0x08
ST a8, C0, 0x0c
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
slli.d C3, TL, 0x03
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10

addi.d I, I, 1
blt I, T0, .L21

.L24:
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L28

.L25: /* for (i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x03
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0

move L, $r0 //cycle param k
beq L, TL, .L27
blt TL, L, .L27

.L26: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

addi.d A0, A0, 0x08
addi.d B0, B0, 0x08

addi.d L, L, 1
blt L, TL, .L26

.L27:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6

ST a5, C0, 0x00
ST a6, C0, 0x04
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x03
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x08

addi.d I, I, 1
blt I, T1, .L25

.L28:
slli.d L, K, 3
add.d B, B, L

slli.d I, LDC, 1
add.d C, C, I

addi.d J, J, 1
andi T0, N, 1
blt J, T0, .L20

.L30:
LDARG $r23, $sp, 0
LDARG $r24, $sp, 8
LDARG $r25, $sp, 16
LDARG $r26, $sp, 24
LDARG $r27, $sp, 32
LD $f23, $sp, 40
LD $f24, $sp, 48
LD $f25, $sp, 56
LD $f26, $sp, 64
LD $f27, $sp, 72
LD $f28, $sp, 80
LD $f29, $sp, 88
LD $f30, $sp, 96
LD $f31, $sp, 104

addi.d $sp, $sp, 128
jirl $r0, $r1, 0x0

EPILOGUE

+ 812
- 0
kernel/loongarch64/cgemm_kernel_2x2_lsx.S View File

@@ -0,0 +1,812 @@
/*******************************************************************************
Copyright (c) 2023, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"


/* Function parameters */
#define M $r4 // param 1: bm
#define N $r5 // param 2: bn
#define K $r6 // param 3: bk
#define ALPHA_R $f0 // param 4: alphar
#define ALPHA_I $f1 // param 5: alphai
#define A $r7 // param 6: ba
#define B $r8 // param 7: bb
#define C $r9 // param 8: bc
#define LDC $r10 // param 9: ldc

#if defined (TRMMKERNEL)
#define OFFSET $r11 // param 10: offset
#endif
#define OFF $r26

#define I $r12
#define J $r13
#define L $r14
#define TL $r15
#define A0 $r16
#define B0 $r17
#define C0 $r18
#define C1 $r19
#define C2 $r20
#define C3 $r23
#define T0 $r24
#define T1 $r25

#define a1 $f2
#define a2 $f3
#define a3 $f4
#define a4 $f5
#define a5 $f6
#define a6 $f7
#define a7 $f8
#define a8 $f9
#define b1 $f10
#define b2 $f11
#define b3 $f12
#define b4 $f13
#define b5 $f14
#define b6 $f15
#define b7 $f16
#define b8 $f17
#define c11 $f18
#define c12 $f19
#define c21 $f20
#define c22 $f21
#define c31 $f22
#define c32 $f23
#define c41 $f24
#define c42 $f25

/* LASX vectors */
#define U0 $vr30
#define U1 $vr31
#define U2 $vr2
#define U3 $vr3
#define U4 $vr4
#define U5 $vr5
#define U6 $vr6
#define U7 $vr7
#define U8 $vr8
#define U9 $vr9
#define U10 $vr10
#define U11 $vr11
#define U12 $vr12
#define U13 $vr13
#define U14 $vr14
#define U15 $vr15
#define D0 $vr16
#define D1 $vr17
#define D2 $vr18
#define D3 $vr19
#define D4 $vr20
#define D5 $vr21
#define D6 $vr22
#define D7 $vr23
#define D8 $vr24
#define D9 $vr25
#define D10 $vr26
#define D11 $vr27
#define VALPHAR $vr28
#define VALPHAI $vr29


#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VNMSUB
#define VMADD4 VFMADD

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 NMSUB
#define MADD4 MADD
#endif

#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define VMADD1 VFMADD
#define VMADD2 VFMADD
#define VMADD3 VFMADD
#define VMADD4 VNMSUB

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 MADD
#define MADD4 NMSUB
#endif

#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VFMADD
#define VMADD4 VFMADD

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 MADD
#define MADD4 MADD
#endif

#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define VMADD1 VFMADD
#define VMADD2 VNMSUB
#define VMADD3 VNMSUB
#define VMADD4 VNMSUB

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 NMSUB
#define MADD4 NMSUB
#endif

PROLOGUE

addi.d $sp, $sp, -128
SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
SDARG $r26, $sp, 24
SDARG $r27, $sp, 32
ST $f23, $sp, 40
ST $f24, $sp, 48
ST $f25, $sp, 56
ST $f26, $sp, 64
ST $f27, $sp, 72
ST $f28, $sp, 80
ST $f29, $sp, 88
ST $f30, $sp, 96
ST $f31, $sp, 104
ST ALPHA_R,$sp, 112
ST ALPHA_I,$sp, 120

vldrepl.w VALPHAR, $sp, 112
vldrepl.w VALPHAI, $sp, 120

#if defined (TRMMKERNEL) && !defined(LEFT)
sub.d OFF, $r0, OFFSET
#else
xor OFF, OFF, OFF
#endif

slli.d LDC, LDC, 2

move J, $r0
srai.d T0, N, 1
beq J, T0, .L19

.L10: /* for(j=0; j<bn/2; j+=1) */
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
move A0, A //ptrba

#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move I, $r0
srai.d T0, M, 1
beq I, T0, .L150

.L11: /* for(i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF //temp
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

vxor.v U0, U0, U0
vxor.v U1, U1, U1

move L, $r0 //cycle param k
srai.d C2, TL, 2
beq L, C2, .L130
blt C2, L, .L130

.L12: /* for(k=0; k<bk/4; k+=1) */
vld D0, A0, 0x00 //a0 a1 a2 a3
vld D1, B0, 0x00 //b0 b1 b2 b3

vshuf4i.w D4, D1, 0xa0 //b0 b0 b2 b2
vshuf4i.w D5, D1, 0xf5 //b1 b1 b3 b3

vshuf4i.w D2, D0, 0x88 //a0 a2 a0 a2
vshuf4i.w D3, D0, 0xdd //a1 a3 a1 a3

VMADD1 U0, D2, D4, U0 //res0 2 4 6
VMADD2 U1, D3, D4, U1 //res1 3 4 7
VMADD3 U0, D3, D5, U0
VMADD4 U1, D2, D5, U1

vld D0, A0, 0x10 //a0 a1 a2 a3
vld D1, B0, 0x10 //b0 b1 b2 b3

vshuf4i.w D4, D1, 0xa0 //b0 b0 b2 b2
vshuf4i.w D5, D1, 0xf5 //b1 b1 b3 b3

vshuf4i.w D2, D0, 0x88 //a0 a2 a0 a2
vshuf4i.w D3, D0, 0xdd //a1 a3 a1 a3

VMADD1 U0, D2, D4, U0 //res0 2 4 6
VMADD2 U1, D3, D4, U1 //res1 3 4 7
VMADD3 U0, D3, D5, U0
VMADD4 U1, D2, D5, U1

vld D0, A0, 0x20 //a0 a1 a2 a3
vld D1, B0, 0x20 //b0 b1 b2 b3

vshuf4i.w D4, D1, 0xa0 //b0 b0 b2 b2
vshuf4i.w D5, D1, 0xf5 //b1 b1 b3 b3

vshuf4i.w D2, D0, 0x88 //a0 a2 a0 a2
vshuf4i.w D3, D0, 0xdd //a1 a3 a1 a3

VMADD1 U0, D2, D4, U0 //res0 2 4 6
VMADD2 U1, D3, D4, U1 //res1 3 4 7
VMADD3 U0, D3, D5, U0
VMADD4 U1, D2, D5, U1

vld D0, A0, 0x30 //a0 a1 a2 a3
vld D1, B0, 0x30 //b0 b1 b2 b3

vshuf4i.w D4, D1, 0xa0 //b0 b0 b2 b2
vshuf4i.w D5, D1, 0xf5 //b1 b1 b3 b3

vshuf4i.w D2, D0, 0x88 //a0 a2 a0 a2
vshuf4i.w D3, D0, 0xdd //a1 a3 a1 a3

VMADD1 U0, D2, D4, U0 //res0 2 4 6
VMADD2 U1, D3, D4, U1 //res1 3 4 7
VMADD3 U0, D3, D5, U0
VMADD4 U1, D2, D5, U1

addi.d A0, A0, 0x40
addi.d B0, B0, 0x40

addi.d L, L, 1
blt L, C2, .L12

.L130:
move L, $r0
andi C2, TL, 3
beq L, C2, .L14

.L13: /* for(k=0; k<(bk&3); k+=1) */
vld D0, A0, 0x00 //a0 a1 a2 a3
vld D1, B0, 0x00 //b0 b1 b2 b3

vshuf4i.w D4, D1, 0xa0 //b0 b0 b2 b2
vshuf4i.w D5, D1, 0xf5 //b1 b1 b3 b3

vshuf4i.w D2, D0, 0x88 //a0 a2 a0 a2
vshuf4i.w D3, D0, 0xdd //a1 a3 a1 a3

VMADD1 U0, D2, D4, U0 //res0 2 4 6
VMADD2 U1, D3, D4, U1 //res1 3 5 7
VMADD3 U0, D3, D5, U0
VMADD4 U1, D2, D5, U1

addi.d A0, A0, 0x10
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, C2, .L13

.L14:
#if defined(TRMMKERNEL)
vld U8, C0, 0x00 //0 1 2 3
vld U9, C1, 0x00 //4 5 6 7

vpackev.w U10, U9, U8 //0 4 2 6
vpermi.w U10, U10, 0xd8 //0 2 4 6

vpackod.w U11, U9, U8 //1 5 3 7
vpermi.w U11, U11, 0xd8 //1 3 5 7

vfmul.s U10, U0, VALPHAR
vfmul.s U11, U1, VALPHAR
VNMSUB U10, U1, VALPHAI, U10
VFMADD U11, U0, VALPHAI, U11

vilvl.w U8, U11, U10 //0 1 2 3

vilvh.w U9, U11, U10 //4 5 6 7

vst U8, C0, 0x00
vst U9, C1, 0x00
#else
vld U8, C0, 0x00 //0 1 2 3
vld U9, C1, 0x00 //4 5 6 7

vpackev.w U10, U9, U8 //0 4 2 6
vpermi.w U10, U10, 0xd8 //0 2 4 6

vpackod.w U11, U9, U8 //1 5 3 7
vpermi.w U11, U11, 0xd8 //1 3 5 7

VFMADD U10, U0, VALPHAR, U10
VFMADD U11, U1, VALPHAR, U11
VNMSUB U10, U1, VALPHAI, U10
VFMADD U11, U0, VALPHAI, U11

vilvl.w U8, U11, U10 //0 1 2 3

vilvh.w U9, U11, U10 //4 5 6 7

vst U8, C0, 0x00
vst U9, C1, 0x00
#endif

#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10
addi.d C1, C1, 0x10

addi.d I, I, 1
blt I, T0, .L11

.L150:
move I, $r0
andi T0, M, 1
beq I, T0, .L18

.L15: /* for(i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x03
add.d A0, A0, C3
slli.d C3, OFF, 0x04
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L17
blt TL, L, .L17

.L16: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD b3, B0, 0x08 //load4
MADD1 c21, a1, b3, c21 //res2
MADD2 c22, a2, b3, c22 //res3
LD b4, B0, 0x0c //load5
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22

addi.d A0, A0, 0x08
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L16

.L17:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04

MUL b5, c21, ALPHA_R
MUL b6, c22, ALPHA_I
SUB b5, b5, b6
ST b5, C1, 0x00

MUL b5, c22, ALPHA_R
MUL b6, c21, ALPHA_I
ADD b6, b5, b6
ST b6, C1, 0x04
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD b5, C1, 0x00 //C1[0]
LD b6, C1, 0x04 //C1[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x04

MADD b5, c21, ALPHA_R, b5
MADD b6, c22, ALPHA_R, b6
NMSUB b5, c22, ALPHA_I, b5
MADD b6, c21, ALPHA_I, b6
ST b5, C1, 0x00
ST b6, C1, 0x04
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x03
add.d A0, A0, C3
slli.d C3, TL, 0x04
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x08
addi.d C1, C1, 0x08

addi.d I, I, 1
blt I, T0, .L15

.L18:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 2
#endif

slli.d L, K, 0x04
add.d B, B, L

slli.d I, LDC, 0x02
add.d C, C, I

addi.d J, J, 1
srai.d T0, N, 1
blt J, T0, .L10

.L19:
move J, $r0
andi T0, N, 1
beq J, T0, .L30

.L20: /* for (j=0; j<(bn&1); j+=1) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move C0, C
move A0, A //ptrba

move I, $r0
srai.d T0, M, 1
beq I, T0, .L24

.L21: /* for (i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
slli.d C3, OFF, 0x03
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L23
blt TL, L, .L23

.L22: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x08 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x0c //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22

addi.d A0, A0, 0x10
addi.d B0, B0, 0x08

addi.d L, L, 1
blt L, TL, .L22

.L23:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04

MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_I
SUB a7, a7, a8
ST a7, C0, 0x08

MUL a7, c22, ALPHA_R
MUL a8, c21, ALPHA_I
ADD a8, a7, a8
ST a8, C0, 0x0c
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]
LD a7, C0, 0x08 //C1[2]
LD a8, C0, 0x0c //C1[3]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8

ST a5, C0, 0x00
ST a6, C0, 0x04
ST a7, C0, 0x08
ST a8, C0, 0x0c
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
slli.d C3, TL, 0x03
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10

addi.d I, I, 1
blt I, T0, .L21

.L24:
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L28

.L25: /* for (i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x03
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0

move L, $r0 //cycle param k
beq L, TL, .L27
blt TL, L, .L27

.L26: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x04 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x04 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

addi.d A0, A0, 0x08
addi.d B0, B0, 0x08

addi.d L, L, 1
blt L, TL, .L26

.L27:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x04
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x04 //C0[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6

ST a5, C0, 0x00
ST a6, C0, 0x04
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x03
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x08

addi.d I, I, 1
blt I, T1, .L25

.L28:
slli.d L, K, 3
add.d B, B, L

slli.d I, LDC, 1
add.d C, C, I

addi.d J, J, 1
andi T0, N, 1
blt J, T0, .L20

.L30:
LDARG $r23, $sp, 0
LDARG $r24, $sp, 8
LDARG $r25, $sp, 16
LDARG $r26, $sp, 24
LDARG $r27, $sp, 32
LD $f23, $sp, 40
LD $f24, $sp, 48
LD $f25, $sp, 56
LD $f26, $sp, 64
LD $f27, $sp, 72
LD $f28, $sp, 80
LD $f29, $sp, 88
LD $f30, $sp, 96
LD $f31, $sp, 104

addi.d $sp, $sp, 128
jirl $r0, $r1, 0x0

EPILOGUE

+ 193
- 0
kernel/loongarch64/cgemm_ncopy_2_lasx.S View File

@@ -0,0 +1,193 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define S5 $r16
#define S6 $r17
#define S7 $r18
#define S8 $r19
#define TD $r20
#define TS $r11
#define TL $r7
#define T0 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $xr0
#define U1 $xr1
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define D0 $xr8
#define D1 $xr9
#define D2 $xr10
#define D3 $xr11
#define D4 $xr12
#define D5 $xr13
#define D6 $xr14
#define D7 $xr15
#define D8 $xr16

PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TD, DST //boffset
move TS, SRC //aoffset

slli.d TL, LDA, 0x02 //lda
slli.d TL, TL, 0x01
slli.d T0, TL, 0x01

srai.d I, N, 0x01
beq I, ZERO, .L_N0

.L_J1: /* if (i > 0) I-- */
move S1, TS //a_offset1
add.d S2, TS, TL //a_offset2
srai.d J, M, 0x02
add.d TS, TS, T0

beq J, ZERO, .L_I3

.L_I1: /* if (j > 0) J-- */
xvld U0, S1, 0x00
xvld U1, S1, 0x00
xvld U2, S2, 0x00

xvpermi.q U0, U2, 0x02
xvpermi.q U2, U1, 0x31

xvpermi.d U0, U0, 0xd8
xvpermi.d U2, U2, 0xd8

xvst U0, TD, 0x00
xvst U2, TD, 0x20

addi.d S1, S1, 0x20 // a_offset1
addi.d S2, S2, 0x20
addi.d TD, TD, 0x40 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_I1

.L_I3:
andi J, M, 0x03
beq J, ZERO, .L_II20

.L_II1: /* j = (m & 3) if (j > 0) */
fld.s F0, S1, 0x00
fld.s F1, S1, 0x04
fld.s F2, S2, 0x00
fld.s F3, S2, 0x04

fst.s F0, TD, 0x00
fst.s F1, TD, 0x04
fst.s F2, TD, 0x08
fst.s F3, TD, 0x0c

addi.d S1, S1, 0x08
addi.d S2, S2, 0x08
addi.d TD, TD, 0x10

addi.d J, J, -1
blt ZERO, J, .L_II1

.L_II20:
addi.d I, I, -1
blt ZERO, I, .L_J1

.L_N0: /* if(n&1)*/
andi I, N, 0x01
beq ZERO, I, .L_N00

.L_N1:
srai.d J, M, 0x02
beq ZERO, J, .L_N10

.L_N11: /* j = (m >> 2) if (j > 0) */
xvld U0, TS, 0x00

xvst U0, TD, 0x00

addi.d TS, TS, 0x20 // a_offset
addi.d TD, TD, 0x20 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N11

.L_N10:
andi J, M, 0x03
beq J, ZERO, .L_N00

.L_N12: /* j = (m & 3) if (j > 0) */
fld.s F0, TS, 0x00
fld.s F1, TS, 0x04

fst.s F0, TD, 0x00
fst.s F1, TD, 0x04

addi.d TS, TS, 0x08 // a_offset
addi.d TD, TD, 0x08 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N12

.L_N00:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 202
- 0
kernel/loongarch64/cgemm_ncopy_2_lsx.S View File

@@ -0,0 +1,202 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define S5 $r16
#define S6 $r17
#define S7 $r18
#define S8 $r19
#define TD $r20
#define TS $r11
#define TL $r7
#define T0 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $vr0
#define U1 $vr1
#define U2 $vr2
#define U3 $vr3
#define U4 $vr4
#define U5 $vr5
#define U6 $vr6
#define U7 $vr7
#define D0 $vr8
#define D1 $vr9
#define D2 $vr10
#define D3 $vr11
#define D4 $vr12
#define D5 $vr13
#define D6 $vr14
#define D7 $vr15
#define D8 $vr16

PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TD, DST //boffset
move TS, SRC //aoffset

slli.d TL, LDA, 0x02 //lda
slli.d TL, TL, 0x01
slli.d T0, TL, 0x01

srai.d I, N, 0x01
beq I, ZERO, .L_N0

.L_J1: /* if (i > 0) I-- */
move S1, TS //a_offset1
add.d S2, TS, TL //a_offset2
srai.d J, M, 0x02
add.d TS, TS, T0

beq J, ZERO, .L_I3

.L_I1: /* if (j > 0) J-- */
vld U0, S1, 0x00
vld U1, S1, 0x10
vld U2, S2, 0x00
vld U3, S2, 0x10

vand.v D0, U2, U2
vand.v D1, U3, U3
vand.v D2, U2, U2
vand.v D3, U3, U3

vpermi.w D0, U0, 0x44
vpermi.w D2, U0, 0xee
vpermi.w D1, U1, 0x44
vpermi.w D3, U1, 0xee

vst D0, TD, 0x00
vst D2, TD, 0x10
vst D1, TD, 0x20
vst D3, TD, 0x30

addi.d S1, S1, 0x20 // a_offset1
addi.d S2, S2, 0x20
addi.d TD, TD, 0x40 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_I1

.L_I3:
andi J, M, 0x03
beq J, ZERO, .L_II20

.L_II1: /* j = (m & 3) if (j > 0) */
fld.s F0, S1, 0x00
fld.s F1, S1, 0x04
fld.s F2, S2, 0x00
fld.s F3, S2, 0x04

fst.s F0, TD, 0x00
fst.s F1, TD, 0x04
fst.s F2, TD, 0x08
fst.s F3, TD, 0x0c

addi.d S1, S1, 0x08
addi.d S2, S2, 0x08
addi.d TD, TD, 0x10

addi.d J, J, -1
blt ZERO, J, .L_II1

.L_II20:
addi.d I, I, -1
blt ZERO, I, .L_J1

.L_N0: /* if(n&1)*/
andi I, N, 0x01
beq ZERO, I, .L_N00

.L_N1:
srai.d J, M, 0x02
beq ZERO, J, .L_N10

.L_N11: /* j = (m >> 2) if (j > 0) */
vld U0, TS, 0x00
vld U1, TS, 0x10

vst U0, TD, 0x00
vst U1, TD, 0x10

addi.d TS, TS, 0x20 // a_offset
addi.d TD, TD, 0x20 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N11

.L_N10:
andi J, M, 0x03
beq J, ZERO, .L_N00

.L_N12: /* j = (m & 3) if (j > 0) */
fld.s F0, TS, 0x00
fld.s F1, TS, 0x04

fst.s F0, TD, 0x00
fst.s F1, TD, 0x04

addi.d TS, TS, 0x08 // a_offset
addi.d TD, TD, 0x08 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N12

.L_N00:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 218
- 0
kernel/loongarch64/cgemm_tcopy_2_lasx.S View File

@@ -0,0 +1,218 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define TD $r16
#define TS $r17
#define TL $r7
#define T0 $r18
#define S8 $r19
#define S9 $r20
#define S10 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $xr0
#define U1 $xr1
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define D0 $xr8
#define D1 $xr9
#define D2 $xr10
#define D3 $xr11
#define D4 $xr12
#define D5 $xr13
#define D6 $xr14
#define D7 $xr15


PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TS, SRC //aoffset
move TD, DST //boffset
slli.d TL, LDA, 0x02 //lda
slli.d TL, TL, 0x01

ori T0, ZERO, 0x01
andn T0, N, T0
mul.d T0, M, T0
slli.d T0, T0, 0x01
slli.d T0, T0, 0x02
add.d S9, DST, T0 //boffset2

srai.d J, M, 0x01 //j

beq J, ZERO, .L_M1

.L_J1: /* if(j>0) j--*/
move S1, TS //aoffset1
slli.d T0, TL, 0x01
add.d S2, S1, TL //aoffset2
add.d TS, TS, T0

move S8, TD //boffset1
addi.d TD, TD, 0x20

srai.d I, N, 0x02
beq ZERO, I, .L_JN1

.L_JI1: /* if(i>0) i--*/
xvld U0, S1, 0x00
xvld U1, S1, 0x00
xvld U2, S2, 0x00

xvpermi.q U0, U2, 0x02
xvpermi.q U2, U1, 0x31

xvst U0, S8, 0x00

slli.d T0, M, 0x04
add.d S8, S8, T0

xvst U2, S8, 0x00

add.d S8, S8, T0
addi.d S1, S1, 0x20
addi.d S2, S2, 0x20

addi.d I, I, -1
blt ZERO, I, .L_JI1

.L_JN1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_JN2

vld $vr0, S1, 0x00
vld $vr1, S2, 0x00

vst $vr0, S8, 0x00
vst $vr1, S8, 0x10

addi.d S1, S1, 0x10
addi.d S2, S2, 0x10

.L_JN2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_J0

fld.s F0, S1, 0x00
fld.s F1, S1, 0x04
fld.s F2, S2, 0x00
fld.s F3, S2, 0x04

fst.s F0, S9, 0x00
fst.s F1, S9, 0x04
fst.s F2, S9, 0x08
fst.s F3, S9, 0x0c

addi.d S9, S9, 0x10

.L_J0:
addi.d J, J, -1
blt ZERO, J, .L_J1

.L_M1: /* if(m&1) */
andi I, M, 0x01
beq ZERO, I, .L_M0

srai.d I, N, 0x02
beq ZERO, I, .L_M1N1

.L_M1I1: /* if(i>0) */
vld $vr0, TS, 0x00
vld $vr1, TS, 0x10

vst $vr0, TD, 0x00

slli.d T0, M, 0x04
add.d TD, TD, T0

vst $vr1, TD, 0x00

add.d TD, TD, T0
addi.d TS, TS, 0x20

addi.d I, I, -1
blt ZERO, I, .L_M1I1

.L_M1N1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_M1N2

vld $vr0, TS, 0x00

vst $vr0, TD, 0x00

addi.d TS, TS, 0x10

.L_M1N2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_M0

fld.s F0, TS, 0x00
fld.s F1, TS, 0x04

fst.s F0, S9, 0x00
fst.s F1, S9, 0x04

.L_M0:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 218
- 0
kernel/loongarch64/cgemm_tcopy_2_lsx.S View File

@@ -0,0 +1,218 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define TD $r16
#define TS $r17
#define TL $r7
#define T0 $r18
#define S8 $r19
#define S9 $r20
#define S10 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $vr0
#define U1 $vr1
#define U2 $vr2
#define U3 $vr3
#define U4 $vr4
#define U5 $vr5
#define U6 $vr6
#define U7 $vr7
#define D0 $vr8
#define D1 $vr9
#define D2 $vr10
#define D3 $vr11
#define D4 $vr12
#define D5 $vr13
#define D6 $vr14
#define D7 $vr15


PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TS, SRC //aoffset
move TD, DST //boffset
slli.d TL, LDA, 0x02 //lda
slli.d TL, TL, 0x01

ori T0, ZERO, 0x01
andn T0, N, T0
mul.d T0, M, T0
slli.d T0, T0, 0x01
slli.d T0, T0, 0x02
add.d S9, DST, T0 //boffset2

srai.d J, M, 0x01 //j

beq J, ZERO, .L_M1

.L_J1: /* if(j>0) j--*/
move S1, TS //aoffset1
slli.d T0, TL, 0x01
add.d S2, S1, TL //aoffset2
add.d TS, TS, T0

move S8, TD //boffset1
addi.d TD, TD, 0x20

srai.d I, N, 0x02
beq ZERO, I, .L_JN1

.L_JI1: /* if(i>0) i--*/
vld U0, S1, 0x00
vld U1, S1, 0x10
vld U2, S2, 0x00
vld U3, S2, 0x10

vst U0, S8, 0x00
vst U2, S8, 0x10

slli.d T0, M, 0x04
add.d S8, S8, T0

vst U1, S8, 0x00
vst U3, S8, 0x10

add.d S8, S8, T0
addi.d S1, S1, 0x20
addi.d S2, S2, 0x20

addi.d I, I, -1
blt ZERO, I, .L_JI1

.L_JN1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_JN2

vld U0, S1, 0x00
vld U1, S2, 0x00

vst U0, S8, 0x00
vst U1, S8, 0x10

addi.d S1, S1, 0x10
addi.d S2, S2, 0x10

.L_JN2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_J0

fld.s F0, S1, 0x00
fld.s F1, S1, 0x04
fld.s F2, S2, 0x00
fld.s F3, S2, 0x04

fst.s F0, S9, 0x00
fst.s F1, S9, 0x04
fst.s F2, S9, 0x08
fst.s F3, S9, 0x0c

addi.d S9, S9, 0x10

.L_J0:
addi.d J, J, -1
blt ZERO, J, .L_J1

.L_M1: /* if(m&1) */
andi I, M, 0x01
beq ZERO, I, .L_M0

srai.d I, N, 0x02
beq ZERO, I, .L_M1N1

.L_M1I1: /* if(i>0) */
vld U0, TS, 0x00
vld U1, TS, 0x10

vst U0, TD, 0x00

slli.d T0, M, 0x04
add.d TD, TD, T0

vst U1, TD, 0x00

add.d TD, TD, T0
addi.d TS, TS, 0x20

addi.d I, I, -1
blt ZERO, I, .L_M1I1

.L_M1N1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_M1N2

vld U0, TS, 0x00

vst U0, TD, 0x00

addi.d TS, TS, 0x10

.L_M1N2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_M0

fld.s F0, TS, 0x00
fld.s F1, TS, 0x04

fst.s F0, S9, 0x00
fst.s F1, S9, 0x04

.L_M0:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 848
- 0
kernel/loongarch64/zgemm_kernel_2x2.S View File

@@ -0,0 +1,848 @@
/*******************************************************************************
Copyright (c) 2023, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"


/* Function parameters */
#define M $r4 // param 1: bm
#define N $r5 // param 2: bn
#define K $r6 // param 3: bk
#define ALPHA_R $f0 // param 4: alphar
#define ALPHA_I $f1 // param 5: alphai
#define A $r7 // param 6: ba
#define B $r8 // param 7: bb
#define C $r9 // param 8: bc
#define LDC $r10 // param 9: ldc

#if defined (TRMMKERNEL)
#define OFFSET $r11 // param 10: offset
#endif
#define OFF $r26

#define I $r12
#define J $r13
#define L $r14
#define TL $r15
#define A0 $r16
#define B0 $r17
#define C0 $r18
#define C1 $r19
#define C2 $r20
#define C3 $r23
#define T0 $r24
#define T1 $r25

#define a1 $f2
#define a2 $f3
#define a3 $f4
#define a4 $f5
#define a5 $f6
#define a6 $f7
#define a7 $f8
#define a8 $f9
#define b1 $f10
#define b2 $f11
#define b3 $f12
#define b4 $f13
#define b5 $f14
#define b6 $f15
#define b7 $f16
#define b8 $f17
#define c11 $f18
#define c12 $f19
#define c21 $f20
#define c22 $f21
#define c31 $f22
#define c32 $f23
#define c41 $f24
#define c42 $f25
#define c51 $f26
#define c52 $f27
#define c61 $f28
#define c62 $f29


#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define MADD1 MADD
#define MADD2 MADD
#define MADD3 NMSUB
#define MADD4 MADD
#endif

#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define MADD1 MADD
#define MADD2 MADD
#define MADD3 MADD
#define MADD4 NMSUB
#endif

#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 MADD
#define MADD4 MADD
#endif

#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 NMSUB
#define MADD4 NMSUB
#endif

PROLOGUE

addi.d $sp, $sp, -88
SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
SDARG $r26, $sp, 24
ST $f23, $sp, 32
ST $f24, $sp, 40
ST $f25, $sp, 48
ST $f26, $sp, 56
ST $f27, $sp, 64
ST $f28, $sp, 72
ST $f29, $sp, 80


#if defined (TRMMKERNEL) && !defined(LEFT)
sub.d OFF, $r0, OFFSET
#else
xor OFF, OFF, OFF
#endif

slli.d LDC, LDC, BASE_SHIFT

move J, $r0
srai.d T0, N, 1
beq J, T0, .L19

.L10: /* for(j=0; j<bn/2; j+=1) */
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
move A0, A //ptrba

#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move I, $r0
srai.d T0, M, 1
beq I, T0, .L150

.L11: /* for(i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x05
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF //temp
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MOV c12, c11
MOV c21, c11
MOV c22, c11
MOV c31, c11
MOV c32, c11
MOV c41, c11
MOV c42, c11

move L, $r0 //cycle param k
srai.d C2, TL, 2
beq L, C2, .L130
blt C2, L, .L130

.L12: /* for(k=0; k<bk/4; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x08 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x08 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x10 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x18 //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22
LD b3, B0, 0x10 //load6
MADD1 c31, a1, b3, c31 //res4
MADD2 c32, a2, b3, c32 //res5
LD b4, B0, 0x18 //load7
MADD3 c31, a2, b4, c31
MADD4 c32, a1, b4, c32
MADD1 c41, a3, b3, c41 //res6
MADD2 c42, a4, b3, c42 //res7
MADD3 c41, a4, b4, c41
MADD4 c42, a3, b4, c42
LD a5, A0, 0x20 //load8
LD b5, B0, 0x20 //load9
MADD1 c11, a5, b5, c11
LD a6, A0, 0x28 //load10
MADD2 c12, a6, b5, c12
LD b6, B0, 0x28 //load11
MADD3 c11, a6, b6, c11
MADD4 c12, a5, b6, c12
LD a7, A0, 0x30 //load12
MADD1 c21, a7, b5, c21
LD a8, A0, 0x38 //load13
MADD2 c22, a8, b5, c22
MADD3 c21, a8, b6, c21
MADD4 c22, a7, b6, c22
LD b7, B0, 0x30 //load14
MADD1 c31, a5, b7, c31
MADD2 c32, a6, b7, c32
LD b8, B0, 0x38 //load15
MADD3 c31, a6, b8, c31
MADD4 c32, a5, b8, c32
MADD1 c41, a7, b7, c41
MADD2 c42, a8, b7, c42
MADD3 c41, a8, b8, c41
MADD4 c42, a7, b8, c42

LD a1, A0, 0x40 //load0
LD b1, B0, 0x40 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x48 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x48 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x50 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x58 //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22
LD b3, B0, 0x50 //load6
MADD1 c31, a1, b3, c31 //res4
MADD2 c32, a2, b3, c32 //res5
LD b4, B0, 0x58 //load7
MADD3 c31, a2, b4, c31
MADD4 c32, a1, b4, c32
MADD1 c41, a3, b3, c41 //res6
MADD2 c42, a4, b3, c42 //res7
MADD3 c41, a4, b4, c41
MADD4 c42, a3, b4, c42
LD a5, A0, 0x60 //load8
LD b5, B0, 0x60 //load9
MADD1 c11, a5, b5, c11
LD a6, A0, 0x68 //load10
MADD2 c12, a6, b5, c12
LD b6, B0, 0x68 //load11
MADD3 c11, a6, b6, c11
MADD4 c12, a5, b6, c12
LD a7, A0, 0x70 //load12
MADD1 c21, a7, b5, c21
LD a8, A0, 0x78 //load13
MADD2 c22, a8, b5, c22
MADD3 c21, a8, b6, c21
MADD4 c22, a7, b6, c22
LD b7, B0, 0x70 //load14
MADD1 c31, a5, b7, c31
MADD2 c32, a6, b7, c32
LD b8, B0, 0x78 //load15
MADD3 c31, a6, b8, c31
MADD4 c32, a5, b8, c32
MADD1 c41, a7, b7, c41
MADD2 c42, a8, b7, c42
MADD3 c41, a8, b8, c41
MADD4 c42, a7, b8, c42

addi.d A0, A0, 0x80
addi.d B0, B0, 0x80

addi.d L, L, 1
blt L, C2, .L12

.L130:
move L, $r0
andi C2, TL, 3
beq L, C2, .L14

.L13: /* for(k=0; k<(bk&3); k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x08 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x08 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x10 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x18 //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22
LD b3, B0, 0x10 //load6
MADD1 c31, a1, b3, c31 //res4
MADD2 c32, a2, b3, c32 //res5
LD b4, B0, 0x18 //load7
MADD3 c31, a2, b4, c31
MADD4 c32, a1, b4, c32
MADD1 c41, a3, b3, c41 //res6
MADD2 c42, a4, b3, c42 //res7
MADD3 c41, a4, b4, c41
MADD4 c42, a3, b4, c42

addi.d A0, A0, 0x20
addi.d B0, B0, 0x20

addi.d L, L, 1
blt L, C2, .L13

.L14:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08

MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_I
SUB a7, a7, a8
ST a7, C0, 0x10

MUL a7, c22, ALPHA_R
MUL a8, c21, ALPHA_I
ADD a8, a7, a8
ST a8, C0, 0x18

MUL b5, c31, ALPHA_R
MUL b6, c32, ALPHA_I
SUB b5, b5, b6
ST b5, C1, 0x00

MUL b5, c32, ALPHA_R
MUL b6, c31, ALPHA_I
ADD b6, b5, b6
ST b6, C1, 0x08

MUL b7, c41, ALPHA_R
MUL b8, c42, ALPHA_I
SUB b7, b7, b8
ST b7, C1, 0x10

MUL b7, c42, ALPHA_R
MUL b8, c41, ALPHA_I
ADD b8, b7, b8
ST b8, C1, 0x18
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]
LD a7, C0, 0x10 //C0[2]
LD a8, C0, 0x18 //C0[3]
LD b5, C1, 0x00 //C1[0]
LD b6, C1, 0x08 //C1[1]
LD b7, C1, 0x10 //C1[2]
LD b8, C1, 0x18 //C1[3]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x08

MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8
ST a7, C0, 0x10
ST a8, C0, 0x18

MADD b5, c31, ALPHA_R, b5
MADD b6, c32, ALPHA_R, b6
NMSUB b5, c32, ALPHA_I, b5
MADD b6, c31, ALPHA_I, b6
ST b5, C1, 0x00
ST b6, C1, 0x08

MADD b7, c41, ALPHA_R, b7
MADD b8, c42, ALPHA_R, b8
NMSUB b7, c42, ALPHA_I, b7
MADD b8, c41, ALPHA_I, b8
ST b7, C1, 0x10
ST b8, C1, 0x18
#endif

#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x05
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x20
addi.d C1, C1, 0x20

addi.d I, I, 1
blt I, T0, .L11

.L150:
move I, $r0
andi T0, M, 1
beq I, T0, .L18

.L15: /* for(i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
slli.d C3, OFF, 0x05
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L17
blt TL, L, .L17

.L16: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x08 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x08 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD b3, B0, 0x10 //load4
MADD1 c21, a1, b3, c21 //res2
MADD2 c22, a2, b3, c22 //res3
LD b4, B0, 0x18 //load5
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22

addi.d A0, A0, 0x10
addi.d B0, B0, 0x20

addi.d L, L, 1
blt L, TL, .L16

.L17:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08

MUL b5, c21, ALPHA_R
MUL b6, c22, ALPHA_I
SUB b5, b5, b6
ST b5, C1, 0x00

MUL b5, c22, ALPHA_R
MUL b6, c21, ALPHA_I
ADD b6, b5, b6
ST b6, C1, 0x08
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]
LD b5, C1, 0x00 //C1[0]
LD b6, C1, 0x08 //C1[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
ST a5, C0, 0x00
ST a6, C0, 0x08

MADD b5, c21, ALPHA_R, b5
MADD b6, c22, ALPHA_R, b6
NMSUB b5, c22, ALPHA_I, b5
MADD b6, c21, ALPHA_I, b6
ST b5, C1, 0x00
ST b6, C1, 0x08
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
slli.d C3, TL, 0x05
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10
addi.d C1, C1, 0x10

addi.d I, I, 1
blt I, T0, .L15

.L18:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 2
#endif

slli.d L, K, 0x05
add.d B, B, L

slli.d I, LDC, 0x02
add.d C, C, I

addi.d J, J, 1
srai.d T0, N, 1
blt J, T0, .L10

.L19:
move J, $r0
andi T0, N, 1
beq J, T0, .L30

.L20: /* for (j=0; j<(bn&1); j+=1) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move C0, C
move A0, A //ptrba

move I, $r0
srai.d T0, M, 1
beq I, T0, .L24

.L21: /* for (i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x05
add.d A0, A0, C3
slli.d C3, OFF, 0x04
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L23
blt TL, L, .L23

.L22: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x08 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x08 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12
LD a3, A0, 0x10 //load4
MADD1 c21, a3, b1, c21 //res2
LD a4, A0, 0x18 //load5
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22

addi.d A0, A0, 0x20
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L22

.L23:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08

MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_I
SUB a7, a7, a8
ST a7, C0, 0x10

MUL a7, c22, ALPHA_R
MUL a8, c21, ALPHA_I
ADD a8, a7, a8
ST a8, C0, 0x18
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]
LD a7, C0, 0x10 //C1[2]
LD a8, C0, 0x18 //C1[3]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8

ST a5, C0, 0x00
ST a6, C0, 0x08
ST a7, C0, 0x10
ST a8, C0, 0x18
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x05
add.d A0, A0, C3
slli.d C3, TL, 0x04
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x20

addi.d I, I, 1
blt I, T0, .L21

.L24:
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L28

.L25: /* for (i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0

move L, $r0 //cycle param k
beq L, TL, .L27
blt TL, L, .L27

.L26: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
MADD1 c11, a1, b1, c11 //res0
LD a2, A0, 0x08 //load2
MADD2 c12, a2, b1, c12 //res1
LD b2, B0, 0x08 //load3
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

addi.d A0, A0, 0x10
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L26

.L27:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6

ST a5, C0, 0x00
ST a6, C0, 0x08
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10

addi.d I, I, 1
blt I, T1, .L25

.L28:
slli.d L, K, 4
add.d B, B, L

slli.d I, LDC, 1
add.d C, C, I

addi.d J, J, 1
andi T0, N, 1
blt J, T0, .L20

.L30:
LDARG $r23, $sp, 0
LDARG $r24, $sp, 8
LDARG $r25, $sp, 16
LDARG $r26, $sp, 24
LD $f23, $sp, 32
LD $f24, $sp, 40
LD $f25, $sp, 48
LD $f26, $sp, 56
LD $f27, $sp, 64
LD $f28, $sp, 72
LD $f29, $sp, 80

addi.d $sp, $sp, 88
jirl $r0, $r1, 0x0

EPILOGUE

+ 822
- 0
kernel/loongarch64/zgemm_kernel_2x2_lasx.S View File

@@ -0,0 +1,822 @@
/*******************************************************************************
Copyright (c) 2023, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"


/* Function parameters */
#define M $r4 // param 1: bm
#define N $r5 // param 2: bn
#define K $r6 // param 3: bk
#define ALPHA_R $f0 // param 4: alphar
#define ALPHA_I $f1 // param 5: alphai
#define A $r7 // param 6: ba
#define B $r8 // param 7: bb
#define C $r9 // param 8: bc
#define LDC $r10 // param 9: ldc

#if defined (TRMMKERNEL)
#define OFFSET $r11 // param 10: offset
#endif
#define OFF $r26

#define I $r12
#define J $r13
#define L $r14
#define TL $r15
#define A0 $r16
#define B0 $r17
#define C0 $r18
#define C1 $r19
#define C2 $r20
#define C3 $r23
#define T0 $r24
#define T1 $r25

#define a1 $f2
#define a2 $f3
#define a3 $f4
#define a4 $f5
#define a5 $f6
#define a6 $f7
#define a7 $f8
#define a8 $f9
#define b1 $f10
#define b2 $f11
#define b3 $f12
#define b4 $f13
#define b5 $f14
#define b6 $f15
#define b7 $f16
#define b8 $f17
#define c11 $f18
#define c12 $f19
#define c21 $f20
#define c22 $f21
#define c31 $f22
#define c32 $f23
#define c41 $f24
#define c42 $f25

/* LASX vectors */
#define U0 $xr30
#define U1 $xr31
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define U8 $xr8
#define U9 $xr9
#define U10 $xr10
#define U11 $xr11
#define U12 $xr12
#define U13 $xr13
#define U14 $xr14
#define U15 $xr15
#define D0 $xr16
#define D1 $xr17
#define D2 $xr18
#define D3 $xr19
#define D4 $xr20
#define D5 $xr21
#define D6 $xr22
#define D7 $xr23
#define D8 $xr24
#define D9 $xr25
#define D10 $xr26
#define D11 $xr27
#define D12 $xr28
#define D13 $xr29
#define VALPHAR $xr28
#define VALPHAI $xr29


#if defined(NN) || defined(NT) || defined(TN) || defined(TT)
#define XVMADD1 XVFMADD
#define XVMADD2 XVFMADD
#define XVMADD3 XVNMSUB
#define XVMADD4 XVFMADD

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 NMSUB
#define MADD4 MADD
#endif

#if defined(NR) || defined(NC) || defined(TR) || defined(TC)
#define XVMADD1 XVFMADD
#define XVMADD2 XVFMADD
#define XVMADD3 XVFMADD
#define XVMADD4 XVNMSUB

#define MADD1 MADD
#define MADD2 MADD
#define MADD3 MADD
#define MADD4 NMSUB
#endif

#if defined(RN) || defined(RT) || defined(CN) || defined(CT)
#define XVMADD1 XVFMADD
#define XVMADD2 XVNMSUB
#define XVMADD3 XVFMADD
#define XVMADD4 XVFMADD

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 MADD
#define MADD4 MADD
#endif

#if defined(RR) || defined(RC) || defined(CR) || defined(CC)
#define XVMADD1 XVFMADD
#define XVMADD2 XVNMSUB
#define XVMADD3 XVNMSUB
#define XVMADD4 XVNMSUB

#define MADD1 MADD
#define MADD2 NMSUB
#define MADD3 NMSUB
#define MADD4 NMSUB
#endif

PROLOGUE

addi.d $sp, $sp, -128
SDARG $r23, $sp, 0
SDARG $r24, $sp, 8
SDARG $r25, $sp, 16
SDARG $r26, $sp, 24
SDARG $r27, $sp, 32
ST $f23, $sp, 40
ST $f24, $sp, 48
ST $f25, $sp, 56
ST $f26, $sp, 64
ST $f27, $sp, 72
ST $f28, $sp, 80
ST $f29, $sp, 88
ST $f30, $sp, 96
ST $f31, $sp, 104
ST ALPHA_R,$sp, 112
ST ALPHA_I,$sp, 120

xvldrepl.d VALPHAR, $sp, 112
xvldrepl.d VALPHAI, $sp, 120

#if defined (TRMMKERNEL) && !defined(LEFT)
sub.d OFF, $r0, OFFSET
#else
xor OFF, OFF, OFF
#endif

slli.d LDC, LDC, BASE_SHIFT

move J, $r0
srai.d T0, N, 1
beq J, T0, .L19

.L10: /* for(j=0; j<bn/2; j+=1) */
move C0, C
slli.d TL, LDC, 1
add.d C1, C0, TL
move A0, A //ptrba

#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move I, $r0
srai.d T0, M, 1
beq I, T0, .L150

.L11: /* for(i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x05
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF //temp
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

xvxor.v U0, U0, U0
move L, $r0 //cycle param k
srai.d C2, TL, 2
xvxor.v U1, U1, U1
beq L, C2, .L130
blt C2, L, .L130

.L12: /* for(k=0; k<bk/4; k+=1) */
xvld D0, A0, 0x00 //a0 a1 a2 a3
xvld D1, B0, 0x00 //b0 b1 b2 b3

xvpermi.d D4, D1, 0xa0 //b0 b0 b2 b2
xvpermi.d D5, D1, 0xf5 //b1 b1 b3 b3
xvpermi.d D2, D0, 0x88 //a0 a2 a0 a2
xvpermi.d D3, D0, 0xdd //a1 a3 a1 a3

XVMADD1 U0, D2, D4, U0 //res0 2 4 6
XVMADD2 U1, D3, D4, U1 //res1 3 4 7
XVMADD3 U0, D3, D5, U0
XVMADD4 U1, D2, D5, U1

xvld D0, A0, 0x20 //a0 a1 a2 a3
xvld D1, B0, 0x20 //b0 b1 b2 b3

xvpermi.d D4, D1, 0xa0 //b0 b0 b2 b2
xvpermi.d D5, D1, 0xf5 //b1 b1 b3 b3
xvpermi.d D2, D0, 0x88 //a0 a2 a0 a2
xvpermi.d D3, D0, 0xdd //a1 a3 a1 a3

XVMADD1 U0, D2, D4, U0 //res0 2 4 6
XVMADD2 U1, D3, D4, U1 //res1 3 4 7
XVMADD3 U0, D3, D5, U0
XVMADD4 U1, D2, D5, U1

xvld D0, A0, 0x40 //a0 a1 a2 a3
xvld D1, B0, 0x40 //b0 b1 b2 b3

xvpermi.d D4, D1, 0xa0 //b0 b0 b2 b2
xvpermi.d D5, D1, 0xf5 //b1 b1 b3 b3
xvpermi.d D2, D0, 0x88 //a0 a2 a0 a2
xvpermi.d D3, D0, 0xdd //a1 a3 a1 a3

XVMADD1 U0, D2, D4, U0 //res0 2 4 6
XVMADD2 U1, D3, D4, U1 //res1 3 4 7
XVMADD3 U0, D3, D5, U0
XVMADD4 U1, D2, D5, U1

xvld D0, A0, 0x60 //a0 a1 a2 a3
xvld D1, B0, 0x60 //b0 b1 b2 b3

xvpermi.d D4, D1, 0xa0 //b0 b0 b2 b2
xvpermi.d D5, D1, 0xf5 //b1 b1 b3 b3
xvpermi.d D2, D0, 0x88 //a0 a2 a0 a2
xvpermi.d D3, D0, 0xdd //a1 a3 a1 a3

XVMADD1 U0, D2, D4, U0 //res0 2 4 6
XVMADD2 U1, D3, D4, U1 //res1 3 4 7
XVMADD3 U0, D3, D5, U0
XVMADD4 U1, D2, D5, U1

addi.d A0, A0, 0x80
addi.d B0, B0, 0x80

addi.d L, L, 1
blt L, C2, .L12

.L130:
move L, $r0
andi C2, TL, 3
beq L, C2, .L14

.L13: /* for(k=0; k<(bk&3); k+=1) */
xvld D0, A0, 0x00 //a0 a1 a2 a3
xvld D1, B0, 0x00 //b0 b1 b2 b3

xvpermi.d D4, D1, 0xa0 //b0 b0 b2 b2
xvpermi.d D5, D1, 0xf5 //b1 b1 b3 b3
xvpermi.d D2, D0, 0x88 //a0 a2 a0 a2
xvpermi.d D3, D0, 0xdd //a1 a3 a1 a3

XVMADD1 U0, D2, D4, U0 //res0 2 4 6
XVMADD2 U1, D3, D4, U1 //res1 3 5 7
XVMADD3 U0, D3, D5, U0
XVMADD4 U1, D2, D5, U1

addi.d A0, A0, 0x20
addi.d B0, B0, 0x20

addi.d L, L, 1
blt L, C2, .L13

.L14:
#if defined(TRMMKERNEL)
xvld U8, C0, 0x00 //0 1 2 3
xvld U9, C1, 0x00 //4 5 6 7

xvpackev.d U10, U9, U8
xvpermi.d U10, U10, 0xd8 //0 2 4 6
xvpackod.d U11, U9, U8
xvpermi.d U11, U11, 0xd8 //1 3 5 7

xvfmul.d U10, U0, VALPHAR
xvfmul.d U11, U1, VALPHAR
XVNMSUB U10, U1, VALPHAI, U10
XVFMADD U11, U0, VALPHAI, U11

xvand.v U8, U10, U10 //0 2 4 6
xvpermi.q U8, U11, 0x02 //0 2 1 3
xvpermi.d U8, U8, 0xd8 //0 1 2 3

xvand.v U9, U11, U11 //1 3 5 7
xvpermi.q U9, U10, 0x31 //4 6 5 7
xvpermi.d U9, U9, 0xd8 //4 5 6 7

xvst U8, C0, 0x00
xvst U9, C1, 0x00
#else
xvld U8, C0, 0x00 //0 1 2 3
xvld U9, C1, 0x00 //4 5 6 7

xvpackev.d U10, U9, U8
xvpermi.d U10, U10, 0xd8 //0 2 4 6
xvpackod.d U11, U9, U8
xvpermi.d U11, U11, 0xd8 //1 3 5 7

XVFMADD U10, U0, VALPHAR, U10
XVFMADD U11, U1, VALPHAR, U11
XVNMSUB U10, U1, VALPHAI, U10
XVFMADD U11, U0, VALPHAI, U11

xvand.v U8, U10, U10 //0 2 4 6
xvpermi.q U8, U11, 0x02 //0 2 1 3
xvpermi.d U8, U8, 0xd8 //0 1 2 3

xvand.v U9, U11, U11 //1 3 5 7
xvpermi.q U9, U10, 0x31 //4 6 5 7
xvpermi.d U9, U9, 0xd8 //4 5 6 7

xvst U8, C0, 0x00
xvst U9, C1, 0x00
#endif

#if defined(TRMMKERNEL)

#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x05
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x20
addi.d C1, C1, 0x20

addi.d I, I, 1
blt I, T0, .L11

.L150:
move I, $r0
andi T0, M, 1
beq I, T0, .L18

.L15: /* for(i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
slli.d C3, OFF, 0x05
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 2
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L17
blt TL, L, .L17

.L16: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
LD a2, A0, 0x08 //load2
LD b2, B0, 0x08 //load3
LD b3, B0, 0x10 //load4
LD b4, B0, 0x18 //load5

MADD1 c11, a1, b1, c11 //res0
MADD2 c12, a2, b1, c12 //res1
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

MADD1 c21, a1, b3, c21 //res2
MADD2 c22, a2, b3, c22 //res3
MADD3 c21, a2, b4, c21
MADD4 c22, a1, b4, c22

addi.d A0, A0, 0x10
addi.d B0, B0, 0x20

addi.d L, L, 1
blt L, TL, .L16

.L17:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08

MUL b5, c21, ALPHA_R
MUL b6, c22, ALPHA_I
SUB b5, b5, b6
ST b5, C1, 0x00

MUL b5, c22, ALPHA_R
MUL b6, c21, ALPHA_I
ADD b6, b5, b6
ST b6, C1, 0x08
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]
LD b5, C1, 0x00 //C1[0]
LD b6, C1, 0x08 //C1[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6

MADD b5, c21, ALPHA_R, b5
MADD b6, c22, ALPHA_R, b6
NMSUB b5, c22, ALPHA_I, b5
MADD b6, c21, ALPHA_I, b6

ST a5, C0, 0x00
ST a6, C0, 0x08

ST b5, C1, 0x00
ST b6, C1, 0x08
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -2
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
slli.d C3, TL, 0x05
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10
addi.d C1, C1, 0x10

addi.d I, I, 1
blt I, T0, .L15

.L18:
#if defined(TRMMKERNEL) && !defined(LEFT)
addi.d OFF, OFF, 2
#endif

slli.d L, K, 0x05
add.d B, B, L

slli.d I, LDC, 0x02
add.d C, C, I

addi.d J, J, 1
srai.d T0, N, 1
blt J, T0, .L10

.L19:
move J, $r0
andi T0, N, 1
beq J, T0, .L30

.L20: /* for (j=0; j<(bn&1); j+=1) */
#if defined(TRMMKERNEL) && defined(LEFT)
move OFF, OFFSET
#endif

move C0, C
move A0, A //ptrba

move I, $r0
srai.d T0, M, 1
beq I, T0, .L24

.L21: /* for (i=0; i<bm/2; i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x05
add.d A0, A0, C3
slli.d C3, OFF, 0x04
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 2
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0
MTC c21, $r0
MTC c22, $r0

move L, $r0 //cycle param k
beq L, TL, .L23
blt TL, L, .L23

.L22: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
LD a2, A0, 0x08 //load2
LD b2, B0, 0x08 //load3
LD a3, A0, 0x10 //load4
LD a4, A0, 0x18 //load5

MADD1 c11, a1, b1, c11 //res0
MADD2 c12, a2, b1, c12 //res1
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

MADD1 c21, a3, b1, c21 //res2
MADD2 c22, a4, b1, c22 //res3
MADD3 c21, a4, b2, c21
MADD4 c22, a3, b2, c22

addi.d A0, A0, 0x20
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L22

.L23:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08

MUL a7, c21, ALPHA_R
MUL a8, c22, ALPHA_I
SUB a7, a7, a8
ST a7, C0, 0x10

MUL a7, c22, ALPHA_R
MUL a8, c21, ALPHA_I
ADD a8, a7, a8
ST a8, C0, 0x18
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]
LD a7, C0, 0x10 //C1[2]
LD a8, C0, 0x18 //C1[3]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6
MADD a7, c21, ALPHA_R, a7
MADD a8, c22, ALPHA_R, a8
NMSUB a7, c22, ALPHA_I, a7
MADD a8, c21, ALPHA_I, a8

ST a5, C0, 0x00
ST a6, C0, 0x08

ST a7, C0, 0x10
ST a8, C0, 0x18
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -2
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x05
add.d A0, A0, C3
slli.d C3, TL, 0x04
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 2
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x20

addi.d I, I, 1
blt I, T0, .L21

.L24:
move I, $r0
andi T1, M, 1 //bm&1
beq I, T1, .L28

.L25: /* for (i=0; i<(bm&1); i+=1) */
move B0, B //ptrbb
move TL, K /* TL = bk */
#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))
move B0, B //ptrbb
#else
slli.d C3, OFF, 0x04
add.d A0, A0, C3
add.d B0, B, C3
#endif

#if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
sub.d TL, K, OFF
#elif defined(LEFT)
addi.d TL, OFF, 1
#else
addi.d TL, OFF, 1
#endif

#endif // #if defined(TRMMKERNEL)

MTC c11, $r0
MTC c12, $r0

move L, $r0 //cycle param k
beq L, TL, .L27
blt TL, L, .L27

.L26: /* for (k=0; k<bk; k+=1) */
LD a1, A0, 0x00 //load0
LD b1, B0, 0x00 //load1
LD a2, A0, 0x08 //load2
LD b2, B0, 0x08 //load3

MADD1 c11, a1, b1, c11 //res0
MADD2 c12, a2, b1, c12 //res1
MADD3 c11, a2, b2, c11
MADD4 c12, a1, b2, c12

addi.d A0, A0, 0x10
addi.d B0, B0, 0x10

addi.d L, L, 1
blt L, TL, .L26

.L27:
#if defined(TRMMKERNEL)
MUL a5, c11, ALPHA_R
MUL a6, c12, ALPHA_I
SUB a5, a5, a6
ST a5, C0, 0x00

MUL a5, c12, ALPHA_R
MUL a6, c11, ALPHA_I
ADD a6, a5, a6
ST a6, C0, 0x08
#else
LD a5, C0, 0x00 //C0[0]
LD a6, C0, 0x08 //C0[1]

MADD a5, c11, ALPHA_R, a5
MADD a6, c12, ALPHA_R, a6
NMSUB a5, c12, ALPHA_I, a5
MADD a6, c11, ALPHA_I, a6

ST a5, C0, 0x00
ST a6, C0, 0x08
#endif

#if defined(TRMMKERNEL)
#if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
sub.d TL, K, OFF
#ifdef LEFT
addi.d TL, TL, -1
#else
addi.d TL, TL, -1
#endif
slli.d C3, TL, 0x04
add.d A0, A0, C3
add.d B0, B0, C3
#endif

#ifdef LEFT
addi.d OFF, OFF, 1
#endif
#endif // #if defined(TRMMKERNEL)

addi.d C0, C0, 0x10

addi.d I, I, 1
blt I, T1, .L25

.L28:
slli.d L, K, 4
add.d B, B, L

slli.d I, LDC, 1
add.d C, C, I

addi.d J, J, 1
andi T0, N, 1
blt J, T0, .L20

.L30:
LDARG $r23, $sp, 0
LDARG $r24, $sp, 8
LDARG $r25, $sp, 16
LDARG $r26, $sp, 24
LDARG $r27, $sp, 32
LD $f23, $sp, 40
LD $f24, $sp, 48
LD $f25, $sp, 56
LD $f26, $sp, 64
LD $f27, $sp, 72
LD $f28, $sp, 80
LD $f29, $sp, 88
LD $f30, $sp, 96
LD $f31, $sp, 104

addi.d $sp, $sp, 128
jirl $r0, $r1, 0x0

EPILOGUE

+ 196
- 0
kernel/loongarch64/zgemm_ncopy_2_lasx.S View File

@@ -0,0 +1,196 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define S5 $r16
#define S6 $r17
#define S7 $r18
#define S8 $r19
#define TD $r20
#define TS $r11
#define TL $r7
#define T0 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $xr0
#define U1 $xr1
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define D0 $xr8
#define D1 $xr9
#define D2 $xr10
#define D3 $xr11
#define D4 $xr12
#define D5 $xr13
#define D6 $xr14
#define D7 $xr15
#define D8 $xr16

PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TD, DST //boffset
move TS, SRC //aoffset

slli.d TL, LDA, 0x03 //lda
slli.d TL, TL, 0x01
slli.d T0, TL, 0x01

srai.d I, N, 0x01
beq I, ZERO, .L_N0

.L_J1: /* if (i > 0) I-- */
move S1, TS //a_offset1
add.d S2, TS, TL //a_offset2
srai.d J, M, 0x02
add.d TS, TS, T0

beq J, ZERO, .L_I3

.L_I1: /* if (j > 0) J-- */
xvld U0, S1, 0x00
xvld U1, S1, 0x20
xvld U2, S2, 0x00
xvld U3, S2, 0x20

xvand.v D0, U0, U0
xvand.v D1, U1, U1
xvand.v D2, U2, U2
xvand.v D3, U3, U3

xvpermi.q D0, U2, 0x02
xvpermi.q D2, U0, 0x31
xvpermi.q D1, U3, 0x02
xvpermi.q D3, U1, 0x31

xvst D0, TD, 0x00
xvst D2, TD, 0x20
xvst D1, TD, 0x40
xvst D3, TD, 0x60

addi.d S1, S1, 0x40 // a_offset1
addi.d S2, S2, 0x40
addi.d TD, TD, 0x80 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_I1

.L_I3:
andi J, M, 0x03
beq J, ZERO, .L_II20

.L_II1: /* j = (m & 3) if (j > 0) */
vld $vr0, S1, 0x00
vld $vr1, S2, 0x00

vst $vr0, TD, 0x00
vst $vr1, TD, 0x10

addi.d S1, S1, 0x10
addi.d S2, S2, 0x10
addi.d TD, TD, 0x20

addi.d J, J, -1
blt ZERO, J, .L_II1

.L_II20:
addi.d I, I, -1
blt ZERO, I, .L_J1

.L_N0: /* if(n&1)*/
andi I, N, 0x01
beq ZERO, I, .L_N00

.L_N1:
srai.d J, M, 0x02
beq ZERO, J, .L_N10

.L_N11: /* j = (m >> 2) if (j > 0) */
xvld U0, TS, 0x00
xvld U1, TS, 0x20

xvst U0, TD, 0x00
xvst U1, TD, 0x20

addi.d TS, TS, 0x40 // a_offset
addi.d TD, TD, 0x40 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N11

.L_N10:
andi J, M, 0x03
beq J, ZERO, .L_N00

.L_N12: /* j = (m & 3) if (j > 0) */
vld $vr0, TS, 0x00
vst $vr0, TD, 0x00


addi.d TS, TS, 0x10 // a_offset
addi.d TD, TD, 0x10 // b_offset

addi.d J, J, -1
blt ZERO, J, .L_N12

.L_N00:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 212
- 0
kernel/loongarch64/zgemm_tcopy_2_lasx.S View File

@@ -0,0 +1,212 @@
/*******************************************************************************
Copyright (c) 2021, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/
#define ASSEMBLER

#include "common.h"

/* Function parameters */
#define M $r4 // param 1: m
#define N $r5 // param 2: n
#define SRC $r6 // param 3: src
#define LDA $r7 // param 4: lda
#define DST $r8 // param 5: dst

#define I $r9
#define J $r10
#define S1 $r12
#define S2 $r13
#define S3 $r14
#define S4 $r15
#define TD $r16
#define TS $r17
#define TL $r7
#define T0 $r18
#define S8 $r19
#define S9 $r20
#define S10 $r23
#define ZERO $r0

#define F0 $f0
#define F1 $f1
#define F2 $f2
#define F3 $f3
#define F4 $f4
#define F5 $f5
#define F6 $f6
#define F7 $f7

/* LASX vectors */
#define U0 $xr0
#define U1 $xr1
#define U2 $xr2
#define U3 $xr3
#define U4 $xr4
#define U5 $xr5
#define U6 $xr6
#define U7 $xr7
#define D0 $xr8
#define D1 $xr9
#define D2 $xr10
#define D3 $xr11
#define D4 $xr12
#define D5 $xr13
#define D6 $xr14
#define D7 $xr15


PROLOGUE

addi.d $sp, $sp, -8
SDARG $r23, $sp, 0

move TS, SRC //aoffset
move TD, DST //boffset
slli.d TL, LDA, 0x03 //lda
slli.d TL, TL, 0x01

ori T0, ZERO, 0x01
andn T0, N, T0
mul.d T0, M, T0
slli.d T0, T0, 0x01
slli.d T0, T0, 0x03
add.d S9, DST, T0 //boffset2

srai.d J, M, 0x01 //j

beq J, ZERO, .L_M1

.L_J1: /* if(j>0) j--*/
move S1, TS //aoffset1
slli.d T0, TL, 0x01
add.d S2, S1, TL //aoffset2
add.d TS, TS, T0

move S8, TD //boffset1
addi.d TD, TD, 0x40

srai.d I, N, 0x02
beq ZERO, I, .L_JN1

.L_JI1: /* if(i>0) i--*/
xvld U0, S1, 0x00
xvld U1, S1, 0x20
xvld U2, S2, 0x00
xvld U3, S2, 0x20

xvst U0, S8, 0x00
xvst U2, S8, 0x20

slli.d T0, M, 0x05
add.d S8, S8, T0

xvst U1, S8, 0x00
xvst U3, S8, 0x20

add.d S8, S8, T0
addi.d S1, S1, 0x40
addi.d S2, S2, 0x40

addi.d I, I, -1
blt ZERO, I, .L_JI1

.L_JN1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_JN2

xvld U0, S1, 0x00
xvld U1, S2, 0x00

xvst U0, S8, 0x00
xvst U1, S8, 0x20

addi.d S1, S1, 0x20
addi.d S2, S2, 0x20

.L_JN2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_J0

vld $vr0, S1, 0x00
vld $vr1, S2, 0x00

vst $vr0, S9, 0x00
vst $vr1, S9, 0x10

addi.d S9, S9, 0x20

.L_J0:
addi.d J, J, -1
blt ZERO, J, .L_J1

.L_M1: /* if(m&1) */
andi I, M, 0x01
beq ZERO, I, .L_M0

srai.d I, N, 0x02
beq ZERO, I, .L_M1N1

.L_M1I1: /* if(i>0) */
xvld U0, TS, 0x00
xvld U1, TS, 0x20

xvst U0, TD, 0x00

slli.d T0, M, 0x05
add.d TD, TD, T0

xvst U1, TD, 0x00

add.d TD, TD, T0
addi.d TS, TS, 0x40

addi.d I, I, -1
blt ZERO, I, .L_M1I1

.L_M1N1: /* if(n&2) */
andi I, N, 0x02
beq ZERO, I, .L_M1N2

xvld U0, TS, 0x00

xvst U0, TD, 0x00

addi.d TS, TS, 0x20

.L_M1N2: /* if(n&1) */
andi I, N, 0x01
beq ZERO, I, .L_M0

vld $vr0, TS, 0x00

vst $vr0, S9, 0x00

.L_M0:
LDARG $r23, $sp, 0
addi.d $sp, $sp, 8
jirl $r0, $r1, 0x00

EPILOGUE

+ 4
- 4
param.h View File

@@ -2853,13 +2853,13 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#endif

#define QGEMM_DEFAULT_UNROLL_N 2
#define CGEMM_DEFAULT_UNROLL_N 4
#define ZGEMM_DEFAULT_UNROLL_N 4
#define CGEMM_DEFAULT_UNROLL_N 2
#define ZGEMM_DEFAULT_UNROLL_N 2
#define XGEMM_DEFAULT_UNROLL_N 1

#define QGEMM_DEFAULT_UNROLL_M 2
#define CGEMM_DEFAULT_UNROLL_M 1
#define ZGEMM_DEFAULT_UNROLL_M 1
#define CGEMM_DEFAULT_UNROLL_M 2
#define ZGEMM_DEFAULT_UNROLL_M 2
#define XGEMM_DEFAULT_UNROLL_M 1

#define SGEMM_DEFAULT_P 256


Write Preview
Loading…
Cancel
Save