OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c31861ea62
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c31861ea62'
${ noResults }
OpenBLAS/kernel/loongarch64/cgemm_kernel_2x2_lasx.S

857 lines
24 kB
Raw Blame History 

  1. /*******************************************************************************

  2. Copyright (c) 2023, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *******************************************************************************/

  27. #define ASSEMBLER

  28. 

  29. #include "common.h"

  30. 

  31. 

  32. /* Function parameters */

  33. #define M      $r4   // param 1: bm

  34. #define N      $r5   // param 2: bn

  35. #define K      $r6   // param 3: bk

  36. #define ALPHA_R $f0   // param 4: alphar

  37. #define ALPHA_I $f1   // param 5: alphai

  38. #define A      $r7   // param 6: ba

  39. #define B      $r8  // param 7: bb

  40. #define C      $r9  // param 8: bc

  41. #define LDC    $r10  // param 9: ldc

  42. 

  43. #if defined (TRMMKERNEL)

  44. #define OFFSET $r11  // param 10: offset

  45. #endif

  46. #define OFF    $r26

  47. 

  48. #define I      $r12

  49. #define J      $r13

  50. #define L      $r14

  51. #define TL     $r15

  52. #define A0     $r16

  53. #define B0     $r17

  54. #define C0     $r18

  55. #define C1     $r19

  56. #define C2     $r20

  57. #define C3     $r23

  58. #define T0     $r24

  59. #define T1     $r25

  60. 

  61. #define a1     $f2

  62. #define a2     $f3

  63. #define a3     $f4

  64. #define a4     $f5

  65. #define a5     $f6

  66. #define a6     $f7

  67. #define a7     $f8

  68. #define a8     $f9

  69. #define b1     $f10

  70. #define b2     $f11

  71. #define b3     $f12

  72. #define b4     $f13

  73. #define b5     $f14

  74. #define b6     $f15

  75. #define b7     $f16

  76. #define b8     $f17

  77. #define c11    $f18

  78. #define c12    $f19

  79. #define c21    $f20

  80. #define c22    $f21

  81. #define c31    $f22

  82. #define c32    $f23

  83. #define c41    $f24

  84. #define c42    $f25

  85. 

  86. /* LASX vectors */

  87. #define U0     $xr30

  88. #define U1     $xr31

  89. #define U2     $xr2

  90. #define U3     $xr3

  91. #define U4     $xr4

  92. #define U5     $xr5

  93. #define U6     $xr6

  94. #define U7     $xr7

  95. #define U8     $xr8

  96. #define U9     $xr9

  97. #define U10    $xr10

  98. #define U11    $xr11

  99. #define U12    $xr12

  100. #define U13    $xr13

  101. #define U14    $xr14

  102. #define U15    $xr15

  103. #define D0     $xr16

  104. #define D1     $xr17

  105. #define D2     $xr18

  106. #define D3     $xr19

  107. #define D4     $xr20

  108. #define D5     $xr21

  109. #define D6     $xr22

  110. #define D7     $xr23

  111. #define D8     $xr24

  112. #define D9     $xr25

  113. #define D10    $xr26

  114. #define D11    $xr27

  115. #define VALPHAR    $xr28

  116. #define VALPHAI    $xr29

  117. 

  118. 

  119. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  120. #define    XVMADD1       XVFMADD

  121. #define    XVMADD2       XVFMADD

  122. #define    XVMADD3       XVNMSUB

  123. #define    XVMADD4       XVFMADD

  124. 

  125. #define    VMADD1       VFMADD

  126. #define    VMADD2       VFMADD

  127. #define    VMADD3       VNMSUB

  128. #define    VMADD4       VFMADD

  129. 

  130. #define    XVFADD1       XVFADD

  131. #define    XVFADD2       XVFADD

  132. #define    XVFADD3       XVFSUB

  133. #define    XVFADD4       XVFADD

  134. 

  135. #define    MADD1       MADD

  136. #define    MADD2       MADD

  137. #define    MADD3       NMSUB

  138. #define    MADD4       MADD

  139. #endif

  140. 

  141. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  142. #define    XVMADD1       XVFMADD

  143. #define    XVMADD2       XVFMADD

  144. #define    XVMADD3       XVFMADD

  145. #define    XVMADD4       XVNMSUB

  146. 

  147. #define    VMADD1       VFMADD

  148. #define    VMADD2       VFMADD

  149. #define    VMADD3       VFMADD

  150. #define    VMADD4       VNMSUB

  151. 

  152. #define    XVFADD1       XVFADD

  153. #define    XVFADD2       XVFADD

  154. #define    XVFADD3       XVFADD

  155. #define    XVFADD4       XVFSUB

  156. 

  157. #define    MADD1       MADD

  158. #define    MADD2       MADD

  159. #define    MADD3       MADD

  160. #define    MADD4       NMSUB

  161. #endif

  162. 

  163. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  164. #define    XVMADD1       XVFMADD

  165. #define    XVMADD2       XVNMSUB

  166. #define    XVMADD3       XVFMADD

  167. #define    XVMADD4       XVFMADD

  168. 

  169. #define    VMADD1       VFMADD

  170. #define    VMADD2       VNMSUB

  171. #define    VMADD3       VFMADD

  172. #define    VMADD4       VFMADD

  173. 

  174. #define    XVFADD1       XVFADD

  175. #define    XVFADD2       XVFSUB

  176. #define    XVFADD3       XVFADD

  177. #define    XVFADD4       XVFADD

  178. 

  179. #define    MADD1       MADD

  180. #define    MADD2       NMSUB

  181. #define    MADD3       MADD

  182. #define    MADD4       MADD

  183. #endif

  184. 

  185. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  186. #define    XVMADD1       XVFMADD

  187. #define    XVMADD2       XVNMSUB

  188. #define    XVMADD3       XVNMSUB

  189. #define    XVMADD4       XVNMSUB

  190. 

  191. #define    VMADD1       VFMADD

  192. #define    VMADD2       VNMSUB

  193. #define    VMADD3       VNMSUB

  194. #define    VMADD4       VNMSUB

  195. 

  196. #define    XVFADD1       XVFADD

  197. #define    XVFADD2       XVFSUB

  198. #define    XVFADD3       XVFSUB

  199. #define    XVFADD4       XVFSUB

  200. 

  201. #define    MADD1       MADD

  202. #define    MADD2       NMSUB

  203. #define    MADD3       NMSUB

  204. #define    MADD4       NMSUB

  205. #endif

  206. 

  207.     PROLOGUE

  208. 

  209.     addi.d     $sp,    $sp,   -128

  210.     SDARG      $r23,   $sp,   0

  211.     SDARG      $r24,   $sp,   8

  212.     SDARG      $r25,   $sp,   16

  213.     SDARG      $r26,   $sp,   24

  214.     SDARG      $r27,   $sp,   32

  215.     ST         $f23,   $sp,   40

  216.     ST         $f24,   $sp,   48

  217.     ST         $f25,   $sp,   56

  218.     ST         $f26,   $sp,   64

  219.     ST         $f27,   $sp,   72

  220.     ST         $f28,   $sp,   80

  221.     ST         $f29,   $sp,   88

  222.     ST         $f30,   $sp,   96

  223.     ST         $f31,   $sp,   104

  224.     ST         ALPHA_R,$sp,   112

  225.     ST         ALPHA_I,$sp,   120

  226. 

  227.     xvldrepl.w  VALPHAR, $sp, 112

  228.     xvldrepl.w  VALPHAI, $sp, 120

  229. 

  230. #if defined (TRMMKERNEL) && !defined(LEFT)

  231.     sub.d      OFF,    $r0,   OFFSET

  232. #else

  233.     xor        OFF,    OFF,   OFF

  234. #endif

  235. 

  236.     slli.d     LDC,    LDC,   2

  237. 

  238.     move       J,      $r0

  239.     srai.d     T0,     N,     1

  240.     beq        J,      T0,    .L19

  241. 

  242. .L10:  /* for(j=0; j<bn/2; j+=1) */

  243.     move       C0,     C

  244.     slli.d     TL,     LDC,   1

  245.     add.d      C1,     C0,    TL

  246.     move       A0,     A    //ptrba

  247. 

  248. #if defined(TRMMKERNEL) && defined(LEFT)

  249.     move       OFF,    OFFSET

  250. #endif

  251. 

  252.     move       I,      $r0

  253.     srai.d     T0,     M,     1

  254.     beq        I,      T0,    .L150

  255. 

  256. .L11:  /* for(i=0; i<bm/2; i+=1) */

  257.     move       B0,     B      //ptrbb

  258.     move       TL,     K      /* TL = bk */

  259. #if defined(TRMMKERNEL)

  260. 

  261. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  262.     move       B0,     B     //ptrbb

  263. #else

  264.     slli.d     C3,     OFF,   0x04

  265.     add.d      A0,     A0,    C3

  266.     add.d      B0,     B,     C3

  267. #endif

  268. 

  269. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  270.     sub.d      TL,     K,     OFF   //temp

  271. #elif defined(LEFT)

  272.     addi.d     TL,     OFF,   2

  273. #else

  274.     addi.d     TL,     OFF,   2

  275. #endif

  276. 

  277. #endif  // #if defined(TRMMKERNEL)

  278. 

  279.     xvxor.v    U0,     U0,   U0

  280.     xvxor.v    U1,     U1,   U1

  281. 

  282.     move       L,      $r0   //cycle param k

  283.     srai.d     C2,     TL,     2

  284.     beq        L,      C2,    .L130

  285.     blt        C2,     L,     .L130

  286. 

  287. .L12:  /* for(k=0; k<bk/4; k+=1) */

  288.     xvld       D0,     A0,    0x00  //a 0-7

  289.     xvld       D1,     A0,    0x20  //a 8-15

  290.     xvld       D2,     B0,    0x00  //b 0-7

  291.     xvld       D3,     B0,    0x20  //b 8-15

  292. 

  293.     xvand.v    D4,     D0,    D0

  294.     xvpermi.q  D4,     D1,    0x02  //a 0 1 2 3 8 9 10 11

  295.     xvand.v    D5,     D4,    D4

  296.     xvshuf4i.w D4,     D4,    0x88  //a 0 2 0 2 8 10 8 10

  297.     xvshuf4i.w D5,     D5,    0xdd  //a 1 3 1 3 9 11 9 11

  298. 

  299.     xvand.v    D6,     D1,    D1

  300.     xvpermi.q  D6,     D0,    0x31  //a 4 5 6 7 12 13 14 15

  301.     xvand.v    D7,     D6,    D6

  302.     xvshuf4i.w D6,     D6,    0x88  //a 4 6 4 6 12 14 12 14

  303.     xvshuf4i.w D7,     D7,    0xdd  //a 5 7 5 7 13 15 13 15

  304. 

  305.     xvand.v    D8,     D2,    D2

  306.     xvpermi.q  D8,     D3,    0x02  //b 0 1 2 3 8 9 10 11

  307.     xvand.v    D9,     D8,    D8

  308.     xvshuf4i.w D8,     D8,    0xa0  //b 0 0 2 2 8 8 10 10

  309.     xvshuf4i.w D9,     D9,    0xf5  //a 1 1 3 3 9 9 11 11

  310. 

  311.     xvand.v    D10,     D3,    D3

  312.     xvpermi.q  D10,     D2,    0x31  //b 4 5 6 7 12 13 14 15

  313.     xvand.v    D11,     D10,    D10

  314.     xvshuf4i.w D10,     D10,    0xa0  //b 4 4 6 6 12 12 14 14

  315.     xvshuf4i.w D11,     D11,    0xf5  //a 5 5 7 7 13 13 15 15

  316. 

  317.     XVMADD1      U0,     D4,    D8,     U0  //res0 2 4 6 0 2 4 6

  318.     XVMADD2      U1,     D5,    D8,     U1  //res1 3 4 7 1 3 4 7

  319. 

  320.     xvpermi.q    U0,     U0,    0x01

  321.     xvpermi.q    U1,     U1,    0x01

  322.     XVMADD1      U0,     D4,    D8,     U0

  323.     XVMADD2      U1,     D5,    D8,     U1

  324. 

  325.     XVMADD3      U0,     D5,    D9,     U0

  326.     XVMADD4      U1,     D4,    D9,     U1

  327. 

  328.     xvpermi.q    U0,     U0,    0x01

  329.     xvpermi.q    U1,     U1,    0x01

  330.     XVMADD3      U0,     D5,    D9,     U0

  331.     XVMADD4      U1,     D4,    D9,     U1

  332. 

  333.     XVMADD1      U0,     D6,    D10,     U0  //res0 2 4 6 0 2 4 6

  334.     XVMADD2      U1,     D7,    D10,     U1  //res1 3 4 7 1 3 4 7

  335. 

  336.     xvpermi.q    U0,     U0,    0x01

  337.     xvpermi.q    U1,     U1,    0x01

  338.     XVMADD1      U0,     D6,    D10,     U0

  339.     XVMADD2      U1,     D7,    D10,     U1

  340. 

  341.     XVMADD3      U0,     D7,    D11,     U0

  342.     XVMADD4      U1,     D6,    D11,     U1

  343. 

  344.     xvpermi.q    U0,     U0,    0x01

  345.     xvpermi.q    U1,     U1,    0x01

  346.     XVMADD3      U0,     D7,    D11,     U0

  347.     XVMADD4      U1,     D6,    D11,     U1

  348. 

  349.     addi.d     A0,     A0,    0x40

  350.     addi.d     B0,     B0,    0x40

  351. 

  352.     addi.d     L,      L,     1

  353.     blt        L,      C2,    .L12

  354. 

  355. .L130:

  356.     move       L,      $r0

  357.     andi       C2,     TL,     3

  358.     beq        L,      C2,    .L14

  359. 

  360. .L13:  /* for(k=0; k<(bk&3); k+=1) */

  361.     vld       $vr16,     A0,    0x00  //a0 a1 a2 a3

  362.     vld       $vr17,     B0,    0x00  //b0 b1 b2 b3

  363. 

  364.     vshuf4i.w $vr20,     $vr17,    0xa0    //b0 b0 b2 b2

  365.     vshuf4i.w $vr21,     $vr17,    0xf5    //b1 b1 b3 b3

  366. 

  367.     vshuf4i.w  $vr18,     $vr16,    0x88  //a0 a2 a0 a2

  368.     vshuf4i.w  $vr19,     $vr16,    0xdd  //a1 a3 a1 a3

  369. 

  370.     VMADD1      $vr30,     $vr18,    $vr20,     $vr30  //res0 2 4 6

  371.     VMADD2      $vr31,     $vr19,    $vr20,     $vr31  //res1 3 5 7

  372.     VMADD3      $vr30,     $vr19,    $vr21,     $vr30

  373.     VMADD4      $vr31,     $vr18,    $vr21,     $vr31

  374. 

  375.     addi.d     A0,     A0,    0x10

  376.     addi.d     B0,     B0,    0x10

  377. 

  378.     addi.d     L,      L,     1

  379.     blt        L,      C2,    .L13

  380. 

  381. .L14:

  382. #if defined(TRMMKERNEL)

  383.     vld       $vr8,     C0,    0x00  //0 1 2 3

  384.     vld       $vr9,     C1,    0x00  //4 5 6 7

  385. 

  386.     vpackev.w $vr10,    $vr9,    $vr8    //0 4 2 6

  387.     vpermi.w  $vr10,    $vr10,   0xd8    //0 2 4 6

  388. 

  389.     vpackod.w $vr11,    $vr9,    $vr8    //1 5 3 7

  390.     vpermi.w  $vr11,    $vr11,   0xd8    //1 3 5 7

  391. 

  392.     vfmul.s      $vr10,    $vr30,    $vr28

  393.     vfmul.s      $vr11,    $vr31,    $vr28

  394.     VNMSUB    $vr10,    $vr31,    $vr29, $vr10

  395.     VFMADD     $vr11,    $vr30,    $vr29, $vr11

  396. 

  397.     vilvl.w   $vr8,     $vr11,   $vr10  //0 1 2 3

  398. 

  399.     vilvh.w   $vr9,     $vr11,   $vr10  //4 5 6 7

  400. 

  401.     vst       $vr8,     C0,    0x00

  402.     vst       $vr9,     C1,    0x00

  403. #else

  404.     vld       $vr8,     C0,    0x00  //0 1 2 3

  405.     vld       $vr9,     C1,    0x00  //4 5 6 7

  406. 

  407.     vpackev.w $vr10,    $vr9,    $vr8    //0 4 2 6

  408.     vpermi.w  $vr10,    $vr10,   0xd8    //0 2 4 6

  409. 

  410.     vpackod.w $vr11,    $vr9,    $vr8    //1 5 3 7

  411.     vpermi.w  $vr11,    $vr11,   0xd8    //1 3 5 7

  412. 

  413.     VFMADD      $vr10,    $vr30,    $vr28, $vr10

  414.     VFMADD      $vr11,    $vr31,    $vr28, $vr11

  415.     VNMSUB     $vr10,    $vr31,    $vr29, $vr10

  416.     VFMADD      $vr11,    $vr30,    $vr29, $vr11

  417. 

  418.     vilvl.w   $vr8,     $vr11,   $vr10  //0 1 2 3

  419. 

  420.     vilvh.w   $vr9,     $vr11,   $vr10  //4 5 6 7

  421. 

  422.     vst       $vr8,     C0,    0x00

  423.     vst       $vr9,     C1,    0x00

  424. #endif

  425. 

  426. #if defined(TRMMKERNEL)

  427. 

  428. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  429.     sub.d      TL,     K,    OFF

  430. #ifdef LEFT

  431.     addi.d     TL,     TL,   -2

  432. #else

  433.     addi.d     TL,     TL,   -2

  434. #endif

  435.     slli.d     C3,     TL,   0x04

  436.     add.d      A0,     A0,   C3

  437.     add.d      B0,     B0,   C3

  438. #endif

  439. 

  440. #ifdef LEFT

  441.     addi.d     OFF,    OFF,   2

  442. #endif

  443. 

  444. #endif   // #if defined(TRMMKERNEL)

  445. 

  446.     addi.d     C0,     C0,    0x10

  447.     addi.d     C1,     C1,    0x10

  448. 

  449.     addi.d     I,      I,     1

  450.     blt        I,      T0,    .L11

  451. 

  452. .L150:

  453.     move       I,      $r0

  454.     andi       T0,     M,     1

  455.     beq        I,      T0,    .L18

  456. 

  457. .L15:  /* for(i=0; i<(bm&1); i+=1) */

  458.     move       B0,     B      //ptrbb

  459.     move       TL,     K      /* TL = bk */

  460. #if defined(TRMMKERNEL)

  461. #if (defined(LEFT) &&  defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))

  462.     move       B0,     B    //ptrbb

  463. #else

  464.     slli.d     C3,     OFF,  0x03

  465.     add.d      A0,     A0,   C3

  466.     slli.d     C3,     OFF,  0x04

  467.     add.d      B0,     B,    C3

  468. #endif

  469. 

  470. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  471.     sub.d      TL,     K,     OFF

  472. #elif defined(LEFT)

  473.     addi.d     TL,     OFF,   1

  474. #else

  475.     addi.d     TL,     OFF,   2

  476. #endif

  477. 

  478. #endif  // #if defined(TRMMKERNEL)

  479. 

  480.     MTC        c11,    $r0

  481.     MTC        c12,    $r0

  482.     MTC        c21,    $r0

  483.     MTC        c22,    $r0

  484. 

  485.     move       L,      $r0   //cycle param k

  486.     beq        L,      TL,    .L17

  487.     blt        TL,     L,     .L17

  488. 

  489. .L16:  /* for (k=0; k<bk; k+=1) */

  490.     LD         a1,     A0,    0x00        //load0

  491.     LD         b1,     B0,    0x00        //load1

  492.     MADD1      c11,    a1,    b1,     c11  //res0

  493.     LD         a2,     A0,    0x04        //load2

  494.     MADD2      c12,    a2,    b1,     c12  //res1

  495.     LD         b2,     B0,    0x04        //load3

  496.     MADD3      c11,    a2,    b2,     c11

  497.     MADD4      c12,    a1,    b2,     c12

  498.     LD         b3,     B0,    0x08        //load4

  499.     MADD1      c21,    a1,    b3,     c21  //res2

  500.     MADD2      c22,    a2,    b3,     c22  //res3

  501.     LD         b4,     B0,    0x0c        //load5

  502.     MADD3      c21,    a2,    b4,     c21

  503.     MADD4      c22,    a1,    b4,     c22

  504. 

  505.     addi.d     A0,     A0,    0x08

  506.     addi.d     B0,     B0,    0x10

  507. 

  508.     addi.d     L,      L,     1

  509.     blt        L,      TL,     .L16

  510. 

  511. .L17:

  512. #if defined(TRMMKERNEL)

  513.     MUL        a5,     c11,   ALPHA_R

  514.     MUL        a6,     c12,   ALPHA_I

  515.     SUB        a5,     a5,    a6

  516.     ST         a5,     C0,    0x00

  517. 

  518.     MUL        a5,     c12,   ALPHA_R

  519.     MUL        a6,     c11,   ALPHA_I

  520.     ADD        a6,     a5,    a6

  521.     ST         a6,     C0,    0x04

  522. 

  523.     MUL        b5,     c21,   ALPHA_R

  524.     MUL        b6,     c22,   ALPHA_I

  525.     SUB        b5,     b5,    b6

  526.     ST         b5,     C1,    0x00

  527. 

  528.     MUL        b5,     c22,   ALPHA_R

  529.     MUL        b6,     c21,   ALPHA_I

  530.     ADD        b6,     b5,    b6

  531.     ST         b6,     C1,    0x04

  532. #else

  533.     LD         a5,     C0,    0x00    //C0[0]

  534.     LD         a6,     C0,    0x04    //C0[1]

  535.     LD         b5,     C1,    0x00    //C1[0]

  536.     LD         b6,     C1,    0x04    //C1[1]

  537. 

  538.     MADD       a5,     c11,   ALPHA_R, a5

  539.     MADD       a6,     c12,   ALPHA_R, a6

  540.     NMSUB      a5,     c12,   ALPHA_I, a5

  541.     MADD       a6,     c11,   ALPHA_I, a6

  542.     ST         a5,     C0,    0x00

  543.     ST         a6,     C0,    0x04

  544. 

  545.     MADD       b5,     c21,   ALPHA_R, b5

  546.     MADD       b6,     c22,   ALPHA_R, b6

  547.     NMSUB      b5,     c22,   ALPHA_I, b5

  548.     MADD       b6,     c21,   ALPHA_I, b6

  549.     ST         b5,     C1,    0x00

  550.     ST         b6,     C1,    0x04

  551. #endif

  552. 

  553. #if defined(TRMMKERNEL)

  554. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  555.     sub.d      TL,     K,    OFF

  556. #ifdef LEFT

  557.     addi.d     TL,     TL,   -1

  558. #else

  559.     addi.d     TL,     TL,   -2

  560. #endif

  561.     slli.d     C3,     TL,   0x03

  562.     add.d      A0,     A0,   C3

  563.     slli.d     C3,     TL,   0x04

  564.     add.d      B0,     B0,   C3

  565. #endif

  566. 

  567. #ifdef LEFT

  568.     addi.d     OFF,    OFF,   1

  569. #endif

  570. #endif   // #if defined(TRMMKERNEL)

  571. 

  572.     addi.d     C0,     C0,    0x08

  573.     addi.d     C1,     C1,    0x08

  574. 

  575.     addi.d     I,      I,     1

  576.     blt        I,      T0,    .L15

  577. 

  578. .L18:

  579. #if defined(TRMMKERNEL) && !defined(LEFT)

  580.     addi.d     OFF,    OFF,   2

  581. #endif

  582. 

  583.     slli.d     L,      K,     0x04

  584.     add.d      B,      B,     L

  585. 

  586.     slli.d     I,      LDC,   0x02

  587.     add.d      C,      C,     I

  588. 

  589.     addi.d     J,      J,     1

  590.     srai.d     T0,     N,     1

  591.     blt        J,      T0,    .L10

  592. 

  593. .L19:

  594.     move       J,      $r0

  595.     andi       T0,     N,     1

  596.     beq        J,      T0,    .L30

  597. 

  598. .L20: /* for (j=0; j<(bn&1); j+=1) */

  599. #if defined(TRMMKERNEL) && defined(LEFT)

  600.     move       OFF,    OFFSET

  601. #endif

  602. 

  603.     move       C0,     C

  604.     move       A0,     A    //ptrba

  605. 

  606.     move       I,      $r0

  607.     srai.d     T0,     M,     1

  608.     beq        I,      T0,    .L24

  609. 

  610. .L21:  /* for (i=0; i<bm/2; i+=1) */

  611.     move       B0,     B      //ptrbb

  612.     move       TL,     K      /* TL = bk */

  613. #if defined(TRMMKERNEL)

  614. #if (defined(LEFT) &&  defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))

  615.     move       B0,     B    //ptrbb

  616. #else

  617.     slli.d     C3,     OFF,  0x04

  618.     add.d      A0,     A0,   C3

  619.     slli.d     C3,     OFF,  0x03

  620.     add.d      B0,     B,    C3

  621. #endif

  622. 

  623. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  624.     sub.d      TL,     K,     OFF

  625. #elif defined(LEFT)

  626.     addi.d     TL,     OFF,   2

  627. #else

  628.     addi.d     TL,     OFF,   1

  629. #endif

  630. 

  631. #endif  // #if defined(TRMMKERNEL)

  632. 

  633.     MTC        c11,    $r0

  634.     MTC        c12,    $r0

  635.     MTC        c21,    $r0

  636.     MTC        c22,    $r0

  637. 

  638.     move       L,      $r0   //cycle param k

  639.     beq        L,      TL,    .L23

  640.     blt        TL,     L,     .L23

  641. 

  642. .L22:  /* for (k=0; k<bk; k+=1) */

  643.     LD         a1,     A0,    0x00        //load0

  644.     LD         b1,     B0,    0x00        //load1

  645.     MADD1      c11,    a1,    b1,     c11  //res0

  646.     LD         a2,     A0,    0x04        //load2

  647.     MADD2      c12,    a2,    b1,     c12  //res1

  648.     LD         b2,     B0,    0x04        //load3

  649.     MADD3      c11,    a2,    b2,     c11

  650.     MADD4      c12,    a1,    b2,     c12

  651.     LD         a3,     A0,    0x08        //load4

  652.     MADD1      c21,    a3,    b1,     c21  //res2

  653.     LD         a4,     A0,    0x0c        //load5

  654.     MADD2      c22,    a4,    b1,     c22  //res3

  655.     MADD3      c21,    a4,    b2,     c21

  656.     MADD4      c22,    a3,    b2,     c22

  657. 

  658.     addi.d     A0,     A0,    0x10

  659.     addi.d     B0,     B0,    0x08

  660. 

  661.     addi.d     L,      L,     1

  662.     blt        L,      TL,    .L22

  663. 

  664. .L23:

  665. #if defined(TRMMKERNEL)

  666.     MUL        a5,     c11,   ALPHA_R

  667.     MUL        a6,     c12,   ALPHA_I

  668.     SUB        a5,     a5,    a6

  669.     ST         a5,     C0,    0x00

  670. 

  671.     MUL        a5,     c12,   ALPHA_R

  672.     MUL        a6,     c11,   ALPHA_I

  673.     ADD        a6,     a5,    a6

  674.     ST         a6,     C0,    0x04

  675. 

  676.     MUL        a7,     c21,   ALPHA_R

  677.     MUL        a8,     c22,   ALPHA_I

  678.     SUB        a7,     a7,    a8

  679.     ST         a7,     C0,    0x08

  680. 

  681.     MUL        a7,     c22,   ALPHA_R

  682.     MUL        a8,     c21,   ALPHA_I

  683.     ADD        a8,     a7,    a8

  684.     ST         a8,     C0,    0x0c

  685. #else

  686.     LD         a5,     C0,    0x00    //C0[0]

  687.     LD         a6,     C0,    0x04    //C0[1]

  688.     LD         a7,     C0,    0x08    //C1[2]

  689.     LD         a8,     C0,    0x0c    //C1[3]

  690. 

  691.     MADD       a5,     c11,   ALPHA_R, a5

  692.     MADD       a6,     c12,   ALPHA_R, a6

  693.     NMSUB      a5,     c12,   ALPHA_I, a5

  694.     MADD       a6,     c11,   ALPHA_I, a6

  695.     MADD       a7,     c21,   ALPHA_R, a7

  696.     MADD       a8,     c22,   ALPHA_R, a8

  697.     NMSUB      a7,     c22,   ALPHA_I, a7

  698.     MADD       a8,     c21,   ALPHA_I, a8

  699. 

  700.     ST         a5,     C0,    0x00

  701.     ST         a6,     C0,    0x04

  702.     ST         a7,     C0,    0x08

  703.     ST         a8,     C0,    0x0c

  704. #endif

  705. 

  706. #if defined(TRMMKERNEL)

  707. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  708.     sub.d      TL,     K,    OFF

  709. #ifdef LEFT

  710.     addi.d     TL,     TL,   -2

  711. #else

  712.     addi.d     TL,     TL,   -1

  713. #endif

  714.     slli.d     C3,     TL,   0x04

  715.     add.d      A0,     A0,   C3

  716.     slli.d     C3,     TL,   0x03

  717.     add.d      B0,     B0,   C3

  718. #endif

  719. 

  720. #ifdef LEFT

  721.     addi.d     OFF,    OFF,   2

  722. #endif

  723. #endif   // #if defined(TRMMKERNEL)

  724. 

  725.     addi.d     C0,     C0,    0x10

  726. 

  727.     addi.d     I,      I,     1

  728.     blt        I,      T0,    .L21

  729. 

  730. .L24:

  731.     move       I,      $r0

  732.     andi       T1,     M,     1    //bm&1

  733.     beq        I,      T1,    .L28

  734. 

  735. .L25:  /* for (i=0; i<(bm&1); i+=1) */

  736.     move       B0,     B      //ptrbb

  737.     move       TL,     K      /* TL = bk */

  738. #if defined(TRMMKERNEL)

  739. #if (defined(LEFT) &&  defined(TRANSA)) ||(!defined(LEFT) && !defined(TRANSA))

  740.     move       B0,     B    //ptrbb

  741. #else

  742.     slli.d     C3,     OFF,   0x03

  743.     add.d      A0,     A0,   C3

  744.     add.d      B0,     B,    C3

  745. #endif

  746. 

  747. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  748.     sub.d      TL,     K,     OFF

  749. #elif defined(LEFT)

  750.     addi.d     TL,     OFF,   1

  751. #else

  752.     addi.d     TL,     OFF,   1

  753. #endif

  754. 

  755. #endif  // #if defined(TRMMKERNEL)

  756. 

  757.     MTC        c11,    $r0

  758.     MTC        c12,    $r0

  759. 

  760.     move       L,      $r0   //cycle param k

  761.     beq        L,      TL,    .L27

  762.     blt        TL,     L,     .L27

  763. 

  764. .L26:  /* for (k=0; k<bk; k+=1) */

  765.     LD         a1,     A0,    0x00        //load0

  766.     LD         b1,     B0,    0x00        //load1

  767.     MADD1      c11,    a1,    b1,     c11  //res0

  768.     LD         a2,     A0,    0x04        //load2

  769.     MADD2      c12,    a2,    b1,     c12  //res1

  770.     LD         b2,     B0,    0x04        //load3

  771.     MADD3      c11,    a2,    b2,     c11

  772.     MADD4      c12,    a1,    b2,     c12

  773. 

  774.     addi.d     A0,     A0,    0x08

  775.     addi.d     B0,     B0,    0x08

  776. 

  777.     addi.d     L,      L,     1

  778.     blt        L,      TL,    .L26

  779. 

  780. .L27:

  781. #if defined(TRMMKERNEL)

  782.     MUL        a5,     c11,   ALPHA_R

  783.     MUL        a6,     c12,   ALPHA_I

  784.     SUB        a5,     a5,    a6

  785.     ST         a5,     C0,    0x00

  786. 

  787.     MUL        a5,     c12,   ALPHA_R

  788.     MUL        a6,     c11,   ALPHA_I

  789.     ADD        a6,     a5,    a6

  790.     ST         a6,     C0,    0x04

  791. #else

  792.     LD         a5,     C0,    0x00    //C0[0]

  793.     LD         a6,     C0,    0x04    //C0[1]

  794. 

  795.     MADD       a5,     c11,   ALPHA_R, a5

  796.     MADD       a6,     c12,   ALPHA_R, a6

  797.     NMSUB      a5,     c12,   ALPHA_I, a5

  798.     MADD       a6,     c11,   ALPHA_I, a6

  799. 

  800.     ST         a5,     C0,    0x00

  801.     ST         a6,     C0,    0x04

  802. #endif

  803. 

  804. #if defined(TRMMKERNEL)

  805. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  806.     sub.d      TL,     K,    OFF

  807. #ifdef LEFT

  808.     addi.d     TL,     TL,   -1

  809. #else

  810.     addi.d     TL,     TL,   -1

  811. #endif

  812.     slli.d     C3,     TL,   0x03

  813.     add.d      A0,     A0,   C3

  814.     add.d      B0,     B0,   C3

  815. #endif

  816. 

  817. #ifdef LEFT

  818.     addi.d     OFF,    OFF,   1

  819. #endif

  820. #endif   // #if defined(TRMMKERNEL)

  821. 

  822.     addi.d     C0,     C0,    0x08

  823. 

  824.     addi.d     I,      I,     1

  825.     blt        I,      T1,    .L25

  826. 

  827. .L28:

  828.     slli.d     L,      K,     3

  829.     add.d      B,      B,     L

  830. 

  831.     slli.d     I,      LDC,   1

  832.     add.d      C,      C,     I

  833. 

  834.     addi.d     J,      J,     1

  835.     andi       T0,     N,     1

  836.     blt        J,      T0,    .L20

  837. 

  838. .L30:

  839.     LDARG      $r23,   $sp,   0

  840.     LDARG      $r24,   $sp,   8

  841.     LDARG      $r25,   $sp,   16

  842.     LDARG      $r26,   $sp,   24

  843.     LDARG      $r27,   $sp,   32

  844.     LD         $f23,   $sp,   40

  845.     LD         $f24,   $sp,   48

  846.     LD         $f25,   $sp,   56

  847.     LD         $f26,   $sp,   64

  848.     LD         $f27,   $sp,   72

  849.     LD         $f28,   $sp,   80

  850.     LD         $f29,   $sp,   88

  851.     LD         $f30,   $sp,   96

  852.     LD         $f31,   $sp,   104

  853. 

  854.     addi.d     $sp,    $sp,   128

  855.     jirl       $r0,    $r1,   0x0

  856. 

  857.     EPILOGUE