OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c31861ea62
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c31861ea62'
${ noResults }
OpenBLAS/kernel/loongarch64/dgemm_small_kernel_nt_lasx.S

501 lines
18 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2024 The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define ASSEMBLER

  29. 

  30. #include "common.h"

  31. #include "loongarch64_asm.S"

  32. 

  33. #define M       $a0

  34. #define N       $a1

  35. #define K       $a2

  36. #define A       $a3

  37. #define LDA     $a4

  38. #define ALPHA   $f0

  39. #define B       $a5

  40. #define LDB     $a6

  41. #define C       $a7

  42. #define LDC     $t0

  43. #ifdef B0

  44. #define BETA    $f1

  45. #endif

  46. #undef ZERO

  47. #define ZERO    $r0

  48. 

  49. #define M16     $t1

  50. #define M8      $t1

  51. #define M4      $t1

  52. #define M2      $t1

  53. #define M1      $t1

  54. #define N4      $t2

  55. #define N2      $t2

  56. #define N1      $t2

  57. #define K_LDB   $t3

  58. #define A0      $t4

  59. #define X0      $t5

  60. #define C0      $t6

  61. #define C1      $t7

  62. #define C2      $t8

  63. #define C3      $s0

  64. #define K1      $s1

  65. 

  66. #define VALPHA  $xr0

  67. #ifndef B0

  68. #define VBETA   $xr1

  69. #endif

  70. #define D0     $xr2

  71. #define D1     $xr3

  72. #define D2     $xr4

  73. #define D3     $xr5

  74. #define D4     $xr6

  75. #define D5     $xr7

  76. #define D6     $xr8

  77. #define D7     $xr9

  78. #define D8     $xr10

  79. #define D9     $xr11

  80. #define D10    $xr12

  81. #define D11    $xr13

  82. #define D12    $xr14

  83. #define D13    $xr15

  84. #define D14    $xr16

  85. #define D15    $xr17

  86. #define S0     $xr18

  87. #define S1     $xr19

  88. #define S2     $xr20

  89. #define S3     $xr21

  90. #define Z0     $xr22

  91. #define Z1     $xr23

  92. #define Z2     $xr24

  93. #define Z3     $xr25

  94. #define V0     $vr2

  95. #define V1     $vr3

  96. #define V2     $vr4

  97. #define V3     $vr5

  98. #define F0     $f2

  99. #define F1     $f3

  100. #define F2     $f4

  101. #define F3     $f5

  102. 

  103. .macro DGEMM_SMALL_KERNEL_NT_TAIL M

  104.     PTR_SRAI    N4,     N,      2 // N >> 2

  105.     move        A0,     A // Restore A0

  106.     move        X0,     B // Restore X0

  107.     move        C0,     C // Restore C0

  108.     PTR_ADD     C1,     C0,     LDC

  109.     PTR_ADD     C2,     C1,     LDC

  110.     PTR_ADD     C3,     C2,     LDC

  111.     beqz        N4,     .L_M\M\()_N3

  112. .L_M\M\()_N4:

  113.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3

  114.     move        K1,     K // Restore K1

  115.     PTR_ADDI    N4,     N4,     -1

  116.     bge         ZERO,   K,      .L_M\M\()_N4_END

  117. .L_M\M\()_N4_K1:

  118.     PTR_ADDI    K1,     K1,    -1

  119.     GLD xv, , S0, A0, 0x00

  120.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18

  121.     GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1, D2, S0, Z2, D2, D3, S0, Z3, D3

  122.     PTR_ADD     X0,     X0,     LDB

  123.     PTR_ADD     A0,     A0,     LDA

  124.     bnez        K1,     .L_M\M\()_N4_K1

  125. .L_M\M\()_N4_END:

  126.     GMUL xvf, d, D0, D0, VALPHA, D1,  D1, VALPHA, D2, D2, VALPHA, D3, D3, VALPHA

  127. #ifndef B0

  128.     GLD xv, , S0, C0, 0x00

  129.     GMADD xvf, d, D0, S0, VBETA, D0

  130.     GLD xv, , S0, C1, 0x00

  131.     GMADD xvf, d, D1, S0, VBETA, D1

  132.     GLD xv, , S0, C2, 0x00

  133.     GMADD xvf, d, D2, S0, VBETA, D2

  134.     GLD xv, , S0, C3, 0x00

  135.     GMADD xvf, d, D3, S0, VBETA, D3

  136. #endif

  137. .if \M == 4

  138.     GST xv, , D0, C0, 0x00, D1, C1, 0x00, D2, C2, 0x00, D3, C3, 0x00

  139. .elseif \M == 2

  140.     GST v, , V0, C0, 0x00, V1, C1, 0x00, V2, C2, 0x00, V3, C3, 0x00

  141. .elseif \M == 1

  142.     GST f, d, F0, C0, 0x00, F1, C1, 0x00, F2, C2, 0x00, F3, C3, 0x00

  143. .endif

  144.     // Update C0, C1, C2, C3

  145.     PTR_ALSL    C0,     LDC,    C0,     2

  146.     PTR_ALSL    C1,     LDC,    C1,     2

  147.     PTR_ALSL    C2,     LDC,    C2,     2

  148.     PTR_ALSL    C3,     LDC,    C3,     2

  149.     // Update X0

  150.     PTR_SUB     X0,     X0,     K_LDB

  151.     PTR_ADDI    X0,     X0,     0x20

  152.     // Restore A0

  153.     move        A0,     A

  154.     bnez        N4,     .L_M\M\()_N4

  155. .L_M\M\()_N3:

  156.     andi        N2,     N,      0x02

  157.     beqz        N2,     .L_M\M\()_N1

  158. .L_M\M\()_N2:

  159.     GXOR xv, v, D0, D0, D0, D1, D1, D1

  160.     move        K1,     K // Restore K1

  161.     bge         ZERO,   K,      .L_M\M\()_N2_END

  162. .L_M\M\()_N2_K1:

  163.     PTR_ADDI    K1,     K1,    -1

  164.     GLD xv, , S0, A0, 0x00

  165.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08

  166.     GMADD xvf, d, D0, S0, Z0, D0, D1, S0, Z1, D1

  167.     PTR_ADD     X0,     X0,     LDB

  168.     PTR_ADD     A0,     A0,     LDA

  169.     bnez        K1,     .L_M\M\()_N2_K1

  170. .L_M\M\()_N2_END:

  171.     GMUL xvf, d, D0,  D0, VALPHA, D1, D1, VALPHA

  172. #ifndef B0

  173.     GLD xv, , S0, C0, 0x00

  174.     GMADD xvf, d, D0, S0, VBETA, D0

  175.     GLD xv, , S0, C1, 0x00

  176.     GMADD xvf, d, D1, S0, VBETA, D1

  177. #endif

  178. .if \M == 4

  179.     GST xv, , D0, C0, 0x00, D1, C1, 0x00

  180. .elseif \M == 2

  181.     GST v, , V0, C0, 0x00, V1, C1, 0x00

  182. .elseif \M == 1

  183.     GST f, d, F0, C0, 0x00, F1, C1, 0x00

  184. .endif

  185.     // Update C0, C1

  186.     PTR_ALSL    C0,     LDC,    C0,     1

  187.     PTR_ALSL    C1,     LDC,    C1,     1

  188.     // Update X0

  189.     PTR_SUB     X0,     X0,     K_LDB

  190.     PTR_ADDI    X0,     X0,     0x10

  191.     // Restore A0

  192.     move        A0,     A

  193. .L_M\M\()_N1:

  194.     andi        N1,     N,      0x01

  195.     beqz        N1,     .L_M\M\()_END

  196.     GXOR xv, v, D0, D0, D0

  197.     move        K1,     K // Restore K1

  198.     bge         ZERO,   K,      .L_M\M\()_N1_END

  199. .L_M\M\()_N1_K1:

  200.     PTR_ADDI    K1,     K1,    -1

  201.     GLD xv, , S0, A0, 0x00

  202.     GLDREPL xv, d, Z0, X0, 0x00

  203.     GMADD xvf, d, D0, S0, Z0, D0

  204.     PTR_ADD     X0,     X0,     LDB

  205.     PTR_ADD     A0,     A0,     LDA

  206.     bnez        K1,     .L_M\M\()_N1_K1

  207. .L_M\M\()_N1_END:

  208.     GMUL xvf, d, D0, D0, VALPHA

  209. #ifndef B0

  210.     GLD xv, , S0, C0, 0x00

  211.     GMADD xvf, d, D0, S0, VBETA, D0

  212. #endif

  213. .if \M == 4

  214.     GST xv, , D0, C0, 0x00

  215. .elseif \M == 2

  216.     GST v, , V0, C0, 0x00

  217. .elseif \M == 1

  218.     GST f, d, F0, C0, 0x00

  219. .endif

  220. .L_M\M\()_END:

  221. .if \M == 4

  222.     PTR_ADDI    A,      A,     0x20

  223.     PTR_ADDI    C,      C,     0x20

  224. .elseif \M == 2

  225.     PTR_ADDI    A,      A,     0x10

  226.     PTR_ADDI    C,      C,     0x10

  227. .elseif \M == 1

  228. .endif

  229. .endm

  230. 

  231.     PROLOGUE

  232.     PTR_LD          LDC,    $sp,    0

  233.     push_if_used    2, 2

  234.     xvreplve0.d     VALPHA, VALPHA

  235. #ifndef B0

  236.     xvreplve0.d     VBETA,  VBETA

  237. #endif

  238.     PTR_SLLI    LDA,    LDA,    3

  239.     PTR_SLLI    LDB,    LDB,    3

  240.     PTR_SLLI    LDC,    LDC,    3

  241.     PTR_MUL     K_LDB,  K,      LDB

  242.     PTR_SRAI    M16,    M,      4 // M >> 4

  243.     beqz        M16,    .L_M15

  244. .L_M16:

  245.     PTR_SRAI    N4,     N,      2 // N >> 2

  246.     move        A0,     A // Restore A0

  247.     move        X0,     B // Restore X0

  248.     move        C0,     C // Restore C0

  249.     PTR_ADD     C1,     C0,     LDC

  250.     PTR_ADD     C2,     C1,     LDC

  251.     PTR_ADD     C3,     C2,     LDC

  252.     beqz        N4,     .L_M16_N3

  253. .L_M16_N4:

  254.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3,       \

  255.                 D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7,       \

  256.                 D8, D8, D8, D9, D9, D9, D10, D10, D10, D11, D11, D11, \

  257.                 D12, D12, D12, D13, D13, D13, D14, D14, D14, D15, D15, D15

  258.     move        K1,     K // Restore K1

  259.     PTR_ADDI    N4,     N4,     -1

  260.     bge         ZERO,   K,      .L_M16_N4_END

  261. .L_M16_N4_K1:

  262.     PTR_ADDI    K1,     K1,    -1

  263.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60

  264.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18

  265.     GMADD xvf, d, D0,  S0, Z0, D0,  D1,  S1, Z0, D1,  D2,  S2, Z0, D2,  D3,  S3, Z0, D3,  \

  266.                   D4,  S0, Z1, D4,  D5,  S1, Z1, D5,  D6,  S2, Z1, D6,  D7,  S3, Z1, D7,  \

  267.                   D8,  S0, Z2, D8,  D9,  S1, Z2, D9,  D10, S2, Z2, D10, D11, S3, Z2, D11, \

  268.                   D12, S0, Z3, D12, D13, S1, Z3, D13, D14, S2, Z3, D14, D15, S3, Z3, D15

  269.     PTR_ADD     X0,     X0,     LDB

  270.     PTR_ADD     A0,     A0,     LDA

  271.     bnez        K1,     .L_M16_N4_K1

  272.  .L_M16_N4_END:

  273.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA, D2,  D2,  VALPHA, D3,  D3,  VALPHA, \

  274.                  D4,  D4,  VALPHA, D5,  D5,  VALPHA, D6,  D6,  VALPHA, D7,  D7,  VALPHA, \

  275.                  D8,  D8,  VALPHA, D9,  D9,  VALPHA, D10, D10, VALPHA, D11, D11, VALPHA, \

  276.                  D12, D12, VALPHA, D13, D13, VALPHA, D14, D14, VALPHA, D15, D15, VALPHA

  277. #ifndef B0

  278.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60

  279.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3

  280.     GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60

  281.     GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7

  282.     GLD xv, , S0, C2, 0x00, S1, C2, 0x20, S2, C2, 0x40, S3, C2, 0x60

  283.     GMADD xvf, d, D8, S0, VBETA, D8, D9, S1, VBETA, D9, D10, S2, VBETA, D10, D11, S3, VBETA, D11

  284.     GLD xv, , S0, C3, 0x00, S1, C3, 0x20, S2, C3, 0x40, S3, C3, 0x60

  285.     GMADD xvf, d, D12, S0, VBETA, D12, D13, S1, VBETA, D13, D14, S2, VBETA, D14, D15, S3, VBETA, D15

  286. #endif

  287.     GST xv, , D12, C3, 0x00, D13, C3, 0x20, D14, C3, 0x40, D15, C3, 0x60, \

  288.               D8,  C2, 0x00, D9,  C2, 0x20, D10, C2, 0x40, D11, C2, 0x60, \

  289.               D4,  C1, 0x00, D5,  C1, 0x20, D6,  C1, 0x40, D7,  C1, 0x60, \

  290.               D0,  C0, 0x00, D1,  C0, 0x20, D2,  C0, 0x40, D3,  C0, 0x60

  291.     // Update C0, C1, C2, C3

  292.     PTR_ALSL    C0,     LDC,    C0,     2

  293.     PTR_ALSL    C1,     LDC,    C1,     2

  294.     PTR_ALSL    C2,     LDC,    C2,     2

  295.     PTR_ALSL    C3,     LDC,    C3,     2

  296.     // Update X0

  297.     PTR_SUB     X0,     X0,     K_LDB

  298.     PTR_ADDI    X0,     X0,     0x20

  299.     // Restore A0

  300.     move        A0,     A

  301.     bnez        N4,     .L_M16_N4

  302. .L_M16_N3:

  303.     andi        N2,     N,      0x02

  304.     beqz        N2,     .L_M16_N1

  305. .L_M16_N2:

  306.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3,       \

  307.                 D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7

  308.     move        K1,     K // Restore K1

  309.     bge         ZERO,   K,      .L_M16_N2_END

  310. .L_M16_N2_K1:

  311.     PTR_ADDI    K1,     K1,    -1

  312.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60

  313.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08

  314.     GMADD xvf, d, D0,  S0, Z0, D0,  D1,  S1, Z0, D1,  D2,  S2, Z0, D2,  D3,  S3, Z0, D3, \

  315.                   D4,  S0, Z1, D4,  D5,  S1, Z1, D5,  D6,  S2, Z1, D6,  D7,  S3, Z1, D7

  316.     PTR_ADD     X0,     X0,     LDB

  317.     PTR_ADD     A0,     A0,     LDA

  318.     bnez        K1,     .L_M16_N2_K1

  319. .L_M16_N2_END:

  320.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA, D2,  D2,  VALPHA, D3,  D3,  VALPHA, \

  321.                  D4,  D4,  VALPHA, D5,  D5,  VALPHA, D6,  D6,  VALPHA, D7,  D7,  VALPHA

  322. #ifndef B0

  323.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60

  324.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3

  325.     GLD xv, , S0, C1, 0x00, S1, C1, 0x20, S2, C1, 0x40, S3, C1, 0x60

  326.     GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5, D6, S2, VBETA, D6, D7, S3, VBETA, D7

  327. #endif

  328.     GST xv, , D4,  C1, 0x00, D5,  C1, 0x20, D6,  C1, 0x40, D7,  C1, 0x60, \

  329.               D0,  C0, 0x00, D1,  C0, 0x20, D2,  C0, 0x40, D3,  C0, 0x60

  330.     // Update C0, C1

  331.     PTR_ALSL    C0,     LDC,    C0,     1

  332.     PTR_ALSL    C1,     LDC,    C1,     1

  333.     // Update X0

  334.     PTR_SUB     X0,     X0,    K_LDB

  335.     PTR_ADDI    X0,     X0,    0x10

  336.     // Restore A0

  337.     move        A0,     A

  338. .L_M16_N1:

  339.     andi        N1,     N,      0x01

  340.     beqz        N1,     .L_M16_END

  341.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3

  342.     move        K1,     K // Restore K1

  343.     bge         ZERO,   K,      .L_M16_N1_END

  344. .L_M16_N1_K1:

  345.     PTR_ADDI    K1,     K1,    -1

  346.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20, S2, A0, 0x40, S3, A0, 0x60

  347.     GLDREPL xv, d, Z0, X0, 0x00

  348.     GMADD xvf, d, D0,  S0, Z0, D0,  D1,  S1, Z0, D1,  D2,  S2, Z0, D2,  D3,  S3, Z0, D3

  349.     PTR_ADD     X0,     X0,     LDB

  350.     PTR_ADD     A0,     A0,     LDA

  351.     bnez        K1,     .L_M16_N1_K1

  352. .L_M16_N1_END:

  353.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA, D2,  D2,  VALPHA, D3,  D3,  VALPHA

  354. #ifndef B0

  355.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20, S2, C0, 0x40, S3, C0, 0x60

  356.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1, D2, S2, VBETA, D2, D3, S3, VBETA, D3

  357. #endif

  358.     GST xv, , D0,  C0, 0x00, D1,  C0, 0x20, D2,  C0, 0x40, D3,  C0, 0x60

  359.     // Update C0

  360.     PTR_ALSL    C0,     LDC,    C0,     2

  361.     // Update X0

  362.     PTR_SUB     X0,     X0,     K_LDB

  363.     PTR_ADDI    X0,     X0,     0x08

  364.     // Restore A0

  365.     move        A0,     A

  366. .L_M16_END:

  367.     PTR_ADDI    M16,    M16,   -1

  368.     PTR_ADDI    A,      A,     0x80

  369.     PTR_ADDI    C,      C,     0x80

  370.     bnez        M16,    .L_M16

  371. .L_M15:

  372.     andi        M8,     M,      0x08

  373.     beqz        M8,     .L_M7

  374. .L_M8:

  375.     PTR_SRAI    N4,     N,      2 // N >> 2

  376.     move        A0,     A // Restore A0

  377.     move        X0,     B // Restore X0

  378.     move        C0,     C // Restore C0

  379.     PTR_ADD     C1,     C0,     LDC

  380.     PTR_ADD     C2,     C1,     LDC

  381.     PTR_ADD     C3,     C2,     LDC

  382.     beqz        N4,     .L_M8_N3

  383. .L_M8_N4:

  384.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3, \

  385.                 D4, D4, D4, D5, D5, D5, D6, D6, D6, D7, D7, D7

  386.     move        K1,     K // Restore K1

  387.     PTR_ADDI    N4,     N4,     -1

  388.     bge         ZERO,   K,      .L_M8_N4_END

  389. .L_M8_N4_K1:

  390.     PTR_ADDI    K1,     K1,    -1

  391.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20

  392.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08, Z2, X0, 0x10, Z3, X0, 0x18

  393.     GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \

  394.                   D2, S0, Z1, D2, D3, S1, Z1, D3, \

  395.                   D4, S0, Z2, D4, D5, S1, Z2, D5, \

  396.                   D6, S0, Z3, D6, D7, S1, Z3, D7,

  397.     PTR_ADD     X0,     X0,     LDB

  398.     PTR_ADD     A0,     A0,     LDA

  399.     bnez        K1,     .L_M8_N4_K1

  400. .L_M8_N4_END:

  401.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA, D2,  D2,  VALPHA, D3,  D3,  VALPHA, \

  402.                  D4,  D4,  VALPHA, D5,  D5,  VALPHA, D6,  D6,  VALPHA, D7,  D7,  VALPHA

  403. #ifndef B0

  404.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20

  405.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1

  406.     GLD xv, , S0, C1, 0x00, S1, C1, 0x20

  407.     GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3

  408.     GLD xv, , S0, C2, 0x00, S1, C2, 0x20

  409.     GMADD xvf, d, D4, S0, VBETA, D4, D5, S1, VBETA, D5

  410.     GLD xv, , S0, C3, 0x00, S1, C3, 0x20

  411.     GMADD xvf, d, D6, S0, VBETA, D6, D7, S1, VBETA, D7

  412. #endif

  413.     GST xv, , D4,  C2, 0x00, D5,  C2, 0x20, D6,  C3, 0x00, D7,  C3, 0x20, \

  414.               D0,  C0, 0x00, D1,  C0, 0x20, D2,  C1, 0x00, D3,  C1, 0x20

  415.     // Update C0, C1, C2, C3

  416.     PTR_ALSL    C0,     LDC,    C0,     2

  417.     PTR_ALSL    C1,     LDC,    C1,     2

  418.     PTR_ALSL    C2,     LDC,    C2,     2

  419.     PTR_ALSL    C3,     LDC,    C3,     2

  420.     // Update X0

  421.     PTR_SUB     X0,     X0,     K_LDB

  422.     PTR_ADDI    X0,     X0,     0x20

  423.     // Restore A0

  424.     move        A0,     A

  425.     bnez        N4,     .L_M8_N4

  426. .L_M8_N3:

  427.     andi        N2,     N,      0x02

  428.     beqz        N2,     .L_M8_N1

  429. .L_M8_N2:

  430.     GXOR xv, v, D0, D0, D0, D1, D1, D1, D2, D2, D2, D3, D3, D3

  431.     move        K1,     K // Restore K1

  432.     bge         ZERO,   K,      .L_M8_N2_END

  433. .L_M8_N2_K1:

  434.     PTR_ADDI    K1,     K1,    -1

  435.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20

  436.     GLDREPL xv, d, Z0, X0, 0x00, Z1, X0, 0x08

  437.     GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1, \

  438.                   D2, S0, Z1, D2, D3, S1, Z1, D3

  439.     PTR_ADD     X0,     X0,     LDB

  440.     PTR_ADD     A0,     A0,     LDA

  441.     bnez        K1,     .L_M8_N2_K1

  442. .L_M8_N2_END:

  443.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA, D2,  D2,  VALPHA, D3,  D3,  VALPHA

  444. #ifndef B0

  445.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20

  446.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1

  447.     GLD xv, , S0, C1, 0x00, S1, C1, 0x20

  448.     GMADD xvf, d, D2, S0, VBETA, D2, D3, S1, VBETA, D3

  449. #endif

  450.     GST xv, , D0,  C0, 0x00, D1,  C0, 0x20, D2,  C1, 0x00, D3,  C1, 0x20

  451.     // Update C0, C1

  452.     PTR_ALSL    C0,     LDC,    C0,     1

  453.     PTR_ALSL    C1,     LDC,    C1,     1

  454.     // Update X0

  455.     PTR_SUB     X0,     X0,     K_LDB

  456.     PTR_ADDI    X0,     X0,     0x10

  457.     // Restore A0

  458.     move        A0,     A

  459. .L_M8_N1:

  460.     andi        N1,     N,      0x01

  461.     beqz        N1,     .L_M8_END

  462.     GXOR xv, v, D0, D0, D0, D1, D1, D1

  463.     move        K1,     K // Restore K1

  464.     bge         ZERO,   K,      .L_M8_N1_END

  465. .L_M8_N1_K1:

  466.     PTR_ADDI    K1,     K1,    -1

  467.     GLD xv, , S0, A0, 0x00, S1, A0, 0x20

  468.     GLDREPL xv, d, Z0, X0, 0x00

  469.     GMADD xvf, d, D0, S0, Z0, D0, D1, S1, Z0, D1

  470.     PTR_ADD     X0,     X0,     LDB

  471.     PTR_ADD     A0,     A0,     LDA

  472.     bnez        K1,     .L_M8_N1_K1

  473. .L_M8_N1_END:

  474.     GMUL xvf, d, D0,  D0,  VALPHA, D1,  D1,  VALPHA

  475. #ifndef B0

  476.     GLD xv, , S0, C0, 0x00, S1, C0, 0x20

  477.     GMADD xvf, d, D0, S0, VBETA, D0, D1, S1, VBETA, D1

  478. #endif

  479.     GST xv, , D0, C0, 0x00, D1, C0, 0x20

  480. .L_M8_END:

  481.     PTR_ADDI    A,      A,     0x40

  482.     PTR_ADDI    C,      C,     0x40

  483. .L_M7:

  484.     andi        M4,     M,     0x04

  485.     beqz        M4,     .L_M3

  486. .L_M4:

  487.     DGEMM_SMALL_KERNEL_NT_TAIL 4

  488. .L_M3:

  489.     andi        M2,     M,     0x02

  490.     beqz        M2,     .L_M1

  491. .L_M2:

  492.     DGEMM_SMALL_KERNEL_NT_TAIL 2

  493. .L_M1:

  494.     andi        M1,     M,     0x01

  495.     beqz        M1,     .L_M0

  496.     DGEMM_SMALL_KERNEL_NT_TAIL 1

  497. .L_M0:

  498.     pop_if_used  2, 2

  499.     jirl    $r0, $r1, 0x0

  500.     EPILOGUE