OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 855945befb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '855945befb'
${ noResults }
OpenBLAS/kernel/power/sgemm_small_kernel_tt_power...

1560 lines
53 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2021, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. #include <altivec.h>

  30. 

  31. typedef __vector unsigned char vec_t;

  32. 

  33. #if !defined(B0)

  34. #define SAVE_4x4_ACC(ACC, N, M)                         \

  35.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  36.   rc0 = vec_xl(0, C+(N+0)*ldc+M);                       \

  37.   rc0 = vec_mul(rc0, vbeta);                            \

  38.   result[0] = vec_madd(result[0], valpha, rc0);         \

  39.   vec_xst(result[0], 0, C+(N+0)*ldc+M);                 \

  40.   rc0 = vec_xl(0, C+(N+1)*ldc+M);                       \

  41.   rc0 = vec_mul(rc0, vbeta);                            \

  42.   result[1] = vec_madd(result[1], valpha, rc0);         \

  43.   vec_xst(result[1], 0, C+(N+1)*ldc+M);                 \

  44.   rc0 = vec_xl(0, C+(N+2)*ldc+M);                       \

  45.   rc0 = vec_mul(rc0, vbeta);                            \

  46.   result[2] = vec_madd(result[2], valpha, rc0);         \

  47.   vec_xst(result[2], 0, C+(N+2)*ldc+M);                 \

  48.   rc0 = vec_xl(0, C+(N+3)*ldc+M);                       \

  49.   rc0 = vec_mul(rc0, vbeta);                            \

  50.   result[3] = vec_madd(result[3], valpha, rc0);         \

  51.   vec_xst(result[3], 0, C+(N+3)*ldc+M);

  52. 

  53. #define SAVE_4x2_ACC(ACC, N, M)                         \

  54.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  55.   rc0 = vec_xl_len(C+(N+0)*ldc+M, 8);                   \

  56.   rc0 = vec_mul(rc0, vbeta);                            \

  57.   result[0] = vec_madd(result[0], valpha, rc0);         \

  58.   vec_xst_len(result[0], C+(N+0)*ldc+M, 8);             \

  59.   rc0 = vec_xl_len(C+(N+1)*ldc+M, 8);                   \

  60.   rc0 = vec_mul(rc0, vbeta);                            \

  61.   result[1] = vec_madd(result[1], valpha, rc0);         \

  62.   vec_xst_len(result[1], C+(N+1)*ldc+M, 8);             \

  63.   rc0 = vec_xl_len(C+(N+2)*ldc+M, 8);                   \

  64.   rc0 = vec_mul(rc0, vbeta);                            \

  65.   result[2] = vec_madd(result[2], valpha, rc0);         \

  66.   vec_xst_len(result[2], C+(N+2)*ldc+M, 8);             \

  67.   rc0 = vec_xl_len(C+(N+3)*ldc+M, 8);                   \

  68.   rc0 = vec_mul(rc0, vbeta);                            \

  69.   result[3] = vec_madd(result[3], valpha, rc0);         \

  70.   vec_xst_len(result[3], C+(N+3)*ldc+M, 8);

  71. 

  72. #define SAVE_2x4_ACC(ACC, N, M)                         \

  73.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  74.   rc0 = vec_xl(0, C+(N+0)*ldc+M);                       \

  75.   rc0 = vec_mul(rc0, vbeta);                            \

  76.   result[0] = vec_madd(result[0], valpha, rc0);         \

  77.   vec_xst(result[0], 0, C+(N+0)*ldc+M);                 \

  78.   rc0 = vec_xl(0, C+(N+1)*ldc+M);                       \

  79.   rc0 = vec_mul(rc0, vbeta);                            \

  80.   result[1] = vec_madd(result[1], valpha, rc0);         \

  81.   vec_xst(result[1], 0, C+(N+1)*ldc+M);

  82. 

  83. #define SAVE_1x4_VSR(result, N, M)        \

  84.   rc0 = vec_xl(0, C+((N)*ldc)+M);         \

  85.   rc0 = vec_mul(rc0, vbeta);              \

  86.   result = vec_madd(result, valpha, rc0); \

  87.   vec_xst(result, 0, C+((N)*ldc)+M);

  88. 

  89. #define SAVE_2x2_VSR(result, N, M)            \

  90.   rc0 = vec_xl_len(C+(N*ldc)+M, 8);           \

  91.   rc0 = vec_insert(C[(N+1)*ldc+M+0], rc0, 2); \

  92.   rc0 = vec_insert(C[(N+1)*ldc+M+1], rc0, 3); \

  93.   rc0 = vec_mul(rc0, vbeta);                  \

  94.   result = vec_madd(result, valpha, rc0);     \

  95.   vec_xst_len(result, C+(N*ldc)+M, 8);        \

  96.   C[(N+1)*ldc+M+0] = result[2];               \

  97.   C[(N+1)*ldc+M+1] = result[3];

  98. 

  99. #define SAVE_1x2_VSR(result, N, M)        \

  100.   rc0 = vec_xl_len(C+(N*ldc)+M, 8);       \

  101.   rc0 = vec_mul(rc0, vbeta);              \

  102.   result = vec_madd(result, valpha, rc0); \

  103.   vec_xst_len(result, C+(N*ldc)+M, 8);

  104. 

  105. #define SAVE_4x1_VSR(result, N, M)                      \

  106.   result = vec_mul(result, valpha);                     \

  107.   C[(N+0)*ldc+M] = (C[(N+0)*ldc+M] * beta) + result[0]; \

  108.   C[(N+1)*ldc+M] = (C[(N+1)*ldc+M] * beta) + result[1]; \

  109.   C[(N+2)*ldc+M] = (C[(N+2)*ldc+M] * beta) + result[2]; \

  110.   C[(N+3)*ldc+M] = (C[(N+3)*ldc+M] * beta) + result[3];

  111. 

  112. #define SAVE_2x1_VSR(result, N, M)                      \

  113.   result = vec_mul(result, valpha);                     \

  114.   C[(N+0)*ldc+M] = (C[(N+0)*ldc+M] * beta) + result[0]; \

  115.   C[(N+1)*ldc+M] = (C[(N+1)*ldc+M] * beta) + result[1];

  116. 

  117. #else

  118. 

  119. #define SAVE_4x4_ACC(ACC, N, M)                         \

  120.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  121.   result[0] = vec_mul(result[0], valpha);               \

  122.   vec_xst(result[0], 0, C+(N+0)*ldc+M);                 \

  123.   result[1] = vec_mul(result[1], valpha);               \

  124.   vec_xst(result[1], 0, C+(N+1)*ldc+M);                 \

  125.   result[2] = vec_mul(result[2], valpha);               \

  126.   vec_xst(result[2], 0, C+(N+2)*ldc+M);                 \

  127.   result[3] = vec_mul(result[3], valpha);               \

  128.   vec_xst(result[3], 0, C+(N+3)*ldc+M);

  129. 

  130. #define SAVE_4x2_ACC(ACC, N, M)                         \

  131.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  132.   result[0] = vec_mul(result[0], valpha);               \

  133.   vec_xst_len(result[0], C+(N+0)*ldc+M, 8);             \

  134.   result[1] = vec_mul(result[1], valpha);               \

  135.   vec_xst_len(result[1], C+(N+1)*ldc+M, 8);             \

  136.   result[2] = vec_mul(result[2], valpha);               \

  137.   vec_xst_len(result[2], C+(N+2)*ldc+M, 8);             \

  138.   result[3] = vec_mul(result[3], valpha);               \

  139.   vec_xst_len(result[3], C+(N+3)*ldc+M, 8);

  140. 

  141. #define SAVE_2x4_ACC(ACC, N, M)                         \

  142.   __builtin_mma_disassemble_acc ((void *)result, ACC);  \

  143.   result[0] = vec_mul(result[0], valpha);               \

  144.   vec_xst(result[0], 0, C+(N+0)*ldc+M);                 \

  145.   result[1] = vec_mul(result[1], valpha);               \

  146.   vec_xst(result[1], 0, C+(N+1)*ldc+M);

  147. 

  148. #define SAVE_1x4_VSR(result, N, M)    \

  149.   result = vec_mul(result, valpha);   \

  150.   vec_xst(result, 0, C+((N)*ldc)+M);

  151. 

  152. #define SAVE_2x2_VSR(result, N, M)      \

  153.   result = vec_mul(result, valpha);     \

  154.   vec_xst_len(result, C+(N*ldc)+M, 8);  \

  155.   C[(N+1)*ldc+M+0] = result[2];         \

  156.   C[(N+1)*ldc+M+1] = result[3];

  157. 

  158. #define SAVE_1x2_VSR(result, N, M)    \

  159.   result = vec_mul(result, valpha);   \

  160.   vec_xst_len(result, C+(N*ldc)+M, 8);

  161. 

  162. #define SAVE_4x1_VSR(result, N, M)  \

  163.   result = vec_mul(result, valpha); \

  164.   C[(N+0)*ldc+M] = result[0];       \

  165.   C[(N+1)*ldc+M] = result[1];       \

  166.   C[(N+2)*ldc+M] = result[2];       \

  167.   C[(N+3)*ldc+M] = result[3];

  168. 

  169. #define SAVE_2x1_VSR(result, N, M)  \

  170.   result = vec_mul(result, valpha); \

  171.   C[(N+0)*ldc+M] = result[0];       \

  172.   C[(N+1)*ldc+M] = result[1];

  173. 

  174. #endif

  175. 

  176. #define INIT_8ACCS()              \

  177.   __builtin_mma_xxsetaccz(&acc0); \

  178.   __builtin_mma_xxsetaccz(&acc1); \

  179.   __builtin_mma_xxsetaccz(&acc2); \

  180.   __builtin_mma_xxsetaccz(&acc3); \

  181.   __builtin_mma_xxsetaccz(&acc4); \

  182.   __builtin_mma_xxsetaccz(&acc5); \

  183.   __builtin_mma_xxsetaccz(&acc6); \

  184.   __builtin_mma_xxsetaccz(&acc7);

  185. 

  186. #define INIT_4ACCS()              \

  187.   __builtin_mma_xxsetaccz(&acc0); \

  188.   __builtin_mma_xxsetaccz(&acc1); \

  189.   __builtin_mma_xxsetaccz(&acc2); \

  190.   __builtin_mma_xxsetaccz(&acc3);

  191. 

  192. #define INIT_2ACCS()              \

  193.   __builtin_mma_xxsetaccz(&acc0); \

  194.   __builtin_mma_xxsetaccz(&acc1);

  195. 

  196. #define INIT_1ACC() __builtin_mma_xxsetaccz(&acc0);

  197. 

  198. #define LOAD_AT_16x4(M, K)           \

  199.   ra0 = vec_xl(0, A+(M+0)*lda+K);    \

  200.   ra1 = vec_xl(0, A+(M+1)*lda+K);    \

  201.   t0 = vec_mergeh(ra0, ra1);         \

  202.   t1 = vec_mergel(ra0, ra1);         \

  203.   ra2 = vec_xl(0, A+(M+2)*lda+K);    \

  204.   ra3 = vec_xl(0, A+(M+3)*lda+K);    \

  205.   t2 = vec_mergeh(ra2, ra3);         \

  206.   t3 = vec_mergel(ra2, ra3);         \

  207.   ra0 = vec_xxpermdi(t0, t2, 0b00);  \

  208.   ra1 = vec_xxpermdi(t0, t2, 0b11);  \

  209.   ra2 = vec_xxpermdi(t1, t3, 0b00);  \

  210.   ra3 = vec_xxpermdi(t1, t3, 0b11);  \

  211.   ra4 = vec_xl(0, A+(M+4)*lda+K);    \

  212.   ra5 = vec_xl(0, A+(M+5)*lda+K);    \

  213.   t0 = vec_mergeh(ra4, ra5);         \

  214.   t1 = vec_mergel(ra4, ra5);         \

  215.   ra6 = vec_xl(0, A+(M+6)*lda+K);    \

  216.   ra7 = vec_xl(0, A+(M+7)*lda+K);    \

  217.   t2 = vec_mergeh(ra6, ra7);         \

  218.   t3 = vec_mergel(ra6, ra7);         \

  219.   ra4 = vec_xxpermdi(t0, t2, 0b00);  \

  220.   ra5 = vec_xxpermdi(t0, t2, 0b11);  \

  221.   ra6 = vec_xxpermdi(t1, t3, 0b00);  \

  222.   ra7 = vec_xxpermdi(t1, t3, 0b11);  \

  223.   ra8 = vec_xl(0, A+(M+8)*lda+K);    \

  224.   ra9 = vec_xl(0, A+(M+9)*lda+K);    \

  225.   t0 = vec_mergeh(ra8, ra9);         \

  226.   t1 = vec_mergel(ra8, ra9);         \

  227.   ra10 = vec_xl(0, A+(M+10)*lda+K);  \

  228.   ra11 = vec_xl(0, A+(M+11)*lda+K);  \

  229.   t2 = vec_mergeh(ra10, ra11);       \

  230.   t3 = vec_mergel(ra10, ra11);       \

  231.   ra8 = vec_xxpermdi(t0, t2, 0b00);  \

  232.   ra9 = vec_xxpermdi(t0, t2, 0b11);  \

  233.   ra10 = vec_xxpermdi(t1, t3, 0b00); \

  234.   ra11 = vec_xxpermdi(t1, t3, 0b11); \

  235.   ra12 = vec_xl(0, A+(M+12)*lda+K);  \

  236.   ra13 = vec_xl(0, A+(M+13)*lda+K);  \

  237.   t0 = vec_mergeh(ra12, ra13);       \

  238.   t1 = vec_mergel(ra12, ra13);       \

  239.   ra14 = vec_xl(0, A+(M+14)*lda+K);  \

  240.   ra15 = vec_xl(0, A+(M+15)*lda+K);  \

  241.   t2 = vec_mergeh(ra14, ra15);       \

  242.   t3 = vec_mergel(ra14, ra15);       \

  243.   ra12 = vec_xxpermdi(t0, t2, 0b00); \

  244.   ra13 = vec_xxpermdi(t0, t2, 0b11); \

  245.   ra14 = vec_xxpermdi(t1, t3, 0b00); \

  246.   ra15 = vec_xxpermdi(t1, t3, 0b11);

  247. 

  248. #define LOAD_AT_16x2(M, K)              \

  249.   ra0 = vec_xl_len(A+(M+0)*lda+K, 8);   \

  250.   ra1 = vec_xl_len(A+(M+1)*lda+K, 8);   \

  251.   t0 = vec_mergeh(ra0, ra1);            \

  252.   ra2 = vec_xl_len(A+(M+2)*lda+K, 8);   \

  253.   ra3 = vec_xl_len(A+(M+3)*lda+K, 8);   \

  254.   t1 = vec_mergeh(ra2, ra3);            \

  255.   ra0 = vec_xxpermdi(t0, t1, 0b00);     \

  256.   ra1 = vec_xxpermdi(t0, t1, 0b11);     \

  257.   ra4 = vec_xl_len(A+(M+4)*lda+K, 8);   \

  258.   ra5 = vec_xl_len(A+(M+5)*lda+K, 8);   \

  259.   t0 = vec_mergeh(ra4, ra5);            \

  260.   ra6 = vec_xl_len(A+(M+6)*lda+K, 8);   \

  261.   ra7 = vec_xl_len(A+(M+7)*lda+K, 8);   \

  262.   t1 = vec_mergeh(ra6, ra7);            \

  263.   ra2 = vec_xxpermdi(t0, t1, 0b00);     \

  264.   ra3 = vec_xxpermdi(t0, t1, 0b11);     \

  265.   ra8 = vec_xl_len(A+(M+8)*lda+K, 8);   \

  266.   ra9 = vec_xl_len(A+(M+9)*lda+K, 8);   \

  267.   t0 = vec_mergeh(ra8, ra9);            \

  268.   ra10 = vec_xl_len(A+(M+10)*lda+K, 8); \

  269.   ra11 = vec_xl_len(A+(M+11)*lda+K, 8); \

  270.   t1 = vec_mergeh(ra10, ra11);          \

  271.   ra4 = vec_xxpermdi(t0, t1, 0b00);     \

  272.   ra5 = vec_xxpermdi(t0, t1, 0b11);     \

  273.   ra12 = vec_xl_len(A+(M+12)*lda+K, 8); \

  274.   ra13 = vec_xl_len(A+(M+13)*lda+K, 8); \

  275.   t0 = vec_mergeh(ra12, ra13);          \

  276.   ra14 = vec_xl_len(A+(M+14)*lda+K, 8); \

  277.   ra15 = vec_xl_len(A+(M+15)*lda+K, 8); \

  278.   t1 = vec_mergeh(ra14, ra15);          \

  279.   ra6 = vec_xxpermdi(t0, t1, 0b00);     \

  280.   ra7 = vec_xxpermdi(t0, t1, 0b11);

  281. 

  282. #define LOAD_AT_16x1(M, K)                   \

  283.   ra0 = vec_xor(ra0, ra0);                   \

  284.   ra0 = vec_insert(A[(M+0)*lda+K], ra0, 0);  \

  285.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 1);  \

  286.   ra0 = vec_insert(A[(M+2)*lda+K], ra0, 2);  \

  287.   ra0 = vec_insert(A[(M+3)*lda+K], ra0, 3);  \

  288.   ra1 = vec_xor(ra1, ra1);                   \

  289.   ra1 = vec_insert(A[(M+4)*lda+K], ra1, 0);  \

  290.   ra1 = vec_insert(A[(M+5)*lda+K], ra1, 1);  \

  291.   ra1 = vec_insert(A[(M+6)*lda+K], ra1, 2);  \

  292.   ra1 = vec_insert(A[(M+7)*lda+K], ra1, 3);  \

  293.   ra2 = vec_xor(ra2, ra2);                   \

  294.   ra2 = vec_insert(A[(M+8)*lda+K], ra2, 0);  \

  295.   ra2 = vec_insert(A[(M+9)*lda+K], ra2, 1);  \

  296.   ra2 = vec_insert(A[(M+10)*lda+K], ra2, 2); \

  297.   ra2 = vec_insert(A[(M+11)*lda+K], ra2, 3); \

  298.   ra3 = vec_xor(ra3, ra3);                   \

  299.   ra3 = vec_insert(A[(M+12)*lda+K], ra3, 0); \

  300.   ra3 = vec_insert(A[(M+13)*lda+K], ra3, 1); \

  301.   ra3 = vec_insert(A[(M+14)*lda+K], ra3, 2); \

  302.   ra3 = vec_insert(A[(M+15)*lda+K], ra3, 3);

  303. 

  304. #define LOAD_AT_8x4(M, K)           \

  305.   ra0 = vec_xl(0, A+(M+0)*lda+K);   \

  306.   ra1 = vec_xl(0, A+(M+1)*lda+K);   \

  307.   t0 = vec_mergeh(ra0, ra1);        \

  308.   t1 = vec_mergel(ra0, ra1);        \

  309.   ra2 = vec_xl(0, A+(M+2)*lda+K);   \

  310.   ra3 = vec_xl(0, A+(M+3)*lda+K);   \

  311.   t2 = vec_mergeh(ra2, ra3);        \

  312.   t3 = vec_mergel(ra2, ra3);        \

  313.   ra0 = vec_xxpermdi(t0, t2, 0b00); \

  314.   ra1 = vec_xxpermdi(t0, t2, 0b11); \

  315.   ra2 = vec_xxpermdi(t1, t3, 0b00); \

  316.   ra3 = vec_xxpermdi(t1, t3, 0b11); \

  317.   ra4 = vec_xl(0, A+(M+4)*lda+K);   \

  318.   ra5 = vec_xl(0, A+(M+5)*lda+K);   \

  319.   t0 = vec_mergeh(ra4, ra5);        \

  320.   t1 = vec_mergel(ra4, ra5);        \

  321.   ra6 = vec_xl(0, A+(M+6)*lda+K);   \

  322.   ra7 = vec_xl(0, A+(M+7)*lda+K);   \

  323.   t2 = vec_mergeh(ra6, ra7);        \

  324.   t3 = vec_mergel(ra6, ra7);        \

  325.   ra4 = vec_xxpermdi(t0, t2, 0b00); \

  326.   ra5 = vec_xxpermdi(t0, t2, 0b11); \

  327.   ra6 = vec_xxpermdi(t1, t3, 0b00); \

  328.   ra7 = vec_xxpermdi(t1, t3, 0b11);

  329. 

  330. #define LOAD_AT_8x2(M, K)             \

  331.   ra0 = vec_xl_len(A+(M+0)*lda+K, 8); \

  332.   ra1 = vec_xl_len(A+(M+1)*lda+K, 8); \

  333.   t0 = vec_mergeh(ra0, ra1);          \

  334.   ra2 = vec_xl_len(A+(M+2)*lda+K, 8); \

  335.   ra3 = vec_xl_len(A+(M+3)*lda+K, 8); \

  336.   t1 = vec_mergeh(ra2, ra3);          \

  337.   ra0 = vec_xxpermdi(t0, t1, 0b00);   \

  338.   ra1 = vec_xxpermdi(t0, t1, 0b11);   \

  339.   ra4 = vec_xl_len(A+(M+4)*lda+K, 8); \

  340.   ra5 = vec_xl_len(A+(M+5)*lda+K, 8); \

  341.   t0 = vec_mergeh(ra4, ra5);          \

  342.   ra6 = vec_xl_len(A+(M+6)*lda+K, 8); \

  343.   ra7 = vec_xl_len(A+(M+7)*lda+K, 8); \

  344.   t1 = vec_mergeh(ra6, ra7);          \

  345.   ra2 = vec_xxpermdi(t0, t1, 0b00);   \

  346.   ra3 = vec_xxpermdi(t0, t1, 0b11);

  347. 

  348. #define LOAD_AT_8x1(M, K)                   \

  349.   ra0 = vec_xor(ra0, ra0);                  \

  350.   ra0 = vec_insert(A[(M+0)*lda+K], ra0, 0); \

  351.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 1); \

  352.   ra0 = vec_insert(A[(M+2)*lda+K], ra0, 2); \

  353.   ra0 = vec_insert(A[(M+3)*lda+K], ra0, 3); \

  354.   ra1 = vec_xor(ra1, ra1);                  \

  355.   ra1 = vec_insert(A[(M+4)*lda+K], ra1, 0); \

  356.   ra1 = vec_insert(A[(M+5)*lda+K], ra1, 1); \

  357.   ra1 = vec_insert(A[(M+6)*lda+K], ra1, 2); \

  358.   ra1 = vec_insert(A[(M+7)*lda+K], ra1, 3);

  359. 

  360. #define LOAD_AT_4x4(M, K)           \

  361.   ra0 = vec_xl(0, A+(M+0)*lda+K);   \

  362.   ra1 = vec_xl(0, A+(M+1)*lda+K);   \

  363.   t0 = vec_mergeh(ra0, ra1);        \

  364.   t1 = vec_mergel(ra0, ra1);        \

  365.   ra2 = vec_xl(0, A+(M+2)*lda+K);   \

  366.   ra3 = vec_xl(0, A+(M+3)*lda+K);   \

  367.   t2 = vec_mergeh(ra2, ra3);        \

  368.   t3 = vec_mergel(ra2, ra3);        \

  369.   ra0 = vec_xxpermdi(t0, t2, 0b00); \

  370.   ra1 = vec_xxpermdi(t0, t2, 0b11); \

  371.   ra2 = vec_xxpermdi(t1, t3, 0b00); \

  372.   ra3 = vec_xxpermdi(t1, t3, 0b11);

  373. 

  374. #define LOAD_AT_4x2(M, K)             \

  375.   ra0 = vec_xl_len(A+(M+0)*lda+K, 8); \

  376.   ra1 = vec_xl_len(A+(M+1)*lda+K, 8); \

  377.   t0 = vec_mergeh(ra0, ra1);          \

  378.   ra2 = vec_xl_len(A+(M+2)*lda+K, 8); \

  379.   ra3 = vec_xl_len(A+(M+3)*lda+K, 8); \

  380.   t1 = vec_mergeh(ra2, ra3);          \

  381.   ra0 = vec_xxpermdi(t0, t1, 0b00);   \

  382.   ra1 = vec_xxpermdi(t0, t1, 0b11);

  383. 

  384. #define LOAD_AT_4x1(M, K)                   \

  385.   ra0 = vec_xor(ra0, ra0);                  \

  386.   ra0 = vec_insert(A[(M+0)*lda+K], ra0, 0); \

  387.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 1); \

  388.   ra0 = vec_insert(A[(M+2)*lda+K], ra0, 2); \

  389.   ra0 = vec_insert(A[(M+3)*lda+K], ra0, 3);

  390. 

  391. #define LOAD_AT_2x4(M, K)                       \

  392.   ra0 = vec_xl(0, A+(M+0)*lda+K);               \

  393.   ra1 = vec_xl(0, A+(M+1)*lda+K);               \

  394.   t0 = vec_mergeh(ra0, ra1);                    \

  395.   t1 = vec_mergeo(ra0, ra1);                    \

  396.   t2 = vec_mergel(ra0, ra1);                    \

  397.   ra0 = t0;                                     \

  398.   ra1 = t1;                                     \

  399.   ra2 = t2;                                     \

  400.   ra3 = vec_xor(ra3, ra3);                      \

  401.   ra3 = vec_insert(vec_extract(t2,2), ra3, 0);  \

  402.   ra3 = vec_insert(vec_extract(t2,3), ra3, 1);

  403. 

  404. #define LOAD_AT_2x2(M, K)             \

  405.   ra0 = vec_xl_len(A+(M+0)*lda+K, 8); \

  406.   ra1 = vec_xl_len(A+(M+1)*lda+K, 8); \

  407.   t0 = vec_mergee(ra0, ra1);          \

  408.   t1 = vec_mergeo(ra0, ra1);          \

  409.   ra0 = t0;                           \

  410.   ra1 = t1;

  411. 

  412. #define LOAD_AT_2x1(M, K)                   \

  413.   ra0 = vec_xor(ra0, ra0);                  \

  414.   ra0 = vec_insert(A[(M+0)*lda+K], ra0, 0); \

  415.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 1);

  416. 

  417. #define LOAD_A_2x2(M, K)                    \

  418.   ra0 = vec_splats(A[M*lda+K]);             \

  419.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 1); \

  420.   ra0 = vec_insert(A[(M+1)*lda+K], ra0, 3);

  421. 

  422. #define LOAD_A_1x1(M, K) ra0 = vec_splats(A[M*lda+K]);

  423. 

  424. #define LOAD_B_1x16(K, N)            \

  425.   rb0 = vec_xl(0, B+((K)*ldb)+N+0); \

  426.   rb1 = vec_xl(0, B+((K)*ldb)+N+4); \

  427.   rb2 = vec_xl(0, B+((K)*ldb)+N+8); \

  428.   rb3 = vec_xl(0, B+((K)*ldb)+N+12);

  429. 

  430. #define LOAD_B_1x8(K, N)            \

  431.   rb0 = vec_xl(0, B+((K)*ldb)+N+0); \

  432.   rb1 = vec_xl(0, B+((K)*ldb)+N+4);

  433. 

  434. #define LOAD_B_1x4(K, N) rb0 = vec_xl(0, B+((K)*ldb)+N);

  435. 

  436. #define LOAD_B_1x2(K, N) rb0 = vec_xl_len(B+((K)*ldb)+N, 8);

  437. 

  438. #define LOAD_B_2x2(K, N)                  \

  439.   rb0 = vec_splats(B[K*ldb+N]);         \

  440.   rb0 = vec_insert(B[K*ldb+N+1], rb0, 2); \

  441.   rb0 = vec_insert(B[K*ldb+N+1], rb0, 3);

  442. 

  443. #define LOAD_B_1x1(K, N) rb0 = vec_splats(B[(K)*ldb+N]);

  444. 

  445. #define KERNEL_MMA_8ACC(b0, b1, b2, b3, b4, b5, b6, b7,   \

  446.                         a0, a1, a2, a3, a4, a5, a6, a7)   \

  447.   __builtin_mma_xvf32gerpp(&acc0, (vec_t)b0, (vec_t)a0);  \

  448.   __builtin_mma_xvf32gerpp(&acc1, (vec_t)b1, (vec_t)a1);  \

  449.   __builtin_mma_xvf32gerpp(&acc2, (vec_t)b2, (vec_t)a2);  \

  450.   __builtin_mma_xvf32gerpp(&acc3, (vec_t)b3, (vec_t)a3);  \

  451.   __builtin_mma_xvf32gerpp(&acc4, (vec_t)b4, (vec_t)a4);  \

  452.   __builtin_mma_xvf32gerpp(&acc5, (vec_t)b5, (vec_t)a5);  \

  453.   __builtin_mma_xvf32gerpp(&acc6, (vec_t)b6, (vec_t)a6);  \

  454.   __builtin_mma_xvf32gerpp(&acc7, (vec_t)b7, (vec_t)a7);

  455. 

  456. #define KERNEL_MMA_4ACC(b0, b1, b2, b3, a0, a1, a2, a3)   \

  457.   __builtin_mma_xvf32gerpp(&acc0, (vec_t)b0, (vec_t)a0);  \

  458.   __builtin_mma_xvf32gerpp(&acc1, (vec_t)b1, (vec_t)a1);  \

  459.   __builtin_mma_xvf32gerpp(&acc2, (vec_t)b2, (vec_t)a2);  \

  460.   __builtin_mma_xvf32gerpp(&acc3, (vec_t)b3, (vec_t)a3);

  461. 

  462. #define KERNEL_MMA_2ACC(b0, b1, a0, a1)                   \

  463.   __builtin_mma_xvf32gerpp(&acc0, (vec_t)b0, (vec_t)a0);  \

  464.   __builtin_mma_xvf32gerpp(&acc1, (vec_t)b1, (vec_t)a1);

  465. 

  466. #define KERNEL_MMA_1ACC(b0, a0)                           \

  467.   __builtin_mma_xvf32gerpp(&acc0, (vec_t)b0, (vec_t)a0);

  468. 

  469. #define KERNEL_VMADD_4VSR(a0, a1, a2, a3, b0, b1, b2, b3) \

  470.   result = vec_madd(a0, b0, result);                      \

  471.   result1 = vec_madd(a1, b1, result1);                    \

  472.   result2 = vec_madd(a2, b2, result2);                    \

  473.   result3 = vec_madd(a3, b3, result3);

  474. 

  475. #define KERNEL_VMADD_2VSR(a0, a1, b0, b1) \

  476.   result = vec_madd(a0, b0, result);      \

  477.   result1 = vec_madd(a1, b1, result1);

  478. 

  479. #define KERNEL_VMADD_1VSR(a0, b0)     \

  480.   result = vec_madd(a0, b0, result);

  481. 

  482. #define PACK_A(ra0, ra1, ra2, ra3, offset) \

  483.   vec_xst(ra0, 0, packA+(k*16)+0+offset);  \

  484.   vec_xst(ra1, 0, packA+(k*16)+4+offset);  \

  485.   vec_xst(ra2, 0, packA+(k*16)+8+offset);  \

  486.   vec_xst(ra3, 0, packA+(k*16)+12+offset);

  487. 

  488. #define LOAD_PACKED_A(ra0, ra1, ra2, ra3, offset) \

  489.   ra0 = vec_xl(0, packA+(k*16)+0+offset);         \

  490.   ra1 = vec_xl(0, packA+(k*16)+4+offset);         \

  491.   ra2 = vec_xl(0, packA+(k*16)+8+offset);         \

  492.   ra3 = vec_xl(0, packA+(k*16)+12+offset);

  493. 

  494. #ifdef B0

  495. int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT * C, BLASLONG ldc)

  496. #else

  497. int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, IFLOAT * A, BLASLONG lda, FLOAT alpha, IFLOAT * B, BLASLONG ldb, FLOAT beta, FLOAT * C, BLASLONG ldc)

  498. #endif

  499. {

  500.   BLASLONG m, n, k;

  501. 

  502.   BLASLONG m16 = M & ~15;

  503.   BLASLONG m8 = M & ~7;

  504.   BLASLONG m4 = M & ~3;

  505.   BLASLONG m2 = M & ~1;

  506. 

  507.   BLASLONG n16 = N & ~15;

  508.   BLASLONG n8 = N & ~7;

  509.   BLASLONG n4 = N & ~3;

  510.   BLASLONG n2 = N & ~1;

  511. 

  512.   BLASLONG k4 = K & ~3;

  513.   BLASLONG k2 = K & ~1;

  514. 

  515.   vector float valpha = vec_splats(alpha);

  516. #if !defined(B0)

  517.   vector float vbeta = vec_splats(beta);

  518. #endif

  519. 

  520. #if defined(__GNUC__) && !defined(__clang__)

  521.   int has_packing = (M >= 32 && N >= 32 && K >= 32) ? 1 : 0;

  522. #else

  523.   int has_packing = 0;

  524. #endif

  525. 

  526.   float *packA;

  527.   if (has_packing) packA = (float *)malloc(K*16*sizeof(float));

  528. 

  529.   for (m = 0; m < m16; m += 16) {

  530.     for (n = 0; n < n8; n += 8) {

  531.       __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  532. 

  533.       INIT_8ACCS();

  534. 

  535.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7, ra8, ra9,

  536.           ra10, ra11, ra12, ra13, ra14, ra15;

  537.       register vector float rb0, rb1;

  538.       register vector float t0, t1, t2, t3;

  539. 

  540.       if (has_packing) {

  541.         if (n == 0) {

  542.           for (k = 0; k < k4; k += 4) {

  543.             LOAD_AT_16x4(m, k);

  544.             LOAD_B_1x8(k, n);

  545.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  546.                             ra0, ra4, ra8, ra12, ra0, ra4, ra8, ra12);

  547.             PACK_A(ra0, ra4, ra8, ra12, 0);

  548.             LOAD_B_1x8(k+1, n);

  549.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  550.                             ra1, ra5, ra9, ra13, ra1, ra5, ra9, ra13);

  551.             PACK_A(ra1, ra5, ra9, ra13, 16);

  552.             LOAD_B_1x8(k+2, n);

  553.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  554.                             ra2, ra6, ra10, ra14, ra2, ra6, ra10, ra14);

  555.             PACK_A(ra2, ra6, ra10, ra14, 32);

  556.             LOAD_B_1x8(k+3, n);

  557.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  558.                             ra3, ra7, ra11, ra15, ra3, ra7, ra11, ra15);

  559.             PACK_A(ra3, ra7, ra11, ra15, 48);

  560.           }

  561.           for (; k < k2; k += 2) {

  562.             LOAD_AT_16x2(m, k);

  563.             LOAD_B_1x8(k, n);

  564.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  565.                             ra0, ra2, ra4, ra6, ra0, ra2, ra4, ra6);

  566.             PACK_A(ra0, ra2, ra4, ra6, 0);

  567.             LOAD_B_1x8(k+1, n);

  568.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  569.                             ra1, ra3, ra5, ra7, ra1, ra3, ra5, ra7);

  570.             PACK_A(ra1, ra3, ra5, ra7, 16);

  571.           }

  572.           for (; k < K; k++) {

  573.             LOAD_AT_16x1(m, k);

  574.             LOAD_B_1x8(k, n);

  575.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  576.                             ra0, ra1, ra2, ra3, ra0, ra1, ra2, ra3);

  577.             PACK_A(ra0, ra1, ra2, ra3, 0);

  578.           }

  579.         } else {

  580.           for (k = 0; k < k4; k += 4) {

  581.             LOAD_PACKED_A(ra0, ra4, ra8, ra12, 0);

  582.             LOAD_B_1x8(k, n);

  583.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  584.                             ra0, ra4, ra8, ra12, ra0, ra4, ra8, ra12);

  585.             LOAD_PACKED_A(ra1, ra5, ra9, ra13, 16);

  586.             LOAD_B_1x8(k+1, n);

  587.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  588.                             ra1, ra5, ra9, ra13, ra1, ra5, ra9, ra13);

  589.             LOAD_PACKED_A(ra2, ra6, ra10, ra14, 32);

  590.             LOAD_B_1x8(k+2, n);

  591.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  592.                             ra2, ra6, ra10, ra14, ra2, ra6, ra10, ra14);

  593.             LOAD_PACKED_A(ra3, ra7, ra11, ra15, 48);

  594.             LOAD_B_1x8(k+3, n);

  595.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  596.                             ra3, ra7, ra11, ra15, ra3, ra7, ra11, ra15);

  597.           }

  598.           for (; k < k2; k += 2) {

  599.             LOAD_PACKED_A(ra0, ra2, ra4, ra6, 0);

  600.             LOAD_B_1x8(k, n);

  601.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  602.                             ra0, ra2, ra4, ra6, ra0, ra2, ra4, ra6);

  603.             LOAD_PACKED_A(ra1, ra3, ra5, ra7, 16);

  604.             LOAD_B_1x8(k+1, n);

  605.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  606.                             ra1, ra3, ra5, ra7, ra1, ra3, ra5, ra7);

  607.           }

  608.           for (; k < K; k++) {

  609.             LOAD_PACKED_A(ra0, ra1, ra2, ra3, 0);

  610.             LOAD_B_1x8(k, n);

  611.             KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  612.                             ra0, ra1, ra2, ra3, ra0, ra1, ra2, ra3);

  613.           }

  614.         }

  615.       } else {

  616.         for (k = 0; k < k4; k += 4) {

  617.           LOAD_AT_16x4(m, k);

  618.           LOAD_B_1x8(k, n);

  619.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  620.                           ra0, ra4, ra8, ra12, ra0, ra4, ra8, ra12);

  621.           LOAD_B_1x8(k+1, n);

  622.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  623.                           ra1, ra5, ra9, ra13, ra1, ra5, ra9, ra13);

  624.           LOAD_B_1x8(k+2, n);

  625.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  626.                           ra2, ra6, ra10, ra14, ra2, ra6, ra10, ra14);

  627.           LOAD_B_1x8(k+3, n);

  628.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  629.                           ra3, ra7, ra11, ra15, ra3, ra7, ra11, ra15);

  630.         }

  631.         for (; k < k2; k += 2) {

  632.           LOAD_AT_16x2(m, k);

  633.           LOAD_B_1x8(k, n);

  634.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  635.                           ra0, ra2, ra4, ra6, ra0, ra2, ra4, ra6);

  636.           LOAD_B_1x8(k+1, n);

  637.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  638.                           ra1, ra3, ra5, ra7, ra1, ra3, ra5, ra7);

  639.         }

  640.         for (; k < K; k++) {

  641.           LOAD_AT_16x1(m, k);

  642.           LOAD_B_1x8(k, n);

  643.           KERNEL_MMA_8ACC(rb0, rb0, rb0, rb0, rb1, rb1, rb1, rb1,

  644.                           ra0, ra1, ra2, ra3, ra0, ra1, ra2, ra3);

  645.         }

  646.       }

  647. 

  648. #if !defined(B0)

  649.       register vector float rc0;

  650. #endif

  651.       vector float result[4];

  652.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  653.       SAVE_4x4_ACC(&acc1, n+0, m+4);

  654.       SAVE_4x4_ACC(&acc2, n+0, m+8);

  655.       SAVE_4x4_ACC(&acc3, n+0, m+12);

  656.       SAVE_4x4_ACC(&acc4, n+4, m+0);

  657.       SAVE_4x4_ACC(&acc5, n+4, m+4);

  658.       SAVE_4x4_ACC(&acc6, n+4, m+8);

  659.       SAVE_4x4_ACC(&acc7, n+4, m+12);

  660.     }

  661. 

  662.     for (; n < n4; n += 4) {

  663.       __vector_quad acc0, acc1, acc2, acc3;

  664. 

  665.       INIT_4ACCS();

  666. 

  667.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7, ra8, ra9,

  668.           ra10, ra11, ra12, ra13, ra14, ra15;

  669.       register vector float rb0;

  670.       register vector float t0, t1, t2, t3;

  671. 

  672.       if (!has_packing) {

  673.         for (k = 0; k < k4; k += 4) {

  674.           LOAD_AT_16x4(m, k);

  675.           LOAD_B_1x4(k, n);

  676.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra4, ra8, ra12);

  677.           LOAD_B_1x4(k+1, n);

  678.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra5, ra9, ra13);

  679.           LOAD_B_1x4(k+2, n);

  680.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra2, ra6, ra10, ra14);

  681.           LOAD_B_1x4(k+3, n);

  682.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra3, ra7, ra11, ra15);

  683.         }

  684.         for (; k < k2; k += 2) {

  685.           LOAD_AT_16x2(m, k);

  686.           LOAD_B_1x4(k, n);

  687.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra2, ra4, ra6);

  688.           LOAD_B_1x4(k+1, n);

  689.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra3, ra5, ra7);

  690.         }

  691.         for (; k < K; k++) {

  692.           LOAD_AT_16x1(m, k);

  693.           LOAD_B_1x4(k, n);

  694.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra1, ra2, ra3);

  695.         }

  696.       } else {

  697.         for (k = 0; k < k4; k += 4) {

  698.           LOAD_PACKED_A(ra0, ra4, ra8, ra12, 0);

  699.           LOAD_B_1x4(k, n);

  700.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra4, ra8, ra12);

  701.           LOAD_PACKED_A(ra1, ra5, ra9, ra13, 16);

  702.           LOAD_B_1x4(k+1, n);

  703.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra5, ra9, ra13);

  704.           LOAD_PACKED_A(ra2, ra6, ra10, ra14, 32);

  705.           LOAD_B_1x4(k+2, n);

  706.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra2, ra6, ra10, ra14);

  707.           LOAD_PACKED_A(ra3, ra7, ra11, ra15, 48);

  708.           LOAD_B_1x4(k+3, n);

  709.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra3, ra7, ra11, ra15);

  710.         }

  711.         for (; k < k2; k += 2) {

  712.           LOAD_PACKED_A(ra0, ra2, ra4, ra6, 0);

  713.           LOAD_B_1x4(k, n);

  714.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra2, ra4, ra6);

  715.           LOAD_PACKED_A(ra1, ra3, ra5, ra7, 16);

  716.           LOAD_B_1x4(k+1, n);

  717.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra3, ra5, ra7);

  718.         }

  719.         for (; k < K; k++) {

  720.           LOAD_PACKED_A(ra0, ra1, ra2, ra3, 0);

  721.           LOAD_B_1x4(k, n);

  722.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra1, ra2, ra3);

  723.         }

  724.       }

  725. 

  726. #if !defined(B0)

  727.       register vector float rc0;

  728. #endif

  729.       vector float result[4];

  730.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  731.       SAVE_4x4_ACC(&acc1, n+0, m+4);

  732.       SAVE_4x4_ACC(&acc2, n+0, m+8);

  733.       SAVE_4x4_ACC(&acc3, n+0, m+12);

  734.     }

  735. 

  736.     for (; n < n2; n += 2) {

  737.       __vector_quad acc0, acc1, acc2, acc3;

  738. 

  739.       INIT_4ACCS();

  740. 

  741.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7, ra8, ra9,

  742.           ra10, ra11, ra12, ra13, ra14, ra15;

  743.       register vector float rb0;

  744.       register vector float t0, t1, t2, t3;

  745. 

  746.       if (!has_packing) {

  747.         for (k = 0; k < k4; k += 4) {

  748.           LOAD_AT_16x4(m, k);

  749.           LOAD_B_1x2(k, n);

  750.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra4, ra8, ra12);

  751.           LOAD_B_1x2(k+1, n);

  752.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra5, ra9, ra13);

  753.           LOAD_B_1x2(k+2, n);

  754.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra2, ra6, ra10, ra14);

  755.           LOAD_B_1x2(k+3, n);

  756.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra3, ra7, ra11, ra15);

  757.         }

  758.         for (; k < k2; k += 2) {

  759.           LOAD_AT_16x2(m, k);

  760.           LOAD_B_1x2(k, n);

  761.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra2, ra4, ra6);

  762.           LOAD_B_1x2(k+1, n);

  763.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra3, ra5, ra7);

  764.         }

  765.         for (; k < K; k++) {

  766.           LOAD_AT_16x1(m, k);

  767.           LOAD_B_1x2(k, n);

  768.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra1, ra2, ra3);

  769.         }

  770.       } else {

  771.         for (k = 0; k < k4; k += 4) {

  772.           LOAD_PACKED_A(ra0, ra4, ra8, ra12, 0);

  773.           LOAD_B_1x2(k, n);

  774.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra4, ra8, ra12);

  775.           LOAD_PACKED_A(ra1, ra5, ra9, ra13, 16);

  776.           LOAD_B_1x2(k+1, n);

  777.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra5, ra9, ra13);

  778.           LOAD_PACKED_A(ra2, ra6, ra10, ra14, 32);

  779.           LOAD_B_1x2(k+2, n);

  780.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra2, ra6, ra10, ra14);

  781.           LOAD_PACKED_A(ra3, ra7, ra11, ra15, 48);

  782.           LOAD_B_1x2(k+3, n);

  783.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra3, ra7, ra11, ra15);

  784.         }

  785.         for (; k < k2; k += 2) {

  786.           LOAD_B_1x2(k, n);

  787.           LOAD_PACKED_A(ra0, ra2, ra4, ra6, 0);

  788.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra2, ra4, ra6);

  789.           LOAD_B_1x2(k+1, n);

  790.           LOAD_PACKED_A(ra1, ra3, ra5, ra7, 16);

  791.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra1, ra3, ra5, ra7);

  792.         }

  793.         for (; k < K; k++) {

  794.           LOAD_PACKED_A(ra0, ra1, ra2, ra3, 0);

  795.           LOAD_B_1x2(k, n);

  796.           KERNEL_MMA_4ACC(rb0, rb0, rb0, rb0, ra0, ra1, ra2, ra3);

  797.         }

  798.       }

  799. 

  800. #if !defined(B0)

  801.       register vector float rc0;

  802. #endif

  803.       vector float result[4];

  804.       SAVE_2x4_ACC(&acc0, n, m+0);

  805.       SAVE_2x4_ACC(&acc1, n, m+4);

  806.       SAVE_2x4_ACC(&acc2, n, m+8);

  807.       SAVE_2x4_ACC(&acc3, n, m+12);

  808.     }

  809. 

  810.     for (; n < N; n++) {

  811.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7, ra8, ra9,

  812.           ra10, ra11, ra12, ra13, ra14, ra15;

  813.       register vector float rb0;

  814.       register vector float t0, t1, t2, t3;

  815. 

  816.       vector float result = ((vector float){0.,0.,0.,0.});

  817.       vector float result1 = ((vector float){0.,0.,0.,0.});

  818.       vector float result2 = ((vector float){0.,0.,0.,0.});

  819.       vector float result3 = ((vector float){0.,0.,0.,0.});

  820. 

  821.       if (!has_packing) {

  822.         for (k = 0; k < k4; k += 4) {

  823.           LOAD_AT_16x4(m, k);

  824.           LOAD_B_1x1(k, n);

  825.           KERNEL_VMADD_4VSR(ra0, ra4, ra8, ra12, rb0, rb0, rb0, rb0);

  826.           LOAD_B_1x1(k+1, n);

  827.           KERNEL_VMADD_4VSR(ra1, ra5, ra9, ra13, rb0, rb0, rb0, rb0);

  828.           LOAD_B_1x1(k+2, n);

  829.           KERNEL_VMADD_4VSR(ra2, ra6, ra10, ra14, rb0, rb0, rb0, rb0);

  830.           LOAD_B_1x1(k+3, n);

  831.           KERNEL_VMADD_4VSR(ra3, ra7, ra11, ra15, rb0, rb0, rb0, rb0);

  832.         }

  833.         for (; k < k2; k += 2) {

  834.           LOAD_AT_16x2(m, k);

  835.           LOAD_B_1x1(k, n);

  836.           KERNEL_VMADD_4VSR(ra0, ra2, ra4, ra6, rb0, rb0, rb0, rb0);

  837.           LOAD_B_1x1(k+1, n);

  838.           KERNEL_VMADD_4VSR(ra1, ra3, ra5, ra7, rb0, rb0, rb0, rb0);

  839.         }

  840.         for (; k < K; k++) {

  841.           LOAD_AT_16x1(m, k);

  842.           LOAD_B_1x1(k, n);

  843.           KERNEL_VMADD_4VSR(ra0, ra1, ra2, ra3, rb0, rb0, rb0, rb0);

  844.         }

  845.       } else {

  846.         for (k = 0; k < k4; k += 4) {

  847.           LOAD_PACKED_A(ra0, ra4, ra8, ra12, 0);

  848.           LOAD_B_1x1(k, n);

  849.           KERNEL_VMADD_4VSR(ra0, ra4, ra8, ra12, rb0, rb0, rb0, rb0);

  850.           LOAD_PACKED_A(ra1, ra5, ra9, ra13, 16);

  851.           LOAD_B_1x1(k+1, n);

  852.           KERNEL_VMADD_4VSR(ra1, ra5, ra9, ra13, rb0, rb0, rb0, rb0);

  853.           LOAD_PACKED_A(ra2, ra6, ra10, ra14, 32);

  854.           LOAD_B_1x1(k+2, n);

  855.           KERNEL_VMADD_4VSR(ra2, ra6, ra10, ra14, rb0, rb0, rb0, rb0);

  856.           LOAD_PACKED_A(ra3, ra7, ra11, ra15, 48);

  857.           LOAD_B_1x1(k+3, n);

  858.           KERNEL_VMADD_4VSR(ra3, ra7, ra11, ra15, rb0, rb0, rb0, rb0);

  859.         }

  860.         for (; k < k2; k += 2) {

  861.           LOAD_PACKED_A(ra0, ra2, ra4, ra6, 0);

  862.           LOAD_B_1x1(k, n);

  863.           KERNEL_VMADD_4VSR(ra0, ra2, ra4, ra6, rb0, rb0, rb0, rb0);

  864.           LOAD_PACKED_A(ra1, ra3, ra5, ra7, 16);

  865.           LOAD_B_1x1(k+1, n);

  866.           KERNEL_VMADD_4VSR(ra1, ra3, ra5, ra7, rb0, rb0, rb0, rb0);

  867.         }

  868.         for (; k < K; k++) {

  869.           LOAD_PACKED_A(ra0, ra1, ra2, ra3, 0);

  870.           LOAD_B_1x1(k, n);

  871.           KERNEL_VMADD_4VSR(ra0, ra1, ra2, ra3, rb0, rb0, rb0, rb0);

  872.         }

  873.       }

  874. 

  875. #if !defined(B0)

  876.       register vector float rc0;

  877. #endif

  878.       SAVE_1x4_VSR(result, n, m);

  879.       SAVE_1x4_VSR(result1, n, m+4);

  880.       SAVE_1x4_VSR(result2, n, m+8);

  881.       SAVE_1x4_VSR(result3, n, m+12);

  882.     }

  883.   }

  884. 

  885.   for (; m < m8; m += 8) {

  886.     for (n = 0; n < n16; n += 16) {

  887.       __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  888. 

  889.       INIT_8ACCS();

  890. 

  891.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7;

  892.       register vector float rb0, rb1, rb2, rb3;

  893.       register vector float t0, t1, t2, t3;

  894. 

  895.       for (k = 0; k < k4; k += 4) {

  896.         LOAD_AT_8x4(m, k);

  897.         LOAD_B_1x16(k, n);

  898.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  899.                         ra0, ra0, ra0, ra0, ra4, ra4, ra4, ra4);

  900.         LOAD_B_1x16(k+1, n);

  901.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  902.                         ra1, ra1, ra1, ra1, ra5, ra5, ra5, ra5);

  903.         LOAD_B_1x16(k+2, n);

  904.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  905.                         ra2, ra2, ra2, ra2, ra6, ra6, ra6, ra6);

  906.         LOAD_B_1x16(k+3, n);

  907.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  908.                         ra3, ra3, ra3, ra3, ra7, ra7, ra7, ra7);

  909.       }

  910.       for (; k < k2; k += 2) {

  911.         LOAD_AT_8x2(m, k);

  912.         LOAD_B_1x16(k, n);

  913.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  914.                         ra0, ra0, ra0, ra0, ra2, ra2, ra2, ra2);

  915.         LOAD_B_1x16(k+1, n);

  916.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  917.                         ra1, ra1, ra1, ra1, ra3, ra3, ra3, ra3);

  918.       }

  919.       for (; k < K; k++) {

  920.         LOAD_AT_8x1(m, k);

  921.         LOAD_B_1x16(k, n);

  922.         KERNEL_MMA_8ACC(rb0, rb1, rb2, rb3, rb0, rb1, rb2, rb3,

  923.                         ra0, ra0, ra0, ra0, ra1, ra1, ra1, ra1);

  924.       }

  925. 

  926. #if !defined(B0)

  927.       register vector float rc0;

  928. #endif

  929.       vector float result[4];

  930.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  931.       SAVE_4x4_ACC(&acc4, n+0, m+4);

  932.       SAVE_4x4_ACC(&acc1, n+4, m+0);

  933.       SAVE_4x4_ACC(&acc5, n+4, m+4);

  934.       SAVE_4x4_ACC(&acc2, n+8, m+0);

  935.       SAVE_4x4_ACC(&acc6, n+8, m+4);

  936.       SAVE_4x4_ACC(&acc3, n+12, m+0);

  937.       SAVE_4x4_ACC(&acc7, n+12, m+4);

  938.     }

  939. 

  940.     for (; n < n8; n += 8) {

  941.       __vector_quad acc0, acc1, acc2, acc3;

  942. 

  943.       INIT_4ACCS();

  944. 

  945.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7;

  946.       register vector float rb0, rb1;

  947.       register vector float t0, t1, t2, t3;

  948. 

  949.       for (k = 0; k < k4; k += 4) {

  950.         LOAD_AT_8x4(m, k);

  951.         LOAD_B_1x8(k, n);

  952.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra0, ra0, ra4, ra4);

  953.         LOAD_B_1x8(k+1, n);

  954.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra1, ra1, ra5, ra5);

  955.         LOAD_B_1x8(k+2, n);

  956.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra2, ra2, ra6, ra6);

  957.         LOAD_B_1x8(k+3, n);

  958.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra3, ra3, ra7, ra7);

  959.       }

  960.       for (; k < k2; k += 2) {

  961.         LOAD_AT_8x2(m, k);

  962.         LOAD_B_1x8(k, n);

  963.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra0, ra0, ra2, ra2);

  964.         LOAD_B_1x8(k+1, n);

  965.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra1, ra1, ra3, ra3);

  966.       }

  967.       for (; k < K; k++) {

  968.         LOAD_AT_8x1(m, k);

  969.         LOAD_B_1x8(k, n);

  970.         KERNEL_MMA_4ACC(rb0, rb1, rb0, rb1, ra0, ra0, ra1, ra1);

  971.       }

  972. 

  973. #if !defined(B0)

  974.       register vector float rc0;

  975. #endif

  976.       vector float result[4];

  977.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  978.       SAVE_4x4_ACC(&acc2, n+0, m+4);

  979.       SAVE_4x4_ACC(&acc1, n+4, m+0);

  980.       SAVE_4x4_ACC(&acc3, n+4, m+4);

  981.     }

  982. 

  983.     for (; n < n4; n += 4) {

  984.       __vector_quad acc0, acc1;

  985. 

  986.       INIT_2ACCS();

  987. 

  988.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7;

  989.       register vector float rb0;

  990.       register vector float t0, t1, t2, t3;

  991. 

  992.       for (k = 0; k < k4; k += 4) {

  993.         LOAD_AT_8x4(m, k);

  994.         LOAD_B_1x4(k, n);

  995.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra4);

  996.         LOAD_B_1x4(k+1, n);

  997.         KERNEL_MMA_2ACC(rb0, rb0, ra1, ra5);

  998.         LOAD_B_1x4(k+2, n);

  999.         KERNEL_MMA_2ACC(rb0, rb0, ra2, ra6);

  1000.         LOAD_B_1x4(k+3, n);

  1001.         KERNEL_MMA_2ACC(rb0, rb0, ra3, ra7);

  1002.       }

  1003.       for (; k < k2; k += 2) {

  1004.         LOAD_AT_8x2(m, k);

  1005.         LOAD_B_1x4(k, n);

  1006.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra2);

  1007.         LOAD_B_1x4(k+1, n);

  1008.         KERNEL_MMA_2ACC(rb0, rb0, ra1, ra3);

  1009.       }

  1010.       for (; k < K; k++) {

  1011.         LOAD_AT_8x1(m, k);

  1012.         LOAD_B_1x4(k, n);

  1013.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra1);

  1014.       }

  1015. 

  1016. #if !defined(B0)

  1017.       register vector float rc0;

  1018. #endif

  1019.       vector float result[4];

  1020.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  1021.       SAVE_4x4_ACC(&acc1, n+0, m+4);

  1022.     }

  1023. 

  1024.     for (; n < n2; n += 2) {

  1025.       __vector_quad acc0, acc1;

  1026. 

  1027.       INIT_2ACCS();

  1028. 

  1029.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7;

  1030.       register vector float rb0;

  1031.       register vector float t0, t1, t2, t3;

  1032. 

  1033.       for (k = 0; k < k4; k += 4) {

  1034.         LOAD_AT_8x4(m, k);

  1035.         LOAD_B_1x2(k, n);

  1036.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra4);

  1037.         LOAD_B_1x2(k+1, n);

  1038.         KERNEL_MMA_2ACC(rb0, rb0, ra1, ra5);

  1039.         LOAD_B_1x2(k+2, n);

  1040.         KERNEL_MMA_2ACC(rb0, rb0, ra2, ra6);

  1041.         LOAD_B_1x2(k+3, n);

  1042.         KERNEL_MMA_2ACC(rb0, rb0, ra3, ra7);

  1043.       }

  1044.       for (; k < k2; k += 2) {

  1045.         LOAD_AT_8x2(m, k);

  1046.         LOAD_B_1x2(k, n);

  1047.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra2);

  1048.         LOAD_B_1x2(k+1, n);

  1049.         KERNEL_MMA_2ACC(rb0, rb0, ra1, ra3);

  1050.       }

  1051.       for (; k < K; k++) {

  1052.         LOAD_AT_8x1(m, k);

  1053.         LOAD_B_1x2(k, n);

  1054.         KERNEL_MMA_2ACC(rb0, rb0, ra0, ra1);

  1055.       }

  1056. 

  1057. #if !defined(B0)

  1058.       register vector float rc0;

  1059. #endif

  1060.       vector float result[4];

  1061.       SAVE_2x4_ACC(&acc0, n, m+0);

  1062.       SAVE_2x4_ACC(&acc1, n, m+4);

  1063.     }

  1064. 

  1065.     for (; n < N; n++) {

  1066.       register vector float ra0, ra1, ra2, ra3, ra4, ra5, ra6, ra7;

  1067.       register vector float rb0;

  1068.       register vector float t0, t1, t2, t3;

  1069. 

  1070.       vector float result = ((vector float){0.,0.,0.,0.});

  1071.       vector float result1 = ((vector float){0.,0.,0.,0.});

  1072. 

  1073.       for (k = 0; k < k4; k += 4) {

  1074.         LOAD_AT_8x4(m, k);

  1075.         LOAD_B_1x1(k, n);

  1076.         KERNEL_VMADD_2VSR(ra0, ra4, rb0, rb0);

  1077.         LOAD_B_1x1(k+1, n);

  1078.         KERNEL_VMADD_2VSR(ra1, ra5, rb0, rb0);

  1079.         LOAD_B_1x1(k+2, n);

  1080.         KERNEL_VMADD_2VSR(ra2, ra6, rb0, rb0);

  1081.         LOAD_B_1x1(k+3, n);

  1082.         KERNEL_VMADD_2VSR(ra3, ra7, rb0, rb0);

  1083.       }

  1084.       for (; k < k2; k += 2) {

  1085.         LOAD_AT_8x2(m, k);

  1086.         LOAD_B_1x1(k, n);

  1087.         KERNEL_VMADD_2VSR(ra0, ra2, rb0, rb0);

  1088.         LOAD_B_1x1(k+1, n);

  1089.         KERNEL_VMADD_2VSR(ra1, ra3, rb0, rb0);

  1090.       }

  1091.       for (; k < K; k++) {

  1092.         LOAD_AT_8x1(m, k);

  1093.         LOAD_B_1x1(k, n);

  1094.         KERNEL_VMADD_2VSR(ra0, ra1, rb0, rb0);

  1095.       }

  1096. 

  1097. #if !defined(B0)

  1098.       register vector float rc0;

  1099. #endif

  1100.       SAVE_1x4_VSR(result, n, m);

  1101.       SAVE_1x4_VSR(result1, n, m+4);

  1102.     }

  1103.   }

  1104. 

  1105.   for (; m < m4; m += 4) {

  1106.     for (n = 0; n < n16; n += 16) {

  1107.       __vector_quad acc0, acc1, acc2, acc3;

  1108. 

  1109.       INIT_4ACCS();

  1110. 

  1111.       register vector float ra0, ra1, ra2, ra3;

  1112.       register vector float rb0, rb1, rb2, rb3;

  1113.       register vector float t0, t1, t2, t3;

  1114. 

  1115.       for (k = 0; k < k4; k += 4) {

  1116.         LOAD_AT_4x4(m, k);

  1117.         LOAD_B_1x16(k, n);

  1118.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1119.         LOAD_B_1x16(k+1, n);

  1120.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra1, ra1, ra1, ra1);

  1121.         LOAD_B_1x16(k+2, n);

  1122.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra2, ra2, ra2, ra2);

  1123.         LOAD_B_1x16(k+3, n);

  1124.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra3, ra3, ra3, ra3);

  1125.       }

  1126.       for (; k < k2; k += 2) {

  1127.         LOAD_AT_4x2(m, k);

  1128.         LOAD_B_1x16(k, n);

  1129.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1130.         LOAD_B_1x16(k+1, n);

  1131.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra1, ra1, ra1, ra1);

  1132.       }

  1133.       for (; k < K; k++) {

  1134.         LOAD_AT_4x1(m, k);

  1135.         LOAD_B_1x16(k, n);

  1136.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1137.       }

  1138. 

  1139. #if !defined(B0)

  1140.       register vector float rc0;

  1141. #endif

  1142.       vector float result[4];

  1143.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  1144.       SAVE_4x4_ACC(&acc1, n+4, m+0);

  1145.       SAVE_4x4_ACC(&acc2, n+8, m+0);

  1146.       SAVE_4x4_ACC(&acc3, n+12, m+0);

  1147.     }

  1148. 

  1149.     for (; n < n8; n += 8) {

  1150.       __vector_quad acc0, acc1;

  1151. 

  1152.       INIT_2ACCS();

  1153. 

  1154.       register vector float ra0, ra1, ra2, ra3;

  1155.       register vector float rb0, rb1;

  1156.       register vector float t0, t1, t2, t3;

  1157. 

  1158.       for (k = 0; k < k4; k += 4) {

  1159.         LOAD_AT_4x4(m, k);

  1160.         LOAD_B_1x8(k, n);

  1161.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1162.         LOAD_B_1x8(k+1, n);

  1163.         KERNEL_MMA_2ACC(rb0, rb1, ra1, ra1);

  1164.         LOAD_B_1x8(k+2, n);

  1165.         KERNEL_MMA_2ACC(rb0, rb1, ra2, ra2);

  1166.         LOAD_B_1x8(k+3, n);

  1167.         KERNEL_MMA_2ACC(rb0, rb1, ra3, ra3);

  1168.       }

  1169.       for (; k < k2; k += 2) {

  1170.         LOAD_AT_4x2(m, k);

  1171.         LOAD_B_1x8(k, n);

  1172.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1173.         LOAD_B_1x8(k+1, n);

  1174.         KERNEL_MMA_2ACC(rb0, rb1, ra1, ra1);

  1175.       }

  1176.       for (; k < K; k++) {

  1177.         LOAD_AT_4x1(m, k);

  1178.         LOAD_B_1x8(k, n);

  1179.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1180.       }

  1181. 

  1182. #if !defined(B0)

  1183.       register vector float rc0;

  1184. #endif

  1185.       vector float result[4];

  1186.       SAVE_4x4_ACC(&acc0, n+0, m+0);

  1187.       SAVE_4x4_ACC(&acc1, n+4, m+0);

  1188.     }

  1189. 

  1190.     for (; n < n4; n += 4) {

  1191.       __vector_quad acc0;

  1192. 

  1193.       INIT_1ACC();

  1194. 

  1195.       register vector float ra0, ra1, ra2, ra3;

  1196.       register vector float rb0;

  1197.       register vector float t0, t1, t2, t3;

  1198. 

  1199.       for (k = 0; k < k4; k += 4) {

  1200.         LOAD_AT_4x4(m, k);

  1201.         LOAD_B_1x4(k, n);

  1202.         KERNEL_MMA_1ACC(rb0, ra0);

  1203.         LOAD_B_1x4(k+1, n);

  1204.         KERNEL_MMA_1ACC(rb0, ra1);

  1205.         LOAD_B_1x4(k+2, n);

  1206.         KERNEL_MMA_1ACC(rb0, ra2);

  1207.         LOAD_B_1x4(k+3, n);

  1208.         KERNEL_MMA_1ACC(rb0, ra3);

  1209.       }

  1210.       for (; k < k2; k += 2) {

  1211.         LOAD_AT_4x2(m, k);

  1212.         LOAD_B_1x4(k, n);

  1213.         KERNEL_MMA_1ACC(rb0, ra0);

  1214.         LOAD_B_1x4(k+1, n);

  1215.         KERNEL_MMA_1ACC(rb0, ra1);

  1216.       }

  1217.       for (; k < K; k++) {

  1218.         LOAD_AT_4x1(m, k);

  1219.         LOAD_B_1x4(k, n);

  1220.         KERNEL_MMA_1ACC(rb0, ra0);

  1221.       }

  1222. 

  1223. #if !defined(B0)

  1224.       register vector float rc0;

  1225. #endif

  1226.       vector float result[4];

  1227.       SAVE_4x4_ACC(&acc0, n, m);

  1228.     }

  1229. 

  1230.     for (; n < n2; n += 2) {

  1231.       __vector_quad acc0;

  1232. 

  1233.       INIT_1ACC();

  1234. 

  1235.       register vector float ra0, ra1, ra2, ra3;

  1236.       register vector float rb0;

  1237.       register vector float t0, t1, t2, t3;

  1238. 

  1239.       for (k = 0; k < k4; k += 4) {

  1240.         LOAD_AT_4x4(m, k);

  1241.         LOAD_B_1x2(k, n);

  1242.         KERNEL_MMA_1ACC(rb0, ra0);

  1243.         LOAD_B_1x2(k+1, n);

  1244.         KERNEL_MMA_1ACC(rb0, ra1);

  1245.         LOAD_B_1x2(k+2, n);

  1246.         KERNEL_MMA_1ACC(rb0, ra2);

  1247.         LOAD_B_1x2(k+3, n);

  1248.         KERNEL_MMA_1ACC(rb0, ra3);

  1249.       }

  1250.       for (; k < k2; k += 2) {

  1251.         LOAD_AT_4x2(m, k);

  1252.         LOAD_B_1x2(k, n);

  1253.         KERNEL_MMA_1ACC(rb0, ra0);

  1254.         LOAD_B_1x2(k+1, n);

  1255.         KERNEL_MMA_1ACC(rb0, ra1);

  1256.       }

  1257.       for (; k < K; k++) {

  1258.         LOAD_AT_4x1(m, k);

  1259.         LOAD_B_1x2(k, n);

  1260.         KERNEL_MMA_1ACC(rb0, ra0);

  1261.       }

  1262. 

  1263. #if !defined(B0)

  1264.       register vector float rc0;

  1265. #endif

  1266.       vector float result[4];

  1267.       SAVE_2x4_ACC(&acc0, n, m);

  1268.     }

  1269. 

  1270.     for (; n < N; n++) {

  1271.       register vector float ra0, ra1, ra2, ra3;

  1272.       register vector float rb0;

  1273.       register vector float t0, t1, t2, t3;

  1274. 

  1275.       vector float result = ((vector float){0.,0.,0.,0.});

  1276. 

  1277.       for (k = 0; k < k4; k += 4) {

  1278.         LOAD_AT_4x4(m, k);

  1279.         LOAD_B_1x1(k, n);

  1280.         KERNEL_VMADD_1VSR(ra0, rb0);

  1281.         LOAD_B_1x1(k+1, n);

  1282.         KERNEL_VMADD_1VSR(ra1, rb0);

  1283.         LOAD_B_1x1(k+2, n);

  1284.         KERNEL_VMADD_1VSR(ra2, rb0);

  1285.         LOAD_B_1x1(k+3, n);

  1286.         KERNEL_VMADD_1VSR(ra3, rb0);

  1287.       }

  1288.       for (; k < k2; k += 2) {

  1289.         LOAD_AT_4x2(m, k);

  1290.         LOAD_B_1x1(k, n);

  1291.         KERNEL_VMADD_1VSR(ra0, rb0);

  1292.         LOAD_B_1x1(k+1, n);

  1293.         KERNEL_VMADD_1VSR(ra1, rb0);

  1294.       }

  1295.       for (; k < K; k++) {

  1296.         LOAD_AT_4x1(m, k);

  1297.         LOAD_B_1x1(k, n);

  1298.         KERNEL_VMADD_1VSR(ra0, rb0);

  1299.       }

  1300. 

  1301. #if !defined(B0)

  1302.       register vector float rc0;

  1303. #endif

  1304.       SAVE_1x4_VSR(result, n, m);

  1305.     }

  1306.   }

  1307. 

  1308.   for (; m < m2; m += 2) {

  1309.     for (n = 0; n < n16; n += 16) {

  1310.       __vector_quad acc0, acc1, acc2, acc3;

  1311. 

  1312.       INIT_4ACCS();

  1313. 

  1314.       register vector float ra0, ra1, ra2, ra3;

  1315.       register vector float rb0, rb1, rb2, rb3;

  1316.       register vector float t0, t1, t2;

  1317. 

  1318.       for (k = 0; k < k4; k += 4) {

  1319.         LOAD_AT_2x4(m, k);

  1320.         LOAD_B_1x16(k, n);

  1321.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1322.         LOAD_B_1x16(k+1, n);

  1323.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra1, ra1, ra1, ra1);

  1324.         LOAD_B_1x16(k+2, n);

  1325.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra2, ra2, ra2, ra2);

  1326.         LOAD_B_1x16(k+3, n);

  1327.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra3, ra3, ra3, ra3);

  1328.       }

  1329.       for (; k < k2; k += 2) {

  1330.         LOAD_AT_2x2(m, k);

  1331.         LOAD_B_1x16(k, n);

  1332.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1333.         LOAD_B_1x16(k+1, n);

  1334.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra1, ra1, ra1, ra1);

  1335.       }

  1336.       for (; k < K; k++) {

  1337.         LOAD_AT_2x1(m, k);

  1338.         LOAD_B_1x16(k, n);

  1339.         KERNEL_MMA_4ACC(rb0, rb1, rb2, rb3, ra0, ra0, ra0, ra0);

  1340.       }

  1341. 

  1342. #if !defined(B0)

  1343.       register vector float rc0;

  1344. #endif

  1345.       vector float result[4];

  1346.       SAVE_4x2_ACC(&acc0, n+0, m+0);

  1347.       SAVE_4x2_ACC(&acc1, n+4, m+0);

  1348.       SAVE_4x2_ACC(&acc2, n+8, m+0);

  1349.       SAVE_4x2_ACC(&acc3, n+12, m+0);

  1350.     }

  1351. 

  1352.     for (; n < n8; n += 8) {

  1353.       __vector_quad acc0, acc1;

  1354. 

  1355.       INIT_2ACCS();

  1356. 

  1357.       register vector float ra0, ra1, ra2, ra3;

  1358.       register vector float rb0, rb1;

  1359.       register vector float t0, t1, t2;

  1360. 

  1361.       for (k = 0; k < k4; k += 4) {

  1362.         LOAD_AT_2x4(m, k);

  1363.         LOAD_B_1x8(k, n);

  1364.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1365.         LOAD_B_1x8(k+1, n);

  1366.         KERNEL_MMA_2ACC(rb0, rb1, ra1, ra1);

  1367.         LOAD_B_1x8(k+2, n);

  1368.         KERNEL_MMA_2ACC(rb0, rb1, ra2, ra2);

  1369.         LOAD_B_1x8(k+3, n);

  1370.         KERNEL_MMA_2ACC(rb0, rb1, ra3, ra3);

  1371.       }

  1372.       for (; k < k2; k += 2) {

  1373.         LOAD_AT_2x2(m, k);

  1374.         LOAD_B_1x8(k, n);

  1375.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1376.         LOAD_B_1x8(k+1, n);

  1377.         KERNEL_MMA_2ACC(rb0, rb1, ra1, ra1);

  1378.       }

  1379.       for (; k < K; k++) {

  1380.         LOAD_AT_2x1(m, k);

  1381.         LOAD_B_1x8(k, n);

  1382.         KERNEL_MMA_2ACC(rb0, rb1, ra0, ra0);

  1383.       }

  1384. 

  1385. #if !defined(B0)

  1386.       register vector float rc0;

  1387. #endif

  1388.       vector float result[4];

  1389.       SAVE_4x2_ACC(&acc0, n+0, m+0);

  1390.       SAVE_4x2_ACC(&acc1, n+4, m+0);

  1391.     }

  1392. 

  1393.     for (; n < n4; n += 4) {

  1394.       __vector_quad acc0;

  1395. 

  1396.       INIT_1ACC();

  1397. 

  1398.       register vector float ra0, ra1, ra2, ra3;

  1399.       register vector float rb0;

  1400.       register vector float t0, t1, t2;

  1401. 

  1402.       for (k = 0; k < k4; k += 4) {

  1403.         LOAD_AT_2x4(m, k);

  1404.         LOAD_B_1x4(k, n);

  1405.         KERNEL_MMA_1ACC(rb0, ra0);

  1406.         LOAD_B_1x4(k+1, n);

  1407.         KERNEL_MMA_1ACC(rb0, ra1);

  1408.         LOAD_B_1x4(k+2, n);

  1409.         KERNEL_MMA_1ACC(rb0, ra2);

  1410.         LOAD_B_1x4(k+3, n);

  1411.         KERNEL_MMA_1ACC(rb0, ra3);

  1412.       }

  1413.       for (; k < k2; k += 2) {

  1414.         LOAD_AT_2x2(m, k);

  1415.         LOAD_B_1x4(k, n);

  1416.         KERNEL_MMA_1ACC(rb0, ra0);

  1417.         LOAD_B_1x4(k+1, n);

  1418.         KERNEL_MMA_1ACC(rb0, ra1);

  1419.       }

  1420.       for (; k < K; k++) {

  1421.         LOAD_AT_2x1(m, k);

  1422.         LOAD_B_1x4(k, n);

  1423.         KERNEL_MMA_1ACC(rb0, ra0);

  1424.       }

  1425. 

  1426. #if !defined(B0)

  1427.       register vector float rc0;

  1428. #endif

  1429.       vector float result[4];

  1430.       SAVE_4x2_ACC(&acc0, n, m);

  1431.     }

  1432. 

  1433.     for (; n < n2; n += 2) {

  1434.       vector float result = ((vector float){0.,0.,0.,0.});

  1435. 

  1436.       register vector float ra0;

  1437.       register vector float rb0;

  1438. 

  1439.       for (k = 0; k < K; k++) {

  1440.         LOAD_A_2x2(m, k);

  1441.         LOAD_B_2x2(k, n);

  1442.         KERNEL_VMADD_1VSR(ra0, rb0);

  1443.       }

  1444. 

  1445. #if !defined(B0)

  1446.       register vector float rc0;

  1447. #endif

  1448.       SAVE_2x2_VSR(result, n, m);

  1449.     }

  1450. 

  1451.     for (; n < N; n++) {

  1452.       vector float result = ((vector float){0.,0.,0.,0.});

  1453.       register vector float ra0;

  1454.       register vector float rb0;

  1455. 

  1456.       for (k = 0; k < K; k++) {

  1457.         LOAD_A_1x1(m, k);

  1458.         ra0 = vec_insert(A[(m+1)*lda+k], ra0, 1);

  1459.         ra0 = vec_insert(A[(m+1)*lda+k], ra0, 3);

  1460.         LOAD_B_1x1(k, n);

  1461.         KERNEL_VMADD_1VSR(ra0, rb0);

  1462.       }

  1463. 

  1464. #if !defined(B0)

  1465.       register vector float rc0;

  1466. #endif

  1467.       SAVE_1x2_VSR(result, n, m);

  1468.     }

  1469.   }

  1470. 

  1471.   for (; m < M; m++) {

  1472.     for (n = 0; n < n16; n += 16) {

  1473.       vector float result = ((vector float){0.,0.,0.,0.});

  1474.       vector float result1 = ((vector float){0.,0.,0.,0.});

  1475.       vector float result2 = ((vector float){0.,0.,0.,0.});

  1476.       vector float result3 = ((vector float){0.,0.,0.,0.});

  1477. 

  1478.       register vector float ra0;

  1479.       register vector float rb0, rb1, rb2, rb3;

  1480. 

  1481.       for (k = 0; k < K; k++) {

  1482.         LOAD_A_1x1(m, k);

  1483.         LOAD_B_1x16(k, n);

  1484.         KERNEL_VMADD_4VSR(ra0, ra0, ra0, ra0, rb0, rb1, rb2, rb3);

  1485.       }

  1486. 

  1487.       SAVE_4x1_VSR(result, n+0, m);

  1488.       SAVE_4x1_VSR(result1, n+4, m);

  1489.       SAVE_4x1_VSR(result2, n+8, m);

  1490.       SAVE_4x1_VSR(result3, n+12, m);

  1491.     }

  1492. 

  1493.     for (; n < n8; n += 8) {

  1494.       vector float result = ((vector float){0.,0.,0.,0.});

  1495.       vector float result1 = ((vector float){0.,0.,0.,0.});

  1496. 

  1497.       register vector float ra0;

  1498.       register vector float rb0, rb1;

  1499. 

  1500.       for (k = 0; k < K; k++) {

  1501.         LOAD_A_1x1(m, k);

  1502.         LOAD_B_1x8(k, n);

  1503.         KERNEL_VMADD_2VSR(ra0, ra0, rb0, rb1);

  1504.       }

  1505. 

  1506.       SAVE_4x1_VSR(result, n+0, m);

  1507.       SAVE_4x1_VSR(result1, n+4, m);

  1508.     }

  1509. 

  1510.     for (; n < n4; n += 4) {

  1511.       vector float result = ((vector float){0.,0.,0.,0.});

  1512. 

  1513.       register vector float ra0;

  1514.       register vector float rb0;

  1515. 

  1516.       for (k = 0; k < K; k++) {

  1517.         LOAD_A_1x1(m, k);

  1518.         LOAD_B_1x4(k, n);

  1519.         KERNEL_VMADD_1VSR(ra0, rb0);

  1520.       }

  1521. 

  1522.       SAVE_4x1_VSR(result, n, m);

  1523.     }

  1524. 

  1525.     for (; n < n2; n += 2) {

  1526.       vector float result = ((vector float){0.,0.,0.,0.});

  1527. 

  1528.       register vector float ra0;

  1529.       register vector float rb0;

  1530. 

  1531.       for (k = 0; k < K; k++) {

  1532.         LOAD_A_1x1(m, k);

  1533.         LOAD_B_1x2(k, n);

  1534.         KERNEL_VMADD_1VSR(ra0, rb0);

  1535.       }

  1536. 

  1537.       SAVE_2x1_VSR(result, n, m);

  1538.     }

  1539. 

  1540.     for (; n < N; n++) {

  1541.       FLOAT result = 0.0f;

  1542. 

  1543.       for (k = 0; k < K; k++) {

  1544.         result += A[m*lda+k] * B[k*ldb+n];

  1545.       }

  1546.       result = result * alpha;

  1547. 

  1548. #if !defined(B0)

  1549.       C[n*ldc+m] = (C[n*ldc+m] * beta) + result;

  1550. #else

  1551.       C[n*ldc+m] = result;

  1552. #endif

  1553.     }

  1554.   }

  1555. 

  1556.   if (has_packing) free (packA);

  1557. 

  1558.   return 0;

  1559. }