OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: d96daa220d
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'd96daa220d'
${ noResults }
OpenBLAS/kernel/riscv64/gemmkernel_rvv_v1x8.c

602 lines
23 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2022, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. 

  30. #if !defined(DOUBLE)

  31. #define VSETVL(n)               __riscv_vsetvl_e32m2(n)

  32. #define FLOAT_V_T               vfloat32m2_t

  33. #define VLEV_FLOAT              __riscv_vle32_v_f32m2

  34. #define VSEV_FLOAT              __riscv_vse32_v_f32m2

  35. #define VFMVVF_FLOAT            __riscv_vfmv_v_f_f32m2

  36. #define VFMACCVF_FLOAT          __riscv_vfmacc_vf_f32m2

  37. #else

  38. #define VSETVL(n)               __riscv_vsetvl_e64m2(n)

  39. #define FLOAT_V_T               vfloat64m2_t

  40. #define VLEV_FLOAT              __riscv_vle64_v_f64m2

  41. #define VSEV_FLOAT              __riscv_vse64_v_f64m2

  42. #define VFMVVF_FLOAT            __riscv_vfmv_v_f_f64m2

  43. #define VFMACCVF_FLOAT          __riscv_vfmacc_vf_f64m2

  44. #endif

  45. 

  46. int CNAME(BLASLONG bm, BLASLONG bn, BLASLONG bk, FLOAT alpha, IFLOAT* ba, IFLOAT* bb, FLOAT* C, BLASLONG ldc

  47. #ifdef TRMMKERNEL

  48. 		,BLASLONG offset

  49. #endif

  50. 		)

  51. {

  52.     BLASLONG i,j,k;

  53.     FLOAT *C0,*C1,*C2,*C3,*C4,*C5,*C6,*C7;

  54.     IFLOAT *ptrba,*ptrbb;

  55. 

  56.     //fprintf(stderr, "%s, bm=%ld bn=%ld bk=%ld alpha=%f ldc=%ld\n", __FUNCTION__, bm, bn, bk, alpha, ldc); // Debug

  57. 

  58.     FLOAT_V_T va0, va1, va2, va3, va4, va5, va6, va7;

  59.     FLOAT_V_T vres0, vres1, vres2, vres3, vres4, vres5, vres6, vres7;

  60.     size_t vl;

  61. 

  62.     // N:8

  63.     for (j = bn/8; j > 0; j--) {

  64.         C0 = C;

  65.         C1 = C0 + ldc;

  66.         C2 = C1 + ldc;

  67.         C3 = C2 + ldc;

  68.         C4 = C3 + ldc;

  69.         C5 = C4 + ldc;

  70.         C6 = C5 + ldc;

  71.         C7 = C6 + ldc;

  72.         ptrba = ba;

  73. 

  74.         for (i = bm; i > 0; i -= vl) {

  75.             vl = VSETVL(i);

  76. 

  77.             ptrbb = bb;

  78. 

  79.             vres0 = VFMVVF_FLOAT(0.0, vl);

  80.             vres1 = VFMVVF_FLOAT(0.0, vl);

  81.             vres2 = VFMVVF_FLOAT(0.0, vl);

  82.             vres3 = VFMVVF_FLOAT(0.0, vl);

  83.             vres4 = VFMVVF_FLOAT(0.0, vl);

  84.             vres5 = VFMVVF_FLOAT(0.0, vl);

  85.             vres6 = VFMVVF_FLOAT(0.0, vl);

  86.             vres7 = VFMVVF_FLOAT(0.0, vl);

  87. #if 0

  88.             for (k = bk; k > 0; k--) {

  89.                 va0 = VLEV_FLOAT(ptrba, vl); 

  90. 

  91.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  92.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  93.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  94.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  95.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va0, vl);

  96.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va0, vl);

  97.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va0, vl);

  98.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va0, vl);

  99. 

  100.                 ptrba += vl;

  101.                 ptrbb += 8;

  102.             }

  103. #else

  104.             // Unroll K

  105.             for (k = bk/8; k > 0; k--) {

  106.                 va0 = VLEV_FLOAT(ptrba, vl);

  107.                 ptrba += vl;

  108.                 va1 = VLEV_FLOAT(ptrba, vl);

  109.                 ptrba += vl;

  110.  

  111.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  112.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  113.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  114.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  115.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va0, vl);

  116.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va0, vl);

  117.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va0, vl);

  118.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va0, vl);

  119.                 ptrbb += 8;

  120.   

  121.                 va2 = VLEV_FLOAT(ptrba, vl);

  122.                 ptrba += vl;

  123. 

  124.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va1, vl);

  125.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va1, vl);

  126.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va1, vl);

  127.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va1, vl);

  128.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va1, vl);

  129.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va1, vl);

  130.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va1, vl);

  131.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va1, vl);

  132.                 ptrbb += 8;

  133. 

  134.                 va3 = VLEV_FLOAT(ptrba, vl);

  135.                 ptrba += vl;

  136. 

  137.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va2, vl);

  138.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va2, vl);

  139.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va2, vl);

  140.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va2, vl);

  141.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va2, vl);

  142.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va2, vl);

  143.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va2, vl);

  144.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va2, vl);

  145.                 ptrbb += 8;

  146. 

  147.                 va4 = VLEV_FLOAT(ptrba, vl);

  148.                 ptrba += vl;

  149. 

  150.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va3, vl);

  151.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va3, vl);

  152.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va3, vl);

  153.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va3, vl);

  154.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va3, vl);

  155.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va3, vl);

  156.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va3, vl);

  157.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va3, vl);

  158.                 ptrbb += 8;

  159. 

  160.                 va5 = VLEV_FLOAT(ptrba, vl);

  161.                 ptrba += vl;

  162. 

  163.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va4, vl);

  164.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va4, vl);

  165.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va4, vl);

  166.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va4, vl);

  167.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va4, vl);

  168.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va4, vl);

  169.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va4, vl);

  170.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va4, vl);

  171.                 ptrbb += 8;

  172. 

  173.                 va6 = VLEV_FLOAT(ptrba, vl);

  174.                 ptrba += vl;

  175. 

  176.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va5, vl);

  177.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va5, vl);

  178.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va5, vl);

  179.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va5, vl);

  180.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va5, vl);

  181.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va5, vl);

  182.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va5, vl);

  183.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va5, vl);

  184.                 ptrbb += 8;

  185. 

  186.                 va7 = VLEV_FLOAT(ptrba, vl);

  187.                 ptrba += vl;

  188. 

  189.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va6, vl);

  190.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va6, vl);

  191.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va6, vl);

  192.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va6, vl);

  193.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va6, vl);

  194.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va6, vl);

  195.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va6, vl);

  196.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va6, vl);

  197.                 ptrbb += 8;

  198. 

  199.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va7, vl);

  200.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va7, vl);

  201.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va7, vl);

  202.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va7, vl);

  203.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va7, vl);

  204.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va7, vl);

  205.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va7, vl);

  206.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va7, vl);

  207.                 ptrbb += 8;

  208.             }

  209. 

  210.             // K remainder

  211.             for (k = bk&7; k > 0; k--) {

  212.                 va0 = VLEV_FLOAT(ptrba, vl);

  213. 

  214.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  215.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  216.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  217.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  218.                 vres4 = VFMACCVF_FLOAT(vres4, *(ptrbb + 4), va0, vl);

  219.                 vres5 = VFMACCVF_FLOAT(vres5, *(ptrbb + 5), va0, vl);

  220.                 vres6 = VFMACCVF_FLOAT(vres6, *(ptrbb + 6), va0, vl);

  221.                 vres7 = VFMACCVF_FLOAT(vres7, *(ptrbb + 7), va0, vl);

  222. 

  223.                 ptrbb += 8;

  224.                 ptrba += vl;

  225.             }

  226. #endif

  227.             va0 = VLEV_FLOAT(C0, vl);

  228.             va0 = VFMACCVF_FLOAT(va0, alpha, vres0, vl);

  229.             VSEV_FLOAT(C0, va0, vl);

  230. 

  231.             va1 = VLEV_FLOAT(C1, vl);

  232.             va1 = VFMACCVF_FLOAT(va1, alpha, vres1, vl);

  233.             VSEV_FLOAT(C1, va1, vl);

  234. 

  235.             va2 = VLEV_FLOAT(C2, vl);

  236.             va2 = VFMACCVF_FLOAT(va2, alpha, vres2, vl);

  237.             VSEV_FLOAT(C2, va2, vl);

  238. 

  239.             va3 = VLEV_FLOAT(C3, vl);

  240.             va3 = VFMACCVF_FLOAT(va3, alpha, vres3, vl);

  241.             VSEV_FLOAT(C3, va3, vl);

  242. 

  243.             va4 = VLEV_FLOAT(C4, vl);

  244.             va4 = VFMACCVF_FLOAT(va4, alpha, vres4, vl);

  245.             VSEV_FLOAT(C4, va4, vl);

  246. 

  247.             va5 = VLEV_FLOAT(C5, vl);

  248.             va5 = VFMACCVF_FLOAT(va5, alpha, vres5, vl);

  249.             VSEV_FLOAT(C5, va5, vl);

  250. 

  251.             va6 = VLEV_FLOAT(C6, vl);

  252.             va6 = VFMACCVF_FLOAT(va6, alpha, vres6, vl);

  253.             VSEV_FLOAT(C6, va6, vl);

  254. 

  255.             va7 = VLEV_FLOAT(C7, vl);

  256.             va7 = VFMACCVF_FLOAT(va7, alpha, vres7, vl);

  257.             VSEV_FLOAT(C7, va7, vl);

  258. 

  259.             C0 += vl;

  260.             C1 += vl;

  261.             C2 += vl;

  262.             C3 += vl;

  263.             C4 += vl;

  264.             C5 += vl;

  265.             C6 += vl;

  266.             C7 += vl;

  267.         }

  268. 

  269.         bb += (bk<<3);

  270.         C += (ldc<<3);

  271.     }

  272. 

  273.     // N:4

  274.     if (bn & 4) {

  275.         C0 = C;

  276.         C1 = C0 + ldc;

  277.         C2 = C1 + ldc;

  278.         C3 = C2 + ldc;

  279.         ptrba = ba;

  280. 

  281.         for (i = bm; i > 0; i -= vl) {

  282.             vl = VSETVL(i);

  283. 

  284.             ptrbb = bb;

  285. 

  286.             vres0 = VFMVVF_FLOAT(0.0, vl);

  287.             vres1 = VFMVVF_FLOAT(0.0, vl);

  288.             vres2 = VFMVVF_FLOAT(0.0, vl);

  289.             vres3 = VFMVVF_FLOAT(0.0, vl);

  290. 

  291. #if 0

  292.             for (k = bk; k > 0; k--) {

  293.                 va0 = VLEV_FLOAT(ptrba, vl); 

  294. 

  295.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  296.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  297.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  298.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  299. 

  300.                 ptrba += vl;

  301.                 ptrbb += 4;

  302.             }

  303. #else

  304.             // Unroll K

  305.             for (k = bk/8; k > 0; k--) {

  306.                 va0 = VLEV_FLOAT(ptrba, vl);

  307.                 ptrba += vl;

  308.                 va1 = VLEV_FLOAT(ptrba, vl);

  309.                 ptrba += vl;

  310. 

  311.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  312.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  313.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  314.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  315.                 ptrbb += 4;

  316.                 va2 = VLEV_FLOAT(ptrba, vl);

  317.                 ptrba += vl;

  318.   

  319.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va1, vl);

  320.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va1, vl);

  321.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va1, vl);

  322.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va1, vl);

  323.                 ptrbb += 4;

  324.                 va3 = VLEV_FLOAT(ptrba, vl);

  325.                 ptrba += vl;

  326. 

  327.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va2, vl);

  328.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va2, vl);

  329.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va2, vl);

  330.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va2, vl);

  331.                 ptrbb += 4;

  332.                 va4 = VLEV_FLOAT(ptrba, vl);

  333.                 ptrba += vl;

  334. 

  335.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va3, vl);

  336.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va3, vl);

  337.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va3, vl);

  338.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va3, vl);

  339.                 ptrbb += 4;

  340.                 va5 = VLEV_FLOAT(ptrba, vl);

  341.                 ptrba += vl;

  342. 

  343.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va4, vl);

  344.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va4, vl);

  345.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va4, vl);

  346.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va4, vl);

  347.                 ptrbb += 4;

  348.                 va6 = VLEV_FLOAT(ptrba, vl);

  349.                 ptrba += vl;

  350. 

  351.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va5, vl);

  352.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va5, vl);

  353.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va5, vl);

  354.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va5, vl);

  355.                 ptrbb += 4;

  356.                 va7 = VLEV_FLOAT(ptrba, vl);

  357.                 ptrba += vl;

  358. 

  359.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va6, vl);

  360.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va6, vl);

  361.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va6, vl);

  362.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va6, vl);

  363.                 ptrbb += 4;

  364. 

  365.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va7, vl);

  366.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va7, vl);

  367.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va7, vl);

  368.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va7, vl);

  369.                 ptrbb += 4;

  370.             }

  371. 

  372.             // K remainder

  373.             for (k = bk&7; k > 0; k--) {

  374.                 va0 = VLEV_FLOAT(ptrba, vl);

  375. 

  376.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  377.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  378.                 vres2 = VFMACCVF_FLOAT(vres2, *(ptrbb + 2), va0, vl);

  379.                 vres3 = VFMACCVF_FLOAT(vres3, *(ptrbb + 3), va0, vl);

  380. 

  381.                 ptrbb += 4;

  382.                 ptrba += vl;

  383.             }

  384. #endif

  385.             va0 = VLEV_FLOAT(C0, vl);

  386.             va0 = VFMACCVF_FLOAT(va0, alpha, vres0, vl);

  387.             VSEV_FLOAT(C0, va0, vl);

  388. 

  389.             va1 = VLEV_FLOAT(C1, vl);

  390.             va1 = VFMACCVF_FLOAT(va1, alpha, vres1, vl);

  391.             VSEV_FLOAT(C1, va1, vl);

  392. 

  393.             va2 = VLEV_FLOAT(C2, vl);

  394.             va2 = VFMACCVF_FLOAT(va2, alpha, vres2, vl);

  395.             VSEV_FLOAT(C2, va2, vl);

  396. 

  397.             va3 = VLEV_FLOAT(C3, vl);

  398.             va3 = VFMACCVF_FLOAT(va3, alpha, vres3, vl);

  399.             VSEV_FLOAT(C3, va3, vl);

  400. 

  401.             C0 += vl;

  402.             C1 += vl;

  403.             C2 += vl;

  404.             C3 += vl;

  405.         }

  406. 

  407.         bb += (bk<<2);

  408.         C += (ldc<<2);

  409.     }

  410. 

  411.     // N:2

  412.     if (bn & 2) {

  413.         C0 = C;

  414.         C1 = C0 + ldc;

  415.         ptrba = ba;

  416. 

  417.         for (i = bm; i > 0; i -= vl) {

  418.             vl = VSETVL(i);

  419. 

  420.             ptrbb = bb;

  421. 

  422.             vres0 = VFMVVF_FLOAT(0.0, vl);

  423.             vres1 = VFMVVF_FLOAT(0.0, vl);

  424. #if 0

  425.             for (k = bk; k > 0; k--) {

  426.                 va0 = VLEV_FLOAT(ptrba, vl); 

  427. 

  428.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  429.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  430. 

  431.                 ptrba += vl;

  432.                 ptrbb += 2;

  433.             }

  434. #else

  435.             // Unroll K

  436.             for (k = bk/8; k > 0; k--) {

  437.                 va0 = VLEV_FLOAT(ptrba, vl);

  438.                 ptrba += vl;

  439.                 va1 = VLEV_FLOAT(ptrba, vl);

  440.                 ptrba += vl;

  441. 

  442.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  443.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  444.                 ptrbb += 2;

  445.                 va2 = VLEV_FLOAT(ptrba, vl);

  446.                 ptrba += vl;

  447. 

  448.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va1, vl);

  449.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va1, vl);

  450.                 ptrbb += 2;

  451.                 va3 = VLEV_FLOAT(ptrba, vl);

  452.                 ptrba += vl;

  453. 

  454.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va2, vl);

  455.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va2, vl);

  456.                 ptrbb += 2;

  457.                 va4 = VLEV_FLOAT(ptrba, vl);

  458.                 ptrba += vl;

  459. 

  460.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va3, vl);

  461.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va3, vl);

  462.                 ptrbb += 2;

  463.                 va5 = VLEV_FLOAT(ptrba, vl);

  464.                 ptrba += vl;

  465. 

  466.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va4, vl);

  467.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va4, vl);

  468.                 ptrbb += 2;

  469.                 va6 = VLEV_FLOAT(ptrba, vl);

  470.                 ptrba += vl;

  471. 

  472.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va5, vl);

  473.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va5, vl);

  474.                 ptrbb += 2;

  475.                 va7 = VLEV_FLOAT(ptrba, vl);

  476.                 ptrba += vl; 

  477. 

  478.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va6, vl);

  479.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va6, vl);

  480.                 ptrbb += 2;

  481. 

  482.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va7, vl);

  483.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va7, vl);

  484.                 ptrbb += 2;

  485.             }

  486. 

  487.             // K remainder

  488.             for (k = bk&7; k > 0; k--) {

  489.                 va0 = VLEV_FLOAT(ptrba, vl);

  490. 

  491.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  492.                 vres1 = VFMACCVF_FLOAT(vres1, *(ptrbb + 1), va0, vl);

  493. 

  494.                 ptrbb += 2;

  495.                 ptrba += vl;

  496.             }

  497. #endif

  498.             va0 = VLEV_FLOAT(C0, vl);

  499.             va0 = VFMACCVF_FLOAT(va0, alpha, vres0, vl);

  500.             VSEV_FLOAT(C0, va0, vl);

  501. 

  502.             va1 = VLEV_FLOAT(C1, vl);

  503.             va1 = VFMACCVF_FLOAT(va1, alpha, vres1, vl);

  504.             VSEV_FLOAT(C1, va1, vl);

  505. 

  506.             C0 += vl;

  507.             C1 += vl;

  508.         }

  509. 

  510.         bb += (bk<<1);

  511.         C += (ldc<<1);

  512.     }

  513. 

  514.     // N:1

  515.     if (bn & 1) {

  516.         C0 = C;

  517.         ptrba = ba;

  518. 

  519.         for (i = bm; i > 0; i -= vl) {

  520.             vl = VSETVL(i);

  521. 

  522.             ptrbb = bb;

  523. 

  524.             vres0 = VFMVVF_FLOAT(0.0, vl);

  525. #if 0

  526.             for (k = bk; k > 0; k--) {

  527.                 va0 = VLEV_FLOAT(ptrba, vl); 

  528. 

  529.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  530.  

  531.                 ptrba += vl;

  532.                 ptrbb += 1;

  533.             }

  534. #else

  535.             // Unroll K

  536.             for (k = bk/8; k > 0; k--) {

  537.                 va0 = VLEV_FLOAT(ptrba, vl);

  538.                 ptrba += vl;

  539.                 va1 = VLEV_FLOAT(ptrba, vl);

  540.                 ptrba += vl;

  541. 

  542.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  543.                 ptrbb += 1;

  544.                 va2 = VLEV_FLOAT(ptrba, vl);

  545.                 ptrba += vl;

  546. 

  547.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va1, vl);

  548.                 ptrbb += 1;

  549.                 va3 = VLEV_FLOAT(ptrba, vl);

  550.                 ptrba += vl;

  551. 

  552.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va2, vl);

  553.                 ptrbb += 1;

  554.                 va4 = VLEV_FLOAT(ptrba, vl);

  555.                 ptrba += vl;

  556. 

  557.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va3, vl);

  558.                 ptrbb += 1;

  559.                 va5 = VLEV_FLOAT(ptrba, vl);

  560.                 ptrba += vl;

  561. 

  562.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va4, vl);

  563.                 ptrbb += 1;

  564.                 va6 = VLEV_FLOAT(ptrba, vl);

  565.                 ptrba += vl;

  566. 

  567.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va5, vl);

  568.                 ptrbb += 1;

  569.                 va7 = VLEV_FLOAT(ptrba, vl);

  570.                 ptrba += vl;

  571. 

  572.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va6, vl);

  573.                 ptrbb += 1;

  574. 

  575.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va7, vl);

  576.                 ptrbb += 1;

  577.             }

  578. 

  579.             // K remainder

  580.             for (k = bk&7; k > 0; k--) {

  581.                 va0 = VLEV_FLOAT(ptrba, vl);

  582. 

  583.                 vres0 = VFMACCVF_FLOAT(vres0, *(ptrbb + 0), va0, vl);

  584. 

  585.                 ptrbb += 1;

  586.                 ptrba += vl;

  587.             }

  588. #endif

  589.             va0 = VLEV_FLOAT(C0, vl);

  590.             va0 = VFMACCVF_FLOAT(va0, alpha, vres0, vl);

  591.             VSEV_FLOAT(C0, va0, vl);

  592.   

  593.             C0 += vl;

  594.         }

  595. 

  596.         bb += (bk);

  597.         C += (ldc);

  598.     }

  599. 

  600.     return 0;

  601. }