OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 2406958629
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2406958629'
${ noResults }
OpenBLAS/kernel/riscv64/ctrmm_kernel_8x8_zvl256b.c

2008 lines
86 kB
Raw Blame History 

  1. /*

  2. 

  3. AUTOGENERATED KERNEL

  4. Settings:

  5.  LMUL=1

  6.  M=8

  7.  M_tail_scalar_from=1

  8.  N=8

  9.  __riscv_='__riscv_'

  10.  complex=True

  11.  conjugate=False

  12.  cpu='zvl256b'

  13.  force_acc_double=False

  14.  index_type='BLASLONG'

  15.  op='trmm'

  16.  param_precision='float'

  17.  reg_width_bits=256

  18.  tail_policy=''

  19.  trace=False

  20. 

  21. Derived:

  22.  ELEN_ACC=32

  23.  ELEN_PARAM=32

  24.  LMUL_ACC=1

  25.  VFMACC='__riscv_vfmacc_vf_f32m1'

  26.  VFMUL='__riscv_vfmul_vf_f32m1'

  27.  VLEV='__riscv_vle32_v_f32m1'

  28.  VLSEV='__riscv_vlse32_v_f32m1'

  29.  VMACC_TO_ACC='__riscv_vfmacc_vf_f32m1'

  30.  VMUL_TO_ACC='__riscv_vfmul_vf_f32m1'

  31.  VSETVL='__riscv_vsetvl_e32m1'

  32.  VSEV='__riscv_vse32_v_f32m1'

  33.  VSSEV='__riscv_vsse32_v_f32m1'

  34.  acc_vector_t='vfloat32m1_t'

  35.  output='ctrmm_kernel_8x8_zvl256b.c'

  36.  param_scalar_t='float'

  37.  param_vector_t='vfloat32m1_t'

  38. 

  39. */

  40. 

  41. #include "common.h"

  42. 

  43. 

  44. 

  45. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  46.     #define S0  1

  47.     #define S1 -1

  48.     #define S2  1

  49.     #define S3  1

  50.     #define VFMACC_RR __riscv_vfmsac

  51.     #define VFMACC_RI __riscv_vfmacc

  52. #endif

  53. #if   defined(NR) || defined(NC) || defined(TR) || defined(TC)

  54.     #define S0  1

  55.     #define S1  1

  56.     #define S2  1

  57.     #define S3 -1

  58.     #define VFMACC_RR __riscv_vfmacc

  59.     #define VFMACC_RI __riscv_vfmsac

  60. #endif

  61. #if   defined(RN) || defined(RT) || defined(CN) || defined(CT)

  62.     #define S0  1

  63.     #define S1  1

  64.     #define S2 -1

  65.     #define S3  1

  66.     #define VFMACC_RR __riscv_vfmacc

  67.     #define VFMACC_RI __riscv_vfnmsac

  68. #endif

  69. #if   defined(RR) || defined(RC) || defined(CR) || defined(CC)

  70.     #define S0  1

  71.     #define S1 -1

  72.     #define S2 -1

  73.     #define S3 -1

  74.     #define VFMACC_RR __riscv_vfmsac

  75.     #define VFMACC_RI __riscv_vfnmacc

  76. #endif

  77. 

  78. 

  79. #if defined(LEFT) != defined(TRANSA)

  80.     #define BACKWARDS

  81. #endif

  82. 

  83. int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alphar, FLOAT alphai, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc, BLASLONG offset)

  84. 

  85. {

  86.     BLASLONG gvl = 0;

  87.     BLASLONG m_top = 0;

  88.     BLASLONG n_top = 0;

  89. 

  90. 

  91.     // -- MAIN PASS

  92. 

  93.     for (BLASLONG j=0; j<N/8; j+=1) {

  94.         m_top = 0;

  95.         BLASLONG gvl = __riscv_vsetvl_e32m1(8);

  96. 

  97. 

  98.         for (BLASLONG i=0; i<M/8; i+=1) {

  99.             BLASLONG ai=m_top*K*2;

  100.             BLASLONG bi=n_top*K*2;

  101.             BLASLONG pass_K = K;

  102.             #ifdef LEFT

  103.                 BLASLONG off = offset + m_top;

  104.             #else

  105.                 BLASLONG off = -offset + n_top;

  106.             #endif

  107.             #ifdef BACKWARDS

  108.                 ai += off*8*2;

  109.                 bi += off*8*2;

  110.                 pass_K -= off;

  111.             #else

  112.                 #ifdef LEFT

  113.                     pass_K = off + 8;

  114.                 #else

  115.                     pass_K = off + 8;

  116.                 #endif

  117.             #endif

  118.             float B0r = B[bi+0*2+0];

  119.             float B0i = B[bi+0*2+1];

  120.             float B1r = B[bi+1*2+0];

  121.             float B1i = B[bi+1*2+1];

  122.             float B2r = B[bi+2*2+0];

  123.             float B2i = B[bi+2*2+1];

  124.             float B3r = B[bi+3*2+0];

  125.             float B3i = B[bi+3*2+1];

  126.             float B4r = B[bi+4*2+0];

  127.             float B4i = B[bi+4*2+1];

  128.             float B5r = B[bi+5*2+0];

  129.             float B5i = B[bi+5*2+1];

  130.             float B6r = B[bi+6*2+0];

  131.             float B6i = B[bi+6*2+1];

  132.             float B7r = B[bi+7*2+0];

  133.             float B7i = B[bi+7*2+1];

  134.             bi += 8*2;

  135. 

  136.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  137.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  138.             ai += 8*2;

  139. 

  140.             // 2 vector regs to hold A array contents, 16 regs to hold values accumulated over k

  141.             // leaving 14 vector registers for temporaries

  142.             // performing 4 operations between reuses of temporaries

  143.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  144.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  145.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  146.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  147.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  148.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  149.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  150.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  151.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  152.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  153.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  154.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  155.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  156.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  157.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  158.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  159.             vfloat32m1_t ACC0r = tmp0r;

  160.             vfloat32m1_t ACC0i = tmp0i;

  161.             vfloat32m1_t ACC1r = tmp1r;

  162.             vfloat32m1_t ACC1i = tmp1i;

  163.             vfloat32m1_t ACC2r = tmp2r;

  164.             vfloat32m1_t ACC2i = tmp2i;

  165.             vfloat32m1_t ACC3r = tmp3r;

  166.             vfloat32m1_t ACC3i = tmp3i;

  167.             tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  168.             tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  169.             tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  170.             tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  171.             tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  172.             tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  173.             tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  174.             tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  175.             tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  176.             tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  177.             tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  178.             tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  179.             tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  180.             tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  181.             tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  182.             tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  183.             vfloat32m1_t ACC4r = tmp0r;

  184.             vfloat32m1_t ACC4i = tmp0i;

  185.             vfloat32m1_t ACC5r = tmp1r;

  186.             vfloat32m1_t ACC5i = tmp1i;

  187.             vfloat32m1_t ACC6r = tmp2r;

  188.             vfloat32m1_t ACC6i = tmp2i;

  189.             vfloat32m1_t ACC7r = tmp3r;

  190.             vfloat32m1_t ACC7i = tmp3i;

  191. 

  192.             for(BLASLONG k=1; k<pass_K; k++) {

  193.                 B0r = B[bi+0*2+0];

  194.                 B0i = B[bi+0*2+1];

  195.                 B1r = B[bi+1*2+0];

  196.                 B1i = B[bi+1*2+1];

  197.                 B2r = B[bi+2*2+0];

  198.                 B2i = B[bi+2*2+1];

  199.                 B3r = B[bi+3*2+0];

  200.                 B3i = B[bi+3*2+1];

  201.                 B4r = B[bi+4*2+0];

  202.                 B4i = B[bi+4*2+1];

  203.                 B5r = B[bi+5*2+0];

  204.                 B5i = B[bi+5*2+1];

  205.                 B6r = B[bi+6*2+0];

  206.                 B6i = B[bi+6*2+1];

  207.                 B7r = B[bi+7*2+0];

  208.                 B7i = B[bi+7*2+1];

  209.                 bi += 8*2;

  210. 

  211.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  212.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  213.                 ai += 8*2;

  214. 

  215.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  216.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  217.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  218.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  219.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  220.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  221.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  222.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  223.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  224.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  225.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  226.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  227.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  228.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  229.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  230.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  231.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  232.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  233.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  234.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  235.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  236.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  237.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  238.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  239.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  240.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  241.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  242.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  243.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  244.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  245.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  246.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  247.                 tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  248.                 tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  249.                 tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  250.                 tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  251.                 tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  252.                 tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  253.                 tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  254.                 tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  255.                 ACC4r = __riscv_vfadd( ACC4r, tmp0r, gvl);

  256.                 ACC4i = __riscv_vfadd( ACC4i, tmp0i, gvl);

  257.                 ACC5r = __riscv_vfadd( ACC5r, tmp1r, gvl);

  258.                 ACC5i = __riscv_vfadd( ACC5i, tmp1i, gvl);

  259.                 ACC6r = __riscv_vfadd( ACC6r, tmp2r, gvl);

  260.                 ACC6i = __riscv_vfadd( ACC6i, tmp2i, gvl);

  261.                 ACC7r = __riscv_vfadd( ACC7r, tmp3r, gvl);

  262.                 ACC7i = __riscv_vfadd( ACC7i, tmp3i, gvl);

  263.             }

  264. 

  265. 

  266.             BLASLONG ci=n_top*ldc+m_top;

  267. 

  268.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  269.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  270.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  271.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  272.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  273.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  274.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  275.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  276.             vfloat32m1_t C4r = __riscv_vfmul( ACC4r, alphar, gvl );

  277.             vfloat32m1_t C4i = __riscv_vfmul( ACC4i, alphar, gvl );

  278.             vfloat32m1_t C5r = __riscv_vfmul( ACC5r, alphar, gvl );

  279.             vfloat32m1_t C5i = __riscv_vfmul( ACC5i, alphar, gvl );

  280.             vfloat32m1_t C6r = __riscv_vfmul( ACC6r, alphar, gvl );

  281.             vfloat32m1_t C6i = __riscv_vfmul( ACC6i, alphar, gvl );

  282.             vfloat32m1_t C7r = __riscv_vfmul( ACC7r, alphar, gvl );

  283.             vfloat32m1_t C7i = __riscv_vfmul( ACC7i, alphar, gvl );

  284.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  285.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  286.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  287.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  288.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  289.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  290.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  291.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  292.             C4r = __riscv_vfnmsac( C4r, alphai, ACC4i, gvl );

  293.             C4i = __riscv_vfmacc ( C4i, alphai, ACC4r, gvl );

  294.             C5r = __riscv_vfnmsac( C5r, alphai, ACC5i, gvl );

  295.             C5i = __riscv_vfmacc ( C5i, alphai, ACC5r, gvl );

  296.             C6r = __riscv_vfnmsac( C6r, alphai, ACC6i, gvl );

  297.             C6i = __riscv_vfmacc ( C6i, alphai, ACC6r, gvl );

  298.             C7r = __riscv_vfnmsac( C7r, alphai, ACC7i, gvl );

  299.             C7i = __riscv_vfmacc ( C7i, alphai, ACC7r, gvl );

  300.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  301.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  302.             ci += ldc-gvl*0;

  303.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  304.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  305.             ci += ldc-gvl*0;

  306.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  307.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  308.             ci += ldc-gvl*0;

  309.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  310.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  311.             ci += ldc-gvl*0;

  312.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C4r, gvl);

  313.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C4i, gvl);

  314.             ci += ldc-gvl*0;

  315.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C5r, gvl);

  316.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C5i, gvl);

  317.             ci += ldc-gvl*0;

  318.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C6r, gvl);

  319.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C6i, gvl);

  320.             ci += ldc-gvl*0;

  321.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C7r, gvl);

  322.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C7i, gvl);

  323.             

  324.             m_top += 8;

  325.         }

  326. 

  327. 

  328. 

  329.         // -- tails for main pass

  330. 

  331.         if( M & 4 ) {

  332.             gvl = __riscv_vsetvl_e32m1(4);

  333. 

  334.             BLASLONG ai=m_top*K*2;

  335.             BLASLONG bi=n_top*K*2;

  336.             BLASLONG pass_K = K;

  337.             #ifdef LEFT

  338.                 BLASLONG off = offset + m_top;

  339.             #else

  340.                 BLASLONG off = -offset + n_top;

  341.             #endif

  342.             #ifdef BACKWARDS

  343.                 ai += off*4*2;

  344.                 bi += off*8*2;

  345.                 pass_K -= off;

  346.             #else

  347.                 #ifdef LEFT

  348.                     pass_K = off + 4;

  349.                 #else

  350.                     pass_K = off + 8;

  351.                 #endif

  352.             #endif

  353.             float B0r = B[bi+0*2+0];

  354.             float B0i = B[bi+0*2+1];

  355.             float B1r = B[bi+1*2+0];

  356.             float B1i = B[bi+1*2+1];

  357.             float B2r = B[bi+2*2+0];

  358.             float B2i = B[bi+2*2+1];

  359.             float B3r = B[bi+3*2+0];

  360.             float B3i = B[bi+3*2+1];

  361.             float B4r = B[bi+4*2+0];

  362.             float B4i = B[bi+4*2+1];

  363.             float B5r = B[bi+5*2+0];

  364.             float B5i = B[bi+5*2+1];

  365.             float B6r = B[bi+6*2+0];

  366.             float B6i = B[bi+6*2+1];

  367.             float B7r = B[bi+7*2+0];

  368.             float B7i = B[bi+7*2+1];

  369.             bi += 8*2;

  370. 

  371.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  372.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  373.             ai += 4*2;

  374. 

  375.             // 2 vector regs to hold A array contents, 16 regs to hold values accumulated over k

  376.             // leaving 14 vector registers for temporaries

  377.             // performing 4 operations between reuses of temporaries

  378.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  379.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  380.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  381.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  382.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  383.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  384.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  385.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  386.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  387.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  388.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  389.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  390.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  391.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  392.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  393.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  394.             vfloat32m1_t ACC0r = tmp0r;

  395.             vfloat32m1_t ACC0i = tmp0i;

  396.             vfloat32m1_t ACC1r = tmp1r;

  397.             vfloat32m1_t ACC1i = tmp1i;

  398.             vfloat32m1_t ACC2r = tmp2r;

  399.             vfloat32m1_t ACC2i = tmp2i;

  400.             vfloat32m1_t ACC3r = tmp3r;

  401.             vfloat32m1_t ACC3i = tmp3i;

  402.             tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  403.             tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  404.             tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  405.             tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  406.             tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  407.             tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  408.             tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  409.             tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  410.             tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  411.             tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  412.             tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  413.             tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  414.             tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  415.             tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  416.             tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  417.             tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  418.             vfloat32m1_t ACC4r = tmp0r;

  419.             vfloat32m1_t ACC4i = tmp0i;

  420.             vfloat32m1_t ACC5r = tmp1r;

  421.             vfloat32m1_t ACC5i = tmp1i;

  422.             vfloat32m1_t ACC6r = tmp2r;

  423.             vfloat32m1_t ACC6i = tmp2i;

  424.             vfloat32m1_t ACC7r = tmp3r;

  425.             vfloat32m1_t ACC7i = tmp3i;

  426. 

  427.             for(BLASLONG k=1; k<pass_K; k++) {

  428.                 B0r = B[bi+0*2+0];

  429.                 B0i = B[bi+0*2+1];

  430.                 B1r = B[bi+1*2+0];

  431.                 B1i = B[bi+1*2+1];

  432.                 B2r = B[bi+2*2+0];

  433.                 B2i = B[bi+2*2+1];

  434.                 B3r = B[bi+3*2+0];

  435.                 B3i = B[bi+3*2+1];

  436.                 B4r = B[bi+4*2+0];

  437.                 B4i = B[bi+4*2+1];

  438.                 B5r = B[bi+5*2+0];

  439.                 B5i = B[bi+5*2+1];

  440.                 B6r = B[bi+6*2+0];

  441.                 B6i = B[bi+6*2+1];

  442.                 B7r = B[bi+7*2+0];

  443.                 B7i = B[bi+7*2+1];

  444.                 bi += 8*2;

  445. 

  446.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  447.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  448.                 ai += 4*2;

  449. 

  450.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  451.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  452.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  453.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  454.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  455.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  456.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  457.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  458.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  459.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  460.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  461.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  462.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  463.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  464.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  465.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  466.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  467.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  468.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  469.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  470.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  471.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  472.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  473.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  474.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  475.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  476.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  477.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  478.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  479.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  480.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  481.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  482.                 tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  483.                 tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  484.                 tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  485.                 tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  486.                 tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  487.                 tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  488.                 tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  489.                 tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  490.                 ACC4r = __riscv_vfadd( ACC4r, tmp0r, gvl);

  491.                 ACC4i = __riscv_vfadd( ACC4i, tmp0i, gvl);

  492.                 ACC5r = __riscv_vfadd( ACC5r, tmp1r, gvl);

  493.                 ACC5i = __riscv_vfadd( ACC5i, tmp1i, gvl);

  494.                 ACC6r = __riscv_vfadd( ACC6r, tmp2r, gvl);

  495.                 ACC6i = __riscv_vfadd( ACC6i, tmp2i, gvl);

  496.                 ACC7r = __riscv_vfadd( ACC7r, tmp3r, gvl);

  497.                 ACC7i = __riscv_vfadd( ACC7i, tmp3i, gvl);

  498.             }

  499. 

  500. 

  501.             BLASLONG ci=n_top*ldc+m_top;

  502. 

  503.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  504.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  505.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  506.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  507.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  508.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  509.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  510.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  511.             vfloat32m1_t C4r = __riscv_vfmul( ACC4r, alphar, gvl );

  512.             vfloat32m1_t C4i = __riscv_vfmul( ACC4i, alphar, gvl );

  513.             vfloat32m1_t C5r = __riscv_vfmul( ACC5r, alphar, gvl );

  514.             vfloat32m1_t C5i = __riscv_vfmul( ACC5i, alphar, gvl );

  515.             vfloat32m1_t C6r = __riscv_vfmul( ACC6r, alphar, gvl );

  516.             vfloat32m1_t C6i = __riscv_vfmul( ACC6i, alphar, gvl );

  517.             vfloat32m1_t C7r = __riscv_vfmul( ACC7r, alphar, gvl );

  518.             vfloat32m1_t C7i = __riscv_vfmul( ACC7i, alphar, gvl );

  519.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  520.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  521.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  522.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  523.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  524.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  525.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  526.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  527.             C4r = __riscv_vfnmsac( C4r, alphai, ACC4i, gvl );

  528.             C4i = __riscv_vfmacc ( C4i, alphai, ACC4r, gvl );

  529.             C5r = __riscv_vfnmsac( C5r, alphai, ACC5i, gvl );

  530.             C5i = __riscv_vfmacc ( C5i, alphai, ACC5r, gvl );

  531.             C6r = __riscv_vfnmsac( C6r, alphai, ACC6i, gvl );

  532.             C6i = __riscv_vfmacc ( C6i, alphai, ACC6r, gvl );

  533.             C7r = __riscv_vfnmsac( C7r, alphai, ACC7i, gvl );

  534.             C7i = __riscv_vfmacc ( C7i, alphai, ACC7r, gvl );

  535.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  536.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  537.             ci += ldc-gvl*0;

  538.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  539.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  540.             ci += ldc-gvl*0;

  541.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  542.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  543.             ci += ldc-gvl*0;

  544.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  545.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  546.             ci += ldc-gvl*0;

  547.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C4r, gvl);

  548.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C4i, gvl);

  549.             ci += ldc-gvl*0;

  550.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C5r, gvl);

  551.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C5i, gvl);

  552.             ci += ldc-gvl*0;

  553.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C6r, gvl);

  554.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C6i, gvl);

  555.             ci += ldc-gvl*0;

  556.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C7r, gvl);

  557.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C7i, gvl);

  558.             

  559.             m_top += 4;

  560.         }

  561. 

  562. 

  563.         if( M & 2 ) {

  564.             gvl = __riscv_vsetvl_e32m1(2);

  565. 

  566.             BLASLONG ai=m_top*K*2;

  567.             BLASLONG bi=n_top*K*2;

  568.             BLASLONG pass_K = K;

  569.             #ifdef LEFT

  570.                 BLASLONG off = offset + m_top;

  571.             #else

  572.                 BLASLONG off = -offset + n_top;

  573.             #endif

  574.             #ifdef BACKWARDS

  575.                 ai += off*2*2;

  576.                 bi += off*8*2;

  577.                 pass_K -= off;

  578.             #else

  579.                 #ifdef LEFT

  580.                     pass_K = off + 2;

  581.                 #else

  582.                     pass_K = off + 8;

  583.                 #endif

  584.             #endif

  585.             float B0r = B[bi+0*2+0];

  586.             float B0i = B[bi+0*2+1];

  587.             float B1r = B[bi+1*2+0];

  588.             float B1i = B[bi+1*2+1];

  589.             float B2r = B[bi+2*2+0];

  590.             float B2i = B[bi+2*2+1];

  591.             float B3r = B[bi+3*2+0];

  592.             float B3i = B[bi+3*2+1];

  593.             float B4r = B[bi+4*2+0];

  594.             float B4i = B[bi+4*2+1];

  595.             float B5r = B[bi+5*2+0];

  596.             float B5i = B[bi+5*2+1];

  597.             float B6r = B[bi+6*2+0];

  598.             float B6i = B[bi+6*2+1];

  599.             float B7r = B[bi+7*2+0];

  600.             float B7i = B[bi+7*2+1];

  601.             bi += 8*2;

  602. 

  603.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  604.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  605.             ai += 2*2;

  606. 

  607.             // 2 vector regs to hold A array contents, 16 regs to hold values accumulated over k

  608.             // leaving 14 vector registers for temporaries

  609.             // performing 4 operations between reuses of temporaries

  610.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  611.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  612.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  613.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  614.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  615.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  616.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  617.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  618.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  619.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  620.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  621.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  622.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  623.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  624.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  625.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  626.             vfloat32m1_t ACC0r = tmp0r;

  627.             vfloat32m1_t ACC0i = tmp0i;

  628.             vfloat32m1_t ACC1r = tmp1r;

  629.             vfloat32m1_t ACC1i = tmp1i;

  630.             vfloat32m1_t ACC2r = tmp2r;

  631.             vfloat32m1_t ACC2i = tmp2i;

  632.             vfloat32m1_t ACC3r = tmp3r;

  633.             vfloat32m1_t ACC3i = tmp3i;

  634.             tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  635.             tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  636.             tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  637.             tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  638.             tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  639.             tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  640.             tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  641.             tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  642.             tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  643.             tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  644.             tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  645.             tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  646.             tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  647.             tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  648.             tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  649.             tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  650.             vfloat32m1_t ACC4r = tmp0r;

  651.             vfloat32m1_t ACC4i = tmp0i;

  652.             vfloat32m1_t ACC5r = tmp1r;

  653.             vfloat32m1_t ACC5i = tmp1i;

  654.             vfloat32m1_t ACC6r = tmp2r;

  655.             vfloat32m1_t ACC6i = tmp2i;

  656.             vfloat32m1_t ACC7r = tmp3r;

  657.             vfloat32m1_t ACC7i = tmp3i;

  658. 

  659.             for(BLASLONG k=1; k<pass_K; k++) {

  660.                 B0r = B[bi+0*2+0];

  661.                 B0i = B[bi+0*2+1];

  662.                 B1r = B[bi+1*2+0];

  663.                 B1i = B[bi+1*2+1];

  664.                 B2r = B[bi+2*2+0];

  665.                 B2i = B[bi+2*2+1];

  666.                 B3r = B[bi+3*2+0];

  667.                 B3i = B[bi+3*2+1];

  668.                 B4r = B[bi+4*2+0];

  669.                 B4i = B[bi+4*2+1];

  670.                 B5r = B[bi+5*2+0];

  671.                 B5i = B[bi+5*2+1];

  672.                 B6r = B[bi+6*2+0];

  673.                 B6i = B[bi+6*2+1];

  674.                 B7r = B[bi+7*2+0];

  675.                 B7i = B[bi+7*2+1];

  676.                 bi += 8*2;

  677. 

  678.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  679.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  680.                 ai += 2*2;

  681. 

  682.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  683.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  684.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  685.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  686.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  687.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  688.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  689.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  690.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  691.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  692.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  693.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  694.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  695.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  696.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  697.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  698.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  699.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  700.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  701.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  702.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  703.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  704.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  705.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  706.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B4i, gvl);

  707.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B4i, gvl);

  708.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B5i, gvl);

  709.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B5i, gvl);

  710.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B6i, gvl);

  711.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B6i, gvl);

  712.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B7i, gvl);

  713.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B7i, gvl);

  714.                 tmp0r = VFMACC_RR( tmp0r, B4r, A0r, gvl);

  715.                 tmp0i = VFMACC_RI( tmp0i, B4r, A0i, gvl);

  716.                 tmp1r = VFMACC_RR( tmp1r, B5r, A0r, gvl);

  717.                 tmp1i = VFMACC_RI( tmp1i, B5r, A0i, gvl);

  718.                 tmp2r = VFMACC_RR( tmp2r, B6r, A0r, gvl);

  719.                 tmp2i = VFMACC_RI( tmp2i, B6r, A0i, gvl);

  720.                 tmp3r = VFMACC_RR( tmp3r, B7r, A0r, gvl);

  721.                 tmp3i = VFMACC_RI( tmp3i, B7r, A0i, gvl);

  722.                 ACC4r = __riscv_vfadd( ACC4r, tmp0r, gvl);

  723.                 ACC4i = __riscv_vfadd( ACC4i, tmp0i, gvl);

  724.                 ACC5r = __riscv_vfadd( ACC5r, tmp1r, gvl);

  725.                 ACC5i = __riscv_vfadd( ACC5i, tmp1i, gvl);

  726.                 ACC6r = __riscv_vfadd( ACC6r, tmp2r, gvl);

  727.                 ACC6i = __riscv_vfadd( ACC6i, tmp2i, gvl);

  728.                 ACC7r = __riscv_vfadd( ACC7r, tmp3r, gvl);

  729.                 ACC7i = __riscv_vfadd( ACC7i, tmp3i, gvl);

  730.             }

  731. 

  732. 

  733.             BLASLONG ci=n_top*ldc+m_top;

  734. 

  735.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  736.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  737.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  738.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  739.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  740.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  741.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  742.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  743.             vfloat32m1_t C4r = __riscv_vfmul( ACC4r, alphar, gvl );

  744.             vfloat32m1_t C4i = __riscv_vfmul( ACC4i, alphar, gvl );

  745.             vfloat32m1_t C5r = __riscv_vfmul( ACC5r, alphar, gvl );

  746.             vfloat32m1_t C5i = __riscv_vfmul( ACC5i, alphar, gvl );

  747.             vfloat32m1_t C6r = __riscv_vfmul( ACC6r, alphar, gvl );

  748.             vfloat32m1_t C6i = __riscv_vfmul( ACC6i, alphar, gvl );

  749.             vfloat32m1_t C7r = __riscv_vfmul( ACC7r, alphar, gvl );

  750.             vfloat32m1_t C7i = __riscv_vfmul( ACC7i, alphar, gvl );

  751.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  752.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  753.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  754.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  755.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  756.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  757.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  758.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  759.             C4r = __riscv_vfnmsac( C4r, alphai, ACC4i, gvl );

  760.             C4i = __riscv_vfmacc ( C4i, alphai, ACC4r, gvl );

  761.             C5r = __riscv_vfnmsac( C5r, alphai, ACC5i, gvl );

  762.             C5i = __riscv_vfmacc ( C5i, alphai, ACC5r, gvl );

  763.             C6r = __riscv_vfnmsac( C6r, alphai, ACC6i, gvl );

  764.             C6i = __riscv_vfmacc ( C6i, alphai, ACC6r, gvl );

  765.             C7r = __riscv_vfnmsac( C7r, alphai, ACC7i, gvl );

  766.             C7i = __riscv_vfmacc ( C7i, alphai, ACC7r, gvl );

  767.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  768.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  769.             ci += ldc-gvl*0;

  770.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  771.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  772.             ci += ldc-gvl*0;

  773.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  774.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  775.             ci += ldc-gvl*0;

  776.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  777.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  778.             ci += ldc-gvl*0;

  779.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C4r, gvl);

  780.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C4i, gvl);

  781.             ci += ldc-gvl*0;

  782.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C5r, gvl);

  783.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C5i, gvl);

  784.             ci += ldc-gvl*0;

  785.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C6r, gvl);

  786.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C6i, gvl);

  787.             ci += ldc-gvl*0;

  788.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C7r, gvl);

  789.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C7i, gvl);

  790.             

  791.             m_top += 2;

  792.         }

  793. 

  794. 

  795.         if( M & 1 ) {

  796.             float result0 = 0;

  797.             float result1 = 0;

  798.             float result2 = 0;

  799.             float result3 = 0;

  800.             float result4 = 0;

  801.             float result5 = 0;

  802.             float result6 = 0;

  803.             float result7 = 0;

  804.             float result8 = 0;

  805.             float result9 = 0;

  806.             float result10 = 0;

  807.             float result11 = 0;

  808.             float result12 = 0;

  809.             float result13 = 0;

  810.             float result14 = 0;

  811.             float result15 = 0;

  812.             BLASLONG ai=m_top*K*2;

  813.             BLASLONG bi=n_top*K*2;

  814.             BLASLONG pass_K = K;

  815.             #ifdef LEFT

  816.                 BLASLONG off = offset + m_top;

  817.             #else

  818.                 BLASLONG off = -offset + n_top;

  819.             #endif

  820.             #ifdef BACKWARDS

  821.                 ai += off*1*2;

  822.                 bi += off*8*2;

  823.                 pass_K -= off;

  824.             #else

  825.                 #ifdef LEFT

  826.                     pass_K = off + 1;

  827.                 #else

  828.                     pass_K = off + 8;

  829.                 #endif

  830.             #endif

  831. 

  832.             for(BLASLONG k=0; k<pass_K; k++) {

  833.                 result0+=S0*A[ai+0+0]*B[bi+0+0] + S1*A[ai+0+1]*B[bi+0+1];

  834.                 result1+=S2*A[ai+0+1]*B[bi+0+0] + S3*A[ai+0+0]*B[bi+0+1];

  835.                 result2+=S0*A[ai+0+0]*B[bi+2+0] + S1*A[ai+0+1]*B[bi+2+1];

  836.                 result3+=S2*A[ai+0+1]*B[bi+2+0] + S3*A[ai+0+0]*B[bi+2+1];

  837.                 result4+=S0*A[ai+0+0]*B[bi+4+0] + S1*A[ai+0+1]*B[bi+4+1];

  838.                 result5+=S2*A[ai+0+1]*B[bi+4+0] + S3*A[ai+0+0]*B[bi+4+1];

  839.                 result6+=S0*A[ai+0+0]*B[bi+6+0] + S1*A[ai+0+1]*B[bi+6+1];

  840.                 result7+=S2*A[ai+0+1]*B[bi+6+0] + S3*A[ai+0+0]*B[bi+6+1];

  841.                 result8+=S0*A[ai+0+0]*B[bi+8+0] + S1*A[ai+0+1]*B[bi+8+1];

  842.                 result9+=S2*A[ai+0+1]*B[bi+8+0] + S3*A[ai+0+0]*B[bi+8+1];

  843.                 result10+=S0*A[ai+0+0]*B[bi+10+0] + S1*A[ai+0+1]*B[bi+10+1];

  844.                 result11+=S2*A[ai+0+1]*B[bi+10+0] + S3*A[ai+0+0]*B[bi+10+1];

  845.                 result12+=S0*A[ai+0+0]*B[bi+12+0] + S1*A[ai+0+1]*B[bi+12+1];

  846.                 result13+=S2*A[ai+0+1]*B[bi+12+0] + S3*A[ai+0+0]*B[bi+12+1];

  847.                 result14+=S0*A[ai+0+0]*B[bi+14+0] + S1*A[ai+0+1]*B[bi+14+1];

  848.                 result15+=S2*A[ai+0+1]*B[bi+14+0] + S3*A[ai+0+0]*B[bi+14+1];

  849.                 ai+=1*2;

  850.                 bi+=8*2;

  851.             }

  852. 

  853.             BLASLONG ci=n_top*ldc+m_top;

  854.             float Cr, Ci;

  855.             Cr = result0*alphar;

  856.             Ci = result1*alphar;

  857.             Cr -= result1*alphai;

  858.             Ci += result0*alphai;

  859.             C[(ci+0*ldc+0)*2+0] = Cr;

  860.             C[(ci+0*ldc+0)*2+1] = Ci;

  861.             Cr = result2*alphar;

  862.             Ci = result3*alphar;

  863.             Cr -= result3*alphai;

  864.             Ci += result2*alphai;

  865.             C[(ci+1*ldc+0)*2+0] = Cr;

  866.             C[(ci+1*ldc+0)*2+1] = Ci;

  867.             Cr = result4*alphar;

  868.             Ci = result5*alphar;

  869.             Cr -= result5*alphai;

  870.             Ci += result4*alphai;

  871.             C[(ci+2*ldc+0)*2+0] = Cr;

  872.             C[(ci+2*ldc+0)*2+1] = Ci;

  873.             Cr = result6*alphar;

  874.             Ci = result7*alphar;

  875.             Cr -= result7*alphai;

  876.             Ci += result6*alphai;

  877.             C[(ci+3*ldc+0)*2+0] = Cr;

  878.             C[(ci+3*ldc+0)*2+1] = Ci;

  879.             Cr = result8*alphar;

  880.             Ci = result9*alphar;

  881.             Cr -= result9*alphai;

  882.             Ci += result8*alphai;

  883.             C[(ci+4*ldc+0)*2+0] = Cr;

  884.             C[(ci+4*ldc+0)*2+1] = Ci;

  885.             Cr = result10*alphar;

  886.             Ci = result11*alphar;

  887.             Cr -= result11*alphai;

  888.             Ci += result10*alphai;

  889.             C[(ci+5*ldc+0)*2+0] = Cr;

  890.             C[(ci+5*ldc+0)*2+1] = Ci;

  891.             Cr = result12*alphar;

  892.             Ci = result13*alphar;

  893.             Cr -= result13*alphai;

  894.             Ci += result12*alphai;

  895.             C[(ci+6*ldc+0)*2+0] = Cr;

  896.             C[(ci+6*ldc+0)*2+1] = Ci;

  897.             Cr = result14*alphar;

  898.             Ci = result15*alphar;

  899.             Cr -= result15*alphai;

  900.             Ci += result14*alphai;

  901.             C[(ci+7*ldc+0)*2+0] = Cr;

  902.             C[(ci+7*ldc+0)*2+1] = Ci;

  903.             m_top+=1;

  904.         }

  905. 

  906.         n_top += 8;

  907.     }

  908. 

  909. 

  910. 

  911.     // -- tails for N=4

  912. 

  913.     if( N & 4 ) {

  914.         gvl = __riscv_vsetvl_e32m1(8);

  915.         m_top = 0;

  916. 

  917.         for (BLASLONG i=0; i<M/8; i+=1) {

  918.             BLASLONG ai=m_top*K*2;

  919.             BLASLONG bi=n_top*K*2;

  920.             BLASLONG pass_K = K;

  921.             #ifdef LEFT

  922.                 BLASLONG off = offset + m_top;

  923.             #else

  924.                 BLASLONG off = -offset + n_top;

  925.             #endif

  926.             #ifdef BACKWARDS

  927.                 ai += off*8*2;

  928.                 bi += off*4*2;

  929.                 pass_K -= off;

  930.             #else

  931.                 #ifdef LEFT

  932.                     pass_K = off + 8;

  933.                 #else

  934.                     pass_K = off + 4;

  935.                 #endif

  936.             #endif

  937.             float B0r = B[bi+0*2+0];

  938.             float B0i = B[bi+0*2+1];

  939.             float B1r = B[bi+1*2+0];

  940.             float B1i = B[bi+1*2+1];

  941.             float B2r = B[bi+2*2+0];

  942.             float B2i = B[bi+2*2+1];

  943.             float B3r = B[bi+3*2+0];

  944.             float B3i = B[bi+3*2+1];

  945.             bi += 4*2;

  946. 

  947.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  948.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  949.             ai += 8*2;

  950. 

  951.             // 2 vector regs to hold A array contents, 8 regs to hold values accumulated over k

  952.             // leaving 22 vector registers for temporaries

  953.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  954.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  955.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  956.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  957.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  958.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  959.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  960.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  961.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  962.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  963.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  964.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  965.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  966.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  967.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  968.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  969.             vfloat32m1_t ACC0r = tmp0r;

  970.             vfloat32m1_t ACC0i = tmp0i;

  971.             vfloat32m1_t ACC1r = tmp1r;

  972.             vfloat32m1_t ACC1i = tmp1i;

  973.             vfloat32m1_t ACC2r = tmp2r;

  974.             vfloat32m1_t ACC2i = tmp2i;

  975.             vfloat32m1_t ACC3r = tmp3r;

  976.             vfloat32m1_t ACC3i = tmp3i;

  977. 

  978.             for(BLASLONG k=1; k<pass_K; k++) {

  979.                 B0r = B[bi+0*2+0];

  980.                 B0i = B[bi+0*2+1];

  981.                 B1r = B[bi+1*2+0];

  982.                 B1i = B[bi+1*2+1];

  983.                 B2r = B[bi+2*2+0];

  984.                 B2i = B[bi+2*2+1];

  985.                 B3r = B[bi+3*2+0];

  986.                 B3i = B[bi+3*2+1];

  987.                 bi += 4*2;

  988. 

  989.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  990.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  991.                 ai += 8*2;

  992. 

  993.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  994.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  995.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  996.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  997.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  998.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  999.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  1000.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  1001.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1002.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1003.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1004.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1005.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  1006.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  1007.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  1008.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  1009.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1010.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1011.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1012.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1013.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  1014.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  1015.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  1016.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  1017.             }

  1018. 

  1019. 

  1020.             BLASLONG ci=n_top*ldc+m_top;

  1021. 

  1022.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1023.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1024.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1025.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1026.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  1027.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  1028.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  1029.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  1030.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1031.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1032.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1033.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1034.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  1035.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  1036.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  1037.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  1038.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1039.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1040.             ci += ldc-gvl*0;

  1041.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1042.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1043.             ci += ldc-gvl*0;

  1044.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  1045.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  1046.             ci += ldc-gvl*0;

  1047.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  1048.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  1049.             

  1050.             m_top += 8;

  1051.         }

  1052. 

  1053. 

  1054.         if( M & 4 ) {

  1055.             gvl = __riscv_vsetvl_e32m1(4);

  1056. 

  1057.             BLASLONG ai=m_top*K*2;

  1058.             BLASLONG bi=n_top*K*2;

  1059.             BLASLONG pass_K = K;

  1060.             #ifdef LEFT

  1061.                 BLASLONG off = offset + m_top;

  1062.             #else

  1063.                 BLASLONG off = -offset + n_top;

  1064.             #endif

  1065.             #ifdef BACKWARDS

  1066.                 ai += off*4*2;

  1067.                 bi += off*4*2;

  1068.                 pass_K -= off;

  1069.             #else

  1070.                 #ifdef LEFT

  1071.                     pass_K = off + 4;

  1072.                 #else

  1073.                     pass_K = off + 4;

  1074.                 #endif

  1075.             #endif

  1076.             float B0r = B[bi+0*2+0];

  1077.             float B0i = B[bi+0*2+1];

  1078.             float B1r = B[bi+1*2+0];

  1079.             float B1i = B[bi+1*2+1];

  1080.             float B2r = B[bi+2*2+0];

  1081.             float B2i = B[bi+2*2+1];

  1082.             float B3r = B[bi+3*2+0];

  1083.             float B3i = B[bi+3*2+1];

  1084.             bi += 4*2;

  1085. 

  1086.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1087.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1088.             ai += 4*2;

  1089. 

  1090.             // 2 vector regs to hold A array contents, 8 regs to hold values accumulated over k

  1091.             // leaving 22 vector registers for temporaries

  1092.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1093.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1094.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1095.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1096.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  1097.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  1098.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  1099.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  1100.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1101.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1102.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1103.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1104.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  1105.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  1106.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  1107.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  1108.             vfloat32m1_t ACC0r = tmp0r;

  1109.             vfloat32m1_t ACC0i = tmp0i;

  1110.             vfloat32m1_t ACC1r = tmp1r;

  1111.             vfloat32m1_t ACC1i = tmp1i;

  1112.             vfloat32m1_t ACC2r = tmp2r;

  1113.             vfloat32m1_t ACC2i = tmp2i;

  1114.             vfloat32m1_t ACC3r = tmp3r;

  1115.             vfloat32m1_t ACC3i = tmp3i;

  1116. 

  1117.             for(BLASLONG k=1; k<pass_K; k++) {

  1118.                 B0r = B[bi+0*2+0];

  1119.                 B0i = B[bi+0*2+1];

  1120.                 B1r = B[bi+1*2+0];

  1121.                 B1i = B[bi+1*2+1];

  1122.                 B2r = B[bi+2*2+0];

  1123.                 B2i = B[bi+2*2+1];

  1124.                 B3r = B[bi+3*2+0];

  1125.                 B3i = B[bi+3*2+1];

  1126.                 bi += 4*2;

  1127. 

  1128.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1129.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1130.                 ai += 4*2;

  1131. 

  1132.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1133.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1134.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1135.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1136.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  1137.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  1138.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  1139.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  1140.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1141.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1142.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1143.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1144.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  1145.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  1146.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  1147.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  1148.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1149.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1150.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1151.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1152.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  1153.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  1154.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  1155.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  1156.             }

  1157. 

  1158. 

  1159.             BLASLONG ci=n_top*ldc+m_top;

  1160. 

  1161.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1162.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1163.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1164.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1165.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  1166.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  1167.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  1168.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  1169.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1170.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1171.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1172.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1173.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  1174.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  1175.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  1176.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  1177.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1178.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1179.             ci += ldc-gvl*0;

  1180.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1181.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1182.             ci += ldc-gvl*0;

  1183.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  1184.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  1185.             ci += ldc-gvl*0;

  1186.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  1187.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  1188.             

  1189.             m_top += 4;

  1190.         }

  1191. 

  1192. 

  1193.         if( M & 2 ) {

  1194.             gvl = __riscv_vsetvl_e32m1(2);

  1195. 

  1196.             BLASLONG ai=m_top*K*2;

  1197.             BLASLONG bi=n_top*K*2;

  1198.             BLASLONG pass_K = K;

  1199.             #ifdef LEFT

  1200.                 BLASLONG off = offset + m_top;

  1201.             #else

  1202.                 BLASLONG off = -offset + n_top;

  1203.             #endif

  1204.             #ifdef BACKWARDS

  1205.                 ai += off*2*2;

  1206.                 bi += off*4*2;

  1207.                 pass_K -= off;

  1208.             #else

  1209.                 #ifdef LEFT

  1210.                     pass_K = off + 2;

  1211.                 #else

  1212.                     pass_K = off + 4;

  1213.                 #endif

  1214.             #endif

  1215.             float B0r = B[bi+0*2+0];

  1216.             float B0i = B[bi+0*2+1];

  1217.             float B1r = B[bi+1*2+0];

  1218.             float B1i = B[bi+1*2+1];

  1219.             float B2r = B[bi+2*2+0];

  1220.             float B2i = B[bi+2*2+1];

  1221.             float B3r = B[bi+3*2+0];

  1222.             float B3i = B[bi+3*2+1];

  1223.             bi += 4*2;

  1224. 

  1225.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1226.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1227.             ai += 2*2;

  1228. 

  1229.             // 2 vector regs to hold A array contents, 8 regs to hold values accumulated over k

  1230.             // leaving 22 vector registers for temporaries

  1231.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1232.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1233.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1234.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1235.             vfloat32m1_t tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  1236.             vfloat32m1_t tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  1237.             vfloat32m1_t tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  1238.             vfloat32m1_t tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  1239.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1240.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1241.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1242.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1243.             tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  1244.             tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  1245.             tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  1246.             tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  1247.             vfloat32m1_t ACC0r = tmp0r;

  1248.             vfloat32m1_t ACC0i = tmp0i;

  1249.             vfloat32m1_t ACC1r = tmp1r;

  1250.             vfloat32m1_t ACC1i = tmp1i;

  1251.             vfloat32m1_t ACC2r = tmp2r;

  1252.             vfloat32m1_t ACC2i = tmp2i;

  1253.             vfloat32m1_t ACC3r = tmp3r;

  1254.             vfloat32m1_t ACC3i = tmp3i;

  1255. 

  1256.             for(BLASLONG k=1; k<pass_K; k++) {

  1257.                 B0r = B[bi+0*2+0];

  1258.                 B0i = B[bi+0*2+1];

  1259.                 B1r = B[bi+1*2+0];

  1260.                 B1i = B[bi+1*2+1];

  1261.                 B2r = B[bi+2*2+0];

  1262.                 B2i = B[bi+2*2+1];

  1263.                 B3r = B[bi+3*2+0];

  1264.                 B3i = B[bi+3*2+1];

  1265.                 bi += 4*2;

  1266. 

  1267.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1268.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1269.                 ai += 2*2;

  1270. 

  1271.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1272.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1273.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1274.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1275.                 tmp2r = __riscv_vfmul_vf_f32m1( A0i, B2i, gvl);

  1276.                 tmp2i = __riscv_vfmul_vf_f32m1( A0r, B2i, gvl);

  1277.                 tmp3r = __riscv_vfmul_vf_f32m1( A0i, B3i, gvl);

  1278.                 tmp3i = __riscv_vfmul_vf_f32m1( A0r, B3i, gvl);

  1279.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1280.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1281.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1282.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1283.                 tmp2r = VFMACC_RR( tmp2r, B2r, A0r, gvl);

  1284.                 tmp2i = VFMACC_RI( tmp2i, B2r, A0i, gvl);

  1285.                 tmp3r = VFMACC_RR( tmp3r, B3r, A0r, gvl);

  1286.                 tmp3i = VFMACC_RI( tmp3i, B3r, A0i, gvl);

  1287.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1288.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1289.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1290.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1291.                 ACC2r = __riscv_vfadd( ACC2r, tmp2r, gvl);

  1292.                 ACC2i = __riscv_vfadd( ACC2i, tmp2i, gvl);

  1293.                 ACC3r = __riscv_vfadd( ACC3r, tmp3r, gvl);

  1294.                 ACC3i = __riscv_vfadd( ACC3i, tmp3i, gvl);

  1295.             }

  1296. 

  1297. 

  1298.             BLASLONG ci=n_top*ldc+m_top;

  1299. 

  1300.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1301.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1302.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1303.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1304.             vfloat32m1_t C2r = __riscv_vfmul( ACC2r, alphar, gvl );

  1305.             vfloat32m1_t C2i = __riscv_vfmul( ACC2i, alphar, gvl );

  1306.             vfloat32m1_t C3r = __riscv_vfmul( ACC3r, alphar, gvl );

  1307.             vfloat32m1_t C3i = __riscv_vfmul( ACC3i, alphar, gvl );

  1308.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1309.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1310.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1311.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1312.             C2r = __riscv_vfnmsac( C2r, alphai, ACC2i, gvl );

  1313.             C2i = __riscv_vfmacc ( C2i, alphai, ACC2r, gvl );

  1314.             C3r = __riscv_vfnmsac( C3r, alphai, ACC3i, gvl );

  1315.             C3i = __riscv_vfmacc ( C3i, alphai, ACC3r, gvl );

  1316.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1317.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1318.             ci += ldc-gvl*0;

  1319.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1320.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1321.             ci += ldc-gvl*0;

  1322.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C2r, gvl);

  1323.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C2i, gvl);

  1324.             ci += ldc-gvl*0;

  1325.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C3r, gvl);

  1326.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C3i, gvl);

  1327.             

  1328.             m_top += 2;

  1329.         }

  1330. 

  1331. 

  1332.         if( M & 1 ) {

  1333.             float result0 = 0;

  1334.             float result1 = 0;

  1335.             float result2 = 0;

  1336.             float result3 = 0;

  1337.             float result4 = 0;

  1338.             float result5 = 0;

  1339.             float result6 = 0;

  1340.             float result7 = 0;

  1341.             BLASLONG ai=m_top*K*2;

  1342.             BLASLONG bi=n_top*K*2;

  1343.             BLASLONG pass_K = K;

  1344.             #ifdef LEFT

  1345.                 BLASLONG off = offset + m_top;

  1346.             #else

  1347.                 BLASLONG off = -offset + n_top;

  1348.             #endif

  1349.             #ifdef BACKWARDS

  1350.                 ai += off*1*2;

  1351.                 bi += off*4*2;

  1352.                 pass_K -= off;

  1353.             #else

  1354.                 #ifdef LEFT

  1355.                     pass_K = off + 1;

  1356.                 #else

  1357.                     pass_K = off + 4;

  1358.                 #endif

  1359.             #endif

  1360. 

  1361.             for(BLASLONG k=0; k<pass_K; k++) {

  1362.                 result0+=S0*A[ai+0+0]*B[bi+0+0] + S1*A[ai+0+1]*B[bi+0+1];

  1363.                 result1+=S2*A[ai+0+1]*B[bi+0+0] + S3*A[ai+0+0]*B[bi+0+1];

  1364.                 result2+=S0*A[ai+0+0]*B[bi+2+0] + S1*A[ai+0+1]*B[bi+2+1];

  1365.                 result3+=S2*A[ai+0+1]*B[bi+2+0] + S3*A[ai+0+0]*B[bi+2+1];

  1366.                 result4+=S0*A[ai+0+0]*B[bi+4+0] + S1*A[ai+0+1]*B[bi+4+1];

  1367.                 result5+=S2*A[ai+0+1]*B[bi+4+0] + S3*A[ai+0+0]*B[bi+4+1];

  1368.                 result6+=S0*A[ai+0+0]*B[bi+6+0] + S1*A[ai+0+1]*B[bi+6+1];

  1369.                 result7+=S2*A[ai+0+1]*B[bi+6+0] + S3*A[ai+0+0]*B[bi+6+1];

  1370.                 ai+=1*2;

  1371.                 bi+=4*2;

  1372.             }

  1373. 

  1374.             BLASLONG ci=n_top*ldc+m_top;

  1375.             float Cr, Ci;

  1376.             Cr = result0*alphar;

  1377.             Ci = result1*alphar;

  1378.             Cr -= result1*alphai;

  1379.             Ci += result0*alphai;

  1380.             C[(ci+0*ldc+0)*2+0] = Cr;

  1381.             C[(ci+0*ldc+0)*2+1] = Ci;

  1382.             Cr = result2*alphar;

  1383.             Ci = result3*alphar;

  1384.             Cr -= result3*alphai;

  1385.             Ci += result2*alphai;

  1386.             C[(ci+1*ldc+0)*2+0] = Cr;

  1387.             C[(ci+1*ldc+0)*2+1] = Ci;

  1388.             Cr = result4*alphar;

  1389.             Ci = result5*alphar;

  1390.             Cr -= result5*alphai;

  1391.             Ci += result4*alphai;

  1392.             C[(ci+2*ldc+0)*2+0] = Cr;

  1393.             C[(ci+2*ldc+0)*2+1] = Ci;

  1394.             Cr = result6*alphar;

  1395.             Ci = result7*alphar;

  1396.             Cr -= result7*alphai;

  1397.             Ci += result6*alphai;

  1398.             C[(ci+3*ldc+0)*2+0] = Cr;

  1399.             C[(ci+3*ldc+0)*2+1] = Ci;

  1400.             m_top+=1;

  1401.         }

  1402. 

  1403.         n_top += 4;

  1404.     }

  1405. 

  1406. 

  1407. 

  1408.     // -- tails for N=2

  1409. 

  1410.     if( N & 2 ) {

  1411.         gvl = __riscv_vsetvl_e32m1(8);

  1412.         m_top = 0;

  1413. 

  1414.         for (BLASLONG i=0; i<M/8; i+=1) {

  1415.             BLASLONG ai=m_top*K*2;

  1416.             BLASLONG bi=n_top*K*2;

  1417.             BLASLONG pass_K = K;

  1418.             #ifdef LEFT

  1419.                 BLASLONG off = offset + m_top;

  1420.             #else

  1421.                 BLASLONG off = -offset + n_top;

  1422.             #endif

  1423.             #ifdef BACKWARDS

  1424.                 ai += off*8*2;

  1425.                 bi += off*2*2;

  1426.                 pass_K -= off;

  1427.             #else

  1428.                 #ifdef LEFT

  1429.                     pass_K = off + 8;

  1430.                 #else

  1431.                     pass_K = off + 2;

  1432.                 #endif

  1433.             #endif

  1434.             float B0r = B[bi+0*2+0];

  1435.             float B0i = B[bi+0*2+1];

  1436.             float B1r = B[bi+1*2+0];

  1437.             float B1i = B[bi+1*2+1];

  1438.             bi += 2*2;

  1439. 

  1440.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1441.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1442.             ai += 8*2;

  1443. 

  1444.             // 2 vector regs to hold A array contents, 4 regs to hold values accumulated over k

  1445.             // leaving 26 vector registers for temporaries

  1446.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1447.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1448.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1449.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1450.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1451.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1452.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1453.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1454.             vfloat32m1_t ACC0r = tmp0r;

  1455.             vfloat32m1_t ACC0i = tmp0i;

  1456.             vfloat32m1_t ACC1r = tmp1r;

  1457.             vfloat32m1_t ACC1i = tmp1i;

  1458. 

  1459.             for(BLASLONG k=1; k<pass_K; k++) {

  1460.                 B0r = B[bi+0*2+0];

  1461.                 B0i = B[bi+0*2+1];

  1462.                 B1r = B[bi+1*2+0];

  1463.                 B1i = B[bi+1*2+1];

  1464.                 bi += 2*2;

  1465. 

  1466.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1467.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1468.                 ai += 8*2;

  1469. 

  1470.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1471.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1472.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1473.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1474.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1475.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1476.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1477.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1478.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1479.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1480.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1481.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1482.             }

  1483. 

  1484. 

  1485.             BLASLONG ci=n_top*ldc+m_top;

  1486. 

  1487.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1488.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1489.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1490.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1491.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1492.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1493.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1494.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1495.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1496.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1497.             ci += ldc-gvl*0;

  1498.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1499.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1500.             

  1501.             m_top += 8;

  1502.         }

  1503. 

  1504. 

  1505.         if( M & 4 ) {

  1506.             gvl = __riscv_vsetvl_e32m1(4);

  1507. 

  1508.             BLASLONG ai=m_top*K*2;

  1509.             BLASLONG bi=n_top*K*2;

  1510.             BLASLONG pass_K = K;

  1511.             #ifdef LEFT

  1512.                 BLASLONG off = offset + m_top;

  1513.             #else

  1514.                 BLASLONG off = -offset + n_top;

  1515.             #endif

  1516.             #ifdef BACKWARDS

  1517.                 ai += off*4*2;

  1518.                 bi += off*2*2;

  1519.                 pass_K -= off;

  1520.             #else

  1521.                 #ifdef LEFT

  1522.                     pass_K = off + 4;

  1523.                 #else

  1524.                     pass_K = off + 2;

  1525.                 #endif

  1526.             #endif

  1527.             float B0r = B[bi+0*2+0];

  1528.             float B0i = B[bi+0*2+1];

  1529.             float B1r = B[bi+1*2+0];

  1530.             float B1i = B[bi+1*2+1];

  1531.             bi += 2*2;

  1532. 

  1533.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1534.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1535.             ai += 4*2;

  1536. 

  1537.             // 2 vector regs to hold A array contents, 4 regs to hold values accumulated over k

  1538.             // leaving 26 vector registers for temporaries

  1539.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1540.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1541.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1542.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1543.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1544.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1545.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1546.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1547.             vfloat32m1_t ACC0r = tmp0r;

  1548.             vfloat32m1_t ACC0i = tmp0i;

  1549.             vfloat32m1_t ACC1r = tmp1r;

  1550.             vfloat32m1_t ACC1i = tmp1i;

  1551. 

  1552.             for(BLASLONG k=1; k<pass_K; k++) {

  1553.                 B0r = B[bi+0*2+0];

  1554.                 B0i = B[bi+0*2+1];

  1555.                 B1r = B[bi+1*2+0];

  1556.                 B1i = B[bi+1*2+1];

  1557.                 bi += 2*2;

  1558. 

  1559.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1560.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1561.                 ai += 4*2;

  1562. 

  1563.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1564.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1565.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1566.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1567.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1568.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1569.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1570.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1571.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1572.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1573.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1574.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1575.             }

  1576. 

  1577. 

  1578.             BLASLONG ci=n_top*ldc+m_top;

  1579. 

  1580.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1581.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1582.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1583.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1584.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1585.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1586.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1587.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1588.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1589.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1590.             ci += ldc-gvl*0;

  1591.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1592.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1593.             

  1594.             m_top += 4;

  1595.         }

  1596. 

  1597. 

  1598.         if( M & 2 ) {

  1599.             gvl = __riscv_vsetvl_e32m1(2);

  1600. 

  1601.             BLASLONG ai=m_top*K*2;

  1602.             BLASLONG bi=n_top*K*2;

  1603.             BLASLONG pass_K = K;

  1604.             #ifdef LEFT

  1605.                 BLASLONG off = offset + m_top;

  1606.             #else

  1607.                 BLASLONG off = -offset + n_top;

  1608.             #endif

  1609.             #ifdef BACKWARDS

  1610.                 ai += off*2*2;

  1611.                 bi += off*2*2;

  1612.                 pass_K -= off;

  1613.             #else

  1614.                 #ifdef LEFT

  1615.                     pass_K = off + 2;

  1616.                 #else

  1617.                     pass_K = off + 2;

  1618.                 #endif

  1619.             #endif

  1620.             float B0r = B[bi+0*2+0];

  1621.             float B0i = B[bi+0*2+1];

  1622.             float B1r = B[bi+1*2+0];

  1623.             float B1i = B[bi+1*2+1];

  1624.             bi += 2*2;

  1625. 

  1626.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1627.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1628.             ai += 2*2;

  1629. 

  1630.             // 2 vector regs to hold A array contents, 4 regs to hold values accumulated over k

  1631.             // leaving 26 vector registers for temporaries

  1632.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1633.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1634.             vfloat32m1_t tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1635.             vfloat32m1_t tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1636.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1637.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1638.             tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1639.             tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1640.             vfloat32m1_t ACC0r = tmp0r;

  1641.             vfloat32m1_t ACC0i = tmp0i;

  1642.             vfloat32m1_t ACC1r = tmp1r;

  1643.             vfloat32m1_t ACC1i = tmp1i;

  1644. 

  1645.             for(BLASLONG k=1; k<pass_K; k++) {

  1646.                 B0r = B[bi+0*2+0];

  1647.                 B0i = B[bi+0*2+1];

  1648.                 B1r = B[bi+1*2+0];

  1649.                 B1i = B[bi+1*2+1];

  1650.                 bi += 2*2;

  1651. 

  1652.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1653.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1654.                 ai += 2*2;

  1655. 

  1656.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1657.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1658.                 tmp1r = __riscv_vfmul_vf_f32m1( A0i, B1i, gvl);

  1659.                 tmp1i = __riscv_vfmul_vf_f32m1( A0r, B1i, gvl);

  1660.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1661.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1662.                 tmp1r = VFMACC_RR( tmp1r, B1r, A0r, gvl);

  1663.                 tmp1i = VFMACC_RI( tmp1i, B1r, A0i, gvl);

  1664.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1665.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1666.                 ACC1r = __riscv_vfadd( ACC1r, tmp1r, gvl);

  1667.                 ACC1i = __riscv_vfadd( ACC1i, tmp1i, gvl);

  1668.             }

  1669. 

  1670. 

  1671.             BLASLONG ci=n_top*ldc+m_top;

  1672. 

  1673.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1674.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1675.             vfloat32m1_t C1r = __riscv_vfmul( ACC1r, alphar, gvl );

  1676.             vfloat32m1_t C1i = __riscv_vfmul( ACC1i, alphar, gvl );

  1677.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1678.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1679.             C1r = __riscv_vfnmsac( C1r, alphai, ACC1i, gvl );

  1680.             C1i = __riscv_vfmacc ( C1i, alphai, ACC1r, gvl );

  1681.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1682.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1683.             ci += ldc-gvl*0;

  1684.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C1r, gvl);

  1685.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C1i, gvl);

  1686.             

  1687.             m_top += 2;

  1688.         }

  1689. 

  1690. 

  1691.         if( M & 1 ) {

  1692.             float result0 = 0;

  1693.             float result1 = 0;

  1694.             float result2 = 0;

  1695.             float result3 = 0;

  1696.             BLASLONG ai=m_top*K*2;

  1697.             BLASLONG bi=n_top*K*2;

  1698.             BLASLONG pass_K = K;

  1699.             #ifdef LEFT

  1700.                 BLASLONG off = offset + m_top;

  1701.             #else

  1702.                 BLASLONG off = -offset + n_top;

  1703.             #endif

  1704.             #ifdef BACKWARDS

  1705.                 ai += off*1*2;

  1706.                 bi += off*2*2;

  1707.                 pass_K -= off;

  1708.             #else

  1709.                 #ifdef LEFT

  1710.                     pass_K = off + 1;

  1711.                 #else

  1712.                     pass_K = off + 2;

  1713.                 #endif

  1714.             #endif

  1715. 

  1716.             for(BLASLONG k=0; k<pass_K; k++) {

  1717.                 result0+=S0*A[ai+0+0]*B[bi+0+0] + S1*A[ai+0+1]*B[bi+0+1];

  1718.                 result1+=S2*A[ai+0+1]*B[bi+0+0] + S3*A[ai+0+0]*B[bi+0+1];

  1719.                 result2+=S0*A[ai+0+0]*B[bi+2+0] + S1*A[ai+0+1]*B[bi+2+1];

  1720.                 result3+=S2*A[ai+0+1]*B[bi+2+0] + S3*A[ai+0+0]*B[bi+2+1];

  1721.                 ai+=1*2;

  1722.                 bi+=2*2;

  1723.             }

  1724. 

  1725.             BLASLONG ci=n_top*ldc+m_top;

  1726.             float Cr, Ci;

  1727.             Cr = result0*alphar;

  1728.             Ci = result1*alphar;

  1729.             Cr -= result1*alphai;

  1730.             Ci += result0*alphai;

  1731.             C[(ci+0*ldc+0)*2+0] = Cr;

  1732.             C[(ci+0*ldc+0)*2+1] = Ci;

  1733.             Cr = result2*alphar;

  1734.             Ci = result3*alphar;

  1735.             Cr -= result3*alphai;

  1736.             Ci += result2*alphai;

  1737.             C[(ci+1*ldc+0)*2+0] = Cr;

  1738.             C[(ci+1*ldc+0)*2+1] = Ci;

  1739.             m_top+=1;

  1740.         }

  1741. 

  1742.         n_top += 2;

  1743.     }

  1744. 

  1745. 

  1746. 

  1747.     // -- tails for N=1

  1748. 

  1749.     if( N & 1 ) {

  1750.         gvl = __riscv_vsetvl_e32m1(8);

  1751.         m_top = 0;

  1752. 

  1753.         for (BLASLONG i=0; i<M/8; i+=1) {

  1754.             BLASLONG ai=m_top*K*2;

  1755.             BLASLONG bi=n_top*K*2;

  1756.             BLASLONG pass_K = K;

  1757.             #ifdef LEFT

  1758.                 BLASLONG off = offset + m_top;

  1759.             #else

  1760.                 BLASLONG off = -offset + n_top;

  1761.             #endif

  1762.             #ifdef BACKWARDS

  1763.                 ai += off*8*2;

  1764.                 bi += off*1*2;

  1765.                 pass_K -= off;

  1766.             #else

  1767.                 #ifdef LEFT

  1768.                     pass_K = off + 8;

  1769.                 #else

  1770.                     pass_K = off + 1;

  1771.                 #endif

  1772.             #endif

  1773.             float B0r = B[bi+0*2+0];

  1774.             float B0i = B[bi+0*2+1];

  1775.             bi += 1*2;

  1776. 

  1777.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1778.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1779.             ai += 8*2;

  1780. 

  1781.             // 2 vector regs to hold A array contents, 2 regs to hold values accumulated over k

  1782.             // leaving 28 vector registers for temporaries

  1783.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1784.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1785.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1786.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1787.             vfloat32m1_t ACC0r = tmp0r;

  1788.             vfloat32m1_t ACC0i = tmp0i;

  1789. 

  1790.             for(BLASLONG k=1; k<pass_K; k++) {

  1791.                 B0r = B[bi+0*2+0];

  1792.                 B0i = B[bi+0*2+1];

  1793.                 bi += 1*2;

  1794. 

  1795.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1796.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1797.                 ai += 8*2;

  1798. 

  1799.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1800.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1801.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1802.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1803.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1804.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1805.             }

  1806. 

  1807. 

  1808.             BLASLONG ci=n_top*ldc+m_top;

  1809. 

  1810.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1811.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1812.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1813.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1814.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1815.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1816.             

  1817.             m_top += 8;

  1818.         }

  1819. 

  1820. 

  1821.         if( M & 4 ) {

  1822.             gvl = __riscv_vsetvl_e32m1(4);

  1823. 

  1824.             BLASLONG ai=m_top*K*2;

  1825.             BLASLONG bi=n_top*K*2;

  1826.             BLASLONG pass_K = K;

  1827.             #ifdef LEFT

  1828.                 BLASLONG off = offset + m_top;

  1829.             #else

  1830.                 BLASLONG off = -offset + n_top;

  1831.             #endif

  1832.             #ifdef BACKWARDS

  1833.                 ai += off*4*2;

  1834.                 bi += off*1*2;

  1835.                 pass_K -= off;

  1836.             #else

  1837.                 #ifdef LEFT

  1838.                     pass_K = off + 4;

  1839.                 #else

  1840.                     pass_K = off + 1;

  1841.                 #endif

  1842.             #endif

  1843.             float B0r = B[bi+0*2+0];

  1844.             float B0i = B[bi+0*2+1];

  1845.             bi += 1*2;

  1846. 

  1847.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1848.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1849.             ai += 4*2;

  1850. 

  1851.             // 2 vector regs to hold A array contents, 2 regs to hold values accumulated over k

  1852.             // leaving 28 vector registers for temporaries

  1853.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1854.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1855.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1856.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1857.             vfloat32m1_t ACC0r = tmp0r;

  1858.             vfloat32m1_t ACC0i = tmp0i;

  1859. 

  1860.             for(BLASLONG k=1; k<pass_K; k++) {

  1861.                 B0r = B[bi+0*2+0];

  1862.                 B0i = B[bi+0*2+1];

  1863.                 bi += 1*2;

  1864. 

  1865.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1866.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1867.                 ai += 4*2;

  1868. 

  1869.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1870.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1871.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1872.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1873.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1874.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1875.             }

  1876. 

  1877. 

  1878.             BLASLONG ci=n_top*ldc+m_top;

  1879. 

  1880.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1881.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1882.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1883.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1884.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1885.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1886.             

  1887.             m_top += 4;

  1888.         }

  1889. 

  1890. 

  1891.         if( M & 2 ) {

  1892.             gvl = __riscv_vsetvl_e32m1(2);

  1893. 

  1894.             BLASLONG ai=m_top*K*2;

  1895.             BLASLONG bi=n_top*K*2;

  1896.             BLASLONG pass_K = K;

  1897.             #ifdef LEFT

  1898.                 BLASLONG off = offset + m_top;

  1899.             #else

  1900.                 BLASLONG off = -offset + n_top;

  1901.             #endif

  1902.             #ifdef BACKWARDS

  1903.                 ai += off*2*2;

  1904.                 bi += off*1*2;

  1905.                 pass_K -= off;

  1906.             #else

  1907.                 #ifdef LEFT

  1908.                     pass_K = off + 2;

  1909.                 #else

  1910.                     pass_K = off + 1;

  1911.                 #endif

  1912.             #endif

  1913.             float B0r = B[bi+0*2+0];

  1914.             float B0i = B[bi+0*2+1];

  1915.             bi += 1*2;

  1916. 

  1917.             vfloat32m1_t A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1918.             vfloat32m1_t A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1919.             ai += 2*2;

  1920. 

  1921.             // 2 vector regs to hold A array contents, 2 regs to hold values accumulated over k

  1922.             // leaving 28 vector registers for temporaries

  1923.             vfloat32m1_t tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1924.             vfloat32m1_t tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1925.             tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1926.             tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1927.             vfloat32m1_t ACC0r = tmp0r;

  1928.             vfloat32m1_t ACC0i = tmp0i;

  1929. 

  1930.             for(BLASLONG k=1; k<pass_K; k++) {

  1931.                 B0r = B[bi+0*2+0];

  1932.                 B0i = B[bi+0*2+1];

  1933.                 bi += 1*2;

  1934. 

  1935.                 A0r = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2], sizeof(FLOAT)*2, gvl );

  1936.                 A0i = __riscv_vlse32_v_f32m1( &A[ai+0*gvl*2+1], sizeof(FLOAT)*2, gvl );

  1937.                 ai += 2*2;

  1938. 

  1939.                 tmp0r = __riscv_vfmul_vf_f32m1( A0i, B0i, gvl);

  1940.                 tmp0i = __riscv_vfmul_vf_f32m1( A0r, B0i, gvl);

  1941.                 tmp0r = VFMACC_RR( tmp0r, B0r, A0r, gvl);

  1942.                 tmp0i = VFMACC_RI( tmp0i, B0r, A0i, gvl);

  1943.                 ACC0r = __riscv_vfadd( ACC0r, tmp0r, gvl);

  1944.                 ACC0i = __riscv_vfadd( ACC0i, tmp0i, gvl);

  1945.             }

  1946. 

  1947. 

  1948.             BLASLONG ci=n_top*ldc+m_top;

  1949. 

  1950.             vfloat32m1_t C0r = __riscv_vfmul( ACC0r, alphar, gvl );

  1951.             vfloat32m1_t C0i = __riscv_vfmul( ACC0i, alphar, gvl );

  1952.             C0r = __riscv_vfnmsac( C0r, alphai, ACC0i, gvl );

  1953.             C0i = __riscv_vfmacc ( C0i, alphai, ACC0r, gvl );

  1954.             __riscv_vsse32_v_f32m1( &C[ci*2+0], sizeof(FLOAT)*2, C0r, gvl);

  1955.             __riscv_vsse32_v_f32m1( &C[ci*2+1], sizeof(FLOAT)*2, C0i, gvl);

  1956.             

  1957.             m_top += 2;

  1958.         }

  1959. 

  1960. 

  1961.         if( M & 1 ) {

  1962.             float result0 = 0;

  1963.             float result1 = 0;

  1964.             BLASLONG ai=m_top*K*2;

  1965.             BLASLONG bi=n_top*K*2;

  1966.             BLASLONG pass_K = K;

  1967.             #ifdef LEFT

  1968.                 BLASLONG off = offset + m_top;

  1969.             #else

  1970.                 BLASLONG off = -offset + n_top;

  1971.             #endif

  1972.             #ifdef BACKWARDS

  1973.                 ai += off*1*2;

  1974.                 bi += off*1*2;

  1975.                 pass_K -= off;

  1976.             #else

  1977.                 #ifdef LEFT

  1978.                     pass_K = off + 1;

  1979.                 #else

  1980.                     pass_K = off + 1;

  1981.                 #endif

  1982.             #endif

  1983. 

  1984.             for(BLASLONG k=0; k<pass_K; k++) {

  1985.                 result0+=S0*A[ai+0+0]*B[bi+0+0] + S1*A[ai+0+1]*B[bi+0+1];

  1986.                 result1+=S2*A[ai+0+1]*B[bi+0+0] + S3*A[ai+0+0]*B[bi+0+1];

  1987.                 ai+=1*2;

  1988.                 bi+=1*2;

  1989.             }

  1990. 

  1991.             BLASLONG ci=n_top*ldc+m_top;

  1992.             float Cr, Ci;

  1993.             Cr = result0*alphar;

  1994.             Ci = result1*alphar;

  1995.             Cr -= result1*alphai;

  1996.             Ci += result0*alphai;

  1997.             C[(ci+0*ldc+0)*2+0] = Cr;

  1998.             C[(ci+0*ldc+0)*2+1] = Ci;

  1999.             m_top+=1;

  2000.         }

  2001. 

  2002.         n_top += 1;

  2003.     }

  2004. 

  2005.     return 0;

  2006. }