OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: eebc189287
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'eebc189287'
${ noResults }
OpenBLAS/kernel/mips/sgemm_kernel_8x8_msa.c

2475 lines
63 kB
Raw Blame History 

  1. /*******************************************************************************

  2. Copyright (c) 2016, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *******************************************************************************/

  27. 

  28. #include "common.h"

  29. #include "macros_msa.h"

  30. 

  31. int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT *A, FLOAT *B,

  32.           FLOAT *C, BLASLONG ldc

  33. #ifdef TRMMKERNEL

  34.           , BLASLONG offset

  35. #endif

  36.           )

  37. {

  38.     BLASLONG i, j, l, temp;

  39. #if defined(TRMMKERNEL)

  40.     BLASLONG off;

  41. #endif

  42.     FLOAT *pc0, *pc1, *pc2, *pc3, *pc4, *pc5, *pc6, *pc7;

  43.     FLOAT *pa0, *pb0;

  44.     FLOAT tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;

  45.     FLOAT tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15;

  46.     FLOAT a0, a1, b0, b1, b2, b3, b4, b5, b6, b7;

  47.     v4f32 v_alpha = {alpha, alpha, alpha, alpha};

  48.     v4f32 src_a0, src_a1, src_b, src_b0, src_b1;

  49.     v4f32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;

  50.     v4f32 res0, res1, res2, res3, res4, res5, res6, res7;

  51.     v4f32 res8, res9, res10, res11, res12, res13, res14, res15;

  52. 

  53. #if defined(TRMMKERNEL) && !defined(LEFT)

  54.     off = -offset;

  55. #endif

  56. 

  57.     for (j = (n >> 3); j--;)

  58.     {

  59.         pc0 = C;

  60.         pc1 = pc0 + ldc;

  61.         pc2 = pc1 + ldc;

  62.         pc3 = pc2 + ldc;

  63.         pc4 = pc3 + ldc;

  64.         pc5 = pc4 + ldc;

  65.         pc6 = pc5 + ldc;

  66.         pc7 = pc6 + ldc;

  67. 

  68. #if defined(TRMMKERNEL) && defined(LEFT)

  69.         off = offset;

  70. #endif

  71. 

  72.         pa0 = A;

  73.         for (i = (m >> 3); i--;)

  74.         {

  75. #if defined(TRMMKERNEL)

  76. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  77.             pb0 = B;

  78. #else

  79.             pa0 += off * 8;

  80.             pb0 = B + off * 8;

  81. #endif

  82. 

  83. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  84.             temp = k - off;

  85. #elif defined(LEFT)

  86.             temp = off + 8; // number of values in A

  87. #else

  88.             temp = off + 8; // number of values in B

  89. #endif

  90. #else

  91.             pb0 = B;

  92.             temp = k;

  93. #endif

  94. #ifdef ENABLE_PREFETCH

  95.             __asm__ __volatile__(

  96.                 "pref   0,   32(%[pa0])   \n\t"

  97.                 "pref   0,   32(%[pb0])   \n\t"

  98. 

  99.                 :

  100.                 : [pa0] "r" (pa0), [pb0] "r" (pb0)

  101.             );

  102. #endif

  103. 

  104.             LD_SP2_INC(pa0, 4, src_a0, src_a1);

  105.             LD_SP2_INC(pb0, 4, src_b0, src_b1);

  106. 

  107.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  108.             res0 = src_a0 * src_b;

  109.             res1 = src_a1 * src_b;

  110. 

  111.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  112.             res2 = src_a0 * src_b;

  113.             res3 = src_a1 * src_b;

  114. 

  115.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  116.             res4 = src_a0 * src_b;

  117.             res5 = src_a1 * src_b;

  118. 

  119.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  120.             res6 = src_a0 * src_b;

  121.             res7 = src_a1 * src_b;

  122. 

  123.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  124.             res8 = src_a0 * src_b;

  125.             res9 = src_a1 * src_b;

  126. 

  127.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  128.             res10 = src_a0 * src_b;

  129.             res11 = src_a1 * src_b;

  130. 

  131.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  132.             res12 = src_a0 * src_b;

  133.             res13 = src_a1 * src_b;

  134. 

  135.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  136.             res14 = src_a0 * src_b;

  137.             res15 = src_a1 * src_b;

  138. 

  139.             for (l = ((temp - 1) >> 1); l--;)

  140.             {

  141. #ifdef ENABLE_PREFETCH

  142.             __asm__ __volatile__(

  143.                 "pref   0,   64(%[pa0])   \n\t"

  144.                 "pref   0,   96(%[pa0])   \n\t"

  145.                 "pref   0,   64(%[pb0])   \n\t"

  146.                 "pref   0,   96(%[pb0])   \n\t"

  147. 

  148.                 :

  149.                 : [pa0] "r" (pa0), [pb0] "r" (pb0)

  150.             );

  151. #endif

  152. 

  153.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  154.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  155. 

  156.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  157.                 res0 += src_a0 * src_b;

  158.                 res1 += src_a1 * src_b;

  159. 

  160.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  161.                 res2 += src_a0 * src_b;

  162.                 res3 += src_a1 * src_b;

  163. 

  164.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  165.                 res4 += src_a0 * src_b;

  166.                 res5 += src_a1 * src_b;

  167. 

  168.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  169.                 res6 += src_a0 * src_b;

  170.                 res7 += src_a1 * src_b;

  171. 

  172.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  173.                 res8 += src_a0 * src_b;

  174.                 res9 += src_a1 * src_b;

  175. 

  176.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  177.                 res10 += src_a0 * src_b;

  178.                 res11 += src_a1 * src_b;

  179. 

  180.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  181.                 res12 += src_a0 * src_b;

  182.                 res13 += src_a1 * src_b;

  183. 

  184.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  185.                 res14 += src_a0 * src_b;

  186.                 res15 += src_a1 * src_b;

  187. 

  188.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  189.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  190. 

  191.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  192.                 res0 += src_a0 * src_b;

  193.                 res1 += src_a1 * src_b;

  194. 

  195.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  196.                 res2 += src_a0 * src_b;

  197.                 res3 += src_a1 * src_b;

  198. 

  199.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  200.                 res4 += src_a0 * src_b;

  201.                 res5 += src_a1 * src_b;

  202. 

  203.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  204.                 res6 += src_a0 * src_b;

  205.                 res7 += src_a1 * src_b;

  206. 

  207.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  208.                 res8 += src_a0 * src_b;

  209.                 res9 += src_a1 * src_b;

  210. 

  211.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  212.                 res10 += src_a0 * src_b;

  213.                 res11 += src_a1 * src_b;

  214. 

  215.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  216.                 res12 += src_a0 * src_b;

  217.                 res13 += src_a1 * src_b;

  218. 

  219.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  220.                 res14 += src_a0 * src_b;

  221.                 res15 += src_a1 * src_b;

  222.             }

  223. 

  224.             if ((temp - 1) & 1)

  225.             {

  226.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  227.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  228. 

  229.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  230.                 res0 += src_a0 * src_b;

  231.                 res1 += src_a1 * src_b;

  232. 

  233.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  234.                 res2 += src_a0 * src_b;

  235.                 res3 += src_a1 * src_b;

  236. 

  237.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  238.                 res4 += src_a0 * src_b;

  239.                 res5 += src_a1 * src_b;

  240. 

  241.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  242.                 res6 += src_a0 * src_b;

  243.                 res7 += src_a1 * src_b;

  244. 

  245.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  246.                 res8 += src_a0 * src_b;

  247.                 res9 += src_a1 * src_b;

  248. 

  249.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  250.                 res10 += src_a0 * src_b;

  251.                 res11 += src_a1 * src_b;

  252. 

  253.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  254.                 res12 += src_a0 * src_b;

  255.                 res13 += src_a1 * src_b;

  256. 

  257.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  258.                 res14 += src_a0 * src_b;

  259.                 res15 += src_a1 * src_b;

  260.             }

  261. 

  262. #if defined(TRMMKERNEL)

  263.             dst0 = res0 * v_alpha;

  264.             dst1 = res1 * v_alpha;

  265.             dst2 = res2 * v_alpha;

  266.             dst3 = res3 * v_alpha;

  267.             dst4 = res4 * v_alpha;

  268.             dst5 = res5 * v_alpha;

  269.             dst6 = res6 * v_alpha;

  270.             dst7 = res7 * v_alpha;

  271. #else

  272.             LD_SP2(pc0, 4, dst0, dst1);

  273.             LD_SP2(pc1, 4, dst2, dst3);

  274.             LD_SP2(pc2, 4, dst4, dst5);

  275.             LD_SP2(pc3, 4, dst6, dst7);

  276. 

  277.             dst0 += res0 * v_alpha;

  278.             dst1 += res1 * v_alpha;

  279.             dst2 += res2 * v_alpha;

  280.             dst3 += res3 * v_alpha;

  281.             dst4 += res4 * v_alpha;

  282.             dst5 += res5 * v_alpha;

  283.             dst6 += res6 * v_alpha;

  284.             dst7 += res7 * v_alpha;

  285. #endif

  286.             ST_SP2_INC(dst0, dst1, pc0, 4);

  287.             ST_SP2_INC(dst2, dst3, pc1, 4);

  288.             ST_SP2_INC(dst4, dst5, pc2, 4);

  289.             ST_SP2_INC(dst6, dst7, pc3, 4);

  290. 

  291. #if defined(TRMMKERNEL)

  292.             dst0 = res8 * v_alpha;

  293.             dst1 = res9 * v_alpha;

  294.             dst2 = res10 * v_alpha;

  295.             dst3 = res11 * v_alpha;

  296.             dst4 = res12 * v_alpha;

  297.             dst5 = res13 * v_alpha;

  298.             dst6 = res14 * v_alpha;

  299.             dst7 = res15 * v_alpha;

  300. #else

  301.             LD_SP2(pc4, 4, dst0, dst1);

  302.             LD_SP2(pc5, 4, dst2, dst3);

  303.             LD_SP2(pc6, 4, dst4, dst5);

  304.             LD_SP2(pc7, 4, dst6, dst7);

  305. 

  306.             dst0 += res8 * v_alpha;

  307.             dst1 += res9 * v_alpha;

  308.             dst2 += res10 * v_alpha;

  309.             dst3 += res11 * v_alpha;

  310.             dst4 += res12 * v_alpha;

  311.             dst5 += res13 * v_alpha;

  312.             dst6 += res14 * v_alpha;

  313.             dst7 += res15 * v_alpha;

  314. #endif

  315.             ST_SP2_INC(dst0, dst1, pc4, 4);

  316.             ST_SP2_INC(dst2, dst3, pc5, 4);

  317.             ST_SP2_INC(dst4, dst5, pc6, 4);

  318.             ST_SP2_INC(dst6, dst7, pc7, 4);

  319. 

  320. #if defined(TRMMKERNEL)

  321. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  322.             temp = k - off;

  323. #ifdef LEFT

  324.             temp -= 8; // number of values in A

  325. #else

  326.             temp -= 8; // number of values in B

  327. #endif

  328.             pa0 += temp * 8;

  329.             pb0 += temp * 8;

  330. #endif

  331. 

  332. #ifdef LEFT

  333.             off += 8; // number of values in A

  334. #endif

  335. #endif

  336.         }

  337. 

  338.         if (m & 4)

  339.         {

  340. #if defined(TRMMKERNEL)

  341. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  342.             pb0 = B;

  343. #else

  344.             pa0 += off * 4;

  345.             pb0 = B + off * 8;

  346. #endif

  347. 

  348. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  349.             temp = k - off;

  350. #elif defined(LEFT)

  351.             temp = off + 4; // number of values in A

  352. #else

  353.             temp = off + 8; // number of values in B

  354. #endif

  355. #else

  356.             pb0 = B;

  357.             temp = k;

  358. #endif

  359. 

  360.             src_a0 = LD_SP(pa0);

  361.             LD_SP2_INC(pb0, 4, src_b0, src_b1);

  362. 

  363.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  364.             res0 = src_a0 * src_b;

  365. 

  366.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  367.             res1 = src_a0 * src_b;

  368. 

  369.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  370.             res2 = src_a0 * src_b;

  371. 

  372.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  373.             res3 = src_a0 * src_b;

  374. 

  375.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  376.             res4 = src_a0 * src_b;

  377. 

  378.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  379.             res5 = src_a0 * src_b;

  380. 

  381.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  382.             res6 = src_a0 * src_b;

  383. 

  384.             src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  385.             res7 = src_a0 * src_b;

  386. 

  387.             pa0 += 4;

  388. 

  389.             for (l = ((temp - 1) >> 1); l--;)

  390.             {

  391.                 src_a0 = LD_SP(pa0);

  392.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  393. 

  394.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  395.                 res0 += src_a0 * src_b;

  396. 

  397.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  398.                 res1 += src_a0 * src_b;

  399. 

  400.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  401.                 res2 += src_a0 * src_b;

  402. 

  403.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  404.                 res3 += src_a0 * src_b;

  405. 

  406.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  407.                 res4 += src_a0 * src_b;

  408. 

  409.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  410.                 res5 += src_a0 * src_b;

  411. 

  412.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  413.                 res6 += src_a0 * src_b;

  414. 

  415.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  416.                 res7 += src_a0 * src_b;

  417. 

  418.                 pa0 += 4;

  419. 

  420.                 src_a0 = LD_SP(pa0);

  421.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  422. 

  423.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  424.                 res0 += src_a0 * src_b;

  425. 

  426.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  427.                 res1 += src_a0 * src_b;

  428. 

  429.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  430.                 res2 += src_a0 * src_b;

  431. 

  432.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  433.                 res3 += src_a0 * src_b;

  434. 

  435.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  436.                 res4 += src_a0 * src_b;

  437. 

  438.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  439.                 res5 += src_a0 * src_b;

  440. 

  441.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  442.                 res6 += src_a0 * src_b;

  443. 

  444.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  445.                 res7 += src_a0 * src_b;

  446. 

  447.                 pa0 += 4;

  448.             }

  449. 

  450.             if ((temp - 1) & 1)

  451.             {

  452.                 src_a0 = LD_SP(pa0);

  453.                 LD_SP2_INC(pb0, 4, src_b0, src_b1);

  454. 

  455.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  456.                 res0 += src_a0 * src_b;

  457. 

  458.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  459.                 res1 += src_a0 * src_b;

  460. 

  461.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  462.                 res2 += src_a0 * src_b;

  463. 

  464.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  465.                 res3 += src_a0 * src_b;

  466. 

  467.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0);

  468.                 res4 += src_a0 * src_b;

  469. 

  470.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0x55);

  471.                 res5 += src_a0 * src_b;

  472. 

  473.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xAA);

  474.                 res6 += src_a0 * src_b;

  475. 

  476.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b1, 0xFF);

  477.                 res7 += src_a0 * src_b;

  478. 

  479.                 pa0 += 4;

  480.             }

  481. 

  482. #if defined(TRMMKERNEL)

  483.             dst0 = res0 * v_alpha;

  484.             dst1 = res1 * v_alpha;

  485.             dst2 = res2 * v_alpha;

  486.             dst3 = res3 * v_alpha;

  487. #else

  488.             dst0 = LD_SP(pc0);

  489.             dst1 = LD_SP(pc1);

  490.             dst2 = LD_SP(pc2);

  491.             dst3 = LD_SP(pc3);

  492. 

  493.             dst0 += res0 * v_alpha;

  494.             dst1 += res1 * v_alpha;

  495.             dst2 += res2 * v_alpha;

  496.             dst3 += res3 * v_alpha;

  497. #endif

  498.             ST_SP(dst0, pc0);

  499.             ST_SP(dst1, pc1);

  500.             ST_SP(dst2, pc2);

  501.             ST_SP(dst3, pc3);

  502. 

  503. #if defined(TRMMKERNEL)

  504.             dst0 = res4 * v_alpha;

  505.             dst1 = res5 * v_alpha;

  506.             dst2 = res6 * v_alpha;

  507.             dst3 = res7 * v_alpha;

  508. #else

  509.             dst0 = LD_SP(pc4);

  510.             dst1 = LD_SP(pc5);

  511.             dst2 = LD_SP(pc6);

  512.             dst3 = LD_SP(pc7);

  513. 

  514.             dst0 += res4 * v_alpha;

  515.             dst1 += res5 * v_alpha;

  516.             dst2 += res6 * v_alpha;

  517.             dst3 += res7 * v_alpha;

  518. #endif

  519.             ST_SP(dst0, pc4);

  520.             ST_SP(dst1, pc5);

  521.             ST_SP(dst2, pc6);

  522.             ST_SP(dst3, pc7);

  523. 

  524.             pc0 += 4;

  525.             pc1 += 4;

  526.             pc2 += 4;

  527.             pc3 += 4;

  528.             pc4 += 4;

  529.             pc5 += 4;

  530.             pc6 += 4;

  531.             pc7 += 4;

  532. 

  533. #if defined(TRMMKERNEL)

  534. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  535.             temp = k - off;

  536. #ifdef LEFT

  537.             temp -= 4; // number of values in A

  538. #else

  539.             temp -= 8; // number of values in B

  540. #endif

  541.             pa0 += temp * 4;

  542.             pb0 += temp * 8;

  543. #endif

  544. 

  545. #ifdef LEFT

  546.             off += 4; // number of values in A

  547. #endif

  548. #endif

  549.         }

  550. 

  551.         if (m & 2)

  552.         {

  553. #if defined(TRMMKERNEL)

  554. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  555.             pb0 = B;

  556. #else

  557.             pa0 += off * 2;

  558.             pb0 = B + off * 8;

  559. #endif

  560. 

  561. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  562.             temp = k - off;

  563. #elif defined(LEFT)

  564.             temp = off + 2; // number of values in A

  565. #else

  566.             temp = off + 8; // number of values in B

  567. #endif

  568. #else

  569.             pb0 = B;

  570.             temp = k;

  571. #endif

  572. 

  573.             a0 = pa0[0];

  574.             b0 = pb0[0];

  575.             tmp0 = a0 * b0;

  576. 

  577.             a1 = pa0[1];

  578.             tmp1 = a1 * b0;

  579. 

  580.             b1 = pb0[1];

  581.             tmp2 = a0 * b1;

  582.             tmp3 = a1 * b1;

  583. 

  584.             b2 = pb0[2];

  585.             tmp4 = a0 * b2;

  586.             tmp5 = a1 * b2;

  587. 

  588.             b3 = pb0[3];

  589.             tmp6 = a0 * b3;

  590.             tmp7 = a1 * b3;

  591. 

  592.             b4 = pb0[4];

  593.             tmp8 = a0 * b4;

  594.             tmp9 = a1 * b4;

  595. 

  596.             b5 = pb0[5];

  597.             tmp10 = a0 * b5;

  598.             tmp11 = a1 * b5;

  599. 

  600.             b6 = pb0[6];

  601.             tmp12 = a0 * b6;

  602.             tmp13 = a1 * b6;

  603. 

  604.             b7 = pb0[7];

  605.             tmp14 = a0 * b7;

  606.             tmp15 = a1 * b7;

  607. 

  608.             pa0 += 2;

  609.             pb0 += 8;

  610. 

  611.             for (l = ((temp - 1) >> 1); l--;)

  612.             {

  613.                 a0 = pa0[0];

  614.                 b0 = pb0[0];

  615.                 tmp0 += a0 * b0;

  616. 

  617.                 a1 = pa0[1];

  618.                 tmp1 += a1 * b0;

  619. 

  620.                 b1 = pb0[1];

  621.                 tmp2 += a0 * b1;

  622.                 tmp3 += a1 * b1;

  623. 

  624.                 b2 = pb0[2];

  625.                 tmp4 += a0 * b2;

  626.                 tmp5 += a1 * b2;

  627. 

  628.                 b3 = pb0[3];

  629.                 tmp6 += a0 * b3;

  630.                 tmp7 += a1 * b3;

  631. 

  632.                 b4 = pb0[4];

  633.                 tmp8 += a0 * b4;

  634.                 tmp9 += a1 * b4;

  635. 

  636.                 b5 = pb0[5];

  637.                 tmp10 += a0 * b5;

  638.                 tmp11 += a1 * b5;

  639. 

  640.                 b6 = pb0[6];

  641.                 tmp12 += a0 * b6;

  642.                 tmp13 += a1 * b6;

  643. 

  644.                 b7 = pb0[7];

  645.                 tmp14 += a0 * b7;

  646.                 tmp15 += a1 * b7;

  647. 

  648.                 pa0 += 2;

  649.                 pb0 += 8;

  650. 

  651.                 a0 = pa0[0];

  652.                 b0 = pb0[0];

  653.                 tmp0 += a0 * b0;

  654. 

  655.                 a1 = pa0[1];

  656.                 tmp1 += a1 * b0;

  657. 

  658.                 b1 = pb0[1];

  659.                 tmp2 += a0 * b1;

  660.                 tmp3 += a1 * b1;

  661. 

  662.                 b2 = pb0[2];

  663.                 tmp4 += a0 * b2;

  664.                 tmp5 += a1 * b2;

  665. 

  666.                 b3 = pb0[3];

  667.                 tmp6 += a0 * b3;

  668.                 tmp7 += a1 * b3;

  669. 

  670.                 b4 = pb0[4];

  671.                 tmp8 += a0 * b4;

  672.                 tmp9 += a1 * b4;

  673. 

  674.                 b5 = pb0[5];

  675.                 tmp10 += a0 * b5;

  676.                 tmp11 += a1 * b5;

  677. 

  678.                 b6 = pb0[6];

  679.                 tmp12 += a0 * b6;

  680.                 tmp13 += a1 * b6;

  681. 

  682.                 b7 = pb0[7];

  683.                 tmp14 += a0 * b7;

  684.                 tmp15 += a1 * b7;

  685. 

  686.                 pa0 += 2;

  687.                 pb0 += 8;

  688.             }

  689. 

  690.             if ((temp - 1) & 1)

  691.             {

  692.                 a0 = pa0[0];

  693.                 b0 = pb0[0];

  694.                 tmp0 += a0 * b0;

  695. 

  696.                 a1 = pa0[1];

  697.                 tmp1 += a1 * b0;

  698. 

  699.                 b1 = pb0[1];

  700.                 tmp2 += a0 * b1;

  701.                 tmp3 += a1 * b1;

  702. 

  703.                 b2 = pb0[2];

  704.                 tmp4 += a0 * b2;

  705.                 tmp5 += a1 * b2;

  706. 

  707.                 b3 = pb0[3];

  708.                 tmp6 += a0 * b3;

  709.                 tmp7 += a1 * b3;

  710. 

  711.                 b4 = pb0[4];

  712.                 tmp8 += a0 * b4;

  713.                 tmp9 += a1 * b4;

  714. 

  715.                 b5 = pb0[5];

  716.                 tmp10 += a0 * b5;

  717.                 tmp11 += a1 * b5;

  718. 

  719.                 b6 = pb0[6];

  720.                 tmp12 += a0 * b6;

  721.                 tmp13 += a1 * b6;

  722. 

  723.                 b7 = pb0[7];

  724.                 tmp14 += a0 * b7;

  725.                 tmp15 += a1 * b7;

  726. 

  727.                 pa0 += 2;

  728.                 pb0 += 8;

  729.             }

  730. 

  731.             tmp0 = alpha * tmp0;

  732.             tmp2 = alpha * tmp2;

  733.             tmp4 = alpha * tmp4;

  734.             tmp6 = alpha * tmp6;

  735.             tmp8 = alpha * tmp8;

  736.             tmp10 = alpha * tmp10;

  737.             tmp12 = alpha * tmp12;

  738.             tmp14 = alpha * tmp14;

  739. 

  740. #if defined(TRMMKERNEL)

  741.             pc0[0] = tmp0;

  742.             pc1[0] = tmp2;

  743.             pc2[0] = tmp4;

  744.             pc3[0] = tmp6;

  745.             pc4[0] = tmp8;

  746.             pc5[0] = tmp10;

  747.             pc6[0] = tmp12;

  748.             pc7[0] = tmp14;

  749. #else

  750.             pc0[0] += tmp0;

  751.             pc1[0] += tmp2;

  752.             pc2[0] += tmp4;

  753.             pc3[0] += tmp6;

  754.             pc4[0] += tmp8;

  755.             pc5[0] += tmp10;

  756.             pc6[0] += tmp12;

  757.             pc7[0] += tmp14;

  758. #endif

  759.             tmp1 = alpha * tmp1;

  760.             tmp3 = alpha * tmp3;

  761.             tmp5 = alpha * tmp5;

  762.             tmp7 = alpha * tmp7;

  763.             tmp9 = alpha * tmp9;

  764.             tmp11 = alpha * tmp11;

  765.             tmp13 = alpha * tmp13;

  766.             tmp15 = alpha * tmp15;

  767. 

  768. #if defined(TRMMKERNEL)

  769.             pc0[1] = tmp1;

  770.             pc1[1] = tmp3;

  771.             pc2[1] = tmp5;

  772.             pc3[1] = tmp7;

  773.             pc4[1] = tmp9;

  774.             pc5[1] = tmp11;

  775.             pc6[1] = tmp13;

  776.             pc7[1] = tmp15;

  777. #else

  778.             pc0[1] += tmp1;

  779.             pc1[1] += tmp3;

  780.             pc2[1] += tmp5;

  781.             pc3[1] += tmp7;

  782.             pc4[1] += tmp9;

  783.             pc5[1] += tmp11;

  784.             pc6[1] += tmp13;

  785.             pc7[1] += tmp15;

  786. #endif

  787.             pc0 += 2;

  788.             pc1 += 2;

  789.             pc2 += 2;

  790.             pc3 += 2;

  791.             pc4 += 2;

  792.             pc5 += 2;

  793.             pc6 += 2;

  794.             pc7 += 2;

  795. 

  796. #if defined(TRMMKERNEL)

  797. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  798.             temp = k - off;

  799. #ifdef LEFT

  800.             temp -= 2; // number of values in A

  801. #else

  802.             temp -= 8; // number of values in B

  803. #endif

  804.             pa0 += temp * 2;

  805.             pb0 += temp * 8;

  806. #endif

  807. 

  808. #ifdef LEFT

  809.             off += 2; // number of values in A

  810. #endif

  811. #endif

  812.         }

  813. 

  814.         if (m & 1)

  815.         {

  816. #if defined(TRMMKERNEL)

  817. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  818.             pb0 = B;

  819. #else

  820.             pa0 += off * 1;

  821.             pb0 = B + off * 8;

  822. #endif

  823. 

  824. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  825.             temp = k - off;

  826. #elif defined(LEFT)

  827.             temp = off + 1; // number of values in A

  828. #else

  829.             temp = off + 8; // number of values in B

  830. #endif

  831. #else

  832.             pb0 = B;

  833.             temp = k;

  834. #endif

  835. 

  836.             a0 = pa0[0];

  837.             b0 = pb0[0];

  838.             tmp0 = a0 * b0;

  839. 

  840.             b1 = pb0[1];

  841.             tmp1 = a0 * b1;

  842. 

  843.             b2 = pb0[2];

  844.             tmp2 = a0 * b2;

  845. 

  846.             b3 = pb0[3];

  847.             tmp3 = a0 * b3;

  848. 

  849.             b4 = pb0[4];

  850.             tmp4 = a0 * b4;

  851. 

  852.             b5 = pb0[5];

  853.             tmp5 = a0 * b5;

  854. 

  855.             b6 = pb0[6];

  856.             tmp6 = a0 * b6;

  857. 

  858.             b7 = pb0[7];

  859.             tmp7 = a0 * b7;

  860. 

  861.             pa0 += 1;

  862.             pb0 += 8;

  863. 

  864.             for (l = ((temp - 1) >> 1); l--;)

  865.             {

  866.                 a0 = pa0[0];

  867.                 b0 = pb0[0];

  868.                 tmp0 += a0 * b0;

  869. 

  870.                 b1 = pb0[1];

  871.                 tmp1  += a0 * b1;

  872. 

  873.                 b2 = pb0[2];

  874.                 tmp2 += a0 * b2;

  875. 

  876.                 b3 = pb0[3];

  877.                 tmp3 += a0 * b3;

  878. 

  879.                 b4 = pb0[4];

  880.                 tmp4 += a0 * b4;

  881. 

  882.                 b5 = pb0[5];

  883.                 tmp5 += a0 * b5;

  884. 

  885.                 b6 = pb0[6];

  886.                 tmp6 += a0 * b6;

  887. 

  888.                 b7 = pb0[7];

  889.                 tmp7 += a0 * b7;

  890. 

  891.                 pa0 += 1;

  892.                 pb0 += 8;

  893. 

  894.                 a0 = pa0[0];

  895.                 b0 = pb0[0];

  896.                 tmp0 += a0 * b0;

  897. 

  898.                 b1 = pb0[1];

  899.                 tmp1  += a0 * b1;

  900. 

  901.                 b2 = pb0[2];

  902.                 tmp2 += a0 * b2;

  903. 

  904.                 b3 = pb0[3];

  905.                 tmp3 += a0 * b3;

  906. 

  907.                 b4 = pb0[4];

  908.                 tmp4 += a0 * b4;

  909. 

  910.                 b5 = pb0[5];

  911.                 tmp5 += a0 * b5;

  912. 

  913.                 b6 = pb0[6];

  914.                 tmp6 += a0 * b6;

  915. 

  916.                 b7 = pb0[7];

  917.                 tmp7 += a0 * b7;

  918. 

  919.                 pa0 += 1;

  920.                 pb0 += 8;

  921.             }

  922. 

  923.             if ((temp - 1) & 1)

  924.             {

  925.                 a0 = pa0[0];

  926.                 b0 = pb0[0];

  927.                 tmp0 += a0 * b0;

  928. 

  929.                 b1 = pb0[1];

  930.                 tmp1 += a0 * b1;

  931. 

  932.                 b2 = pb0[2];

  933.                 tmp2 += a0 * b2;

  934. 

  935.                 b3 = pb0[3];

  936.                 tmp3 += a0 * b3;

  937. 

  938.                 b4 = pb0[4];

  939.                 tmp4 += a0 * b4;

  940. 

  941.                 b5 = pb0[5];

  942.                 tmp5 += a0 * b5;

  943. 

  944.                 b6 = pb0[6];

  945.                 tmp6 += a0 * b6;

  946. 

  947.                 b7 = pb0[7];

  948.                 tmp7 += a0 * b7;

  949. 

  950.                 pa0 += 1;

  951.                 pb0 += 8;

  952.             }

  953. 

  954.             tmp0 = alpha * tmp0;

  955.             tmp1 = alpha * tmp1;

  956.             tmp2 = alpha * tmp2;

  957.             tmp3 = alpha * tmp3;

  958.             tmp4 = alpha * tmp4;

  959.             tmp5 = alpha * tmp5;

  960.             tmp6 = alpha * tmp6;

  961.             tmp7 = alpha * tmp7;

  962. 

  963. #if defined(TRMMKERNEL)

  964.             pc0[0] = tmp0;

  965.             pc1[0] = tmp1;

  966.             pc2[0] = tmp2;

  967.             pc3[0] = tmp3;

  968.             pc4[0] = tmp4;

  969.             pc5[0] = tmp5;

  970.             pc6[0] = tmp6;

  971.             pc7[0] = tmp7;

  972. #else

  973.             pc0[0] += tmp0;

  974.             pc1[0] += tmp1;

  975.             pc2[0] += tmp2;

  976.             pc3[0] += tmp3;

  977.             pc4[0] += tmp4;

  978.             pc5[0] += tmp5;

  979.             pc6[0] += tmp6;

  980.             pc7[0] += tmp7;

  981. #endif

  982.             pc0 += 1;

  983.             pc1 += 1;

  984.             pc2 += 1;

  985.             pc3 += 1;

  986.             pc4 += 1;

  987.             pc5 += 1;

  988.             pc6 += 1;

  989.             pc7 += 1;

  990. 

  991. #if defined(TRMMKERNEL)

  992. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  993.             temp = k - off;

  994. #ifdef LEFT

  995.             temp -= 1; // number of values in A

  996. #else

  997.             temp -= 8; // number of values in B

  998. #endif

  999.             pa0 += temp * 1;

  1000.             pb0 += temp * 8;

  1001. #endif

  1002. 

  1003. #ifdef LEFT

  1004.             off += 1; // number of values in A

  1005. #endif

  1006. #endif

  1007.         }

  1008. 

  1009. #if defined(TRMMKERNEL) && !defined(LEFT)

  1010.         off += 8; // number of values in A

  1011. #endif

  1012. 

  1013.         B += (k << 3);

  1014.         C += (ldc << 3);

  1015.     }

  1016. 

  1017.     if (n & 4)

  1018.     {

  1019.         pc0 = C;

  1020.         pc1 = pc0 + ldc;

  1021.         pc2 = pc1 + ldc;

  1022.         pc3 = pc2 + ldc;

  1023. 

  1024. #if defined(TRMMKERNEL) && defined(LEFT)

  1025.         off = offset;

  1026. #endif

  1027. 

  1028.         pa0 = A;

  1029. 

  1030.         for (i = (m >> 3); i--;)

  1031.         {

  1032. #if defined(TRMMKERNEL)

  1033. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1034.             pb0 = B;

  1035. #else

  1036.             pa0 += off * 8;

  1037.             pb0 = B + off * 4;

  1038. #endif

  1039. 

  1040. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1041.             temp = k - off;

  1042. #elif defined(LEFT)

  1043.             temp = off + 8; // number of values in A

  1044. #else

  1045.             temp = off + 4; // number of values in B

  1046. #endif

  1047. #else

  1048.             pb0 = B;

  1049.             temp = k;

  1050. #endif

  1051. 

  1052.             LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1053.             src_b0 = LD_SP(pb0);

  1054. 

  1055.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1056.             res0 = src_a0 * src_b;

  1057.             res1 = src_a1 * src_b;

  1058. 

  1059.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1060.             res2 = src_a0 * src_b;

  1061.             res3 = src_a1 * src_b;

  1062. 

  1063.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1064.             res4 = src_a0 * src_b;

  1065.             res5 = src_a1 * src_b;

  1066. 

  1067.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1068.             res6 = src_a0 * src_b;

  1069.             res7 = src_a1 * src_b;

  1070. 

  1071.             pb0 += 4;

  1072. 

  1073.             for (l = ((temp - 1) >> 1); l--;)

  1074.             {

  1075.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1076.                 src_b0 = LD_SP(pb0);

  1077. 

  1078.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1079.                 res0 += src_a0 * src_b;

  1080.                 res1 += src_a1 * src_b;

  1081. 

  1082.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1083.                 res2 += src_a0 * src_b;

  1084.                 res3 += src_a1 * src_b;

  1085. 

  1086.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1087.                 res4 += src_a0 * src_b;

  1088.                 res5 += src_a1 * src_b;

  1089. 

  1090.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1091.                 res6 += src_a0 * src_b;

  1092.                 res7 += src_a1 * src_b;

  1093. 

  1094.                 pb0 += 4;

  1095. 

  1096.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1097.                 src_b0 = LD_SP(pb0);

  1098. 

  1099.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1100.                 res0 += src_a0 * src_b;

  1101.                 res1 += src_a1 * src_b;

  1102. 

  1103.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1104.                 res2 += src_a0 * src_b;

  1105.                 res3 += src_a1 * src_b;

  1106. 

  1107.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1108.                 res4 += src_a0 * src_b;

  1109.                 res5 += src_a1 * src_b;

  1110. 

  1111.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1112.                 res6 += src_a0 * src_b;

  1113.                 res7 += src_a1 * src_b;

  1114. 

  1115.                 pb0 += 4;

  1116.             }

  1117. 

  1118.             if ((temp - 1) & 1)

  1119.             {

  1120.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1121.                 src_b0 = LD_SP(pb0);

  1122. 

  1123.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1124.                 res0 += src_a0 * src_b;

  1125.                 res1 += src_a1 * src_b;

  1126. 

  1127.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1128.                 res2 += src_a0 * src_b;

  1129.                 res3 += src_a1 * src_b;

  1130. 

  1131.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1132.                 res4 += src_a0 * src_b;

  1133.                 res5 += src_a1 * src_b;

  1134. 

  1135.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1136.                 res6 += src_a0 * src_b;

  1137.                 res7 += src_a1 * src_b;

  1138. 

  1139.                 pb0 += 4;

  1140.             }

  1141. 

  1142. #if defined(TRMMKERNEL)

  1143.             dst0 = res0 * v_alpha;

  1144.             dst1 = res1 * v_alpha;

  1145.             dst2 = res2 * v_alpha;

  1146.             dst3 = res3 * v_alpha;

  1147.             dst4 = res4 * v_alpha;

  1148.             dst5 = res5 * v_alpha;

  1149.             dst6 = res6 * v_alpha;

  1150.             dst7 = res7 * v_alpha;

  1151. #else

  1152.             LD_SP2(pc0, 4, dst0, dst1);

  1153.             LD_SP2(pc1, 4, dst2, dst3);

  1154.             LD_SP2(pc2, 4, dst4, dst5);

  1155.             LD_SP2(pc3, 4, dst6, dst7);

  1156. 

  1157.             dst0 += res0 * v_alpha;

  1158.             dst1 += res1 * v_alpha;

  1159.             dst2 += res2 * v_alpha;

  1160.             dst3 += res3 * v_alpha;

  1161.             dst4 += res4 * v_alpha;

  1162.             dst5 += res5 * v_alpha;

  1163.             dst6 += res6 * v_alpha;

  1164.             dst7 += res7 * v_alpha;

  1165. #endif

  1166.             ST_SP2_INC(dst0, dst1, pc0, 4);

  1167.             ST_SP2_INC(dst2, dst3, pc1, 4);

  1168.             ST_SP2_INC(dst4, dst5, pc2, 4);

  1169.             ST_SP2_INC(dst6, dst7, pc3, 4);

  1170. 

  1171. #if defined(TRMMKERNEL)

  1172. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1173.             temp = k - off;

  1174. #ifdef LEFT

  1175.             temp -= 8; // number of values in A

  1176. #else

  1177.             temp -= 4; // number of values in B

  1178. #endif

  1179.             pa0 += temp * 8;

  1180.             pb0 += temp * 4;

  1181. #endif

  1182. 

  1183. #ifdef LEFT

  1184.             off += 8; // number of values in A

  1185. #endif

  1186. #endif

  1187.         }

  1188. 

  1189.         if (m & 4)

  1190.         {

  1191. #if defined(TRMMKERNEL)

  1192. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1193.             pb0 = B;

  1194. #else

  1195.             pa0 += off * 4;

  1196.             pb0 = B + off * 4;

  1197. #endif

  1198. 

  1199. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1200.             temp = k - off;

  1201. #elif defined(LEFT)

  1202.             temp = off + 4; // number of values in A

  1203. #else

  1204.             temp = off + 4; // number of values in B

  1205. #endif

  1206. #else

  1207.             pb0 = B;

  1208.             temp = k;

  1209. #endif

  1210. 

  1211.             src_a0 = LD_SP(pa0);

  1212.             src_b0 = LD_SP(pb0);

  1213. 

  1214.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1215.             res0 = src_a0 * src_b;

  1216. 

  1217.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1218.             res1 = src_a0 * src_b;

  1219. 

  1220.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1221.             res2 = src_a0 * src_b;

  1222. 

  1223.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1224.             res3 = src_a0 * src_b;

  1225. 

  1226.             pa0 += 4;

  1227.             pb0 += 4;

  1228. 

  1229.             for (l = ((temp - 1) >> 1); l--;)

  1230.             {

  1231.                 src_a0 = LD_SP(pa0);

  1232.                 src_b0 = LD_SP(pb0);

  1233. 

  1234.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1235.                 res0 += src_a0 * src_b;

  1236. 

  1237.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1238.                 res1 += src_a0 * src_b;

  1239. 

  1240.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1241.                 res2 += src_a0 * src_b;

  1242. 

  1243.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1244.                 res3 += src_a0 * src_b;

  1245. 

  1246.                 pa0 += 4;

  1247.                 pb0 += 4;

  1248. 

  1249.                 src_a0 = LD_SP(pa0);

  1250.                 src_b0 = LD_SP(pb0);

  1251. 

  1252.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1253.                 res0 += src_a0 * src_b;

  1254. 

  1255.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1256.                 res1 += src_a0 * src_b;

  1257. 

  1258.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1259.                 res2 += src_a0 * src_b;

  1260. 

  1261.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1262.                 res3 += src_a0 * src_b;

  1263. 

  1264.                 pa0 += 4;

  1265.                 pb0 += 4;

  1266.             }

  1267. 

  1268.             if ((temp - 1) & 1)

  1269.             {

  1270.                 src_a0 = LD_SP(pa0);

  1271.                 src_b0 = LD_SP(pb0);

  1272. 

  1273.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1274.                 res0 += src_a0 * src_b;

  1275. 

  1276.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1277.                 res1 += src_a0 * src_b;

  1278. 

  1279.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xAA);

  1280.                 res2 += src_a0 * src_b;

  1281. 

  1282.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0xFF);

  1283.                 res3 += src_a0 * src_b;

  1284. 

  1285.                 pa0 += 4;

  1286.                 pb0 += 4;

  1287.             }

  1288. 

  1289. #if defined(TRMMKERNEL)

  1290.             dst0 = res0 * v_alpha;

  1291.             dst1 = res1 * v_alpha;

  1292.             dst2 = res2 * v_alpha;

  1293.             dst3 = res3 * v_alpha;

  1294. #else

  1295.             dst0 = LD_SP(pc0);

  1296.             dst1 = LD_SP(pc1);

  1297.             dst2 = LD_SP(pc2);

  1298.             dst3 = LD_SP(pc3);

  1299. 

  1300.             dst0 += res0 * v_alpha;

  1301.             dst1 += res1 * v_alpha;

  1302.             dst2 += res2 * v_alpha;

  1303.             dst3 += res3 * v_alpha;

  1304. #endif

  1305.             ST_SP(dst0, pc0);

  1306.             ST_SP(dst1, pc1);

  1307.             ST_SP(dst2, pc2);

  1308.             ST_SP(dst3, pc3);

  1309. 

  1310.             pc0 += 4;

  1311.             pc1 += 4;

  1312.             pc2 += 4;

  1313.             pc3 += 4;

  1314. 

  1315. #if defined(TRMMKERNEL)

  1316. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1317.             temp = k - off;

  1318. #ifdef LEFT

  1319.             temp -= 4; // number of values in A

  1320. #else

  1321.             temp -= 4; // number of values in B

  1322. #endif

  1323.             pa0 += temp * 4;

  1324.             pb0 += temp * 4;

  1325. #endif

  1326. 

  1327. #ifdef LEFT

  1328.             off += 4; // number of values in A

  1329. #endif

  1330. #endif

  1331.         }

  1332. 

  1333.         if (m & 2)

  1334.         {

  1335. #if defined(TRMMKERNEL)

  1336. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1337.             pb0 = B;

  1338. #else

  1339.             pa0 += off * 2;

  1340.             pb0 = B + off * 4;

  1341. #endif

  1342. 

  1343. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1344.             temp = k - off;

  1345. #elif defined(LEFT)

  1346.             temp = off + 2; // number of values in A

  1347. #else

  1348.             temp = off + 4; // number of values in B

  1349. #endif

  1350. #else

  1351.             pb0 = B;

  1352.             temp = k;

  1353. #endif

  1354. 

  1355.             a0 = pa0[0];

  1356.             b0 = pb0[0];

  1357.             tmp0 = a0 * b0;

  1358. 

  1359.             a1 = pa0[1];

  1360.             tmp1 = a1 * b0;

  1361. 

  1362.             b1 = pb0[1];

  1363.             tmp2 = a0 * b1;

  1364.             tmp3 = a1 * b1;

  1365. 

  1366.             b2 = pb0[2];

  1367.             tmp4 = a0 * b2;

  1368.             tmp5 = a1 * b2;

  1369. 

  1370.             b3 = pb0[3];

  1371.             tmp6 = a0 * b3;

  1372.             tmp7 = a1 * b3;

  1373. 

  1374.             pa0 += 2;

  1375.             pb0 += 4;

  1376. 

  1377.             for (l = ((temp - 1) >> 1); l--;)

  1378.             {

  1379.                 a0 = pa0[0];

  1380.                 b0 = pb0[0];

  1381.                 tmp0 += a0 * b0;

  1382. 

  1383.                 a1 = pa0[1];

  1384.                 tmp1 += a1 * b0;

  1385. 

  1386.                 b1 = pb0[1];

  1387.                 tmp2  += a0 * b1;

  1388.                 tmp3  += a1 * b1;

  1389. 

  1390.                 b2 = pb0[2];

  1391.                 tmp4 += a0 * b2;

  1392.                 tmp5 += a1 * b2;

  1393. 

  1394.                 b3 = pb0[3];

  1395.                 tmp6 += a0 * b3;

  1396.                 tmp7 += a1 * b3;

  1397. 

  1398.                 pa0 += 2;

  1399.                 pb0 += 4;

  1400. 

  1401.                 a0 = pa0[0];

  1402.                 b0 = pb0[0];

  1403.                 tmp0 += a0 * b0;

  1404. 

  1405.                 a1 = pa0[1];

  1406.                 tmp1 += a1 * b0;

  1407. 

  1408.                 b1 = pb0[1];

  1409.                 tmp2  += a0 * b1;

  1410.                 tmp3  += a1 * b1;

  1411. 

  1412.                 b2 = pb0[2];

  1413.                 tmp4 += a0 * b2;

  1414.                 tmp5 += a1 * b2;

  1415. 

  1416.                 b3 = pb0[3];

  1417.                 tmp6 += a0 * b3;

  1418.                 tmp7 += a1 * b3;

  1419. 

  1420.                 pa0 += 2;

  1421.                 pb0 += 4;

  1422.             }

  1423. 

  1424.             if ((temp - 1) & 1)

  1425.             {

  1426.                 a0 = pa0[0];

  1427.                 b0 = pb0[0];

  1428.                 tmp0 += a0 * b0;

  1429. 

  1430.                 a1 = pa0[1];

  1431.                 tmp1 += a1 * b0;

  1432. 

  1433.                 b1 = pb0[1];

  1434.                 tmp2  += a0 * b1;

  1435.                 tmp3  += a1 * b1;

  1436. 

  1437.                 b2 = pb0[2];

  1438.                 tmp4 += a0 * b2;

  1439.                 tmp5 += a1 * b2;

  1440. 

  1441.                 b3 = pb0[3];

  1442.                 tmp6 += a0 * b3;

  1443.                 tmp7 += a1 * b3;

  1444. 

  1445.                 pa0 += 2;

  1446.                 pb0 += 4;

  1447.             }

  1448. 

  1449.             tmp0 = alpha * tmp0;

  1450.             tmp2 = alpha * tmp2;

  1451.             tmp4 = alpha * tmp4;

  1452.             tmp6 = alpha * tmp6;

  1453. 

  1454. #if defined(TRMMKERNEL)

  1455.             pc0[0] = tmp0;

  1456.             pc1[0] = tmp2;

  1457.             pc2[0] = tmp4;

  1458.             pc3[0] = tmp6;

  1459. #else

  1460.             pc0[0] += tmp0;

  1461.             pc1[0] += tmp2;

  1462.             pc2[0] += tmp4;

  1463.             pc3[0] += tmp6;

  1464. #endif

  1465.             tmp1 = alpha * tmp1;

  1466.             tmp3 = alpha * tmp3;

  1467.             tmp5 = alpha * tmp5;

  1468.             tmp7 = alpha * tmp7;

  1469. 

  1470. #if defined(TRMMKERNEL)

  1471.             pc0[1] = tmp1;

  1472.             pc1[1] = tmp3;

  1473.             pc2[1] = tmp5;

  1474.             pc3[1] = tmp7;

  1475. #else

  1476.             pc0[1] += tmp1;

  1477.             pc1[1] += tmp3;

  1478.             pc2[1] += tmp5;

  1479.             pc3[1] += tmp7;

  1480. #endif

  1481.             pc0 += 2;

  1482.             pc1 += 2;

  1483.             pc2 += 2;

  1484.             pc3 += 2;

  1485. 

  1486. #if defined(TRMMKERNEL)

  1487. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1488.             temp = k - off;

  1489. #ifdef LEFT

  1490.             temp -= 2; // number of values in A

  1491. #else

  1492.             temp -= 4; // number of values in B

  1493. #endif

  1494.             pa0 += temp * 2;

  1495.             pb0 += temp * 4;

  1496. #endif

  1497. 

  1498. #ifdef LEFT

  1499.             off += 2; // number of values in A

  1500. #endif

  1501. #endif

  1502.         }

  1503. 

  1504.         if (m & 1)

  1505.         {

  1506. #if defined(TRMMKERNEL)

  1507. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1508.             pb0 = B;

  1509. #else

  1510.             pa0 += off * 1;

  1511.             pb0 = B + off * 4;

  1512. #endif

  1513. 

  1514. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1515.             temp = k - off;

  1516. #elif defined(LEFT)

  1517.             temp = off + 1; // number of values in A

  1518. #else

  1519.             temp = off + 4; // number of values in B

  1520. #endif

  1521. #else

  1522.             pb0 = B;

  1523.             temp = k;

  1524. #endif

  1525. 

  1526.             a0 = pa0[0];

  1527.             b0 = pb0[0];

  1528.             tmp0 = a0 * b0;

  1529. 

  1530.             b1 = pb0[1];

  1531.             tmp1 = a0 * b1;

  1532. 

  1533.             b2 = pb0[2];

  1534.             tmp2 = a0 * b2;

  1535. 

  1536.             b3 = pb0[3];

  1537.             tmp3 = a0 * b3;

  1538. 

  1539.             pa0 += 1;

  1540.             pb0 += 4;

  1541. 

  1542.             for (l = ((temp - 1) >> 1); l--;)

  1543.             {

  1544.                 a0 = pa0[0];

  1545.                 b0 = pb0[0];

  1546.                 tmp0 += a0 * b0;

  1547. 

  1548.                 b1 = pb0[1];

  1549.                 tmp1  += a0 * b1;

  1550. 

  1551.                 b2 = pb0[2];

  1552.                 tmp2 += a0 * b2;

  1553. 

  1554.                 b3 = pb0[3];

  1555.                 tmp3 += a0 * b3;

  1556. 

  1557.                 pa0 += 1;

  1558.                 pb0 += 4;

  1559. 

  1560.                 a0 = pa0[0];

  1561.                 b0 = pb0[0];

  1562.                 tmp0 += a0 * b0;

  1563. 

  1564.                 b1 = pb0[1];

  1565.                 tmp1  += a0 * b1;

  1566. 

  1567.                 b2 = pb0[2];

  1568.                 tmp2 += a0 * b2;

  1569. 

  1570.                 b3 = pb0[3];

  1571.                 tmp3 += a0 * b3;

  1572. 

  1573.                 pa0 += 1;

  1574.                 pb0 += 4;

  1575.             }

  1576. 

  1577.             if ((temp - 1) & 1)

  1578.             {

  1579.                 a0 = pa0[0];

  1580.                 b0 = pb0[0];

  1581.                 tmp0 += a0 * b0;

  1582. 

  1583.                 b1 = pb0[1];

  1584.                 tmp1  += a0 * b1;

  1585. 

  1586.                 b2 = pb0[2];

  1587.                 tmp2 += a0 * b2;

  1588. 

  1589.                 b3 = pb0[3];

  1590.                 tmp3 += a0 * b3;

  1591. 

  1592.                 pa0 += 1;

  1593.                 pb0 += 4;

  1594.             }

  1595. 

  1596.             tmp0 = alpha * tmp0;

  1597.             tmp1 = alpha * tmp1;

  1598.             tmp2 = alpha * tmp2;

  1599.             tmp3 = alpha * tmp3;

  1600. 

  1601. #if defined(TRMMKERNEL)

  1602.             pc0[0] = tmp0;

  1603.             pc1[0] = tmp1;

  1604.             pc2[0] = tmp2;

  1605.             pc3[0] = tmp3;

  1606. #else

  1607.             pc0[0] += tmp0;

  1608.             pc1[0] += tmp1;

  1609.             pc2[0] += tmp2;

  1610.             pc3[0] += tmp3;

  1611. #endif

  1612.             pc0 += 1;

  1613.             pc1 += 1;

  1614.             pc2 += 1;

  1615.             pc3 += 1;

  1616. 

  1617. #if defined(TRMMKERNEL)

  1618. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1619.             temp = k - off;

  1620. #ifdef LEFT

  1621.             temp -= 1; // number of values in A

  1622. #else

  1623.             temp -= 4; // number of values in B

  1624. #endif

  1625.             pa0 += temp * 1;

  1626.             pb0 += temp * 4;

  1627. #endif

  1628. 

  1629. #ifdef LEFT

  1630.             off += 1; // number of values in A

  1631. #endif

  1632. #endif

  1633.         }

  1634. 

  1635. #if defined(TRMMKERNEL) && !defined(LEFT)

  1636.         off += 4; // number of values in A

  1637. #endif

  1638. 

  1639.         B += (k << 2);

  1640.         C += (ldc << 2);

  1641.     }

  1642. 

  1643.     if (n & 2)

  1644.     {

  1645.         pc0 = C;

  1646.         pc1 = pc0 + ldc;

  1647. 

  1648. #if defined(TRMMKERNEL) && defined(LEFT)

  1649.         off = offset;

  1650. #endif

  1651. 

  1652.         pa0 = A;

  1653. 

  1654.         for (i = (m >> 3); i--;)

  1655.         {

  1656. #if defined(TRMMKERNEL)

  1657. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1658.             pb0 = B;

  1659. #else

  1660.             pa0 += off * 8;

  1661.             pb0 = B + off * 2;

  1662. #endif

  1663. 

  1664. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1665.             temp = k - off;

  1666. #elif defined(LEFT)

  1667.             temp = off + 8; // number of values in A

  1668. #else

  1669.             temp = off + 2; // number of values in B

  1670. #endif

  1671. #else

  1672.             pb0 = B;

  1673.             temp = k;

  1674. #endif

  1675. 

  1676.             LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1677.             src_b0[0] = pb0[0];

  1678.             src_b0[1] = pb0[1];

  1679. 

  1680.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1681.             res0 = src_a0 * src_b;

  1682.             res1 = src_a1 * src_b;

  1683. 

  1684.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1685.             res2 = src_a0 * src_b;

  1686.             res3 = src_a1 * src_b;

  1687. 

  1688.             pb0 += 2;

  1689. 

  1690.             for (l = ((temp - 1) >> 1); l--;)

  1691.             {

  1692.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1693.                 src_b0[0] = pb0[0];

  1694.                 src_b0[1] = pb0[1];

  1695. 

  1696.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1697.                 res0 += src_a0 * src_b;

  1698.                 res1 += src_a1 * src_b;

  1699. 

  1700.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1701.                 res2 += src_a0 * src_b;

  1702.                 res3 += src_a1 * src_b;

  1703. 

  1704.                 pb0 += 2;

  1705. 

  1706.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1707.                 src_b0[0] = pb0[0];

  1708.                 src_b0[1] = pb0[1];

  1709. 

  1710.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1711.                 res0 += src_a0 * src_b;

  1712.                 res1 += src_a1 * src_b;

  1713. 

  1714.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1715.                 res2 += src_a0 * src_b;

  1716.                 res3 += src_a1 * src_b;

  1717. 

  1718.                 pb0 += 2;

  1719.             }

  1720. 

  1721.             if ((temp - 1) & 1)

  1722.             {

  1723.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  1724.                 src_b0[0] = pb0[0];

  1725.                 src_b0[1] = pb0[1];

  1726. 

  1727.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1728.                 res0 += src_a0 * src_b;

  1729.                 res1 += src_a1 * src_b;

  1730. 

  1731.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1732.                 res2 += src_a0 * src_b;

  1733.                 res3 += src_a1 * src_b;

  1734. 

  1735.                 pb0 += 2;

  1736.             }

  1737. 

  1738. #if defined(TRMMKERNEL)

  1739.             dst0 = res0 * v_alpha;

  1740.             dst1 = res1 * v_alpha;

  1741.             dst2 = res2 * v_alpha;

  1742.             dst3 = res3 * v_alpha;

  1743. #else

  1744.             LD_SP2(pc0, 4, dst0, dst1);

  1745.             LD_SP2(pc1, 4, dst2, dst3);

  1746. 

  1747.             dst0 += res0 * v_alpha;

  1748.             dst1 += res1 * v_alpha;

  1749.             dst2 += res2 * v_alpha;

  1750.             dst3 += res3 * v_alpha;

  1751. #endif

  1752.             ST_SP2_INC(dst0, dst1, pc0, 4);

  1753.             ST_SP2_INC(dst2, dst3, pc1, 4);

  1754. 

  1755. #if defined(TRMMKERNEL)

  1756. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1757.             temp = k - off;

  1758. #ifdef LEFT

  1759.             temp -= 8; // number of values in A

  1760. #else

  1761.             temp -= 2; // number of values in B

  1762. #endif

  1763.             pa0 += temp * 8;

  1764.             pb0 += temp * 2;

  1765. #endif

  1766. 

  1767. #ifdef LEFT

  1768.             off += 8; // number of values in A

  1769. #endif

  1770. #endif

  1771.         }

  1772. 

  1773.         if (m & 4)

  1774.         {

  1775. #if defined(TRMMKERNEL)

  1776. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1777.             pb0 = B;

  1778. #else

  1779.             pa0 += off * 4;

  1780.             pb0 = B + off * 2;

  1781. #endif

  1782. 

  1783. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1784.             temp = k - off;

  1785. #elif defined(LEFT)

  1786.             temp = off + 4; // number of values in A

  1787. #else

  1788.             temp = off + 2; // number of values in B

  1789. #endif

  1790. #else

  1791.             pb0 = B;

  1792.             temp = k;

  1793. #endif

  1794. 

  1795.             src_a0 = LD_SP(pa0);

  1796.             src_b0[0] = pb0[0];

  1797.             src_b0[1] = pb0[1];

  1798. 

  1799.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1800.             res0 = src_a0 * src_b;

  1801. 

  1802.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1803.             res1 = src_a0 * src_b;

  1804. 

  1805.             pa0 += 4;

  1806.             pb0 += 2;

  1807. 

  1808.             for (l = ((temp - 1) >> 1); l--;)

  1809.             {

  1810.                 src_a0 = LD_SP(pa0);

  1811.                 src_b0[0] = pb0[0];

  1812.                 src_b0[1] = pb0[1];

  1813. 

  1814.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1815.                 res0 += src_a0 * src_b;

  1816. 

  1817.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1818.                 res1 += src_a0 * src_b;

  1819. 

  1820.                 pa0 += 4;

  1821.                 pb0 += 2;

  1822. 

  1823.                 src_a0 = LD_SP(pa0);

  1824.                 src_b0[0] = pb0[0];

  1825.                 src_b0[1] = pb0[1];

  1826. 

  1827.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1828.                 res0 += src_a0 * src_b;

  1829. 

  1830.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1831.                 res1 += src_a0 * src_b;

  1832. 

  1833.                 pa0 += 4;

  1834.                 pb0 += 2;

  1835.             }

  1836. 

  1837.             if ((temp - 1) & 1)

  1838.             {

  1839.                 src_a0 = LD_SP(pa0);

  1840.                 src_b0[0] = pb0[0];

  1841.                 src_b0[1] = pb0[1];

  1842. 

  1843.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  1844.                 res0 += src_a0 * src_b;

  1845. 

  1846.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0x55);

  1847.                 res1 += src_a0 * src_b;

  1848. 

  1849.                 pa0 += 4;

  1850.                 pb0 += 2;

  1851.             }

  1852. 

  1853. #if defined(TRMMKERNEL)

  1854.             dst0 = res0 * v_alpha;

  1855.             dst1 = res1 * v_alpha;

  1856. #else

  1857.             dst0 = LD_SP(pc0);

  1858.             dst1 = LD_SP(pc1);

  1859. 

  1860.             dst0 += res0 * v_alpha;

  1861.             dst1 += res1 * v_alpha;

  1862. #endif

  1863.             ST_SP(dst0, pc0);

  1864.             ST_SP(dst1, pc1);

  1865. 

  1866.             pc0 += 4;

  1867.             pc1 += 4;

  1868. 

  1869. #if defined(TRMMKERNEL)

  1870. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1871.             temp = k - off;

  1872. #ifdef LEFT

  1873.             temp -= 4; // number of values in A

  1874. #else

  1875.             temp -= 2; // number of values in B

  1876. #endif

  1877.             pa0 += temp * 4;

  1878.             pb0 += temp * 2;

  1879. #endif

  1880. 

  1881. #ifdef LEFT

  1882.             off += 4; // number of values in A

  1883. #endif

  1884. #endif

  1885.         }

  1886. 

  1887.         if (m & 2)

  1888.         {

  1889. #if defined(TRMMKERNEL)

  1890. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1891.             pb0 = B;

  1892. #else

  1893.             pa0 += off * 2;

  1894.             pb0 = B + off * 2;

  1895. #endif

  1896. 

  1897. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1898.             temp = k - off;

  1899. #elif defined(LEFT)

  1900.             temp = off + 2; // number of values in A

  1901. #else

  1902.             temp = off + 2; // number of values in B

  1903. #endif

  1904. #else

  1905.             pb0 = B;

  1906.             temp = k;

  1907. #endif

  1908. 

  1909.             a0 = pa0[0];

  1910.             b0 = pb0[0];

  1911.             tmp0 = a0 * b0;

  1912. 

  1913.             a1 = pa0[1];

  1914.             tmp1 = a1 * b0;

  1915. 

  1916.             b1 = pb0[1];

  1917.             tmp2 = a0 * b1;

  1918.             tmp3 = a1 * b1;

  1919. 

  1920.             pa0 += 2;

  1921.             pb0 += 2;

  1922. 

  1923.             for (l = ((temp - 1) >> 1); l--;)

  1924.             {

  1925.                 a0 = pa0[0];

  1926.                 b0 = pb0[0];

  1927.                 tmp0 += a0 * b0;

  1928. 

  1929.                 a1 = pa0[1];

  1930.                 tmp1 += a1 * b0;

  1931. 

  1932.                 b1 = pb0[1];

  1933.                 tmp2  += a0 * b1;

  1934.                 tmp3  += a1 * b1;

  1935. 

  1936.                 pa0 += 2;

  1937.                 pb0 += 2;

  1938. 

  1939.                 a0 = pa0[0];

  1940.                 b0 = pb0[0];

  1941.                 tmp0 += a0 * b0;

  1942. 

  1943.                 a1 = pa0[1];

  1944.                 tmp1 += a1 * b0;

  1945. 

  1946.                 b1 = pb0[1];

  1947.                 tmp2  += a0 * b1;

  1948.                 tmp3  += a1 * b1;

  1949. 

  1950.                 pa0 += 2;

  1951.                 pb0 += 2;

  1952.             }

  1953. 

  1954.             if ((temp - 1) & 1)

  1955.             {

  1956.                 a0 = pa0[0];

  1957.                 b0 = pb0[0];

  1958.                 tmp0 += a0 * b0;

  1959. 

  1960.                 a1 = pa0[1];

  1961.                 tmp1 += a1 * b0;

  1962. 

  1963.                 b1 = pb0[1];

  1964.                 tmp2  += a0 * b1;

  1965.                 tmp3  += a1 * b1;

  1966. 

  1967.                 pa0 += 2;

  1968.                 pb0 += 2;

  1969.             }

  1970. 

  1971.             tmp0 = alpha * tmp0;

  1972.             tmp1 = alpha * tmp1;

  1973.             tmp2 = alpha * tmp2;

  1974.             tmp3 = alpha * tmp3;

  1975. 

  1976. #if defined(TRMMKERNEL)

  1977.             pc0[0] = tmp0;

  1978.             pc1[0] = tmp2;

  1979.             pc0[1] = tmp1;

  1980.             pc1[1] = tmp3;

  1981. #else

  1982.             pc0[0] += tmp0;

  1983.             pc1[0] += tmp2;

  1984.             pc0[1] += tmp1;

  1985.             pc1[1] += tmp3;

  1986. #endif

  1987.             pc0 += 2;

  1988.             pc1 += 2;

  1989. 

  1990. #if defined(TRMMKERNEL)

  1991. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1992.             temp = k - off;

  1993. #ifdef LEFT

  1994.             temp -= 2; // number of values in A

  1995. #else

  1996.             temp -= 2; // number of values in B

  1997. #endif

  1998.             pa0 += temp * 2;

  1999.             pb0 += temp * 2;

  2000. #endif

  2001. 

  2002. #ifdef LEFT

  2003.             off += 2; // number of values in A

  2004. #endif

  2005. #endif

  2006.         }

  2007. 

  2008.         if (m & 1)

  2009.         {

  2010. #if defined(TRMMKERNEL)

  2011. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2012.             pb0 = B;

  2013. #else

  2014.             pa0 += off * 1;

  2015.             pb0 = B + off * 2;

  2016. #endif

  2017. 

  2018. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2019.             temp = k - off;

  2020. #elif defined(LEFT)

  2021.             temp = off + 1; // number of values in A

  2022. #else

  2023.             temp = off + 2; // number of values in B

  2024. #endif

  2025. #else

  2026.             pb0 = B;

  2027.             temp = k;

  2028. #endif

  2029. 

  2030.             a0 = pa0[0];

  2031.             b0 = pb0[0];

  2032.             tmp0 = a0 * b0;

  2033. 

  2034.             b1 = pb0[1];

  2035.             tmp1 = a0 * b1;

  2036. 

  2037.             pa0 += 1;

  2038.             pb0 += 2;

  2039. 

  2040.             for (l = ((temp - 1) >> 1); l--;)

  2041.             {

  2042.                 a0 = pa0[0];

  2043.                 b0 = pb0[0];

  2044.                 tmp0 += a0 * b0;

  2045. 

  2046.                 b1 = pb0[1];

  2047.                 tmp1  += a0 * b1;

  2048. 

  2049.                 pa0 += 1;

  2050.                 pb0 += 2;

  2051. 

  2052.                 a0 = pa0[0];

  2053.                 b0 = pb0[0];

  2054.                 tmp0 += a0 * b0;

  2055. 

  2056.                 b1 = pb0[1];

  2057.                 tmp1  += a0 * b1;

  2058. 

  2059.                 pa0 += 1;

  2060.                 pb0 += 2;

  2061.             }

  2062. 

  2063.             if ((temp - 1) & 1)

  2064.             {

  2065.                 a0 = pa0[0];

  2066.                 b0 = pb0[0];

  2067.                 tmp0 += a0 * b0;

  2068. 

  2069.                 b1 = pb0[1];

  2070.                 tmp1  += a0 * b1;

  2071. 

  2072.                 pa0 += 1;

  2073.                 pb0 += 2;

  2074.             }

  2075. 

  2076.             tmp0 = alpha * tmp0;

  2077.             tmp1 = alpha * tmp1;

  2078. 

  2079. #if defined(TRMMKERNEL)

  2080.             pc0[0] = tmp0;

  2081.             pc1[0] = tmp1;

  2082. #else

  2083.             pc0[0] += tmp0;

  2084.             pc1[0] += tmp1;

  2085. #endif

  2086.             pc0 += 1;

  2087.             pc1 += 1;

  2088. 

  2089. #if defined(TRMMKERNEL)

  2090. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2091.             temp = k - off;

  2092. #ifdef LEFT

  2093.             temp -= 1; // number of values in A

  2094. #else

  2095.             temp -= 2; // number of values in B

  2096. #endif

  2097.             pa0 += temp * 1;

  2098.             pb0 += temp * 2;

  2099. #endif

  2100. 

  2101. #ifdef LEFT

  2102.             off += 1; // number of values in A

  2103. #endif

  2104. #endif

  2105.         }

  2106. 

  2107. #if defined(TRMMKERNEL) && !defined(LEFT)

  2108.         off += 2; // number of values in A

  2109. #endif

  2110. 

  2111.         B += (k << 1);

  2112.         C += (ldc << 1);

  2113.     }

  2114. 

  2115.     if (n & 1)

  2116.     {

  2117.         pc0 = C;

  2118. 

  2119. #if defined(TRMMKERNEL) && defined(LEFT)

  2120.         off = offset;

  2121. #endif

  2122. 

  2123.         pa0 = A;

  2124. 

  2125.         for (i = (m >> 3); i--;)

  2126.         {

  2127. #if defined(TRMMKERNEL)

  2128. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2129.             pb0 = B;

  2130. #else

  2131.             pa0 += off * 8;

  2132.             pb0 = B + off * 1;

  2133. #endif

  2134. 

  2135. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2136.             temp = k - off;

  2137. #elif defined(LEFT)

  2138.             temp = off + 8; // number of values in A

  2139. #else

  2140.             temp = off + 1; // number of values in B

  2141. #endif

  2142. #else

  2143.             pb0 = B;

  2144.             temp = k;

  2145. #endif

  2146. 

  2147.             LD_SP2_INC(pa0, 4, src_a0, src_a1);

  2148.             src_b0[0] = pb0[0];

  2149. 

  2150.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2151.             res0 = src_a0 * src_b;

  2152.             res1 = src_a1 * src_b;

  2153. 

  2154.             pb0 += 1;

  2155. 

  2156.             for (l = ((temp - 1) >> 1); l--;)

  2157.             {

  2158.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  2159.                 src_b0[0] = pb0[0];

  2160. 

  2161.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2162.                 res0 += src_a0 * src_b;

  2163.                 res1 += src_a1 * src_b;

  2164. 

  2165.                 pb0 += 1;

  2166. 

  2167.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  2168.                 src_b0[0] = pb0[0];

  2169. 

  2170.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2171.                 res0 += src_a0 * src_b;

  2172.                 res1 += src_a1 * src_b;

  2173. 

  2174.                 pb0 += 1;

  2175.             }

  2176. 

  2177.             if ((temp - 1) & 1)

  2178.             {

  2179.                 LD_SP2_INC(pa0, 4, src_a0, src_a1);

  2180.                 src_b0[0] = pb0[0];

  2181. 

  2182.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2183.                 res0 += src_a0 * src_b;

  2184.                 res1 += src_a1 * src_b;

  2185. 

  2186.                 pb0 += 1;

  2187.             }

  2188. 

  2189. #if defined(TRMMKERNEL)

  2190.             dst0 = res0 * v_alpha;

  2191.             dst1 = res1 * v_alpha;

  2192. #else

  2193.             LD_SP2(pc0, 4, dst0, dst1);

  2194. 

  2195.             dst0 += res0 * v_alpha;

  2196.             dst1 += res1 * v_alpha;

  2197. #endif

  2198.             ST_SP2_INC(dst0, dst1, pc0, 4);

  2199. 

  2200. #if defined(TRMMKERNEL)

  2201. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2202.             temp = k - off;

  2203. #ifdef LEFT

  2204.             temp -= 8; // number of values in A

  2205. #else

  2206.             temp -= 1; // number of values in B

  2207. #endif

  2208.             pa0 += temp * 8;

  2209.             pb0 += temp * 1;

  2210. #endif

  2211. 

  2212. #ifdef LEFT

  2213.             off += 8; // number of values in A

  2214. #endif

  2215. #endif

  2216.         }

  2217. 

  2218.         if (m & 4)

  2219.         {

  2220. #if defined(TRMMKERNEL)

  2221. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2222.             pb0 = B;

  2223. #else

  2224.             pa0 += off * 4;

  2225.             pb0 = B + off * 1;

  2226. #endif

  2227. 

  2228. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2229.             temp = k - off;

  2230. #elif defined(LEFT)

  2231.             temp = off + 4; // number of values in A

  2232. #else

  2233.             temp = off + 1; // number of values in B

  2234. #endif

  2235. #else

  2236.             pb0 = B;

  2237.             temp = k;

  2238. #endif

  2239. 

  2240.             src_a0 = LD_SP(pa0);

  2241.             src_b0[0] = pb0[0];

  2242. 

  2243.             src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2244.             res0 = src_a0 * src_b;

  2245. 

  2246.             pa0 += 4;

  2247.             pb0 += 1;

  2248. 

  2249.             for (l = ((temp - 1) >> 1); l--;)

  2250.             {

  2251.                 src_a0 = LD_SP(pa0);

  2252.                 src_b0[0] = pb0[0];

  2253. 

  2254.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2255.                 res0 += src_a0 * src_b;

  2256. 

  2257.                 pa0 += 4;

  2258.                 pb0 += 1;

  2259. 

  2260.                 src_a0 = LD_SP(pa0);

  2261.                 src_b0[0] = pb0[0];

  2262. 

  2263.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2264.                 res0 += src_a0 * src_b;

  2265. 

  2266.                 pa0 += 4;

  2267.                 pb0 += 1;

  2268.             }

  2269. 

  2270.             if ((temp - 1) & 1)

  2271.             {

  2272.                 src_a0 = LD_SP(pa0);

  2273.                 src_b0[0] = pb0[0];

  2274. 

  2275.                 src_b = (v4f32) __msa_shf_w((v4i32) src_b0, 0);

  2276.                 res0 += src_a0 * src_b;

  2277. 

  2278.                 pa0 += 4;

  2279.                 pb0 += 1;

  2280.             }

  2281. 

  2282. #if defined(TRMMKERNEL)

  2283.             dst0 = res0 * v_alpha;

  2284. #else

  2285.             dst0 = LD_SP(pc0);

  2286. 

  2287.             dst0 += res0 * v_alpha;

  2288. #endif

  2289.             ST_SP(dst0, pc0);

  2290. 

  2291.             pc0 += 4;

  2292. 

  2293. #if defined(TRMMKERNEL)

  2294. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2295.             temp = k - off;

  2296. #ifdef LEFT

  2297.             temp -= 4; // number of values in A

  2298. #else

  2299.             temp -= 1; // number of values in B

  2300. #endif

  2301.             pa0 += temp * 4;

  2302.             pb0 += temp * 1;

  2303. #endif

  2304. 

  2305. #ifdef LEFT

  2306.             off += 4; // number of values in A

  2307. #endif

  2308. #endif

  2309.         }

  2310. 

  2311.         if (m & 2)

  2312.         {

  2313. #if defined(TRMMKERNEL)

  2314. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2315.             pb0 = B;

  2316. #else

  2317.             pa0 += off * 2;

  2318.             pb0 = B + off * 1;

  2319. #endif

  2320. 

  2321. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2322.             temp = k - off;

  2323. #elif defined(LEFT)

  2324.             temp = off + 2; // number of values in A

  2325. #else

  2326.             temp = off + 1; // number of values in B

  2327. #endif

  2328. #else

  2329.             pb0 = B;

  2330.             temp = k;

  2331. #endif

  2332. 

  2333.             a0 = pa0[0];

  2334.             b0 = pb0[0];

  2335.             tmp0 = a0 * b0;

  2336. 

  2337.             a1 = pa0[1];

  2338.             tmp1 = a1 * b0;

  2339. 

  2340.             pa0 += 2;

  2341.             pb0 += 1;

  2342. 

  2343.             for (l = ((temp - 1) >> 1); l--;)

  2344.             {

  2345.                 a0 = pa0[0];

  2346.                 b0 = pb0[0];

  2347.                 tmp0 += a0 * b0;

  2348. 

  2349.                 a1 = pa0[1];

  2350.                 tmp1 += a1 * b0;

  2351. 

  2352.                 pa0 += 2;

  2353.                 pb0 += 1;

  2354. 

  2355.                 a0 = pa0[0];

  2356.                 b0 = pb0[0];

  2357.                 tmp0 += a0 * b0;

  2358. 

  2359.                 a1 = pa0[1];

  2360.                 tmp1 += a1 * b0;

  2361. 

  2362.                 pa0 += 2;

  2363.                 pb0 += 1;

  2364.             }

  2365. 

  2366.             if ((temp - 1) & 1)

  2367.             {

  2368.                 a0 = pa0[0];

  2369.                 b0 = pb0[0];

  2370.                 tmp0 += a0 * b0;

  2371. 

  2372.                 a1 = pa0[1];

  2373.                 tmp1 += a1 * b0;

  2374. 

  2375.                 pa0 += 2;

  2376.                 pb0 += 1;

  2377.             }

  2378. 

  2379.             tmp0 = alpha * tmp0;

  2380.             tmp1 = alpha * tmp1;

  2381. 

  2382. #if defined(TRMMKERNEL)

  2383.             pc0[0] = tmp0;

  2384.             pc0[1] = tmp1;

  2385. #else

  2386.             pc0[0] += tmp0;

  2387.             pc0[1] += tmp1;

  2388. #endif

  2389.             pc0 += 2;

  2390. 

  2391. #if defined(TRMMKERNEL)

  2392. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2393.             temp = k - off;

  2394. #ifdef LEFT

  2395.             temp -= 2; // number of values in A

  2396. #else

  2397.             temp -= 1; // number of values in B

  2398. #endif

  2399.             pa0 += temp * 2;

  2400.             pb0 += temp * 1;

  2401. #endif

  2402. 

  2403. #ifdef LEFT

  2404.             off += 2; // number of values in A

  2405. #endif

  2406. #endif

  2407.         }

  2408. 

  2409.         if (m & 1)

  2410.         {

  2411. #if defined(TRMMKERNEL)

  2412. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2413.             pb0 = B;

  2414. #else

  2415.             pa0 += off * 1;

  2416.             pb0 = B + off * 1;

  2417. #endif

  2418. 

  2419. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2420.             temp = k - off;

  2421. #elif defined(LEFT)

  2422.             temp = off + 1; // number of values in A

  2423. #else

  2424.             temp = off + 1; // number of values in B

  2425. #endif

  2426. #else

  2427.             pb0 = B;

  2428.             temp = k;

  2429. #endif

  2430. 

  2431.             a0 = pa0[0];

  2432.             b0 = pb0[0];

  2433.             tmp0 = a0 * b0;

  2434. 

  2435.             pa0 += 1;

  2436.             pb0 += 1;

  2437. 

  2438.             for (l = ((temp - 1) >> 1); l--;)

  2439.             {

  2440.                 a0 = pa0[0];

  2441.                 b0 = pb0[0];

  2442.                 tmp0 += a0 * b0;

  2443. 

  2444.                 pa0 += 1;

  2445.                 pb0 += 1;

  2446. 

  2447.                 a0 = pa0[0];

  2448.                 b0 = pb0[0];

  2449.                 tmp0 += a0 * b0;

  2450. 

  2451.                 pa0 += 1;

  2452.                 pb0 += 1;

  2453.             }

  2454. 

  2455.             if ((temp - 1) & 1)

  2456.             {

  2457.                 a0 = pa0[0];

  2458.                 b0 = pb0[0];

  2459.                 tmp0 += a0 * b0;

  2460. 

  2461.                 pa0 += 1;

  2462.                 pb0 += 1;

  2463.             }

  2464. 

  2465. #if defined(TRMMKERNEL)

  2466.             pc0[0] = alpha * tmp0;

  2467. #else

  2468.             pc0[0] += alpha * tmp0;

  2469. #endif

  2470.         }

  2471.     }

  2472. 

  2473.     return 0;

  2474. }