OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 571eadb880
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '571eadb880'
${ noResults }
OpenBLAS/kernel/power/dgemm_kernel_power10.c

865 lines
23 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2020, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. #include "common.h"

  28. #include <altivec.h>

  29. 

  30. typedef unsigned char vec_t __attribute__ ((vector_size (16)));

  31. typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));

  32. typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));

  33. 

  34. #ifdef TRMMKERNEL

  35. #define SAVE_ACC(ACC, J)  \

  36.           __builtin_mma_disassemble_acc (result, ACC); \

  37.           rowC = (v4sf_t *) &CO[0* ldc+J]; \

  38.           rowC[0] = result[3] * alpha; \

  39.           rowC = (v4sf_t *) &CO[1*ldc+J]; \

  40.           rowC[0] = result[2] * alpha; \

  41.           rowC = (v4sf_t *) &CO[2*ldc+J]; \

  42.           rowC[0] = result[1] * alpha; \

  43.           rowC = (v4sf_t *) &CO[3*ldc+J]; \

  44.           rowC[0] = result[0] * alpha;

  45. #define SAVE_ACC1(ACC, J)  \

  46.           __builtin_mma_disassemble_acc (result, ACC); \

  47.           rowC = (v4sf_t *) &CO[4* ldc+J]; \

  48.           rowC[0] = result[3] * alpha; \

  49.           rowC = (v4sf_t *) &CO[5*ldc+J]; \

  50.           rowC[0] = result[2] * alpha; \

  51.           rowC = (v4sf_t *) &CO[6*ldc+J]; \

  52.           rowC[0] = result[1] * alpha; \

  53.           rowC = (v4sf_t *) &CO[7*ldc+J]; \

  54.           rowC[0] = result[0] * alpha;

  55. #define  SAVE2x4_ACC(ACC, J)  \

  56. 	  __builtin_mma_disassemble_acc (result, ACC); \

  57. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  58.           rowC[0] = result[3] * alpha; \

  59. 	  rowC = (v4sf_t *) &CO[1* ldc+J]; \

  60.           rowC[0] = result[2] * alpha;

  61. #else

  62. #define SAVE_ACC(ACC, J)  \

  63.           __builtin_mma_disassemble_acc (result, ACC); \

  64.           rowC = (v4sf_t *) &CO[0* ldc+J]; \

  65.           rowC[0] += result[3] * alpha; \

  66.           rowC = (v4sf_t *) &CO[1*ldc+J]; \

  67.           rowC[0] += result[2] * alpha; \

  68.           rowC = (v4sf_t *) &CO[2*ldc+J]; \

  69.           rowC[0] += result[1] * alpha; \

  70.           rowC = (v4sf_t *) &CO[3*ldc+J]; \

  71.           rowC[0] += result[0] * alpha;

  72. #define SAVE_ACC1(ACC, J)  \

  73.           __builtin_mma_disassemble_acc (result, ACC); \

  74.           rowC = (v4sf_t *) &CO[4* ldc+J]; \

  75.           rowC[0] += result[3] * alpha; \

  76.           rowC = (v4sf_t *) &CO[5*ldc+J]; \

  77.           rowC[0] += result[2] * alpha; \

  78.           rowC = (v4sf_t *) &CO[6*ldc+J]; \

  79.           rowC[0] += result[1] * alpha; \

  80.           rowC = (v4sf_t *) &CO[7*ldc+J]; \

  81.           rowC[0] += result[0] * alpha;

  82. #define  SAVE2x4_ACC(ACC, J)  \

  83. 	  __builtin_mma_disassemble_acc (result, ACC); \

  84. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  85.           rowC[0] += result[3] * alpha; \

  86. 	  rowC = (v4sf_t *) &CO[1* ldc+J]; \

  87.           rowC[0] += result[2] * alpha;

  88. #endif

  89. 

  90. #define SET_ACC_ZERO4() \

  91.           __builtin_mma_xxsetaccz (&acc0); \

  92.           __builtin_mma_xxsetaccz (&acc1); \

  93.           __builtin_mma_xxsetaccz (&acc2); \

  94.           __builtin_mma_xxsetaccz (&acc3);

  95. 

  96. #define SET_ACC_ZERO8() \

  97.           __builtin_mma_xxsetaccz (&acc0); \

  98.           __builtin_mma_xxsetaccz (&acc1); \

  99.           __builtin_mma_xxsetaccz (&acc2); \

  100.           __builtin_mma_xxsetaccz (&acc3); \

  101.           __builtin_mma_xxsetaccz (&acc4); \

  102.           __builtin_mma_xxsetaccz (&acc5); \

  103.           __builtin_mma_xxsetaccz (&acc6); \

  104.           __builtin_mma_xxsetaccz (&acc7);

  105. 

  106. #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");

  107. 

  108. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  109. #define REFRESH_TEMP_BK(x, y) \

  110.             temp = k - off;

  111. #elif defined(LEFT)

  112. #define REFRESH_TEMP_BK(x, y) \

  113.             temp = off + x;

  114. #else

  115. #define REFRESH_TEMP_BK(x, y) \

  116.             temp = off + y;

  117. #endif

  118. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  119. #define REFRESH_POINTERS(x, y) \

  120.           BO = B; \

  121.           REFRESH_TEMP_BK(x, y)

  122. #else

  123. #define REFRESH_POINTERS(x, y) \

  124.           AO += off * x; \

  125.           BO = B + off * y; \

  126.           REFRESH_TEMP_BK(x, y)

  127. #endif

  128. 

  129. #ifdef LEFT

  130. #define REFRESH_OFF(x) \

  131.             off += x;

  132. #else

  133. #define REFRESH_OFF(x)

  134. #endif

  135. 

  136. #ifdef LEFT

  137. #define UPDATE_TEMP(x, y) \

  138.             temp -= x;

  139. #else

  140. #define UPDATE_TEMP(x, y) \

  141.             temp -= y;

  142. #endif

  143. 

  144. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  145. #define REFRESH_TMP_AFTER_SAVE(x, y) \

  146.             temp = k - off; \

  147.             UPDATE_TEMP(x, y) \

  148.             AO += temp * x; \

  149.             BO += temp * y;

  150. #else

  151. #define REFRESH_TMP_AFTER_SAVE(x, y)

  152. #endif

  153. 

  154. #define REFRESH_AFTER_SAVE(x,y) \

  155.         REFRESH_TMP_AFTER_SAVE(x, y) \

  156.         REFRESH_OFF(x)

  157. /*************************************************************************************

  158. * GEMM Kernel

  159. *************************************************************************************/

  160. int

  161. CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,

  162.        FLOAT * C, BLASLONG ldc

  163. #ifdef TRMMKERNEL

  164.        , BLASLONG offset

  165. #endif

  166.   )

  167. {

  168.   BLASLONG N = n;

  169.   BLASLONG i1;

  170. #if defined(TRMMKERNEL)

  171.   BLASLONG off;

  172. #endif

  173. #if defined(TRMMKERNEL) && !defined(LEFT)

  174.   off = -offset;

  175. #endif

  176.   v4sf_t valpha = { alpha, alpha };

  177.   N = n >> 2;

  178.   for (i1 = 0; i1 < N; i1++)

  179.     {

  180.       BLASLONG i, j, temp;

  181.       FLOAT *CO;

  182.       FLOAT *AO;

  183. #if defined(TRMMKERNEL) && defined(LEFT)

  184.       off = offset;

  185. #endif

  186.       CO = C;

  187.       C += ldc << 2;

  188.       AO = A;

  189.       PREFETCH1 (A, 128);

  190.       PREFETCH1 (A, 256);

  191.       i = m >> 4;

  192.       for (j = 0; j < i; j++)

  193. 	{

  194.           FLOAT *BO;

  195. #if defined(TRMMKERNEL)

  196.           REFRESH_POINTERS (16, 4);

  197. #else

  198.           BO = B;

  199.           temp = k;

  200. #endif

  201. 	  v4sf_t *rowC;

  202. 	  v4sf_t result[4];

  203. 	  BLASLONG l = 0;

  204. 	  PREFETCH1 (CO, 0);

  205. 	  PREFETCH1 (CO + ldc, 0);

  206. 	  PREFETCH1 (CO + ldc + ldc, 0);

  207. 	  PREFETCH1 (CO + ldc + ldc + ldc, 0);

  208. 	  PREFETCH1 (CO, 128);

  209. 	  PREFETCH1 (CO + ldc, 128);

  210. 	  PREFETCH1 (CO + ldc + ldc, 128);

  211. 	  PREFETCH1 (CO + ldc + ldc + ldc, 128);

  212. 	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  213. 	  SET_ACC_ZERO8 ();

  214. 	  for (l = 0; l < temp; l++)

  215. 	    {

  216. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  217. 	      __vector_pair rowB;

  218. 	      vec_t *rb = (vec_t *) & BO[l << 2];

  219. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  220. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  221. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  222. 	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);

  223. 	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);

  224. 	      __builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);

  225. 	      __builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);

  226. 	      __builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);

  227. 	      __builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);

  228. 	    }

  229. 	  SAVE_ACC (&acc0, 0);

  230. 	  SAVE_ACC (&acc2, 4);

  231. 	  SAVE_ACC (&acc1, 2);

  232. 	  SAVE_ACC (&acc3, 6);

  233. 	  SAVE_ACC (&acc4, 8);

  234. 	  SAVE_ACC (&acc6, 12);

  235. 	  SAVE_ACC (&acc5, 10);

  236. 	  SAVE_ACC (&acc7, 14);

  237. 	  AO += temp << 4;

  238. 	  BO += temp << 2;

  239. #if defined(TRMMKERNEL)

  240.           REFRESH_AFTER_SAVE (16, 4)

  241. #endif

  242. 	  CO += 16;

  243. 	}

  244.       i = (m & 15) >> 3;

  245.       for (j = 0; j < i; j++)

  246. 	{

  247. 	  FLOAT *BO;

  248. #if defined(TRMMKERNEL)

  249.           REFRESH_POINTERS (8, 4);

  250. #else

  251.           BO = B;

  252.           temp = k;

  253. #endif

  254. 	  v4sf_t *rowC;

  255. 	  v4sf_t result[4];

  256. 	  __vector_quad acc0, acc1, acc2, acc3;

  257. 	  SET_ACC_ZERO4 ();

  258. 	  BLASLONG l = 0;

  259. 	  for (l = 0; l < temp; l++)

  260. 	    {

  261. 	      vec_t *rowA = (vec_t *) & AO[l << 3];

  262. 	      __vector_pair rowB;

  263. 	      vec_t *rb = (vec_t *) & BO[l << 2];

  264. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  265. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  266. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  267. 	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);

  268. 	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);

  269. 	    }

  270. 	  SAVE_ACC (&acc0, 0);

  271. 	  SAVE_ACC (&acc2, 4);

  272. 	  SAVE_ACC (&acc1, 2);

  273. 	  SAVE_ACC (&acc3, 6);

  274. 	  CO += 8;

  275. 	  AO += temp << 3;

  276. 	  BO += temp << 2;

  277. #if defined(TRMMKERNEL)

  278.           REFRESH_AFTER_SAVE (8, 4)

  279. #endif

  280. 	}

  281.       i = (m & 7) >> 2;

  282.       for (j = 0; j < i; j++)

  283. 	{

  284. 	  FLOAT *BO;

  285. #if defined(TRMMKERNEL)

  286.           REFRESH_POINTERS (4, 4);

  287. #else

  288.           BO = B;

  289.           temp = k;

  290. #endif

  291. 	  v4sf_t *rowC;

  292. 	  v4sf_t result[4];

  293. 	  __vector_quad acc0, acc1;

  294. 	  __builtin_mma_xxsetaccz (&acc0);

  295. 	  __builtin_mma_xxsetaccz (&acc1);

  296. 	  BLASLONG l = 0;

  297. 	  for (l = 0; l < temp; l++)

  298. 	    {

  299. 	      vec_t *rowA = (vec_t *) & AO[l << 2];

  300. 	      __vector_pair rowB;

  301. 	      vec_t *rb = (vec_t *) & BO[l << 2];

  302. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  303. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  304. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  305. 	    }

  306. 	  SAVE_ACC (&acc0, 0);

  307. 	  SAVE_ACC (&acc1, 2);

  308. 	  CO += 4;

  309. 	  AO += temp << 2;

  310. 	  BO += temp << 2;

  311. #if defined(TRMMKERNEL)

  312.           REFRESH_AFTER_SAVE (4, 4)

  313. #endif

  314. 	}

  315.       i = (m & 3) >> 1;

  316.       for (j = 0; j < i; j++)

  317. 	{

  318. 	  FLOAT *BO;

  319. #if defined(TRMMKERNEL)

  320.           REFRESH_POINTERS (2, 4);

  321. #else

  322.           BO = B;

  323.           temp = k;

  324. #endif

  325. 	  v4sf_t *rowC;

  326. 	  v4sf_t result[4];

  327. 	  __vector_quad acc0;

  328. 	  __builtin_mma_xxsetaccz (&acc0);

  329. 	  BLASLONG l = 0;

  330. 	  for (l = 0; l < temp; l++)

  331. 	    {

  332. 	      vec_t *rowA = (vec_t *) & AO[l << 1];

  333. 	      __vector_pair rowB;

  334. 	      vec_t *rb = (vec_t *) & BO[l << 2];

  335. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  336. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  337. 	    }

  338. 	  SAVE_ACC (&acc0, 0);

  339. 	  CO += 2;

  340. 	  AO += temp << 1;

  341. 	  BO += temp << 2;

  342. #if defined(TRMMKERNEL)

  343.           REFRESH_AFTER_SAVE (2, 4)

  344. #endif

  345. 	}

  346.       i = (m & 1) >> 0;

  347.       for (j = 0; j < i; j++)

  348. 	{

  349. 	  FLOAT *BO;

  350. #if defined(TRMMKERNEL)

  351.           REFRESH_POINTERS (1, 4);

  352. #else

  353.           BO = B;

  354.           temp = k;

  355. #endif

  356. 	  BLASLONG l = 0;

  357. 	  v4sf_t t = { 0, 0 };

  358. 	  v4sf_t t1 = { 0, 0 };

  359. 	  for (l = 0; l < temp; l++)

  360. 	    {

  361. 	      v4sf_t rowA = { AO[l], AO[l] };

  362. 	      v4sf_t rowB = { BO[l << 2], BO[(l << 2) + 1] };

  363. 	      v4sf_t rowB1 = { BO[(l << 2) + 2], BO[(l << 2) + 3] };

  364. 	      t += rowA * rowB;

  365. 	      t1 += rowA * rowB1;

  366. 	    }

  367. 	  t = t * valpha;

  368. 	  t1 = t1 * valpha;

  369. #if defined(TRMMKERNEL)

  370. 	  CO[0 * ldc] = t[0];

  371. 	  CO[1 * ldc] = t[1];

  372. 	  CO[2 * ldc] = t1[0];

  373. 	  CO[3 * ldc] = t1[1];

  374. #else

  375. 	  CO[0 * ldc] += t[0];

  376. 	  CO[1 * ldc] += t[1];

  377. 	  CO[2 * ldc] += t1[0];

  378. 	  CO[3 * ldc] += t1[1];

  379. #endif

  380. 	  CO += 1;

  381. 	  AO += temp;

  382. 	  BO += temp << 2;

  383. #if defined(TRMMKERNEL)

  384.           REFRESH_AFTER_SAVE (1, 4)

  385. #endif

  386. 	}

  387. #if defined(TRMMKERNEL) && !defined(LEFT)

  388.       off += 4;                 // number of values in A

  389. #endif

  390.       B += k << 2;

  391.     }

  392.   N = (n & 3) >> 1;

  393.   for (i1 = 0; i1 < N; i1++)

  394.     {

  395.       BLASLONG i, j, temp;

  396. #if defined(TRMMKERNEL) && defined(LEFT)

  397.       off = offset;

  398. #endif

  399.       FLOAT *CO;

  400.       FLOAT *AO;

  401.       CO = C;

  402.       C += ldc << 1;

  403.       AO = A;

  404.       i = m >> 4;

  405.       for (j = 0; j < i; j++)

  406. 	{

  407. 	  FLOAT *BO;

  408. #if defined(TRMMKERNEL)

  409.           REFRESH_POINTERS (16, 2);

  410. #else

  411.           BO = B;

  412.           temp = k;

  413. #endif

  414. 	  v4sf_t *rowC;

  415. 	  v4sf_t result[4];

  416. 	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  417. 	  SET_ACC_ZERO8 ();

  418. 	  BLASLONG l = 0;

  419. 	  for (l = 0; l < temp; l++)

  420. 	    {

  421. 	      FLOAT t[4] = { 0, 0, 0, 0 };

  422. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  423. 	      __vector_pair rowB;

  424. 	      vec_t *rb = (vec_t *) & t[0];

  425. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  426. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  427. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  428. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  429. 	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);

  430. 	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);

  431. 	      __builtin_mma_xvf64gerpp (&acc4, rowB, rowA[4]);

  432. 	      __builtin_mma_xvf64gerpp (&acc5, rowB, rowA[5]);

  433. 	      __builtin_mma_xvf64gerpp (&acc6, rowB, rowA[6]);

  434. 	      __builtin_mma_xvf64gerpp (&acc7, rowB, rowA[7]);

  435. 	    }

  436. 	  SAVE2x4_ACC (&acc0, 0);

  437. 	  SAVE2x4_ACC (&acc1, 2);

  438. 	  SAVE2x4_ACC (&acc2, 4);

  439. 	  SAVE2x4_ACC (&acc3, 6);

  440. 	  SAVE2x4_ACC (&acc4, 8);

  441. 	  SAVE2x4_ACC (&acc5, 10);

  442. 	  SAVE2x4_ACC (&acc6, 12);

  443. 	  SAVE2x4_ACC (&acc7, 14);

  444. 	  CO += 16;

  445. 	  AO += temp << 4;

  446. 	  BO += temp << 1;

  447. #if defined(TRMMKERNEL)

  448.           REFRESH_AFTER_SAVE (16, 2)

  449. #endif

  450. 	}

  451.       i = (m & 15) >> 3;

  452.       for (j = 0; j < i; j++)

  453. 	{

  454. 	  FLOAT *BO;

  455. #if defined(TRMMKERNEL)

  456.           REFRESH_POINTERS (8, 2);

  457. #else

  458.           BO = B;

  459.           temp = k;

  460. #endif

  461. 	  v4sf_t *rowC;

  462. 	  v4sf_t result[4];

  463. 	  __vector_quad acc0, acc1, acc2, acc3;

  464. 	  SET_ACC_ZERO4 ();

  465. 	  BLASLONG l = 0;

  466. 	  for (l = 0; l < temp; l++)

  467. 	    {

  468. 	      FLOAT t[4] = { 0, 0, 0, 0 };

  469. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  470. 	      __vector_pair rowB;

  471. 	      vec_t *rb = (vec_t *) & t[0];

  472. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  473. 	      vec_t *rowA = (vec_t *) & AO[l << 3];

  474. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  475. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  476. 	      __builtin_mma_xvf64gerpp (&acc2, rowB, rowA[2]);

  477. 	      __builtin_mma_xvf64gerpp (&acc3, rowB, rowA[3]);

  478. 	    }

  479. 	  SAVE2x4_ACC (&acc0, 0);

  480. 	  SAVE2x4_ACC (&acc1, 2);

  481. 	  SAVE2x4_ACC (&acc2, 4);

  482. 	  SAVE2x4_ACC (&acc3, 6);

  483. 	  CO += 8;

  484. 	  AO += temp << 3;

  485. 	  BO += temp << 1;

  486. #if defined(TRMMKERNEL)

  487.           REFRESH_AFTER_SAVE (8, 2)

  488. #endif

  489. 	}

  490.       i = (m & 7) >> 2;

  491.       for (j = 0; j < i; j++)

  492. 	{

  493. 	  FLOAT *BO;

  494. #if defined(TRMMKERNEL)

  495.           REFRESH_POINTERS (4, 2);

  496. #else

  497.           BO = B;

  498.           temp = k;

  499. #endif

  500. 	  v4sf_t *rowC;

  501. 	  v4sf_t result[4];

  502. 	  __vector_quad acc0, acc1;

  503. 	  __builtin_mma_xxsetaccz (&acc0);

  504. 	  __builtin_mma_xxsetaccz (&acc1);

  505. 	  BLASLONG l = 0;

  506. 	  for (l = 0; l < temp; l++)

  507. 	    {

  508. 	      FLOAT t[4] = { 0, 0, 0, 0 };

  509. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  510. 	      __vector_pair rowB;

  511. 	      vec_t *rb = (vec_t *) & t[0];

  512. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  513. 	      vec_t *rowA = (vec_t *) & AO[l << 2];

  514. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  515. 	      __builtin_mma_xvf64gerpp (&acc1, rowB, rowA[1]);

  516. 	    }

  517. 	  SAVE2x4_ACC (&acc0, 0);

  518. 	  SAVE2x4_ACC (&acc1, 2);

  519. 	  CO += 4;

  520. 	  AO += temp << 2;

  521. 	  BO += temp << 1;

  522. #if defined(TRMMKERNEL)

  523.           REFRESH_AFTER_SAVE (4, 2)

  524. #endif

  525. 	}

  526.       i = (m & 3) >> 1;

  527.       for (j = 0; j < i; j++)

  528. 	{

  529. 	  FLOAT *BO;

  530. #if defined(TRMMKERNEL)

  531.           REFRESH_POINTERS (2, 2);

  532. #else

  533.           BO = B;

  534.           temp = k;

  535. #endif

  536. 	  v4sf_t *rowC;

  537. 	  v4sf_t result[4];

  538. 	  __vector_quad acc0;

  539. 	  __builtin_mma_xxsetaccz (&acc0);

  540. 	  BLASLONG l = 0;

  541. 	  for (l = 0; l < temp; l++)

  542. 	    {

  543. 	      FLOAT t[4] = { 0, 0, 0, 0 };

  544. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  545. 	      __vector_pair rowB;

  546. 	      vec_t *rb = (vec_t *) & t[0];

  547. 	      __builtin_mma_assemble_pair (&rowB, rb[1], rb[0]);

  548. 	      vec_t *rowA = (vec_t *) & AO[l << 1];

  549. 	      __builtin_mma_xvf64gerpp (&acc0, rowB, rowA[0]);

  550. 	    }

  551. 	  SAVE2x4_ACC (&acc0, 0);

  552. 	  CO += 2;

  553. 	  AO += temp << 1;

  554. 	  BO += temp << 1;

  555. #if defined(TRMMKERNEL)

  556.           REFRESH_AFTER_SAVE (2, 2)

  557. #endif

  558. 	}

  559.       i = (m & 1) >> 0;

  560.       for (j = 0; j < i; j++)

  561. 	{

  562. 	  FLOAT *BO;

  563. #if defined(TRMMKERNEL)

  564.           REFRESH_POINTERS (1, 2);

  565. #else

  566.           BO = B;

  567.           temp = k;

  568. #endif

  569. 	  BLASLONG l = 0;

  570. 	  v4sf_t t = { 0, 0 };

  571. 	  for (l = 0; l < temp; l++)

  572. 	    {

  573. 	      v4sf_t rowA = { AO[l], AO[l] };

  574. 	      v4sf_t rowB = { BO[l << 1], BO[(l << 1) + 1] };

  575. 	      t += rowA * rowB;

  576. 	    }

  577. 	  t = t * valpha;

  578. #if defined(TRMMKERNEL)

  579. 	  CO[0 * ldc] = t[0];

  580. 	  CO[1 * ldc] = t[1];

  581. #else

  582. 	  CO[0 * ldc] += t[0];

  583. 	  CO[1 * ldc] += t[1];

  584. #endif

  585. 	  CO += 1;

  586. 	  AO += temp;

  587. 	  BO += temp << 1;

  588. #if defined(TRMMKERNEL)

  589.           REFRESH_AFTER_SAVE (1, 2)

  590. #endif

  591. 	}

  592. #if defined(TRMMKERNEL) && !defined(LEFT)

  593.       off += 2;                 // number of values in A

  594. #endif

  595.       B += k << 1;

  596.     }

  597.   N = (n & 1) >> 0;

  598.   for (i1 = 0; i1 < N; i1++)

  599.     {

  600.       BLASLONG i, temp;

  601. #if defined(TRMMKERNEL) && defined(LEFT)

  602.       off = offset;

  603. #endif

  604.       FLOAT *CO;

  605.       FLOAT *AO;

  606.       CO = C;

  607.       C += ldc;

  608.       AO = A;

  609.       i = m;

  610.       while (i >= 16)

  611. 	{

  612. 	  FLOAT *BO;

  613. #if defined(TRMMKERNEL)

  614.           REFRESH_POINTERS (16, 1)

  615. #else

  616.           BO = B;

  617.           temp = k;

  618. #endif

  619. 	  BLASLONG l = 0;

  620. 	  v4sf_t t = { 0, 0 };

  621. 	  v4sf_t t1 = { 0, 0 };

  622. 	  v4sf_t t2 = { 0, 0 };

  623. 	  v4sf_t t3 = { 0, 0 };

  624. 	  v4sf_t t4 = { 0, 0 };

  625. 	  v4sf_t t5 = { 0, 0 };

  626. 	  v4sf_t t6 = { 0, 0 };

  627. 	  v4sf_t t7 = { 0, 0 };

  628. 	  for (l = 0; l < temp; l++)

  629. 	    {

  630. 	      v4sf_t rowB = { BO[l], BO[l] };

  631. 	      v4sf_t rowA = { AO[l << 4], AO[(l << 4) + 1] };

  632. 	      v4sf_t rowA1 = { AO[(l << 4) + 2], AO[(l << 4) + 3] };

  633. 	      v4sf_t rowA2 = { AO[(l << 4) + 4], AO[(l << 4) + 5] };

  634. 	      v4sf_t rowA3 = { AO[(l << 4) + 6], AO[(l << 4) + 7] };

  635. 	      v4sf_t rowA4 = { AO[(l << 4) + 8], AO[(l << 4) + 9] };

  636. 	      v4sf_t rowA5 = { AO[(l << 4) + 10], AO[(l << 4) + 11] };

  637. 	      v4sf_t rowA6 = { AO[(l << 4) + 12], AO[(l << 4) + 13] };

  638. 	      v4sf_t rowA7 = { AO[(l << 4) + 14], AO[(l << 4) + 15] };

  639. 	      t += rowA * rowB;

  640. 	      t1 += rowA1 * rowB;

  641. 	      t2 += rowA2 * rowB;

  642. 	      t3 += rowA3 * rowB;

  643. 	      t4 += rowA4 * rowB;

  644. 	      t5 += rowA5 * rowB;

  645. 	      t6 += rowA6 * rowB;

  646. 	      t7 += rowA7 * rowB;

  647. 	    }

  648. 	  t = t * valpha;

  649. 	  t1 = t1 * valpha;

  650. 	  t2 = t2 * valpha;

  651. 	  t3 = t3 * valpha;

  652. 	  t4 = t4 * valpha;

  653. 	  t5 = t5 * valpha;

  654. 	  t6 = t6 * valpha;

  655. 	  t7 = t7 * valpha;

  656. #if defined(TRMMKERNEL)

  657. 	  CO[0] = t[0];

  658. 	  CO[1] = t[1];

  659. 	  CO[2] = t1[0];

  660. 	  CO[3] = t1[1];

  661. 	  CO[4] = t2[0];

  662. 	  CO[5] = t2[1];

  663. 	  CO[6] = t3[0];

  664. 	  CO[7] = t3[1];

  665. 	  CO[8] = t4[0];

  666. 	  CO[9] = t4[1];

  667. 	  CO[10] = t5[0];

  668. 	  CO[11] = t5[1];

  669. 	  CO[12] = t6[0];

  670. 	  CO[13] = t6[1];

  671. 	  CO[14] = t7[0];

  672. 	  CO[15] = t7[1];

  673. #else

  674. 	  CO[0] += t[0];

  675. 	  CO[1] += t[1];

  676. 	  CO[2] += t1[0];

  677. 	  CO[3] += t1[1];

  678. 	  CO[4] += t2[0];

  679. 	  CO[5] += t2[1];

  680. 	  CO[6] += t3[0];

  681. 	  CO[7] += t3[1];

  682. 	  CO[8] += t4[0];

  683. 	  CO[9] += t4[1];

  684. 	  CO[10] += t5[0];

  685. 	  CO[11] += t5[1];

  686. 	  CO[12] += t6[0];

  687. 	  CO[13] += t6[1];

  688. 	  CO[14] += t7[0];

  689. 	  CO[15] += t7[1];

  690. #endif

  691. 	  AO += temp << 4;

  692. 	  BO += temp;

  693. 	  CO += 16;

  694. 	  i -= 16;

  695. #if defined(TRMMKERNEL)

  696.           REFRESH_AFTER_SAVE (16, 1)

  697. #endif

  698. 	}

  699.       while (i >= 8)

  700. 	{

  701. 	  FLOAT *BO;

  702. #if defined(TRMMKERNEL)

  703.           REFRESH_POINTERS (8, 1)

  704. #else

  705.           BO = B;

  706.           temp = k;

  707. #endif

  708. 	  BLASLONG l = 0;

  709. 	  v4sf_t t = { 0, 0 };

  710. 	  v4sf_t t1 = { 0, 0 };

  711. 	  v4sf_t t2 = { 0, 0 };

  712. 	  v4sf_t t3 = { 0, 0 };

  713. 	  for (l = 0; l < temp; l++)

  714. 	    {

  715. 	      v4sf_t rowB = { BO[l], BO[l] };

  716. 	      v4sf_t rowA = { AO[l << 3], AO[(l << 3) + 1] };

  717. 	      v4sf_t rowA1 = { AO[(l << 3) + 2], AO[(l << 3) + 3] };

  718. 	      v4sf_t rowA2 = { AO[(l << 3) + 4], AO[(l << 3) + 5] };

  719. 	      v4sf_t rowA3 = { AO[(l << 3) + 6], AO[(l << 3) + 7] };

  720. 	      t += rowA * rowB;

  721. 	      t1 += rowA1 * rowB;

  722. 	      t2 += rowA2 * rowB;

  723. 	      t3 += rowA3 * rowB;

  724. 	    }

  725. 	  t = t * valpha;

  726. 	  t1 = t1 * valpha;

  727. 	  t2 = t2 * valpha;

  728. 	  t3 = t3 * valpha;

  729. #if defined(TRMMKERNEL)

  730. 	  CO[0] = t[0];

  731. 	  CO[1] = t[1];

  732. 	  CO[2] = t1[0];

  733. 	  CO[3] = t1[1];

  734. 	  CO[4] = t2[0];

  735. 	  CO[5] = t2[1];

  736. 	  CO[6] = t3[0];

  737. 	  CO[7] = t3[1];

  738. #else

  739. 	  CO[0] += t[0];

  740. 	  CO[1] += t[1];

  741. 	  CO[2] += t1[0];

  742. 	  CO[3] += t1[1];

  743. 	  CO[4] += t2[0];

  744. 	  CO[5] += t2[1];

  745. 	  CO[6] += t3[0];

  746. 	  CO[7] += t3[1];

  747. #endif

  748. 	  AO += temp << 3;

  749. 	  BO += temp;

  750. 	  CO += 8;

  751. 	  i -= 8;

  752. #if defined(TRMMKERNEL)

  753.           REFRESH_AFTER_SAVE (8, 1)

  754. #endif

  755. 	}

  756.       while (i >= 4)

  757. 	{

  758. 	  FLOAT *BO;

  759. #if defined(TRMMKERNEL)

  760.           REFRESH_POINTERS (4, 1)

  761. #else

  762.           BO = B;

  763.           temp = k;

  764. #endif

  765. 	  BLASLONG l = 0;

  766. 	  v4sf_t t = { 0, 0 };

  767. 	  v4sf_t t1 = { 0, 0 };

  768. 	  for (l = 0; l < temp; l++)

  769. 	    {

  770. 	      v4sf_t rowB = { BO[l], BO[l] };

  771. 	      v4sf_t rowA = { AO[l << 2], AO[(l << 2) + 1] };

  772. 	      v4sf_t rowA1 = { AO[(l << 2) + 2], AO[(l << 2) + 3] };

  773. 	      t += rowA * rowB;

  774. 	      t1 += rowA1 * rowB;

  775. 	    }

  776. 	  t = t * valpha;

  777. 	  t1 = t1 * valpha;

  778. #if defined(TRMMKERNEL)

  779. 	  CO[0] = t[0];

  780. 	  CO[1] = t[1];

  781. 	  CO[2] = t1[0];

  782. 	  CO[3] = t1[1];

  783. #else

  784. 	  CO[0] += t[0];

  785. 	  CO[1] += t[1];

  786. 	  CO[2] += t1[0];

  787. 	  CO[3] += t1[1];

  788. #endif

  789. 	  AO += temp << 2;

  790. 	  BO += temp;

  791. 	  CO += 4;

  792. 	  i -= 4;

  793. #if defined(TRMMKERNEL)

  794.           REFRESH_AFTER_SAVE (4, 1)

  795. #endif

  796. 	}

  797.       while (i >= 2)

  798. 	{

  799. 	  FLOAT *BO;

  800. #if defined(TRMMKERNEL)

  801.           REFRESH_POINTERS (2, 1)

  802. #else

  803.           BO = B;

  804.           temp = k;

  805. #endif

  806. 	  BLASLONG l = 0;

  807. 	  v4sf_t t = { 0, 0 };

  808. 	  for (l = 0; l < temp; l++)

  809. 	    {

  810. 	      v4sf_t rowB = { BO[l], BO[l] };

  811. 	      v4sf_t rowA = { AO[l << 1], AO[(l << 1) + 1] };

  812. 	      t += rowA * rowB;

  813. 	    }

  814. 	  t = t * valpha;

  815. #if defined(TRMMKERNEL)

  816. 	  CO[0] = t[0];

  817. 	  CO[1] = t[1];

  818. #else

  819. 	  CO[0] += t[0];

  820. 	  CO[1] += t[1];

  821. #endif

  822. 	  AO += temp << 1;

  823. 	  BO += temp;

  824. 	  CO += 2;

  825. 	  i -= 2;

  826. #if defined(TRMMKERNEL)

  827.           REFRESH_AFTER_SAVE (2, 1)

  828. #endif

  829. 	}

  830.       while (i >= 1)

  831. 	{

  832. 	  FLOAT *BO;

  833. #if defined(TRMMKERNEL)

  834.           REFRESH_POINTERS (1, 1)

  835. #else

  836.           BO = B;

  837.           temp = k;

  838. #endif

  839. 	  BLASLONG l = 0;

  840. 	  FLOAT t = 0;

  841. 	  for (l = 0; l < temp; l++)

  842. 	    {

  843. 	      t += AO[l] * BO[l];

  844. 	    }

  845. 	  AO += temp;

  846. 	  BO += temp;

  847. #if defined(TRMMKERNEL)

  848. 	  CO[0] = t * alpha;

  849. #else

  850. 	  CO[0] += t * alpha;

  851. #endif

  852. 	  CO += 1;

  853. 	  i -= 1;

  854. #if defined(TRMMKERNEL)

  855.           REFRESH_AFTER_SAVE (1, 1)

  856. #endif

  857. 	}

  858. #if defined(TRMMKERNEL) && !defined(LEFT)

  859.       off += 1;                 // number of values in A

  860. #endif

  861.       B += k;

  862.     }

  863.   return 0;

  864. }