OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 571eadb880
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '571eadb880'
${ noResults }
OpenBLAS/kernel/power/sgemm_kernel_power10.c

1335 lines
33 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2020, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. #include "common.h"

  28. #include <altivec.h>

  29. 

  30. typedef unsigned char vec_t __attribute__ ((vector_size (16)));

  31. typedef FLOAT v4sf_t __attribute__ ((vector_size (16)));

  32. typedef FLOAT v2sf_t __attribute__ ((vector_size (8)));

  33. #if defined(TRMMKERNEL)

  34. #define SAVE_ACC(ACC, J)  \

  35. 	  __builtin_mma_disassemble_acc (result, ACC); \

  36. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  37.           rowC[0] = result[3] * alpha; \

  38.           rowC = (v4sf_t *) &CO[1*ldc+J]; \

  39.           rowC[0] = result[2] * alpha; \

  40.           rowC = (v4sf_t *) &CO[2*ldc+J]; \

  41.           rowC[0] = result[1] * alpha; \

  42.           rowC = (v4sf_t *) &CO[3*ldc+J]; \

  43.           rowC[0] = result[0] * alpha;

  44. #define SAVE_ACC1(ACC, J)  \

  45. 	  __builtin_mma_disassemble_acc (result, ACC); \

  46. 	  rowC = (v4sf_t *) &CO[4* ldc+J]; \

  47.           rowC[0] = result[3] * alpha; \

  48.           rowC = (v4sf_t *) &CO[5*ldc+J]; \

  49.           rowC[0] = result[2] * alpha; \

  50.           rowC = (v4sf_t *) &CO[6*ldc+J]; \

  51.           rowC[0] = result[1] * alpha; \

  52.           rowC = (v4sf_t *) &CO[7*ldc+J]; \

  53.           rowC[0] = result[0] * alpha;

  54. #define  SAVE4x2_ACC(ACC, J)  \

  55. 	  __builtin_mma_disassemble_acc (result, ACC); \

  56. 	  rowC = (v2sf_t *) &CO[0* ldc+J]; \

  57.           rowC[0] = result[6] * alpha; \

  58. 	  rowC = (v2sf_t *) &CO[1* ldc+J]; \

  59.           rowC[0] = result[4] * alpha; \

  60. 	  rowC = (v2sf_t *) &CO[2* ldc+J]; \

  61.           rowC[0] = result[2] * alpha; \

  62. 	  rowC = (v2sf_t *) &CO[3* ldc+J]; \

  63.           rowC[0] = result[0] * alpha;

  64. #define  SAVE4x2_ACC1(ACC, J)  \

  65. 	  __builtin_mma_disassemble_acc (result, ACC); \

  66. 	  rowC = (v2sf_t *) &CO[4* ldc+J]; \

  67.           rowC[0] = result[6] * alpha; \

  68. 	  rowC = (v2sf_t *) &CO[5* ldc+J]; \

  69.           rowC[0] = result[4] * alpha; \

  70. 	  rowC = (v2sf_t *) &CO[6* ldc+J]; \

  71.           rowC[0] = result[2] * alpha; \

  72. 	  rowC = (v2sf_t *) &CO[7* ldc+J]; \

  73.           rowC[0] = result[0] * alpha;

  74. #define  SAVE2x4_ACC(ACC, J)  \

  75. 	  __builtin_mma_disassemble_acc (result, ACC); \

  76. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  77.           rowC[0] = result[3] * alpha; \

  78. 	  rowC = (v4sf_t *) &CO[1* ldc+J]; \

  79.           rowC[0] = result[2] * alpha;

  80. #else

  81. #define SAVE_ACC(ACC, J)  \

  82. 	  __builtin_mma_disassemble_acc (result, ACC); \

  83. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  84.           rowC[0] += result[3] * alpha; \

  85.           rowC = (v4sf_t *) &CO[1*ldc+J]; \

  86.           rowC[0] += result[2] * alpha; \

  87.           rowC = (v4sf_t *) &CO[2*ldc+J]; \

  88.           rowC[0] += result[1] * alpha; \

  89.           rowC = (v4sf_t *) &CO[3*ldc+J]; \

  90.           rowC[0] += result[0] * alpha;

  91. #define SAVE_ACC1(ACC, J)  \

  92. 	  __builtin_mma_disassemble_acc (result, ACC); \

  93. 	  rowC = (v4sf_t *) &CO[4* ldc+J]; \

  94.           rowC[0] += result[3] * alpha; \

  95.           rowC = (v4sf_t *) &CO[5*ldc+J]; \

  96.           rowC[0] += result[2] * alpha; \

  97.           rowC = (v4sf_t *) &CO[6*ldc+J]; \

  98.           rowC[0] += result[1] * alpha; \

  99.           rowC = (v4sf_t *) &CO[7*ldc+J]; \

  100.           rowC[0] += result[0] * alpha;

  101. #define  SAVE4x2_ACC(ACC, J)  \

  102. 	  __builtin_mma_disassemble_acc (result, ACC); \

  103. 	  rowC = (v2sf_t *) &CO[0* ldc+J]; \

  104.           rowC[0] += result[6] * alpha; \

  105. 	  rowC = (v2sf_t *) &CO[1* ldc+J]; \

  106.           rowC[0] += result[4] * alpha; \

  107. 	  rowC = (v2sf_t *) &CO[2* ldc+J]; \

  108.           rowC[0] += result[2] * alpha; \

  109. 	  rowC = (v2sf_t *) &CO[3* ldc+J]; \

  110.           rowC[0] += result[0] * alpha;

  111. #define  SAVE4x2_ACC1(ACC, J)  \

  112. 	  __builtin_mma_disassemble_acc (result, ACC); \

  113. 	  rowC = (v2sf_t *) &CO[4* ldc+J]; \

  114.           rowC[0] += result[6] * alpha; \

  115. 	  rowC = (v2sf_t *) &CO[5* ldc+J]; \

  116.           rowC[0] += result[4] * alpha; \

  117. 	  rowC = (v2sf_t *) &CO[6* ldc+J]; \

  118.           rowC[0] += result[2] * alpha; \

  119. 	  rowC = (v2sf_t *) &CO[7* ldc+J]; \

  120.           rowC[0] += result[0] * alpha;

  121. #define  SAVE2x4_ACC(ACC, J)  \

  122. 	  __builtin_mma_disassemble_acc (result, ACC); \

  123. 	  rowC = (v4sf_t *) &CO[0* ldc+J]; \

  124.           rowC[0] += result[3] * alpha; \

  125. 	  rowC = (v4sf_t *) &CO[1* ldc+J]; \

  126.           rowC[0] += result[2] * alpha;

  127. #endif

  128. #define KERNEL(i, j) \

  129.           __builtin_mma_xvf32gerpp (&acc0, rowB[i], rowA[j]); \

  130.           __builtin_mma_xvf32gerpp (&acc1, rowB[i+1], rowA[j]); \

  131.           __builtin_mma_xvf32gerpp (&acc2, rowB[i], rowA[j+1]); \

  132.           __builtin_mma_xvf32gerpp (&acc3, rowB[i+1], rowA[j+1]); \

  133.           __builtin_mma_xvf32gerpp (&acc4, rowB[i], rowA[j+2]); \

  134.           __builtin_mma_xvf32gerpp (&acc5, rowB[i+1], rowA[j+2]); \

  135.           __builtin_mma_xvf32gerpp (&acc6, rowB[i], rowA[j+3]); \

  136.           __builtin_mma_xvf32gerpp (&acc7, rowB[i+1], rowA[j+3]);

  137. #define SET_ACC_ZERO4() \

  138.           __builtin_mma_xxsetaccz (&acc0); \

  139.           __builtin_mma_xxsetaccz (&acc1); \

  140.           __builtin_mma_xxsetaccz (&acc2); \

  141.           __builtin_mma_xxsetaccz (&acc3);

  142. 

  143. #define SET_ACC_ZERO8() \

  144.           __builtin_mma_xxsetaccz (&acc0); \

  145.           __builtin_mma_xxsetaccz (&acc1); \

  146.           __builtin_mma_xxsetaccz (&acc2); \

  147.           __builtin_mma_xxsetaccz (&acc3); \

  148.           __builtin_mma_xxsetaccz (&acc4); \

  149.           __builtin_mma_xxsetaccz (&acc5); \

  150.           __builtin_mma_xxsetaccz (&acc6); \

  151.           __builtin_mma_xxsetaccz (&acc7);

  152. 

  153. #define PREFETCH1(x, y) asm volatile ("dcbt %0, %1" : : "r" (x), "b" (y) : "memory");

  154. 

  155. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  156. #define REFRESH_TEMP_BK(x, y) \

  157.             temp = k - off;

  158. #elif defined(LEFT)

  159. #define REFRESH_TEMP_BK(x, y) \

  160.             temp = off + x;

  161. #else

  162. #define REFRESH_TEMP_BK(x, y) \

  163.             temp = off + y;

  164. #endif

  165. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  166. #define REFRESH_POINTERS(x, y) \

  167. 	  BO = B; \

  168.           REFRESH_TEMP_BK(x, y)

  169. #else

  170. #define REFRESH_POINTERS(x, y) \

  171.           AO += off * x; \

  172.           BO = B + off * y; \

  173.           REFRESH_TEMP_BK(x, y)

  174. #endif

  175. 

  176. #ifdef LEFT

  177. #define REFRESH_OFF(x) \

  178.             off += x;

  179. #else

  180. #define REFRESH_OFF(x)

  181. #endif

  182. 

  183. #ifdef LEFT

  184. #define UPDATE_TEMP(x, y) \

  185.             temp -= x;

  186. #else

  187. #define UPDATE_TEMP(x, y) \

  188.             temp -= y;

  189. #endif

  190. 

  191. #if (defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  192. #define REFRESH_TMP_AFTER_SAVE(x, y) \

  193.             temp = k - off; \

  194.             UPDATE_TEMP(x, y) \

  195.             AO += temp * x; \

  196.             BO += temp * y;

  197. #else

  198. #define REFRESH_TMP_AFTER_SAVE(x, y)

  199. #endif

  200. 

  201. #define REFRESH_AFTER_SAVE(x,y) \

  202.         REFRESH_TMP_AFTER_SAVE(x, y) \

  203. 	REFRESH_OFF(x)

  204. /*************************************************************************************

  205. * GEMM Kernel

  206. *************************************************************************************/

  207. int

  208. CNAME (BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, FLOAT * A, FLOAT * B,

  209.        FLOAT * C, BLASLONG ldc

  210. #ifdef TRMMKERNEL

  211.        , BLASLONG offset

  212. #endif

  213.   )

  214. {

  215.   BLASLONG N = n;

  216.   BLASLONG i1;

  217. #if defined(TRMMKERNEL)

  218.   BLASLONG off;

  219. #endif

  220. #if defined(TRMMKERNEL) && !defined(LEFT)

  221.   off = -offset;

  222. #endif

  223. 

  224.   v4sf_t valpha = { alpha, alpha, alpha, alpha };

  225.   N = n >> 3;

  226.   for (i1 = 0; i1 < N; i1++)

  227.     {

  228.       BLASLONG i, j, temp;

  229.       FLOAT *CO;

  230.       FLOAT *AO;

  231. #if defined(TRMMKERNEL) && defined(LEFT)

  232.       off = offset;

  233. #endif

  234.       CO = C;

  235.       C += ldc << 3;

  236.       AO = A;

  237.       PREFETCH1 (A, 128);

  238.       PREFETCH1 (A, 256);

  239.       i = m >> 4;

  240.       for (j = 0; j < i; j++)

  241. 	{

  242. 	  FLOAT *BO;

  243. #if defined(TRMMKERNEL)

  244. 	  REFRESH_POINTERS (16, 8);

  245. #else

  246. 	  BO = B;

  247. 	  temp = k;

  248. #endif

  249. 	  v4sf_t *rowC;

  250. 	  v4sf_t result[4];

  251. 	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  252. 	  SET_ACC_ZERO8 ();

  253. 	  BLASLONG l = 0;

  254. 	  BLASLONG K = temp / 64;

  255. 	  for (l = 0; l < K; l++)

  256. 	    {

  257. 	      vec_t *rowA = (vec_t *) & AO[0];

  258. 	      vec_t *rowB = (vec_t *) & BO[0];

  259. 	      KERNEL (0, 0);

  260. 	      KERNEL (2, 4);

  261. 	      KERNEL (4, 8);

  262. 	      KERNEL (6, 12);

  263. 	      KERNEL (8, 16);

  264. 	      KERNEL (10, 20);

  265. 	      KERNEL (12, 24);

  266. 	      KERNEL (14, 28);

  267. 	      KERNEL (16, 32);

  268. 	      KERNEL (18, 36);

  269. 	      KERNEL (20, 40);

  270. 	      KERNEL (22, 44);

  271. 	      KERNEL (24, 48);

  272. 	      KERNEL (26, 52);

  273. 	      KERNEL (28, 56);

  274. 	      KERNEL (30, 60);

  275. 	      KERNEL (32, 64);

  276. 	      KERNEL (34, 68);

  277. 	      KERNEL (36, 72);

  278. 	      KERNEL (38, 76);

  279. 	      KERNEL (40, 80);

  280. 	      KERNEL (42, 84);

  281. 	      KERNEL (44, 88);

  282. 	      KERNEL (46, 92);

  283. 	      KERNEL (48, 96);

  284. 	      KERNEL (50, 100);

  285. 	      KERNEL (52, 104);

  286. 	      KERNEL (54, 108);

  287. 	      KERNEL (56, 112);

  288. 	      KERNEL (58, 116);

  289. 	      KERNEL (60, 120);

  290. 	      KERNEL (62, 124);

  291. 	      KERNEL (64, 128);

  292. 	      KERNEL (66, 132);

  293. 	      KERNEL (68, 136);

  294. 	      KERNEL (70, 140);

  295. 	      KERNEL (72, 144);

  296. 	      KERNEL (74, 148);

  297. 	      KERNEL (76, 152);

  298. 	      KERNEL (78, 156);

  299. 	      KERNEL (80, 160);

  300. 	      KERNEL (82, 164);

  301. 	      KERNEL (84, 168);

  302. 	      KERNEL (86, 172);

  303. 	      KERNEL (88, 176);

  304. 	      KERNEL (90, 180);

  305. 	      KERNEL (92, 184);

  306. 	      KERNEL (94, 188);

  307. 	      KERNEL (96, 192);

  308. 	      KERNEL (98, 196);

  309. 	      KERNEL (100, 200);

  310. 	      KERNEL (102, 204);

  311. 	      KERNEL (104, 208);

  312. 	      KERNEL (106, 212);

  313. 	      KERNEL (108, 216);

  314. 	      KERNEL (110, 220);

  315. 	      KERNEL (112, 224);

  316. 	      KERNEL (114, 228);

  317. 	      KERNEL (116, 232);

  318. 	      KERNEL (118, 236);

  319. 	      KERNEL (120, 240);

  320. 	      KERNEL (122, 244);

  321. 	      KERNEL (124, 248);

  322. 	      KERNEL (126, 252);

  323. 	      AO += 1024;

  324. 	      BO += 512;

  325. 	    }

  326. 	  if ((temp & 63) >> 5)

  327. 	    {

  328. 	      vec_t *rowA = (vec_t *) & AO[0];

  329. 	      vec_t *rowB = (vec_t *) & BO[0];

  330. 	      KERNEL (0, 0);

  331. 	      KERNEL (2, 4);

  332. 	      KERNEL (4, 8);

  333. 	      KERNEL (6, 12);

  334. 	      KERNEL (8, 16);

  335. 	      KERNEL (10, 20);

  336. 	      KERNEL (12, 24);

  337. 	      KERNEL (14, 28);

  338. 	      KERNEL (16, 32);

  339. 	      KERNEL (18, 36);

  340. 	      KERNEL (20, 40);

  341. 	      KERNEL (22, 44);

  342. 	      KERNEL (24, 48);

  343. 	      KERNEL (26, 52);

  344. 	      KERNEL (28, 56);

  345. 	      KERNEL (30, 60);

  346. 	      KERNEL (32, 64);

  347. 	      KERNEL (34, 68);

  348. 	      KERNEL (36, 72);

  349. 	      KERNEL (38, 76);

  350. 	      KERNEL (40, 80);

  351. 	      KERNEL (42, 84);

  352. 	      KERNEL (44, 88);

  353. 	      KERNEL (46, 92);

  354. 	      KERNEL (48, 96);

  355. 	      KERNEL (50, 100);

  356. 	      KERNEL (52, 104);

  357. 	      KERNEL (54, 108);

  358. 	      KERNEL (56, 112);

  359. 	      KERNEL (58, 116);

  360. 	      KERNEL (60, 120);

  361. 	      KERNEL (62, 124);

  362. 	      AO += 512;

  363. 	      BO += 256;

  364. 	    }

  365. 	  if ((temp & 31) >> 4)

  366. 	    {

  367. 	      vec_t *rowA = (vec_t *) & AO[0];

  368. 	      vec_t *rowB = (vec_t *) & BO[0];

  369. 	      KERNEL (0, 0);

  370. 	      KERNEL (2, 4);

  371. 	      KERNEL (4, 8);

  372. 	      KERNEL (6, 12);

  373. 	      KERNEL (8, 16);

  374. 	      KERNEL (10, 20);

  375. 	      KERNEL (12, 24);

  376. 	      KERNEL (14, 28);

  377. 	      KERNEL (16, 32);

  378. 	      KERNEL (18, 36);

  379. 	      KERNEL (20, 40);

  380. 	      KERNEL (22, 44);

  381. 	      KERNEL (24, 48);

  382. 	      KERNEL (26, 52);

  383. 	      KERNEL (28, 56);

  384. 	      KERNEL (30, 60);

  385. 	      AO += 256;

  386. 	      BO += 128;

  387. 	    }

  388. 	  if ((temp & 15) >> 3)

  389. 	    {

  390. 	      vec_t *rowA = (vec_t *) & AO[0];

  391. 	      vec_t *rowB = (vec_t *) & BO[0];

  392. 	      KERNEL (0, 0);

  393. 	      KERNEL (2, 4);

  394. 	      KERNEL (4, 8);

  395. 	      KERNEL (6, 12);

  396. 	      KERNEL (8, 16);

  397. 	      KERNEL (10, 20);

  398. 	      KERNEL (12, 24);

  399. 	      KERNEL (14, 28);

  400. 	      AO += 128;

  401. 	      BO += 64;

  402. 	    }

  403. 	  if ((temp & 7) >> 2)

  404. 	    {

  405. 	      vec_t *rowA = (vec_t *) & AO[0];

  406. 	      vec_t *rowB = (vec_t *) & BO[0];

  407. 	      KERNEL (0, 0);

  408. 	      KERNEL (2, 4);

  409. 	      KERNEL (4, 8);

  410. 	      KERNEL (6, 12);

  411. 	      AO += 64;

  412. 	      BO += 32;

  413. 	    }

  414. 	  if ((temp & 3) >> 1)

  415. 	    {

  416. 	      vec_t *rowA = (vec_t *) & AO[0];

  417. 	      vec_t *rowB = (vec_t *) & BO[0];

  418. 	      KERNEL (0, 0);

  419. 	      KERNEL (2, 4);

  420. 	      AO += 32;

  421. 	      BO += 16;

  422. 	    }

  423. 	  if ((temp & 1) >> 0)

  424. 	    {

  425. 	      vec_t *rowA = (vec_t *) & AO[0];

  426. 	      vec_t *rowB = (vec_t *) & BO[0];

  427. 	      KERNEL (0, 0);

  428. 	      AO += 16;

  429. 	      BO += 8;

  430. 	    }

  431. 	  SAVE_ACC (&acc0, 0);

  432. 	  SAVE_ACC (&acc2, 4);

  433. 	  SAVE_ACC1 (&acc1, 0);

  434. 	  SAVE_ACC1 (&acc3, 4);

  435. 	  SAVE_ACC (&acc4, 8);

  436. 	  SAVE_ACC (&acc6, 12);

  437. 	  SAVE_ACC1 (&acc5, 8);

  438. 	  SAVE_ACC1 (&acc7, 12);

  439. #if defined(TRMMKERNEL)

  440. 	  REFRESH_AFTER_SAVE (16, 8)

  441. #endif

  442. 	    CO += 16;

  443. 	}

  444.       i = (m & 15) >> 3;

  445.       for (j = 0; j < i; j++)

  446. 	{

  447. 	  FLOAT *BO;

  448. #if defined(TRMMKERNEL)

  449. 	  REFRESH_POINTERS (8, 8);

  450. #else

  451. 	  BO = B;

  452. 	  temp = k;

  453. #endif

  454. 	  v4sf_t *rowC;

  455. 	  v4sf_t result[4];

  456. 	  __vector_quad acc0, acc1, acc2, acc3;

  457. 	  SET_ACC_ZERO4 ();

  458. 	  BLASLONG l = 0;

  459. 	  for (l = 0; l < temp; l++)

  460. 	    {

  461. 	      vec_t *rowA = (vec_t *) & AO[l << 3];

  462. 	      vec_t *rowB = (vec_t *) & BO[l << 3];

  463. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  464. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);

  465. 	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[1]);

  466. 	      __builtin_mma_xvf32gerpp (&acc3, rowB[1], rowA[1]);

  467. 	    }

  468. 	  SAVE_ACC (&acc0, 0);

  469. 	  SAVE_ACC (&acc2, 4);

  470. 	  SAVE_ACC1 (&acc1, 0);

  471. 	  SAVE_ACC1 (&acc3, 4);

  472. 	  AO += (temp << 3);

  473. 	  BO += (temp << 3);

  474. 	  CO += 8;

  475. #if defined(TRMMKERNEL)

  476. 	  REFRESH_AFTER_SAVE (8, 8)

  477. #endif

  478. 	}

  479.       i = (m & 7) >> 2;

  480.       for (j = 0; j < i; j++)

  481. 	{

  482. 	  FLOAT *BO;

  483. #if defined(TRMMKERNEL)

  484. 	  REFRESH_POINTERS (4, 8);

  485. #else

  486. 	  BO = B;

  487. 	  temp = k;

  488. #endif

  489. 	  v4sf_t *rowC;

  490. 	  v4sf_t result[4];

  491. 	  __vector_quad acc0, acc1;

  492. 	  __builtin_mma_xxsetaccz (&acc0);

  493. 	  __builtin_mma_xxsetaccz (&acc1);

  494. 	  BLASLONG l = 0;

  495. 	  for (l = 0; l < temp; l++)

  496. 	    {

  497. 	      vec_t *rowA = (vec_t *) & AO[l << 2];

  498. 	      vec_t *rowB = (vec_t *) & BO[l << 3];

  499. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  500. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);

  501. 	    }

  502. 	  SAVE_ACC (&acc0, 0);

  503. 	  SAVE_ACC1 (&acc1, 0);

  504. 	  CO += 4;

  505. 	  AO += (temp << 2);

  506. 	  BO += (temp << 3);

  507. #if defined(TRMMKERNEL)

  508. 	  REFRESH_AFTER_SAVE (4, 8)

  509. #endif

  510. 	}

  511.       i = (m & 3) >> 1;

  512.       for (j = 0; j < i; j++)

  513. 	{

  514. 	  FLOAT *BO;

  515. #if defined(TRMMKERNEL)

  516. 	  REFRESH_POINTERS (2, 8);

  517. #else

  518. 	  BO = B;

  519. 	  temp = k;

  520. #endif

  521. 

  522. 	  v2sf_t *rowC;

  523. 	  v2sf_t result[8];

  524. 	  __vector_quad acc0, acc1;

  525. 	  __builtin_mma_xxsetaccz (&acc0);

  526. 	  __builtin_mma_xxsetaccz (&acc1);

  527. 	  BLASLONG l = 0;

  528. 	  for (l = 0; l < temp; l++)

  529. 	    {

  530. 	      FLOAT t[4] = { 0 };

  531. 	      t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];

  532. 	      vec_t *rowA = (vec_t *) & t[0];

  533. 	      vec_t *rowB = (vec_t *) & BO[l << 3];

  534. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  535. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[1], rowA[0]);

  536. 	    }

  537. 	  SAVE4x2_ACC (&acc0, 0);

  538. 	  SAVE4x2_ACC1 (&acc1, 0);

  539. 	  CO += 2;

  540. 	  AO += (temp << 1);

  541. 	  BO += (temp << 3);

  542. #if defined(TRMMKERNEL)

  543. 	  REFRESH_AFTER_SAVE (2, 8)

  544. #endif

  545. 	}

  546.       i = (m & 1) >> 0;

  547.       for (j = 0; j < i; j++)

  548. 	{

  549. 	  FLOAT *BO;

  550. #if defined(TRMMKERNEL)

  551. 	  REFRESH_POINTERS (1, 8);

  552. #else

  553. 	  BO = B;

  554. 	  temp = k;

  555. #endif

  556. 	  BLASLONG l = 0;

  557. 	  v4sf_t t = { 0, 0, 0, 0 };

  558. 	  v4sf_t t1 = { 0, 0, 0, 0 };

  559. 	  for (l = 0; l < temp; l++)

  560. 	    {

  561. 	      v4sf_t rowA = { AO[l], AO[l], AO[l], AO[l] };

  562. 	      v4sf_t rowB = { BO[l << 3], BO[(l << 3) + 1], BO[(l << 3) + 2],

  563. 		BO[(l << 3) + 3]

  564. 	      };

  565. 	      v4sf_t rowB1 =

  566. 		{ BO[(l << 3) + 4], BO[(l << 3) + 5], BO[(l << 3) + 6],

  567. 		BO[(l << 3) + 7]

  568. 	      };

  569. 	      t += rowA * rowB;

  570. 	      t1 += rowA * rowB1;

  571. 	    }

  572. 	  t = t * valpha;

  573. 	  t1 = t1 * valpha;

  574. #if defined(TRMMKERNEL)

  575. 	  CO[0 * ldc] = t[0];

  576. 	  CO[1 * ldc] = t[1];

  577. 	  CO[2 * ldc] = t[2];

  578. 	  CO[3 * ldc] = t[3];

  579. 	  CO[4 * ldc] = t1[0];

  580. 	  CO[5 * ldc] = t1[1];

  581. 	  CO[6 * ldc] = t1[2];

  582. 	  CO[7 * ldc] = t1[3];

  583. #else

  584. 	  CO[0 * ldc] += t[0];

  585. 	  CO[1 * ldc] += t[1];

  586. 	  CO[2 * ldc] += t[2];

  587. 	  CO[3 * ldc] += t[3];

  588. 	  CO[4 * ldc] += t1[0];

  589. 	  CO[5 * ldc] += t1[1];

  590. 	  CO[6 * ldc] += t1[2];

  591. 	  CO[7 * ldc] += t1[3];

  592. #endif

  593. 	  CO += 1;

  594. 	  AO += temp;

  595. 	  BO += (temp << 3);

  596. #if defined(TRMMKERNEL)

  597. 	  REFRESH_AFTER_SAVE (1, 8)

  598. #endif

  599. 	}

  600. #if defined(TRMMKERNEL) && !defined(LEFT)

  601.       off += 8;			// number of values in A

  602. #endif

  603. 

  604.       B += k << 3;

  605.     }

  606.   N = (n & 7) >> 2;

  607.   for (i1 = 0; i1 < N; i1++)

  608.     {

  609.       BLASLONG i, j, temp;

  610. #if defined(TRMMKERNEL) && defined(LEFT)

  611.       off = offset;

  612. #endif

  613.       FLOAT *CO;

  614.       FLOAT *AO;

  615.       CO = C;

  616.       C += ldc << 2;

  617.       AO = A;

  618. #if !defined(TRMMKERNEL)

  619.       i = m >> 5;

  620.       for (j = 0; j < i; j++)

  621. 	{

  622. 	  FLOAT *BO = B;

  623. 	  v4sf_t *rowC;

  624. 	  v4sf_t result[4];

  625. 	  FLOAT *A1;

  626. 	  A1 = AO + (16 * k);

  627. 	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  628. 	  SET_ACC_ZERO8 ();

  629. 	  BLASLONG l = 0;

  630. 	  for (l = 0; l < k; l++)

  631. 	    {

  632. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  633. 	      vec_t *rowA1 = (vec_t *) & A1[l << 4];

  634. 	      vec_t *rowB = (vec_t *) & BO[l << 2];

  635. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  636. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  637. 	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);

  638. 	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);

  639. 	      __builtin_mma_xvf32gerpp (&acc4, rowB[0], rowA1[0]);

  640. 	      __builtin_mma_xvf32gerpp (&acc5, rowB[0], rowA1[1]);

  641. 	      __builtin_mma_xvf32gerpp (&acc6, rowB[0], rowA1[2]);

  642. 	      __builtin_mma_xvf32gerpp (&acc7, rowB[0], rowA1[3]);

  643. 	    }

  644. 

  645. 	  SAVE_ACC (&acc0, 0);

  646. 	  SAVE_ACC (&acc1, 4);

  647. 	  CO += 8;

  648. 	  SAVE_ACC (&acc2, 0);

  649. 	  SAVE_ACC (&acc3, 4);

  650. 	  CO += 8;

  651. 	  SAVE_ACC (&acc4, 0);

  652. 	  SAVE_ACC (&acc5, 4);

  653. 	  CO += 8;

  654. 	  SAVE_ACC (&acc6, 0);

  655. 	  SAVE_ACC (&acc7, 4);

  656. 	  CO += 8;

  657. 	  AO += k << 5;

  658. 	  BO += k << 2;

  659. 	}

  660.       i = (m & 31) >> 4;

  661. #else

  662.       i = m >> 4;

  663. #endif

  664.       for (j = 0; j < i; j++)

  665. 	{

  666. 	  FLOAT *BO;

  667. #if defined(TRMMKERNEL)

  668. 	  REFRESH_POINTERS (16, 4);

  669. #else

  670. 	  BO = B;

  671. 	  temp = k;

  672. #endif

  673. 	  v4sf_t *rowC;

  674. 	  v4sf_t result[4];

  675. 	  __vector_quad acc0, acc1, acc2, acc3;

  676. 	  SET_ACC_ZERO4 ();

  677. 	  BLASLONG l = 0;

  678. 	  for (l = 0; l < temp; l++)

  679. 	    {

  680. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  681. 	      vec_t *rowB = (vec_t *) & BO[l << 2];

  682. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  683. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  684. 	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);

  685. 	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);

  686. 	    }

  687. 

  688. 	  SAVE_ACC (&acc0, 0);

  689. 	  SAVE_ACC (&acc1, 4);

  690. 	  CO += 8;

  691. 	  SAVE_ACC (&acc2, 0);

  692. 	  SAVE_ACC (&acc3, 4);

  693. 	  CO += 8;

  694. 	  AO += temp << 4;

  695. 	  BO += temp << 2;

  696. #if defined(TRMMKERNEL)

  697. 	  REFRESH_AFTER_SAVE (16, 4)

  698. #endif

  699. 	}

  700.       i = (m & 15) >> 3;

  701.       for (j = 0; j < i; j++)

  702. 	{

  703. 	  FLOAT *BO;

  704. #if defined(TRMMKERNEL)

  705. 	  REFRESH_POINTERS (8, 4);

  706. #else

  707. 	  BO = B;

  708. 	  temp = k;

  709. #endif

  710. 	  v4sf_t *rowC;

  711. 	  v4sf_t result[4];

  712. 	  __vector_quad acc0, acc1;

  713. 	  __builtin_mma_xxsetaccz (&acc0);

  714. 	  __builtin_mma_xxsetaccz (&acc1);

  715. 	  BLASLONG l = 0;

  716. 	  for (l = 0; l < temp; l++)

  717. 	    {

  718. 	      vec_t *rowA = (vec_t *) & AO[l << 3];

  719. 	      vec_t *rowB = (vec_t *) & BO[l << 2];

  720. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  721. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  722. 	    }

  723. 	  SAVE_ACC (&acc0, 0);

  724. 	  SAVE_ACC (&acc1, 4);

  725. 	  CO += 8;

  726. 	  AO += temp << 3;

  727. 	  BO += temp << 2;

  728. #if defined(TRMMKERNEL)

  729. 	  REFRESH_AFTER_SAVE (8, 4)

  730. #endif

  731. 	}

  732.       i = (m & 7) >> 2;

  733.       for (j = 0; j < i; j++)

  734. 	{

  735. 	  FLOAT *BO;

  736. #if defined(TRMMKERNEL)

  737. 	  REFRESH_POINTERS (4, 4);

  738. #else

  739. 	  BO = B;

  740. 	  temp = k;

  741. #endif

  742. 	  v4sf_t *rowC;

  743. 	  __vector_quad acc0;

  744. 	  v4sf_t result[4];

  745. 	  __builtin_mma_xxsetaccz (&acc0);

  746. 	  BLASLONG l = 0;

  747. 	  for (l = 0; l < temp; l++)

  748. 	    {

  749. 	      vec_t *rowA = (vec_t *) & AO[l << 2];

  750. 	      vec_t *rowB = (vec_t *) & BO[l << 2];

  751. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  752. 	    }

  753. 	  SAVE_ACC (&acc0, 0);

  754. 	  CO += 4;

  755. 	  AO += temp << 2;

  756. 	  BO += temp << 2;

  757. #if defined(TRMMKERNEL)

  758. 	  REFRESH_AFTER_SAVE (4, 4)

  759. #endif

  760. 	}

  761.       i = (m & 3) >> 1;

  762.       for (j = 0; j < i; j++)

  763. 	{

  764. 	  FLOAT *BO;

  765. #if defined(TRMMKERNEL)

  766. 	  REFRESH_POINTERS (2, 4);

  767. #else

  768. 	  BO = B;

  769. 	  temp = k;

  770. #endif

  771. 	  v2sf_t *rowC;

  772. 	  v2sf_t result[8];

  773. 	  __vector_quad acc0;

  774. 	  __builtin_mma_xxsetaccz (&acc0);

  775. 	  BLASLONG l = 0;

  776. 	  for (l = 0; l < temp; l++)

  777. 	    {

  778. 	      FLOAT t[4] = { 0 };

  779. 	      t[0] = AO[l << 1], t[1] = AO[(l << 1) + 1];

  780. 	      vec_t *rowA = (vec_t *) & t[0];

  781. 	      vec_t *rowB = (vec_t *) & BO[l << 2];

  782. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  783. 	    }

  784. 	  SAVE4x2_ACC (&acc0, 0);

  785. 	  CO += 2;

  786. 	  AO += temp << 1;

  787. 	  BO += temp << 2;

  788. #if defined(TRMMKERNEL)

  789. 	  REFRESH_AFTER_SAVE (2, 4)

  790. #endif

  791. 	}

  792.       i = (m & 1) >> 0;

  793.       for (j = 0; j < i; j++)

  794. 	{

  795. 	  FLOAT *BO;

  796. #if defined(TRMMKERNEL)

  797. 	  REFRESH_POINTERS (1, 4)

  798. #else

  799. 	  BO = B;

  800. 	  temp = k;

  801. #endif

  802. 	  BLASLONG l = 0;

  803. 	  v4sf_t t = { 0, 0, 0, 0 };

  804. 	  for (l = 0; l < temp; l++)

  805. 	    {

  806. 	      v4sf_t rowA = { AO[l], AO[l], AO[l], AO[l] };

  807. 	      v4sf_t rowB = { BO[l << 2], BO[(l << 2) + 1], BO[(l << 2) + 2],

  808. 		BO[(l << 2) + 3]

  809. 	      };

  810. 	      t += rowA * rowB;

  811. 	    }

  812. 	  t = t * valpha;

  813. #if defined(TRMMKERNEL)

  814. 	  CO[0 * ldc] = t[0];

  815. 	  CO[1 * ldc] = t[1];

  816. 	  CO[2 * ldc] = t[2];

  817. 	  CO[3 * ldc] = t[3];

  818. #else

  819. 	  CO[0 * ldc] += t[0];

  820. 	  CO[1 * ldc] += t[1];

  821. 	  CO[2 * ldc] += t[2];

  822. 	  CO[3 * ldc] += t[3];

  823. #endif

  824. 	  CO += 1;

  825. 	  AO += temp;

  826. 	  BO += temp << 2;

  827. #if defined(TRMMKERNEL)

  828. 	  REFRESH_AFTER_SAVE (1, 4)

  829. #endif

  830. 	}

  831. #if defined(TRMMKERNEL) && !defined(LEFT)

  832.       off += 4;			// number of values in A

  833. #endif

  834. 

  835.       B += k << 2;

  836.     }

  837.   N = (n & 3) >> 1;

  838.   for (i1 = 0; i1 < N; i1++)

  839.     {

  840.       BLASLONG i, j, temp;

  841. #if defined(TRMMKERNEL) && defined(LEFT)

  842.       off = offset;

  843. #endif

  844.       FLOAT *CO;

  845.       FLOAT *AO;

  846.       CO = C;

  847.       C += ldc << 1;

  848.       AO = A;

  849. #if !defined(TRMMKERNEL)

  850.       i = m >> 5;

  851.       for (j = 0; j < i; j++)

  852. 	{

  853. 	  FLOAT *BO = B;

  854. 	  v4sf_t *rowC;

  855. 	  v4sf_t result[4];

  856. 	  FLOAT *A1;

  857. 	  A1 = AO + (16 * k);

  858. 	  __vector_quad acc0, acc1, acc2, acc3, acc4, acc5, acc6, acc7;

  859. 	  SET_ACC_ZERO8 ();

  860. 	  BLASLONG l = 0;

  861. 	  for (l = 0; l < k; l++)

  862. 	    {

  863. 	      FLOAT t[4] = { 0 };

  864. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  865. 	      vec_t *rowB = (vec_t *) & t[0];

  866. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  867. 	      vec_t *rowA1 = (vec_t *) & A1[l << 4];

  868. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  869. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  870. 	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);

  871. 	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);

  872. 	      __builtin_mma_xvf32gerpp (&acc4, rowB[0], rowA1[0]);

  873. 	      __builtin_mma_xvf32gerpp (&acc5, rowB[0], rowA1[1]);

  874. 	      __builtin_mma_xvf32gerpp (&acc6, rowB[0], rowA1[2]);

  875. 	      __builtin_mma_xvf32gerpp (&acc7, rowB[0], rowA1[3]);

  876. 	    }

  877. 	  SAVE2x4_ACC (&acc0, 0);

  878. 	  SAVE2x4_ACC (&acc1, 4);

  879. 	  SAVE2x4_ACC (&acc2, 8);

  880. 	  SAVE2x4_ACC (&acc3, 12);

  881. 	  CO += 16;

  882. 	  SAVE2x4_ACC (&acc4, 0);

  883. 	  SAVE2x4_ACC (&acc5, 4);

  884. 	  SAVE2x4_ACC (&acc6, 8);

  885. 	  SAVE2x4_ACC (&acc7, 12);

  886. 	  CO += 16;

  887. 	  AO += k << 5;

  888. 	  BO += k << 1;

  889. 	}

  890.       i = (m & 31) >> 4;

  891. #else

  892.       i = m >> 4;

  893. #endif

  894.       for (j = 0; j < i; j++)

  895. 	{

  896. 	  FLOAT *BO;

  897. 	  v4sf_t *rowC;

  898. 	  v4sf_t result[4];

  899. 	  __vector_quad acc0, acc1, acc2, acc3;

  900. 	  SET_ACC_ZERO4 ();

  901. 	  BLASLONG l = 0;

  902. #if defined(TRMMKERNEL)

  903. 	  REFRESH_POINTERS (16, 2)

  904. #else

  905. 	  BO = B;

  906. 	  temp = k;

  907. #endif

  908. 	  for (l = 0; l < temp; l++)

  909. 	    {

  910. 	      FLOAT t[4] = { 0 };

  911. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  912. 	      vec_t *rowB = (vec_t *) & t[0];

  913. 	      vec_t *rowA = (vec_t *) & AO[l << 4];

  914. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  915. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  916. 	      __builtin_mma_xvf32gerpp (&acc2, rowB[0], rowA[2]);

  917. 	      __builtin_mma_xvf32gerpp (&acc3, rowB[0], rowA[3]);

  918. 	    }

  919. 	  SAVE2x4_ACC (&acc0, 0);

  920. 	  SAVE2x4_ACC (&acc1, 4);

  921. 	  SAVE2x4_ACC (&acc2, 8);

  922. 	  SAVE2x4_ACC (&acc3, 12);

  923. 	  CO += 16;

  924. 	  AO += temp << 4;

  925. 	  BO += temp << 1;

  926. #if defined(TRMMKERNEL)

  927. 	  REFRESH_AFTER_SAVE (16, 2)

  928. #endif

  929. 	}

  930.       i = (m & 15) >> 3;

  931.       for (j = 0; j < i; j++)

  932. 	{

  933. 	  FLOAT *BO;

  934. 	  v4sf_t *rowC;

  935. 	  v4sf_t result[4];

  936. 	  __vector_quad acc0, acc1;

  937. 	  __builtin_mma_xxsetaccz (&acc0);

  938. 	  __builtin_mma_xxsetaccz (&acc1);

  939. #if defined(TRMMKERNEL)

  940. 	  REFRESH_POINTERS (8, 2)

  941. #else

  942. 	  BO = B;

  943. 	  temp = k;

  944. #endif

  945. 	  BLASLONG l = 0;

  946. 	  for (l = 0; l < temp; l++)

  947. 	    {

  948. 	      FLOAT t[4] = { 0 };

  949. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  950. 	      vec_t *rowB = (vec_t *) & t[0];

  951. 	      vec_t *rowA = (vec_t *) & AO[l << 3];

  952. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  953. 	      __builtin_mma_xvf32gerpp (&acc1, rowB[0], rowA[1]);

  954. 	    }

  955. 	  SAVE2x4_ACC (&acc0, 0);

  956. 	  SAVE2x4_ACC (&acc1, 4);

  957. 	  CO += 8;

  958. 	  AO += temp << 3;

  959. 	  BO += temp << 1;

  960. #if defined(TRMMKERNEL)

  961. 	  REFRESH_AFTER_SAVE (8, 2)

  962. #endif

  963. 	}

  964.       i = (m & 7) >> 2;

  965.       for (j = 0; j < i; j++)

  966. 	{

  967. 	  FLOAT *BO;

  968. 	  v4sf_t *rowC;

  969. 	  v4sf_t result[4];

  970. 	  __vector_quad acc0;

  971. 	  __builtin_mma_xxsetaccz (&acc0);

  972. #if defined(TRMMKERNEL)

  973. 	  REFRESH_POINTERS (4, 2)

  974. #else

  975. 	  BO = B;

  976. 	  temp = k;

  977. #endif

  978. 	  BLASLONG l = 0;

  979. 	  for (l = 0; l < temp; l++)

  980. 	    {

  981. 	      FLOAT t[4] = { 0 };

  982. 	      t[0] = BO[l << 1], t[1] = BO[(l << 1) + 1];

  983. 	      vec_t *rowB = (vec_t *) & t[0];

  984. 	      vec_t *rowA = (vec_t *) & AO[l << 2];

  985. 	      __builtin_mma_xvf32gerpp (&acc0, rowB[0], rowA[0]);

  986. 	    }

  987. 	  SAVE2x4_ACC (&acc0, 0);

  988. 	  CO += 4;

  989. 	  AO += temp << 2;

  990. 	  BO += temp << 1;

  991. #if defined(TRMMKERNEL)

  992. 	  REFRESH_AFTER_SAVE (4, 2)

  993. #endif

  994. 	}

  995.       i = (m & 3) >> 1;

  996.       for (j = 0; j < i; j++)

  997. 	{

  998. 	  FLOAT *BO;

  999. 	  BLASLONG l = 0;

  1000. #if defined(TRMMKERNEL)

  1001. 	  REFRESH_POINTERS (2, 2)

  1002. #else

  1003. 	  BO = B;

  1004. 	  temp = k;

  1005. #endif

  1006. 	  v4sf_t t = { 0, 0, 0, 0 };

  1007. 	  for (l = 0; l < (temp << 1); l += 2)

  1008. 	    {

  1009. 	      v4sf_t rowA = { AO[l], AO[l], AO[l + 1], AO[l + 1] };

  1010. 	      v4sf_t rowB = { BO[l], BO[l + 1], BO[l], BO[l + 1] };

  1011. 	      t += rowA * rowB;

  1012. 	    }

  1013. 	  t = t * valpha;

  1014. #if defined(TRMMKERNEL)

  1015. 	  CO[0 * ldc] = t[0];

  1016. 	  CO[1 * ldc] = t[1];

  1017. 	  CO[0 * ldc + 1] = t[2];

  1018. 	  CO[1 * ldc + 1] = t[3];

  1019. #else

  1020. 	  CO[0 * ldc] += t[0];

  1021. 	  CO[1 * ldc] += t[1];

  1022. 	  CO[0 * ldc + 1] += t[2];

  1023. 	  CO[1 * ldc + 1] += t[3];

  1024. #endif

  1025. 	  CO += 2;

  1026. 	  AO += temp << 1;

  1027. 	  BO += temp << 1;

  1028. #if defined(TRMMKERNEL)

  1029. 	  REFRESH_AFTER_SAVE (2, 2)

  1030. #endif

  1031. 	}

  1032.       i = (m & 1) >> 0;

  1033.       for (j = 0; j < i; j++)

  1034. 	{

  1035. 	  FLOAT *BO;

  1036. 	  BLASLONG l = 0;

  1037. #if defined(TRMMKERNEL)

  1038. 	  REFRESH_POINTERS (1, 2)

  1039. #else

  1040. 	  BO = B;

  1041. 	  temp = k;

  1042. #endif

  1043. 	  v4sf_t t = { 0, 0, 0, 0 };

  1044. 	  for (l = 0; l < temp; l++)

  1045. 	    {

  1046. 	      v4sf_t rowA = { AO[l], AO[l], 0, 0 };

  1047. 	      v4sf_t rowB = { BO[l << 1], BO[(l << 1) + 1], 0, 0 };

  1048. 	      t += rowA * rowB;

  1049. 	    }

  1050. 	  t = t * valpha;

  1051. #if defined(TRMMKERNEL)

  1052. 	  CO[0 * ldc] = t[0];

  1053. 	  CO[1 * ldc] = t[1];

  1054. #else

  1055. 	  CO[0 * ldc] += t[0];

  1056. 	  CO[1 * ldc] += t[1];

  1057. #endif

  1058. 	  CO += 1;

  1059. 	  AO += temp;

  1060. 	  BO += temp << 1;

  1061. #if defined(TRMMKERNEL)

  1062. 	  REFRESH_AFTER_SAVE (1, 2)

  1063. #endif

  1064. 	}

  1065. #if defined(TRMMKERNEL) && !defined(LEFT)

  1066.       off += 2;			// number of values in A

  1067. #endif

  1068. 

  1069.       B += k << 1;

  1070.     }

  1071.   N = (n & 1) >> 0;

  1072.   for (i1 = 0; i1 < N; i1++)

  1073.     {

  1074.       BLASLONG i, temp;

  1075. #if defined(TRMMKERNEL) && defined(LEFT)

  1076.       off = offset;

  1077. #endif

  1078.       FLOAT *CO;

  1079.       FLOAT *AO;

  1080.       CO = C;

  1081.       C += ldc;

  1082.       AO = A;

  1083.       i = m;

  1084.       while (i >= 16)

  1085. 	{

  1086. 	  FLOAT *BO;

  1087. 	  BLASLONG l = 0;

  1088. #if defined(TRMMKERNEL)

  1089. 	  REFRESH_POINTERS (16, 1)

  1090. #else

  1091. 	  BO = B;

  1092. 	  temp = k;

  1093. #endif

  1094. 

  1095. 	  v4sf_t t = { 0, 0, 0, 0 };

  1096. 	  v4sf_t t1 = { 0, 0, 0, 0 };

  1097. 	  v4sf_t t2 = { 0, 0, 0, 0 };

  1098. 	  v4sf_t t3 = { 0, 0, 0, 0 };

  1099. 	  for (l = 0; l < temp; l++)

  1100. 	    {

  1101. 	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };

  1102. 	      v4sf_t rowA = { AO[l << 4], AO[(l << 4) + 1], AO[(l << 4) + 2],

  1103. 		AO[(l << 4) + 3]

  1104. 	      };

  1105. 	      v4sf_t rowA1 =

  1106. 		{ AO[(l << 4) + 4], AO[(l << 4) + 5], AO[(l << 4) + 6],

  1107. 		AO[(l << 4) + 7]

  1108. 	      };

  1109. 	      v4sf_t rowA2 =

  1110. 		{ AO[(l << 4) + 8], AO[(l << 4) + 9], AO[(l << 4) + 10],

  1111. 		AO[(l << 4) + 11]

  1112. 	      };

  1113. 	      v4sf_t rowA3 =

  1114. 		{ AO[(l << 4) + 12], AO[(l << 4) + 13], AO[(l << 4) + 14],

  1115. 		AO[(l << 4) + 15]

  1116. 	      };

  1117. 	      t += rowA * rowB;

  1118. 	      t1 += rowA1 * rowB;

  1119. 	      t2 += rowA2 * rowB;

  1120. 	      t3 += rowA3 * rowB;

  1121. 	    }

  1122. 	  t = t * valpha;

  1123. 	  t1 = t1 * valpha;

  1124. 	  t2 = t2 * valpha;

  1125. 	  t3 = t3 * valpha;

  1126. #if defined(TRMMKERNEL)

  1127. 	  CO[0] = t[0];

  1128. 	  CO[1] = t[1];

  1129. 	  CO[2] = t[2];

  1130. 	  CO[3] = t[3];

  1131. 	  CO[4] = t1[0];

  1132. 	  CO[5] = t1[1];

  1133. 	  CO[6] = t1[2];

  1134. 	  CO[7] = t1[3];

  1135. 	  CO[8] = t2[0];

  1136. 	  CO[9] = t2[1];

  1137. 	  CO[10] = t2[2];

  1138. 	  CO[11] = t2[3];

  1139. 	  CO[12] = t3[0];

  1140. 	  CO[13] = t3[1];

  1141. 	  CO[14] = t3[2];

  1142. 	  CO[15] = t3[3];

  1143. #else

  1144. 	  CO[0] += t[0];

  1145. 	  CO[1] += t[1];

  1146. 	  CO[2] += t[2];

  1147. 	  CO[3] += t[3];

  1148. 	  CO[4] += t1[0];

  1149. 	  CO[5] += t1[1];

  1150. 	  CO[6] += t1[2];

  1151. 	  CO[7] += t1[3];

  1152. 	  CO[8] += t2[0];

  1153. 	  CO[9] += t2[1];

  1154. 	  CO[10] += t2[2];

  1155. 	  CO[11] += t2[3];

  1156. 	  CO[12] += t3[0];

  1157. 	  CO[13] += t3[1];

  1158. 	  CO[14] += t3[2];

  1159. 	  CO[15] += t3[3];

  1160. #endif

  1161. 	  AO += temp << 4;

  1162. 	  BO += temp;

  1163. 	  CO += 16;

  1164. 	  i -= 16;

  1165. #if defined(TRMMKERNEL)

  1166. 	  REFRESH_AFTER_SAVE (16, 1)

  1167. #endif

  1168. 	}

  1169.       while (i >= 8)

  1170. 	{

  1171. 	  FLOAT *BO;

  1172. 	  BLASLONG l = 0;

  1173. 	  v4sf_t t = { 0, 0, 0, 0 };

  1174. 	  v4sf_t t1 = { 0, 0, 0, 0 };

  1175. #if defined(TRMMKERNEL)

  1176. 	  REFRESH_POINTERS (8, 1)

  1177. #else

  1178. 	  BO = B;

  1179. 	  temp = k;

  1180. #endif

  1181. 

  1182. 	  for (l = 0; l < temp; l++)

  1183. 	    {

  1184. 	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };

  1185. 	      v4sf_t rowA = { AO[l << 3], AO[(l << 3) + 1], AO[(l << 3) + 2],

  1186. 		AO[(l << 3) + 3]

  1187. 	      };

  1188. 	      v4sf_t rowA1 =

  1189. 		{ AO[(l << 3) + 4], AO[(l << 3) + 5], AO[(l << 3) + 6],

  1190. 		AO[(l << 3) + 7]

  1191. 	      };

  1192. 	      t += rowA * rowB;

  1193. 	      t1 += rowA1 * rowB;

  1194. 	    }

  1195. 	  t = t * valpha;

  1196. 	  t1 = t1 * valpha;

  1197. #if defined(TRMMKERNEL)

  1198. 	  CO[0] = t[0];

  1199. 	  CO[1] = t[1];

  1200. 	  CO[2] = t[2];

  1201. 	  CO[3] = t[3];

  1202. 	  CO[4] = t1[0];

  1203. 	  CO[5] = t1[1];

  1204. 	  CO[6] = t1[2];

  1205. 	  CO[7] = t1[3];

  1206. #else

  1207. 	  CO[0] += t[0];

  1208. 	  CO[1] += t[1];

  1209. 	  CO[2] += t[2];

  1210. 	  CO[3] += t[3];

  1211. 	  CO[4] += t1[0];

  1212. 	  CO[5] += t1[1];

  1213. 	  CO[6] += t1[2];

  1214. 	  CO[7] += t1[3];

  1215. #endif

  1216. 	  AO += temp << 3;

  1217. 	  BO += temp;

  1218. 	  CO += 8;

  1219. 	  i -= 8;

  1220. #if defined(TRMMKERNEL)

  1221. 	  REFRESH_AFTER_SAVE (8, 1)

  1222. #endif

  1223. 	}

  1224.       while (i >= 4)

  1225. 	{

  1226. 	  FLOAT *BO;

  1227. 	  BLASLONG l = 0;

  1228. 	  v4sf_t t = { 0, 0, 0, 0 };

  1229. #if defined(TRMMKERNEL)

  1230. 	  REFRESH_POINTERS (4, 1)

  1231. #else

  1232. 	  BO = B;

  1233. 	  temp = k;

  1234. #endif

  1235. 

  1236. 	  for (l = 0; l < temp; l++)

  1237. 	    {

  1238. 	      v4sf_t rowB = { BO[l], BO[l], BO[l], BO[l] };

  1239. 	      v4sf_t rowA = { AO[l << 2], AO[(l << 2) + 1], AO[(l << 2) + 2],

  1240. 		AO[(l << 2) + 3]

  1241. 	      };

  1242. 	      t += rowA * rowB;

  1243. 	    }

  1244. 	  t = t * valpha;

  1245. #if defined(TRMMKERNEL)

  1246. 	  CO[0] = t[0];

  1247. 	  CO[1] = t[1];

  1248. 	  CO[2] = t[2];

  1249. 	  CO[3] = t[3];

  1250. #else

  1251. 	  CO[0] += t[0];

  1252. 	  CO[1] += t[1];

  1253. 	  CO[2] += t[2];

  1254. 	  CO[3] += t[3];

  1255. #endif

  1256. 	  AO += temp << 2;

  1257. 	  BO += temp;

  1258. 	  CO += 4;

  1259. 	  i -= 4;

  1260. #if defined(TRMMKERNEL)

  1261. 	  REFRESH_AFTER_SAVE (4, 1)

  1262. #endif

  1263. 	}

  1264.       while (i >= 2)

  1265. 	{

  1266. 	  FLOAT *BO;

  1267. 	  BLASLONG l = 0;

  1268. #if defined(TRMMKERNEL)

  1269. 	  REFRESH_POINTERS (2, 1)

  1270. #else

  1271. 	  BO = B;

  1272. 	  temp = k;

  1273. #endif

  1274. 

  1275. 	  v4sf_t t = { 0, 0, 0, 0 };

  1276. 	  for (l = 0; l < temp; l++)

  1277. 	    {

  1278. 	      v4sf_t rowB = { BO[l], BO[l], 0, 0 };

  1279. 	      v4sf_t rowA = { AO[l << 1], AO[(l << 1) + 1], 0, 0 };

  1280. 	      t += rowA * rowB;

  1281. 	    }

  1282. 	  t = t * valpha;

  1283. #if defined(TRMMKERNEL)

  1284. 	  CO[0] = t[0];

  1285. 	  CO[1] = t[1];

  1286. #else

  1287. 	  CO[0] += t[0];

  1288. 	  CO[1] += t[1];

  1289. #endif

  1290. 	  AO += temp << 1;

  1291. 	  BO += temp;

  1292. 	  CO += 2;

  1293. 	  i -= 2;

  1294. #if defined(TRMMKERNEL)

  1295. 	  REFRESH_AFTER_SAVE (2, 1)

  1296. #endif

  1297. 	}

  1298.       while (i >= 1)

  1299. 	{

  1300. 	  FLOAT *BO;

  1301. #if defined(TRMMKERNEL)

  1302. 	  REFRESH_POINTERS (1, 1)

  1303. #else

  1304. 	  BO = B;

  1305. 	  temp = k;

  1306. #endif

  1307. 

  1308. 	  BLASLONG l = 0;

  1309. 	  FLOAT t = 0;

  1310. 	  for (l = 0; l < temp; l++)

  1311. 	    {

  1312. 	      t += AO[l] * BO[l];

  1313. 	    }

  1314. 	  AO += temp;

  1315. 	  BO += temp;

  1316. #if defined(TRMMKERNEL)

  1317. 	  CO[0] = t * alpha;

  1318. #else

  1319. 	  CO[0] += t * alpha;

  1320. #endif

  1321. 	  CO += 1;

  1322. 	  i -= 1;

  1323. #if defined(TRMMKERNEL)

  1324. 	  REFRESH_AFTER_SAVE (1, 1)

  1325. #endif

  1326. 	}

  1327. 

  1328. #if defined(TRMMKERNEL) && !defined(LEFT)

  1329.       off += 1;			// number of values in A

  1330. #endif

  1331.       B += k;

  1332.     }

  1333.   return 0;

  1334. }