OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 180ba5e7d0
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '180ba5e7d0'
${ noResults }
OpenBLAS/kernel/x86_64/dtrmm_kernel_4x8_haswell.c

1547 lines
27 kB
Raw Blame History 

  1. #include "common.h"

  2. #include <stdbool.h>

  3. 

  4. 

  5. static void dtrmm_kernel_4x8( BLASLONG n, FLOAT *alpha ,FLOAT *a, FLOAT *b, FLOAT *C0, FLOAT *C1, FLOAT *C2,FLOAT *C3, FLOAT *C4, FLOAT *C5,FLOAT *C6, FLOAT *C7) __attribute__ ((noinline));

  6. 

  7. static void dtrmm_kernel_4x8( BLASLONG n, FLOAT *alpha ,FLOAT *a, FLOAT *b, FLOAT *C0, FLOAT *C1, FLOAT *C2,FLOAT *C3, FLOAT *C4, FLOAT *C5,FLOAT *C6, FLOAT *C7)

  8. {

  9. 

  10. 		BLASLONG i = 0;

  11. 		BLASLONG temp1 = n * 8;

  12. 

  13. 		 __asm__  __volatile__

  14.         	(

  15. 		"	vxorpd	%%ymm4 , %%ymm4 , %%ymm4			\n\t"

  16. 		"	vxorpd	%%ymm5 , %%ymm5 , %%ymm5			\n\t"

  17. 		"	vxorpd	%%ymm6 , %%ymm6 , %%ymm6			\n\t"

  18. 		"	vxorpd	%%ymm7 , %%ymm7 , %%ymm7			\n\t"

  19. 		"	vxorpd	%%ymm8 , %%ymm8 , %%ymm8			\n\t"

  20. 		"	vxorpd	%%ymm9 , %%ymm9 , %%ymm9			\n\t"

  21. 		"	vxorpd	%%ymm10, %%ymm10, %%ymm10			\n\t"

  22. 		"	vxorpd	%%ymm11, %%ymm11, %%ymm11			\n\t"

  23. 

  24. 		"	cmp $0, %1						\n\t"

  25. 		"	jz 2f							\n\t"

  26. 

  27. 		"	.p2align 4						\n\t"

  28. 		"1:								\n\t"

  29. 		"	vmovups   	(%2,%0,4) , %%ymm0			\n\t"

  30. 		"	vmovups   	(%3,%0,8) , %%ymm1			\n\t"

  31. 		"	vmovups       32(%3,%0,8) , %%ymm2			\n\t"

  32. 

  33. 		"	vfmadd231pd	%%ymm0 , %%ymm1 , %%ymm4		\n\t"

  34. 		"	vfmadd231pd	%%ymm0 , %%ymm2 , %%ymm8		\n\t"

  35. 

  36. 		"	vpermilpd         $0x05  , %%ymm0 , %%ymm0		\n\t"

  37. 		"	vfmadd231pd	%%ymm0 , %%ymm1 , %%ymm5		\n\t"

  38. 		"	vfmadd231pd	%%ymm0 , %%ymm2 , %%ymm9		\n\t"

  39. 

  40. 		"	vpermpd         $0x1b  , %%ymm0 , %%ymm0		\n\t"

  41. 		"	vfmadd231pd	%%ymm0 , %%ymm1 , %%ymm6		\n\t"

  42. 		"	vfmadd231pd	%%ymm0 , %%ymm2 , %%ymm10		\n\t"

  43. 

  44. 		"	vpermilpd         $0x05  , %%ymm0 , %%ymm0		\n\t"

  45. 		"	vfmadd231pd	%%ymm0 , %%ymm1 , %%ymm7		\n\t"

  46. 		"	vfmadd231pd	%%ymm0 , %%ymm2 , %%ymm11		\n\t"

  47. 

  48. 		"	addq	$8 , %0						\n\t"

  49. 		"	cmp	%0 , %1						\n\t"

  50. 		"	jne	1b						\n\t"

  51. 

  52. 		"2:								\n\t"

  53. 

  54. 		"	vbroadcastsd	(%4), %%ymm0				\n\t"

  55. 

  56. 		"	vmulpd		%%ymm0 , %%ymm4 , %%ymm4		\n\t"

  57. 		"	vmulpd		%%ymm0 , %%ymm5 , %%ymm5		\n\t"

  58. 		"	vmulpd		%%ymm0 , %%ymm6 , %%ymm6		\n\t"

  59. 		"	vmulpd		%%ymm0 , %%ymm7 , %%ymm7		\n\t"

  60. 		"	vmulpd		%%ymm0 , %%ymm8 , %%ymm8		\n\t"

  61. 		"	vmulpd		%%ymm0 , %%ymm9 , %%ymm9		\n\t"

  62. 		"	vmulpd		%%ymm0 , %%ymm10, %%ymm10		\n\t"

  63. 		"	vmulpd		%%ymm0 , %%ymm11, %%ymm11		\n\t"

  64. 

  65. 		"	vpermilpd 	$0x05  , %%ymm5 , %%ymm5		\n\t"

  66. 		"	vpermilpd 	$0x05  , %%ymm7 , %%ymm7		\n\t"

  67. 

  68. 		"	vblendpd 	$0x0a  , %%ymm5 , %%ymm4 , %%ymm0	\n\t"

  69. 		"	vblendpd 	$0x05  , %%ymm5 , %%ymm4 , %%ymm1	\n\t"

  70. 		"	vblendpd 	$0x0a  , %%ymm7 , %%ymm6 , %%ymm2	\n\t"

  71. 		"	vblendpd 	$0x05  , %%ymm7 , %%ymm6 , %%ymm3	\n\t"

  72. 

  73. 		"	vperm2f128 	$0x01  , %%ymm2 , %%ymm2 , %%ymm2	\n\t"

  74. 		"	vperm2f128 	$0x01  , %%ymm3 , %%ymm3 , %%ymm3	\n\t"

  75. 

  76. 		"	vblendpd 	$0x03  , %%ymm0 , %%ymm2 , %%ymm4	\n\t"

  77. 		"	vblendpd 	$0x03  , %%ymm1 , %%ymm3 , %%ymm5	\n\t"

  78. 		"	vblendpd 	$0x03  , %%ymm2 , %%ymm0 , %%ymm6	\n\t"

  79. 		"	vblendpd 	$0x03  , %%ymm3 , %%ymm1 , %%ymm7	\n\t"

  80. 

  81. 		"	vmovups		%%ymm4 , (%5)				\n\t"

  82. 		"	vmovups		%%ymm5 , (%6)				\n\t"

  83. 		"	vmovups		%%ymm6 , (%7)				\n\t"

  84. 		"	vmovups		%%ymm7 , (%8)				\n\t"

  85. 

  86. 		"	vpermilpd 	$0x05  , %%ymm9 , %%ymm9		\n\t"

  87. 		"	vpermilpd 	$0x05  , %%ymm11, %%ymm11		\n\t"

  88. 

  89. 		"	vblendpd 	$0x0a  , %%ymm9 , %%ymm8 , %%ymm0	\n\t"

  90. 		"	vblendpd 	$0x05  , %%ymm9 , %%ymm8 , %%ymm1	\n\t"

  91. 		"	vblendpd 	$0x0a  , %%ymm11, %%ymm10, %%ymm2	\n\t"

  92. 		"	vblendpd 	$0x05  , %%ymm11, %%ymm10, %%ymm3	\n\t"

  93. 

  94. 		"	vperm2f128 	$0x01  , %%ymm2 , %%ymm2 , %%ymm2	\n\t"

  95. 		"	vperm2f128 	$0x01  , %%ymm3 , %%ymm3 , %%ymm3	\n\t"

  96. 

  97. 		"	vblendpd 	$0x03  , %%ymm0 , %%ymm2 , %%ymm4	\n\t"

  98. 		"	vblendpd 	$0x03  , %%ymm1 , %%ymm3 , %%ymm5	\n\t"

  99. 		"	vblendpd 	$0x03  , %%ymm2 , %%ymm0 , %%ymm6	\n\t"

  100. 		"	vblendpd 	$0x03  , %%ymm3 , %%ymm1 , %%ymm7	\n\t"

  101. 

  102. 		"	vmovups		%%ymm4 , (%9)				\n\t"

  103. 		"	vmovups		%%ymm5 , (%10)				\n\t"

  104. 		"	vmovups		%%ymm6 , (%11)				\n\t"

  105. 		"	vmovups		%%ymm7 , (%12)				\n\t"

  106. 

  107. 	        :

  108.         	:

  109. 		"a" (i),	 // 0

  110.           	"r" (temp1),     // 1    

  111.           	"S" (a),         // 2

  112.           	"D" (b),         // 3

  113.           	"r" (alpha),     // 4

  114. 		"r" (C0),	 // 5

  115. 		"r" (C1),	 // 6

  116. 		"r" (C2),	 // 7

  117. 		"r" (C3),	 // 8

  118. 		"r" (C4),	 // 9

  119. 		"r" (C5),	 // 10

  120. 		"r" (C6),	 // 11

  121. 		"r" (C7) 	 // 12

  122. 		: "cc",

  123.           	"%xmm0", "%xmm1", "%xmm2", "%xmm3",

  124.           	"%xmm4", "%xmm5", "%xmm6", "%xmm7",

  125.           	"%xmm8", "%xmm9", "%xmm10", "%xmm11",

  126.           	"%xmm12", "%xmm13", "%xmm14", "%xmm15",

  127.           	"memory"

  128.         	);

  129. 

  130. 

  131. 

  132. 

  133. }

  134. 

  135. 

  136. 

  137. 

  138. int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc ,BLASLONG offset)

  139. {

  140. 

  141.    BLASLONG i,j,k;

  142.    FLOAT *C0,*C1,*C2,*C3,*C4,*C5,*C6,*C7,*ptrba,*ptrbb;

  143. 

  144.    FLOAT res0_0;

  145.    FLOAT res0_1;

  146.    FLOAT res0_2;

  147.    FLOAT res0_3;

  148. 

  149.    FLOAT res1_0;

  150.    FLOAT res1_1;

  151.    FLOAT res1_2;

  152.    FLOAT res1_3;

  153. 

  154.    FLOAT res2_0;

  155.    FLOAT res2_1;

  156.    FLOAT res2_2;

  157.    FLOAT res2_3;

  158. 

  159.    FLOAT res3_0;

  160.    FLOAT res3_1;

  161.    FLOAT res3_2;

  162.    FLOAT res3_3;

  163. 

  164.    FLOAT res4_0;

  165.    FLOAT res4_1;

  166. /*

  167.    FLOAT res4_2;

  168.    FLOAT res4_3;

  169. */

  170.    FLOAT res5_0;

  171.    FLOAT res5_1;

  172. /*

  173.    FLOAT res5_2;

  174.    FLOAT res5_3;

  175. */

  176.    FLOAT res6_0;

  177.    FLOAT res6_1;

  178. /*

  179.    FLOAT res6_2;

  180.    FLOAT res6_3;

  181. */

  182.    FLOAT res7_0;

  183.    FLOAT res7_1;

  184. /*

  185.    FLOAT res7_2;

  186.    FLOAT res7_3;

  187. */

  188.    FLOAT a0;

  189.    FLOAT a1;

  190. 

  191.    FLOAT b0;

  192.    FLOAT b1;

  193.    FLOAT b2;

  194.    FLOAT b3;

  195.    FLOAT b4;

  196.    FLOAT b5;

  197.    FLOAT b6;

  198.    FLOAT b7;

  199. 

  200.    BLASLONG off, temp ;

  201. 

  202.    bool left;

  203.    bool transposed;

  204.    bool backwards;

  205. 

  206. #ifdef LEFT

  207.    left = true;

  208. #else

  209.    left = false;

  210. #endif

  211. 

  212. #ifdef TRANSA

  213.    transposed = true;

  214. #else

  215.    transposed = false;

  216. #endif

  217. 

  218.    backwards = left != transposed;

  219. 

  220.    if (!left) {

  221.       off = -offset;

  222.    }

  223. 

  224. 

  225.    for (j=0; j<bn/8; j+=1) // do blocks of the Mx8 loops 

  226.    {

  227.         C0 = C;

  228.         C1 = C0+ldc;

  229.         C2 = C1+ldc;

  230.         C3 = C2+ldc;

  231.         C4 = C3+ldc;

  232.         C5 = C4+ldc;

  233.         C6 = C5+ldc;

  234.         C7 = C6+ldc;

  235. 

  236. 

  237.         if (left) {

  238.             off = offset;

  239.         }

  240. 

  241.         ptrba = ba;

  242. 

  243.         for (i=0; i<bm/4; i+=1) // do blocks of 4x4

  244. 	{

  245. 

  246. 		ptrbb = bb;

  247.                 if (backwards)

  248.                 {

  249. 		   ptrba += off*4; // number of values in A

  250. 		   ptrbb += off*8; // number of values in B

  251.                 }

  252. /*

  253. 		res0_0 = 0;

  254. 		res0_1 = 0;

  255. 		res0_2 = 0;

  256. 		res0_3 = 0;

  257. 

  258. 		res1_0 = 0;

  259. 		res1_1 = 0;

  260. 		res1_2 = 0;

  261. 		res1_3 = 0;

  262. 

  263. 		res2_0 = 0;

  264. 		res2_1 = 0;

  265. 		res2_2 = 0;

  266. 		res2_3 = 0;

  267. 

  268. 		res3_0 = 0;

  269. 		res3_1 = 0;

  270. 		res3_2 = 0;

  271. 		res3_3 = 0;

  272. 

  273. 		res4_0 = 0;

  274. 		res4_1 = 0;

  275. 		res4_2 = 0;

  276. 		res4_3 = 0;

  277. 

  278. 		res5_0 = 0;

  279. 		res5_1 = 0;

  280. 		res5_2 = 0;

  281. 		res5_3 = 0;

  282. 

  283. 		res6_0 = 0;

  284. 		res6_1 = 0;

  285. 		res6_2 = 0;

  286. 		res6_3 = 0;

  287. 

  288. 		res7_0 = 0;

  289. 		res7_1 = 0;

  290. 		res7_2 = 0;

  291. 		res7_3 = 0;

  292. */

  293.                 temp = backwards ? bk-off :

  294.                              left ? off + 4 : // number of values in A

  295.                                     off + 8;  // number of values in B

  296. 

  297. 	        dtrmm_kernel_4x8( temp, &alpha , ptrba, ptrbb, C0, C1, C2, C3, C4, C5, C6, C7);

  298. 

  299. 		ptrba = ptrba + temp * 4;

  300. 		ptrbb = ptrbb + temp * 8;

  301. 

  302. /*

  303. 		for (k=0; k<temp; k++)

  304.                 {

  305. 			b0 = ptrbb[0];

  306. 			b1 = ptrbb[1];

  307. 			b2 = ptrbb[2];

  308. 			b3 = ptrbb[3];

  309. 			b4 = ptrbb[4];

  310. 			b5 = ptrbb[5];

  311. 			b6 = ptrbb[6];

  312. 			b7 = ptrbb[7];

  313. 

  314. 			a0 = ptrba[0];

  315. 			res0_0 += a0*b0;

  316. 			res1_0 += a0*b1;

  317. 			res2_0 += a0*b2;

  318. 			res3_0 += a0*b3;

  319. 			res4_0 += a0*b4;

  320. 			res5_0 += a0*b5;

  321. 			res6_0 += a0*b6;

  322. 			res7_0 += a0*b7;

  323. 

  324. 			a1 = ptrba[1];

  325. 			res0_1 += a1*b0;

  326. 			res1_1 += a1*b1;

  327. 			res2_1 += a1*b2;

  328. 			res3_1 += a1*b3;

  329. 			res4_1 += a1*b4;

  330. 			res5_1 += a1*b5;

  331. 			res6_1 += a1*b6;

  332. 			res7_1 += a1*b7;

  333. 

  334. 			a0 = ptrba[2];

  335. 			res0_2 += a0*b0;

  336. 			res1_2 += a0*b1;

  337. 			res2_2 += a0*b2;

  338. 			res3_2 += a0*b3;

  339. 			res4_2 += a0*b4;

  340. 			res5_2 += a0*b5;

  341. 			res6_2 += a0*b6;

  342. 			res7_2 += a0*b7;

  343. 

  344. 			a1 = ptrba[3];

  345. 			res0_3 += a1*b0;

  346. 			res1_3 += a1*b1;

  347. 			res2_3 += a1*b2;

  348. 			res3_3 += a1*b3;

  349. 			res4_3 += a1*b4;

  350. 			res5_3 += a1*b5;

  351. 			res6_3 += a1*b6;

  352. 			res7_3 += a1*b7;

  353. 

  354. 			ptrba = ptrba+4;

  355. 			ptrbb = ptrbb+8;

  356.                 }

  357. 

  358. 		res0_0 *= alpha;

  359. 		res0_1 *= alpha;

  360. 		res0_2 *= alpha;

  361. 		res0_3 *= alpha;

  362. 

  363. 		res1_0 *= alpha;

  364. 		res1_1 *= alpha;

  365. 		res1_2 *= alpha;

  366. 		res1_3 *= alpha;

  367. 

  368. 		res2_0 *= alpha;

  369. 		res2_1 *= alpha;

  370. 		res2_2 *= alpha;

  371. 		res2_3 *= alpha;

  372. 

  373. 		res3_0 *= alpha;

  374. 		res3_1 *= alpha;

  375. 		res3_2 *= alpha;

  376. 		res3_3 *= alpha;

  377. 

  378. 		res4_0 *= alpha;

  379. 		res4_1 *= alpha;

  380. 		res4_2 *= alpha;

  381. 		res4_3 *= alpha;

  382. 

  383. 		res5_0 *= alpha;

  384. 		res5_1 *= alpha;

  385. 		res5_2 *= alpha;

  386. 		res5_3 *= alpha;

  387. 

  388. 		res6_0 *= alpha;

  389. 		res6_1 *= alpha;

  390. 		res6_2 *= alpha;

  391. 		res6_3 *= alpha;

  392. 

  393. 		res7_0 *= alpha;

  394. 		res7_1 *= alpha;

  395. 		res7_2 *= alpha;

  396. 		res7_3 *= alpha;

  397. 

  398. 

  399. 		C0[0] = res0_0;

  400. 		C0[1] = res0_1;

  401. 		C0[2] = res0_2;

  402. 		C0[3] = res0_3;

  403. 

  404. 		C1[0] = res1_0;

  405. 		C1[1] = res1_1;

  406. 		C1[2] = res1_2;

  407. 		C1[3] = res1_3;

  408. 

  409. 		C2[0] = res2_0;

  410. 		C2[1] = res2_1;

  411. 		C2[2] = res2_2;

  412. 		C2[3] = res2_3;

  413. 

  414. 		C3[0] = res3_0;

  415. 		C3[1] = res3_1;

  416. 		C3[2] = res3_2;

  417. 		C3[3] = res3_3;

  418. 

  419. 		C4[0] = res4_0;

  420. 		C4[1] = res4_1;

  421. 		C4[2] = res4_2;

  422. 		C4[3] = res4_3;

  423. 

  424. 		C5[0] = res5_0;

  425. 		C5[1] = res5_1;

  426. 		C5[2] = res5_2;

  427. 		C5[3] = res5_3;

  428. 

  429. 		C6[0] = res6_0;

  430. 		C6[1] = res6_1;

  431. 		C6[2] = res6_2;

  432. 		C6[3] = res6_3;

  433. 

  434. 		C7[0] = res7_0;

  435. 		C7[1] = res7_1;

  436. 		C7[2] = res7_2;

  437. 		C7[3] = res7_3;

  438. */

  439. 		if (!backwards) {

  440.                     temp = bk-off;

  441.                     temp = left ? temp - 4 : // number of values in A

  442.                                   temp - 8;  // number of values in B

  443. 

  444.                     ptrba += temp*4; // number of values in A

  445. 		    ptrbb += temp*8; // number of values in B

  446.                 }

  447. #ifdef LEFT

  448. 		off += 4; // number of values in A

  449. #endif

  450. 

  451. 		C0 = C0+4;

  452. 		C1 = C1+4;

  453. 		C2 = C2+4;

  454. 		C3 = C3+4;

  455. 		C4 = C4+4;

  456. 		C5 = C5+4;

  457. 		C6 = C6+4;

  458. 		C7 = C7+4;

  459. 

  460. 	}

  461. 

  462. 	if ( bm & 2 ) // do any 2x4 loop

  463. 	{

  464. 

  465. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  466. 		ptrbb = bb;

  467. #else

  468. 		ptrba += off*2;

  469. 		ptrbb = bb + off*8;

  470. #endif

  471. 

  472. 		res0_0 = 0;

  473. 		res0_1 = 0;

  474. 

  475. 		res1_0 = 0;

  476. 		res1_1 = 0;

  477. 

  478. 		res2_0 = 0;

  479. 		res2_1 = 0;

  480. 

  481. 		res3_0 = 0;

  482. 		res3_1 = 0;

  483. 

  484. 		res4_0 = 0;

  485. 		res4_1 = 0;

  486. 

  487. 		res5_0 = 0;

  488. 		res5_1 = 0;

  489. 

  490. 		res6_0 = 0;

  491. 		res6_1 = 0;

  492. 

  493. 		res7_0 = 0;

  494. 		res7_1 = 0;

  495. 

  496. 

  497. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  498. 		temp = bk-off;

  499. #elif defined(LEFT)

  500. 		temp = off+2;	// number of values in A

  501. #else

  502. 		temp = off+8;	// number of values in B

  503. #endif

  504. 

  505. 		for (k=0; k<temp; k++)

  506.                 {

  507. 			b0 = ptrbb[0];

  508. 			b1 = ptrbb[1];

  509. 			b2 = ptrbb[2];

  510. 			b3 = ptrbb[3];

  511. 			b4 = ptrbb[4];

  512. 			b5 = ptrbb[5];

  513. 			b6 = ptrbb[6];

  514. 			b7 = ptrbb[7];

  515. 

  516. 			a0 = ptrba[0];

  517. 			res0_0 += a0*b0;

  518. 			res1_0 += a0*b1;

  519. 			res2_0 += a0*b2;

  520. 			res3_0 += a0*b3;

  521. 			res4_0 += a0*b4;

  522. 			res5_0 += a0*b5;

  523. 			res6_0 += a0*b6;

  524. 			res7_0 += a0*b7;

  525. 

  526. 			a1 = ptrba[1];

  527. 			res0_1 += a1*b0;

  528. 			res1_1 += a1*b1;

  529. 			res2_1 += a1*b2;

  530. 			res3_1 += a1*b3;

  531. 			res4_1 += a1*b4;

  532. 			res5_1 += a1*b5;

  533. 			res6_1 += a1*b6;

  534. 			res7_1 += a1*b7;

  535. 

  536. 			ptrba = ptrba+2;

  537. 			ptrbb = ptrbb+8;

  538.                 }

  539. 

  540. 		res0_0 *= alpha;

  541. 		res0_1 *= alpha;

  542. 

  543. 		res1_0 *= alpha;

  544. 		res1_1 *= alpha;

  545. 

  546. 		res2_0 *= alpha;

  547. 		res2_1 *= alpha;

  548. 

  549. 		res3_0 *= alpha;

  550. 		res3_1 *= alpha;

  551. 

  552. 		res4_0 *= alpha;

  553. 		res4_1 *= alpha;

  554. 

  555. 		res5_0 *= alpha;

  556. 		res5_1 *= alpha;

  557. 

  558. 		res6_0 *= alpha;

  559. 		res6_1 *= alpha;

  560. 

  561. 		res7_0 *= alpha;

  562. 		res7_1 *= alpha;

  563. 

  564. 		C0[0] = res0_0;

  565. 		C0[1] = res0_1;

  566. 

  567. 		C1[0] = res1_0;

  568. 		C1[1] = res1_1;

  569. 

  570. 		C2[0] = res2_0;

  571. 		C2[1] = res2_1;

  572. 

  573. 		C3[0] = res3_0;

  574. 		C3[1] = res3_1;

  575. 

  576. 		C4[0] = res4_0;

  577. 		C4[1] = res4_1;

  578. 

  579. 		C5[0] = res5_0;

  580. 		C5[1] = res5_1;

  581. 

  582. 		C6[0] = res6_0;

  583. 		C6[1] = res6_1;

  584. 

  585. 		C7[0] = res7_0;

  586. 		C7[1] = res7_1;

  587. 

  588. 

  589. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  590. 		temp = bk - off;

  591. #ifdef LEFT

  592. 		temp -= 2; // number of values in A

  593. #else

  594. 		temp -= 8; // number of values in B

  595. #endif

  596. 		ptrba += temp*2;

  597. 		ptrbb += temp*8;

  598. #endif

  599. 

  600. #ifdef LEFT

  601. 		off += 2; // number of values in A

  602. #endif

  603. 

  604. 		C0 = C0+2;

  605. 		C1 = C1+2;

  606. 		C2 = C2+2;

  607. 		C3 = C3+2;

  608. 		C4 = C4+2;

  609. 		C5 = C5+2;

  610. 		C6 = C6+2;

  611. 		C7 = C7+2;

  612. 

  613. 	}

  614. 

  615. 	if ( bm & 1 ) // do any 1x4 loop

  616. 	{

  617. 

  618. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  619. 		ptrbb = bb;

  620. #else

  621. 		ptrba += off*1;

  622. 		ptrbb = bb + off*8;

  623. #endif

  624. 

  625. 		res0_0 = 0;

  626. 		res1_0 = 0;

  627. 		res2_0 = 0;

  628. 		res3_0 = 0;

  629. 		res4_0 = 0;

  630. 		res5_0 = 0;

  631. 		res6_0 = 0;

  632. 		res7_0 = 0;

  633. 

  634. 

  635. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  636. 		temp = bk-off;

  637. #elif defined(LEFT)

  638. 		temp = off+1;	// number of values in A

  639. #else

  640. 		temp = off+8;	// number of values in B

  641. #endif

  642. 

  643. 		for (k=0; k<temp; k++)

  644.                 {

  645. 			b0 = ptrbb[0];

  646. 			b1 = ptrbb[1];

  647. 			b2 = ptrbb[2];

  648. 			b3 = ptrbb[3];

  649. 			b4 = ptrbb[4];

  650. 			b5 = ptrbb[5];

  651. 			b6 = ptrbb[6];

  652. 			b7 = ptrbb[7];

  653. 

  654. 			a0 = ptrba[0];

  655. 			res0_0 += a0*b0;

  656. 			res1_0 += a0*b1;

  657. 			res2_0 += a0*b2;

  658. 			res3_0 += a0*b3;

  659. 			res4_0 += a0*b4;

  660. 			res5_0 += a0*b5;

  661. 			res6_0 += a0*b6;

  662. 			res7_0 += a0*b7;

  663. 

  664. 			ptrba = ptrba+1;

  665. 			ptrbb = ptrbb+8;

  666.                 }

  667. 

  668. 		res0_0 *= alpha;

  669. 

  670. 		res1_0 *= alpha;

  671. 

  672. 		res2_0 *= alpha;

  673. 

  674. 		res3_0 *= alpha;

  675. 		res4_0 *= alpha;

  676. 		res5_0 *= alpha;

  677. 		res6_0 *= alpha;

  678. 		res7_0 *= alpha;

  679. 

  680. 		C0[0] = res0_0;

  681. 

  682. 		C1[0] = res1_0;

  683. 

  684. 		C2[0] = res2_0;

  685. 

  686. 		C3[0] = res3_0;

  687. 		C4[0] = res4_0;

  688. 		C5[0] = res5_0;

  689. 		C6[0] = res6_0;

  690. 		C7[0] = res7_0;

  691. 

  692. 

  693. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  694. 		temp = bk - off;

  695. #ifdef LEFT

  696. 		temp -= 1; // number of values in A

  697. #else

  698. 		temp -= 8; // number of values in B

  699. #endif

  700. 		ptrba += temp*1;

  701. 		ptrbb += temp*8;

  702. #endif

  703. 

  704. #ifdef LEFT

  705. 		off += 1; // number of values in A

  706. #endif

  707. 

  708. 		C0 = C0+1;

  709. 		C1 = C1+1;

  710. 		C2 = C2+1;

  711. 		C3 = C3+1;

  712. 		C4 = C4+1;

  713. 		C5 = C5+1;

  714. 		C6 = C6+1;

  715. 		C7 = C7+1;

  716. 

  717. 	}

  718. 

  719. 

  720. #if defined(TRMMKERNEL) && !defined(LEFT)

  721. 		off += 8;

  722. #endif

  723. 

  724.         k = (bk<<3);

  725.         bb = bb+k;

  726.         i = (ldc<<3);

  727.         C = C+i;

  728.     }

  729. 

  730. 

  731. 

  732.    for (j=0; j<(bn&4); j+=4) // do blocks of the Mx4 loops 

  733.    {

  734.         C0 = C;

  735.         C1 = C0+ldc;

  736.         C2 = C1+ldc;

  737.         C3 = C2+ldc;

  738. 

  739. 

  740.         if (left) {

  741.             off = offset;

  742.         }

  743. 

  744.         ptrba = ba;

  745. 

  746.         for (i=0; i<bm/4; i+=1) // do blocks of 4x4

  747. 	{

  748. 

  749. 		ptrbb = bb;

  750.                 if (backwards)

  751.                 {

  752. 		   ptrba += off*4; // number of values in A

  753. 		   ptrbb += off*4; // number of values in B

  754.                 }

  755. 

  756. 		res0_0 = 0;

  757. 		res0_1 = 0;

  758. 		res0_2 = 0;

  759. 		res0_3 = 0;

  760. 

  761. 		res1_0 = 0;

  762. 		res1_1 = 0;

  763. 		res1_2 = 0;

  764. 		res1_3 = 0;

  765. 

  766. 		res2_0 = 0;

  767. 		res2_1 = 0;

  768. 		res2_2 = 0;

  769. 		res2_3 = 0;

  770. 

  771. 		res3_0 = 0;

  772. 		res3_1 = 0;

  773. 		res3_2 = 0;

  774. 		res3_3 = 0;

  775. 

  776.                 temp = backwards ? bk-off : off + 4;

  777.                             /* left ? off + 4 : // number of values in A

  778.                                     off + 4;  // number of values in B */

  779. 

  780. 		for (k=0; k<temp; k++)

  781.                 {

  782. 			b0 = ptrbb[0];

  783. 			b1 = ptrbb[1];

  784. 			b2 = ptrbb[2];

  785. 			b3 = ptrbb[3];

  786. 

  787. 			a0 = ptrba[0];

  788. 			res0_0 += a0*b0;

  789. 			res1_0 += a0*b1;

  790. 			res2_0 += a0*b2;

  791. 			res3_0 += a0*b3;

  792. 

  793. 			a1 = ptrba[1];

  794. 			res0_1 += a1*b0;

  795. 			res1_1 += a1*b1;

  796. 			res2_1 += a1*b2;

  797. 			res3_1 += a1*b3;

  798. 

  799. 			a0 = ptrba[2];

  800. 			res0_2 += a0*b0;

  801. 			res1_2 += a0*b1;

  802. 			res2_2 += a0*b2;

  803. 			res3_2 += a0*b3;

  804. 

  805. 			a1 = ptrba[3];

  806. 			res0_3 += a1*b0;

  807. 			res1_3 += a1*b1;

  808. 			res2_3 += a1*b2;

  809. 			res3_3 += a1*b3;

  810. 

  811. 			ptrba = ptrba+4;

  812. 			ptrbb = ptrbb+4;

  813.                 }

  814. 

  815. 		res0_0 *= alpha;

  816. 		res0_1 *= alpha;

  817. 		res0_2 *= alpha;

  818. 		res0_3 *= alpha;

  819. 

  820. 		res1_0 *= alpha;

  821. 		res1_1 *= alpha;

  822. 		res1_2 *= alpha;

  823. 		res1_3 *= alpha;

  824. 

  825. 		res2_0 *= alpha;

  826. 		res2_1 *= alpha;

  827. 		res2_2 *= alpha;

  828. 		res2_3 *= alpha;

  829. 

  830. 		res3_0 *= alpha;

  831. 		res3_1 *= alpha;

  832. 		res3_2 *= alpha;

  833. 		res3_3 *= alpha;

  834. 

  835. 		C0[0] = res0_0;

  836. 		C0[1] = res0_1;

  837. 		C0[2] = res0_2;

  838. 		C0[3] = res0_3;

  839. 

  840. 		C1[0] = res1_0;

  841. 		C1[1] = res1_1;

  842. 		C1[2] = res1_2;

  843. 		C1[3] = res1_3;

  844. 

  845. 		C2[0] = res2_0;

  846. 		C2[1] = res2_1;

  847. 		C2[2] = res2_2;

  848. 		C2[3] = res2_3;

  849. 

  850. 		C3[0] = res3_0;

  851. 		C3[1] = res3_1;

  852. 		C3[2] = res3_2;

  853. 		C3[3] = res3_3;

  854. 

  855. 		if (!backwards) {

  856.                     temp = bk-off - 4;

  857.                     /* temp = left ? temp - 4 : // number of values in A

  858.                                   temp - 4;  // number of values in B */

  859. 

  860.                     ptrba += temp*4; // number of values in A

  861. 		    ptrbb += temp*4; // number of values in B

  862.                 }

  863. #ifdef LEFT

  864. 		off += 4; // number of values in A

  865. #endif

  866. 

  867. 		C0 = C0+4;

  868. 		C1 = C1+4;

  869. 		C2 = C2+4;

  870. 		C3 = C3+4;

  871. 

  872. 	}

  873. 

  874. 	if ( bm & 2 ) // do any 2x4 loop

  875. 	{

  876. 

  877. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  878. 		ptrbb = bb;

  879. #else

  880. 		ptrba += off*2;

  881. 		ptrbb = bb + off*4;

  882. #endif

  883. 

  884. 		res0_0 = 0;

  885. 		res0_1 = 0;

  886. 

  887. 		res1_0 = 0;

  888. 		res1_1 = 0;

  889. 

  890. 		res2_0 = 0;

  891. 		res2_1 = 0;

  892. 

  893. 		res3_0 = 0;

  894. 		res3_1 = 0;

  895. 

  896. 

  897. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  898. 		temp = bk-off;

  899. #elif defined(LEFT)

  900. 		temp = off+2;	// number of values in A

  901. #else

  902. 		temp = off+4;	// number of values in B

  903. #endif

  904. 

  905. 		for (k=0; k<temp; k++)

  906.                 {

  907. 			b0 = ptrbb[0];

  908. 			b1 = ptrbb[1];

  909. 			b2 = ptrbb[2];

  910. 			b3 = ptrbb[3];

  911. 

  912. 			a0 = ptrba[0];

  913. 			res0_0 += a0*b0;

  914. 			res1_0 += a0*b1;

  915. 			res2_0 += a0*b2;

  916. 			res3_0 += a0*b3;

  917. 

  918. 			a1 = ptrba[1];

  919. 			res0_1 += a1*b0;

  920. 			res1_1 += a1*b1;

  921. 			res2_1 += a1*b2;

  922. 			res3_1 += a1*b3;

  923. 

  924. 			ptrba = ptrba+2;

  925. 			ptrbb = ptrbb+4;

  926.                 }

  927. 

  928. 		res0_0 *= alpha;

  929. 		res0_1 *= alpha;

  930. 

  931. 		res1_0 *= alpha;

  932. 		res1_1 *= alpha;

  933. 

  934. 		res2_0 *= alpha;

  935. 		res2_1 *= alpha;

  936. 

  937. 		res3_0 *= alpha;

  938. 		res3_1 *= alpha;

  939. 

  940. 		C0[0] = res0_0;

  941. 		C0[1] = res0_1;

  942. 

  943. 		C1[0] = res1_0;

  944. 		C1[1] = res1_1;

  945. 

  946. 		C2[0] = res2_0;

  947. 		C2[1] = res2_1;

  948. 

  949. 		C3[0] = res3_0;

  950. 		C3[1] = res3_1;

  951. 

  952. 

  953. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  954. 		temp = bk - off;

  955. #ifdef LEFT

  956. 		temp -= 2; // number of values in A

  957. #else

  958. 		temp -= 4; // number of values in B

  959. #endif

  960. 		ptrba += temp*2;

  961. 		ptrbb += temp*4;

  962. #endif

  963. 

  964. #ifdef LEFT

  965. 		off += 2; // number of values in A

  966. #endif

  967. 

  968. 		C0 = C0+2;

  969. 		C1 = C1+2;

  970. 		C2 = C2+2;

  971. 		C3 = C3+2;

  972. 

  973. 	}

  974. 

  975. 	if ( bm & 1 ) // do any 1x4 loop

  976. 	{

  977. 

  978. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  979. 		ptrbb = bb;

  980. #else

  981. 		ptrba += off*1;

  982. 		ptrbb = bb + off*4;

  983. #endif

  984. 

  985. 		res0_0 = 0;

  986. 		res1_0 = 0;

  987. 		res2_0 = 0;

  988. 		res3_0 = 0;

  989. 

  990. 

  991. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  992. 		temp = bk-off;

  993. #elif defined(LEFT)

  994. 		temp = off+1;	// number of values in A

  995. #else

  996. 		temp = off+4;	// number of values in B

  997. #endif

  998. 

  999. 		for (k=0; k<temp; k++)

  1000.                 {

  1001. 			b0 = ptrbb[0];

  1002. 			b1 = ptrbb[1];

  1003. 			b2 = ptrbb[2];

  1004. 			b3 = ptrbb[3];

  1005. 

  1006. 			a0 = ptrba[0];

  1007. 			res0_0 += a0*b0;

  1008. 			res1_0 += a0*b1;

  1009. 			res2_0 += a0*b2;

  1010. 			res3_0 += a0*b3;

  1011. 

  1012. 			ptrba = ptrba+1;

  1013. 			ptrbb = ptrbb+4;

  1014.                 }

  1015. 

  1016. 		res0_0 *= alpha;

  1017. 

  1018. 		res1_0 *= alpha;

  1019. 

  1020. 		res2_0 *= alpha;

  1021. 

  1022. 		res3_0 *= alpha;

  1023. 

  1024. 		C0[0] = res0_0;

  1025. 

  1026. 		C1[0] = res1_0;

  1027. 

  1028. 		C2[0] = res2_0;

  1029. 

  1030. 		C3[0] = res3_0;

  1031. 

  1032. 

  1033. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1034. 		temp = bk - off;

  1035. #ifdef LEFT

  1036. 		temp -= 1; // number of values in A

  1037. #else

  1038. 		temp -= 4; // number of values in B

  1039. #endif

  1040. 		ptrba += temp*1;

  1041. 		ptrbb += temp*4;

  1042. #endif

  1043. 

  1044. #ifdef LEFT

  1045. 		off += 1; // number of values in A

  1046. #endif

  1047. 

  1048. 		C0 = C0+1;

  1049. 		C1 = C1+1;

  1050. 		C2 = C2+1;

  1051. 		C3 = C3+1;

  1052. 

  1053. 	}

  1054. 

  1055. 

  1056. #if defined(TRMMKERNEL) && !defined(LEFT)

  1057. 		off += 4;

  1058. #endif

  1059. 

  1060.         k = (bk<<2);

  1061.         bb = bb+k;

  1062.         i = (ldc<<2);

  1063.         C = C+i;

  1064.     }

  1065. 

  1066. 

  1067. 

  1068.    for (j=0; j<(bn&2); j+=2) // do the Mx2 loops 

  1069.    {

  1070.         C0 = C;

  1071.         C1 = C0+ldc;

  1072. 

  1073. #if defined(TRMMKERNEL) && defined(LEFT)

  1074. 		off = offset;

  1075. #endif

  1076. 

  1077. 

  1078.         ptrba = ba;

  1079. 

  1080.         for (i=0; i<bm/4; i+=1) // do blocks of 4x2

  1081. 	{

  1082. 

  1083. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1084. 		ptrbb = bb;

  1085. #else

  1086. 		ptrba += off*4;

  1087. 		ptrbb = bb + off*2;

  1088. #endif

  1089. 

  1090. 		res0_0 = 0;

  1091. 		res0_1 = 0;

  1092. 		res0_2 = 0;

  1093. 		res0_3 = 0;

  1094. 

  1095. 		res1_0 = 0;

  1096. 		res1_1 = 0;

  1097. 		res1_2 = 0;

  1098. 		res1_3 = 0;

  1099. 

  1100. 

  1101. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1102. 		temp = bk-off;

  1103. #elif defined(LEFT)

  1104. 		temp = off+4;	// number of values in A

  1105. #else

  1106. 		temp = off+2;	// number of values in B

  1107. #endif

  1108. 

  1109. 		for (k=0; k<temp; k++)

  1110.                 {

  1111. 			b0 = ptrbb[0];

  1112. 			b1 = ptrbb[1];

  1113. 

  1114. 			a0 = ptrba[0];

  1115. 			res0_0 += a0*b0;

  1116. 			res1_0 += a0*b1;

  1117. 

  1118. 			a1 = ptrba[1];

  1119. 			res0_1 += a1*b0;

  1120. 			res1_1 += a1*b1;

  1121. 

  1122. 			a0 = ptrba[2];

  1123. 			res0_2 += a0*b0;

  1124. 			res1_2 += a0*b1;

  1125. 

  1126. 			a1 = ptrba[3];

  1127. 			res0_3 += a1*b0;

  1128. 			res1_3 += a1*b1;

  1129. 

  1130. 			ptrba = ptrba+4;

  1131. 			ptrbb = ptrbb+2;

  1132.                 }

  1133. 

  1134. 		res0_0 *= alpha;

  1135. 		res0_1 *= alpha;

  1136. 		res0_2 *= alpha;

  1137. 		res0_3 *= alpha;

  1138. 

  1139. 		res1_0 *= alpha;

  1140. 		res1_1 *= alpha;

  1141. 		res1_2 *= alpha;

  1142. 		res1_3 *= alpha;

  1143. 

  1144. 		C0[0] = res0_0;

  1145. 		C0[1] = res0_1;

  1146. 		C0[2] = res0_2;

  1147. 		C0[3] = res0_3;

  1148. 

  1149. 		C1[0] = res1_0;

  1150. 		C1[1] = res1_1;

  1151. 		C1[2] = res1_2;

  1152. 		C1[3] = res1_3;

  1153. 

  1154. 

  1155. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1156. 		temp = bk - off;

  1157. #ifdef LEFT

  1158. 		temp -= 4; // number of values in A

  1159. #else

  1160. 		temp -= 2; // number of values in B

  1161. #endif

  1162. 		ptrba += temp*4;

  1163. 		ptrbb += temp*2;

  1164. #endif

  1165. 

  1166. #ifdef LEFT

  1167. 		off += 4; // number of values in A

  1168. #endif

  1169. 

  1170. 		C0 = C0+4;

  1171. 		C1 = C1+4;

  1172. 

  1173. 	}

  1174. 

  1175. 	if ( bm & 2 ) // do any 2x2 loop

  1176. 	{

  1177. 

  1178. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1179. 		ptrbb = bb;

  1180. #else

  1181. 		ptrba += off*2;

  1182. 		ptrbb = bb + off*2;

  1183. #endif

  1184. 

  1185. 		res0_0 = 0;

  1186. 		res0_1 = 0;

  1187. 

  1188. 		res1_0 = 0;

  1189. 		res1_1 = 0;

  1190. 

  1191. 

  1192. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1193. 		temp = bk-off;

  1194. #elif defined(LEFT)

  1195. 		temp = off+2;	// number of values in A

  1196. #else

  1197. 		temp = off+2;	// number of values in B

  1198. #endif

  1199. 

  1200. 		for (k=0; k<temp; k++)

  1201.                 {

  1202. 			b0 = ptrbb[0];

  1203. 			b1 = ptrbb[1];

  1204. 

  1205. 			a0 = ptrba[0];

  1206. 			res0_0 += a0*b0;

  1207. 			res1_0 += a0*b1;

  1208. 

  1209. 			a1 = ptrba[1];

  1210. 			res0_1 += a1*b0;

  1211. 			res1_1 += a1*b1;

  1212. 

  1213. 			ptrba = ptrba+2;

  1214. 			ptrbb = ptrbb+2;

  1215.                 }

  1216. 

  1217. 		res0_0 *= alpha;

  1218. 		res0_1 *= alpha;

  1219. 

  1220. 		res1_0 *= alpha;

  1221. 		res1_1 *= alpha;

  1222. 

  1223. 		C0[0] = res0_0;

  1224. 		C0[1] = res0_1;

  1225. 

  1226. 		C1[0] = res1_0;

  1227. 		C1[1] = res1_1;

  1228. 

  1229. 

  1230. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1231. 		temp = bk - off;

  1232. #ifdef LEFT

  1233. 		temp -= 2; // number of values in A

  1234. #else

  1235. 		temp -= 2; // number of values in B

  1236. #endif

  1237. 		ptrba += temp*2;

  1238. 		ptrbb += temp*2;

  1239. #endif

  1240. 

  1241. #ifdef LEFT

  1242. 		off += 2; // number of values in A

  1243. #endif

  1244. 

  1245. 		C0 = C0+2;

  1246. 		C1 = C1+2;

  1247. 

  1248. 	}

  1249. 

  1250. 	if ( bm & 1 ) // do any 1x2 loop

  1251. 	{

  1252. 

  1253. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1254. 		ptrbb = bb;

  1255. #else

  1256. 		ptrba += off*1;

  1257. 		ptrbb = bb + off*2;

  1258. #endif

  1259. 

  1260. 		res0_0 = 0;

  1261. 

  1262. 		res1_0 = 0;

  1263. 

  1264. 

  1265. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1266. 		temp = bk-off;

  1267. #elif defined(LEFT)

  1268. 		temp = off+1;	// number of values in A

  1269. #else

  1270. 		temp = off+2;	// number of values in B

  1271. #endif

  1272. 

  1273. 		for (k=0; k<temp; k++)

  1274.                 {

  1275. 			b0 = ptrbb[0];

  1276. 			b1 = ptrbb[1];

  1277. 

  1278. 			a0 = ptrba[0];

  1279. 			res0_0 += a0*b0;

  1280. 			res1_0 += a0*b1;

  1281. 

  1282. 			ptrba = ptrba+1;

  1283. 			ptrbb = ptrbb+2;

  1284.                 }

  1285. 

  1286. 		res0_0 *= alpha;

  1287. 

  1288. 		res1_0 *= alpha;

  1289. 

  1290. 		C0[0] = res0_0;

  1291. 

  1292. 		C1[0] = res1_0;

  1293. 

  1294. 

  1295. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1296. 		temp = bk - off;

  1297. #ifdef LEFT

  1298. 		temp -= 1; // number of values in A

  1299. #else

  1300. 		temp -= 2; // number of values in B

  1301. #endif

  1302. 		ptrba += temp*1;

  1303. 		ptrbb += temp*2;

  1304. #endif

  1305. 

  1306. #ifdef LEFT

  1307. 		off += 1; // number of values in A

  1308. #endif

  1309. 

  1310. 		C0 = C0+1;

  1311. 		C1 = C1+1;

  1312. 

  1313. 	}

  1314. 

  1315. 

  1316. #if defined(TRMMKERNEL) && !defined(LEFT)

  1317. 		off += 2;

  1318. #endif

  1319. 

  1320.         k = (bk<<1);

  1321.         bb = bb+k;

  1322.         i = (ldc<<1);

  1323.         C = C+i;

  1324.     }

  1325. 

  1326. 

  1327. 

  1328. 

  1329. 

  1330. 

  1331. 

  1332.    for (j=0; j<(bn&1); j+=1) // do the Mx1 loops

  1333.    {

  1334.         C0 = C;

  1335. 

  1336. #if defined(TRMMKERNEL) &&  defined(LEFT)

  1337. 	off = offset;

  1338. #endif

  1339. 

  1340.         ptrba = ba;

  1341. 

  1342.         for (i=0; i<bm/4; i+=1) // do blocks of 4x1 loops

  1343. 	{

  1344. 

  1345. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1346. 		ptrbb = bb;

  1347. #else

  1348. 		ptrba += off*4;

  1349. 		ptrbb = bb + off*1;

  1350. #endif

  1351. 

  1352. 		res0_0 = 0;

  1353. 		res0_1 = 0;

  1354. 		res0_2 = 0;

  1355. 		res0_3 = 0;

  1356. 

  1357. 

  1358. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1359. 		temp = bk-off;

  1360. #elif defined(LEFT)

  1361. 		temp = off+4;	// number of values in A

  1362. #else

  1363. 		temp = off+1;	// number of values in B

  1364. #endif

  1365. 

  1366. 		for (k=0; k<temp; k++)

  1367.                 {

  1368. 			b0 = ptrbb[0];

  1369. 

  1370. 			a0 = ptrba[0];

  1371. 			res0_0 += a0*b0;

  1372. 

  1373. 			a1 = ptrba[1];

  1374. 			res0_1 += a1*b0;

  1375. 

  1376. 			a0 = ptrba[2];

  1377. 			res0_2 += a0*b0;

  1378. 

  1379. 			a1 = ptrba[3];

  1380. 			res0_3 += a1*b0;

  1381. 

  1382. 			ptrba = ptrba+4;

  1383. 			ptrbb = ptrbb+1;

  1384.                 }

  1385. 

  1386. 		res0_0 *= alpha;

  1387. 		res0_1 *= alpha;

  1388. 		res0_2 *= alpha;

  1389. 		res0_3 *= alpha;

  1390. 

  1391. 		C0[0] = res0_0;

  1392. 		C0[1] = res0_1;

  1393. 		C0[2] = res0_2;

  1394. 		C0[3] = res0_3;

  1395. 

  1396. 

  1397. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1398. 		temp = bk - off;

  1399. #ifdef LEFT

  1400. 		temp -= 4; // number of values in A

  1401. #else

  1402. 		temp -= 1; // number of values in B

  1403. #endif

  1404. 		ptrba += temp*4;

  1405. 		ptrbb += temp*1;

  1406. #endif

  1407. 

  1408. #ifdef LEFT

  1409. 		off += 4; // number of values in A

  1410. #endif

  1411. 

  1412. 		C0 = C0+4;

  1413. 

  1414. 	}

  1415. 

  1416. 	if ( bm & 2 ) // do any 2x1 loop

  1417. 	{

  1418. 

  1419. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1420. 		ptrbb = bb;

  1421. #else

  1422. 		ptrba += off*2;

  1423. 		ptrbb = bb + off*1;

  1424. #endif

  1425. 

  1426. 		res0_0 = 0;

  1427. 		res0_1 = 0;

  1428. 

  1429. 

  1430. 

  1431. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1432. 		temp = bk-off;

  1433. #elif defined(LEFT)

  1434. 		temp = off+2;	// number of values in A

  1435. #else

  1436. 		temp = off+1;	// number of values in B

  1437. #endif

  1438. 

  1439. 		for (k=0; k<temp; k++)

  1440.                 {

  1441. 			b0 = ptrbb[0];

  1442. 

  1443. 			a0 = ptrba[0];

  1444. 			res0_0 += a0*b0;

  1445. 

  1446. 			a1 = ptrba[1];

  1447. 			res0_1 += a1*b0;

  1448. 

  1449. 			ptrba = ptrba+2;

  1450. 			ptrbb = ptrbb+1;

  1451.                 }

  1452. 

  1453. 		res0_0 *= alpha;

  1454. 		res0_1 *= alpha;

  1455. 

  1456. 		C0[0] = res0_0;

  1457. 		C0[1] = res0_1;

  1458. 

  1459. 

  1460. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1461. 		temp = bk - off;

  1462. #ifdef LEFT

  1463. 		temp -= 2; // number of values in A

  1464. #else

  1465. 		temp -= 1; // number of values in B

  1466. #endif

  1467. 		ptrba += temp*2;

  1468. 		ptrbb += temp*1;

  1469. #endif

  1470. 

  1471. #ifdef LEFT

  1472. 		off += 2; // number of values in A

  1473. #endif

  1474. 

  1475. 		C0 = C0+2;

  1476. 

  1477. 	}

  1478. 

  1479. 	if ( bm & 1 ) // do any 1x1 loop

  1480. 	{

  1481. 

  1482. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1483. 		ptrbb = bb;

  1484. #else

  1485. 		ptrba += off*1;

  1486. 		ptrbb = bb + off*1;

  1487. #endif

  1488. 

  1489. 		res0_0 = 0;

  1490. 

  1491. 

  1492. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1493. 		temp = bk-off;

  1494. #elif defined(LEFT)

  1495. 		temp = off+1;	// number of values in A

  1496. #else

  1497. 		temp = off+1;	// number of values in B

  1498. #endif

  1499. 

  1500. 		for (k=0; k<temp; k++)

  1501.                 {

  1502. 			b0 = ptrbb[0];

  1503. 

  1504. 			a0 = ptrba[0];

  1505. 			res0_0 += a0*b0;

  1506. 

  1507. 			ptrba = ptrba+1;

  1508. 			ptrbb = ptrbb+1;

  1509.                 }

  1510. 

  1511. 		res0_0 *= alpha;

  1512. 

  1513. 		C0[0] = res0_0;

  1514. 

  1515. 

  1516. #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1517. 		temp = bk - off;

  1518. #ifdef LEFT

  1519. 		temp -= 1; // number of values in A

  1520. #else

  1521. 		temp -= 1; // number of values in B

  1522. #endif

  1523. 		ptrba += temp*1;

  1524. 		ptrbb += temp*1;

  1525. #endif

  1526. 

  1527. #ifdef LEFT

  1528. 		off += 1; // number of values in A

  1529. #endif

  1530. 

  1531. 		C0 = C0+1;

  1532. 

  1533. 	}

  1534. 

  1535. 

  1536. 

  1537. #if defined(TRMMKERNEL) && !defined(LEFT)

  1538. 		off += 1;

  1539. #endif

  1540. 

  1541.         k = (bk<<0);

  1542.         bb = bb+k;

  1543.         C = C+ldc;

  1544.    }

  1545.    return 0;

  1546. }