OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 8d12027a79
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '8d12027a79'
${ noResults }
OpenBLAS/kernel/power/sgemm_logic_power9.S

2192 lines
46 kB
Raw Blame History 

  1. #define MY_ALIGN .align 3

  2. b L8

  3. 

  4. 	MY_ALIGN

  5. LSGEMM_L8x16_LMAIN_SUB: 

  6. 	LOAD8x16_2    

  7. 	MY_ALIGN

  8. 

  9. LSGEMM_L8x16_LOOP:

  10.     KERNEL8x16_L2 128,64,0,0 

  11. LSGEMM_L8x16_K128:

  12.     KERNEL8x16_L2 128,64,1,0 

  13.     KERNEL8x16_I1_L4_2  128,64, 1,0

  14.     KERNEL8x16_I1_L4_2  128,64, 2,0

  15.     KERNEL8x16_I1_L4_2  128,64, 3,0

  16.     KERNEL8x16_I1_L4_2  128,64, 4,0

  17.     KERNEL8x16_I1_L4_2  128,64, 5,0        

  18.     KERNEL8x16_I1_L4_2  128,64, 6,0

  19.     KERNEL8x16_I1_L4_2  128,64, 7,0  

  20.     KERNEL8x16_I1_L4_2  128,64, 8,0      

  21.     KERNEL8x16_I1_L4_2  128,64, 9,0

  22.     KERNEL8x16_I1_L4_2  128,64, 10,0

  23.     KERNEL8x16_I1_L4_2  128,64, 11,0

  24.     KERNEL8x16_I1_L4_2  128,64, 12,0

  25.     KERNEL8x16_I1_L4_2  128,64, 13,0    

  26.     KERNEL8x16_I1_L4_2  128,64, 14,0    

  27.     KERNEL8x16_I1_L4_2  128,64, 15,0  	

  28.     KERNEL8x16_I1_L4_2  128,64, 16,0

  29.     KERNEL8x16_I1_L4_2  128,64, 17,0

  30.     KERNEL8x16_I1_L4_2  128,64, 18,0

  31.     KERNEL8x16_I1_L4_2  128,64, 19,0

  32.     KERNEL8x16_I1_L4_2  128,64, 20,0

  33.     KERNEL8x16_I1_L4_2  128,64, 21,0        

  34.     KERNEL8x16_I1_L4_2  128,64, 22,0

  35.     KERNEL8x16_I1_L4_2  128,64, 23,0  

  36.     KERNEL8x16_I1_L4_2  128,64, 24,0      

  37.     KERNEL8x16_I1_L4_2  128,64, 25,0

  38.     KERNEL8x16_I1_L4_2  128,64, 26,0

  39.     KERNEL8x16_I1_L4_2  128,64, 27,0

  40.     KERNEL8x16_I1_L4_2  128,64, 28,0

  41.     KERNEL8x16_I1_L4_2  128,64, 29,0    

  42.     KERNEL8x16_I1_L4_2  128,64, 30,0    

  43.     KERNEL8x16_I1_L4_2  128,64, 31,1 

  44. 	bdnz		LSGEMM_L8x16_LOOP

  45. 

  46. 	MY_ALIGN

  47. LSGEMM_L8x16_LOOP_END: 

  48.     END8x16_2

  49.     blr  

  50. 

  51. 	MY_ALIGN

  52. LSGEMM_L8x16_L64_SUB: 

  53. 	LOAD8x16_2     

  54.     KERNEL8x16_I1_L4_2  128,64, 0,0

  55.     KERNEL8x16_I1_L4_2  128,64, 1,0

  56.     KERNEL8x16_I1_L4_2  128,64, 2,0

  57.     KERNEL8x16_I1_L4_2  128,64,3,0

  58.     KERNEL8x16_I1_L4_2  128,64,4,0

  59.     KERNEL8x16_I1_L4_2  128,64,5,0        

  60.     KERNEL8x16_I1_L4_2  128,64,6,0

  61.     KERNEL8x16_I1_L4_2  128,64,7,0  

  62.     KERNEL8x16_I1_L4_2  128,64,8,0      

  63.     KERNEL8x16_I1_L4_2  128,64,9,0

  64.     KERNEL8x16_I1_L4_2  128,64,10,0

  65.     KERNEL8x16_I1_L4_2  128,64,11,0

  66.     KERNEL8x16_I1_L4_2  128,64,12,0

  67.     KERNEL8x16_I1_L4_2  128,64,13,0    

  68.     KERNEL8x16_I1_L4_2  128,64,14,0    

  69.     KERNEL8x16_I1_L4_3  128,64,15,1 

  70.     blr	

  71. LSGEMM_L8x16_L32_SUB: 

  72. 	LOAD8x16_2     

  73.     KERNEL8x16_I1_L4_2  128,64,0,0

  74.     KERNEL8x16_I1_L4_2  128,64,1,0

  75.     KERNEL8x16_I1_L4_2  128,64,2,0

  76.     KERNEL8x16_I1_L4_2  128,64,3,0

  77.     KERNEL8x16_I1_L4_2  128,64,4,0

  78.     KERNEL8x16_I1_L4_2  128,64,5,0        

  79.     KERNEL8x16_I1_L4_2  128,64,6,0

  80.     KERNEL8x16_I1_L4_3  128,64,7,1

  81.     blr	

  82. 

  83. LSGEMM_L8x16_L16_SUB: 

  84. 	LOAD8x16_2     

  85.     KERNEL8x16_I1_L4_2  128,64,0,0

  86.     KERNEL8x16_I1_L4_2  128,64,1,0

  87.     KERNEL8x16_I1_L4_2  128,64,2,0

  88.     KERNEL8x16_I1_L4_3  128,64,3,1

  89.     blr	

  90. 

  91. L8:

  92. #if defined(TRMMKERNEL) && !defined(LEFT)

  93.    neg TEMP_REG, OFFSET 

  94. #endif

  95. 

  96. 	srawi.		J,	N,	3

  97. 

  98. 	ble		LSGEMM_L8_END

  99. 

  100. LSGEMM_L8_BEGIN:

  101. 

  102. 	li		T1,	128

  103. 	li		T2,	256

  104.  

  105. 	mr		AO,	A

  106. 	mr		CO,	C

  107. 	slwi		T3,	LDC	,	3

  108. 	add		C,	C,	T3

  109. 

  110. 	dcbt		A,	T1

  111. 	dcbt		A,	T2

  112. #if defined(TRMMKERNEL) && defined(LEFT)

  113. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  114. #endif 

  115. 	srawi.		I,	M,	4

  116. 	ble		LSGEMM_L8x16_END

  117. 

  118. 	MY_ALIGN

  119. LSGEMM_L8x16_BEGIN:

  120. 

  121. #if defined(TRMMKERNEL)

  122.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,8

  123. #else

  124. 	mr		BO,	B

  125. #endif	

  126. 

  127. #if defined(TRMMKERNEL)

  128.    REFRESH_TEMP_BK T11,K,TEMP_REG,16,8

  129.    mr T12, T11

  130.    addi T12,T12, -2

  131.    srawi.		L, T12,	7 /**(T11-2) % 128x */

  132. #else

  133.    mr T12, K

  134.    addi T12,T12, -2

  135.    srawi.		L,	T12,	7 /**(K-2) % 128x */

  136. #endif 

  137.  

  138.     ZERO8x16 

  139. 	ble		LSGEMM_L8x16_SUB0

  140. 	mtctr		L 

  141.     bl      LSGEMM_L8x16_LMAIN_SUB

  142. 	andi.		L,	T12,	127

  143. 	ble		LSGEMM_L8x16_SAVE

  144. 	b		LSGEMM_L8x16_SUB2   

  145. 	MY_ALIGN

  146. LSGEMM_L8x16_SUB0:

  147. #if defined(TRMMKERNEL)

  148. 	andi.		L,	T11,	255

  149.     cmpwi   T11,129

  150. #else

  151. 	andi.		L,	K,	255

  152.     cmpwi   K,129

  153. #endif       

  154.     li T10,1

  155.     bne CMP8x16_128K

  156.     addi BO,BO,-32

  157.     addi AO,AO,-64 

  158.     LOAD8x16 64,32 

  159.     END8x16_WITHOUT_ADD   

  160.     LOAD8x16_2O AO,BO,  128, 64 

  161.     mtctr   T10   

  162.     bl LSGEMM_L8x16_K128   

  163.     b LSGEMM_L8x16_SAVE  

  164. CMP8x16_128K:

  165. /*----------------------------------------*/   

  166. #if defined(TRMMKERNEL)    

  167.     cmpwi   T11,128

  168. #else    

  169.     cmpwi   K,128

  170. #endif        

  171.     bne LSGEMM_L8x16_SUB2 

  172.     MY_ALIGN   

  173.     mtctr   T10

  174.     addi BO,BO,-64

  175.     addi AO,AO,-128   

  176.     LOAD8x16_2O  AO,BO,  128,64

  177.     bl LSGEMM_L8x16_K128   

  178.     b LSGEMM_L8x16_SAVE

  179. 	MY_ALIGN

  180. LSGEMM_L8x16_SUB2:

  181.     andi.   T10,L,64

  182.     ble   LSGEMM_L8x16_SUB2_32

  183.     bl   LSGEMM_L8x16_L64_SUB

  184.     MY_ALIGN 

  185. LSGEMM_L8x16_SUB2_32:

  186.     andi.      T10,L, 32

  187.     ble LSGEMM_L8x16_SUB2_16

  188.     bl   LSGEMM_L8x16_L32_SUB

  189.     MY_ALIGN                

  190. LSGEMM_L8x16_SUB2_16:

  191.     andi.      T10,L, 16

  192.     ble LSGEMM_L8x16_SUB2_8

  193. 	bl  LSGEMM_L8x16_L16_SUB

  194.     MY_ALIGN 

  195. LSGEMM_L8x16_SUB2_8:

  196.     andi.      T10,L, 8

  197.     ble LSGEMM_L8x16_SUB2_4 

  198. 	LOAD8x16_2

  199.     KERNEL8x16_I1_L4_2  128,64, 0,0

  200.     KERNEL8x16_I1_L4_3  128,64, 1,1

  201. 	MY_ALIGN	

  202. LSGEMM_L8x16_SUB2_4:

  203.     andi.      T10,L, 4

  204.     ble LSGEMM_L8x16_SUB2_2

  205.     LOAD8x16_2

  206.     KERNEL8x16_I1_L4_3  128,64, 0,1

  207.     MY_ALIGN

  208. LSGEMM_L8x16_SUB2_2:

  209.     andi.      T10,L, 2

  210.     ble LSGEMM_L8x16_SUB2_1

  211.     LOAD8x16_2

  212.     KERNEL8x16_E2  128,64, 0,1

  213.     MY_ALIGN    

  214. LSGEMM_L8x16_SUB2_1:

  215.     andi.      T10,L, 1

  216.     ble LSGEMM_L8x16_SAVE	

  217.     KERNEL8x16 0

  218. 

  219. 

  220. 	MY_ALIGN

  221. LSGEMM_L8x16_SAVE:

  222. 	SAVE8x16

  223. #if defined(TRMMKERNEL)	

  224. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,8

  225. #endif	

  226. 	addic.		I,	I,	-1

  227. 	bgt+		LSGEMM_L8x16_BEGIN

  228.     MY_ALIGN

  229. LSGEMM_L8x16_END:

  230. LSGEMM_L8x8_BEGIN:

  231.     andi.       T2, M,  15

  232.     ble     LSGEMM_L8x1_END

  233. 

  234.     andi.       T1, M,  8

  235.     ble     LSGEMM_L8x8_END

  236. 

  237. #if defined(TRMMKERNEL)

  238.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,8

  239. #else

  240.     mr      BO, B

  241. #endif  

  242. 

  243. #if defined(TRMMKERNEL)

  244.    REFRESH_TEMP_BK T11,K,TEMP_REG,8,8

  245.    mr T12, T11

  246.    addi T12,T12, -1

  247.    srawi.       L, T12, 4 /**(T11-1) % 16x */

  248. #else

  249.    mr T12, K

  250.    addi T12,T12, -1

  251.    srawi.       L,  T12,    4 /**(K-1) % 16x */

  252. #endif 

  253.     

  254.     ZERO8x8

  255.     ble     LSGEMM_L8x8_SUB0

  256. 

  257.     MY_ALIGN

  258. LSGEMM_L8x8_LOOP_START:

  259.  

  260.     LOAD8x8_0  /*we already zeroed */ 

  261.     mtctr       L

  262. 

  263.     MY_ALIGN

  264. 

  265. LSGEMM_L8x8_LOOP:

  266. 

  267.     KERNEL8x8_I1_L4_2  32,32, 0,0

  268.     KERNEL8x8_I1_L4_2  32,32, 1,0

  269.     KERNEL8x8_I1_L4_2  32,32, 2,0

  270.     KERNEL8x8_I1_L4_2  32,32, 3,1    

  271. 

  272.     bdnz        LSGEMM_L8x8_LOOP

  273. 

  274.     MY_ALIGN

  275. LSGEMM_L8x8_LOOP_END:

  276. 

  277.     END8x8 0, AO, BO, 32, 32    

  278. 

  279.     b       LSGEMM_L8x8_SUB1 

  280.     MY_ALIGN

  281. LSGEMM_L8x8_SUB0:

  282. #if defined(TRMMKERNEL)

  283.     andi.       L,  T11,    31

  284. #else

  285.     andi.       L,  K,  31

  286. #endif   

  287.     b       LSGEMM_L8x8_SUB2

  288.     MY_ALIGN

  289. LSGEMM_L8x8_SUB1:

  290. #if defined(TRMMKERNEL)

  291.     andi.       L,  T12,    15

  292. #else

  293.     andi.       L,  T12,    15

  294. #endif  

  295.     ble     LSGEMM_L8x8_SAVE

  296.     MY_ALIGN

  297. LSGEMM_L8x8_SUB2:

  298.  

  299.     srawi.      T1,L, 3

  300.     ble LSGEMM_L8x8_SUB2_4 

  301.     mtctr		T1

  302.     MY_ALIGN

  303. LSGEMM_L8x8_SUB2_LOOP:    

  304.     LOAD8x8_0

  305.     KERNEL8x8_I1_L4_2  32,32, 0,0

  306.     KERNEL8x8_I1_L4_3  32,32, 1,1

  307.     bdnz LSGEMM_L8x8_SUB2_LOOP

  308.     MY_ALIGN    

  309. LSGEMM_L8x8_SUB2_4:

  310.     andi.      T1,L, 4

  311.     ble LSGEMM_L8x8_SUB2_2

  312.     LOAD8x8_0

  313.     KERNEL8x8_I1_L4_3  32,32, 0,1

  314.     MY_ALIGN

  315. LSGEMM_L8x8_SUB2_2:

  316.     andi.      T1,L, 2

  317.     ble LSGEMM_L8x8_SUB2_1

  318.     LOAD8x8_0

  319.     KERNEL8x8_I1_L2_3  32,32, 0,1

  320.     MY_ALIGN    

  321. LSGEMM_L8x8_SUB2_1:

  322.     andi.      T1,L, 1

  323.     ble LSGEMM_L8x8_SAVE   

  324.     KERNEL8x8 0

  325.  

  326. 

  327.     MY_ALIGN

  328. LSGEMM_L8x8_SAVE:

  329.     SAVE8x8

  330. #if defined(TRMMKERNEL) 

  331.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,8

  332. #endif  

  333.     MY_ALIGN  

  334. LSGEMM_L8x8_END:

  335. LSGEMM_L8x4_BEGIN:

  336.     andi.       T2, M,  15

  337.     ble     LSGEMM_L8x1_END

  338. 

  339.     andi.       T1, M,  4

  340.     ble     LSGEMM_L8x4_END

  341. 

  342. #if defined(TRMMKERNEL)

  343.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,8

  344. #else

  345.     mr      BO, B

  346. #endif  

  347. 

  348. #if defined(TRMMKERNEL)

  349.    REFRESH_TEMP_BK T11,K,TEMP_REG,4,8

  350.    mr T12, T11

  351.    addi T12,T12, -1

  352.    srawi.       L, T12, 4 /**(T11-1) % 16x */

  353. #else

  354.    mr T12, K

  355.    addi T12,T12, -1

  356.    srawi.       L,  T12,    4 /**(K-1) % 16x */

  357. #endif 

  358.     

  359.     ZERO8x4

  360.     ble     LSGEMM_L8x4_SUB0

  361. 

  362.     MY_ALIGN

  363. LSGEMM_L8x4_LOOP_START:

  364.  

  365.     LOAD8x4_0  /*we already zeroed */ 

  366.     mtctr       L

  367. 

  368.     MY_ALIGN

  369. 

  370. LSGEMM_L8x4_LOOP:

  371. 

  372.     KERNEL8x4_I1_L4_2  16,32, 0,0

  373.     KERNEL8x4_I1_L4_2  16,32, 1,0

  374.     KERNEL8x4_I1_L4_2  16,32, 2,0

  375.     KERNEL8x4_I1_L4_2  16,32, 3,1    

  376. 

  377.     bdnz        LSGEMM_L8x4_LOOP

  378. 

  379.     MY_ALIGN

  380. LSGEMM_L8x4_LOOP_END:

  381. 

  382.     END8x4 0, AO, BO, 16, 32    

  383. 

  384.     b       LSGEMM_L8x4_SUB1 

  385.     MY_ALIGN

  386. LSGEMM_L8x4_SUB0:

  387. #if defined(TRMMKERNEL)

  388.     andi.       L,  T11,    31

  389. #else

  390.     andi.       L,  K,  31

  391. #endif   

  392.     b       LSGEMM_L8x4_SUB2

  393.     MY_ALIGN

  394. LSGEMM_L8x4_SUB1:

  395. #if defined(TRMMKERNEL)

  396.     andi.       L,  T12,    15

  397. #else

  398.     andi.       L,  T12,    15

  399. #endif  

  400.     ble     LSGEMM_L8x4_SAVE

  401.     MY_ALIGN

  402. LSGEMM_L8x4_SUB2:

  403. 

  404.     srawi.      T1,L, 3

  405.     ble LSGEMM_L8x4_SUB2_4 

  406.     mtctr		T1

  407.     MY_ALIGN

  408. LSGEMM_L8x4_SUB2_LOOP:      

  409.     LOAD8x4_0

  410.     KERNEL8x4_I1_L4_2  16,32, 0,0

  411.     KERNEL8x4_I1_L4_3  16,32, 1,1

  412.     bdnz LSGEMM_L8x4_SUB2_LOOP

  413.     MY_ALIGN    

  414. LSGEMM_L8x4_SUB2_4:

  415.     andi.      T1,L, 4

  416.     ble LSGEMM_L8x4_SUB2_2

  417.     LOAD8x4_0

  418.     KERNEL8x4_I1_L4_3  16,32, 0,1

  419.     MY_ALIGN

  420. LSGEMM_L8x4_SUB2_2:

  421.     andi.      T1,L, 2

  422.     ble LSGEMM_L8x4_SUB2_1

  423.     LOAD8x4_0

  424.     KERNEL8x4_I1_L2_3  16,32, 0,1

  425.     MY_ALIGN    

  426. LSGEMM_L8x4_SUB2_1:

  427.     andi.      T1,L, 1

  428.     ble LSGEMM_L8x4_SAVE   

  429.     KERNEL8x4 0

  430.  

  431. 

  432.     MY_ALIGN

  433. LSGEMM_L8x4_SAVE:

  434.     SAVE8x4

  435. #if defined(TRMMKERNEL) 

  436.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,8

  437. #endif  

  438.     MY_ALIGN  

  439. LSGEMM_L8x4_END:

  440. LSGEMM_L8x2_BEGIN:

  441.     andi.       T1, M,  2

  442.     ble     LSGEMM_L8x2_END

  443. 

  444. #if defined(TRMMKERNEL)

  445.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,8

  446. #else

  447.     mr      BO, B

  448. #endif  

  449. 

  450. #if defined(TRMMKERNEL)

  451.    REFRESH_TEMP_BK T11,K,TEMP_REG,2,8 

  452.    srawi.       L, T11, 3 /**(T11) % 8x */

  453. #else

  454.    srawi.       L,  K,    3 /**(K) % 8x */

  455. #endif 

  456.     

  457.     ZERO8x2

  458.     ble     LSGEMM_L8x2_SUB0

  459. 

  460.     MY_ALIGN

  461. LSGEMM_L8x2_LOOP_START: 

  462.     mtctr       L

  463. 

  464.     MY_ALIGN

  465. 

  466. LSGEMM_L8x2_LOOP:

  467. 

  468.     KERNEL8x2_2  0,0, 0,0

  469.     KERNEL8x2_2  0,0, 1,0

  470.     KERNEL8x2_2  0,0, 2,0

  471.     KERNEL8x2_2  0,0, 3,1    

  472. 

  473.     bdnz        LSGEMM_L8x2_LOOP

  474. 

  475.     MY_ALIGN

  476. LSGEMM_L8x2_LOOP_END:   

  477.  

  478. LSGEMM_L8x2_SUB0:

  479. #if defined(TRMMKERNEL)

  480.     andi.       L,  T11,    7

  481. #else

  482.     andi.       L,  K,  7

  483. #endif    

  484.     ble     LSGEMM_L8x2_SAVE

  485.     MY_ALIGN

  486. LSGEMM_L8x2_SUB2:

  487.     andi.      T1,L, 4

  488.     ble LSGEMM_L8x2_SUB2_2

  489.     KERNEL8x2_2  0,0, 0,0

  490.     KERNEL8x2_2  0,0, 1,1

  491.     MY_ALIGN

  492. LSGEMM_L8x2_SUB2_2:

  493.     andi.      T1,L, 2

  494.     ble LSGEMM_L8x2_SUB2_1

  495.     KERNEL8x2_2  0,0, 0,1 

  496.     MY_ALIGN    

  497. LSGEMM_L8x2_SUB2_1:

  498.     andi.      T1,L, 1

  499.     ble LSGEMM_L8x2_SAVE   

  500.     KERNEL8x2

  501.   

  502.     MY_ALIGN

  503. LSGEMM_L8x2_SAVE:

  504.     SAVE8x2

  505. #if defined(TRMMKERNEL) 

  506.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,8

  507. #endif  

  508.     MY_ALIGN  

  509. LSGEMM_L8x2_END:

  510. LSGEMM_L8x1_BEGIN: 

  511.     andi.       T1, M,  1

  512.     ble     LSGEMM_L8x1_END

  513. #if defined(TRMMKERNEL)

  514.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,8

  515. #else

  516.     mr      BO, B

  517. #endif  

  518. 

  519. #if defined(TRMMKERNEL)

  520.    REFRESH_TEMP_BK T11,K,TEMP_REG,1,8 

  521.    srawi.       L, T11, 3 /**(T11) % 8x */

  522. #else

  523.    srawi.       L,  K,    3 /**(K) % 8x */

  524. #endif 

  525.     

  526.     ZERO8x1

  527.     ble     LSGEMM_L8x1_SUB0

  528. 

  529.     MY_ALIGN

  530. LSGEMM_L8x1_LOOP_START: 

  531.     mtctr       L

  532. 

  533.     MY_ALIGN

  534. 

  535. LSGEMM_L8x1_LOOP:

  536. 

  537.     KERNEL8x1_4  0,0, 0,0

  538.     KERNEL8x1_4  0,0, 1,1     

  539. 

  540.     bdnz        LSGEMM_L8x1_LOOP

  541. 

  542.     MY_ALIGN

  543. LSGEMM_L8x1_LOOP_END:   

  544.  

  545. LSGEMM_L8x1_SUB0:

  546. #if defined(TRMMKERNEL)

  547.     andi.       L,  T11,    7

  548. #else

  549.     andi.       L,  K,  7

  550. #endif    

  551.     ble     LSGEMM_L8x1_SAVE

  552.     MY_ALIGN

  553. LSGEMM_L8x1_SUB2:

  554.     andi.      T1,L, 4

  555.     ble LSGEMM_L8x1_SUB2_2

  556.     KERNEL8x1_4  0,0, 0,1 

  557.     MY_ALIGN

  558. LSGEMM_L8x1_SUB2_2:

  559.     andi.      T1,L, 2

  560.     ble LSGEMM_L8x1_SUB2_1

  561.     KERNEL8x1_2 

  562.     MY_ALIGN    

  563. LSGEMM_L8x1_SUB2_1:

  564.     andi.      T1,L, 1

  565.     ble LSGEMM_L8x1_SAVE   

  566.     KERNEL8x1

  567.   

  568.     MY_ALIGN

  569. LSGEMM_L8x1_SAVE:

  570.     SAVE8x1

  571. #if defined(TRMMKERNEL) 

  572.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,8

  573. #endif  

  574.     MY_ALIGN  

  575. LSGEMM_L8x1_END:

  576. 

  577. 	slwi		T1,	K,	5

  578. 	add		B,	B,	T1

  579. #if defined(TRMMKERNEL) && !defined(LEFT)

  580.     addi TEMP_REG, TEMP_REG, 8

  581. #endif

  582. 	addic.		J,	J,	-1

  583. 	bgt		LSGEMM_L8_BEGIN

  584.  

  585. 

  586. LSGEMM_L8_END:

  587. 

  588. /*	b		LSGEMM_L4_BEGIN*/

  589.     andi.       T1, N,  4

  590.     ble     LSGEMM_L4_END

  591. LSGEMM_L4_BEGIN:

  592.   

  593. 

  594. 	mr		AO,	A

  595. 	mr		CO,	C

  596. 	slwi		T3,	LDC	,	2

  597. 	add		C,	C,	T3

  598.  

  599. #if defined(TRMMKERNEL) && defined(LEFT)

  600. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  601. #endif 

  602. 	srawi.		I,	M,	4

  603. 	ble		LSGEMM_L4x16_END

  604. 

  605. 	MY_ALIGN

  606. LSGEMM_L4x16_BEGIN:

  607. 

  608. #if defined(TRMMKERNEL)

  609.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,4

  610. #else

  611. 	mr		BO,	B

  612. #endif	

  613. 

  614. #if defined(TRMMKERNEL)

  615.    REFRESH_TEMP_BK T11,K,TEMP_REG,16,4

  616.    mr T12, T11

  617.    addi T12,T12, -1

  618.    srawi.		L, T12,	6 /**(T11-1) % 64x */

  619. #else

  620.    mr T12, K

  621.    addi T12,T12, -1

  622.    srawi.		L,	T12,	6 /**(K-1) % 64x */

  623. #endif 

  624.  

  625.     ZERO4x16

  626. 	ble		LSGEMM_L4x16_SUB0

  627. 

  628. 	MY_ALIGN

  629. LSGEMM_L4x16_LOOP_START:

  630.  

  631. 	LOAD4x16_0  /*we already zeroed */

  632.     ##OffsetA=64 OffsetB=16

  633.     addi AO,AO,2112

  634.     addi BO,BO,16  

  635. 

  636. 	mtctr		L

  637. 

  638. 	MY_ALIGN

  639. 

  640. LSGEMM_L4x16_LOOP:

  641. 

  642.     KERNEL4x16_I1_L4_2  -2048,0, 0,0

  643.     KERNEL4x16_I1_L4_2  -2048,0, 1,0

  644.     KERNEL4x16_I1_L4_2  -2048,0, 2,0

  645.     KERNEL4x16_I1_L4_2  -2048,0, 3,0

  646.     KERNEL4x16_I1_L4_2  -2048,0, 4,0

  647.     KERNEL4x16_I1_L4_2  -2048,0, 5,0        

  648.     KERNEL4x16_I1_L4_2  -2048,0, 6,0

  649.     KERNEL4x16_I1_L4_2  -2048,0, 7,0  

  650.     KERNEL4x16_I1_L4_2  -2048,0, 8,0      

  651.     KERNEL4x16_I1_L4_2  -2048,0, 9,0

  652.     KERNEL4x16_I1_L4_2  -2048,0, 10,0

  653.     KERNEL4x16_I1_L4_2  -2048,0, 11,0

  654.     KERNEL4x16_I1_L4_2  -2048,0, 12,0

  655.     KERNEL4x16_I1_L4_2  -2048,0, 13,0    

  656.     KERNEL4x16_I1_L4_2  -2048,0, 14,0    

  657.     KERNEL4x16_I1_L4_2  -2048,0, 15,1  	

  658. 

  659. 	bdnz		LSGEMM_L4x16_LOOP

  660. 

  661. 	MY_ALIGN

  662. LSGEMM_L4x16_LOOP_END:

  663. 

  664.     END4x16 0, AO, BO, -2048, 0    

  665. 

  666. 	b		LSGEMM_L4x16_SUB1 

  667. 	MY_ALIGN

  668. LSGEMM_L4x16_SUB0:

  669. #if defined(TRMMKERNEL)

  670. 	andi.		L,	T11,	127

  671. #else

  672. 	andi.		L,	K,	127

  673. #endif   

  674. 	b		LSGEMM_L4x16_SUB2

  675. 	MY_ALIGN

  676. LSGEMM_L4x16_SUB1:

  677. #if defined(TRMMKERNEL)

  678. 	andi.		L,	T12,	63

  679. #else

  680. 	andi.		L,  T12,	63

  681. #endif	

  682. 	ble		LSGEMM_L4x16_SAVE

  683. 	MY_ALIGN

  684. LSGEMM_L4x16_SUB2:

  685. 

  686.     srawi.      T10,L, 5

  687.     ble LSGEMM_L4x16_SUB2_16

  688.     mtctr		T10

  689.     MY_ALIGN

  690. LSGEMM_L4x16_SUB2_LOOP:

  691. 	LOAD4x16_0 

  692.     KERNEL4x16_I1_L4_2  64,16, 0,0

  693.     KERNEL4x16_I1_L4_2  64,16, 1,0

  694.     KERNEL4x16_I1_L4_2  64,16, 2,0

  695.     KERNEL4x16_I1_L4_2  64,16, 3,0

  696.     KERNEL4x16_I1_L4_2  64,16, 4,0

  697.     KERNEL4x16_I1_L4_2  64,16, 5,0

  698.     KERNEL4x16_I1_L4_2  64,16, 6,0

  699.     KERNEL4x16_I1_L4_3  64,16, 7,1

  700.     bdnz LSGEMM_L4x16_SUB2_LOOP 

  701.     MY_ALIGN        

  702. LSGEMM_L4x16_SUB2_16:

  703.     andi.      T10,L, 16

  704.     ble LSGEMM_L4x16_SUB2_8

  705. 	LOAD4x16_0 

  706.     KERNEL4x16_I1_L4_2  64,16, 0,0

  707.     KERNEL4x16_I1_L4_2  64,16, 1,0

  708.     KERNEL4x16_I1_L4_2  64,16, 2,0

  709.     KERNEL4x16_I1_L4_3  64,16, 3,1

  710.     MY_ALIGN 

  711. LSGEMM_L4x16_SUB2_8:

  712.     andi.      T10,L, 8

  713.     ble LSGEMM_L4x16_SUB2_4 

  714. 	LOAD4x16_0

  715.     KERNEL4x16_I1_L4_2  64,16, 0,0

  716.     KERNEL4x16_I1_L4_3  64,16, 1,1

  717. 	MY_ALIGN	

  718. LSGEMM_L4x16_SUB2_4:

  719.     andi.      T10,L, 4

  720.     ble LSGEMM_L4x16_SUB2_2

  721.     LOAD4x16_0

  722.     KERNEL4x16_I1_L4_3  64,16, 0,1

  723.     MY_ALIGN

  724. LSGEMM_L4x16_SUB2_2:

  725.     andi.      T10,L, 2

  726.     ble LSGEMM_L4x16_SUB2_1

  727.     LOAD4x16_0

  728.     KERNEL4x16_I1_L2_3  64,16, 0,1

  729.     MY_ALIGN    

  730. LSGEMM_L4x16_SUB2_1:

  731.     andi.      T10,L, 1

  732.     ble LSGEMM_L4x16_SAVE	

  733.     KERNEL4x16 0

  734. #	addic.		L,	L,	-1

  735. #	bgt		LSGEMM_L4x16_SUB2

  736. 

  737. 	MY_ALIGN

  738. LSGEMM_L4x16_SAVE:

  739. 	SAVE4x16

  740. #if defined(TRMMKERNEL)	

  741. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,4

  742. #endif	

  743. 	addic.		I,	I,	-1

  744. 	bgt+		LSGEMM_L4x16_BEGIN

  745.     MY_ALIGN

  746. LSGEMM_L4x16_END:

  747. LSGEMM_L4x8_BEGIN:

  748.     andi.       T2, M,  15

  749.     ble     LSGEMM_L4x1_END

  750. 

  751.     andi.       T1, M,  8

  752.     ble     LSGEMM_L4x8_END

  753. 

  754. #if defined(TRMMKERNEL)

  755.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4

  756. #else

  757.     mr      BO, B

  758. #endif  

  759. 

  760. #if defined(TRMMKERNEL)

  761.    REFRESH_TEMP_BK T11,K,TEMP_REG,8,4

  762.    mr T12, T11

  763.    addi T12,T12, -1

  764.    srawi.       L, T12, 4 /**(T11-1) % 16x */

  765. #else

  766.    mr T12, K

  767.    addi T12,T12, -1

  768.    srawi.       L,  T12,    4 /**(K-1) % 16x */

  769. #endif 

  770.     

  771.     ZERO4x8

  772.     ble     LSGEMM_L4x8_SUB0

  773. 

  774.     MY_ALIGN

  775. LSGEMM_L4x8_LOOP_START:

  776.  

  777.     LOAD4x8_0  /*we already zeroed */ 

  778.     mtctr       L

  779. 

  780.     MY_ALIGN

  781. 

  782. LSGEMM_L4x8_LOOP:

  783. 

  784.     KERNEL4x8_I1_L4_2  32,16, 0,0

  785.     KERNEL4x8_I1_L4_2  32,16, 1,0

  786.     KERNEL4x8_I1_L4_2  32,16, 2,0

  787.     KERNEL4x8_I1_L4_2  32,16, 3,1    

  788. 

  789.     bdnz        LSGEMM_L4x8_LOOP

  790. 

  791.     MY_ALIGN

  792. LSGEMM_L4x8_LOOP_END:

  793. 

  794.     END4x8 0, AO, BO, 32, 16    

  795. 

  796.     b       LSGEMM_L4x8_SUB1 

  797.     MY_ALIGN

  798. LSGEMM_L4x8_SUB0:

  799. #if defined(TRMMKERNEL)

  800.     andi.       L,  T11,    31

  801. #else

  802.     andi.       L,  K,  31

  803. #endif   

  804.     b       LSGEMM_L4x8_SUB2

  805.     MY_ALIGN

  806. LSGEMM_L4x8_SUB1:

  807. #if defined(TRMMKERNEL)

  808.     andi.       L,  T12,    15

  809. #else

  810.     andi.       L,  T12,    15

  811. #endif  

  812.     ble     LSGEMM_L4x8_SAVE

  813.     MY_ALIGN

  814. LSGEMM_L4x8_SUB2:

  815.  

  816.     srawi.      T1,L, 3

  817.     ble LSGEMM_L4x8_SUB2_4 

  818.     mtctr		T1

  819.     MY_ALIGN

  820. LSGEMM_L4x8_SUB2_LOOP:    

  821.     LOAD4x8_0

  822.     KERNEL4x8_I1_L4_2  32,16, 0,0

  823.     KERNEL4x8_I1_L4_3  32,16, 1,1

  824.     bdnz LSGEMM_L4x8_SUB2_LOOP

  825.     MY_ALIGN    

  826. LSGEMM_L4x8_SUB2_4:

  827.     andi.      T1,L, 4

  828.     ble LSGEMM_L4x8_SUB2_2

  829.     LOAD4x8_0

  830.     KERNEL4x8_I1_L4_3  32,16, 0,1

  831.     MY_ALIGN

  832. LSGEMM_L4x8_SUB2_2:

  833.     andi.      T1,L, 2

  834.     ble LSGEMM_L4x8_SUB2_1

  835.     LOAD4x8_0

  836.     KERNEL4x8_I1_L2_3  32,16, 0,1

  837.     MY_ALIGN    

  838. LSGEMM_L4x8_SUB2_1:

  839.     andi.      T1,L, 1

  840.     ble LSGEMM_L4x8_SAVE   

  841.     KERNEL4x8 0

  842.  

  843. 

  844.     MY_ALIGN

  845. LSGEMM_L4x8_SAVE:

  846.     SAVE4x8

  847. #if defined(TRMMKERNEL) 

  848.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,4

  849. #endif  

  850.     MY_ALIGN  

  851. LSGEMM_L4x8_END:

  852. LSGEMM_L4x4_BEGIN:

  853.     andi.       T2, M,  15

  854.     ble     LSGEMM_L4x1_END

  855. 

  856.     andi.       T1, M,  4

  857.     ble     LSGEMM_L4x4_END

  858. 

  859. #if defined(TRMMKERNEL)

  860.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4

  861. #else

  862.     mr      BO, B

  863. #endif  

  864. 

  865. #if defined(TRMMKERNEL)

  866.    REFRESH_TEMP_BK T11,K,TEMP_REG,4,4

  867.    mr T12, T11

  868.    addi T12,T12, -1

  869.    srawi.       L, T12, 4 /**(T11-1) % 16x */

  870. #else

  871.    mr T12, K

  872.    addi T12,T12, -1

  873.    srawi.       L,  T12,    4 /**(K-1) % 16x */

  874. #endif 

  875.     

  876.     ZERO4x4

  877.     ble     LSGEMM_L4x4_SUB0

  878. 

  879.     MY_ALIGN

  880. LSGEMM_L4x4_LOOP_START:

  881.  

  882.     LOAD4x4_0  /*we already zeroed */ 

  883.     mtctr       L

  884. 

  885.     MY_ALIGN

  886. 

  887. LSGEMM_L4x4_LOOP:

  888. 

  889.     KERNEL4x4_I1_L4_2  16,16, 0,0

  890.     KERNEL4x4_I1_L4_2  16,16, 1,0

  891.     KERNEL4x4_I1_L4_2  16,16, 2,0

  892.     KERNEL4x4_I1_L4_2  16,16, 3,1    

  893. 

  894.     bdnz        LSGEMM_L4x4_LOOP

  895. 

  896.     MY_ALIGN

  897. LSGEMM_L4x4_LOOP_END:

  898. 

  899.     END4x4 0, AO, BO, 16, 16    

  900. 

  901.     b       LSGEMM_L4x4_SUB1 

  902.     MY_ALIGN

  903. LSGEMM_L4x4_SUB0:

  904. #if defined(TRMMKERNEL)

  905.     andi.       L,  T11,    31

  906. #else

  907.     andi.       L,  K,  31

  908. #endif   

  909.     b       LSGEMM_L4x4_SUB2

  910.     MY_ALIGN

  911. LSGEMM_L4x4_SUB1:

  912. #if defined(TRMMKERNEL)

  913.     andi.       L,  T12,    15

  914. #else

  915.     andi.       L,  T12,    15

  916. #endif  

  917.     ble     LSGEMM_L4x4_SAVE

  918.     MY_ALIGN

  919. LSGEMM_L4x4_SUB2:

  920. 

  921.     srawi.      T1,L, 3 

  922.     ble LSGEMM_L4x4_SUB2_4  

  923.     mtctr		T1

  924.     MY_ALIGN

  925. LSGEMM_L4x4_SUB2_LOOP:     

  926.     LOAD4x4_0

  927.     KERNEL4x4_I1_L4_2  16,16, 0,0

  928.     KERNEL4x4_I1_L4_3  16,16, 1,1

  929.     bdnz LSGEMM_L4x4_SUB2_LOOP

  930.     MY_ALIGN    

  931. LSGEMM_L4x4_SUB2_4:

  932.     andi.      T1,L, 4

  933.     ble LSGEMM_L4x4_SUB2_2

  934.     LOAD4x4_0

  935.     KERNEL4x4_I1_L4_3  16,16, 0,1

  936.     MY_ALIGN

  937. LSGEMM_L4x4_SUB2_2:

  938.     andi.      T1,L, 2

  939.     ble LSGEMM_L4x4_SUB2_1

  940.     LOAD4x4_0

  941.     KERNEL4x4_I1_L2_3  16,16, 0,1

  942.     MY_ALIGN    

  943. LSGEMM_L4x4_SUB2_1:

  944.     andi.      T1,L, 1

  945.     ble LSGEMM_L4x4_SAVE   

  946.     KERNEL4x4 0

  947.  

  948. 

  949.     MY_ALIGN

  950. LSGEMM_L4x4_SAVE:

  951.     SAVE4x4

  952. #if defined(TRMMKERNEL) 

  953.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,4

  954. #endif  

  955.     MY_ALIGN  

  956. LSGEMM_L4x4_END:

  957. LSGEMM_L4x2_BEGIN:

  958.     andi.       T1, M,  2

  959.     ble     LSGEMM_L4x2_END

  960. 

  961. #if defined(TRMMKERNEL)

  962.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4

  963. #else

  964.     mr      BO, B

  965. #endif  

  966. 

  967. #if defined(TRMMKERNEL)

  968.    REFRESH_TEMP_BK T11,K,TEMP_REG,2,4 

  969.    srawi.       L, T11, 3 /**(T11) % 8x */

  970. #else

  971.    srawi.       L,  K,    3 /**(K) % 8x */

  972. #endif 

  973.     

  974.     ZERO4x2

  975.     ble     LSGEMM_L4x2_SUB0

  976. 

  977.     MY_ALIGN

  978. LSGEMM_L4x2_LOOP_START: 

  979.     mtctr       L

  980. 

  981.     MY_ALIGN

  982. 

  983. LSGEMM_L4x2_LOOP:

  984. 

  985.     KERNEL4x2_2  0,0, 0,0

  986.     KERNEL4x2_2  0,0, 1,0

  987.     KERNEL4x2_2  0,0, 2,0

  988.     KERNEL4x2_2  0,0, 3,1    

  989. 

  990.     bdnz        LSGEMM_L4x2_LOOP

  991. 

  992.     MY_ALIGN

  993. LSGEMM_L4x2_LOOP_END:   

  994.  

  995. LSGEMM_L4x2_SUB0:

  996. #if defined(TRMMKERNEL)

  997.     andi.       L,  T11,    7

  998. #else

  999.     andi.       L,  K,  7

  1000. #endif    

  1001.     ble     LSGEMM_L4x2_SAVE

  1002.     MY_ALIGN

  1003. LSGEMM_L4x2_SUB2:

  1004.     andi.      T1,L, 4

  1005.     ble LSGEMM_L4x2_SUB2_2

  1006.     KERNEL4x2_2  0,0, 0,0

  1007.     KERNEL4x2_2  0,0, 1,1

  1008.     MY_ALIGN

  1009. LSGEMM_L4x2_SUB2_2:

  1010.     andi.      T1,L, 2

  1011.     ble LSGEMM_L4x2_SUB2_1

  1012.     KERNEL4x2_2  0,0, 0,1 

  1013.     MY_ALIGN    

  1014. LSGEMM_L4x2_SUB2_1:

  1015.     andi.      T1,L, 1

  1016.     ble LSGEMM_L4x2_SAVE   

  1017.     KERNEL4x2

  1018.   

  1019.     MY_ALIGN

  1020. LSGEMM_L4x2_SAVE:

  1021.     SAVE4x2

  1022. #if defined(TRMMKERNEL) 

  1023.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,4

  1024. #endif  

  1025.     MY_ALIGN  

  1026. LSGEMM_L4x2_END:

  1027. LSGEMM_L4x1_BEGIN: 

  1028.     andi.       T1, M,  1

  1029.     ble     LSGEMM_L4x1_END

  1030. #if defined(TRMMKERNEL)

  1031.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4

  1032. #else

  1033.     mr      BO, B

  1034. #endif  

  1035. 

  1036. #if defined(TRMMKERNEL)

  1037.    REFRESH_TEMP_BK T11,K,TEMP_REG,1,4 

  1038.    srawi.       L, T11, 3 /**(T11) % 8x */

  1039. #else

  1040.    srawi.       L,  K,    3 /**(K) % 8x */

  1041. #endif 

  1042.     

  1043.     ZERO4x1

  1044.     ble     LSGEMM_L4x1_SUB0

  1045. 

  1046.     MY_ALIGN

  1047. LSGEMM_L4x1_LOOP_START: 

  1048.     mtctr       L

  1049. 

  1050.     MY_ALIGN

  1051. 

  1052. LSGEMM_L4x1_LOOP:

  1053. 

  1054.     KERNEL4x1_4  0,0, 0,0

  1055.     KERNEL4x1_4  0,0, 1,1     

  1056. 

  1057.     bdnz        LSGEMM_L4x1_LOOP

  1058. 

  1059.     MY_ALIGN

  1060. LSGEMM_L4x1_LOOP_END:   

  1061.  

  1062. LSGEMM_L4x1_SUB0:

  1063. #if defined(TRMMKERNEL)

  1064.     andi.       L,  T11,    7

  1065. #else

  1066.     andi.       L,  K,  7

  1067. #endif    

  1068.     ble     LSGEMM_L4x1_SAVE

  1069.     MY_ALIGN

  1070. LSGEMM_L4x1_SUB2:

  1071.     andi.      T1,L, 4

  1072.     ble LSGEMM_L4x1_SUB2_2

  1073.     KERNEL4x1_4  0,0, 0,1 

  1074.     MY_ALIGN

  1075. LSGEMM_L4x1_SUB2_2:

  1076.     andi.      T1,L, 2

  1077.     ble LSGEMM_L4x1_SUB2_1

  1078.     KERNEL4x1_2 

  1079.     MY_ALIGN    

  1080. LSGEMM_L4x1_SUB2_1:

  1081.     andi.      T1,L, 1

  1082.     ble LSGEMM_L4x1_SAVE   

  1083.     KERNEL4x1

  1084.   

  1085.     MY_ALIGN

  1086. LSGEMM_L4x1_SAVE:

  1087.     SAVE4x1

  1088. #if defined(TRMMKERNEL) 

  1089.     REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,4

  1090. #endif  

  1091.     MY_ALIGN  

  1092. LSGEMM_L4x1_END:

  1093. 

  1094. 	slwi		T1,	K,	4

  1095. 	add		B,	B,	T1

  1096. #if defined(TRMMKERNEL) && !defined(LEFT)

  1097.     addi TEMP_REG, TEMP_REG, 4

  1098. #endif

  1099. 

  1100. 	andi.		T2,	N,	3

  1101. 	ble		.L999

  1102. 

  1103. LSGEMM_L4_END:

  1104.     andi.       T1, N,  2

  1105.     ble     LSGEMM_L2_END

  1106. LSGEMM_L2_BEGIN:

  1107.   

  1108. 

  1109. 	mr		AO,	A

  1110. 	mr		CO,	C

  1111. 	slwi		T3,	LDC	,	1

  1112. 	add		C,	C,	T3

  1113.  

  1114. #if defined(TRMMKERNEL) && defined(LEFT)

  1115. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  1116. #endif 

  1117. 	srawi.		I,	M,	4

  1118. 	ble		LSGEMM_L2x16_END

  1119. 

  1120. 	MY_ALIGN

  1121. LSGEMM_L2x16_BEGIN:

  1122. 

  1123. #if defined(TRMMKERNEL)

  1124.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,2

  1125. #else

  1126. 	mr		BO,	B

  1127. #endif	

  1128. 

  1129. #if defined(TRMMKERNEL)

  1130.    REFRESH_TEMP_BK T11,K,TEMP_REG,16,2 

  1131.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1132. #else 

  1133.    srawi.		L,	K,	6 /**(K ) % 64x */

  1134. #endif 

  1135.  

  1136.     ZERO2x16

  1137. 	ble		LSGEMM_L2x16_SUB0

  1138.     addi AO,AO,2048

  1139.   

  1140. 	mtctr		L

  1141. 

  1142. 	MY_ALIGN

  1143. 

  1144. LSGEMM_L2x16_LOOP:

  1145. 

  1146.     KERNEL2x16_4  -2048,0, 0,0

  1147.     KERNEL2x16_4  -2048,0, 1,0

  1148.     KERNEL2x16_4  -2048,0, 2,0

  1149.     KERNEL2x16_4  -2048,0, 3,0

  1150.     KERNEL2x16_4  -2048,0, 4,0

  1151.     KERNEL2x16_4  -2048,0, 5,0        

  1152.     KERNEL2x16_4  -2048,0, 6,0

  1153.     KERNEL2x16_4  -2048,0, 7,0  

  1154.     KERNEL2x16_4  -2048,0, 8,0      

  1155.     KERNEL2x16_4  -2048,0, 9,0

  1156.     KERNEL2x16_4  -2048,0, 10,0

  1157.     KERNEL2x16_4  -2048,0, 11,0

  1158.     KERNEL2x16_4  -2048,0, 12,0

  1159.     KERNEL2x16_4  -2048,0, 13,0    

  1160.     KERNEL2x16_4  -2048,0, 14,0    

  1161.     KERNEL2x16_4  -2048,0, 15,1  	

  1162. 

  1163. 	bdnz		LSGEMM_L2x16_LOOP

  1164.     MY_ALIGN

  1165.     addi AO,AO, -2048

  1166. 	MY_ALIGN

  1167. LSGEMM_L2x16_SUB0: 

  1168. #if defined(TRMMKERNEL)

  1169. 	andi.		L,	T11,	63

  1170. #else

  1171. 	andi.		L,  K,	63

  1172. #endif	

  1173. 	ble		LSGEMM_L2x16_SAVE

  1174. 	MY_ALIGN

  1175. LSGEMM_L2x16_SUB2:

  1176.     andi.      T10,L, 32

  1177.     ble LSGEMM_L2x16_SUB2_16 

  1178.     KERNEL2x16_4  0,0, 0,0

  1179.     KERNEL2x16_4  0,0, 1,0

  1180.     KERNEL2x16_4  0,0, 2,0

  1181.     KERNEL2x16_4  0,0, 3,0

  1182.     KERNEL2x16_4  0,0, 4,0

  1183.     KERNEL2x16_4  0,0, 5,0

  1184.     KERNEL2x16_4  0,0, 6,0

  1185.     KERNEL2x16_4  0,0, 7,1 

  1186.     MY_ALIGN        

  1187. LSGEMM_L2x16_SUB2_16:

  1188.     andi.      T10,L, 16

  1189.     ble LSGEMM_L2x16_SUB2_8 

  1190.     KERNEL2x16_4  0,0, 0,0

  1191.     KERNEL2x16_4  0,0, 1,0

  1192.     KERNEL2x16_4  0,0, 2,0

  1193.     KERNEL2x16_4  0,0, 3,1

  1194.     MY_ALIGN 

  1195. LSGEMM_L2x16_SUB2_8:

  1196.     andi.      T10,L, 8

  1197.     ble LSGEMM_L2x16_SUB2_4  

  1198.     KERNEL2x16_4  0,0, 0,0

  1199.     KERNEL2x16_4  0,0, 1,1

  1200. 	MY_ALIGN	

  1201. LSGEMM_L2x16_SUB2_4:

  1202.     andi.      T10,L, 4

  1203.     ble LSGEMM_L2x16_SUB2_2 

  1204.     KERNEL2x16_4  0,0, 0,1

  1205.     MY_ALIGN

  1206. LSGEMM_L2x16_SUB2_2:

  1207.     andi.      T10,L, 2

  1208.     ble LSGEMM_L2x16_SUB2_1 

  1209.     KERNEL2x16_2  0,0, 0,1

  1210.     MY_ALIGN    

  1211. LSGEMM_L2x16_SUB2_1:

  1212.     andi.      T10,L, 1

  1213.     ble LSGEMM_L2x16_SAVE	

  1214.     KERNEL2x16

  1215. 

  1216. 	MY_ALIGN

  1217. LSGEMM_L2x16_SAVE:

  1218. 	SAVE2x16

  1219. #if defined(TRMMKERNEL)	

  1220. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,2

  1221. #endif	

  1222. 	addic.		I,	I,	-1

  1223. 	bgt+		LSGEMM_L2x16_BEGIN

  1224.     MY_ALIGN

  1225. LSGEMM_L2x16_END:

  1226. 	andi.		I,	M,	8

  1227. 	ble		LSGEMM_L2x8_END

  1228. 

  1229. 	MY_ALIGN

  1230. LSGEMM_L2x8_BEGIN:

  1231. 

  1232. #if defined(TRMMKERNEL)

  1233.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2

  1234. #else

  1235. 	mr		BO,	B

  1236. #endif	

  1237. 

  1238. #if defined(TRMMKERNEL)

  1239.    REFRESH_TEMP_BK T11,K,TEMP_REG,8,2 

  1240.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1241. #else 

  1242.    srawi.		L,	K,	6 /**(K ) % 64x */

  1243. #endif 

  1244.  

  1245.     ZERO2x8

  1246. 	ble		LSGEMM_L2x8_SUB0

  1247.     addi AO,AO,2048

  1248.   

  1249. 	mtctr		L

  1250. 

  1251. 	MY_ALIGN

  1252. 

  1253. LSGEMM_L2x8_LOOP:

  1254. 

  1255.     KERNEL2x8_4  -2048,0, 0,0

  1256.     KERNEL2x8_4  -2048,0, 1,0

  1257.     KERNEL2x8_4  -2048,0, 2,0

  1258.     KERNEL2x8_4  -2048,0, 3,0

  1259.     KERNEL2x8_4  -2048,0, 4,0

  1260.     KERNEL2x8_4  -2048,0, 5,0        

  1261.     KERNEL2x8_4  -2048,0, 6,0

  1262.     KERNEL2x8_4  -2048,0, 7,0  

  1263.     KERNEL2x8_4  -2048,0, 8,0      

  1264.     KERNEL2x8_4  -2048,0, 9,0

  1265.     KERNEL2x8_4  -2048,0, 10,0

  1266.     KERNEL2x8_4  -2048,0, 11,0

  1267.     KERNEL2x8_4  -2048,0, 12,0

  1268.     KERNEL2x8_4  -2048,0, 13,0    

  1269.     KERNEL2x8_4  -2048,0, 14,0    

  1270.     KERNEL2x8_4  -2048,0, 15,1  	

  1271. 

  1272. 	bdnz		LSGEMM_L2x8_LOOP

  1273.     MY_ALIGN

  1274.     addi AO,AO, -2048

  1275. 	MY_ALIGN

  1276. LSGEMM_L2x8_SUB0: 

  1277. #if defined(TRMMKERNEL)

  1278. 	andi.		L,	T11,	63

  1279. #else

  1280. 	andi.		L,  K,	63

  1281. #endif	

  1282. 	ble		LSGEMM_L2x8_SAVE

  1283. 	MY_ALIGN

  1284. LSGEMM_L2x8_SUB2:

  1285.     andi.      T10,L, 32

  1286.     ble LSGEMM_L2x8_SUB2_16 

  1287.     KERNEL2x8_4  0,0, 0,0

  1288.     KERNEL2x8_4  0,0, 1,0

  1289.     KERNEL2x8_4  0,0, 2,0

  1290.     KERNEL2x8_4  0,0, 3,0

  1291.     KERNEL2x8_4  0,0, 4,0

  1292.     KERNEL2x8_4  0,0, 5,0

  1293.     KERNEL2x8_4  0,0, 6,0

  1294.     KERNEL2x8_4  0,0, 7,1 

  1295.     MY_ALIGN        

  1296. LSGEMM_L2x8_SUB2_16:

  1297.     andi.      T10,L, 16

  1298.     ble LSGEMM_L2x8_SUB2_8 

  1299.     KERNEL2x8_4  0,0, 0,0

  1300.     KERNEL2x8_4  0,0, 1,0

  1301.     KERNEL2x8_4  0,0, 2,0

  1302.     KERNEL2x8_4  0,0, 3,1

  1303.     MY_ALIGN 

  1304. LSGEMM_L2x8_SUB2_8:

  1305.     andi.      T10,L, 8

  1306.     ble LSGEMM_L2x8_SUB2_4  

  1307.     KERNEL2x8_4  0,0, 0,0

  1308.     KERNEL2x8_4  0,0, 1,1

  1309. 	MY_ALIGN	

  1310. LSGEMM_L2x8_SUB2_4:

  1311.     andi.      T10,L, 4

  1312.     ble LSGEMM_L2x8_SUB2_2 

  1313.     KERNEL2x8_4  0,0, 0,1

  1314.     MY_ALIGN

  1315. LSGEMM_L2x8_SUB2_2:

  1316.     andi.      T10,L, 2

  1317.     ble LSGEMM_L2x8_SUB2_1 

  1318.     KERNEL2x8_2  0,0, 0,1

  1319.     MY_ALIGN    

  1320. LSGEMM_L2x8_SUB2_1:

  1321.     andi.      T10,L, 1

  1322.     ble LSGEMM_L2x8_SAVE	

  1323.     KERNEL2x8

  1324. 

  1325. 	MY_ALIGN

  1326. LSGEMM_L2x8_SAVE:

  1327. 	SAVE2x8

  1328. #if defined(TRMMKERNEL)	

  1329. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,2

  1330. #endif	 

  1331.     MY_ALIGN

  1332. LSGEMM_L2x8_END:

  1333. 	andi.		I,	M,	4

  1334. 	ble		LSGEMM_L2x4_END

  1335. 

  1336. 	MY_ALIGN

  1337. LSGEMM_L2x4_BEGIN:

  1338. 

  1339. #if defined(TRMMKERNEL)

  1340.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2

  1341. #else

  1342. 	mr		BO,	B

  1343. #endif	

  1344. 

  1345. #if defined(TRMMKERNEL)

  1346.    REFRESH_TEMP_BK T11,K,TEMP_REG,4,2 

  1347.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1348. #else 

  1349.    srawi.		L,	K,	6 /**(K ) % 64x */

  1350. #endif 

  1351.  

  1352.     ZERO2x4

  1353. 	ble		LSGEMM_L2x4_SUB0

  1354.  

  1355.   

  1356. 	mtctr		L

  1357. 

  1358. 	MY_ALIGN

  1359. 

  1360. LSGEMM_L2x4_LOOP:

  1361. 

  1362.     KERNEL2x4_4  0,0, 0,0

  1363.     KERNEL2x4_4  0,0, 1,0

  1364.     KERNEL2x4_4  0,0, 2,0

  1365.     KERNEL2x4_4  0,0, 3,0

  1366.     KERNEL2x4_4  0,0, 4,0

  1367.     KERNEL2x4_4  0,0, 5,0        

  1368.     KERNEL2x4_4  0,0, 6,0

  1369.     KERNEL2x4_4  0,0, 7,0  

  1370.     KERNEL2x4_4  0,0, 8,0      

  1371.     KERNEL2x4_4  0,0, 9,0

  1372.     KERNEL2x4_4  0,0, 10,0

  1373.     KERNEL2x4_4  0,0, 11,0

  1374.     KERNEL2x4_4  0,0, 12,0

  1375.     KERNEL2x4_4  0,0, 13,0    

  1376.     KERNEL2x4_4  0,0, 14,0    

  1377.     KERNEL2x4_4  0,0, 15,1  	

  1378. 

  1379. 	bdnz		LSGEMM_L2x4_LOOP

  1380.     MY_ALIGN

  1381.  

  1382. 	MY_ALIGN

  1383. LSGEMM_L2x4_SUB0: 

  1384. #if defined(TRMMKERNEL)

  1385. 	andi.		L,	T11,	63

  1386. #else

  1387. 	andi.		L,  K,	63

  1388. #endif	

  1389. 	ble		LSGEMM_L2x4_SAVE

  1390. 	MY_ALIGN

  1391. LSGEMM_L2x4_SUB2:

  1392.     andi.      T10,L, 32

  1393.     ble LSGEMM_L2x4_SUB2_16 

  1394.     KERNEL2x4_4  0,0, 0,0

  1395.     KERNEL2x4_4  0,0, 1,0

  1396.     KERNEL2x4_4  0,0, 2,0

  1397.     KERNEL2x4_4  0,0, 3,0

  1398.     KERNEL2x4_4  0,0, 4,0

  1399.     KERNEL2x4_4  0,0, 5,0

  1400.     KERNEL2x4_4  0,0, 6,0

  1401.     KERNEL2x4_4  0,0, 7,1 

  1402.     MY_ALIGN        

  1403. LSGEMM_L2x4_SUB2_16:

  1404.     andi.      T10,L, 16

  1405.     ble LSGEMM_L2x4_SUB2_8 

  1406.     KERNEL2x4_4  0,0, 0,0

  1407.     KERNEL2x4_4  0,0, 1,0

  1408.     KERNEL2x4_4  0,0, 2,0

  1409.     KERNEL2x4_4  0,0, 3,1

  1410.     MY_ALIGN 

  1411. LSGEMM_L2x4_SUB2_8:

  1412.     andi.      T10,L, 8

  1413.     ble LSGEMM_L2x4_SUB2_4  

  1414.     KERNEL2x4_4  0,0, 0,0

  1415.     KERNEL2x4_4  0,0, 1,1

  1416. 	MY_ALIGN	

  1417. LSGEMM_L2x4_SUB2_4:

  1418.     andi.      T10,L, 4

  1419.     ble LSGEMM_L2x4_SUB2_2 

  1420.     KERNEL2x4_4  0,0, 0,1

  1421.     MY_ALIGN

  1422. LSGEMM_L2x4_SUB2_2:

  1423.     andi.      T10,L, 2

  1424.     ble LSGEMM_L2x4_SUB2_1 

  1425.     KERNEL2x4_2  0,0, 0,1

  1426.     MY_ALIGN    

  1427. LSGEMM_L2x4_SUB2_1:

  1428.     andi.      T10,L, 1

  1429.     ble LSGEMM_L2x4_SAVE	

  1430.     KERNEL2x4

  1431. 

  1432. 	MY_ALIGN

  1433. LSGEMM_L2x4_SAVE:

  1434. 	SAVE2x4

  1435. #if defined(TRMMKERNEL)	

  1436. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,2

  1437. #endif	 

  1438.     MY_ALIGN

  1439. LSGEMM_L2x4_END:

  1440. 	andi.		I,	M,	2

  1441. 	ble		LSGEMM_L2x2_END

  1442. 

  1443. 	MY_ALIGN

  1444. LSGEMM_L2x2_BEGIN:

  1445. 

  1446. #if defined(TRMMKERNEL)

  1447.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2

  1448. #else

  1449. 	mr		BO,	B

  1450. #endif	

  1451. 

  1452. #if defined(TRMMKERNEL)

  1453.    REFRESH_TEMP_BK T11,K,TEMP_REG,2,2 

  1454.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1455. #else 

  1456.    srawi.		L,	K,	6 /**(K ) % 64x */

  1457. #endif 

  1458.  

  1459.     ZERO2x2

  1460. 	ble		LSGEMM_L2x2_SUB0

  1461.  

  1462.   

  1463. 	mtctr		L

  1464. 

  1465. 	MY_ALIGN

  1466. 

  1467. LSGEMM_L2x2_LOOP:

  1468. 

  1469.     KERNEL2x2_4  0,0, 0,0

  1470.     KERNEL2x2_4  0,0, 1,0

  1471.     KERNEL2x2_4  0,0, 2,0

  1472.     KERNEL2x2_4  0,0, 3,0

  1473.     KERNEL2x2_4  0,0, 4,0

  1474.     KERNEL2x2_4  0,0, 5,0        

  1475.     KERNEL2x2_4  0,0, 6,0

  1476.     KERNEL2x2_4  0,0, 7,0  

  1477.     KERNEL2x2_4  0,0, 8,0      

  1478.     KERNEL2x2_4  0,0, 9,0

  1479.     KERNEL2x2_4  0,0, 10,0

  1480.     KERNEL2x2_4  0,0, 11,0

  1481.     KERNEL2x2_4  0,0, 12,0

  1482.     KERNEL2x2_4  0,0, 13,0    

  1483.     KERNEL2x2_4  0,0, 14,0    

  1484.     KERNEL2x2_4  0,0, 15,1  	

  1485. 

  1486. 	bdnz		LSGEMM_L2x2_LOOP

  1487.     MY_ALIGN

  1488.  

  1489. 	MY_ALIGN

  1490. LSGEMM_L2x2_SUB0: 

  1491. #if defined(TRMMKERNEL)

  1492. 	andi.		L,	T11,	63

  1493. #else

  1494. 	andi.		L,  K,	63

  1495. #endif	

  1496. 	ble		LSGEMM_L2x2_SAVE

  1497. 	MY_ALIGN

  1498. LSGEMM_L2x2_SUB2:

  1499.     andi.      T10,L, 32

  1500.     ble LSGEMM_L2x2_SUB2_16 

  1501.     KERNEL2x2_4  0,0, 0,0

  1502.     KERNEL2x2_4  0,0, 1,0

  1503.     KERNEL2x2_4  0,0, 2,0

  1504.     KERNEL2x2_4  0,0, 3,0

  1505.     KERNEL2x2_4  0,0, 4,0

  1506.     KERNEL2x2_4  0,0, 5,0

  1507.     KERNEL2x2_4  0,0, 6,0

  1508.     KERNEL2x2_4  0,0, 7,1 

  1509.     MY_ALIGN        

  1510. LSGEMM_L2x2_SUB2_16:

  1511.     andi.      T10,L, 16

  1512.     ble LSGEMM_L2x2_SUB2_8 

  1513.     KERNEL2x2_4  0,0, 0,0

  1514.     KERNEL2x2_4  0,0, 1,0

  1515.     KERNEL2x2_4  0,0, 2,0

  1516.     KERNEL2x2_4  0,0, 3,1

  1517.     MY_ALIGN 

  1518. LSGEMM_L2x2_SUB2_8:

  1519.     andi.      T10,L, 8

  1520.     ble LSGEMM_L2x2_SUB2_4  

  1521.     KERNEL2x2_4  0,0, 0,0

  1522.     KERNEL2x2_4  0,0, 1,1

  1523. 	MY_ALIGN	

  1524. LSGEMM_L2x2_SUB2_4:

  1525.     andi.      T10,L, 4

  1526.     ble LSGEMM_L2x2_SUB2_2 

  1527.     KERNEL2x2_4  0,0, 0,1

  1528.     MY_ALIGN

  1529. LSGEMM_L2x2_SUB2_2:

  1530.     andi.      T10,L, 2

  1531.     ble LSGEMM_L2x2_SUB2_1 

  1532.     KERNEL2x2_2  0,0, 0,1

  1533.     MY_ALIGN    

  1534. LSGEMM_L2x2_SUB2_1:

  1535.     andi.      T10,L, 1

  1536.     ble LSGEMM_L2x2_SAVE	

  1537.     KERNEL2x2

  1538. 

  1539. 	MY_ALIGN

  1540. LSGEMM_L2x2_SAVE:

  1541. 	SAVE2x2

  1542. #if defined(TRMMKERNEL)	

  1543. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,2

  1544. #endif	 

  1545.     MY_ALIGN

  1546. LSGEMM_L2x2_END:

  1547. 	andi.		I,	M,	1

  1548. 	ble		LSGEMM_L2x1_END

  1549. 

  1550. 	MY_ALIGN

  1551. LSGEMM_L2x1_BEGIN:

  1552. 

  1553. #if defined(TRMMKERNEL)

  1554.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2

  1555. #else

  1556. 	mr		BO,	B

  1557. #endif	

  1558. 

  1559. #if defined(TRMMKERNEL)

  1560.    REFRESH_TEMP_BK T11,K,TEMP_REG,1,2 

  1561.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1562. #else 

  1563.    srawi.		L,	K,	6 /**(K ) % 64x */

  1564. #endif 

  1565.  

  1566.     ZERO2x1

  1567. 	ble		LSGEMM_L2x1_SUB0

  1568.  

  1569.   

  1570. 	mtctr		L

  1571. 

  1572. 	MY_ALIGN

  1573. 

  1574. LSGEMM_L2x1_LOOP:

  1575. 

  1576.     KERNEL2x1_4  0,0, 0,0

  1577.     KERNEL2x1_4  0,0, 1,0

  1578.     KERNEL2x1_4  0,0, 2,0

  1579.     KERNEL2x1_4  0,0, 3,0

  1580.     KERNEL2x1_4  0,0, 4,0

  1581.     KERNEL2x1_4  0,0, 5,0        

  1582.     KERNEL2x1_4  0,0, 6,0

  1583.     KERNEL2x1_4  0,0, 7,0  

  1584.     KERNEL2x1_4  0,0, 8,0      

  1585.     KERNEL2x1_4  0,0, 9,0

  1586.     KERNEL2x1_4  0,0, 10,0

  1587.     KERNEL2x1_4  0,0, 11,0

  1588.     KERNEL2x1_4  0,0, 12,0

  1589.     KERNEL2x1_4  0,0, 13,0    

  1590.     KERNEL2x1_4  0,0, 14,0    

  1591.     KERNEL2x1_4  0,0, 15,1  	

  1592. 

  1593. 	bdnz		LSGEMM_L2x1_LOOP

  1594.     MY_ALIGN

  1595.  

  1596. 	MY_ALIGN

  1597. LSGEMM_L2x1_SUB0: 

  1598. #if defined(TRMMKERNEL)

  1599. 	andi.		L,	T11,	63

  1600. #else

  1601. 	andi.		L,  K,	63

  1602. #endif	

  1603. 	ble		LSGEMM_L2x1_SAVE

  1604. 	MY_ALIGN

  1605. LSGEMM_L2x1_SUB2:

  1606.     andi.      T10,L, 32

  1607.     ble LSGEMM_L2x1_SUB2_16 

  1608.     KERNEL2x1_4  0,0, 0,0

  1609.     KERNEL2x1_4  0,0, 1,0

  1610.     KERNEL2x1_4  0,0, 2,0

  1611.     KERNEL2x1_4  0,0, 3,0

  1612.     KERNEL2x1_4  0,0, 4,0

  1613.     KERNEL2x1_4  0,0, 5,0

  1614.     KERNEL2x1_4  0,0, 6,0

  1615.     KERNEL2x1_4  0,0, 7,1 

  1616.     MY_ALIGN        

  1617. LSGEMM_L2x1_SUB2_16:

  1618.     andi.      T10,L, 16

  1619.     ble LSGEMM_L2x1_SUB2_8 

  1620.     KERNEL2x1_4  0,0, 0,0

  1621.     KERNEL2x1_4  0,0, 1,0

  1622.     KERNEL2x1_4  0,0, 2,0

  1623.     KERNEL2x1_4  0,0, 3,1

  1624.     MY_ALIGN 

  1625. LSGEMM_L2x1_SUB2_8:

  1626.     andi.      T10,L, 8

  1627.     ble LSGEMM_L2x1_SUB2_4  

  1628.     KERNEL2x1_4  0,0, 0,0

  1629.     KERNEL2x1_4  0,0, 1,1

  1630. 	MY_ALIGN	

  1631. LSGEMM_L2x1_SUB2_4:

  1632.     andi.      T10,L, 4

  1633.     ble LSGEMM_L2x1_SUB2_2 

  1634.     KERNEL2x1_4  0,0, 0,1

  1635.     MY_ALIGN

  1636. LSGEMM_L2x1_SUB2_2:

  1637.     andi.      T10,L, 2

  1638.     ble LSGEMM_L2x1_SUB2_1 

  1639.     KERNEL2x1_2  0,0, 0,1

  1640.     MY_ALIGN    

  1641. LSGEMM_L2x1_SUB2_1:

  1642.     andi.      T10,L, 1

  1643.     ble LSGEMM_L2x1_SAVE	

  1644.     KERNEL2x1

  1645. 

  1646. 	MY_ALIGN

  1647. LSGEMM_L2x1_SAVE:

  1648. 	SAVE2x1

  1649. #if defined(TRMMKERNEL)	

  1650. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,2

  1651. #endif	 

  1652.     MY_ALIGN

  1653. LSGEMM_L2x1_END:

  1654. 	slwi		T1,	K,	3

  1655. 	add		B,	B,	T1

  1656. #if defined(TRMMKERNEL) && !defined(LEFT)

  1657.     addi TEMP_REG, TEMP_REG, 2

  1658. #endif 

  1659. LSGEMM_L2_END:

  1660.    andi.       T1, N,  1

  1661.    ble     LSGEMM_END

  1662. LSGEMM_1_BEGIN:

  1663.   

  1664. 

  1665. 	mr		AO,	A

  1666. 	mr		CO,	C 

  1667. 	add		C,	C,	LDC

  1668.  

  1669. #if defined(TRMMKERNEL) && defined(LEFT)

  1670. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  1671. #endif 

  1672. 	srawi.		I,	M,	4

  1673. 	ble		LSGEMM_1x16_END

  1674. 

  1675. 	MY_ALIGN

  1676. LSGEMM_1x16_BEGIN:

  1677. 

  1678. #if defined(TRMMKERNEL)

  1679.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,1

  1680. #else

  1681. 	mr		BO,	B

  1682. #endif	

  1683. 

  1684. #if defined(TRMMKERNEL)

  1685.    REFRESH_TEMP_BK T11,K,TEMP_REG,16,1 

  1686.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1687. #else 

  1688.    srawi.		L,	K,	6 /**(K ) % 64x */

  1689. #endif 

  1690.  

  1691.     ZERO1x16

  1692. 	ble		LSGEMM_1x16_SUB0

  1693.     addi AO,AO,2048

  1694.   

  1695. 	mtctr		L

  1696. 

  1697. 	MY_ALIGN

  1698. 

  1699. LSGEMM_1x16_LOOP:

  1700. 

  1701.     KERNEL1x16_4  -2048,0, 0,0

  1702.     KERNEL1x16_4  -2048,0, 1,0

  1703.     KERNEL1x16_4  -2048,0, 2,0

  1704.     KERNEL1x16_4  -2048,0, 3,0

  1705.     KERNEL1x16_4  -2048,0, 4,0

  1706.     KERNEL1x16_4  -2048,0, 5,0        

  1707.     KERNEL1x16_4  -2048,0, 6,0

  1708.     KERNEL1x16_4  -2048,0, 7,0  

  1709.     KERNEL1x16_4  -2048,0, 8,0      

  1710.     KERNEL1x16_4  -2048,0, 9,0

  1711.     KERNEL1x16_4  -2048,0, 10,0

  1712.     KERNEL1x16_4  -2048,0, 11,0

  1713.     KERNEL1x16_4  -2048,0, 12,0

  1714.     KERNEL1x16_4  -2048,0, 13,0    

  1715.     KERNEL1x16_4  -2048,0, 14,0    

  1716.     KERNEL1x16_4  -2048,0, 15,1  	

  1717. 

  1718. 	bdnz		LSGEMM_1x16_LOOP

  1719.     MY_ALIGN

  1720.     addi AO,AO, -2048

  1721. 	MY_ALIGN

  1722. LSGEMM_1x16_SUB0: 

  1723. #if defined(TRMMKERNEL)

  1724. 	andi.		L,	T11,	63

  1725. #else

  1726. 	andi.		L,  K,	63

  1727. #endif	

  1728. 	ble		LSGEMM_1x16_SAVE

  1729. 	MY_ALIGN

  1730. LSGEMM_1x16_SUB2:

  1731.     andi.      T10,L, 32

  1732.     ble LSGEMM_1x16_SUB2_16 

  1733.     KERNEL1x16_4  0,0, 0,0

  1734.     KERNEL1x16_4  0,0, 1,0

  1735.     KERNEL1x16_4  0,0, 2,0

  1736.     KERNEL1x16_4  0,0, 3,0

  1737.     KERNEL1x16_4  0,0, 4,0

  1738.     KERNEL1x16_4  0,0, 5,0

  1739.     KERNEL1x16_4  0,0, 6,0

  1740.     KERNEL1x16_4  0,0, 7,1 

  1741.     MY_ALIGN        

  1742. LSGEMM_1x16_SUB2_16:

  1743.     andi.      T10,L, 16

  1744.     ble LSGEMM_1x16_SUB2_8 

  1745.     KERNEL1x16_4  0,0, 0,0

  1746.     KERNEL1x16_4  0,0, 1,0

  1747.     KERNEL1x16_4  0,0, 2,0

  1748.     KERNEL1x16_4  0,0, 3,1

  1749.     MY_ALIGN 

  1750. LSGEMM_1x16_SUB2_8:

  1751.     andi.      T10,L, 8

  1752.     ble LSGEMM_1x16_SUB2_4  

  1753.     KERNEL1x16_4  0,0, 0,0

  1754.     KERNEL1x16_4  0,0, 1,1

  1755. 	MY_ALIGN	

  1756. LSGEMM_1x16_SUB2_4:

  1757.     andi.      T10,L, 4

  1758.     ble LSGEMM_1x16_SUB2_2 

  1759.     KERNEL1x16_4  0,0, 0,1

  1760.     MY_ALIGN

  1761. LSGEMM_1x16_SUB2_2:

  1762.     andi.      T10,L, 2

  1763.     ble LSGEMM_1x16_SUB2_1 

  1764.     KERNEL1x16_2  0,0, 0,1

  1765.     MY_ALIGN    

  1766. LSGEMM_1x16_SUB2_1:

  1767.     andi.      T10,L, 1

  1768.     ble LSGEMM_1x16_SAVE	

  1769.     KERNEL1x16

  1770. 

  1771. 	MY_ALIGN

  1772. LSGEMM_1x16_SAVE:

  1773. 	SAVE1x16

  1774. #if defined(TRMMKERNEL)	

  1775. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,16,1

  1776. #endif	

  1777. 	addic.		I,	I,	-1

  1778. 	bgt+		LSGEMM_1x16_BEGIN

  1779.     MY_ALIGN

  1780. LSGEMM_1x16_END:

  1781. 	andi.		I,	M,	8

  1782. 	ble		LSGEMM_1x8_END

  1783. 

  1784. 	MY_ALIGN

  1785. LSGEMM_1x8_BEGIN:

  1786. 

  1787. #if defined(TRMMKERNEL)

  1788.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1

  1789. #else

  1790. 	mr		BO,	B

  1791. #endif	

  1792. 

  1793. #if defined(TRMMKERNEL)

  1794.    REFRESH_TEMP_BK T11,K,TEMP_REG,8,1 

  1795.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1796. #else 

  1797.    srawi.		L,	K,	6 /**(K ) % 64x */

  1798. #endif 

  1799.  

  1800.     ZERO1x8

  1801. 	ble		LSGEMM_1x8_SUB0

  1802.     addi AO,AO,2048

  1803.   

  1804. 	mtctr		L

  1805. 

  1806. 	MY_ALIGN

  1807. 

  1808. LSGEMM_1x8_LOOP:

  1809. 

  1810.     KERNEL1x8_4  -2048,0, 0,0

  1811.     KERNEL1x8_4  -2048,0, 1,0

  1812.     KERNEL1x8_4  -2048,0, 2,0

  1813.     KERNEL1x8_4  -2048,0, 3,0

  1814.     KERNEL1x8_4  -2048,0, 4,0

  1815.     KERNEL1x8_4  -2048,0, 5,0        

  1816.     KERNEL1x8_4  -2048,0, 6,0

  1817.     KERNEL1x8_4  -2048,0, 7,0  

  1818.     KERNEL1x8_4  -2048,0, 8,0      

  1819.     KERNEL1x8_4  -2048,0, 9,0

  1820.     KERNEL1x8_4  -2048,0, 10,0

  1821.     KERNEL1x8_4  -2048,0, 11,0

  1822.     KERNEL1x8_4  -2048,0, 12,0

  1823.     KERNEL1x8_4  -2048,0, 13,0    

  1824.     KERNEL1x8_4  -2048,0, 14,0    

  1825.     KERNEL1x8_4  -2048,0, 15,1  	

  1826. 

  1827. 	bdnz		LSGEMM_1x8_LOOP

  1828.     MY_ALIGN

  1829.     addi AO,AO, -2048

  1830. 	MY_ALIGN

  1831. LSGEMM_1x8_SUB0: 

  1832. #if defined(TRMMKERNEL)

  1833. 	andi.		L,	T11,	63

  1834. #else

  1835. 	andi.		L,  K,	63

  1836. #endif	

  1837. 	ble		LSGEMM_1x8_SAVE

  1838. 	MY_ALIGN

  1839. LSGEMM_1x8_SUB2:

  1840.     andi.      T10,L, 32

  1841.     ble LSGEMM_1x8_SUB2_16 

  1842.     KERNEL1x8_4  0,0, 0,0

  1843.     KERNEL1x8_4  0,0, 1,0

  1844.     KERNEL1x8_4  0,0, 2,0

  1845.     KERNEL1x8_4  0,0, 3,0

  1846.     KERNEL1x8_4  0,0, 4,0

  1847.     KERNEL1x8_4  0,0, 5,0

  1848.     KERNEL1x8_4  0,0, 6,0

  1849.     KERNEL1x8_4  0,0, 7,1 

  1850.     MY_ALIGN        

  1851. LSGEMM_1x8_SUB2_16:

  1852.     andi.      T10,L, 16

  1853.     ble LSGEMM_1x8_SUB2_8 

  1854.     KERNEL1x8_4  0,0, 0,0

  1855.     KERNEL1x8_4  0,0, 1,0

  1856.     KERNEL1x8_4  0,0, 2,0

  1857.     KERNEL1x8_4  0,0, 3,1

  1858.     MY_ALIGN 

  1859. LSGEMM_1x8_SUB2_8:

  1860.     andi.      T10,L, 8

  1861.     ble LSGEMM_1x8_SUB2_4  

  1862.     KERNEL1x8_4  0,0, 0,0

  1863.     KERNEL1x8_4  0,0, 1,1

  1864. 	MY_ALIGN	

  1865. LSGEMM_1x8_SUB2_4:

  1866.     andi.      T10,L, 4

  1867.     ble LSGEMM_1x8_SUB2_2 

  1868.     KERNEL1x8_4  0,0, 0,1

  1869.     MY_ALIGN

  1870. LSGEMM_1x8_SUB2_2:

  1871.     andi.      T10,L, 2

  1872.     ble LSGEMM_1x8_SUB2_1 

  1873.     KERNEL1x8_2  0,0, 0,1

  1874.     MY_ALIGN    

  1875. LSGEMM_1x8_SUB2_1:

  1876.     andi.      T10,L, 1

  1877.     ble LSGEMM_1x8_SAVE	

  1878.     KERNEL1x8

  1879. 

  1880. 	MY_ALIGN

  1881. LSGEMM_1x8_SAVE:

  1882. 	SAVE1x8

  1883. #if defined(TRMMKERNEL)	

  1884. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,8,1

  1885. #endif	 

  1886.     MY_ALIGN

  1887. LSGEMM_1x8_END:

  1888. 	andi.		I,	M,	4

  1889. 	ble		LSGEMM_1x4_END

  1890. 

  1891. 	MY_ALIGN

  1892. LSGEMM_1x4_BEGIN:

  1893. 

  1894. #if defined(TRMMKERNEL)

  1895.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1

  1896. #else

  1897. 	mr		BO,	B

  1898. #endif	

  1899. 

  1900. #if defined(TRMMKERNEL)

  1901.    REFRESH_TEMP_BK T11,K,TEMP_REG,4,1 

  1902.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  1903. #else 

  1904.    srawi.		L,	K,	6 /**(K ) % 64x */

  1905. #endif 

  1906.  

  1907.     ZERO1x4

  1908. 	ble		LSGEMM_1x4_SUB0

  1909.  

  1910.   

  1911. 	mtctr		L

  1912. 

  1913. 	MY_ALIGN

  1914. 

  1915. LSGEMM_1x4_LOOP:

  1916. 

  1917.     KERNEL1x4_4  0,0, 0,0

  1918.     KERNEL1x4_4  0,0, 1,0

  1919.     KERNEL1x4_4  0,0, 2,0

  1920.     KERNEL1x4_4  0,0, 3,0

  1921.     KERNEL1x4_4  0,0, 4,0

  1922.     KERNEL1x4_4  0,0, 5,0        

  1923.     KERNEL1x4_4  0,0, 6,0

  1924.     KERNEL1x4_4  0,0, 7,0  

  1925.     KERNEL1x4_4  0,0, 8,0      

  1926.     KERNEL1x4_4  0,0, 9,0

  1927.     KERNEL1x4_4  0,0, 10,0

  1928.     KERNEL1x4_4  0,0, 11,0

  1929.     KERNEL1x4_4  0,0, 12,0

  1930.     KERNEL1x4_4  0,0, 13,0    

  1931.     KERNEL1x4_4  0,0, 14,0    

  1932.     KERNEL1x4_4  0,0, 15,1  	

  1933. 

  1934. 	bdnz		LSGEMM_1x4_LOOP

  1935.     MY_ALIGN

  1936.  

  1937. 	MY_ALIGN

  1938. LSGEMM_1x4_SUB0: 

  1939. #if defined(TRMMKERNEL)

  1940. 	andi.		L,	T11,	63

  1941. #else

  1942. 	andi.		L,  K,	63

  1943. #endif	

  1944. 	ble		LSGEMM_1x4_SAVE

  1945. 	MY_ALIGN

  1946. LSGEMM_1x4_SUB2:

  1947.     andi.      T10,L, 32

  1948.     ble LSGEMM_1x4_SUB2_16 

  1949.     KERNEL1x4_4  0,0, 0,0

  1950.     KERNEL1x4_4  0,0, 1,0

  1951.     KERNEL1x4_4  0,0, 2,0

  1952.     KERNEL1x4_4  0,0, 3,0

  1953.     KERNEL1x4_4  0,0, 4,0

  1954.     KERNEL1x4_4  0,0, 5,0

  1955.     KERNEL1x4_4  0,0, 6,0

  1956.     KERNEL1x4_4  0,0, 7,1 

  1957.     MY_ALIGN        

  1958. LSGEMM_1x4_SUB2_16:

  1959.     andi.      T10,L, 16

  1960.     ble LSGEMM_1x4_SUB2_8 

  1961.     KERNEL1x4_4  0,0, 0,0

  1962.     KERNEL1x4_4  0,0, 1,0

  1963.     KERNEL1x4_4  0,0, 2,0

  1964.     KERNEL1x4_4  0,0, 3,1

  1965.     MY_ALIGN 

  1966. LSGEMM_1x4_SUB2_8:

  1967.     andi.      T10,L, 8

  1968.     ble LSGEMM_1x4_SUB2_4  

  1969.     KERNEL1x4_4  0,0, 0,0

  1970.     KERNEL1x4_4  0,0, 1,1

  1971. 	MY_ALIGN	

  1972. LSGEMM_1x4_SUB2_4:

  1973.     andi.      T10,L, 4

  1974.     ble LSGEMM_1x4_SUB2_2 

  1975.     KERNEL1x4_4  0,0, 0,1

  1976.     MY_ALIGN

  1977. LSGEMM_1x4_SUB2_2:

  1978.     andi.      T10,L, 2

  1979.     ble LSGEMM_1x4_SUB2_1 

  1980.     KERNEL1x4_2  0,0, 0,1

  1981.     MY_ALIGN    

  1982. LSGEMM_1x4_SUB2_1:

  1983.     andi.      T10,L, 1

  1984.     ble LSGEMM_1x4_SAVE	

  1985.     KERNEL1x4

  1986. 

  1987. 	MY_ALIGN

  1988. LSGEMM_1x4_SAVE:

  1989. 	SAVE1x4

  1990. #if defined(TRMMKERNEL)	

  1991. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,4,1

  1992. #endif	 

  1993.     MY_ALIGN

  1994. LSGEMM_1x4_END:

  1995. 	andi.		I,	M,	2

  1996. 	ble		LSGEMM_1x2_END

  1997. 

  1998. 	MY_ALIGN

  1999. LSGEMM_1x2_BEGIN:

  2000. 

  2001. #if defined(TRMMKERNEL)

  2002.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1

  2003. #else

  2004. 	mr		BO,	B

  2005. #endif	

  2006. 

  2007. #if defined(TRMMKERNEL)

  2008.    REFRESH_TEMP_BK T11,K,TEMP_REG,2,1 

  2009.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  2010. #else 

  2011.    srawi.		L,	K,	6 /**(K ) % 64x */

  2012. #endif 

  2013.  

  2014.     ZERO1x2

  2015. 	ble		LSGEMM_1x2_SUB0

  2016.  

  2017.   

  2018. 	mtctr		L

  2019. 

  2020. 	MY_ALIGN

  2021. 

  2022. LSGEMM_1x2_LOOP:

  2023. 

  2024.     KERNEL1x2_4  0,0, 0,0

  2025.     KERNEL1x2_4  0,0, 1,0

  2026.     KERNEL1x2_4  0,0, 2,0

  2027.     KERNEL1x2_4  0,0, 3,0

  2028.     KERNEL1x2_4  0,0, 4,0

  2029.     KERNEL1x2_4  0,0, 5,0        

  2030.     KERNEL1x2_4  0,0, 6,0

  2031.     KERNEL1x2_4  0,0, 7,0  

  2032.     KERNEL1x2_4  0,0, 8,0      

  2033.     KERNEL1x2_4  0,0, 9,0

  2034.     KERNEL1x2_4  0,0, 10,0

  2035.     KERNEL1x2_4  0,0, 11,0

  2036.     KERNEL1x2_4  0,0, 12,0

  2037.     KERNEL1x2_4  0,0, 13,0    

  2038.     KERNEL1x2_4  0,0, 14,0    

  2039.     KERNEL1x2_4  0,0, 15,1  	

  2040. 

  2041. 	bdnz		LSGEMM_1x2_LOOP

  2042.     MY_ALIGN

  2043.  

  2044. 	MY_ALIGN

  2045. LSGEMM_1x2_SUB0: 

  2046. #if defined(TRMMKERNEL)

  2047. 	andi.		L,	T11,	63

  2048. #else

  2049. 	andi.		L,  K,	63

  2050. #endif	

  2051. 	ble		LSGEMM_1x2_SAVE

  2052. 	MY_ALIGN

  2053. LSGEMM_1x2_SUB2:

  2054.     andi.      T10,L, 32

  2055.     ble LSGEMM_1x2_SUB2_16 

  2056.     KERNEL1x2_4  0,0, 0,0

  2057.     KERNEL1x2_4  0,0, 1,0

  2058.     KERNEL1x2_4  0,0, 2,0

  2059.     KERNEL1x2_4  0,0, 3,0

  2060.     KERNEL1x2_4  0,0, 4,0

  2061.     KERNEL1x2_4  0,0, 5,0

  2062.     KERNEL1x2_4  0,0, 6,0

  2063.     KERNEL1x2_4  0,0, 7,1 

  2064.     MY_ALIGN        

  2065. LSGEMM_1x2_SUB2_16:

  2066.     andi.      T10,L, 16

  2067.     ble LSGEMM_1x2_SUB2_8 

  2068.     KERNEL1x2_4  0,0, 0,0

  2069.     KERNEL1x2_4  0,0, 1,0

  2070.     KERNEL1x2_4  0,0, 2,0

  2071.     KERNEL1x2_4  0,0, 3,1

  2072.     MY_ALIGN 

  2073. LSGEMM_1x2_SUB2_8:

  2074.     andi.      T10,L, 8

  2075.     ble LSGEMM_1x2_SUB2_4  

  2076.     KERNEL1x2_4  0,0, 0,0

  2077.     KERNEL1x2_4  0,0, 1,1

  2078. 	MY_ALIGN	

  2079. LSGEMM_1x2_SUB2_4:

  2080.     andi.      T10,L, 4

  2081.     ble LSGEMM_1x2_SUB2_2 

  2082.     KERNEL1x2_4  0,0, 0,1

  2083.     MY_ALIGN

  2084. LSGEMM_1x2_SUB2_2:

  2085.     andi.      T10,L, 2

  2086.     ble LSGEMM_1x2_SUB2_1 

  2087.     KERNEL1x2_2  0,0, 0,1

  2088.     MY_ALIGN    

  2089. LSGEMM_1x2_SUB2_1:

  2090.     andi.      T10,L, 1

  2091.     ble LSGEMM_1x2_SAVE	

  2092.     KERNEL1x2

  2093. 

  2094. 	MY_ALIGN

  2095. LSGEMM_1x2_SAVE:

  2096. 	SAVE1x2

  2097. #if defined(TRMMKERNEL)	

  2098. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,2,1

  2099. #endif	 

  2100.     MY_ALIGN

  2101. LSGEMM_1x2_END:

  2102.     andi.		I,	M,	1

  2103. 	ble		LSGEMM_1x1_END

  2104. 

  2105. 	MY_ALIGN

  2106. LSGEMM_1x1_BEGIN:

  2107. 

  2108. #if defined(TRMMKERNEL)

  2109.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1

  2110. #else

  2111. 	mr		BO,	B

  2112. #endif	

  2113. 

  2114. #if defined(TRMMKERNEL)

  2115.    REFRESH_TEMP_BK T11,K,TEMP_REG,1,1 

  2116.    srawi.		L, T11,	6 /**(T11 ) % 64x */

  2117. #else 

  2118.    srawi.		L,	K,	6 /**(K ) % 64x */

  2119. #endif 

  2120.  

  2121.     ZERO1x1

  2122. 	ble		LSGEMM_1x1_SUB0

  2123.  

  2124.   

  2125. 	mtctr		L

  2126. 

  2127. 	MY_ALIGN

  2128. 

  2129. LSGEMM_1x1_LOOP:

  2130. 

  2131.     KERNEL1x1_16  0,0, 0,0

  2132.     KERNEL1x1_16  0,0, 1,0

  2133.     KERNEL1x1_16  0,0, 2,0

  2134.     KERNEL1x1_16  0,0, 3,1 	

  2135. 

  2136. 	bdnz		LSGEMM_1x1_LOOP

  2137.     MY_ALIGN

  2138.  

  2139. 	MY_ALIGN

  2140. LSGEMM_1x1_SUB0: 

  2141. #if defined(TRMMKERNEL)

  2142. 	andi.		L,	T11,	63

  2143. #else

  2144. 	andi.		L,  K,	63

  2145. #endif	

  2146. 	ble		LSGEMM_1x1_SAVE

  2147. 	MY_ALIGN

  2148. LSGEMM_1x1_SUB2:

  2149.     andi.      T10,L, 32

  2150.     ble LSGEMM_1x1_SUB2_16 

  2151.     KERNEL1x1_16  0,0, 0,0

  2152.     KERNEL1x1_16  0,0, 1,1 

  2153.     MY_ALIGN        

  2154. LSGEMM_1x1_SUB2_16:

  2155.     andi.      T10,L, 16

  2156.     ble LSGEMM_1x1_SUB2_8 

  2157.     KERNEL1x1_16  0,0, 0,1

  2158.     MY_ALIGN 

  2159. LSGEMM_1x1_SUB2_8:

  2160.     andi.      T10,L, 8

  2161.     ble LSGEMM_1x1_SUB2_4  

  2162.     KERNEL1x1_8  0,0, 0,1

  2163. 	MY_ALIGN	

  2164. LSGEMM_1x1_SUB2_4:

  2165.     andi.      T10,L, 4

  2166.     ble LSGEMM_1x1_SUB2_2 

  2167.     KERNEL1x1_4  0,0, 0,1

  2168.     MY_ALIGN

  2169. LSGEMM_1x1_SUB2_2:

  2170.     andi.      T10,L, 2

  2171.     ble LSGEMM_1x1_SUB2_1 

  2172.     KERNEL1x1_2  0,0, 0,1

  2173.     MY_ALIGN    

  2174. LSGEMM_1x1_SUB2_1:

  2175.     andi.      T10,L, 1

  2176.     ble LSGEMM_1x1_SAVE	

  2177.     KERNEL1x1

  2178. 

  2179. 	MY_ALIGN

  2180. LSGEMM_1x1_SAVE:

  2181. 	SAVE1x1

  2182. #if defined(TRMMKERNEL)	

  2183. 	REFRESH_AFTER_SAVE T11,K,TEMP_REG,BO,AO,1,1

  2184. #endif	 

  2185.     MY_ALIGN

  2186. LSGEMM_1x1_END:

  2187. 	slwi		T1,	K,	2

  2188. 	add		B,	B,	T1

  2189. #if defined(TRMMKERNEL) && !defined(LEFT)

  2190.     addi TEMP_REG, TEMP_REG, 1

  2191. #endif 

  2192. LSGEMM_END: