OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/dgemm_logic_power9.S

1982 lines
29 kB
Raw Permalink Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2019 The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27.  

  28. 

  29. #define MY_ALIGN .align 3

  30. 

  31. #if defined(TRMMKERNEL) && !defined(LEFT)

  32.    neg TEMP_REG, OFFSET 

  33. #endif

  34. 

  35. 	srawi.		J,	N,	2

  36. 	ble		LDGEMM_L4_END

  37. 

  38. LDGEMM_L4_BEGIN:

  39. 

  40.  

  41. 	li		T1,	128

  42. 	li		T2,	256

  43.  

  44. 	mr		AO,	A

  45. 	mr		CO,	C

  46. 	slwi		T3,	LDC	,	2

  47. 	add		C,	C,	T3

  48. 

  49.  

  50. 	dcbt		A,	T1

  51. 	dcbt		A,	T2

  52. #if defined(TRMMKERNEL) && defined(LEFT)

  53. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  54. #endif 

  55. 	srawi.		I,	M,	4

  56. 	ble		LDGEMM_L4x16_END

  57. 

  58. 	MY_ALIGN

  59. LDGEMM_L4x16_BEGIN:

  60. 

  61. 	li		L,	-128

  62. 

  63. 

  64. 	SAVE4x16_REGS

  65. 

  66. #if defined(TRMMKERNEL)

  67.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,4

  68. #else

  69. 	mr		BO,	B

  70. #endif	

  71.  

  72. 	and		T1,	CO,	L

  73. 	and		T2,	C2,	L

  74. 	and		T3,	C3,	L

  75. 	and		T4,	C4,	L

  76. 

  77. 	dcbt		T1,	r0

  78. 	dcbt		T2,	r0

  79. 	dcbt		T3,	r0

  80. 	dcbt		T4,	r0

  81.  

  82. 

  83. 	addi		T1, T1, 128

  84. 	addi		T2, T2, 128

  85. 	addi		T3, T3, 128

  86. 	addi		T4, T4, 128

  87. 

  88. 	dcbt		T1,	r0

  89. 	dcbt		T2,	r0

  90. 	dcbt		T3,	r0

  91. 	dcbt		T4,	r0

  92. 

  93. #if defined(TRMMKERNEL)

  94.    REFRESH_TEMP_BK T3,K,TEMP_REG,16,4

  95.    srawi.		L, T3,	5

  96. #else

  97.    srawi.		L,	K,	5

  98. #endif 

  99. 	

  100. 	ble		LDGEMM_L4x16_SUB0

  101. 

  102. 

  103. 	MY_ALIGN

  104. LDGEMM_L4x16_LOOP_START:

  105. 

  106. 	li	T2,	512

  107.  

  108.  

  109. 	LOAD4x16_1

  110.     ##OffsetA=128 OffsetB=32

  111.     addi AO,AO,2176

  112.  #   addi BO,BO,32 

  113. 	addic.		L,	L,	-1

  114. 

  115. 	ble		LDGEMM_L4x16_LOOP_END

  116. 

  117. 	

  118. 	mtctr		L

  119. 

  120. 	MY_ALIGN

  121. 

  122. LDGEMM_L4x16_LOOP:

  123. 

  124. 	#dcbt	AO,	PRE

  125.     KERNEL4x16_I1_L2_2  -2048,32, 0,0

  126.     KERNEL4x16_I1_L2_2  -2048,32, 1,0

  127.     KERNEL4x16_I1_L2_2  -2048,32, 2,0

  128.     KERNEL4x16_I1_L2_2  -2048,32, 3,0

  129.     KERNEL4x16_I1_L2_2  -2048,32, 4,0

  130.     KERNEL4x16_I1_L2_2  -2048,32, 5,0        

  131.     KERNEL4x16_I1_L2_2  -2048,32, 6,0

  132.     KERNEL4x16_I1_L2_2  -2048,32, 7,0  

  133.     KERNEL4x16_I1_L2_2  -2048,32, 8,0      

  134.     KERNEL4x16_I1_L2_2  -2048,32, 9,0

  135.     KERNEL4x16_I1_L2_2  -2048,32, 10,0

  136.     KERNEL4x16_I1_L2_2  -2048,32, 11,0

  137.     KERNEL4x16_I1_L2_2  -2048,32, 12,0

  138.     KERNEL4x16_I1_L2_2  -2048,32, 13,0    

  139.     KERNEL4x16_I1_L2_2  -2048,32, 14,0    

  140.     KERNEL4x16_I1_L2_2  -2048,32, 15,1  	

  141. 

  142. 

  143. 	bdnz		LDGEMM_L4x16_LOOP

  144. 

  145. 	MY_ALIGN

  146. 	MY_ALIGN

  147. LDGEMM_L4x16_LOOP_END:

  148. 

  149.     KERNEL4x16_I1_L2_2  -2048,32, 0,0

  150.     KERNEL4x16_I1_L2_2  -2048,32, 1,0

  151.     KERNEL4x16_I1_L2_2  -2048,32, 2,0

  152.     KERNEL4x16_I1_L2_2  -2048,32, 3,0

  153.     KERNEL4x16_I1_L2_2  -2048,32, 4,0

  154.     KERNEL4x16_I1_L2_2  -2048,32, 5,0        

  155.     KERNEL4x16_I1_L2_2  -2048,32, 6,0

  156.     KERNEL4x16_I1_L2_2  -2048,32, 7,0  

  157.     KERNEL4x16_I1_L2_2  -2048,32, 8,0      

  158.     KERNEL4x16_I1_L2_2  -2048,32, 9,0

  159.     KERNEL4x16_I1_L2_2  -2048,32, 10,0

  160.     KERNEL4x16_I1_L2_2  -2048,32, 11,0

  161.     KERNEL4x16_I1_L2_2  -2048,32, 12,0

  162.     KERNEL4x16_I1_L2_2  -2048,32, 13,0    

  163.     KERNEL4x16_I1_L2_2  -2048,32, 14,0    

  164.     KERNEL4x16_I1_L2_3  -2048,32, 15,1    

  165. 	b		LDGEMM_L4x16_SUB1

  166. 

  167. 

  168. 	MY_ALIGN

  169. LDGEMM_L4x16_SUB0:

  170. #if defined(TRMMKERNEL)

  171. 	andi.		L,	T3,	31

  172. #else

  173. 	andi.		L,	K,	31

  174. #endif

  175. 	KERNEL4x16 1

  176. 

  177. 	addic.		L,	L,	-1

  178. 	ble		LDGEMM_L4x16_SAVE

  179. 	b		LDGEMM_L4x16_SUB2

  180. 	MY_ALIGN

  181. LDGEMM_L4x16_SUB1:

  182. #if defined(TRMMKERNEL)

  183. 	andi.		L,	T3,	31

  184. #else

  185. 	andi.		L,	K,	31

  186. #endif	

  187. 	ble		LDGEMM_L4x16_SAVE

  188. 	MY_ALIGN

  189. LDGEMM_L4x16_SUB2:

  190. 

  191.     andi.      T1,L, 16

  192.     ble LDGEMM_L4x16_SUB2_8

  193. 	LOAD4x16_0 

  194.     KERNEL4x16_I1_L2_2  128,32, 0,0

  195.     KERNEL4x16_I1_L2_2  128,32, 1,0

  196.     KERNEL4x16_I1_L2_2  128,32, 2,0

  197.     KERNEL4x16_I1_L2_2  128,32, 3,0

  198.     KERNEL4x16_I1_L2_2  128,32, 4,0

  199.     KERNEL4x16_I1_L2_2  128,32, 5,0        

  200.     KERNEL4x16_I1_L2_2  128,32, 6,0

  201.     KERNEL4x16_I1_L2_3  128,32, 7,1 

  202.     MY_ALIGN

  203. LDGEMM_L4x16_SUB2_8:

  204.     andi.      T1,L, 8

  205.     ble LDGEMM_L4x16_SUB2_4

  206. 	LOAD4x16_0

  207.     KERNEL4x16_I1_L2_2  128,32, 0,0

  208.     KERNEL4x16_I1_L2_2  128,32, 1,0

  209.     KERNEL4x16_I1_L2_2  128,32, 2,0

  210.     KERNEL4x16_I1_L2_3  128,32, 3,1

  211. 	MY_ALIGN

  212. LDGEMM_L4x16_SUB2_4:

  213.     andi.      T1,L, 4

  214.     ble LDGEMM_L4x16_SUB2_2 

  215. 	LOAD4x16_0

  216.     KERNEL4x16_I1_L2_2  128,32, 0,0

  217.     KERNEL4x16_I1_L2_3  128,32, 1,1

  218. 	MY_ALIGN	

  219. LDGEMM_L4x16_SUB2_2:

  220.     andi.      T1,L, 2

  221.     ble LDGEMM_L4x16_SUB2_1

  222.     LOAD4x16_0

  223.     KERNEL4x16_I1_L2_3  128,32, 0,1

  224.     MY_ALIGN

  225. LDGEMM_L4x16_SUB2_1:

  226.     andi.      T1,L, 1

  227.     ble LDGEMM_L4x16_SAVE	

  228.     KERNEL4x16 0

  229. #	addic.		L,	L,	-1

  230. #	bgt		LDGEMM_L4x16_SUB2

  231. 

  232. 	MY_ALIGN

  233. LDGEMM_L4x16_SAVE:

  234. 	SAVE4x16

  235. #if defined(TRMMKERNEL)	

  236. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,4

  237. #endif	

  238. 	addic.		I,	I,	-1

  239. 	bgt+		LDGEMM_L4x16_BEGIN

  240. 

  241. LDGEMM_L4x16_END:

  242. 

  243. LDGEMM_L4x8_BEGIN:

  244. 

  245. 	andi.		T2,	M,	15

  246. 	ble		LDGEMM_L4x1_END

  247. 

  248. 	andi.		T1,	M,	8

  249. 	ble		LDGEMM_L4x8_END

  250. 

  251. 

  252. #if defined(TRMMKERNEL)

  253.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,4

  254.     REFRESH_TEMP_BK T3,K,TEMP_REG,8,4

  255.     srawi.		L, T3,	4	

  256. #else

  257. 	mr		BO,	B

  258. 	srawi.		L,	K,	4	

  259. #endif		 

  260.  

  261. 

  262. 	ble		LDGEMM_L4x8_SUB0

  263. 

  264. LDGEMM_L4x8_LOOP_START:

  265. 

  266. 

  267. 	LOAD4x8_1

  268.     ##OffsetA=64 OffsetB=32

  269. 

  270. 

  271. 	addic.		L,	L,	-1

  272. 

  273. 	ble		LDGEMM_L4x8_LOOP_END

  274. 

  275.     mtctr		L

  276. 	MY_ALIGN

  277. 

  278. LDGEMM_L4x8_LOOP:

  279. 

  280.     KERNEL4x8_I1_L2_2  64,32, 0,0

  281.     KERNEL4x8_I1_L2_2  64,32, 1,0

  282.     KERNEL4x8_I1_L2_2  64,32, 2,0

  283.     KERNEL4x8_I1_L2_2  64,32, 3,0

  284.     KERNEL4x8_I1_L2_2  64,32, 4,0

  285.     KERNEL4x8_I1_L2_2  64,32, 5,0        

  286.     KERNEL4x8_I1_L2_2  64,32, 6,0

  287.     KERNEL4x8_I1_L2_2  64,32, 7,1     

  288. 

  289. 	bdnz		LDGEMM_L4x8_LOOP

  290. 	MY_ALIGN

  291. LDGEMM_L4x8_LOOP_END:

  292. 

  293.     KERNEL4x8_I1_L2_2  64,32, 0,0

  294.     KERNEL4x8_I1_L2_2  64,32, 1,0

  295.     KERNEL4x8_I1_L2_2  64,32, 2,0

  296.     KERNEL4x8_I1_L2_2  64,32, 3,0

  297.     KERNEL4x8_I1_L2_2  64,32, 4,0

  298.     KERNEL4x8_I1_L2_2  64,32, 5,0        

  299.     KERNEL4x8_I1_L2_2  64,32, 6,0

  300.     KERNEL4x8_I1_L2_3  64,32, 7,1  

  301. 

  302. 	b		LDGEMM_L4x8_SUB1

  303. 	MY_ALIGN

  304. LDGEMM_L4x8_SUB0:

  305. #if defined(TRMMKERNEL)

  306. 	andi.		L,	T3,	15

  307. #else

  308. 	andi.		L,	K,	15

  309. #endif

  310. 	KERNEL4x8 1

  311. 

  312. 	addic.		L,	L,	-1

  313. 	ble		LDGEMM_L4x8_SAVE

  314. 	b		LDGEMM_L4x8_SUB2

  315. 	MY_ALIGN

  316. LDGEMM_L4x8_SUB1:

  317. #if defined(TRMMKERNEL)

  318. 	andi.		L,	T3,	15

  319. #else

  320. 	andi.		L,	K,	15

  321. #endif	

  322. 	ble		LDGEMM_L4x8_SAVE

  323. 	MY_ALIGN

  324. LDGEMM_L4x8_SUB2:

  325. 

  326.     andi.      T1,L, 8

  327.     ble LDGEMM_L4x8_SUB2_4

  328. 	LOAD4x8_0

  329.     KERNEL4x8_I1_L2_2  64,32, 0,0

  330.     KERNEL4x8_I1_L2_2  64,32, 1,0

  331.     KERNEL4x8_I1_L2_2  64,32, 2,0

  332.     KERNEL4x8_I1_L2_3  64,32, 3,1

  333. 	MY_ALIGN

  334. LDGEMM_L4x8_SUB2_4:

  335.     andi.      T1,L, 4

  336.     ble LDGEMM_L4x8_SUB2_2 

  337. 	LOAD4x8_0

  338.     KERNEL4x8_I1_L2_2  64,32, 0,0

  339.     KERNEL4x8_I1_L2_3  64,32, 1,1

  340. 	MY_ALIGN	

  341. LDGEMM_L4x8_SUB2_2:

  342.     andi.      T1,L, 2

  343.     ble LDGEMM_L4x8_SUB2_1

  344.     LOAD4x8_0

  345.     KERNEL4x8_I1_L2_3  64,32, 0,1

  346.     MY_ALIGN

  347. LDGEMM_L4x8_SUB2_1:

  348.     andi.      T1,L, 1

  349.     ble LDGEMM_L4x8_SAVE	

  350.     KERNEL4x8 0

  351.  

  352. 	MY_ALIGN

  353. LDGEMM_L4x8_SAVE:

  354. 	SAVE4x8

  355. #if defined(TRMMKERNEL)	

  356. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,4

  357. #endif	

  358. LDGEMM_L4x8_END:

  359. 

  360. LDGEMM_L4x4_BEGIN:

  361. 

  362. 

  363. 	andi.		T1,	M,	4

  364. 	ble		LDGEMM_L4x4_END

  365. 

  366. #if defined(TRMMKERNEL)

  367.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,4

  368.     REFRESH_TEMP_BK T3,K,TEMP_REG,4,4

  369.     srawi.		L, T3,	3	

  370. #else

  371. 	mr		BO,	B

  372. 	srawi.		L,	K,	3	

  373. #endif		

  374. 	ble		LDGEMM_L4x4_SUB0

  375. 	cmpwi		cr0,	L,	1

  376. 	ble		LDGEMM_L4x4_SUB4

  377. 

  378. LDGEMM_L4x4_LOOP_START:

  379. 

  380. 	#dcbt	AO,	PRE

  381. 	LOAD4x4_1

  382. 	KERNEL4x4_I1

  383. 	KERNEL4x4_2

  384. 	KERNEL4x4_1

  385. 	#dcbt	AO,	PRE

  386. 	KERNEL4x4_2

  387. 

  388. 	KERNEL4x4_1

  389. 	KERNEL4x4_2

  390. 	KERNEL4x4_1

  391. 	#dcbt	AO,	PRE

  392. 	KERNEL4x4_2

  393. 

  394. 	addic.		L,	L,	-2

  395. 	ble		LDGEMM_L4x4_LOOP_END

  396. 

  397. 	MY_ALIGN

  398. 

  399. LDGEMM_L4x4_LOOP:

  400. 

  401. 	KERNEL4x4_1

  402. 	KERNEL4x4_2

  403. 	KERNEL4x4_1

  404. 	#dcbt	AO,	PRE

  405. 	KERNEL4x4_2

  406. 

  407. 	KERNEL4x4_1

  408. 	KERNEL4x4_2

  409. 	KERNEL4x4_1

  410. 	#dcbt	AO,	PRE

  411. 	KERNEL4x4_2

  412. 

  413. 	addic.		L,	L,	-1

  414. 	bgt		LDGEMM_L4x4_LOOP

  415. 

  416. LDGEMM_L4x4_LOOP_END:

  417. 

  418. 	KERNEL4x4_1

  419. 	KERNEL4x4_2

  420. 	KERNEL4x4_1

  421. 	KERNEL4x4_2

  422. 

  423. 	KERNEL4x4_1

  424. 	KERNEL4x4_2

  425. 	KERNEL4x4_1

  426. 	KERNEL4x4_E2

  427. 

  428. 	b		LDGEMM_L4x4_SUB1

  429. 

  430. LDGEMM_L4x4_SUB4:

  431. 

  432. 	KERNEL4x4_SUBI1

  433. 	KERNEL4x4_SUB1

  434. 	KERNEL4x4_SUB1

  435. 	KERNEL4x4_SUB1

  436. 

  437. 	KERNEL4x4_SUB1

  438. 	KERNEL4x4_SUB1

  439. 	KERNEL4x4_SUB1

  440. 	KERNEL4x4_SUB1

  441. 

  442. 	b		LDGEMM_L4x4_SUB1

  443. 

  444. LDGEMM_L4x4_SUB0:

  445. #if defined(TRMMKERNEL)

  446. 	andi.		L,	T3,	7

  447. #else

  448. 	andi.		L,	K,	7

  449. #endif

  450. 

  451. 	KERNEL4x4_SUBI1

  452. 

  453. 	addic.		L,	L,	-1

  454. 	ble		LDGEMM_L4x4_SAVE

  455. 	b		LDGEMM_L4x4_SUB2

  456. 

  457. LDGEMM_L4x4_SUB1:

  458. #if defined(TRMMKERNEL)

  459. 	andi.		L,	T3,	7

  460. #else

  461. 	andi.		L,	K,	7

  462. #endif

  463. 	ble		LDGEMM_L4x4_SAVE

  464. 

  465. LDGEMM_L4x4_SUB2:

  466. 

  467. 	KERNEL4x4_SUB1

  468. 

  469. 	addic.		L,	L,	-1

  470. 	bgt		LDGEMM_L4x4_SUB2

  471. 

  472. LDGEMM_L4x4_SAVE:

  473. 

  474. 	SAVE4x4

  475. #if defined(TRMMKERNEL)	

  476. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,4

  477. #endif	

  478. LDGEMM_L4x4_END:

  479. 

  480. LDGEMM_L4x2_BEGIN:

  481. 

  482. 

  483. 	andi.		T1,	M,	2

  484. 	ble		LDGEMM_L4x2_END

  485. #if defined(TRMMKERNEL)

  486.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,4

  487.     REFRESH_TEMP_BK T3,K,TEMP_REG,2,4

  488.     srawi.		L, T3,	3	

  489. #else

  490. 	mr		BO,	B

  491. 	srawi.		L,	K,	3	

  492. #endif	

  493. 	ble		LDGEMM_L4x2_SUB0

  494. 	cmpwi		cr0,	L,	1

  495. 	ble		LDGEMM_L4x2_SUB4

  496. 

  497. LDGEMM_L4x2_LOOP_START:

  498. 

  499. 	LOAD4x2_1

  500. 	KERNEL4x2_I1

  501. 	KERNEL4x2_2

  502. 	KERNEL4x2_1

  503. 	KERNEL4x2_2

  504. 

  505. 	KERNEL4x2_1

  506. 	KERNEL4x2_2

  507. 	KERNEL4x2_1

  508. 	KERNEL4x2_2

  509. 

  510. 	addic.		L,	L,	-2

  511. 	ble		LDGEMM_L4x2_LOOP_END

  512. 

  513. 	MY_ALIGN

  514. 

  515. LDGEMM_L4x2_LOOP:

  516. 

  517. 	KERNEL4x2_1

  518. 	KERNEL4x2_2

  519. 	KERNEL4x2_1

  520. 	KERNEL4x2_2

  521. 

  522. 	KERNEL4x2_1

  523. 	KERNEL4x2_2

  524. 	KERNEL4x2_1

  525. 	KERNEL4x2_2

  526. 

  527. 	addic.		L,	L,	-1

  528. 	bgt		LDGEMM_L4x2_LOOP

  529. 

  530. LDGEMM_L4x2_LOOP_END:

  531. 

  532. 	KERNEL4x2_1

  533. 	KERNEL4x2_2

  534. 	KERNEL4x2_1

  535. 	KERNEL4x2_2

  536. 

  537. 	KERNEL4x2_1

  538. 	KERNEL4x2_2

  539. 	KERNEL4x2_1

  540. 	KERNEL4x2_E2

  541. 

  542. 	b		LDGEMM_L4x2_SUB1

  543. 

  544. LDGEMM_L4x2_SUB4:

  545. 

  546. 	KERNEL4x2_SUBI1

  547. 	KERNEL4x2_SUB1

  548. 	KERNEL4x2_SUB1

  549. 	KERNEL4x2_SUB1

  550. 

  551. 	KERNEL4x2_SUB1

  552. 	KERNEL4x2_SUB1

  553. 	KERNEL4x2_SUB1

  554. 	KERNEL4x2_SUB1

  555. 

  556. 	b		LDGEMM_L4x2_SUB1

  557. 

  558. LDGEMM_L4x2_SUB0:

  559. #if defined(TRMMKERNEL)

  560. 	andi.		L,	T3,	7

  561. #else

  562. 	andi.		L,	K,	7

  563. #endif

  564. 

  565. 	KERNEL4x2_SUBI1

  566. 

  567. 	addic.		L,	L,	-1

  568. 	ble		LDGEMM_L4x2_SAVE

  569. 	b		LDGEMM_L4x2_SUB2

  570. 

  571. LDGEMM_L4x2_SUB1:

  572. #if defined(TRMMKERNEL)

  573. 	andi.		L,	T3,	7

  574. #else

  575. 	andi.		L,	K,	7

  576. #endif

  577. 	ble		LDGEMM_L4x2_SAVE

  578. 

  579. LDGEMM_L4x2_SUB2:

  580. 

  581. 	KERNEL4x2_SUB1

  582. 

  583. 	addic.		L,	L,	-1

  584. 	bgt		LDGEMM_L4x2_SUB2

  585. 

  586. LDGEMM_L4x2_SAVE:

  587. 

  588. 	SAVE4x2

  589. #if defined(TRMMKERNEL)	

  590. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,4

  591. #endif	

  592. LDGEMM_L4x2_END:

  593. 

  594. LDGEMM_L4x1_BEGIN:

  595. 

  596. 

  597. 	andi.		T1,	M,	1

  598. 	ble		LDGEMM_L4x1_END

  599. #if defined(TRMMKERNEL)

  600.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,4

  601.     REFRESH_TEMP_BK T3,K,TEMP_REG,1,4

  602.     srawi.		L, T3,	3	

  603. #else

  604. 	mr		BO,	B

  605. 	srawi.		L,	K,	3	

  606. #endif	

  607. 	ble		LDGEMM_L4x1_SUB0

  608. 	cmpwi		cr0,	L,	1

  609. 	ble		LDGEMM_L4x1_SUB4

  610. 

  611. LDGEMM_L4x1_LOOP_START:

  612. 

  613. 	LOAD4x1_1

  614. 	KERNEL4x1_I1

  615. 	KERNEL4x1_2

  616. 	KERNEL4x1_1

  617. 	KERNEL4x1_2

  618. 

  619. 	KERNEL4x1_1

  620. 	KERNEL4x1_2

  621. 	KERNEL4x1_1

  622. 	KERNEL4x1_2

  623. 

  624. 	addic.		L,	L,	-2

  625. 	ble		LDGEMM_L4x1_LOOP_END

  626. 

  627. 	MY_ALIGN

  628. 

  629. LDGEMM_L4x1_LOOP:

  630. 

  631. 	KERNEL4x1_1

  632. 	KERNEL4x1_2

  633. 	KERNEL4x1_1

  634. 	KERNEL4x1_2

  635. 

  636. 	KERNEL4x1_1

  637. 	KERNEL4x1_2

  638. 	KERNEL4x1_1

  639. 	KERNEL4x1_2

  640. 

  641. 	addic.		L,	L,	-1

  642. 	bgt		LDGEMM_L4x1_LOOP

  643. 

  644. LDGEMM_L4x1_LOOP_END:

  645. 

  646. 	KERNEL4x1_1

  647. 	KERNEL4x1_2

  648. 	KERNEL4x1_1

  649. 	KERNEL4x1_2

  650. 

  651. 	KERNEL4x1_1

  652. 	KERNEL4x1_2

  653. 	KERNEL4x1_1

  654. 	KERNEL4x1_E2

  655. 

  656. 	b		LDGEMM_L4x1_SUB1

  657. 

  658. LDGEMM_L4x1_SUB4:

  659. 

  660. 	KERNEL4x1_SUBI1

  661. 	KERNEL4x1_SUB1

  662. 	KERNEL4x1_SUB1

  663. 	KERNEL4x1_SUB1

  664. 

  665. 	KERNEL4x1_SUB1

  666. 	KERNEL4x1_SUB1

  667. 	KERNEL4x1_SUB1

  668. 	KERNEL4x1_SUB1

  669. 

  670. 	b		LDGEMM_L4x1_SUB1

  671. 

  672. LDGEMM_L4x1_SUB0:

  673. #if defined(TRMMKERNEL)

  674. 	andi.		L,	T3,	7

  675. #else

  676. 	andi.		L,	K,	7

  677. #endif

  678. 

  679. 	KERNEL4x1_SUBI1

  680. 

  681. 	addic.		L,	L,	-1

  682. 	ble		LDGEMM_L4x1_SAVE

  683. 	b		LDGEMM_L4x1_SUB2

  684. 

  685. LDGEMM_L4x1_SUB1:

  686. #if defined(TRMMKERNEL)

  687. 	andi.		L,	T3,	7

  688. #else

  689. 	andi.		L,	K,	7

  690. #endif

  691. 	ble		LDGEMM_L4x1_SAVE

  692. 

  693. LDGEMM_L4x1_SUB2:

  694. 

  695. 	KERNEL4x1_SUB1

  696. 

  697. 	addic.		L,	L,	-1

  698. 	bgt		LDGEMM_L4x1_SUB2

  699. 

  700. LDGEMM_L4x1_SAVE:

  701. 

  702. 	SAVE4x1

  703. #if defined(TRMMKERNEL)	

  704. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,4

  705. #endif	

  706. LDGEMM_L4x1_END:

  707. 

  708. 	slwi		T1,	K,	5

  709. 	add		B,	B,	T1

  710. #if defined(TRMMKERNEL) && !defined(LEFT)

  711.     addi TEMP_REG, TEMP_REG, 4

  712. #endif

  713. 	addic.		J,	J,	-1

  714. 	bgt		LDGEMM_L4_BEGIN

  715. 

  716. 	andi.		T2,	N,	3

  717. 	ble		.L999

  718. 

  719. LDGEMM_L4_END:

  720. 

  721. 	b		LDGEMM_L2_BEGIN

  722. 

  723. .L999_H1:

  724. 

  725. 	b		.L999

  726. 

  727. LDGEMM_L2_BEGIN:

  728. 

  729. #if defined(TRMMKERNEL) && defined(LEFT)

  730. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  731. #endif 

  732. 	andi.		T1,	N,	2

  733. 	ble		LDGEMM_L2_END

  734. 	mr		CO,	C

  735. 	mr		AO,	A

  736. 	slwi		T1,	LDC	,	1

  737. 	add		C,	C,	T1

  738. 	srawi.		I,	M,	4

  739. 	ble		LDGEMM_L2x16_END

  740. 

  741. LDGEMM_L2x16_BEGIN:

  742. 

  743. 

  744. #if defined(TRMMKERNEL)

  745.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,2

  746.     REFRESH_TEMP_BK T3,K,TEMP_REG,16,2

  747.     srawi.		L, T3,	3	

  748. #else

  749. 	mr		BO,	B

  750. 	srawi.		L,	K,	3	

  751. #endif	

  752. 	ble		LDGEMM_L2x16_SUB0

  753. 	cmpwi		cr0,	L,	1

  754. 	ble		LDGEMM_L2x16_SUB4

  755. 

  756. LDGEMM_L2x16_LOOP_START:

  757. 

  758. 	#dcbt		AO,	PRE

  759. 	LOAD2x16_1

  760. 	#dcbt		AO,	PRE

  761. 	KERNEL2x16_I1

  762. 	#dcbt		AO,	PRE

  763. 	KERNEL2x16_2

  764. 	#dcbt		AO,	PRE

  765. 	KERNEL2x16_1

  766. 	#dcbt		AO,	PRE

  767. 	KERNEL2x16_2

  768. 

  769. 	#dcbt		AO,	PRE

  770. 	KERNEL2x16_1

  771. 	#dcbt		AO,	PRE

  772. 	KERNEL2x16_2

  773. 	#dcbt		AO,	PRE

  774. 	KERNEL2x16_1

  775. 	#dcbt		AO,	PRE

  776. 	KERNEL2x16_2

  777. 

  778. 	addic.		L,	L,	-2

  779. 	ble		LDGEMM_L2x16_LOOP_END

  780. 

  781. 	MY_ALIGN

  782. 

  783. LDGEMM_L2x16_LOOP:

  784. 

  785. 	#dcbt		AO,	PRE

  786. 	KERNEL2x16_1

  787. 	#dcbt		AO,	PRE

  788. 	KERNEL2x16_2

  789. 	#dcbt		AO,	PRE

  790. 	KERNEL2x16_1

  791. 	#dcbt		AO,	PRE

  792. 	KERNEL2x16_2

  793. 

  794. 	#dcbt		AO,	PRE

  795. 	KERNEL2x16_1

  796. 	#dcbt		AO,	PRE

  797. 	KERNEL2x16_2

  798. 	#dcbt		AO,	PRE

  799. 	KERNEL2x16_1

  800. 	#dcbt		AO,	PRE

  801. 	KERNEL2x16_2

  802. 

  803. 	addic.		L,	L,	-1

  804. 	bgt		LDGEMM_L2x16_LOOP

  805. 

  806. LDGEMM_L2x16_LOOP_END:

  807. 

  808. 	#dcbt		AO,	PRE

  809. 	KERNEL2x16_1

  810. 	#dcbt		AO,	PRE

  811. 	KERNEL2x16_2

  812. 	#dcbt		AO,	PRE

  813. 	KERNEL2x16_1

  814. 	#dcbt		AO,	PRE

  815. 	KERNEL2x16_2

  816. 

  817. 	#dcbt		AO,	PRE

  818. 	KERNEL2x16_1

  819. 	#dcbt		AO,	PRE

  820. 	KERNEL2x16_2

  821. 	#dcbt		AO,	PRE

  822. 	KERNEL2x16_1

  823. 	KERNEL2x16_E2

  824. 

  825. 	b		LDGEMM_L2x16_SUB1

  826. 

  827. LDGEMM_L2x16_SUB4:

  828. 

  829. 	#dcbt		AO,	PRE

  830. 	KERNEL2x16_SUBI1

  831. 	#dcbt		AO,	PRE

  832. 	KERNEL2x16_SUB1

  833. 	#dcbt		AO,	PRE

  834. 	KERNEL2x16_SUB1

  835. 	#dcbt		AO,	PRE

  836. 	KERNEL2x16_SUB1

  837. 

  838. 	KERNEL2x16_SUB1

  839. 	KERNEL2x16_SUB1

  840. 	KERNEL2x16_SUB1

  841. 	KERNEL2x16_SUB1

  842. 

  843. 	b		LDGEMM_L2x16_SUB1

  844. 

  845. LDGEMM_L2x16_SUB0:

  846. #if defined(TRMMKERNEL)

  847. 	andi.		L,	T3,	7

  848. #else

  849. 	andi.		L,	K,	7

  850. #endif

  851. 

  852. 	KERNEL2x16_SUBI1

  853. 

  854. 	addic.		L,	L,	-1

  855. 	ble		LDGEMM_L2x16_SAVE

  856. 	b		LDGEMM_L2x16_SUB2

  857. 

  858. LDGEMM_L2x16_SUB1:

  859. #if defined(TRMMKERNEL)

  860. 	andi.		L,	T3,	7

  861. #else

  862. 	andi.		L,	K,	7

  863. #endif

  864. 	ble		LDGEMM_L2x16_SAVE

  865. 

  866. LDGEMM_L2x16_SUB2:

  867. 

  868. 	KERNEL2x16_SUB1

  869. 

  870. 	addic.		L,	L,	-1

  871. 	bgt		LDGEMM_L2x16_SUB2

  872. 

  873. LDGEMM_L2x16_SAVE:

  874. 

  875. 	SAVE2x16

  876. #if defined(TRMMKERNEL)	

  877. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,2

  878. #endif	

  879. 	addic.		I,	I,	-1

  880. 	bgt		LDGEMM_L2x16_BEGIN

  881. 

  882. LDGEMM_L2x16_END:

  883. 

  884. LDGEMM_L2x8_BEGIN:

  885. 

  886. 	andi.		T2,	M,	15

  887. 	ble		LDGEMM_L2x1_END

  888. 

  889. 	andi.		T1,	M,	8

  890. 	ble		LDGEMM_L2x8_END

  891. #if defined(TRMMKERNEL)

  892.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,2

  893.     REFRESH_TEMP_BK T3,K,TEMP_REG,8,2

  894.     srawi.		L, T3,	3	

  895. #else

  896. 	mr		BO,	B

  897. 	srawi.		L,	K,	3	

  898. #endif	

  899. 	ble		LDGEMM_L2x8_SUB0

  900. 	cmpwi		cr0,	L,	1

  901. 	ble		LDGEMM_L2x8_SUB4

  902. 

  903. LDGEMM_L2x8_LOOP_START:

  904. 

  905. 	#dcbt	AO,	PRE

  906. 	LOAD2x8_1

  907. 	KERNEL2x8_I1

  908. 	#dcbt	AO,	PRE

  909. 	KERNEL2x8_2

  910. 	KERNEL2x8_1

  911. 	#dcbt	AO,	PRE

  912. 	KERNEL2x8_2

  913. 

  914. 	KERNEL2x8_1

  915. 	#dcbt	AO,	PRE

  916. 	KERNEL2x8_2

  917. 	KERNEL2x8_1

  918. 	#dcbt	AO,	PRE

  919. 	KERNEL2x8_2

  920. 

  921. 	addic.		L,	L,	-2

  922. 	ble		LDGEMM_L2x8_LOOP_END

  923. 

  924. 	MY_ALIGN

  925. 

  926. LDGEMM_L2x8_LOOP:

  927. 

  928. 	KERNEL2x8_1

  929. 	#dcbt	AO,	PRE

  930. 	KERNEL2x8_2

  931. 	KERNEL2x8_1

  932. 	#dcbt	AO,	PRE

  933. 	KERNEL2x8_2

  934. 

  935. 	KERNEL2x8_1

  936. 	#dcbt	AO,	PRE

  937. 	KERNEL2x8_2

  938. 	KERNEL2x8_1

  939. 	#dcbt	AO,	PRE

  940. 	KERNEL2x8_2

  941. 

  942. 	addic.		L,	L,	-1

  943. 	bgt		LDGEMM_L2x8_LOOP

  944. 

  945. LDGEMM_L2x8_LOOP_END:

  946. 

  947. 	KERNEL2x8_1

  948. 	KERNEL2x8_2

  949. 	KERNEL2x8_1

  950. 	KERNEL2x8_2

  951. 

  952. 	KERNEL2x8_1

  953. 	KERNEL2x8_2

  954. 	KERNEL2x8_1

  955. 	KERNEL2x8_E2

  956. 

  957. 	b		LDGEMM_L2x8_SUB1

  958. 

  959. LDGEMM_L2x8_SUB4:

  960. 

  961. 	KERNEL2x8_SUBI1

  962. 	KERNEL2x8_SUB1

  963. 	KERNEL2x8_SUB1

  964. 	KERNEL2x8_SUB1

  965. 

  966. 	KERNEL2x8_SUB1

  967. 	KERNEL2x8_SUB1

  968. 	KERNEL2x8_SUB1

  969. 	KERNEL2x8_SUB1

  970. 

  971. 	b		LDGEMM_L2x8_SUB1

  972. 

  973. LDGEMM_L2x8_SUB0:

  974. #if defined(TRMMKERNEL)

  975. 	andi.		L,	T3,	7

  976. #else

  977. 	andi.		L,	K,	7

  978. #endif

  979. 

  980. 	KERNEL2x8_SUBI1

  981. 

  982. 	addic.		L,	L,	-1

  983. 	ble		LDGEMM_L2x8_SAVE

  984. 	b		LDGEMM_L2x8_SUB2

  985. 

  986. LDGEMM_L2x8_SUB1:

  987. #if defined(TRMMKERNEL)

  988. 	andi.		L,	T3,	7

  989. #else

  990. 	andi.		L,	K,	7

  991. #endif

  992. 	ble		LDGEMM_L2x8_SAVE

  993. 

  994. LDGEMM_L2x8_SUB2:

  995. 

  996. 	KERNEL2x8_SUB1

  997. 

  998. 	addic.		L,	L,	-1

  999. 	bgt		LDGEMM_L2x8_SUB2

  1000. 

  1001. LDGEMM_L2x8_SAVE:

  1002. 

  1003. 	SAVE2x8

  1004. #if defined(TRMMKERNEL)	

  1005. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,2

  1006. #endif

  1007. LDGEMM_L2x8_END:

  1008. 

  1009. LDGEMM_L2x4_BEGIN:

  1010. 

  1011. 

  1012. 	andi.		T1,	M,	4

  1013. 	ble		LDGEMM_L2x4_END

  1014. #if defined(TRMMKERNEL)

  1015.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,2

  1016.     REFRESH_TEMP_BK T3,K,TEMP_REG,4,2

  1017.     srawi.		L, T3,	3	

  1018. #else

  1019. 	mr		BO,	B

  1020. 	srawi.		L,	K,	3	

  1021. #endif	

  1022. 	ble		LDGEMM_L2x4_SUB0

  1023. 	cmpwi		cr0,	L,	1

  1024. 	ble		LDGEMM_L2x4_SUB4

  1025. 

  1026. LDGEMM_L2x4_LOOP_START:

  1027. 

  1028. 	LOAD2x4_1

  1029. 	KERNEL2x4_I1

  1030. 	KERNEL2x4_2

  1031. 	KERNEL2x4_1

  1032. 	KERNEL2x4_2

  1033. 

  1034. 	KERNEL2x4_1

  1035. 	KERNEL2x4_2

  1036. 	KERNEL2x4_1

  1037. 	KERNEL2x4_2

  1038. 

  1039. 	addic.		L,	L,	-2

  1040. 	ble		LDGEMM_L2x4_LOOP_END

  1041. 

  1042. 	MY_ALIGN

  1043. 

  1044. LDGEMM_L2x4_LOOP:

  1045. 

  1046. 	KERNEL2x4_1

  1047. 	KERNEL2x4_2

  1048. 	KERNEL2x4_1

  1049. 	KERNEL2x4_2

  1050. 

  1051. 	KERNEL2x4_1

  1052. 	KERNEL2x4_2

  1053. 	KERNEL2x4_1

  1054. 	KERNEL2x4_2

  1055. 

  1056. 	addic.		L,	L,	-1

  1057. 	bgt		LDGEMM_L2x4_LOOP

  1058. 

  1059. LDGEMM_L2x4_LOOP_END:

  1060. 

  1061. 	KERNEL2x4_1

  1062. 	KERNEL2x4_2

  1063. 	KERNEL2x4_1

  1064. 	KERNEL2x4_2

  1065. 

  1066. 	KERNEL2x4_1

  1067. 	KERNEL2x4_2

  1068. 	KERNEL2x4_1

  1069. 	KERNEL2x4_E2

  1070. 

  1071. 	b		LDGEMM_L2x4_SUB1

  1072. 

  1073. LDGEMM_L2x4_SUB4:

  1074. 

  1075. 	KERNEL2x4_SUBI1

  1076. 	KERNEL2x4_SUB1

  1077. 	KERNEL2x4_SUB1

  1078. 	KERNEL2x4_SUB1

  1079. 

  1080. 	KERNEL2x4_SUB1

  1081. 	KERNEL2x4_SUB1

  1082. 	KERNEL2x4_SUB1

  1083. 	KERNEL2x4_SUB1

  1084. 

  1085. 	b		LDGEMM_L2x4_SUB1

  1086. 

  1087. LDGEMM_L2x4_SUB0:

  1088. #if defined(TRMMKERNEL)

  1089. 	andi.		L,	T3,	7

  1090. #else

  1091. 	andi.		L,	K,	7

  1092. #endif

  1093. 

  1094. 	KERNEL2x4_SUBI1

  1095. 

  1096. 	addic.		L,	L,	-1

  1097. 	ble		LDGEMM_L2x4_SAVE

  1098. 	b		LDGEMM_L2x4_SUB2

  1099. 

  1100. LDGEMM_L2x4_SUB1:

  1101. #if defined(TRMMKERNEL)

  1102. 	andi.		L,	T3,	7

  1103. #else

  1104. 	andi.		L,	K,	7

  1105. #endif

  1106. 	ble		LDGEMM_L2x4_SAVE

  1107. 

  1108. LDGEMM_L2x4_SUB2:

  1109. 

  1110. 	KERNEL2x4_SUB1

  1111. 

  1112. 	addic.		L,	L,	-1

  1113. 	bgt		LDGEMM_L2x4_SUB2

  1114. 

  1115. LDGEMM_L2x4_SAVE:

  1116. 

  1117. 	SAVE2x4

  1118. #if defined(TRMMKERNEL)	

  1119. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,2

  1120. #endif

  1121. LDGEMM_L2x4_END:

  1122. 

  1123. LDGEMM_L2x2_BEGIN:

  1124. 

  1125. 

  1126. 	andi.		T1,	M,	2

  1127. 	ble		LDGEMM_L2x2_END

  1128. #if defined(TRMMKERNEL)

  1129.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,2

  1130.     REFRESH_TEMP_BK T3,K,TEMP_REG,2,2

  1131.     srawi.		L, T3,	3	

  1132. #else

  1133. 	mr		BO,	B

  1134. 	srawi.		L,	K,	3	

  1135. #endif	

  1136. 	ble		LDGEMM_L2x2_SUB0

  1137. 	cmpwi		cr0,	L,	1

  1138. 	ble		LDGEMM_L2x2_SUB4

  1139. 

  1140. LDGEMM_L2x2_LOOP_START:

  1141. 

  1142. 	LOAD2x2_1

  1143. 	KERNEL2x2_I1

  1144. 	KERNEL2x2_2

  1145. 	KERNEL2x2_1

  1146. 	KERNEL2x2_2

  1147. 

  1148. 	KERNEL2x2_1

  1149. 	KERNEL2x2_2

  1150. 	KERNEL2x2_1

  1151. 	KERNEL2x2_2

  1152. 

  1153. 	addic.		L,	L,	-2

  1154. 	ble		LDGEMM_L2x2_LOOP_END

  1155. 

  1156. 	MY_ALIGN

  1157. 

  1158. LDGEMM_L2x2_LOOP:

  1159. 

  1160. 	KERNEL2x2_1

  1161. 	KERNEL2x2_2

  1162. 	KERNEL2x2_1

  1163. 	KERNEL2x2_2

  1164. 

  1165. 	KERNEL2x2_1

  1166. 	KERNEL2x2_2

  1167. 	KERNEL2x2_1

  1168. 	KERNEL2x2_2

  1169. 

  1170. 	addic.		L,	L,	-1

  1171. 	bgt		LDGEMM_L2x2_LOOP

  1172. 

  1173. LDGEMM_L2x2_LOOP_END:

  1174. 

  1175. 	KERNEL2x2_1

  1176. 	KERNEL2x2_2

  1177. 	KERNEL2x2_1

  1178. 	KERNEL2x2_2

  1179. 

  1180. 	KERNEL2x2_1

  1181. 	KERNEL2x2_2

  1182. 	KERNEL2x2_1

  1183. 	KERNEL2x2_E2

  1184. 

  1185. 	b		LDGEMM_L2x2_SUB1

  1186. 

  1187. LDGEMM_L2x2_SUB4:

  1188. 

  1189. 	KERNEL2x2_SUBI1

  1190. 	KERNEL2x2_SUB1

  1191. 	KERNEL2x2_SUB1

  1192. 	KERNEL2x2_SUB1

  1193. 

  1194. 	KERNEL2x2_SUB1

  1195. 	KERNEL2x2_SUB1

  1196. 	KERNEL2x2_SUB1

  1197. 	KERNEL2x2_SUB1

  1198. 

  1199. 	b		LDGEMM_L2x2_SUB1

  1200. 

  1201. LDGEMM_L2x2_SUB0:

  1202. #if defined(TRMMKERNEL)

  1203. 	andi.		L,	T3,	7

  1204. #else

  1205. 	andi.		L,	K,	7

  1206. #endif

  1207. 

  1208. 	KERNEL2x2_SUBI1

  1209. 

  1210. 	addic.		L,	L,	-1

  1211. 	ble		LDGEMM_L2x2_SAVE

  1212. 	b		LDGEMM_L2x2_SUB2

  1213. 

  1214. LDGEMM_L2x2_SUB1:

  1215. #if defined(TRMMKERNEL)

  1216. 	andi.		L,	T3,	7

  1217. #else

  1218. 	andi.		L,	K,	7

  1219. #endif

  1220. 	ble		LDGEMM_L2x2_SAVE

  1221. 

  1222. LDGEMM_L2x2_SUB2:

  1223. 

  1224. 	KERNEL2x2_SUB1

  1225. 

  1226. 	addic.		L,	L,	-1

  1227. 	bgt		LDGEMM_L2x2_SUB2

  1228. 

  1229. LDGEMM_L2x2_SAVE:

  1230. 

  1231. 	SAVE2x2

  1232. #if defined(TRMMKERNEL)	

  1233. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,2

  1234. #endif

  1235. LDGEMM_L2x2_END:

  1236. 

  1237. LDGEMM_L2x1_BEGIN:

  1238. 

  1239. 

  1240. 	andi.		T1,	M,	1

  1241. 	ble		LDGEMM_L2x1_END

  1242. #if defined(TRMMKERNEL)

  1243.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,2

  1244.     REFRESH_TEMP_BK T3,K,TEMP_REG,1,2

  1245.     srawi.		L, T3,	3	

  1246. #else

  1247. 	mr		BO,	B

  1248. 	srawi.		L,	K,	3	

  1249. #endif	

  1250. 	ble		LDGEMM_L2x1_SUB0

  1251. 	cmpwi		cr0,	L,	1

  1252. 	ble		LDGEMM_L2x1_SUB4

  1253. 

  1254. LDGEMM_L2x1_LOOP_START:

  1255. 

  1256. 	LOAD2x1_1

  1257. 	KERNEL2x1_I1

  1258. 	KERNEL2x1_2

  1259. 	KERNEL2x1_1

  1260. 	KERNEL2x1_2

  1261. 

  1262. 	KERNEL2x1_1

  1263. 	KERNEL2x1_2

  1264. 	KERNEL2x1_1

  1265. 	KERNEL2x1_2

  1266. 

  1267. 	addic.		L,	L,	-2

  1268. 	ble		LDGEMM_L2x1_LOOP_END

  1269. 

  1270. 	MY_ALIGN

  1271. 

  1272. LDGEMM_L2x1_LOOP:

  1273. 

  1274. 	KERNEL2x1_1

  1275. 	KERNEL2x1_2

  1276. 	KERNEL2x1_1

  1277. 	KERNEL2x1_2

  1278. 

  1279. 	KERNEL2x1_1

  1280. 	KERNEL2x1_2

  1281. 	KERNEL2x1_1

  1282. 	KERNEL2x1_2

  1283. 

  1284. 	addic.		L,	L,	-1

  1285. 	bgt		LDGEMM_L2x1_LOOP

  1286. 

  1287. LDGEMM_L2x1_LOOP_END:

  1288. 

  1289. 	KERNEL2x1_1

  1290. 	KERNEL2x1_2

  1291. 	KERNEL2x1_1

  1292. 	KERNEL2x1_2

  1293. 

  1294. 	KERNEL2x1_1

  1295. 	KERNEL2x1_2

  1296. 	KERNEL2x1_1

  1297. 	KERNEL2x1_E2

  1298. 

  1299. 	b		LDGEMM_L2x1_SUB1

  1300. 

  1301. LDGEMM_L2x1_SUB4:

  1302. 

  1303. 	KERNEL2x1_SUBI1

  1304. 	KERNEL2x1_SUB1

  1305. 	KERNEL2x1_SUB1

  1306. 	KERNEL2x1_SUB1

  1307. 

  1308. 	KERNEL2x1_SUB1

  1309. 	KERNEL2x1_SUB1

  1310. 	KERNEL2x1_SUB1

  1311. 	KERNEL2x1_SUB1

  1312. 

  1313. 	b		LDGEMM_L2x1_SUB1

  1314. 

  1315. LDGEMM_L2x1_SUB0:

  1316. #if defined(TRMMKERNEL)

  1317. 	andi.		L,	T3,	7

  1318. #else

  1319. 	andi.		L,	K,	7

  1320. #endif

  1321. 

  1322. 	KERNEL2x1_SUBI1

  1323. 

  1324. 	addic.		L,	L,	-1

  1325. 	ble		LDGEMM_L2x1_SAVE

  1326. 	b		LDGEMM_L2x1_SUB2

  1327. 

  1328. LDGEMM_L2x1_SUB1:

  1329. #if defined(TRMMKERNEL)

  1330. 	andi.		L,	T3,	7

  1331. #else

  1332. 	andi.		L,	K,	7

  1333. #endif

  1334. 	ble		LDGEMM_L2x1_SAVE

  1335. 

  1336. LDGEMM_L2x1_SUB2:

  1337. 

  1338. 	KERNEL2x1_SUB1

  1339. 

  1340. 	addic.		L,	L,	-1

  1341. 	bgt		LDGEMM_L2x1_SUB2

  1342. 

  1343. LDGEMM_L2x1_SAVE:

  1344. 

  1345. 	SAVE2x1

  1346. #if defined(TRMMKERNEL)	

  1347. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,2

  1348. #endif

  1349. LDGEMM_L2x1_END:

  1350. 

  1351. 	slwi		T1,	K,	4

  1352. 	add		B,	B,	T1

  1353. #if defined(TRMMKERNEL) && !defined(LEFT)

  1354.     addi TEMP_REG, TEMP_REG, 2

  1355. #endif

  1356. LDGEMM_L2_END:

  1357. LDGEMM_L1_BEGIN:

  1358. 

  1359. #if defined(TRMMKERNEL) && defined(LEFT)

  1360. 	mr TEMP_REG, OFFSET	 /*off = offset;*/

  1361. #endif 

  1362. 	andi.		T1,	N,	1

  1363. 	ble		LDGEMM_L1_END

  1364. 	mr		CO,	C

  1365. 	mr		AO,	A

  1366. 	srawi.		I,	M,	4

  1367. 	ble		LDGEMM_L1x16_END

  1368. 

  1369. LDGEMM_L1x16_BEGIN:

  1370. 

  1371. 

  1372. #if defined(TRMMKERNEL)

  1373.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,16,1

  1374.     REFRESH_TEMP_BK T3,K,TEMP_REG,16,1

  1375.     srawi.		L, T3,	3	

  1376. #else

  1377. 	mr		BO,	B

  1378. 	srawi.		L,	K,	3	

  1379. #endif	

  1380. 	ble		LDGEMM_L1x16_SUB0

  1381. 	cmpwi		cr0,	L,	1

  1382. 	ble		LDGEMM_L1x16_SUB4

  1383. 

  1384. LDGEMM_L1x16_LOOP_START:

  1385. 

  1386. 	#dcbt		AO,	PRE

  1387. 	LOAD1x16_1

  1388. 	#dcbt		AO,	PRE

  1389. 	KERNEL1x16_I1

  1390. 	#dcbt		AO,	PRE

  1391. 	KERNEL1x16_2

  1392. 	#dcbt		AO,	PRE

  1393. 	KERNEL1x16_1

  1394. 	#dcbt		AO,	PRE

  1395. 	KERNEL1x16_2

  1396. 

  1397. 	#dcbt		AO,	PRE

  1398. 	KERNEL1x16_1

  1399. 	#dcbt		AO,	PRE

  1400. 	KERNEL1x16_2

  1401. 	#dcbt		AO,	PRE

  1402. 	KERNEL1x16_1

  1403. 	#dcbt		AO,	PRE

  1404. 	KERNEL1x16_2

  1405. 

  1406. 	addic.		L,	L,	-2

  1407. 	ble		LDGEMM_L1x16_LOOP_END

  1408. 

  1409. 	MY_ALIGN

  1410. 

  1411. LDGEMM_L1x16_LOOP:

  1412. 

  1413. 	#dcbt		AO,	PRE

  1414. 	KERNEL1x16_1

  1415. 	#dcbt		AO,	PRE

  1416. 	KERNEL1x16_2

  1417. 	#dcbt		AO,	PRE

  1418. 	KERNEL1x16_1

  1419. 	#dcbt		AO,	PRE

  1420. 	KERNEL1x16_2

  1421. 

  1422. 	#dcbt		AO,	PRE

  1423. 	KERNEL1x16_1

  1424. 	#dcbt		AO,	PRE

  1425. 	KERNEL1x16_2

  1426. 	#dcbt		AO,	PRE

  1427. 	KERNEL1x16_1

  1428. 	#dcbt		AO,	PRE

  1429. 	KERNEL1x16_2

  1430. 

  1431. 	addic.		L,	L,	-1

  1432. 	bgt		LDGEMM_L1x16_LOOP

  1433. 

  1434. LDGEMM_L1x16_LOOP_END:

  1435. 

  1436. 	#dcbt		AO,	PRE

  1437. 	KERNEL1x16_1

  1438. 	#dcbt		AO,	PRE

  1439. 	KERNEL1x16_2

  1440. 	#dcbt		AO,	PRE

  1441. 	KERNEL1x16_1

  1442. 	#dcbt		AO,	PRE

  1443. 	KERNEL1x16_2

  1444. 

  1445. 	#dcbt		AO,	PRE

  1446. 	KERNEL1x16_1

  1447. 	#dcbt		AO,	PRE

  1448. 	KERNEL1x16_2

  1449. 	#dcbt		AO,	PRE

  1450. 	KERNEL1x16_1

  1451. 	KERNEL1x16_E2

  1452. 

  1453. 	b		LDGEMM_L1x16_SUB1

  1454. 

  1455. LDGEMM_L1x16_SUB4:

  1456. 

  1457. 	#dcbt		AO,	PRE

  1458. 	KERNEL1x16_SUBI1

  1459. 	#dcbt		AO,	PRE

  1460. 	KERNEL1x16_SUB1

  1461. 	#dcbt		AO,	PRE

  1462. 	KERNEL1x16_SUB1

  1463. 	#dcbt		AO,	PRE

  1464. 	KERNEL1x16_SUB1

  1465. 

  1466. 	KERNEL1x16_SUB1

  1467. 	KERNEL1x16_SUB1

  1468. 	KERNEL1x16_SUB1

  1469. 	KERNEL1x16_SUB1

  1470. 

  1471. 	b		LDGEMM_L1x16_SUB1

  1472. 

  1473. LDGEMM_L1x16_SUB0:

  1474. #if defined(TRMMKERNEL)

  1475. 	andi.		L,	T3,	7

  1476. #else

  1477. 	andi.		L,	K,	7

  1478. #endif

  1479. 

  1480. 	KERNEL1x16_SUBI1

  1481. 

  1482. 	addic.		L,	L,	-1

  1483. 	ble		LDGEMM_L1x16_SAVE

  1484. 	b		LDGEMM_L1x16_SUB2

  1485. 

  1486. LDGEMM_L1x16_SUB1:

  1487. #if defined(TRMMKERNEL)

  1488. 	andi.		L,	T3,	7

  1489. #else

  1490. 	andi.		L,	K,	7

  1491. #endif

  1492. 	ble		LDGEMM_L1x16_SAVE

  1493. 

  1494. LDGEMM_L1x16_SUB2:

  1495. 

  1496. 	KERNEL1x16_SUB1

  1497. 

  1498. 	addic.		L,	L,	-1

  1499. 	bgt		LDGEMM_L1x16_SUB2

  1500. 

  1501. LDGEMM_L1x16_SAVE:

  1502. 

  1503. 	SAVE1x16

  1504. #if defined(TRMMKERNEL)	

  1505. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,16,1

  1506. #endif

  1507. 	addic.		I,	I,	-1

  1508. 	bgt		LDGEMM_L1x16_BEGIN

  1509. 

  1510. LDGEMM_L1x16_END:

  1511. 

  1512. LDGEMM_L1x8_BEGIN:

  1513. 

  1514. 	andi.		T2,	M,	15

  1515. 	ble		LDGEMM_L1x1_END

  1516. 

  1517. 	andi.		T1,	M,	8

  1518. 	ble		LDGEMM_L1x8_END

  1519. #if defined(TRMMKERNEL)

  1520.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,8,1

  1521.     REFRESH_TEMP_BK T3,K,TEMP_REG,8,1

  1522.     srawi.		L, T3,	3	

  1523. #else

  1524. 	mr		BO,	B

  1525. 	srawi.		L,	K,	3	

  1526. #endif	

  1527. 	ble		LDGEMM_L1x8_SUB0

  1528. 	cmpwi		cr0,	L,	1

  1529. 	ble		LDGEMM_L1x8_SUB4

  1530. 

  1531. LDGEMM_L1x8_LOOP_START:

  1532. 

  1533. 	#dcbt	AO,	PRE

  1534. 	LOAD1x8_1

  1535. 	KERNEL1x8_I1

  1536. 	#dcbt	AO,	PRE

  1537. 	KERNEL1x8_2

  1538. 	KERNEL1x8_1

  1539. 	#dcbt	AO,	PRE

  1540. 	KERNEL1x8_2

  1541. 

  1542. 	KERNEL1x8_1

  1543. 	#dcbt	AO,	PRE

  1544. 	KERNEL1x8_2

  1545. 	KERNEL1x8_1

  1546. 	#dcbt	AO,	PRE

  1547. 	KERNEL1x8_2

  1548. 

  1549. 	addic.		L,	L,	-2

  1550. 	ble		LDGEMM_L1x8_LOOP_END

  1551. 

  1552. 	MY_ALIGN

  1553. 

  1554. LDGEMM_L1x8_LOOP:

  1555. 

  1556. 	KERNEL1x8_1

  1557. 	#dcbt	AO,	PRE

  1558. 	KERNEL1x8_2

  1559. 	KERNEL1x8_1

  1560. 	#dcbt	AO,	PRE

  1561. 	KERNEL1x8_2

  1562. 

  1563. 	KERNEL1x8_1

  1564. 	#dcbt	AO,	PRE

  1565. 	KERNEL1x8_2

  1566. 	KERNEL1x8_1

  1567. 	#dcbt	AO,	PRE

  1568. 	KERNEL1x8_2

  1569. 

  1570. 	addic.		L,	L,	-1

  1571. 	bgt		LDGEMM_L1x8_LOOP

  1572. 

  1573. LDGEMM_L1x8_LOOP_END:

  1574. 

  1575. 	KERNEL1x8_1

  1576. 	KERNEL1x8_2

  1577. 	KERNEL1x8_1

  1578. 	KERNEL1x8_2

  1579. 

  1580. 	KERNEL1x8_1

  1581. 	KERNEL1x8_2

  1582. 	KERNEL1x8_1

  1583. 	KERNEL1x8_E2

  1584. 

  1585. 	b		LDGEMM_L1x8_SUB1

  1586. 

  1587. LDGEMM_L1x8_SUB4:

  1588. 

  1589. 	KERNEL1x8_SUBI1

  1590. 	KERNEL1x8_SUB1

  1591. 	KERNEL1x8_SUB1

  1592. 	KERNEL1x8_SUB1

  1593. 

  1594. 	KERNEL1x8_SUB1

  1595. 	KERNEL1x8_SUB1

  1596. 	KERNEL1x8_SUB1

  1597. 	KERNEL1x8_SUB1

  1598. 

  1599. 	b		LDGEMM_L1x8_SUB1

  1600. 

  1601. LDGEMM_L1x8_SUB0:

  1602. #if defined(TRMMKERNEL)

  1603. 	andi.		L,	T3,	7

  1604. #else

  1605. 	andi.		L,	K,	7

  1606. #endif

  1607. 

  1608. 	KERNEL1x8_SUBI1

  1609. 

  1610. 	addic.		L,	L,	-1

  1611. 	ble		LDGEMM_L1x8_SAVE

  1612. 	b		LDGEMM_L1x8_SUB2

  1613. 

  1614. LDGEMM_L1x8_SUB1:

  1615. #if defined(TRMMKERNEL)

  1616. 	andi.		L,	T3,	7

  1617. #else

  1618. 	andi.		L,	K,	7

  1619. #endif

  1620. 	ble		LDGEMM_L1x8_SAVE

  1621. 

  1622. LDGEMM_L1x8_SUB2:

  1623. 

  1624. 	KERNEL1x8_SUB1

  1625. 

  1626. 	addic.		L,	L,	-1

  1627. 	bgt		LDGEMM_L1x8_SUB2

  1628. 

  1629. LDGEMM_L1x8_SAVE:

  1630. 

  1631. 	SAVE1x8

  1632. #if defined(TRMMKERNEL)	

  1633. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,8,1

  1634. #endif

  1635. LDGEMM_L1x8_END:

  1636. 

  1637. LDGEMM_L1x4_BEGIN:

  1638. 

  1639. 

  1640. 	andi.		T1,	M,	4

  1641. 	ble		LDGEMM_L1x4_END

  1642. #if defined(TRMMKERNEL)

  1643.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,4,1

  1644.     REFRESH_TEMP_BK T3,K,TEMP_REG,4,1

  1645.     srawi.		L, T3,	3	

  1646. #else

  1647. 	mr		BO,	B

  1648. 	srawi.		L,	K,	3	

  1649. #endif	

  1650. 	ble		LDGEMM_L1x4_SUB0

  1651. 	cmpwi		cr0,	L,	1

  1652. 	ble		LDGEMM_L1x4_SUB4

  1653. 

  1654. LDGEMM_L1x4_LOOP_START:

  1655. 

  1656. 	LOAD1x4_1

  1657. 	KERNEL1x4_I1

  1658. 	KERNEL1x4_2

  1659. 	KERNEL1x4_1

  1660. 	KERNEL1x4_2

  1661. 

  1662. 	KERNEL1x4_1

  1663. 	KERNEL1x4_2

  1664. 	KERNEL1x4_1

  1665. 	KERNEL1x4_2

  1666. 

  1667. 	addic.		L,	L,	-2

  1668. 	ble		LDGEMM_L1x4_LOOP_END

  1669. 

  1670. 	MY_ALIGN

  1671. 

  1672. LDGEMM_L1x4_LOOP:

  1673. 

  1674. 	KERNEL1x4_1

  1675. 	KERNEL1x4_2

  1676. 	KERNEL1x4_1

  1677. 	KERNEL1x4_2

  1678. 

  1679. 	KERNEL1x4_1

  1680. 	KERNEL1x4_2

  1681. 	KERNEL1x4_1

  1682. 	KERNEL1x4_2

  1683. 

  1684. 	addic.		L,	L,	-1

  1685. 	bgt		LDGEMM_L1x4_LOOP

  1686. 

  1687. LDGEMM_L1x4_LOOP_END:

  1688. 

  1689. 	KERNEL1x4_1

  1690. 	KERNEL1x4_2

  1691. 	KERNEL1x4_1

  1692. 	KERNEL1x4_2

  1693. 

  1694. 	KERNEL1x4_1

  1695. 	KERNEL1x4_2

  1696. 	KERNEL1x4_1

  1697. 	KERNEL1x4_E2

  1698. 

  1699. 	b		LDGEMM_L1x4_SUB1

  1700. 

  1701. LDGEMM_L1x4_SUB4:

  1702. 

  1703. 	KERNEL1x4_SUBI1

  1704. 	KERNEL1x4_SUB1

  1705. 	KERNEL1x4_SUB1

  1706. 	KERNEL1x4_SUB1

  1707. 

  1708. 	KERNEL1x4_SUB1

  1709. 	KERNEL1x4_SUB1

  1710. 	KERNEL1x4_SUB1

  1711. 	KERNEL1x4_SUB1

  1712. 

  1713. 	b		LDGEMM_L1x4_SUB1

  1714. 

  1715. LDGEMM_L1x4_SUB0:

  1716. #if defined(TRMMKERNEL)

  1717. 	andi.		L,	T3,	7

  1718. #else

  1719. 	andi.		L,	K,	7

  1720. #endif

  1721. 

  1722. 	KERNEL1x4_SUBI1

  1723. 

  1724. 	addic.		L,	L,	-1

  1725. 	ble		LDGEMM_L1x4_SAVE

  1726. 	b		LDGEMM_L1x4_SUB2

  1727. 

  1728. LDGEMM_L1x4_SUB1:

  1729. #if defined(TRMMKERNEL)

  1730. 	andi.		L,	T3,	7

  1731. #else

  1732. 	andi.		L,	K,	7

  1733. #endif

  1734. 	ble		LDGEMM_L1x4_SAVE

  1735. 

  1736. LDGEMM_L1x4_SUB2:

  1737. 

  1738. 	KERNEL1x4_SUB1

  1739. 

  1740. 	addic.		L,	L,	-1

  1741. 	bgt		LDGEMM_L1x4_SUB2

  1742. 

  1743. LDGEMM_L1x4_SAVE:

  1744. 

  1745. 	SAVE1x4

  1746. #if defined(TRMMKERNEL)	

  1747. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,4,1

  1748. #endif

  1749. LDGEMM_L1x4_END:

  1750. 

  1751. LDGEMM_L1x2_BEGIN:

  1752. 

  1753. 

  1754. 	andi.		T1,	M,	2

  1755. 	ble		LDGEMM_L1x2_END

  1756. #if defined(TRMMKERNEL)

  1757.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,2,1

  1758.     REFRESH_TEMP_BK T3,K,TEMP_REG,2,1

  1759.     srawi.		L, T3,	3	

  1760. #else

  1761. 	mr		BO,	B

  1762. 	srawi.		L,	K,	3	

  1763. #endif	

  1764. 	ble		LDGEMM_L1x2_SUB0

  1765. 	cmpwi		cr0,	L,	1

  1766. 	ble		LDGEMM_L1x2_SUB4

  1767. 

  1768. LDGEMM_L1x2_LOOP_START:

  1769. 

  1770. 	LOAD1x2_1

  1771. 	KERNEL1x2_I1

  1772. 	KERNEL1x2_2

  1773. 	KERNEL1x2_1

  1774. 	KERNEL1x2_2

  1775. 

  1776. 	KERNEL1x2_1

  1777. 	KERNEL1x2_2

  1778. 	KERNEL1x2_1

  1779. 	KERNEL1x2_2

  1780. 

  1781. 	addic.		L,	L,	-2

  1782. 	ble		LDGEMM_L1x2_LOOP_END

  1783. 

  1784. 	MY_ALIGN

  1785. 

  1786. LDGEMM_L1x2_LOOP:

  1787. 

  1788. 	KERNEL1x2_1

  1789. 	KERNEL1x2_2

  1790. 	KERNEL1x2_1

  1791. 	KERNEL1x2_2

  1792. 

  1793. 	KERNEL1x2_1

  1794. 	KERNEL1x2_2

  1795. 	KERNEL1x2_1

  1796. 	KERNEL1x2_2

  1797. 

  1798. 	addic.		L,	L,	-1

  1799. 	bgt		LDGEMM_L1x2_LOOP

  1800. 

  1801. LDGEMM_L1x2_LOOP_END:

  1802. 

  1803. 	KERNEL1x2_1

  1804. 	KERNEL1x2_2

  1805. 	KERNEL1x2_1

  1806. 	KERNEL1x2_2

  1807. 

  1808. 	KERNEL1x2_1

  1809. 	KERNEL1x2_2

  1810. 	KERNEL1x2_1

  1811. 	KERNEL1x2_E2

  1812. 

  1813. 	b		LDGEMM_L1x2_SUB1

  1814. 

  1815. LDGEMM_L1x2_SUB4:

  1816. 

  1817. 	KERNEL1x2_SUBI1

  1818. 	KERNEL1x2_SUB1

  1819. 	KERNEL1x2_SUB1

  1820. 	KERNEL1x2_SUB1

  1821. 

  1822. 	KERNEL1x2_SUB1

  1823. 	KERNEL1x2_SUB1

  1824. 	KERNEL1x2_SUB1

  1825. 	KERNEL1x2_SUB1

  1826. 

  1827. 	b		LDGEMM_L1x2_SUB1

  1828. 

  1829. LDGEMM_L1x2_SUB0:

  1830. #if defined(TRMMKERNEL)

  1831. 	andi.		L,	T3,	7

  1832. #else

  1833. 	andi.		L,	K,	7

  1834. #endif

  1835. 

  1836. 	KERNEL1x2_SUBI1

  1837. 

  1838. 	addic.		L,	L,	-1

  1839. 	ble		LDGEMM_L1x2_SAVE

  1840. 	b		LDGEMM_L1x2_SUB2

  1841. 

  1842. LDGEMM_L1x2_SUB1:

  1843. #if defined(TRMMKERNEL)

  1844. 	andi.		L,	T3,	7

  1845. #else

  1846. 	andi.		L,	K,	7

  1847. #endif

  1848. 	ble		LDGEMM_L1x2_SAVE

  1849. 

  1850. LDGEMM_L1x2_SUB2:

  1851. 

  1852. 	KERNEL1x2_SUB1

  1853. 

  1854. 	addic.		L,	L,	-1

  1855. 	bgt		LDGEMM_L1x2_SUB2

  1856. 

  1857. LDGEMM_L1x2_SAVE:

  1858. 

  1859. 	SAVE1x2

  1860. #if defined(TRMMKERNEL)	

  1861. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,2,1

  1862. #endif

  1863. LDGEMM_L1x2_END:

  1864. 

  1865. LDGEMM_L1x1_BEGIN:

  1866. 

  1867. 

  1868. 	andi.		T1,	M,	1

  1869. 	ble		LDGEMM_L1x1_END

  1870. #if defined(TRMMKERNEL)

  1871.     REFRESH_POINTERS  AO,BO,TEMP_REG,B,1,1

  1872.     REFRESH_TEMP_BK T3,K,TEMP_REG,1,1

  1873.     srawi.		L, T3,	3	

  1874. #else

  1875. 	mr		BO,	B

  1876. 	srawi.		L,	K,	3	

  1877. #endif	

  1878. 	ble		LDGEMM_L1x1_SUB0

  1879. 	cmpwi		cr0,	L,	1

  1880. 	ble		LDGEMM_L1x1_SUB4

  1881. 

  1882. LDGEMM_L1x1_LOOP_START:

  1883. 

  1884. 	LOAD1x1_1

  1885. 	KERNEL1x1_I1

  1886. 	KERNEL1x1_2

  1887. 	KERNEL1x1_1

  1888. 	KERNEL1x1_2

  1889. 

  1890. 	KERNEL1x1_1

  1891. 	KERNEL1x1_2

  1892. 	KERNEL1x1_1

  1893. 	KERNEL1x1_2

  1894. 

  1895. 	addic.		L,	L,	-2

  1896. 	ble		LDGEMM_L1x1_LOOP_END

  1897. 

  1898. 	MY_ALIGN

  1899. 

  1900. LDGEMM_L1x1_LOOP:

  1901. 

  1902. 	KERNEL1x1_1

  1903. 	KERNEL1x1_2

  1904. 	KERNEL1x1_1

  1905. 	KERNEL1x1_2

  1906. 

  1907. 	KERNEL1x1_1

  1908. 	KERNEL1x1_2

  1909. 	KERNEL1x1_1

  1910. 	KERNEL1x1_2

  1911. 

  1912. 	addic.		L,	L,	-1

  1913. 	bgt		LDGEMM_L1x1_LOOP

  1914. 

  1915. LDGEMM_L1x1_LOOP_END:

  1916. 

  1917. 	KERNEL1x1_1

  1918. 	KERNEL1x1_2

  1919. 	KERNEL1x1_1

  1920. 	KERNEL1x1_2

  1921. 

  1922. 	KERNEL1x1_1

  1923. 	KERNEL1x1_2

  1924. 	KERNEL1x1_1

  1925. 	KERNEL1x1_E2

  1926. 

  1927. 	b		LDGEMM_L1x1_SUB1

  1928. 

  1929. LDGEMM_L1x1_SUB4:

  1930. 

  1931. 	KERNEL1x1_SUBI1

  1932. 	KERNEL1x1_SUB1

  1933. 	KERNEL1x1_SUB1

  1934. 	KERNEL1x1_SUB1

  1935. 

  1936. 	KERNEL1x1_SUB1

  1937. 	KERNEL1x1_SUB1

  1938. 	KERNEL1x1_SUB1

  1939. 	KERNEL1x1_SUB1

  1940. 

  1941. 	b		LDGEMM_L1x1_SUB1

  1942. 

  1943. LDGEMM_L1x1_SUB0:

  1944. #if defined(TRMMKERNEL)

  1945. 	andi.		L,	T3,	7

  1946. #else

  1947. 	andi.		L,	K,	7

  1948. #endif

  1949. 

  1950. 	KERNEL1x1_SUBI1

  1951. 

  1952. 	addic.		L,	L,	-1

  1953. 	ble		LDGEMM_L1x1_SAVE

  1954. 	b		LDGEMM_L1x1_SUB2

  1955. 

  1956. LDGEMM_L1x1_SUB1:

  1957. #if defined(TRMMKERNEL)

  1958. 	andi.		L,	T3,	7

  1959. #else

  1960. 	andi.		L,	K,	7

  1961. #endif

  1962. 	ble		LDGEMM_L1x1_SAVE

  1963. 

  1964. LDGEMM_L1x1_SUB2:

  1965. 

  1966. 	KERNEL1x1_SUB1

  1967. 

  1968. 	addic.		L,	L,	-1

  1969. 	bgt		LDGEMM_L1x1_SUB2

  1970. 

  1971. LDGEMM_L1x1_SAVE:

  1972. 

  1973. 	SAVE1x1

  1974. #if defined(TRMMKERNEL)	

  1975. 	REFRESH_AFTER_SAVE T3,K,TEMP_REG,BO,AO,1,1

  1976. #endif

  1977. LDGEMM_L1x1_END:

  1978. #if defined(TRMMKERNEL) && !defined(LEFT)

  1979.     addi TEMP_REG, TEMP_REG, 1

  1980. #endif

  1981. LDGEMM_L1_END: