OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c31861ea62
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c31861ea62'
${ noResults }
OpenBLAS/kernel/power/dgemm_logic_16x4_power8.S

1852 lines
23 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2016, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. /**************************************************************************************

  29. * 2016/03/05 Werner Saar (wernsaar@googlemail.com)

  30. * 	 BLASTEST 		: OK

  31. * 	 CTEST			: OK

  32. * 	 TEST			: OK

  33. *	 LAPACK-TEST		: OK

  34. **************************************************************************************/

  35. 

  36. #define MY_ALIGN .align 3

  37. 

  38. 	srawi.		J,	N,	2

  39. 	ble		LDGEMM_L4_END

  40. 

  41. LDGEMM_L4_BEGIN:

  42. 

  43. 	li		T1,	128

  44. 	li		T2,	256

  45. 	mr		AO,	A

  46. 

  47. 	mr		CO,	C

  48. 	slwi		T3,	LDC	,	2

  49. 	add		C,	C,	T3

  50. 

  51. 	dcbt		A,	T1

  52. 	dcbt		A,	T2

  53. 

  54. 	srawi.		I,	M,	4

  55. 	ble		LDGEMM_L4x16_END

  56. 

  57. 	MY_ALIGN

  58. LDGEMM_L4x16_BEGIN_FIRST:

  59. 

  60. 	li		L,	-128

  61. 

  62. 	mr		T1,	CO

  63. 	add		T2,	T1,	LDC

  64. 	add		T3,	T2,	LDC

  65. 	add		T4,	T3,	LDC

  66. 

  67. 	and		T1,	T1,	L

  68. 	and		T2,	T2,	L

  69. 	and		T3,	T3,	L

  70. 	and		T4,	T4,	L

  71. 

  72. 	dcbt		T1,	r0

  73. 	dcbt		T2,	r0

  74. 	dcbt		T3,	r0

  75. 	dcbt		T4,	r0

  76. 

  77. 	mr		BO,	B

  78. 	srawi.		L,	K,	2

  79. 

  80. 	addi		T1, T1, 128

  81. 	addi		T2, T2, 128

  82. 	addi		T3, T3, 128

  83. 	addi		T4, T4, 128

  84. 

  85. 	dcbt		T1,	r0

  86. 	dcbt		T2,	r0

  87. 	dcbt		T3,	r0

  88. 	dcbt		T4,	r0

  89. 	

  90. 	ble		LDGEMM_L4x16_SUB0_FIRST

  91. 	cmpwi		cr0,	L,	1

  92. 	ble		LDGEMM_L4x16_SUB4_FIRST

  93. 

  94. 	MY_ALIGN

  95. LDGEMM_L4x16_LOOP_START_FIRST:

  96. 

  97. 	li	T2,	512

  98. 	li	o40,	40

  99. 	li	o56,	56

  100. 

  101. 	dcbt	AO,	PRE

  102. 	dcbt	BO,	T2

  103. 	LOAD4x16_1

  104. 	dcbt	AO,	PRE

  105. 	KERNEL4x16_I1

  106. 	dcbt	AO,	PRE

  107. 	addic.		L,	L,	-2

  108. 	KERNEL4x16_L2

  109. 

  110. 	dcbt	AO,	PRE

  111. 	KERNEL4x16_L1

  112. 	dcbt	AO,	PRE

  113. 	dcbt	BO,	T2

  114. 	KERNEL4x16_L2

  115. 

  116. 	ble		LDGEMM_L4x16_LOOP_END_FIRST

  117. 	mtctr		L

  118. 

  119. 	MY_ALIGN

  120. 

  121. LDGEMM_L4x16_LOOP_FIRST:

  122. 

  123. 	dcbt	AO,	PRE

  124. 	KERNEL4x16_L1

  125. 	dcbt	AO,	PRE

  126. 	KERNEL4x16_L2

  127. 

  128. 	dcbt	AO,	PRE

  129. 	KERNEL4x16_L1

  130. 	dcbt	AO,	PRE

  131. 	dcbt	BO,	T2

  132. 	KERNEL4x16_L2

  133. 

  134. 	bdnz		LDGEMM_L4x16_LOOP_FIRST

  135. 

  136. 	MY_ALIGN

  137. 

  138. LDGEMM_L4x16_LOOP_END_FIRST:

  139. 

  140. 	KERNEL4x16_L1

  141. 	KERNEL4x16_L2

  142. 

  143. 	KERNEL4x16_1

  144. 	KERNEL4x16_E2

  145. 

  146. 	b		LDGEMM_L4x16_SUB1_FIRST

  147. 

  148. LDGEMM_L4x16_SUB4_FIRST:

  149. 

  150. 	KERNEL4x16_SUBI1

  151. 	KERNEL4x16_SUB1

  152. 	KERNEL4x16_SUB1

  153. 	KERNEL4x16_SUB1

  154. 

  155. 	b		LDGEMM_L4x16_SUB1_FIRST

  156. 

  157. LDGEMM_L4x16_SUB0_FIRST:

  158. 

  159. 	andi.		L,	K,	3

  160. 

  161. 	KERNEL4x16_SUBI1

  162. 

  163. 	addic.		L,	L,	-1

  164. 	ble		LDGEMM_L4x16_SAVE_FIRST

  165. 	b		LDGEMM_L4x16_SUB2_FIRST

  166. 

  167. LDGEMM_L4x16_SUB1_FIRST:

  168. 

  169. 	andi.		L,	K,	3

  170. 	ble		LDGEMM_L4x16_SAVE_FIRST

  171. 

  172. LDGEMM_L4x16_SUB2_FIRST:

  173. 

  174. 	KERNEL4x16_SUB1

  175. 

  176. 	addic.		L,	L,	-1

  177. 	bgt		LDGEMM_L4x16_SUB2_FIRST

  178. 

  179. 	MY_ALIGN

  180. LDGEMM_L4x16_SAVE_FIRST:

  181. 

  182. 	SAVE4x16

  183. 

  184. 	addic.		I,	I,	-1

  185. 	ble		LDGEMM_L4x16_END

  186. 

  187. LDGEMM_L4x16_END_FIRST:

  188. 

  189. 	MY_ALIGN

  190. 

  191. LDGEMM_L4x16_BEGIN:

  192. 

  193. 	li		L,	-128

  194. 

  195. 	mr		T1,	CO

  196. 	add		T2,	T1,	LDC

  197. 	add		T3,	T2,	LDC

  198. 	add		T4,	T3,	LDC

  199. 

  200. 	and		T1,	T1,	L

  201. 	and		T2,	T2,	L

  202. 	and		T3,	T3,	L

  203. 	and		T4,	T4,	L

  204. 

  205. 	dcbt		T1,	r0

  206. 	dcbt		T2,	r0

  207. 	dcbt		T3,	r0

  208. 	dcbt		T4,	r0

  209. 

  210. 	mr		BO,	B

  211. 	srawi.		L,	K,	1

  212. 

  213. 	addi		T1, T1, 128

  214. 	addi		T2, T2, 128

  215. 	addi		T3, T3, 128

  216. 	addi		T4, T4, 128

  217. 

  218. 	dcbt		T1,	r0

  219. 	dcbt		T2,	r0

  220. 	dcbt		T3,	r0

  221. 	dcbt		T4,	r0

  222. 	

  223. 	ble-		LDGEMM_L4x16_SUB0

  224. 	cmpwi		cr0,	L,	1

  225. 	ble-		LDGEMM_L4x16_SUB4

  226. 

  227. 	MY_ALIGN

  228. 

  229. LDGEMM_L4x16_LOOP_START:

  230. 

  231. 	li	o40,	40

  232. 	li	o56,	56

  233. 

  234. 	dcbt	AO,	PRE

  235. 	LOAD4x16_1

  236. 	dcbt	AO,	PRE

  237. 	KERNEL4x16_I1

  238. 	dcbt	AO,	PRE

  239. 	addic.		L,	L,	-2

  240. 	KERNEL4x16_L2

  241. 

  242. 	ble-		LDGEMM_L4x16_LOOP_END

  243. 	mtctr		L

  244. 

  245. 	MY_ALIGN

  246. 

  247. LDGEMM_L4x16_LOOP:

  248. 

  249. 	dcbt	AO,	PRE

  250. 	KERNEL4x16_L1

  251. 	dcbt	AO,	PRE

  252. 	KERNEL4x16_L2

  253. 

  254. 	bdnz+		LDGEMM_L4x16_LOOP

  255. 

  256. 

  257. 	MY_ALIGN

  258. 

  259. LDGEMM_L4x16_LOOP_END:

  260. 

  261. 	KERNEL4x16_1

  262. 	KERNEL4x16_E2

  263. 

  264. 	b		LDGEMM_L4x16_SUB1

  265. 

  266. 	MY_ALIGN

  267. 

  268. LDGEMM_L4x16_SUB4:

  269. 

  270. 	KERNEL4x16_SUBI1

  271. 	KERNEL4x16_SUB1

  272. 

  273. 	b		LDGEMM_L4x16_SUB1

  274. 

  275. 	MY_ALIGN

  276. 

  277. LDGEMM_L4x16_SUB0:

  278. 

  279. 	andi.		L,	K,	1

  280. 

  281. 	KERNEL4x16_SUBI1

  282. 

  283. 	addic.		L,	L,	-1

  284. 	ble		LDGEMM_L4x16_SAVE

  285. 	b		LDGEMM_L4x16_SUB2

  286. 

  287. 	MY_ALIGN

  288. 

  289. LDGEMM_L4x16_SUB1:

  290. 

  291. 	andi.		L,	K,	1

  292. 	ble		LDGEMM_L4x16_SAVE

  293. 

  294. 	MY_ALIGN

  295. 

  296. LDGEMM_L4x16_SUB2:

  297. 

  298. 	KERNEL4x16_SUB1

  299. 

  300. 	addic.		L,	L,	-1

  301. 	bgt		LDGEMM_L4x16_SUB2

  302. 

  303. 	MY_ALIGN

  304. 

  305. LDGEMM_L4x16_SAVE:

  306. 

  307. 	SAVE4x16

  308. 

  309. 	addic.		I,	I,	-1

  310. 	bgt+		LDGEMM_L4x16_BEGIN

  311. 

  312. LDGEMM_L4x16_END:

  313. 

  314. LDGEMM_L4x8_BEGIN:

  315. 

  316. 	andi.		T2,	M,	15

  317. 	ble		LDGEMM_L4x1_END

  318. 

  319. 	andi.		T1,	M,	8

  320. 	ble		LDGEMM_L4x8_END

  321. 	mr		BO,	B

  322. 	srawi.		L,	K,	3

  323. 	ble		LDGEMM_L4x8_SUB0

  324. 	cmpwi		cr0,	L,	1

  325. 	ble		LDGEMM_L4x8_SUB4

  326. 

  327. LDGEMM_L4x8_LOOP_START:

  328. 

  329. 	dcbt	AO,	PRE

  330. 	LOAD4x8_1

  331. 	KERNEL4x8_I1

  332. 	dcbt	AO,	PRE

  333. 	KERNEL4x8_2

  334. 	KERNEL4x8_1

  335. 	dcbt	AO,	PRE

  336. 	KERNEL4x8_2

  337. 

  338. 	KERNEL4x8_1

  339. 	dcbt	AO,	PRE

  340. 	KERNEL4x8_2

  341. 	KERNEL4x8_1

  342. 	dcbt	AO,	PRE

  343. 	KERNEL4x8_2

  344. 

  345. 	addic.		L,	L,	-2

  346. 	ble		LDGEMM_L4x8_LOOP_END

  347. 

  348. 	MY_ALIGN

  349. 

  350. LDGEMM_L4x8_LOOP:

  351. 

  352. 	KERNEL4x8_1

  353. 	dcbt	AO,	PRE

  354. 	KERNEL4x8_2

  355. 	KERNEL4x8_1

  356. 	dcbt	AO,	PRE

  357. 	KERNEL4x8_2

  358. 

  359. 	KERNEL4x8_1

  360. 	dcbt	AO,	PRE

  361. 	KERNEL4x8_2

  362. 	KERNEL4x8_1

  363. 	dcbt	AO,	PRE

  364. 	KERNEL4x8_2

  365. 

  366. 	addic.		L,	L,	-1

  367. 	bgt		LDGEMM_L4x8_LOOP

  368. 

  369. LDGEMM_L4x8_LOOP_END:

  370. 

  371. 	KERNEL4x8_1

  372. 	KERNEL4x8_2

  373. 	KERNEL4x8_1

  374. 	KERNEL4x8_2

  375. 

  376. 	KERNEL4x8_1

  377. 	KERNEL4x8_2

  378. 	KERNEL4x8_1

  379. 	KERNEL4x8_E2

  380. 

  381. 	b		LDGEMM_L4x8_SUB1

  382. 

  383. LDGEMM_L4x8_SUB4:

  384. 

  385. 	KERNEL4x8_SUBI1

  386. 	KERNEL4x8_SUB1

  387. 	KERNEL4x8_SUB1

  388. 	KERNEL4x8_SUB1

  389. 

  390. 	KERNEL4x8_SUB1

  391. 	KERNEL4x8_SUB1

  392. 	KERNEL4x8_SUB1

  393. 	KERNEL4x8_SUB1

  394. 

  395. 	b		LDGEMM_L4x8_SUB1

  396. 

  397. LDGEMM_L4x8_SUB0:

  398. 

  399. 	andi.		L,	K,	7

  400. 

  401. 	KERNEL4x8_SUBI1

  402. 

  403. 	addic.		L,	L,	-1

  404. 	ble		LDGEMM_L4x8_SAVE

  405. 	b		LDGEMM_L4x8_SUB2

  406. 

  407. LDGEMM_L4x8_SUB1:

  408. 

  409. 	andi.		L,	K,	7

  410. 	ble		LDGEMM_L4x8_SAVE

  411. 

  412. LDGEMM_L4x8_SUB2:

  413. 

  414. 	KERNEL4x8_SUB1

  415. 

  416. 	addic.		L,	L,	-1

  417. 	bgt		LDGEMM_L4x8_SUB2

  418. 

  419. LDGEMM_L4x8_SAVE:

  420. 

  421. 	SAVE4x8

  422. 

  423. LDGEMM_L4x8_END:

  424. 

  425. LDGEMM_L4x4_BEGIN:

  426. 

  427. 

  428. 	andi.		T1,	M,	4

  429. 	ble		LDGEMM_L4x4_END

  430. 	mr		BO,	B

  431. 	srawi.		L,	K,	3

  432. 	ble		LDGEMM_L4x4_SUB0

  433. 	cmpwi		cr0,	L,	1

  434. 	ble		LDGEMM_L4x4_SUB4

  435. 

  436. LDGEMM_L4x4_LOOP_START:

  437. 

  438. 	dcbt	AO,	PRE

  439. 	LOAD4x4_1

  440. 	KERNEL4x4_I1

  441. 	KERNEL4x4_2

  442. 	KERNEL4x4_1

  443. 	dcbt	AO,	PRE

  444. 	KERNEL4x4_2

  445. 

  446. 	KERNEL4x4_1

  447. 	KERNEL4x4_2

  448. 	KERNEL4x4_1

  449. 	dcbt	AO,	PRE

  450. 	KERNEL4x4_2

  451. 

  452. 	addic.		L,	L,	-2

  453. 	ble		LDGEMM_L4x4_LOOP_END

  454. 

  455. 	MY_ALIGN

  456. 

  457. LDGEMM_L4x4_LOOP:

  458. 

  459. 	KERNEL4x4_1

  460. 	KERNEL4x4_2

  461. 	KERNEL4x4_1

  462. 	dcbt	AO,	PRE

  463. 	KERNEL4x4_2

  464. 

  465. 	KERNEL4x4_1

  466. 	KERNEL4x4_2

  467. 	KERNEL4x4_1

  468. 	dcbt	AO,	PRE

  469. 	KERNEL4x4_2

  470. 

  471. 	addic.		L,	L,	-1

  472. 	bgt		LDGEMM_L4x4_LOOP

  473. 

  474. LDGEMM_L4x4_LOOP_END:

  475. 

  476. 	KERNEL4x4_1

  477. 	KERNEL4x4_2

  478. 	KERNEL4x4_1

  479. 	KERNEL4x4_2

  480. 

  481. 	KERNEL4x4_1

  482. 	KERNEL4x4_2

  483. 	KERNEL4x4_1

  484. 	KERNEL4x4_E2

  485. 

  486. 	b		LDGEMM_L4x4_SUB1

  487. 

  488. LDGEMM_L4x4_SUB4:

  489. 

  490. 	KERNEL4x4_SUBI1

  491. 	KERNEL4x4_SUB1

  492. 	KERNEL4x4_SUB1

  493. 	KERNEL4x4_SUB1

  494. 

  495. 	KERNEL4x4_SUB1

  496. 	KERNEL4x4_SUB1

  497. 	KERNEL4x4_SUB1

  498. 	KERNEL4x4_SUB1

  499. 

  500. 	b		LDGEMM_L4x4_SUB1

  501. 

  502. LDGEMM_L4x4_SUB0:

  503. 

  504. 	andi.		L,	K,	7

  505. 

  506. 	KERNEL4x4_SUBI1

  507. 

  508. 	addic.		L,	L,	-1

  509. 	ble		LDGEMM_L4x4_SAVE

  510. 	b		LDGEMM_L4x4_SUB2

  511. 

  512. LDGEMM_L4x4_SUB1:

  513. 

  514. 	andi.		L,	K,	7

  515. 	ble		LDGEMM_L4x4_SAVE

  516. 

  517. LDGEMM_L4x4_SUB2:

  518. 

  519. 	KERNEL4x4_SUB1

  520. 

  521. 	addic.		L,	L,	-1

  522. 	bgt		LDGEMM_L4x4_SUB2

  523. 

  524. LDGEMM_L4x4_SAVE:

  525. 

  526. 	SAVE4x4

  527. 

  528. LDGEMM_L4x4_END:

  529. 

  530. LDGEMM_L4x2_BEGIN:

  531. 

  532. 

  533. 	andi.		T1,	M,	2

  534. 	ble		LDGEMM_L4x2_END

  535. 	mr		BO,	B

  536. 	srawi.		L,	K,	3

  537. 	ble		LDGEMM_L4x2_SUB0

  538. 	cmpwi		cr0,	L,	1

  539. 	ble		LDGEMM_L4x2_SUB4

  540. 

  541. LDGEMM_L4x2_LOOP_START:

  542. 

  543. 	LOAD4x2_1

  544. 	KERNEL4x2_I1

  545. 	KERNEL4x2_2

  546. 	KERNEL4x2_1

  547. 	KERNEL4x2_2

  548. 

  549. 	KERNEL4x2_1

  550. 	KERNEL4x2_2

  551. 	KERNEL4x2_1

  552. 	KERNEL4x2_2

  553. 

  554. 	addic.		L,	L,	-2

  555. 	ble		LDGEMM_L4x2_LOOP_END

  556. 

  557. 	MY_ALIGN

  558. 

  559. LDGEMM_L4x2_LOOP:

  560. 

  561. 	KERNEL4x2_1

  562. 	KERNEL4x2_2

  563. 	KERNEL4x2_1

  564. 	KERNEL4x2_2

  565. 

  566. 	KERNEL4x2_1

  567. 	KERNEL4x2_2

  568. 	KERNEL4x2_1

  569. 	KERNEL4x2_2

  570. 

  571. 	addic.		L,	L,	-1

  572. 	bgt		LDGEMM_L4x2_LOOP

  573. 

  574. LDGEMM_L4x2_LOOP_END:

  575. 

  576. 	KERNEL4x2_1

  577. 	KERNEL4x2_2

  578. 	KERNEL4x2_1

  579. 	KERNEL4x2_2

  580. 

  581. 	KERNEL4x2_1

  582. 	KERNEL4x2_2

  583. 	KERNEL4x2_1

  584. 	KERNEL4x2_E2

  585. 

  586. 	b		LDGEMM_L4x2_SUB1

  587. 

  588. LDGEMM_L4x2_SUB4:

  589. 

  590. 	KERNEL4x2_SUBI1

  591. 	KERNEL4x2_SUB1

  592. 	KERNEL4x2_SUB1

  593. 	KERNEL4x2_SUB1

  594. 

  595. 	KERNEL4x2_SUB1

  596. 	KERNEL4x2_SUB1

  597. 	KERNEL4x2_SUB1

  598. 	KERNEL4x2_SUB1

  599. 

  600. 	b		LDGEMM_L4x2_SUB1

  601. 

  602. LDGEMM_L4x2_SUB0:

  603. 

  604. 	andi.		L,	K,	7

  605. 

  606. 	KERNEL4x2_SUBI1

  607. 

  608. 	addic.		L,	L,	-1

  609. 	ble		LDGEMM_L4x2_SAVE

  610. 	b		LDGEMM_L4x2_SUB2

  611. 

  612. LDGEMM_L4x2_SUB1:

  613. 

  614. 	andi.		L,	K,	7

  615. 	ble		LDGEMM_L4x2_SAVE

  616. 

  617. LDGEMM_L4x2_SUB2:

  618. 

  619. 	KERNEL4x2_SUB1

  620. 

  621. 	addic.		L,	L,	-1

  622. 	bgt		LDGEMM_L4x2_SUB2

  623. 

  624. LDGEMM_L4x2_SAVE:

  625. 

  626. 	SAVE4x2

  627. 

  628. LDGEMM_L4x2_END:

  629. 

  630. LDGEMM_L4x1_BEGIN:

  631. 

  632. 

  633. 	andi.		T1,	M,	1

  634. 	ble		LDGEMM_L4x1_END

  635. 	mr		BO,	B

  636. 	srawi.		L,	K,	3

  637. 	ble		LDGEMM_L4x1_SUB0

  638. 	cmpwi		cr0,	L,	1

  639. 	ble		LDGEMM_L4x1_SUB4

  640. 

  641. LDGEMM_L4x1_LOOP_START:

  642. 

  643. 	LOAD4x1_1

  644. 	KERNEL4x1_I1

  645. 	KERNEL4x1_2

  646. 	KERNEL4x1_1

  647. 	KERNEL4x1_2

  648. 

  649. 	KERNEL4x1_1

  650. 	KERNEL4x1_2

  651. 	KERNEL4x1_1

  652. 	KERNEL4x1_2

  653. 

  654. 	addic.		L,	L,	-2

  655. 	ble		LDGEMM_L4x1_LOOP_END

  656. 

  657. 	MY_ALIGN

  658. 

  659. LDGEMM_L4x1_LOOP:

  660. 

  661. 	KERNEL4x1_1

  662. 	KERNEL4x1_2

  663. 	KERNEL4x1_1

  664. 	KERNEL4x1_2

  665. 

  666. 	KERNEL4x1_1

  667. 	KERNEL4x1_2

  668. 	KERNEL4x1_1

  669. 	KERNEL4x1_2

  670. 

  671. 	addic.		L,	L,	-1

  672. 	bgt		LDGEMM_L4x1_LOOP

  673. 

  674. LDGEMM_L4x1_LOOP_END:

  675. 

  676. 	KERNEL4x1_1

  677. 	KERNEL4x1_2

  678. 	KERNEL4x1_1

  679. 	KERNEL4x1_2

  680. 

  681. 	KERNEL4x1_1

  682. 	KERNEL4x1_2

  683. 	KERNEL4x1_1

  684. 	KERNEL4x1_E2

  685. 

  686. 	b		LDGEMM_L4x1_SUB1

  687. 

  688. LDGEMM_L4x1_SUB4:

  689. 

  690. 	KERNEL4x1_SUBI1

  691. 	KERNEL4x1_SUB1

  692. 	KERNEL4x1_SUB1

  693. 	KERNEL4x1_SUB1

  694. 

  695. 	KERNEL4x1_SUB1

  696. 	KERNEL4x1_SUB1

  697. 	KERNEL4x1_SUB1

  698. 	KERNEL4x1_SUB1

  699. 

  700. 	b		LDGEMM_L4x1_SUB1

  701. 

  702. LDGEMM_L4x1_SUB0:

  703. 

  704. 	andi.		L,	K,	7

  705. 

  706. 	KERNEL4x1_SUBI1

  707. 

  708. 	addic.		L,	L,	-1

  709. 	ble		LDGEMM_L4x1_SAVE

  710. 	b		LDGEMM_L4x1_SUB2

  711. 

  712. LDGEMM_L4x1_SUB1:

  713. 

  714. 	andi.		L,	K,	7

  715. 	ble		LDGEMM_L4x1_SAVE

  716. 

  717. LDGEMM_L4x1_SUB2:

  718. 

  719. 	KERNEL4x1_SUB1

  720. 

  721. 	addic.		L,	L,	-1

  722. 	bgt		LDGEMM_L4x1_SUB2

  723. 

  724. LDGEMM_L4x1_SAVE:

  725. 

  726. 	SAVE4x1

  727. 

  728. LDGEMM_L4x1_END:

  729. 

  730. 	slwi		T1,	K,	5

  731. 	add		B,	B,	T1

  732. 

  733. 	addic.		J,	J,	-1

  734. 	bgt		LDGEMM_L4_BEGIN

  735. 

  736. 	andi.		T2,	N,	3

  737. 	ble		.L999

  738. 

  739. LDGEMM_L4_END:

  740. 

  741. 	b		LDGEMM_L2_BEGIN

  742. 

  743. .L999_H1:

  744. 

  745. 	b		.L999

  746. 

  747. LDGEMM_L2_BEGIN:

  748. 

  749. 	andi.		T1,	N,	2

  750. 	ble		LDGEMM_L2_END

  751. 	mr		CO,	C

  752. 	mr		AO,	A

  753. 	slwi		T1,	LDC	,	1

  754. 	add		C,	C,	T1

  755. 	srawi.		I,	M,	4

  756. 	ble		LDGEMM_L2x16_END

  757. 

  758. LDGEMM_L2x16_BEGIN:

  759. 

  760. 

  761. 	mr		BO,	B

  762. 	srawi.		L,	K,	3

  763. 	ble		LDGEMM_L2x16_SUB0

  764. 	cmpwi		cr0,	L,	1

  765. 	ble		LDGEMM_L2x16_SUB4

  766. 

  767. LDGEMM_L2x16_LOOP_START:

  768. 

  769. 	dcbt		AO,	PRE

  770. 	LOAD2x16_1

  771. 	dcbt		AO,	PRE

  772. 	KERNEL2x16_I1

  773. 	dcbt		AO,	PRE

  774. 	KERNEL2x16_2

  775. 	dcbt		AO,	PRE

  776. 	KERNEL2x16_1

  777. 	dcbt		AO,	PRE

  778. 	KERNEL2x16_2

  779. 

  780. 	dcbt		AO,	PRE

  781. 	KERNEL2x16_1

  782. 	dcbt		AO,	PRE

  783. 	KERNEL2x16_2

  784. 	dcbt		AO,	PRE

  785. 	KERNEL2x16_1

  786. 	dcbt		AO,	PRE

  787. 	KERNEL2x16_2

  788. 

  789. 	addic.		L,	L,	-2

  790. 	ble		LDGEMM_L2x16_LOOP_END

  791. 

  792. 	MY_ALIGN

  793. 

  794. LDGEMM_L2x16_LOOP:

  795. 

  796. 	dcbt		AO,	PRE

  797. 	KERNEL2x16_1

  798. 	dcbt		AO,	PRE

  799. 	KERNEL2x16_2

  800. 	dcbt		AO,	PRE

  801. 	KERNEL2x16_1

  802. 	dcbt		AO,	PRE

  803. 	KERNEL2x16_2

  804. 

  805. 	dcbt		AO,	PRE

  806. 	KERNEL2x16_1

  807. 	dcbt		AO,	PRE

  808. 	KERNEL2x16_2

  809. 	dcbt		AO,	PRE

  810. 	KERNEL2x16_1

  811. 	dcbt		AO,	PRE

  812. 	KERNEL2x16_2

  813. 

  814. 	addic.		L,	L,	-1

  815. 	bgt		LDGEMM_L2x16_LOOP

  816. 

  817. LDGEMM_L2x16_LOOP_END:

  818. 

  819. 	dcbt		AO,	PRE

  820. 	KERNEL2x16_1

  821. 	dcbt		AO,	PRE

  822. 	KERNEL2x16_2

  823. 	dcbt		AO,	PRE

  824. 	KERNEL2x16_1

  825. 	dcbt		AO,	PRE

  826. 	KERNEL2x16_2

  827. 

  828. 	dcbt		AO,	PRE

  829. 	KERNEL2x16_1

  830. 	dcbt		AO,	PRE

  831. 	KERNEL2x16_2

  832. 	dcbt		AO,	PRE

  833. 	KERNEL2x16_1

  834. 	KERNEL2x16_E2

  835. 

  836. 	b		LDGEMM_L2x16_SUB1

  837. 

  838. LDGEMM_L2x16_SUB4:

  839. 

  840. 	dcbt		AO,	PRE

  841. 	KERNEL2x16_SUBI1

  842. 	dcbt		AO,	PRE

  843. 	KERNEL2x16_SUB1

  844. 	dcbt		AO,	PRE

  845. 	KERNEL2x16_SUB1

  846. 	dcbt		AO,	PRE

  847. 	KERNEL2x16_SUB1

  848. 

  849. 	KERNEL2x16_SUB1

  850. 	KERNEL2x16_SUB1

  851. 	KERNEL2x16_SUB1

  852. 	KERNEL2x16_SUB1

  853. 

  854. 	b		LDGEMM_L2x16_SUB1

  855. 

  856. LDGEMM_L2x16_SUB0:

  857. 

  858. 	andi.		L,	K,	7

  859. 

  860. 	KERNEL2x16_SUBI1

  861. 

  862. 	addic.		L,	L,	-1

  863. 	ble		LDGEMM_L2x16_SAVE

  864. 	b		LDGEMM_L2x16_SUB2

  865. 

  866. LDGEMM_L2x16_SUB1:

  867. 

  868. 	andi.		L,	K,	7

  869. 	ble		LDGEMM_L2x16_SAVE

  870. 

  871. LDGEMM_L2x16_SUB2:

  872. 

  873. 	KERNEL2x16_SUB1

  874. 

  875. 	addic.		L,	L,	-1

  876. 	bgt		LDGEMM_L2x16_SUB2

  877. 

  878. LDGEMM_L2x16_SAVE:

  879. 

  880. 	SAVE2x16

  881. 

  882. 	addic.		I,	I,	-1

  883. 	bgt		LDGEMM_L2x16_BEGIN

  884. 

  885. LDGEMM_L2x16_END:

  886. 

  887. LDGEMM_L2x8_BEGIN:

  888. 

  889. 	andi.		T2,	M,	15

  890. 	ble		LDGEMM_L2x1_END

  891. 

  892. 	andi.		T1,	M,	8

  893. 	ble		LDGEMM_L2x8_END

  894. 	mr		BO,	B

  895. 	srawi.		L,	K,	3

  896. 	ble		LDGEMM_L2x8_SUB0

  897. 	cmpwi		cr0,	L,	1

  898. 	ble		LDGEMM_L2x8_SUB4

  899. 

  900. LDGEMM_L2x8_LOOP_START:

  901. 

  902. 	dcbt	AO,	PRE

  903. 	LOAD2x8_1

  904. 	KERNEL2x8_I1

  905. 	dcbt	AO,	PRE

  906. 	KERNEL2x8_2

  907. 	KERNEL2x8_1

  908. 	dcbt	AO,	PRE

  909. 	KERNEL2x8_2

  910. 

  911. 	KERNEL2x8_1

  912. 	dcbt	AO,	PRE

  913. 	KERNEL2x8_2

  914. 	KERNEL2x8_1

  915. 	dcbt	AO,	PRE

  916. 	KERNEL2x8_2

  917. 

  918. 	addic.		L,	L,	-2

  919. 	ble		LDGEMM_L2x8_LOOP_END

  920. 

  921. 	MY_ALIGN

  922. 

  923. LDGEMM_L2x8_LOOP:

  924. 

  925. 	KERNEL2x8_1

  926. 	dcbt	AO,	PRE

  927. 	KERNEL2x8_2

  928. 	KERNEL2x8_1

  929. 	dcbt	AO,	PRE

  930. 	KERNEL2x8_2

  931. 

  932. 	KERNEL2x8_1

  933. 	dcbt	AO,	PRE

  934. 	KERNEL2x8_2

  935. 	KERNEL2x8_1

  936. 	dcbt	AO,	PRE

  937. 	KERNEL2x8_2

  938. 

  939. 	addic.		L,	L,	-1

  940. 	bgt		LDGEMM_L2x8_LOOP

  941. 

  942. LDGEMM_L2x8_LOOP_END:

  943. 

  944. 	KERNEL2x8_1

  945. 	KERNEL2x8_2

  946. 	KERNEL2x8_1

  947. 	KERNEL2x8_2

  948. 

  949. 	KERNEL2x8_1

  950. 	KERNEL2x8_2

  951. 	KERNEL2x8_1

  952. 	KERNEL2x8_E2

  953. 

  954. 	b		LDGEMM_L2x8_SUB1

  955. 

  956. LDGEMM_L2x8_SUB4:

  957. 

  958. 	KERNEL2x8_SUBI1

  959. 	KERNEL2x8_SUB1

  960. 	KERNEL2x8_SUB1

  961. 	KERNEL2x8_SUB1

  962. 

  963. 	KERNEL2x8_SUB1

  964. 	KERNEL2x8_SUB1

  965. 	KERNEL2x8_SUB1

  966. 	KERNEL2x8_SUB1

  967. 

  968. 	b		LDGEMM_L2x8_SUB1

  969. 

  970. LDGEMM_L2x8_SUB0:

  971. 

  972. 	andi.		L,	K,	7

  973. 

  974. 	KERNEL2x8_SUBI1

  975. 

  976. 	addic.		L,	L,	-1

  977. 	ble		LDGEMM_L2x8_SAVE

  978. 	b		LDGEMM_L2x8_SUB2

  979. 

  980. LDGEMM_L2x8_SUB1:

  981. 

  982. 	andi.		L,	K,	7

  983. 	ble		LDGEMM_L2x8_SAVE

  984. 

  985. LDGEMM_L2x8_SUB2:

  986. 

  987. 	KERNEL2x8_SUB1

  988. 

  989. 	addic.		L,	L,	-1

  990. 	bgt		LDGEMM_L2x8_SUB2

  991. 

  992. LDGEMM_L2x8_SAVE:

  993. 

  994. 	SAVE2x8

  995. 

  996. LDGEMM_L2x8_END:

  997. 

  998. LDGEMM_L2x4_BEGIN:

  999. 

  1000. 

  1001. 	andi.		T1,	M,	4

  1002. 	ble		LDGEMM_L2x4_END

  1003. 	mr		BO,	B

  1004. 	srawi.		L,	K,	3

  1005. 	ble		LDGEMM_L2x4_SUB0

  1006. 	cmpwi		cr0,	L,	1

  1007. 	ble		LDGEMM_L2x4_SUB4

  1008. 

  1009. LDGEMM_L2x4_LOOP_START:

  1010. 

  1011. 	LOAD2x4_1

  1012. 	KERNEL2x4_I1

  1013. 	KERNEL2x4_2

  1014. 	KERNEL2x4_1

  1015. 	KERNEL2x4_2

  1016. 

  1017. 	KERNEL2x4_1

  1018. 	KERNEL2x4_2

  1019. 	KERNEL2x4_1

  1020. 	KERNEL2x4_2

  1021. 

  1022. 	addic.		L,	L,	-2

  1023. 	ble		LDGEMM_L2x4_LOOP_END

  1024. 

  1025. 	MY_ALIGN

  1026. 

  1027. LDGEMM_L2x4_LOOP:

  1028. 

  1029. 	KERNEL2x4_1

  1030. 	KERNEL2x4_2

  1031. 	KERNEL2x4_1

  1032. 	KERNEL2x4_2

  1033. 

  1034. 	KERNEL2x4_1

  1035. 	KERNEL2x4_2

  1036. 	KERNEL2x4_1

  1037. 	KERNEL2x4_2

  1038. 

  1039. 	addic.		L,	L,	-1

  1040. 	bgt		LDGEMM_L2x4_LOOP

  1041. 

  1042. LDGEMM_L2x4_LOOP_END:

  1043. 

  1044. 	KERNEL2x4_1

  1045. 	KERNEL2x4_2

  1046. 	KERNEL2x4_1

  1047. 	KERNEL2x4_2

  1048. 

  1049. 	KERNEL2x4_1

  1050. 	KERNEL2x4_2

  1051. 	KERNEL2x4_1

  1052. 	KERNEL2x4_E2

  1053. 

  1054. 	b		LDGEMM_L2x4_SUB1

  1055. 

  1056. LDGEMM_L2x4_SUB4:

  1057. 

  1058. 	KERNEL2x4_SUBI1

  1059. 	KERNEL2x4_SUB1

  1060. 	KERNEL2x4_SUB1

  1061. 	KERNEL2x4_SUB1

  1062. 

  1063. 	KERNEL2x4_SUB1

  1064. 	KERNEL2x4_SUB1

  1065. 	KERNEL2x4_SUB1

  1066. 	KERNEL2x4_SUB1

  1067. 

  1068. 	b		LDGEMM_L2x4_SUB1

  1069. 

  1070. LDGEMM_L2x4_SUB0:

  1071. 

  1072. 	andi.		L,	K,	7

  1073. 

  1074. 	KERNEL2x4_SUBI1

  1075. 

  1076. 	addic.		L,	L,	-1

  1077. 	ble		LDGEMM_L2x4_SAVE

  1078. 	b		LDGEMM_L2x4_SUB2

  1079. 

  1080. LDGEMM_L2x4_SUB1:

  1081. 

  1082. 	andi.		L,	K,	7

  1083. 	ble		LDGEMM_L2x4_SAVE

  1084. 

  1085. LDGEMM_L2x4_SUB2:

  1086. 

  1087. 	KERNEL2x4_SUB1

  1088. 

  1089. 	addic.		L,	L,	-1

  1090. 	bgt		LDGEMM_L2x4_SUB2

  1091. 

  1092. LDGEMM_L2x4_SAVE:

  1093. 

  1094. 	SAVE2x4

  1095. 

  1096. LDGEMM_L2x4_END:

  1097. 

  1098. LDGEMM_L2x2_BEGIN:

  1099. 

  1100. 

  1101. 	andi.		T1,	M,	2

  1102. 	ble		LDGEMM_L2x2_END

  1103. 	mr		BO,	B

  1104. 	srawi.		L,	K,	3

  1105. 	ble		LDGEMM_L2x2_SUB0

  1106. 	cmpwi		cr0,	L,	1

  1107. 	ble		LDGEMM_L2x2_SUB4

  1108. 

  1109. LDGEMM_L2x2_LOOP_START:

  1110. 

  1111. 	LOAD2x2_1

  1112. 	KERNEL2x2_I1

  1113. 	KERNEL2x2_2

  1114. 	KERNEL2x2_1

  1115. 	KERNEL2x2_2

  1116. 

  1117. 	KERNEL2x2_1

  1118. 	KERNEL2x2_2

  1119. 	KERNEL2x2_1

  1120. 	KERNEL2x2_2

  1121. 

  1122. 	addic.		L,	L,	-2

  1123. 	ble		LDGEMM_L2x2_LOOP_END

  1124. 

  1125. 	MY_ALIGN

  1126. 

  1127. LDGEMM_L2x2_LOOP:

  1128. 

  1129. 	KERNEL2x2_1

  1130. 	KERNEL2x2_2

  1131. 	KERNEL2x2_1

  1132. 	KERNEL2x2_2

  1133. 

  1134. 	KERNEL2x2_1

  1135. 	KERNEL2x2_2

  1136. 	KERNEL2x2_1

  1137. 	KERNEL2x2_2

  1138. 

  1139. 	addic.		L,	L,	-1

  1140. 	bgt		LDGEMM_L2x2_LOOP

  1141. 

  1142. LDGEMM_L2x2_LOOP_END:

  1143. 

  1144. 	KERNEL2x2_1

  1145. 	KERNEL2x2_2

  1146. 	KERNEL2x2_1

  1147. 	KERNEL2x2_2

  1148. 

  1149. 	KERNEL2x2_1

  1150. 	KERNEL2x2_2

  1151. 	KERNEL2x2_1

  1152. 	KERNEL2x2_E2

  1153. 

  1154. 	b		LDGEMM_L2x2_SUB1

  1155. 

  1156. LDGEMM_L2x2_SUB4:

  1157. 

  1158. 	KERNEL2x2_SUBI1

  1159. 	KERNEL2x2_SUB1

  1160. 	KERNEL2x2_SUB1

  1161. 	KERNEL2x2_SUB1

  1162. 

  1163. 	KERNEL2x2_SUB1

  1164. 	KERNEL2x2_SUB1

  1165. 	KERNEL2x2_SUB1

  1166. 	KERNEL2x2_SUB1

  1167. 

  1168. 	b		LDGEMM_L2x2_SUB1

  1169. 

  1170. LDGEMM_L2x2_SUB0:

  1171. 

  1172. 	andi.		L,	K,	7

  1173. 

  1174. 	KERNEL2x2_SUBI1

  1175. 

  1176. 	addic.		L,	L,	-1

  1177. 	ble		LDGEMM_L2x2_SAVE

  1178. 	b		LDGEMM_L2x2_SUB2

  1179. 

  1180. LDGEMM_L2x2_SUB1:

  1181. 

  1182. 	andi.		L,	K,	7

  1183. 	ble		LDGEMM_L2x2_SAVE

  1184. 

  1185. LDGEMM_L2x2_SUB2:

  1186. 

  1187. 	KERNEL2x2_SUB1

  1188. 

  1189. 	addic.		L,	L,	-1

  1190. 	bgt		LDGEMM_L2x2_SUB2

  1191. 

  1192. LDGEMM_L2x2_SAVE:

  1193. 

  1194. 	SAVE2x2

  1195. 

  1196. LDGEMM_L2x2_END:

  1197. 

  1198. LDGEMM_L2x1_BEGIN:

  1199. 

  1200. 

  1201. 	andi.		T1,	M,	1

  1202. 	ble		LDGEMM_L2x1_END

  1203. 	mr		BO,	B

  1204. 	srawi.		L,	K,	3

  1205. 	ble		LDGEMM_L2x1_SUB0

  1206. 	cmpwi		cr0,	L,	1

  1207. 	ble		LDGEMM_L2x1_SUB4

  1208. 

  1209. LDGEMM_L2x1_LOOP_START:

  1210. 

  1211. 	LOAD2x1_1

  1212. 	KERNEL2x1_I1

  1213. 	KERNEL2x1_2

  1214. 	KERNEL2x1_1

  1215. 	KERNEL2x1_2

  1216. 

  1217. 	KERNEL2x1_1

  1218. 	KERNEL2x1_2

  1219. 	KERNEL2x1_1

  1220. 	KERNEL2x1_2

  1221. 

  1222. 	addic.		L,	L,	-2

  1223. 	ble		LDGEMM_L2x1_LOOP_END

  1224. 

  1225. 	MY_ALIGN

  1226. 

  1227. LDGEMM_L2x1_LOOP:

  1228. 

  1229. 	KERNEL2x1_1

  1230. 	KERNEL2x1_2

  1231. 	KERNEL2x1_1

  1232. 	KERNEL2x1_2

  1233. 

  1234. 	KERNEL2x1_1

  1235. 	KERNEL2x1_2

  1236. 	KERNEL2x1_1

  1237. 	KERNEL2x1_2

  1238. 

  1239. 	addic.		L,	L,	-1

  1240. 	bgt		LDGEMM_L2x1_LOOP

  1241. 

  1242. LDGEMM_L2x1_LOOP_END:

  1243. 

  1244. 	KERNEL2x1_1

  1245. 	KERNEL2x1_2

  1246. 	KERNEL2x1_1

  1247. 	KERNEL2x1_2

  1248. 

  1249. 	KERNEL2x1_1

  1250. 	KERNEL2x1_2

  1251. 	KERNEL2x1_1

  1252. 	KERNEL2x1_E2

  1253. 

  1254. 	b		LDGEMM_L2x1_SUB1

  1255. 

  1256. LDGEMM_L2x1_SUB4:

  1257. 

  1258. 	KERNEL2x1_SUBI1

  1259. 	KERNEL2x1_SUB1

  1260. 	KERNEL2x1_SUB1

  1261. 	KERNEL2x1_SUB1

  1262. 

  1263. 	KERNEL2x1_SUB1

  1264. 	KERNEL2x1_SUB1

  1265. 	KERNEL2x1_SUB1

  1266. 	KERNEL2x1_SUB1

  1267. 

  1268. 	b		LDGEMM_L2x1_SUB1

  1269. 

  1270. LDGEMM_L2x1_SUB0:

  1271. 

  1272. 	andi.		L,	K,	7

  1273. 

  1274. 	KERNEL2x1_SUBI1

  1275. 

  1276. 	addic.		L,	L,	-1

  1277. 	ble		LDGEMM_L2x1_SAVE

  1278. 	b		LDGEMM_L2x1_SUB2

  1279. 

  1280. LDGEMM_L2x1_SUB1:

  1281. 

  1282. 	andi.		L,	K,	7

  1283. 	ble		LDGEMM_L2x1_SAVE

  1284. 

  1285. LDGEMM_L2x1_SUB2:

  1286. 

  1287. 	KERNEL2x1_SUB1

  1288. 

  1289. 	addic.		L,	L,	-1

  1290. 	bgt		LDGEMM_L2x1_SUB2

  1291. 

  1292. LDGEMM_L2x1_SAVE:

  1293. 

  1294. 	SAVE2x1

  1295. 

  1296. LDGEMM_L2x1_END:

  1297. 

  1298. 	slwi		T1,	K,	4

  1299. 	add		B,	B,	T1

  1300. 

  1301. LDGEMM_L2_END:

  1302. LDGEMM_L1_BEGIN:

  1303. 

  1304. 	andi.		T1,	N,	1

  1305. 	ble		LDGEMM_L1_END

  1306. 	mr		CO,	C

  1307. 	mr		AO,	A

  1308. 	srawi.		I,	M,	4

  1309. 	ble		LDGEMM_L1x16_END

  1310. 

  1311. LDGEMM_L1x16_BEGIN:

  1312. 

  1313. 

  1314. 	mr		BO,	B

  1315. 	srawi.		L,	K,	3

  1316. 	ble		LDGEMM_L1x16_SUB0

  1317. 	cmpwi		cr0,	L,	1

  1318. 	ble		LDGEMM_L1x16_SUB4

  1319. 

  1320. LDGEMM_L1x16_LOOP_START:

  1321. 

  1322. 	dcbt		AO,	PRE

  1323. 	LOAD1x16_1

  1324. 	dcbt		AO,	PRE

  1325. 	KERNEL1x16_I1

  1326. 	dcbt		AO,	PRE

  1327. 	KERNEL1x16_2

  1328. 	dcbt		AO,	PRE

  1329. 	KERNEL1x16_1

  1330. 	dcbt		AO,	PRE

  1331. 	KERNEL1x16_2

  1332. 

  1333. 	dcbt		AO,	PRE

  1334. 	KERNEL1x16_1

  1335. 	dcbt		AO,	PRE

  1336. 	KERNEL1x16_2

  1337. 	dcbt		AO,	PRE

  1338. 	KERNEL1x16_1

  1339. 	dcbt		AO,	PRE

  1340. 	KERNEL1x16_2

  1341. 

  1342. 	addic.		L,	L,	-2

  1343. 	ble		LDGEMM_L1x16_LOOP_END

  1344. 

  1345. 	MY_ALIGN

  1346. 

  1347. LDGEMM_L1x16_LOOP:

  1348. 

  1349. 	dcbt		AO,	PRE

  1350. 	KERNEL1x16_1

  1351. 	dcbt		AO,	PRE

  1352. 	KERNEL1x16_2

  1353. 	dcbt		AO,	PRE

  1354. 	KERNEL1x16_1

  1355. 	dcbt		AO,	PRE

  1356. 	KERNEL1x16_2

  1357. 

  1358. 	dcbt		AO,	PRE

  1359. 	KERNEL1x16_1

  1360. 	dcbt		AO,	PRE

  1361. 	KERNEL1x16_2

  1362. 	dcbt		AO,	PRE

  1363. 	KERNEL1x16_1

  1364. 	dcbt		AO,	PRE

  1365. 	KERNEL1x16_2

  1366. 

  1367. 	addic.		L,	L,	-1

  1368. 	bgt		LDGEMM_L1x16_LOOP

  1369. 

  1370. LDGEMM_L1x16_LOOP_END:

  1371. 

  1372. 	dcbt		AO,	PRE

  1373. 	KERNEL1x16_1

  1374. 	dcbt		AO,	PRE

  1375. 	KERNEL1x16_2

  1376. 	dcbt		AO,	PRE

  1377. 	KERNEL1x16_1

  1378. 	dcbt		AO,	PRE

  1379. 	KERNEL1x16_2

  1380. 

  1381. 	dcbt		AO,	PRE

  1382. 	KERNEL1x16_1

  1383. 	dcbt		AO,	PRE

  1384. 	KERNEL1x16_2

  1385. 	dcbt		AO,	PRE

  1386. 	KERNEL1x16_1

  1387. 	KERNEL1x16_E2

  1388. 

  1389. 	b		LDGEMM_L1x16_SUB1

  1390. 

  1391. LDGEMM_L1x16_SUB4:

  1392. 

  1393. 	dcbt		AO,	PRE

  1394. 	KERNEL1x16_SUBI1

  1395. 	dcbt		AO,	PRE

  1396. 	KERNEL1x16_SUB1

  1397. 	dcbt		AO,	PRE

  1398. 	KERNEL1x16_SUB1

  1399. 	dcbt		AO,	PRE

  1400. 	KERNEL1x16_SUB1

  1401. 

  1402. 	KERNEL1x16_SUB1

  1403. 	KERNEL1x16_SUB1

  1404. 	KERNEL1x16_SUB1

  1405. 	KERNEL1x16_SUB1

  1406. 

  1407. 	b		LDGEMM_L1x16_SUB1

  1408. 

  1409. LDGEMM_L1x16_SUB0:

  1410. 

  1411. 	andi.		L,	K,	7

  1412. 

  1413. 	KERNEL1x16_SUBI1

  1414. 

  1415. 	addic.		L,	L,	-1

  1416. 	ble		LDGEMM_L1x16_SAVE

  1417. 	b		LDGEMM_L1x16_SUB2

  1418. 

  1419. LDGEMM_L1x16_SUB1:

  1420. 

  1421. 	andi.		L,	K,	7

  1422. 	ble		LDGEMM_L1x16_SAVE

  1423. 

  1424. LDGEMM_L1x16_SUB2:

  1425. 

  1426. 	KERNEL1x16_SUB1

  1427. 

  1428. 	addic.		L,	L,	-1

  1429. 	bgt		LDGEMM_L1x16_SUB2

  1430. 

  1431. LDGEMM_L1x16_SAVE:

  1432. 

  1433. 	SAVE1x16

  1434. 

  1435. 	addic.		I,	I,	-1

  1436. 	bgt		LDGEMM_L1x16_BEGIN

  1437. 

  1438. LDGEMM_L1x16_END:

  1439. 

  1440. LDGEMM_L1x8_BEGIN:

  1441. 

  1442. 	andi.		T2,	M,	15

  1443. 	ble		LDGEMM_L1x1_END

  1444. 

  1445. 	andi.		T1,	M,	8

  1446. 	ble		LDGEMM_L1x8_END

  1447. 	mr		BO,	B

  1448. 	srawi.		L,	K,	3

  1449. 	ble		LDGEMM_L1x8_SUB0

  1450. 	cmpwi		cr0,	L,	1

  1451. 	ble		LDGEMM_L1x8_SUB4

  1452. 

  1453. LDGEMM_L1x8_LOOP_START:

  1454. 

  1455. 	dcbt	AO,	PRE

  1456. 	LOAD1x8_1

  1457. 	KERNEL1x8_I1

  1458. 	dcbt	AO,	PRE

  1459. 	KERNEL1x8_2

  1460. 	KERNEL1x8_1

  1461. 	dcbt	AO,	PRE

  1462. 	KERNEL1x8_2

  1463. 

  1464. 	KERNEL1x8_1

  1465. 	dcbt	AO,	PRE

  1466. 	KERNEL1x8_2

  1467. 	KERNEL1x8_1

  1468. 	dcbt	AO,	PRE

  1469. 	KERNEL1x8_2

  1470. 

  1471. 	addic.		L,	L,	-2

  1472. 	ble		LDGEMM_L1x8_LOOP_END

  1473. 

  1474. 	MY_ALIGN

  1475. 

  1476. LDGEMM_L1x8_LOOP:

  1477. 

  1478. 	KERNEL1x8_1

  1479. 	dcbt	AO,	PRE

  1480. 	KERNEL1x8_2

  1481. 	KERNEL1x8_1

  1482. 	dcbt	AO,	PRE

  1483. 	KERNEL1x8_2

  1484. 

  1485. 	KERNEL1x8_1

  1486. 	dcbt	AO,	PRE

  1487. 	KERNEL1x8_2

  1488. 	KERNEL1x8_1

  1489. 	dcbt	AO,	PRE

  1490. 	KERNEL1x8_2

  1491. 

  1492. 	addic.		L,	L,	-1

  1493. 	bgt		LDGEMM_L1x8_LOOP

  1494. 

  1495. LDGEMM_L1x8_LOOP_END:

  1496. 

  1497. 	KERNEL1x8_1

  1498. 	KERNEL1x8_2

  1499. 	KERNEL1x8_1

  1500. 	KERNEL1x8_2

  1501. 

  1502. 	KERNEL1x8_1

  1503. 	KERNEL1x8_2

  1504. 	KERNEL1x8_1

  1505. 	KERNEL1x8_E2

  1506. 

  1507. 	b		LDGEMM_L1x8_SUB1

  1508. 

  1509. LDGEMM_L1x8_SUB4:

  1510. 

  1511. 	KERNEL1x8_SUBI1

  1512. 	KERNEL1x8_SUB1

  1513. 	KERNEL1x8_SUB1

  1514. 	KERNEL1x8_SUB1

  1515. 

  1516. 	KERNEL1x8_SUB1

  1517. 	KERNEL1x8_SUB1

  1518. 	KERNEL1x8_SUB1

  1519. 	KERNEL1x8_SUB1

  1520. 

  1521. 	b		LDGEMM_L1x8_SUB1

  1522. 

  1523. LDGEMM_L1x8_SUB0:

  1524. 

  1525. 	andi.		L,	K,	7

  1526. 

  1527. 	KERNEL1x8_SUBI1

  1528. 

  1529. 	addic.		L,	L,	-1

  1530. 	ble		LDGEMM_L1x8_SAVE

  1531. 	b		LDGEMM_L1x8_SUB2

  1532. 

  1533. LDGEMM_L1x8_SUB1:

  1534. 

  1535. 	andi.		L,	K,	7

  1536. 	ble		LDGEMM_L1x8_SAVE

  1537. 

  1538. LDGEMM_L1x8_SUB2:

  1539. 

  1540. 	KERNEL1x8_SUB1

  1541. 

  1542. 	addic.		L,	L,	-1

  1543. 	bgt		LDGEMM_L1x8_SUB2

  1544. 

  1545. LDGEMM_L1x8_SAVE:

  1546. 

  1547. 	SAVE1x8

  1548. 

  1549. LDGEMM_L1x8_END:

  1550. 

  1551. LDGEMM_L1x4_BEGIN:

  1552. 

  1553. 

  1554. 	andi.		T1,	M,	4

  1555. 	ble		LDGEMM_L1x4_END

  1556. 	mr		BO,	B

  1557. 	srawi.		L,	K,	3

  1558. 	ble		LDGEMM_L1x4_SUB0

  1559. 	cmpwi		cr0,	L,	1

  1560. 	ble		LDGEMM_L1x4_SUB4

  1561. 

  1562. LDGEMM_L1x4_LOOP_START:

  1563. 

  1564. 	LOAD1x4_1

  1565. 	KERNEL1x4_I1

  1566. 	KERNEL1x4_2

  1567. 	KERNEL1x4_1

  1568. 	KERNEL1x4_2

  1569. 

  1570. 	KERNEL1x4_1

  1571. 	KERNEL1x4_2

  1572. 	KERNEL1x4_1

  1573. 	KERNEL1x4_2

  1574. 

  1575. 	addic.		L,	L,	-2

  1576. 	ble		LDGEMM_L1x4_LOOP_END

  1577. 

  1578. 	MY_ALIGN

  1579. 

  1580. LDGEMM_L1x4_LOOP:

  1581. 

  1582. 	KERNEL1x4_1

  1583. 	KERNEL1x4_2

  1584. 	KERNEL1x4_1

  1585. 	KERNEL1x4_2

  1586. 

  1587. 	KERNEL1x4_1

  1588. 	KERNEL1x4_2

  1589. 	KERNEL1x4_1

  1590. 	KERNEL1x4_2

  1591. 

  1592. 	addic.		L,	L,	-1

  1593. 	bgt		LDGEMM_L1x4_LOOP

  1594. 

  1595. LDGEMM_L1x4_LOOP_END:

  1596. 

  1597. 	KERNEL1x4_1

  1598. 	KERNEL1x4_2

  1599. 	KERNEL1x4_1

  1600. 	KERNEL1x4_2

  1601. 

  1602. 	KERNEL1x4_1

  1603. 	KERNEL1x4_2

  1604. 	KERNEL1x4_1

  1605. 	KERNEL1x4_E2

  1606. 

  1607. 	b		LDGEMM_L1x4_SUB1

  1608. 

  1609. LDGEMM_L1x4_SUB4:

  1610. 

  1611. 	KERNEL1x4_SUBI1

  1612. 	KERNEL1x4_SUB1

  1613. 	KERNEL1x4_SUB1

  1614. 	KERNEL1x4_SUB1

  1615. 

  1616. 	KERNEL1x4_SUB1

  1617. 	KERNEL1x4_SUB1

  1618. 	KERNEL1x4_SUB1

  1619. 	KERNEL1x4_SUB1

  1620. 

  1621. 	b		LDGEMM_L1x4_SUB1

  1622. 

  1623. LDGEMM_L1x4_SUB0:

  1624. 

  1625. 	andi.		L,	K,	7

  1626. 

  1627. 	KERNEL1x4_SUBI1

  1628. 

  1629. 	addic.		L,	L,	-1

  1630. 	ble		LDGEMM_L1x4_SAVE

  1631. 	b		LDGEMM_L1x4_SUB2

  1632. 

  1633. LDGEMM_L1x4_SUB1:

  1634. 

  1635. 	andi.		L,	K,	7

  1636. 	ble		LDGEMM_L1x4_SAVE

  1637. 

  1638. LDGEMM_L1x4_SUB2:

  1639. 

  1640. 	KERNEL1x4_SUB1

  1641. 

  1642. 	addic.		L,	L,	-1

  1643. 	bgt		LDGEMM_L1x4_SUB2

  1644. 

  1645. LDGEMM_L1x4_SAVE:

  1646. 

  1647. 	SAVE1x4

  1648. 

  1649. LDGEMM_L1x4_END:

  1650. 

  1651. LDGEMM_L1x2_BEGIN:

  1652. 

  1653. 

  1654. 	andi.		T1,	M,	2

  1655. 	ble		LDGEMM_L1x2_END

  1656. 	mr		BO,	B

  1657. 	srawi.		L,	K,	3

  1658. 	ble		LDGEMM_L1x2_SUB0

  1659. 	cmpwi		cr0,	L,	1

  1660. 	ble		LDGEMM_L1x2_SUB4

  1661. 

  1662. LDGEMM_L1x2_LOOP_START:

  1663. 

  1664. 	LOAD1x2_1

  1665. 	KERNEL1x2_I1

  1666. 	KERNEL1x2_2

  1667. 	KERNEL1x2_1

  1668. 	KERNEL1x2_2

  1669. 

  1670. 	KERNEL1x2_1

  1671. 	KERNEL1x2_2

  1672. 	KERNEL1x2_1

  1673. 	KERNEL1x2_2

  1674. 

  1675. 	addic.		L,	L,	-2

  1676. 	ble		LDGEMM_L1x2_LOOP_END

  1677. 

  1678. 	MY_ALIGN

  1679. 

  1680. LDGEMM_L1x2_LOOP:

  1681. 

  1682. 	KERNEL1x2_1

  1683. 	KERNEL1x2_2

  1684. 	KERNEL1x2_1

  1685. 	KERNEL1x2_2

  1686. 

  1687. 	KERNEL1x2_1

  1688. 	KERNEL1x2_2

  1689. 	KERNEL1x2_1

  1690. 	KERNEL1x2_2

  1691. 

  1692. 	addic.		L,	L,	-1

  1693. 	bgt		LDGEMM_L1x2_LOOP

  1694. 

  1695. LDGEMM_L1x2_LOOP_END:

  1696. 

  1697. 	KERNEL1x2_1

  1698. 	KERNEL1x2_2

  1699. 	KERNEL1x2_1

  1700. 	KERNEL1x2_2

  1701. 

  1702. 	KERNEL1x2_1

  1703. 	KERNEL1x2_2

  1704. 	KERNEL1x2_1

  1705. 	KERNEL1x2_E2

  1706. 

  1707. 	b		LDGEMM_L1x2_SUB1

  1708. 

  1709. LDGEMM_L1x2_SUB4:

  1710. 

  1711. 	KERNEL1x2_SUBI1

  1712. 	KERNEL1x2_SUB1

  1713. 	KERNEL1x2_SUB1

  1714. 	KERNEL1x2_SUB1

  1715. 

  1716. 	KERNEL1x2_SUB1

  1717. 	KERNEL1x2_SUB1

  1718. 	KERNEL1x2_SUB1

  1719. 	KERNEL1x2_SUB1

  1720. 

  1721. 	b		LDGEMM_L1x2_SUB1

  1722. 

  1723. LDGEMM_L1x2_SUB0:

  1724. 

  1725. 	andi.		L,	K,	7

  1726. 

  1727. 	KERNEL1x2_SUBI1

  1728. 

  1729. 	addic.		L,	L,	-1

  1730. 	ble		LDGEMM_L1x2_SAVE

  1731. 	b		LDGEMM_L1x2_SUB2

  1732. 

  1733. LDGEMM_L1x2_SUB1:

  1734. 

  1735. 	andi.		L,	K,	7

  1736. 	ble		LDGEMM_L1x2_SAVE

  1737. 

  1738. LDGEMM_L1x2_SUB2:

  1739. 

  1740. 	KERNEL1x2_SUB1

  1741. 

  1742. 	addic.		L,	L,	-1

  1743. 	bgt		LDGEMM_L1x2_SUB2

  1744. 

  1745. LDGEMM_L1x2_SAVE:

  1746. 

  1747. 	SAVE1x2

  1748. 

  1749. LDGEMM_L1x2_END:

  1750. 

  1751. LDGEMM_L1x1_BEGIN:

  1752. 

  1753. 

  1754. 	andi.		T1,	M,	1

  1755. 	ble		LDGEMM_L1x1_END

  1756. 	mr		BO,	B

  1757. 	srawi.		L,	K,	3

  1758. 	ble		LDGEMM_L1x1_SUB0

  1759. 	cmpwi		cr0,	L,	1

  1760. 	ble		LDGEMM_L1x1_SUB4

  1761. 

  1762. LDGEMM_L1x1_LOOP_START:

  1763. 

  1764. 	LOAD1x1_1

  1765. 	KERNEL1x1_I1

  1766. 	KERNEL1x1_2

  1767. 	KERNEL1x1_1

  1768. 	KERNEL1x1_2

  1769. 

  1770. 	KERNEL1x1_1

  1771. 	KERNEL1x1_2

  1772. 	KERNEL1x1_1

  1773. 	KERNEL1x1_2

  1774. 

  1775. 	addic.		L,	L,	-2

  1776. 	ble		LDGEMM_L1x1_LOOP_END

  1777. 

  1778. 	MY_ALIGN

  1779. 

  1780. LDGEMM_L1x1_LOOP:

  1781. 

  1782. 	KERNEL1x1_1

  1783. 	KERNEL1x1_2

  1784. 	KERNEL1x1_1

  1785. 	KERNEL1x1_2

  1786. 

  1787. 	KERNEL1x1_1

  1788. 	KERNEL1x1_2

  1789. 	KERNEL1x1_1

  1790. 	KERNEL1x1_2

  1791. 

  1792. 	addic.		L,	L,	-1

  1793. 	bgt		LDGEMM_L1x1_LOOP

  1794. 

  1795. LDGEMM_L1x1_LOOP_END:

  1796. 

  1797. 	KERNEL1x1_1

  1798. 	KERNEL1x1_2

  1799. 	KERNEL1x1_1

  1800. 	KERNEL1x1_2

  1801. 

  1802. 	KERNEL1x1_1

  1803. 	KERNEL1x1_2

  1804. 	KERNEL1x1_1

  1805. 	KERNEL1x1_E2

  1806. 

  1807. 	b		LDGEMM_L1x1_SUB1

  1808. 

  1809. LDGEMM_L1x1_SUB4:

  1810. 

  1811. 	KERNEL1x1_SUBI1

  1812. 	KERNEL1x1_SUB1

  1813. 	KERNEL1x1_SUB1

  1814. 	KERNEL1x1_SUB1

  1815. 

  1816. 	KERNEL1x1_SUB1

  1817. 	KERNEL1x1_SUB1

  1818. 	KERNEL1x1_SUB1

  1819. 	KERNEL1x1_SUB1

  1820. 

  1821. 	b		LDGEMM_L1x1_SUB1

  1822. 

  1823. LDGEMM_L1x1_SUB0:

  1824. 

  1825. 	andi.		L,	K,	7

  1826. 

  1827. 	KERNEL1x1_SUBI1

  1828. 

  1829. 	addic.		L,	L,	-1

  1830. 	ble		LDGEMM_L1x1_SAVE

  1831. 	b		LDGEMM_L1x1_SUB2

  1832. 

  1833. LDGEMM_L1x1_SUB1:

  1834. 

  1835. 	andi.		L,	K,	7

  1836. 	ble		LDGEMM_L1x1_SAVE

  1837. 

  1838. LDGEMM_L1x1_SUB2:

  1839. 

  1840. 	KERNEL1x1_SUB1

  1841. 

  1842. 	addic.		L,	L,	-1

  1843. 	bgt		LDGEMM_L1x1_SUB2

  1844. 

  1845. LDGEMM_L1x1_SAVE:

  1846. 

  1847. 	SAVE1x1

  1848. 

  1849. LDGEMM_L1x1_END:

  1850. 

  1851. LDGEMM_L1_END: