OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 02634b549b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '02634b549b'
${ noResults }
OpenBLAS/kernel/power/sgemm_logic_16x8_power8.S

2334 lines
29 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2016, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. /**************************************************************************************

  29. * 2016/04/21 Werner Saar (wernsaar@googlemail.com)

  30. * 	 BLASTEST 		: OK

  31. * 	 CTEST			: OK

  32. * 	 TEST			: OK

  33. *	 LAPACK-TEST		: OK

  34. **************************************************************************************/

  35. 

  36. 	srawi.		J,	N,	3

  37. 	ble		SGEMM_L8_END

  38. 

  39. SGEMM_L8_BEGIN:

  40. 

  41. 	mr		BO,	B

  42. 	mr		BBO,	BBUFFER

  43. 	srawi.		T1,	K,	2

  44. 	ble		SGEMM_L8_COPYB1

  45. 

  46. 

  47. SGEMM_L8_COPYB4:

  48. 

  49. 	dcbt		BO,	PRE

  50. 	dcbtst		BBO,	PRE

  51. 	COPYB_4x8

  52. 	addic.		T1,	T1,	-1

  53. 	ble		SGEMM_L8_COPYB1

  54. 

  55. 	dcbtst		BBO,	PRE

  56. 	COPYB_4x8

  57. 	addic.		T1,	T1,	-1

  58. 	ble		SGEMM_L8_COPYB1

  59. 

  60. 	dcbtst		BBO,	PRE

  61. 	COPYB_4x8

  62. 	addic.		T1,	T1,	-1

  63. 	ble		SGEMM_L8_COPYB1

  64. 

  65. 	dcbtst		BBO,	PRE

  66. 	COPYB_4x8

  67. 	addic.		T1,	T1,	-1

  68. 

  69. 	bgt		SGEMM_L8_COPYB4

  70. 

  71. SGEMM_L8_COPYB1:

  72. 

  73. 	andi.		T1,	K,	3

  74. 	ble		SGEMM_L8_COPYB_END	

  75. 

  76. SGEMM_L8_COPYB1_LOOP:

  77. 

  78. 

  79. 	COPYB_1x8

  80. 	addic.		T1,	T1,	-1

  81. 

  82. 	bgt		SGEMM_L8_COPYB1_LOOP

  83. 

  84. SGEMM_L8_COPYB_END:

  85. 

  86. 	mr		CO,	C

  87. 	mr		AO,	A

  88. 	slwi		T1,	LDC	,	3

  89. 	add		C,	C,	T1

  90. 	srawi.		I,	M,	4

  91. 	ble		SGEMM_L8x16_END

  92. 

  93. SGEMM_L8x16_BEGIN:

  94. 

  95. 

  96. 	mr		BO,	BBUFFER

  97. 	srawi.		L,	K,	3

  98. 	ble		SGEMM_L8x16_SUB0

  99. 	cmpwi		cr0,	L,	1

  100. 	ble		SGEMM_L8x16_SUB4

  101. 

  102. SGEMM_L8x16_LOOP_START:

  103. 

  104. 	dcbt		AO,	PRE

  105. 	dcbt		BO,	PRE

  106. 	LOAD8x16_1

  107. 	dcbt		BO,	PRE

  108. 	KERNEL8x16_I1

  109. 	dcbt		AO,	PRE

  110. 	dcbt		BO,	PRE

  111. 	KERNEL8x16_2

  112. 	dcbt		BO,	PRE

  113. 	KERNEL8x16_1

  114. 	dcbt		AO,	PRE

  115. 	dcbt		BO,	PRE

  116. 	KERNEL8x16_2

  117. 

  118. 	dcbt		BO,	PRE

  119. 	KERNEL8x16_1

  120. 	dcbt		AO,	PRE

  121. 	dcbt		BO,	PRE

  122. 	KERNEL8x16_2

  123. 	dcbt		BO,	PRE

  124. 	KERNEL8x16_1

  125. 	dcbt		AO,	PRE

  126. 	dcbt		BO,	PRE

  127. 	KERNEL8x16_2

  128. 

  129. 	addic.		L,	L,	-2

  130. 	ble		SGEMM_L8x16_LOOP_END

  131. 

  132. 	.align 5

  133. 

  134. SGEMM_L8x16_LOOP:

  135. 

  136. 	dcbt		BO,	PRE

  137. 	KERNEL8x16_1

  138. 	dcbt		AO,	PRE

  139. 	dcbt		BO,	PRE

  140. 	KERNEL8x16_2

  141. 	dcbt		BO,	PRE

  142. 	KERNEL8x16_1

  143. 	dcbt		AO,	PRE

  144. 	dcbt		BO,	PRE

  145. 	KERNEL8x16_2

  146. 

  147. 	dcbt		BO,	PRE

  148. 	KERNEL8x16_1

  149. 	dcbt		AO,	PRE

  150. 	dcbt		BO,	PRE

  151. 	KERNEL8x16_2

  152. 	dcbt		BO,	PRE

  153. 	KERNEL8x16_1

  154. 	dcbt		AO,	PRE

  155. 	dcbt		BO,	PRE

  156. 	KERNEL8x16_2

  157. 

  158. 	addic.		L,	L,	-1

  159. 	bgt		SGEMM_L8x16_LOOP

  160. 

  161. SGEMM_L8x16_LOOP_END:

  162. 

  163. 	dcbt		BO,	PRE

  164. 	KERNEL8x16_1

  165. 	dcbt		AO,	PRE

  166. 	dcbt		BO,	PRE

  167. 	KERNEL8x16_2

  168. 	dcbt		BO,	PRE

  169. 	KERNEL8x16_1

  170. 	dcbt		AO,	PRE

  171. 	KERNEL8x16_2

  172. 

  173. 	KERNEL8x16_1

  174. 	dcbt		AO,	PRE

  175. 	KERNEL8x16_2

  176. 	KERNEL8x16_1

  177. 	KERNEL8x16_E2

  178. 

  179. 	b		SGEMM_L8x16_SUB1

  180. 

  181. SGEMM_L8x16_SUB4:

  182. 

  183. 	dcbt		AO,	PRE

  184. 	KERNEL8x16_SUBI1

  185. 	KERNEL8x16_SUB1

  186. 	dcbt		AO,	PRE

  187. 	KERNEL8x16_SUB1

  188. 	KERNEL8x16_SUB1

  189. 

  190. 	KERNEL8x16_SUB1

  191. 	KERNEL8x16_SUB1

  192. 	KERNEL8x16_SUB1

  193. 	KERNEL8x16_SUB1

  194. 

  195. 	b		SGEMM_L8x16_SUB1

  196. 

  197. SGEMM_L8x16_SUB0:

  198. 

  199. 	andi.		L,	K,	7

  200. 

  201. 	KERNEL8x16_SUBI1

  202. 

  203. 	addic.		L,	L,	-1

  204. 	ble		SGEMM_L8x16_SAVE

  205. 	b		SGEMM_L8x16_SUB2

  206. 

  207. SGEMM_L8x16_SUB1:

  208. 

  209. 	andi.		L,	K,	7

  210. 	ble		SGEMM_L8x16_SAVE

  211. 

  212. SGEMM_L8x16_SUB2:

  213. 

  214. 	KERNEL8x16_SUB1

  215. 

  216. 	addic.		L,	L,	-1

  217. 	bgt		SGEMM_L8x16_SUB2

  218. 

  219. SGEMM_L8x16_SAVE:

  220. 

  221. 	SAVE8x16

  222. 

  223. 	addic.		I,	I,	-1

  224. 	bgt		SGEMM_L8x16_BEGIN

  225. 

  226. SGEMM_L8x16_END:

  227. 

  228. SGEMM_L8x8_BEGIN:

  229. 

  230. 	andi.		T2,	M,	15

  231. 	ble		SGEMM_L8x1_END

  232. 

  233. 	andi.		T1,	M,	8

  234. 	ble		SGEMM_L8x8_END

  235. 	mr		BO,	BBUFFER

  236. 	srawi.		L,	K,	3

  237. 	ble		SGEMM_L8x8_SUB0

  238. 	cmpwi		cr0,	L,	1

  239. 	ble		SGEMM_L8x8_SUB4

  240. 

  241. SGEMM_L8x8_LOOP_START:

  242. 

  243. 	LOAD8x8_1

  244. 	KERNEL8x8_I1

  245. 	KERNEL8x8_2

  246. 	KERNEL8x8_1

  247. 	KERNEL8x8_2

  248. 

  249. 	KERNEL8x8_1

  250. 	KERNEL8x8_2

  251. 	KERNEL8x8_1

  252. 	KERNEL8x8_2

  253. 

  254. 	addic.		L,	L,	-2

  255. 	ble		SGEMM_L8x8_LOOP_END

  256. 

  257. 	.align 5

  258. 

  259. SGEMM_L8x8_LOOP:

  260. 

  261. 	KERNEL8x8_1

  262. 	KERNEL8x8_2

  263. 	KERNEL8x8_1

  264. 	KERNEL8x8_2

  265. 

  266. 	KERNEL8x8_1

  267. 	KERNEL8x8_2

  268. 	KERNEL8x8_1

  269. 	KERNEL8x8_2

  270. 

  271. 	addic.		L,	L,	-1

  272. 	bgt		SGEMM_L8x8_LOOP

  273. 

  274. SGEMM_L8x8_LOOP_END:

  275. 

  276. 	KERNEL8x8_1

  277. 	KERNEL8x8_2

  278. 	KERNEL8x8_1

  279. 	KERNEL8x8_2

  280. 

  281. 	KERNEL8x8_1

  282. 	KERNEL8x8_2

  283. 	KERNEL8x8_1

  284. 	KERNEL8x8_E2

  285. 

  286. 	b		SGEMM_L8x8_SUB1

  287. 

  288. SGEMM_L8x8_SUB4:

  289. 

  290. 	KERNEL8x8_SUBI1

  291. 	KERNEL8x8_SUB1

  292. 	KERNEL8x8_SUB1

  293. 	KERNEL8x8_SUB1

  294. 

  295. 	KERNEL8x8_SUB1

  296. 	KERNEL8x8_SUB1

  297. 	KERNEL8x8_SUB1

  298. 	KERNEL8x8_SUB1

  299. 

  300. 	b		SGEMM_L8x8_SUB1

  301. 

  302. SGEMM_L8x8_SUB0:

  303. 

  304. 	andi.		L,	K,	7

  305. 

  306. 	KERNEL8x8_SUBI1

  307. 

  308. 	addic.		L,	L,	-1

  309. 	ble		SGEMM_L8x8_SAVE

  310. 	b		SGEMM_L8x8_SUB2

  311. 

  312. SGEMM_L8x8_SUB1:

  313. 

  314. 	andi.		L,	K,	7

  315. 	ble		SGEMM_L8x8_SAVE

  316. 

  317. SGEMM_L8x8_SUB2:

  318. 

  319. 	KERNEL8x8_SUB1

  320. 

  321. 	addic.		L,	L,	-1

  322. 	bgt		SGEMM_L8x8_SUB2

  323. 

  324. SGEMM_L8x8_SAVE:

  325. 

  326. 	SAVE8x8

  327. 

  328. SGEMM_L8x8_END:

  329. 

  330. SGEMM_L8x4_BEGIN:

  331. 

  332. 

  333. 	andi.		T1,	M,	4

  334. 	ble		SGEMM_L8x4_END

  335. 	mr		BO,	BBUFFER

  336. 	srawi.		L,	K,	3

  337. 	ble		SGEMM_L8x4_SUB0

  338. 	cmpwi		cr0,	L,	1

  339. 	ble		SGEMM_L8x4_SUB4

  340. 

  341. SGEMM_L8x4_LOOP_START:

  342. 

  343. 	LOAD8x4_1

  344. 	KERNEL8x4_I1

  345. 	KERNEL8x4_2

  346. 	KERNEL8x4_1

  347. 	KERNEL8x4_2

  348. 

  349. 	KERNEL8x4_1

  350. 	KERNEL8x4_2

  351. 	KERNEL8x4_1

  352. 	KERNEL8x4_2

  353. 

  354. 	addic.		L,	L,	-2

  355. 	ble		SGEMM_L8x4_LOOP_END

  356. 

  357. 	.align 5

  358. 

  359. SGEMM_L8x4_LOOP:

  360. 

  361. 	KERNEL8x4_1

  362. 	KERNEL8x4_2

  363. 	KERNEL8x4_1

  364. 	KERNEL8x4_2

  365. 

  366. 	KERNEL8x4_1

  367. 	KERNEL8x4_2

  368. 	KERNEL8x4_1

  369. 	KERNEL8x4_2

  370. 

  371. 	addic.		L,	L,	-1

  372. 	bgt		SGEMM_L8x4_LOOP

  373. 

  374. SGEMM_L8x4_LOOP_END:

  375. 

  376. 	KERNEL8x4_1

  377. 	KERNEL8x4_2

  378. 	KERNEL8x4_1

  379. 	KERNEL8x4_2

  380. 

  381. 	KERNEL8x4_1

  382. 	KERNEL8x4_2

  383. 	KERNEL8x4_1

  384. 	KERNEL8x4_E2

  385. 

  386. 	b		SGEMM_L8x4_SUB1

  387. 

  388. SGEMM_L8x4_SUB4:

  389. 

  390. 	KERNEL8x4_SUBI1

  391. 	KERNEL8x4_SUB1

  392. 	KERNEL8x4_SUB1

  393. 	KERNEL8x4_SUB1

  394. 

  395. 	KERNEL8x4_SUB1

  396. 	KERNEL8x4_SUB1

  397. 	KERNEL8x4_SUB1

  398. 	KERNEL8x4_SUB1

  399. 

  400. 	b		SGEMM_L8x4_SUB1

  401. 

  402. SGEMM_L8x4_SUB0:

  403. 

  404. 	andi.		L,	K,	7

  405. 

  406. 	KERNEL8x4_SUBI1

  407. 

  408. 	addic.		L,	L,	-1

  409. 	ble		SGEMM_L8x4_SAVE

  410. 	b		SGEMM_L8x4_SUB2

  411. 

  412. SGEMM_L8x4_SUB1:

  413. 

  414. 	andi.		L,	K,	7

  415. 	ble		SGEMM_L8x4_SAVE

  416. 

  417. SGEMM_L8x4_SUB2:

  418. 

  419. 	KERNEL8x4_SUB1

  420. 

  421. 	addic.		L,	L,	-1

  422. 	bgt		SGEMM_L8x4_SUB2

  423. 

  424. SGEMM_L8x4_SAVE:

  425. 

  426. 	SAVE8x4

  427. 

  428. SGEMM_L8x4_END:

  429. 

  430. SGEMM_L8x2_BEGIN:

  431. 

  432. 

  433. 	andi.		T1,	M,	2

  434. 	ble		SGEMM_L8x2_END

  435. 	mr		BO,	BBUFFER

  436. 	srawi.		L,	K,	3

  437. 	ble		SGEMM_L8x2_SUB0

  438. 	cmpwi		cr0,	L,	1

  439. 	ble		SGEMM_L8x2_SUB4

  440. 

  441. SGEMM_L8x2_LOOP_START:

  442. 

  443. 	LOAD8x2_1

  444. 	KERNEL8x2_I1

  445. 	KERNEL8x2_2

  446. 	KERNEL8x2_1

  447. 	KERNEL8x2_2

  448. 

  449. 	KERNEL8x2_1

  450. 	KERNEL8x2_2

  451. 	KERNEL8x2_1

  452. 	KERNEL8x2_2

  453. 

  454. 	addic.		L,	L,	-2

  455. 	ble		SGEMM_L8x2_LOOP_END

  456. 

  457. 	.align 5

  458. 

  459. SGEMM_L8x2_LOOP:

  460. 

  461. 	KERNEL8x2_1

  462. 	KERNEL8x2_2

  463. 	KERNEL8x2_1

  464. 	KERNEL8x2_2

  465. 

  466. 	KERNEL8x2_1

  467. 	KERNEL8x2_2

  468. 	KERNEL8x2_1

  469. 	KERNEL8x2_2

  470. 

  471. 	addic.		L,	L,	-1

  472. 	bgt		SGEMM_L8x2_LOOP

  473. 

  474. SGEMM_L8x2_LOOP_END:

  475. 

  476. 	KERNEL8x2_1

  477. 	KERNEL8x2_2

  478. 	KERNEL8x2_1

  479. 	KERNEL8x2_2

  480. 

  481. 	KERNEL8x2_1

  482. 	KERNEL8x2_2

  483. 	KERNEL8x2_1

  484. 	KERNEL8x2_E2

  485. 

  486. 	b		SGEMM_L8x2_SUB1

  487. 

  488. SGEMM_L8x2_SUB4:

  489. 

  490. 	KERNEL8x2_SUBI1

  491. 	KERNEL8x2_SUB1

  492. 	KERNEL8x2_SUB1

  493. 	KERNEL8x2_SUB1

  494. 

  495. 	KERNEL8x2_SUB1

  496. 	KERNEL8x2_SUB1

  497. 	KERNEL8x2_SUB1

  498. 	KERNEL8x2_SUB1

  499. 

  500. 	b		SGEMM_L8x2_SUB1

  501. 

  502. SGEMM_L8x2_SUB0:

  503. 

  504. 	andi.		L,	K,	7

  505. 

  506. 	KERNEL8x2_SUBI1

  507. 

  508. 	addic.		L,	L,	-1

  509. 	ble		SGEMM_L8x2_SAVE

  510. 	b		SGEMM_L8x2_SUB2

  511. 

  512. SGEMM_L8x2_SUB1:

  513. 

  514. 	andi.		L,	K,	7

  515. 	ble		SGEMM_L8x2_SAVE

  516. 

  517. SGEMM_L8x2_SUB2:

  518. 

  519. 	KERNEL8x2_SUB1

  520. 

  521. 	addic.		L,	L,	-1

  522. 	bgt		SGEMM_L8x2_SUB2

  523. 

  524. SGEMM_L8x2_SAVE:

  525. 

  526. 	SAVE8x2

  527. 

  528. SGEMM_L8x2_END:

  529. 

  530. SGEMM_L8x1_BEGIN:

  531. 

  532. 

  533. 	andi.		T1,	M,	1

  534. 	ble		SGEMM_L8x1_END

  535. 	mr		BO,	BBUFFER

  536. 	srawi.		L,	K,	3

  537. 	ble		SGEMM_L8x1_SUB0

  538. 	cmpwi		cr0,	L,	1

  539. 	ble		SGEMM_L8x1_SUB4

  540. 

  541. SGEMM_L8x1_LOOP_START:

  542. 

  543. 	LOAD8x1_1

  544. 	KERNEL8x1_I1

  545. 	KERNEL8x1_2

  546. 	KERNEL8x1_1

  547. 	KERNEL8x1_2

  548. 

  549. 	KERNEL8x1_1

  550. 	KERNEL8x1_2

  551. 	KERNEL8x1_1

  552. 	KERNEL8x1_2

  553. 

  554. 	addic.		L,	L,	-2

  555. 	ble		SGEMM_L8x1_LOOP_END

  556. 

  557. 	.align 5

  558. 

  559. SGEMM_L8x1_LOOP:

  560. 

  561. 	KERNEL8x1_1

  562. 	KERNEL8x1_2

  563. 	KERNEL8x1_1

  564. 	KERNEL8x1_2

  565. 

  566. 	KERNEL8x1_1

  567. 	KERNEL8x1_2

  568. 	KERNEL8x1_1

  569. 	KERNEL8x1_2

  570. 

  571. 	addic.		L,	L,	-1

  572. 	bgt		SGEMM_L8x1_LOOP

  573. 

  574. SGEMM_L8x1_LOOP_END:

  575. 

  576. 	KERNEL8x1_1

  577. 	KERNEL8x1_2

  578. 	KERNEL8x1_1

  579. 	KERNEL8x1_2

  580. 

  581. 	KERNEL8x1_1

  582. 	KERNEL8x1_2

  583. 	KERNEL8x1_1

  584. 	KERNEL8x1_E2

  585. 

  586. 	b		SGEMM_L8x1_SUB1

  587. 

  588. SGEMM_L8x1_SUB4:

  589. 

  590. 	KERNEL8x1_SUBI1

  591. 	KERNEL8x1_SUB1

  592. 	KERNEL8x1_SUB1

  593. 	KERNEL8x1_SUB1

  594. 

  595. 	KERNEL8x1_SUB1

  596. 	KERNEL8x1_SUB1

  597. 	KERNEL8x1_SUB1

  598. 	KERNEL8x1_SUB1

  599. 

  600. 	b		SGEMM_L8x1_SUB1

  601. 

  602. SGEMM_L8x1_SUB0:

  603. 

  604. 	andi.		L,	K,	7

  605. 

  606. 	KERNEL8x1_SUBI1

  607. 

  608. 	addic.		L,	L,	-1

  609. 	ble		SGEMM_L8x1_SAVE

  610. 	b		SGEMM_L8x1_SUB2

  611. 

  612. SGEMM_L8x1_SUB1:

  613. 

  614. 	andi.		L,	K,	7

  615. 	ble		SGEMM_L8x1_SAVE

  616. 

  617. SGEMM_L8x1_SUB2:

  618. 

  619. 	KERNEL8x1_SUB1

  620. 

  621. 	addic.		L,	L,	-1

  622. 	bgt		SGEMM_L8x1_SUB2

  623. 

  624. SGEMM_L8x1_SAVE:

  625. 

  626. 	SAVE8x1

  627. 

  628. SGEMM_L8x1_END:

  629. 

  630. 	slwi		T1,	K,	5

  631. 	add		B,	B,	T1

  632. 

  633. 	addic.		J,	J,	-1

  634. 	bgt		SGEMM_L8_BEGIN

  635. 

  636. 	andi.		T2,	N,	7

  637. 	ble		L999

  638. 

  639. SGEMM_L8_END:

  640. 

  641. 	b		SGEMM_L4_BEGIN

  642. 

  643. L999_H1:

  644. 

  645. 	b		L999

  646. 

  647. SGEMM_L4_BEGIN:

  648. 

  649. 	mr		BO,	B

  650. 	mr		BBO,	BBUFFER

  651. 	slwi		T1,	K,	2

  652. 

  653. SGEMM_L4_COPYB:

  654. 	dcbtst		BBO,	PRE

  655. 

  656. 	lxvw4x		vs3,	o0,	BO

  657. 	lxvw4x		vs11,	o16,	BO

  658. 	xxspltw		vs4,	vs3,	0

  659. 	xxspltw		vs5,	vs3,	1

  660. 	xxspltw		vs6,	vs3,	2

  661. 	xxspltw		vs7,	vs3,	3

  662. 	xxspltw		vs12,	vs11,	0

  663. 	xxspltw		vs13,	vs11,	1

  664. 	xxspltw		vs14,	vs11,	2

  665. 	xxspltw		vs15,	vs11,	3

  666. 	stxvw4x		vs4,	o0,	BBO

  667. 	stxvw4x		vs5,	o16,	BBO

  668. 	stxvw4x		vs6,	o32,	BBO

  669. 	stxvw4x		vs7,	o48,	BBO

  670. 	addi		BO,	BO,	32

  671. 	addi		BBO,	BBO,	64

  672. 	stxvw4x		vs12,	o0,	BBO

  673. 	stxvw4x		vs13,	o16,	BBO

  674. 	stxvw4x		vs14,	o32,	BBO

  675. 	stxvw4x		vs15,	o48,	BBO

  676. 	addic.		T1,	T1,	-8

  677. 	addi		BBO,	BBO,	64

  678. 

  679. 	bge		SGEMM_L4_COPYB

  680. 

  681. 	andi.		T1,	N,	4

  682. 	ble		SGEMM_L4_END

  683. 	mr		CO,	C

  684. 	mr		AO,	A

  685. 	slwi		T1,	LDC	,	2

  686. 	add		C,	C,	T1

  687. 	srawi.		I,	M,	4

  688. 	ble		SGEMM_L4x16_END

  689. 

  690. SGEMM_L4x16_BEGIN:

  691. 

  692. 

  693. 	mr		BO,	BBUFFER

  694. 	srawi.		L,	K,	3

  695. 	ble		SGEMM_L4x16_SUB0

  696. 	cmpwi		cr0,	L,	1

  697. 	ble		SGEMM_L4x16_SUB4

  698. 

  699. SGEMM_L4x16_LOOP_START:

  700. 

  701. 	dcbt		AO,	PRE

  702. 	LOAD4x16_1

  703. 	KERNEL4x16_I1

  704. 	dcbt		AO,	PRE

  705. 	KERNEL4x16_2

  706. 	KERNEL4x16_1

  707. 	dcbt		AO,	PRE

  708. 	KERNEL4x16_2

  709. 

  710. 	KERNEL4x16_1

  711. 	dcbt		AO,	PRE

  712. 	KERNEL4x16_2

  713. 	KERNEL4x16_1

  714. 	dcbt		AO,	PRE

  715. 	KERNEL4x16_2

  716. 

  717. 	addic.		L,	L,	-2

  718. 	ble		SGEMM_L4x16_LOOP_END

  719. 

  720. 	.align 5

  721. 

  722. SGEMM_L4x16_LOOP:

  723. 

  724. 	KERNEL4x16_1

  725. 	dcbt		AO,	PRE

  726. 	KERNEL4x16_2

  727. 	KERNEL4x16_1

  728. 	dcbt		AO,	PRE

  729. 	KERNEL4x16_2

  730. 

  731. 	KERNEL4x16_1

  732. 	dcbt		AO,	PRE

  733. 	KERNEL4x16_2

  734. 	KERNEL4x16_1

  735. 	dcbt		AO,	PRE

  736. 	KERNEL4x16_2

  737. 

  738. 	addic.		L,	L,	-1

  739. 	bgt		SGEMM_L4x16_LOOP

  740. 

  741. SGEMM_L4x16_LOOP_END:

  742. 

  743. 	KERNEL4x16_1

  744. 	dcbt		AO,	PRE

  745. 	KERNEL4x16_2

  746. 	KERNEL4x16_1

  747. 	dcbt		AO,	PRE

  748. 	KERNEL4x16_2

  749. 

  750. 	KERNEL4x16_1

  751. 	dcbt		AO,	PRE

  752. 	KERNEL4x16_2

  753. 	KERNEL4x16_1

  754. 	KERNEL4x16_E2

  755. 

  756. 	b		SGEMM_L4x16_SUB1

  757. 

  758. SGEMM_L4x16_SUB4:

  759. 

  760. 	dcbt		AO,	PRE

  761. 	KERNEL4x16_SUBI1

  762. 	KERNEL4x16_SUB1

  763. 	dcbt		AO,	PRE

  764. 	KERNEL4x16_SUB1

  765. 	KERNEL4x16_SUB1

  766. 

  767. 	KERNEL4x16_SUB1

  768. 	KERNEL4x16_SUB1

  769. 	KERNEL4x16_SUB1

  770. 	KERNEL4x16_SUB1

  771. 

  772. 	b		SGEMM_L4x16_SUB1

  773. 

  774. SGEMM_L4x16_SUB0:

  775. 

  776. 	andi.		L,	K,	7

  777. 

  778. 	KERNEL4x16_SUBI1

  779. 

  780. 	addic.		L,	L,	-1

  781. 	ble		SGEMM_L4x16_SAVE

  782. 	b		SGEMM_L4x16_SUB2

  783. 

  784. SGEMM_L4x16_SUB1:

  785. 

  786. 	andi.		L,	K,	7

  787. 	ble		SGEMM_L4x16_SAVE

  788. 

  789. SGEMM_L4x16_SUB2:

  790. 

  791. 	KERNEL4x16_SUB1

  792. 

  793. 	addic.		L,	L,	-1

  794. 	bgt		SGEMM_L4x16_SUB2

  795. 

  796. SGEMM_L4x16_SAVE:

  797. 

  798. 	SAVE4x16

  799. 

  800. 	addic.		I,	I,	-1

  801. 	bgt		SGEMM_L4x16_BEGIN

  802. 

  803. SGEMM_L4x16_END:

  804. 

  805. SGEMM_L4x8_BEGIN:

  806. 

  807. 	andi.		T2,	M,	15

  808. 	ble		SGEMM_L4x1_END

  809. 

  810. 	andi.		T1,	M,	8

  811. 	ble		SGEMM_L4x8_END

  812. 	mr		BO,	BBUFFER

  813. 	srawi.		L,	K,	3

  814. 	ble		SGEMM_L4x8_SUB0

  815. 	cmpwi		cr0,	L,	1

  816. 	ble		SGEMM_L4x8_SUB4

  817. 

  818. SGEMM_L4x8_LOOP_START:

  819. 

  820. 	LOAD4x8_1

  821. 	KERNEL4x8_I1

  822. 	KERNEL4x8_2

  823. 	KERNEL4x8_1

  824. 	KERNEL4x8_2

  825. 

  826. 	KERNEL4x8_1

  827. 	KERNEL4x8_2

  828. 	KERNEL4x8_1

  829. 	KERNEL4x8_2

  830. 

  831. 	addic.		L,	L,	-2

  832. 	ble		SGEMM_L4x8_LOOP_END

  833. 

  834. 	.align 5

  835. 

  836. SGEMM_L4x8_LOOP:

  837. 

  838. 	KERNEL4x8_1

  839. 	KERNEL4x8_2

  840. 	KERNEL4x8_1

  841. 	KERNEL4x8_2

  842. 

  843. 	KERNEL4x8_1

  844. 	KERNEL4x8_2

  845. 	KERNEL4x8_1

  846. 	KERNEL4x8_2

  847. 

  848. 	addic.		L,	L,	-1

  849. 	bgt		SGEMM_L4x8_LOOP

  850. 

  851. SGEMM_L4x8_LOOP_END:

  852. 

  853. 	KERNEL4x8_1

  854. 	KERNEL4x8_2

  855. 	KERNEL4x8_1

  856. 	KERNEL4x8_2

  857. 

  858. 	KERNEL4x8_1

  859. 	KERNEL4x8_2

  860. 	KERNEL4x8_1

  861. 	KERNEL4x8_E2

  862. 

  863. 	b		SGEMM_L4x8_SUB1

  864. 

  865. SGEMM_L4x8_SUB4:

  866. 

  867. 	KERNEL4x8_SUBI1

  868. 	KERNEL4x8_SUB1

  869. 	KERNEL4x8_SUB1

  870. 	KERNEL4x8_SUB1

  871. 

  872. 	KERNEL4x8_SUB1

  873. 	KERNEL4x8_SUB1

  874. 	KERNEL4x8_SUB1

  875. 	KERNEL4x8_SUB1

  876. 

  877. 	b		SGEMM_L4x8_SUB1

  878. 

  879. SGEMM_L4x8_SUB0:

  880. 

  881. 	andi.		L,	K,	7

  882. 

  883. 	KERNEL4x8_SUBI1

  884. 

  885. 	addic.		L,	L,	-1

  886. 	ble		SGEMM_L4x8_SAVE

  887. 	b		SGEMM_L4x8_SUB2

  888. 

  889. SGEMM_L4x8_SUB1:

  890. 

  891. 	andi.		L,	K,	7

  892. 	ble		SGEMM_L4x8_SAVE

  893. 

  894. SGEMM_L4x8_SUB2:

  895. 

  896. 	KERNEL4x8_SUB1

  897. 

  898. 	addic.		L,	L,	-1

  899. 	bgt		SGEMM_L4x8_SUB2

  900. 

  901. SGEMM_L4x8_SAVE:

  902. 

  903. 	SAVE4x8

  904. 

  905. SGEMM_L4x8_END:

  906. 

  907. SGEMM_L4x4_BEGIN:

  908. 

  909. 

  910. 	andi.		T1,	M,	4

  911. 	ble		SGEMM_L4x4_END

  912. 	mr		BO,	BBUFFER

  913. 	srawi.		L,	K,	3

  914. 	ble		SGEMM_L4x4_SUB0

  915. 	cmpwi		cr0,	L,	1

  916. 	ble		SGEMM_L4x4_SUB4

  917. 

  918. SGEMM_L4x4_LOOP_START:

  919. 

  920. 	LOAD4x4_1

  921. 	KERNEL4x4_I1

  922. 	KERNEL4x4_2

  923. 	KERNEL4x4_1

  924. 	KERNEL4x4_2

  925. 

  926. 	KERNEL4x4_1

  927. 	KERNEL4x4_2

  928. 	KERNEL4x4_1

  929. 	KERNEL4x4_2

  930. 

  931. 	addic.		L,	L,	-2

  932. 	ble		SGEMM_L4x4_LOOP_END

  933. 

  934. 	.align 5

  935. 

  936. SGEMM_L4x4_LOOP:

  937. 

  938. 	KERNEL4x4_1

  939. 	KERNEL4x4_2

  940. 	KERNEL4x4_1

  941. 	KERNEL4x4_2

  942. 

  943. 	KERNEL4x4_1

  944. 	KERNEL4x4_2

  945. 	KERNEL4x4_1

  946. 	KERNEL4x4_2

  947. 

  948. 	addic.		L,	L,	-1

  949. 	bgt		SGEMM_L4x4_LOOP

  950. 

  951. SGEMM_L4x4_LOOP_END:

  952. 

  953. 	KERNEL4x4_1

  954. 	KERNEL4x4_2

  955. 	KERNEL4x4_1

  956. 	KERNEL4x4_2

  957. 

  958. 	KERNEL4x4_1

  959. 	KERNEL4x4_2

  960. 	KERNEL4x4_1

  961. 	KERNEL4x4_E2

  962. 

  963. 	b		SGEMM_L4x4_SUB1

  964. 

  965. SGEMM_L4x4_SUB4:

  966. 

  967. 	KERNEL4x4_SUBI1

  968. 	KERNEL4x4_SUB1

  969. 	KERNEL4x4_SUB1

  970. 	KERNEL4x4_SUB1

  971. 

  972. 	KERNEL4x4_SUB1

  973. 	KERNEL4x4_SUB1

  974. 	KERNEL4x4_SUB1

  975. 	KERNEL4x4_SUB1

  976. 

  977. 	b		SGEMM_L4x4_SUB1

  978. 

  979. SGEMM_L4x4_SUB0:

  980. 

  981. 	andi.		L,	K,	7

  982. 

  983. 	KERNEL4x4_SUBI1

  984. 

  985. 	addic.		L,	L,	-1

  986. 	ble		SGEMM_L4x4_SAVE

  987. 	b		SGEMM_L4x4_SUB2

  988. 

  989. SGEMM_L4x4_SUB1:

  990. 

  991. 	andi.		L,	K,	7

  992. 	ble		SGEMM_L4x4_SAVE

  993. 

  994. SGEMM_L4x4_SUB2:

  995. 

  996. 	KERNEL4x4_SUB1

  997. 

  998. 	addic.		L,	L,	-1

  999. 	bgt		SGEMM_L4x4_SUB2

  1000. 

  1001. SGEMM_L4x4_SAVE:

  1002. 

  1003. 	SAVE4x4

  1004. 

  1005. SGEMM_L4x4_END:

  1006. 

  1007. SGEMM_L4x2_BEGIN:

  1008. 

  1009. 

  1010. 	andi.		T1,	M,	2

  1011. 	ble		SGEMM_L4x2_END

  1012. 	mr		BO,	BBUFFER

  1013. 	srawi.		L,	K,	3

  1014. 	ble		SGEMM_L4x2_SUB0

  1015. 	cmpwi		cr0,	L,	1

  1016. 	ble		SGEMM_L4x2_SUB4

  1017. 

  1018. SGEMM_L4x2_LOOP_START:

  1019. 

  1020. 	LOAD4x2_1

  1021. 	KERNEL4x2_I1

  1022. 	KERNEL4x2_2

  1023. 	KERNEL4x2_1

  1024. 	KERNEL4x2_2

  1025. 

  1026. 	KERNEL4x2_1

  1027. 	KERNEL4x2_2

  1028. 	KERNEL4x2_1

  1029. 	KERNEL4x2_2

  1030. 

  1031. 	addic.		L,	L,	-2

  1032. 	ble		SGEMM_L4x2_LOOP_END

  1033. 

  1034. 	.align 5

  1035. 

  1036. SGEMM_L4x2_LOOP:

  1037. 

  1038. 	KERNEL4x2_1

  1039. 	KERNEL4x2_2

  1040. 	KERNEL4x2_1

  1041. 	KERNEL4x2_2

  1042. 

  1043. 	KERNEL4x2_1

  1044. 	KERNEL4x2_2

  1045. 	KERNEL4x2_1

  1046. 	KERNEL4x2_2

  1047. 

  1048. 	addic.		L,	L,	-1

  1049. 	bgt		SGEMM_L4x2_LOOP

  1050. 

  1051. SGEMM_L4x2_LOOP_END:

  1052. 

  1053. 	KERNEL4x2_1

  1054. 	KERNEL4x2_2

  1055. 	KERNEL4x2_1

  1056. 	KERNEL4x2_2

  1057. 

  1058. 	KERNEL4x2_1

  1059. 	KERNEL4x2_2

  1060. 	KERNEL4x2_1

  1061. 	KERNEL4x2_E2

  1062. 

  1063. 	b		SGEMM_L4x2_SUB1

  1064. 

  1065. SGEMM_L4x2_SUB4:

  1066. 

  1067. 	KERNEL4x2_SUBI1

  1068. 	KERNEL4x2_SUB1

  1069. 	KERNEL4x2_SUB1

  1070. 	KERNEL4x2_SUB1

  1071. 

  1072. 	KERNEL4x2_SUB1

  1073. 	KERNEL4x2_SUB1

  1074. 	KERNEL4x2_SUB1

  1075. 	KERNEL4x2_SUB1

  1076. 

  1077. 	b		SGEMM_L4x2_SUB1

  1078. 

  1079. SGEMM_L4x2_SUB0:

  1080. 

  1081. 	andi.		L,	K,	7

  1082. 

  1083. 	KERNEL4x2_SUBI1

  1084. 

  1085. 	addic.		L,	L,	-1

  1086. 	ble		SGEMM_L4x2_SAVE

  1087. 	b		SGEMM_L4x2_SUB2

  1088. 

  1089. SGEMM_L4x2_SUB1:

  1090. 

  1091. 	andi.		L,	K,	7

  1092. 	ble		SGEMM_L4x2_SAVE

  1093. 

  1094. SGEMM_L4x2_SUB2:

  1095. 

  1096. 	KERNEL4x2_SUB1

  1097. 

  1098. 	addic.		L,	L,	-1

  1099. 	bgt		SGEMM_L4x2_SUB2

  1100. 

  1101. SGEMM_L4x2_SAVE:

  1102. 

  1103. 	SAVE4x2

  1104. 

  1105. SGEMM_L4x2_END:

  1106. 

  1107. SGEMM_L4x1_BEGIN:

  1108. 

  1109. 

  1110. 	andi.		T1,	M,	1

  1111. 	ble		SGEMM_L4x1_END

  1112. 	mr		BO,	BBUFFER

  1113. 	srawi.		L,	K,	3

  1114. 	ble		SGEMM_L4x1_SUB0

  1115. 	cmpwi		cr0,	L,	1

  1116. 	ble		SGEMM_L4x1_SUB4

  1117. 

  1118. SGEMM_L4x1_LOOP_START:

  1119. 

  1120. 	LOAD4x1_1

  1121. 	KERNEL4x1_I1

  1122. 	KERNEL4x1_2

  1123. 	KERNEL4x1_1

  1124. 	KERNEL4x1_2

  1125. 

  1126. 	KERNEL4x1_1

  1127. 	KERNEL4x1_2

  1128. 	KERNEL4x1_1

  1129. 	KERNEL4x1_2

  1130. 

  1131. 	addic.		L,	L,	-2

  1132. 	ble		SGEMM_L4x1_LOOP_END

  1133. 

  1134. 	.align 5

  1135. 

  1136. SGEMM_L4x1_LOOP:

  1137. 

  1138. 	KERNEL4x1_1

  1139. 	KERNEL4x1_2

  1140. 	KERNEL4x1_1

  1141. 	KERNEL4x1_2

  1142. 

  1143. 	KERNEL4x1_1

  1144. 	KERNEL4x1_2

  1145. 	KERNEL4x1_1

  1146. 	KERNEL4x1_2

  1147. 

  1148. 	addic.		L,	L,	-1

  1149. 	bgt		SGEMM_L4x1_LOOP

  1150. 

  1151. SGEMM_L4x1_LOOP_END:

  1152. 

  1153. 	KERNEL4x1_1

  1154. 	KERNEL4x1_2

  1155. 	KERNEL4x1_1

  1156. 	KERNEL4x1_2

  1157. 

  1158. 	KERNEL4x1_1

  1159. 	KERNEL4x1_2

  1160. 	KERNEL4x1_1

  1161. 	KERNEL4x1_E2

  1162. 

  1163. 	b		SGEMM_L4x1_SUB1

  1164. 

  1165. SGEMM_L4x1_SUB4:

  1166. 

  1167. 	KERNEL4x1_SUBI1

  1168. 	KERNEL4x1_SUB1

  1169. 	KERNEL4x1_SUB1

  1170. 	KERNEL4x1_SUB1

  1171. 

  1172. 	KERNEL4x1_SUB1

  1173. 	KERNEL4x1_SUB1

  1174. 	KERNEL4x1_SUB1

  1175. 	KERNEL4x1_SUB1

  1176. 

  1177. 	b		SGEMM_L4x1_SUB1

  1178. 

  1179. SGEMM_L4x1_SUB0:

  1180. 

  1181. 	andi.		L,	K,	7

  1182. 

  1183. 	KERNEL4x1_SUBI1

  1184. 

  1185. 	addic.		L,	L,	-1

  1186. 	ble		SGEMM_L4x1_SAVE

  1187. 	b		SGEMM_L4x1_SUB2

  1188. 

  1189. SGEMM_L4x1_SUB1:

  1190. 

  1191. 	andi.		L,	K,	7

  1192. 	ble		SGEMM_L4x1_SAVE

  1193. 

  1194. SGEMM_L4x1_SUB2:

  1195. 

  1196. 	KERNEL4x1_SUB1

  1197. 

  1198. 	addic.		L,	L,	-1

  1199. 	bgt		SGEMM_L4x1_SUB2

  1200. 

  1201. SGEMM_L4x1_SAVE:

  1202. 

  1203. 	SAVE4x1

  1204. 

  1205. SGEMM_L4x1_END:

  1206. 

  1207. 	slwi		T1,	K,	4

  1208. 	add		B,	B,	T1

  1209. 

  1210. SGEMM_L4_END:

  1211. SGEMM_L2_BEGIN:

  1212. 

  1213. 	mr		BO,	B

  1214. 	mr		BBO,	BBUFFER

  1215. 	slwi		T1,	K,	1

  1216. 

  1217. SGEMM_L2_COPYB:

  1218. 	dcbtst		BBO,	PRE

  1219. 

  1220. 	lxvw4x		vs3,	o0,	BO

  1221. 	lxvw4x		vs11,	o16,	BO

  1222. 	xxspltw		vs4,	vs3,	0

  1223. 	xxspltw		vs5,	vs3,	1

  1224. 	xxspltw		vs6,	vs3,	2

  1225. 	xxspltw		vs7,	vs3,	3

  1226. 	xxspltw		vs12,	vs11,	0

  1227. 	xxspltw		vs13,	vs11,	1

  1228. 	xxspltw		vs14,	vs11,	2

  1229. 	xxspltw		vs15,	vs11,	3

  1230. 	stxvw4x		vs4,	o0,	BBO

  1231. 	stxvw4x		vs5,	o16,	BBO

  1232. 	stxvw4x		vs6,	o32,	BBO

  1233. 	stxvw4x		vs7,	o48,	BBO

  1234. 	addi		BO,	BO,	32

  1235. 	addi		BBO,	BBO,	64

  1236. 	stxvw4x		vs12,	o0,	BBO

  1237. 	stxvw4x		vs13,	o16,	BBO

  1238. 	stxvw4x		vs14,	o32,	BBO

  1239. 	stxvw4x		vs15,	o48,	BBO

  1240. 	addic.		T1,	T1,	-8

  1241. 	addi		BBO,	BBO,	64

  1242. 

  1243. 	bge		SGEMM_L2_COPYB

  1244. 

  1245. 	andi.		T1,	N,	2

  1246. 	ble		SGEMM_L2_END

  1247. 	mr		CO,	C

  1248. 	mr		AO,	A

  1249. 	slwi		T1,	LDC	,	1

  1250. 	add		C,	C,	T1

  1251. 	srawi.		I,	M,	4

  1252. 	ble		SGEMM_L2x16_END

  1253. 

  1254. SGEMM_L2x16_BEGIN:

  1255. 

  1256. 

  1257. 	mr		BO,	BBUFFER

  1258. 	srawi.		L,	K,	3

  1259. 	ble		SGEMM_L2x16_SUB0

  1260. 	cmpwi		cr0,	L,	1

  1261. 	ble		SGEMM_L2x16_SUB4

  1262. 

  1263. SGEMM_L2x16_LOOP_START:

  1264. 

  1265. 	dcbt		AO,	PRE

  1266. 	LOAD2x16_1

  1267. 	KERNEL2x16_I1

  1268. 	dcbt		AO,	PRE

  1269. 	KERNEL2x16_2

  1270. 	KERNEL2x16_1

  1271. 	dcbt		AO,	PRE

  1272. 	KERNEL2x16_2

  1273. 

  1274. 	KERNEL2x16_1

  1275. 	dcbt		AO,	PRE

  1276. 	KERNEL2x16_2

  1277. 	KERNEL2x16_1

  1278. 	dcbt		AO,	PRE

  1279. 	KERNEL2x16_2

  1280. 

  1281. 	addic.		L,	L,	-2

  1282. 	ble		SGEMM_L2x16_LOOP_END

  1283. 

  1284. 	.align 5

  1285. 

  1286. SGEMM_L2x16_LOOP:

  1287. 

  1288. 	KERNEL2x16_1

  1289. 	dcbt		AO,	PRE

  1290. 	KERNEL2x16_2

  1291. 	KERNEL2x16_1

  1292. 	dcbt		AO,	PRE

  1293. 	KERNEL2x16_2

  1294. 

  1295. 	KERNEL2x16_1

  1296. 	dcbt		AO,	PRE

  1297. 	KERNEL2x16_2

  1298. 	KERNEL2x16_1

  1299. 	dcbt		AO,	PRE

  1300. 	KERNEL2x16_2

  1301. 

  1302. 	addic.		L,	L,	-1

  1303. 	bgt		SGEMM_L2x16_LOOP

  1304. 

  1305. SGEMM_L2x16_LOOP_END:

  1306. 

  1307. 	KERNEL2x16_1

  1308. 	dcbt		AO,	PRE

  1309. 	KERNEL2x16_2

  1310. 	KERNEL2x16_1

  1311. 	dcbt		AO,	PRE

  1312. 	KERNEL2x16_2

  1313. 

  1314. 	KERNEL2x16_1

  1315. 	dcbt		AO,	PRE

  1316. 	KERNEL2x16_2

  1317. 	KERNEL2x16_1

  1318. 	KERNEL2x16_E2

  1319. 

  1320. 	b		SGEMM_L2x16_SUB1

  1321. 

  1322. SGEMM_L2x16_SUB4:

  1323. 

  1324. 	dcbt		AO,	PRE

  1325. 	KERNEL2x16_SUBI1

  1326. 	KERNEL2x16_SUB1

  1327. 	dcbt		AO,	PRE

  1328. 	KERNEL2x16_SUB1

  1329. 	KERNEL2x16_SUB1

  1330. 

  1331. 	KERNEL2x16_SUB1

  1332. 	KERNEL2x16_SUB1

  1333. 	KERNEL2x16_SUB1

  1334. 	KERNEL2x16_SUB1

  1335. 

  1336. 	b		SGEMM_L2x16_SUB1

  1337. 

  1338. SGEMM_L2x16_SUB0:

  1339. 

  1340. 	andi.		L,	K,	7

  1341. 

  1342. 	KERNEL2x16_SUBI1

  1343. 

  1344. 	addic.		L,	L,	-1

  1345. 	ble		SGEMM_L2x16_SAVE

  1346. 	b		SGEMM_L2x16_SUB2

  1347. 

  1348. SGEMM_L2x16_SUB1:

  1349. 

  1350. 	andi.		L,	K,	7

  1351. 	ble		SGEMM_L2x16_SAVE

  1352. 

  1353. SGEMM_L2x16_SUB2:

  1354. 

  1355. 	KERNEL2x16_SUB1

  1356. 

  1357. 	addic.		L,	L,	-1

  1358. 	bgt		SGEMM_L2x16_SUB2

  1359. 

  1360. SGEMM_L2x16_SAVE:

  1361. 

  1362. 	SAVE2x16

  1363. 

  1364. 	addic.		I,	I,	-1

  1365. 	bgt		SGEMM_L2x16_BEGIN

  1366. 

  1367. SGEMM_L2x16_END:

  1368. 

  1369. SGEMM_L2x8_BEGIN:

  1370. 

  1371. 	andi.		T2,	M,	15

  1372. 	ble		SGEMM_L2x1_END

  1373. 

  1374. 	andi.		T1,	M,	8

  1375. 	ble		SGEMM_L2x8_END

  1376. 	mr		BO,	BBUFFER

  1377. 	srawi.		L,	K,	3

  1378. 	ble		SGEMM_L2x8_SUB0

  1379. 	cmpwi		cr0,	L,	1

  1380. 	ble		SGEMM_L2x8_SUB4

  1381. 

  1382. SGEMM_L2x8_LOOP_START:

  1383. 

  1384. 	LOAD2x8_1

  1385. 	KERNEL2x8_I1

  1386. 	KERNEL2x8_2

  1387. 	KERNEL2x8_1

  1388. 	KERNEL2x8_2

  1389. 

  1390. 	KERNEL2x8_1

  1391. 	KERNEL2x8_2

  1392. 	KERNEL2x8_1

  1393. 	KERNEL2x8_2

  1394. 

  1395. 	addic.		L,	L,	-2

  1396. 	ble		SGEMM_L2x8_LOOP_END

  1397. 

  1398. 	.align 5

  1399. 

  1400. SGEMM_L2x8_LOOP:

  1401. 

  1402. 	KERNEL2x8_1

  1403. 	KERNEL2x8_2

  1404. 	KERNEL2x8_1

  1405. 	KERNEL2x8_2

  1406. 

  1407. 	KERNEL2x8_1

  1408. 	KERNEL2x8_2

  1409. 	KERNEL2x8_1

  1410. 	KERNEL2x8_2

  1411. 

  1412. 	addic.		L,	L,	-1

  1413. 	bgt		SGEMM_L2x8_LOOP

  1414. 

  1415. SGEMM_L2x8_LOOP_END:

  1416. 

  1417. 	KERNEL2x8_1

  1418. 	KERNEL2x8_2

  1419. 	KERNEL2x8_1

  1420. 	KERNEL2x8_2

  1421. 

  1422. 	KERNEL2x8_1

  1423. 	KERNEL2x8_2

  1424. 	KERNEL2x8_1

  1425. 	KERNEL2x8_E2

  1426. 

  1427. 	b		SGEMM_L2x8_SUB1

  1428. 

  1429. SGEMM_L2x8_SUB4:

  1430. 

  1431. 	KERNEL2x8_SUBI1

  1432. 	KERNEL2x8_SUB1

  1433. 	KERNEL2x8_SUB1

  1434. 	KERNEL2x8_SUB1

  1435. 

  1436. 	KERNEL2x8_SUB1

  1437. 	KERNEL2x8_SUB1

  1438. 	KERNEL2x8_SUB1

  1439. 	KERNEL2x8_SUB1

  1440. 

  1441. 	b		SGEMM_L2x8_SUB1

  1442. 

  1443. SGEMM_L2x8_SUB0:

  1444. 

  1445. 	andi.		L,	K,	7

  1446. 

  1447. 	KERNEL2x8_SUBI1

  1448. 

  1449. 	addic.		L,	L,	-1

  1450. 	ble		SGEMM_L2x8_SAVE

  1451. 	b		SGEMM_L2x8_SUB2

  1452. 

  1453. SGEMM_L2x8_SUB1:

  1454. 

  1455. 	andi.		L,	K,	7

  1456. 	ble		SGEMM_L2x8_SAVE

  1457. 

  1458. SGEMM_L2x8_SUB2:

  1459. 

  1460. 	KERNEL2x8_SUB1

  1461. 

  1462. 	addic.		L,	L,	-1

  1463. 	bgt		SGEMM_L2x8_SUB2

  1464. 

  1465. SGEMM_L2x8_SAVE:

  1466. 

  1467. 	SAVE2x8

  1468. 

  1469. SGEMM_L2x8_END:

  1470. 

  1471. SGEMM_L2x4_BEGIN:

  1472. 

  1473. 

  1474. 	andi.		T1,	M,	4

  1475. 	ble		SGEMM_L2x4_END

  1476. 	mr		BO,	BBUFFER

  1477. 	srawi.		L,	K,	3

  1478. 	ble		SGEMM_L2x4_SUB0

  1479. 	cmpwi		cr0,	L,	1

  1480. 	ble		SGEMM_L2x4_SUB4

  1481. 

  1482. SGEMM_L2x4_LOOP_START:

  1483. 

  1484. 	LOAD2x4_1

  1485. 	KERNEL2x4_I1

  1486. 	KERNEL2x4_2

  1487. 	KERNEL2x4_1

  1488. 	KERNEL2x4_2

  1489. 

  1490. 	KERNEL2x4_1

  1491. 	KERNEL2x4_2

  1492. 	KERNEL2x4_1

  1493. 	KERNEL2x4_2

  1494. 

  1495. 	addic.		L,	L,	-2

  1496. 	ble		SGEMM_L2x4_LOOP_END

  1497. 

  1498. 	.align 5

  1499. 

  1500. SGEMM_L2x4_LOOP:

  1501. 

  1502. 	KERNEL2x4_1

  1503. 	KERNEL2x4_2

  1504. 	KERNEL2x4_1

  1505. 	KERNEL2x4_2

  1506. 

  1507. 	KERNEL2x4_1

  1508. 	KERNEL2x4_2

  1509. 	KERNEL2x4_1

  1510. 	KERNEL2x4_2

  1511. 

  1512. 	addic.		L,	L,	-1

  1513. 	bgt		SGEMM_L2x4_LOOP

  1514. 

  1515. SGEMM_L2x4_LOOP_END:

  1516. 

  1517. 	KERNEL2x4_1

  1518. 	KERNEL2x4_2

  1519. 	KERNEL2x4_1

  1520. 	KERNEL2x4_2

  1521. 

  1522. 	KERNEL2x4_1

  1523. 	KERNEL2x4_2

  1524. 	KERNEL2x4_1

  1525. 	KERNEL2x4_E2

  1526. 

  1527. 	b		SGEMM_L2x4_SUB1

  1528. 

  1529. SGEMM_L2x4_SUB4:

  1530. 

  1531. 	KERNEL2x4_SUBI1

  1532. 	KERNEL2x4_SUB1

  1533. 	KERNEL2x4_SUB1

  1534. 	KERNEL2x4_SUB1

  1535. 

  1536. 	KERNEL2x4_SUB1

  1537. 	KERNEL2x4_SUB1

  1538. 	KERNEL2x4_SUB1

  1539. 	KERNEL2x4_SUB1

  1540. 

  1541. 	b		SGEMM_L2x4_SUB1

  1542. 

  1543. SGEMM_L2x4_SUB0:

  1544. 

  1545. 	andi.		L,	K,	7

  1546. 

  1547. 	KERNEL2x4_SUBI1

  1548. 

  1549. 	addic.		L,	L,	-1

  1550. 	ble		SGEMM_L2x4_SAVE

  1551. 	b		SGEMM_L2x4_SUB2

  1552. 

  1553. SGEMM_L2x4_SUB1:

  1554. 

  1555. 	andi.		L,	K,	7

  1556. 	ble		SGEMM_L2x4_SAVE

  1557. 

  1558. SGEMM_L2x4_SUB2:

  1559. 

  1560. 	KERNEL2x4_SUB1

  1561. 

  1562. 	addic.		L,	L,	-1

  1563. 	bgt		SGEMM_L2x4_SUB2

  1564. 

  1565. SGEMM_L2x4_SAVE:

  1566. 

  1567. 	SAVE2x4

  1568. 

  1569. SGEMM_L2x4_END:

  1570. 

  1571. SGEMM_L2x2_BEGIN:

  1572. 

  1573. 

  1574. 	andi.		T1,	M,	2

  1575. 	ble		SGEMM_L2x2_END

  1576. 	mr		BO,	BBUFFER

  1577. 	srawi.		L,	K,	3

  1578. 	ble		SGEMM_L2x2_SUB0

  1579. 	cmpwi		cr0,	L,	1

  1580. 	ble		SGEMM_L2x2_SUB4

  1581. 

  1582. SGEMM_L2x2_LOOP_START:

  1583. 

  1584. 	LOAD2x2_1

  1585. 	KERNEL2x2_I1

  1586. 	KERNEL2x2_2

  1587. 	KERNEL2x2_1

  1588. 	KERNEL2x2_2

  1589. 

  1590. 	KERNEL2x2_1

  1591. 	KERNEL2x2_2

  1592. 	KERNEL2x2_1

  1593. 	KERNEL2x2_2

  1594. 

  1595. 	addic.		L,	L,	-2

  1596. 	ble		SGEMM_L2x2_LOOP_END

  1597. 

  1598. 	.align 5

  1599. 

  1600. SGEMM_L2x2_LOOP:

  1601. 

  1602. 	KERNEL2x2_1

  1603. 	KERNEL2x2_2

  1604. 	KERNEL2x2_1

  1605. 	KERNEL2x2_2

  1606. 

  1607. 	KERNEL2x2_1

  1608. 	KERNEL2x2_2

  1609. 	KERNEL2x2_1

  1610. 	KERNEL2x2_2

  1611. 

  1612. 	addic.		L,	L,	-1

  1613. 	bgt		SGEMM_L2x2_LOOP

  1614. 

  1615. SGEMM_L2x2_LOOP_END:

  1616. 

  1617. 	KERNEL2x2_1

  1618. 	KERNEL2x2_2

  1619. 	KERNEL2x2_1

  1620. 	KERNEL2x2_2

  1621. 

  1622. 	KERNEL2x2_1

  1623. 	KERNEL2x2_2

  1624. 	KERNEL2x2_1

  1625. 	KERNEL2x2_E2

  1626. 

  1627. 	b		SGEMM_L2x2_SUB1

  1628. 

  1629. SGEMM_L2x2_SUB4:

  1630. 

  1631. 	KERNEL2x2_SUBI1

  1632. 	KERNEL2x2_SUB1

  1633. 	KERNEL2x2_SUB1

  1634. 	KERNEL2x2_SUB1

  1635. 

  1636. 	KERNEL2x2_SUB1

  1637. 	KERNEL2x2_SUB1

  1638. 	KERNEL2x2_SUB1

  1639. 	KERNEL2x2_SUB1

  1640. 

  1641. 	b		SGEMM_L2x2_SUB1

  1642. 

  1643. SGEMM_L2x2_SUB0:

  1644. 

  1645. 	andi.		L,	K,	7

  1646. 

  1647. 	KERNEL2x2_SUBI1

  1648. 

  1649. 	addic.		L,	L,	-1

  1650. 	ble		SGEMM_L2x2_SAVE

  1651. 	b		SGEMM_L2x2_SUB2

  1652. 

  1653. SGEMM_L2x2_SUB1:

  1654. 

  1655. 	andi.		L,	K,	7

  1656. 	ble		SGEMM_L2x2_SAVE

  1657. 

  1658. SGEMM_L2x2_SUB2:

  1659. 

  1660. 	KERNEL2x2_SUB1

  1661. 

  1662. 	addic.		L,	L,	-1

  1663. 	bgt		SGEMM_L2x2_SUB2

  1664. 

  1665. SGEMM_L2x2_SAVE:

  1666. 

  1667. 	SAVE2x2

  1668. 

  1669. SGEMM_L2x2_END:

  1670. 

  1671. SGEMM_L2x1_BEGIN:

  1672. 

  1673. 

  1674. 	andi.		T1,	M,	1

  1675. 	ble		SGEMM_L2x1_END

  1676. 	mr		BO,	BBUFFER

  1677. 	srawi.		L,	K,	3

  1678. 	ble		SGEMM_L2x1_SUB0

  1679. 	cmpwi		cr0,	L,	1

  1680. 	ble		SGEMM_L2x1_SUB4

  1681. 

  1682. SGEMM_L2x1_LOOP_START:

  1683. 

  1684. 	LOAD2x1_1

  1685. 	KERNEL2x1_I1

  1686. 	KERNEL2x1_2

  1687. 	KERNEL2x1_1

  1688. 	KERNEL2x1_2

  1689. 

  1690. 	KERNEL2x1_1

  1691. 	KERNEL2x1_2

  1692. 	KERNEL2x1_1

  1693. 	KERNEL2x1_2

  1694. 

  1695. 	addic.		L,	L,	-2

  1696. 	ble		SGEMM_L2x1_LOOP_END

  1697. 

  1698. 	.align 5

  1699. 

  1700. SGEMM_L2x1_LOOP:

  1701. 

  1702. 	KERNEL2x1_1

  1703. 	KERNEL2x1_2

  1704. 	KERNEL2x1_1

  1705. 	KERNEL2x1_2

  1706. 

  1707. 	KERNEL2x1_1

  1708. 	KERNEL2x1_2

  1709. 	KERNEL2x1_1

  1710. 	KERNEL2x1_2

  1711. 

  1712. 	addic.		L,	L,	-1

  1713. 	bgt		SGEMM_L2x1_LOOP

  1714. 

  1715. SGEMM_L2x1_LOOP_END:

  1716. 

  1717. 	KERNEL2x1_1

  1718. 	KERNEL2x1_2

  1719. 	KERNEL2x1_1

  1720. 	KERNEL2x1_2

  1721. 

  1722. 	KERNEL2x1_1

  1723. 	KERNEL2x1_2

  1724. 	KERNEL2x1_1

  1725. 	KERNEL2x1_E2

  1726. 

  1727. 	b		SGEMM_L2x1_SUB1

  1728. 

  1729. SGEMM_L2x1_SUB4:

  1730. 

  1731. 	KERNEL2x1_SUBI1

  1732. 	KERNEL2x1_SUB1

  1733. 	KERNEL2x1_SUB1

  1734. 	KERNEL2x1_SUB1

  1735. 

  1736. 	KERNEL2x1_SUB1

  1737. 	KERNEL2x1_SUB1

  1738. 	KERNEL2x1_SUB1

  1739. 	KERNEL2x1_SUB1

  1740. 

  1741. 	b		SGEMM_L2x1_SUB1

  1742. 

  1743. SGEMM_L2x1_SUB0:

  1744. 

  1745. 	andi.		L,	K,	7

  1746. 

  1747. 	KERNEL2x1_SUBI1

  1748. 

  1749. 	addic.		L,	L,	-1

  1750. 	ble		SGEMM_L2x1_SAVE

  1751. 	b		SGEMM_L2x1_SUB2

  1752. 

  1753. SGEMM_L2x1_SUB1:

  1754. 

  1755. 	andi.		L,	K,	7

  1756. 	ble		SGEMM_L2x1_SAVE

  1757. 

  1758. SGEMM_L2x1_SUB2:

  1759. 

  1760. 	KERNEL2x1_SUB1

  1761. 

  1762. 	addic.		L,	L,	-1

  1763. 	bgt		SGEMM_L2x1_SUB2

  1764. 

  1765. SGEMM_L2x1_SAVE:

  1766. 

  1767. 	SAVE2x1

  1768. 

  1769. SGEMM_L2x1_END:

  1770. 

  1771. 	slwi		T1,	K,	3

  1772. 	add		B,	B,	T1

  1773. 

  1774. SGEMM_L2_END:

  1775. SGEMM_L1_BEGIN:

  1776. 

  1777. 	mr		BO,	B

  1778. 	mr		BBO,	BBUFFER

  1779. 	slwi		T1,	K,	0

  1780. 

  1781. SGEMM_L1_COPYB:

  1782. 	dcbtst		BBO,	PRE

  1783. 

  1784. 	lxvw4x		vs3,	o0,	BO

  1785. 	lxvw4x		vs11,	o16,	BO

  1786. 	xxspltw		vs4,	vs3,	0

  1787. 	xxspltw		vs5,	vs3,	1

  1788. 	xxspltw		vs6,	vs3,	2

  1789. 	xxspltw		vs7,	vs3,	3

  1790. 	xxspltw		vs12,	vs11,	0

  1791. 	xxspltw		vs13,	vs11,	1

  1792. 	xxspltw		vs14,	vs11,	2

  1793. 	xxspltw		vs15,	vs11,	3

  1794. 	stxvw4x		vs4,	o0,	BBO

  1795. 	stxvw4x		vs5,	o16,	BBO

  1796. 	stxvw4x		vs6,	o32,	BBO

  1797. 	stxvw4x		vs7,	o48,	BBO

  1798. 	addi		BO,	BO,	32

  1799. 	addi		BBO,	BBO,	64

  1800. 	stxvw4x		vs12,	o0,	BBO

  1801. 	stxvw4x		vs13,	o16,	BBO

  1802. 	stxvw4x		vs14,	o32,	BBO

  1803. 	stxvw4x		vs15,	o48,	BBO

  1804. 	addic.		T1,	T1,	-8

  1805. 	addi		BBO,	BBO,	64

  1806. 

  1807. 	bge		SGEMM_L1_COPYB

  1808. 

  1809. 	andi.		T1,	N,	1

  1810. 	ble		SGEMM_L1_END

  1811. 	mr		CO,	C

  1812. 	mr		AO,	A

  1813. 	srawi.		I,	M,	4

  1814. 	ble		SGEMM_L1x16_END

  1815. 

  1816. SGEMM_L1x16_BEGIN:

  1817. 

  1818. 

  1819. 	mr		BO,	BBUFFER

  1820. 	srawi.		L,	K,	3

  1821. 	ble		SGEMM_L1x16_SUB0

  1822. 	cmpwi		cr0,	L,	1

  1823. 	ble		SGEMM_L1x16_SUB4

  1824. 

  1825. SGEMM_L1x16_LOOP_START:

  1826. 

  1827. 	dcbt		AO,	PRE

  1828. 	LOAD1x16_1

  1829. 	KERNEL1x16_I1

  1830. 	dcbt		AO,	PRE

  1831. 	KERNEL1x16_2

  1832. 	KERNEL1x16_1

  1833. 	dcbt		AO,	PRE

  1834. 	KERNEL1x16_2

  1835. 

  1836. 	KERNEL1x16_1

  1837. 	dcbt		AO,	PRE

  1838. 	KERNEL1x16_2

  1839. 	KERNEL1x16_1

  1840. 	dcbt		AO,	PRE

  1841. 	KERNEL1x16_2

  1842. 

  1843. 	addic.		L,	L,	-2

  1844. 	ble		SGEMM_L1x16_LOOP_END

  1845. 

  1846. 	.align 5

  1847. 

  1848. SGEMM_L1x16_LOOP:

  1849. 

  1850. 	KERNEL1x16_1

  1851. 	dcbt		AO,	PRE

  1852. 	KERNEL1x16_2

  1853. 	KERNEL1x16_1

  1854. 	dcbt		AO,	PRE

  1855. 	KERNEL1x16_2

  1856. 

  1857. 	KERNEL1x16_1

  1858. 	dcbt		AO,	PRE

  1859. 	KERNEL1x16_2

  1860. 	KERNEL1x16_1

  1861. 	dcbt		AO,	PRE

  1862. 	KERNEL1x16_2

  1863. 

  1864. 	addic.		L,	L,	-1

  1865. 	bgt		SGEMM_L1x16_LOOP

  1866. 

  1867. SGEMM_L1x16_LOOP_END:

  1868. 

  1869. 	KERNEL1x16_1

  1870. 	dcbt		AO,	PRE

  1871. 	KERNEL1x16_2

  1872. 	KERNEL1x16_1

  1873. 	dcbt		AO,	PRE

  1874. 	KERNEL1x16_2

  1875. 

  1876. 	KERNEL1x16_1

  1877. 	dcbt		AO,	PRE

  1878. 	KERNEL1x16_2

  1879. 	KERNEL1x16_1

  1880. 	KERNEL1x16_E2

  1881. 

  1882. 	b		SGEMM_L1x16_SUB1

  1883. 

  1884. SGEMM_L1x16_SUB4:

  1885. 

  1886. 	dcbt		AO,	PRE

  1887. 	KERNEL1x16_SUBI1

  1888. 	KERNEL1x16_SUB1

  1889. 	dcbt		AO,	PRE

  1890. 	KERNEL1x16_SUB1

  1891. 	KERNEL1x16_SUB1

  1892. 

  1893. 	KERNEL1x16_SUB1

  1894. 	KERNEL1x16_SUB1

  1895. 	KERNEL1x16_SUB1

  1896. 	KERNEL1x16_SUB1

  1897. 

  1898. 	b		SGEMM_L1x16_SUB1

  1899. 

  1900. SGEMM_L1x16_SUB0:

  1901. 

  1902. 	andi.		L,	K,	7

  1903. 

  1904. 	KERNEL1x16_SUBI1

  1905. 

  1906. 	addic.		L,	L,	-1

  1907. 	ble		SGEMM_L1x16_SAVE

  1908. 	b		SGEMM_L1x16_SUB2

  1909. 

  1910. SGEMM_L1x16_SUB1:

  1911. 

  1912. 	andi.		L,	K,	7

  1913. 	ble		SGEMM_L1x16_SAVE

  1914. 

  1915. SGEMM_L1x16_SUB2:

  1916. 

  1917. 	KERNEL1x16_SUB1

  1918. 

  1919. 	addic.		L,	L,	-1

  1920. 	bgt		SGEMM_L1x16_SUB2

  1921. 

  1922. SGEMM_L1x16_SAVE:

  1923. 

  1924. 	SAVE1x16

  1925. 

  1926. 	addic.		I,	I,	-1

  1927. 	bgt		SGEMM_L1x16_BEGIN

  1928. 

  1929. SGEMM_L1x16_END:

  1930. 

  1931. SGEMM_L1x8_BEGIN:

  1932. 

  1933. 	andi.		T2,	M,	15

  1934. 	ble		SGEMM_L1x1_END

  1935. 

  1936. 	andi.		T1,	M,	8

  1937. 	ble		SGEMM_L1x8_END

  1938. 	mr		BO,	BBUFFER

  1939. 	srawi.		L,	K,	3

  1940. 	ble		SGEMM_L1x8_SUB0

  1941. 	cmpwi		cr0,	L,	1

  1942. 	ble		SGEMM_L1x8_SUB4

  1943. 

  1944. SGEMM_L1x8_LOOP_START:

  1945. 

  1946. 	LOAD1x8_1

  1947. 	KERNEL1x8_I1

  1948. 	KERNEL1x8_2

  1949. 	KERNEL1x8_1

  1950. 	KERNEL1x8_2

  1951. 

  1952. 	KERNEL1x8_1

  1953. 	KERNEL1x8_2

  1954. 	KERNEL1x8_1

  1955. 	KERNEL1x8_2

  1956. 

  1957. 	addic.		L,	L,	-2

  1958. 	ble		SGEMM_L1x8_LOOP_END

  1959. 

  1960. 	.align 5

  1961. 

  1962. SGEMM_L1x8_LOOP:

  1963. 

  1964. 	KERNEL1x8_1

  1965. 	KERNEL1x8_2

  1966. 	KERNEL1x8_1

  1967. 	KERNEL1x8_2

  1968. 

  1969. 	KERNEL1x8_1

  1970. 	KERNEL1x8_2

  1971. 	KERNEL1x8_1

  1972. 	KERNEL1x8_2

  1973. 

  1974. 	addic.		L,	L,	-1

  1975. 	bgt		SGEMM_L1x8_LOOP

  1976. 

  1977. SGEMM_L1x8_LOOP_END:

  1978. 

  1979. 	KERNEL1x8_1

  1980. 	KERNEL1x8_2

  1981. 	KERNEL1x8_1

  1982. 	KERNEL1x8_2

  1983. 

  1984. 	KERNEL1x8_1

  1985. 	KERNEL1x8_2

  1986. 	KERNEL1x8_1

  1987. 	KERNEL1x8_E2

  1988. 

  1989. 	b		SGEMM_L1x8_SUB1

  1990. 

  1991. SGEMM_L1x8_SUB4:

  1992. 

  1993. 	KERNEL1x8_SUBI1

  1994. 	KERNEL1x8_SUB1

  1995. 	KERNEL1x8_SUB1

  1996. 	KERNEL1x8_SUB1

  1997. 

  1998. 	KERNEL1x8_SUB1

  1999. 	KERNEL1x8_SUB1

  2000. 	KERNEL1x8_SUB1

  2001. 	KERNEL1x8_SUB1

  2002. 

  2003. 	b		SGEMM_L1x8_SUB1

  2004. 

  2005. SGEMM_L1x8_SUB0:

  2006. 

  2007. 	andi.		L,	K,	7

  2008. 

  2009. 	KERNEL1x8_SUBI1

  2010. 

  2011. 	addic.		L,	L,	-1

  2012. 	ble		SGEMM_L1x8_SAVE

  2013. 	b		SGEMM_L1x8_SUB2

  2014. 

  2015. SGEMM_L1x8_SUB1:

  2016. 

  2017. 	andi.		L,	K,	7

  2018. 	ble		SGEMM_L1x8_SAVE

  2019. 

  2020. SGEMM_L1x8_SUB2:

  2021. 

  2022. 	KERNEL1x8_SUB1

  2023. 

  2024. 	addic.		L,	L,	-1

  2025. 	bgt		SGEMM_L1x8_SUB2

  2026. 

  2027. SGEMM_L1x8_SAVE:

  2028. 

  2029. 	SAVE1x8

  2030. 

  2031. SGEMM_L1x8_END:

  2032. 

  2033. SGEMM_L1x4_BEGIN:

  2034. 

  2035. 

  2036. 	andi.		T1,	M,	4

  2037. 	ble		SGEMM_L1x4_END

  2038. 	mr		BO,	BBUFFER

  2039. 	srawi.		L,	K,	3

  2040. 	ble		SGEMM_L1x4_SUB0

  2041. 	cmpwi		cr0,	L,	1

  2042. 	ble		SGEMM_L1x4_SUB4

  2043. 

  2044. SGEMM_L1x4_LOOP_START:

  2045. 

  2046. 	LOAD1x4_1

  2047. 	KERNEL1x4_I1

  2048. 	KERNEL1x4_2

  2049. 	KERNEL1x4_1

  2050. 	KERNEL1x4_2

  2051. 

  2052. 	KERNEL1x4_1

  2053. 	KERNEL1x4_2

  2054. 	KERNEL1x4_1

  2055. 	KERNEL1x4_2

  2056. 

  2057. 	addic.		L,	L,	-2

  2058. 	ble		SGEMM_L1x4_LOOP_END

  2059. 

  2060. 	.align 5

  2061. 

  2062. SGEMM_L1x4_LOOP:

  2063. 

  2064. 	KERNEL1x4_1

  2065. 	KERNEL1x4_2

  2066. 	KERNEL1x4_1

  2067. 	KERNEL1x4_2

  2068. 

  2069. 	KERNEL1x4_1

  2070. 	KERNEL1x4_2

  2071. 	KERNEL1x4_1

  2072. 	KERNEL1x4_2

  2073. 

  2074. 	addic.		L,	L,	-1

  2075. 	bgt		SGEMM_L1x4_LOOP

  2076. 

  2077. SGEMM_L1x4_LOOP_END:

  2078. 

  2079. 	KERNEL1x4_1

  2080. 	KERNEL1x4_2

  2081. 	KERNEL1x4_1

  2082. 	KERNEL1x4_2

  2083. 

  2084. 	KERNEL1x4_1

  2085. 	KERNEL1x4_2

  2086. 	KERNEL1x4_1

  2087. 	KERNEL1x4_E2

  2088. 

  2089. 	b		SGEMM_L1x4_SUB1

  2090. 

  2091. SGEMM_L1x4_SUB4:

  2092. 

  2093. 	KERNEL1x4_SUBI1

  2094. 	KERNEL1x4_SUB1

  2095. 	KERNEL1x4_SUB1

  2096. 	KERNEL1x4_SUB1

  2097. 

  2098. 	KERNEL1x4_SUB1

  2099. 	KERNEL1x4_SUB1

  2100. 	KERNEL1x4_SUB1

  2101. 	KERNEL1x4_SUB1

  2102. 

  2103. 	b		SGEMM_L1x4_SUB1

  2104. 

  2105. SGEMM_L1x4_SUB0:

  2106. 

  2107. 	andi.		L,	K,	7

  2108. 

  2109. 	KERNEL1x4_SUBI1

  2110. 

  2111. 	addic.		L,	L,	-1

  2112. 	ble		SGEMM_L1x4_SAVE

  2113. 	b		SGEMM_L1x4_SUB2

  2114. 

  2115. SGEMM_L1x4_SUB1:

  2116. 

  2117. 	andi.		L,	K,	7

  2118. 	ble		SGEMM_L1x4_SAVE

  2119. 

  2120. SGEMM_L1x4_SUB2:

  2121. 

  2122. 	KERNEL1x4_SUB1

  2123. 

  2124. 	addic.		L,	L,	-1

  2125. 	bgt		SGEMM_L1x4_SUB2

  2126. 

  2127. SGEMM_L1x4_SAVE:

  2128. 

  2129. 	SAVE1x4

  2130. 

  2131. SGEMM_L1x4_END:

  2132. 

  2133. SGEMM_L1x2_BEGIN:

  2134. 

  2135. 

  2136. 	andi.		T1,	M,	2

  2137. 	ble		SGEMM_L1x2_END

  2138. 	mr		BO,	BBUFFER

  2139. 	srawi.		L,	K,	3

  2140. 	ble		SGEMM_L1x2_SUB0

  2141. 	cmpwi		cr0,	L,	1

  2142. 	ble		SGEMM_L1x2_SUB4

  2143. 

  2144. SGEMM_L1x2_LOOP_START:

  2145. 

  2146. 	LOAD1x2_1

  2147. 	KERNEL1x2_I1

  2148. 	KERNEL1x2_2

  2149. 	KERNEL1x2_1

  2150. 	KERNEL1x2_2

  2151. 

  2152. 	KERNEL1x2_1

  2153. 	KERNEL1x2_2

  2154. 	KERNEL1x2_1

  2155. 	KERNEL1x2_2

  2156. 

  2157. 	addic.		L,	L,	-2

  2158. 	ble		SGEMM_L1x2_LOOP_END

  2159. 

  2160. 	.align 5

  2161. 

  2162. SGEMM_L1x2_LOOP:

  2163. 

  2164. 	KERNEL1x2_1

  2165. 	KERNEL1x2_2

  2166. 	KERNEL1x2_1

  2167. 	KERNEL1x2_2

  2168. 

  2169. 	KERNEL1x2_1

  2170. 	KERNEL1x2_2

  2171. 	KERNEL1x2_1

  2172. 	KERNEL1x2_2

  2173. 

  2174. 	addic.		L,	L,	-1

  2175. 	bgt		SGEMM_L1x2_LOOP

  2176. 

  2177. SGEMM_L1x2_LOOP_END:

  2178. 

  2179. 	KERNEL1x2_1

  2180. 	KERNEL1x2_2

  2181. 	KERNEL1x2_1

  2182. 	KERNEL1x2_2

  2183. 

  2184. 	KERNEL1x2_1

  2185. 	KERNEL1x2_2

  2186. 	KERNEL1x2_1

  2187. 	KERNEL1x2_E2

  2188. 

  2189. 	b		SGEMM_L1x2_SUB1

  2190. 

  2191. SGEMM_L1x2_SUB4:

  2192. 

  2193. 	KERNEL1x2_SUBI1

  2194. 	KERNEL1x2_SUB1

  2195. 	KERNEL1x2_SUB1

  2196. 	KERNEL1x2_SUB1

  2197. 

  2198. 	KERNEL1x2_SUB1

  2199. 	KERNEL1x2_SUB1

  2200. 	KERNEL1x2_SUB1

  2201. 	KERNEL1x2_SUB1

  2202. 

  2203. 	b		SGEMM_L1x2_SUB1

  2204. 

  2205. SGEMM_L1x2_SUB0:

  2206. 

  2207. 	andi.		L,	K,	7

  2208. 

  2209. 	KERNEL1x2_SUBI1

  2210. 

  2211. 	addic.		L,	L,	-1

  2212. 	ble		SGEMM_L1x2_SAVE

  2213. 	b		SGEMM_L1x2_SUB2

  2214. 

  2215. SGEMM_L1x2_SUB1:

  2216. 

  2217. 	andi.		L,	K,	7

  2218. 	ble		SGEMM_L1x2_SAVE

  2219. 

  2220. SGEMM_L1x2_SUB2:

  2221. 

  2222. 	KERNEL1x2_SUB1

  2223. 

  2224. 	addic.		L,	L,	-1

  2225. 	bgt		SGEMM_L1x2_SUB2

  2226. 

  2227. SGEMM_L1x2_SAVE:

  2228. 

  2229. 	SAVE1x2

  2230. 

  2231. SGEMM_L1x2_END:

  2232. 

  2233. SGEMM_L1x1_BEGIN:

  2234. 

  2235. 

  2236. 	andi.		T1,	M,	1

  2237. 	ble		SGEMM_L1x1_END

  2238. 	mr		BO,	BBUFFER

  2239. 	srawi.		L,	K,	3

  2240. 	ble		SGEMM_L1x1_SUB0

  2241. 	cmpwi		cr0,	L,	1

  2242. 	ble		SGEMM_L1x1_SUB4

  2243. 

  2244. SGEMM_L1x1_LOOP_START:

  2245. 

  2246. 	LOAD1x1_1

  2247. 	KERNEL1x1_I1

  2248. 	KERNEL1x1_2

  2249. 	KERNEL1x1_1

  2250. 	KERNEL1x1_2

  2251. 

  2252. 	KERNEL1x1_1

  2253. 	KERNEL1x1_2

  2254. 	KERNEL1x1_1

  2255. 	KERNEL1x1_2

  2256. 

  2257. 	addic.		L,	L,	-2

  2258. 	ble		SGEMM_L1x1_LOOP_END

  2259. 

  2260. 	.align 5

  2261. 

  2262. SGEMM_L1x1_LOOP:

  2263. 

  2264. 	KERNEL1x1_1

  2265. 	KERNEL1x1_2

  2266. 	KERNEL1x1_1

  2267. 	KERNEL1x1_2

  2268. 

  2269. 	KERNEL1x1_1

  2270. 	KERNEL1x1_2

  2271. 	KERNEL1x1_1

  2272. 	KERNEL1x1_2

  2273. 

  2274. 	addic.		L,	L,	-1

  2275. 	bgt		SGEMM_L1x1_LOOP

  2276. 

  2277. SGEMM_L1x1_LOOP_END:

  2278. 

  2279. 	KERNEL1x1_1

  2280. 	KERNEL1x1_2

  2281. 	KERNEL1x1_1

  2282. 	KERNEL1x1_2

  2283. 

  2284. 	KERNEL1x1_1

  2285. 	KERNEL1x1_2

  2286. 	KERNEL1x1_1

  2287. 	KERNEL1x1_E2

  2288. 

  2289. 	b		SGEMM_L1x1_SUB1

  2290. 

  2291. SGEMM_L1x1_SUB4:

  2292. 

  2293. 	KERNEL1x1_SUBI1

  2294. 	KERNEL1x1_SUB1

  2295. 	KERNEL1x1_SUB1

  2296. 	KERNEL1x1_SUB1

  2297. 

  2298. 	KERNEL1x1_SUB1

  2299. 	KERNEL1x1_SUB1

  2300. 	KERNEL1x1_SUB1

  2301. 	KERNEL1x1_SUB1

  2302. 

  2303. 	b		SGEMM_L1x1_SUB1

  2304. 

  2305. SGEMM_L1x1_SUB0:

  2306. 

  2307. 	andi.		L,	K,	7

  2308. 

  2309. 	KERNEL1x1_SUBI1

  2310. 

  2311. 	addic.		L,	L,	-1

  2312. 	ble		SGEMM_L1x1_SAVE

  2313. 	b		SGEMM_L1x1_SUB2

  2314. 

  2315. SGEMM_L1x1_SUB1:

  2316. 

  2317. 	andi.		L,	K,	7

  2318. 	ble		SGEMM_L1x1_SAVE

  2319. 

  2320. SGEMM_L1x1_SUB2:

  2321. 

  2322. 	KERNEL1x1_SUB1

  2323. 

  2324. 	addic.		L,	L,	-1

  2325. 	bgt		SGEMM_L1x1_SUB2

  2326. 

  2327. SGEMM_L1x1_SAVE:

  2328. 

  2329. 	SAVE1x1

  2330. 

  2331. SGEMM_L1x1_END:

  2332. 

  2333. SGEMM_L1_END: