OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 855945befb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '855945befb'
${ noResults }
OpenBLAS/kernel/power/cgemm_logic_8x4_power8.S

1460 lines
19 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2016, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. /**************************************************************************************

  29. * 2016/04/04 Werner Saar (wernsaar@googlemail.com)

  30. * 	 BLASTEST 		: OK

  31. * 	 CTEST			: OK

  32. * 	 TEST			: OK

  33. * 	 LAPACK-TEST		: OK

  34. **************************************************************************************/

  35. 

  36. 	srawi.		J,	N,	2

  37. 	ble		CGEMM_L4_END

  38. 

  39. CGEMM_L4_BEGIN:

  40. 

  41. 	mr		BO,	B

  42. 	mr		BBO,	BBUFFER

  43. 	slwi		T1,	K,	3

  44. 

  45. CGEMM_L4_COPYB:

  46. 	dcbtst		BBO,	PRE

  47. 

  48. 	lxvw4x		vs3,	o0,	BO

  49. 	lxvw4x		vs11,	o16,	BO

  50. 	xxspltw		vs4,	vs3,	0

  51. 	xxspltw		vs5,	vs3,	1

  52. 	xxspltw		vs6,	vs3,	2

  53. 	xxspltw		vs7,	vs3,	3

  54. 	xxspltw		vs12,	vs11,	0

  55. 	xxspltw		vs13,	vs11,	1

  56. 	xxspltw		vs14,	vs11,	2

  57. 	xxspltw		vs15,	vs11,	3

  58. 	stxvw4x		vs4,	o0,	BBO

  59. 	stxvw4x		vs5,	o16,	BBO

  60. 	stxvw4x		vs6,	o32,	BBO

  61. 	stxvw4x		vs7,	o48,	BBO

  62. 	addi		BO,	BO,	32

  63. 	addi		BBO,	BBO,	64

  64. 	stxvw4x		vs12,	o0,	BBO

  65. 	stxvw4x		vs13,	o16,	BBO

  66. 	stxvw4x		vs14,	o32,	BBO

  67. 	stxvw4x		vs15,	o48,	BBO

  68. 	addic.		T1,	T1,	-8

  69. 	addi		BBO,	BBO,	64

  70. 

  71. 	bge		CGEMM_L4_COPYB

  72. 

  73. 

  74. 	mr		CO,	C

  75. 	mr		AO,	A

  76. 	slwi		T1,	LDC	,	2

  77. 	add		C,	C,	T1

  78. 	srawi.		I,	M,	3

  79. 	ble		CGEMM_L4x8_END

  80. 

  81. CGEMM_L4x8_BEGIN:

  82. 

  83. 

  84. 	mr		BO,	BBUFFER

  85. 	srawi.		L,	K,	3

  86. 	ble		CGEMM_L4x8_SUB0

  87. 	cmpwi		cr0,	L,	1

  88. 	ble		CGEMM_L4x8_SUB4

  89. 

  90. CGEMM_L4x8_LOOP_START:

  91. 

  92. 	dcbt		AO,	PRE

  93. 	dcbt		BO,	PRE

  94. 	LOAD4x8_1

  95. 	dcbt		BO,	PRE

  96. 	KERNEL4x8_I1

  97. 	dcbt		BO,	PRE

  98. 	dcbt		AO,	PRE

  99. 	KERNEL4x8_2

  100. 	dcbt		BO,	PRE

  101. 	KERNEL4x8_1

  102. 	dcbt		BO,	PRE

  103. 	dcbt		AO,	PRE

  104. 	KERNEL4x8_2

  105. 

  106. 	dcbt		BO,	PRE

  107. 	KERNEL4x8_1

  108. 	dcbt		BO,	PRE

  109. 	dcbt		AO,	PRE

  110. 	KERNEL4x8_2

  111. 	dcbt		BO,	PRE

  112. 	KERNEL4x8_1

  113. 	dcbt		BO,	PRE

  114. 	dcbt		AO,	PRE

  115. 	KERNEL4x8_2

  116. 

  117. 	addic.		L,	L,	-2

  118. 	ble		CGEMM_L4x8_LOOP_END

  119. 

  120. 	.align 5

  121. 

  122. CGEMM_L4x8_LOOP:

  123. 

  124. 	dcbt		BO,	PRE

  125. 	KERNEL4x8_1

  126. 	dcbt		BO,	PRE

  127. 	dcbt		AO,	PRE

  128. 	KERNEL4x8_2

  129. 	dcbt		BO,	PRE

  130. 	KERNEL4x8_1

  131. 	dcbt		BO,	PRE

  132. 	dcbt		AO,	PRE

  133. 	KERNEL4x8_2

  134. 

  135. 	dcbt		BO,	PRE

  136. 	KERNEL4x8_1

  137. 	dcbt		BO,	PRE

  138. 	dcbt		AO,	PRE

  139. 	KERNEL4x8_2

  140. 	dcbt		BO,	PRE

  141. 	KERNEL4x8_1

  142. 	dcbt		BO,	PRE

  143. 	dcbt		AO,	PRE

  144. 	KERNEL4x8_2

  145. 

  146. 	addic.		L,	L,	-1

  147. 	bgt		CGEMM_L4x8_LOOP

  148. 

  149. CGEMM_L4x8_LOOP_END:

  150. 

  151. 	dcbt		BO,	PRE

  152. 	KERNEL4x8_1

  153. 	dcbt		BO,	PRE

  154. 	dcbt		AO,	PRE

  155. 	KERNEL4x8_2

  156. 	KERNEL4x8_1

  157. 	dcbt		AO,	PRE

  158. 	KERNEL4x8_2

  159. 

  160. 	KERNEL4x8_1

  161. 	dcbt		AO,	PRE

  162. 	KERNEL4x8_2

  163. 	KERNEL4x8_1

  164. 	KERNEL4x8_E2

  165. 

  166. 	b		CGEMM_L4x8_SUB1

  167. 

  168. CGEMM_L4x8_SUB4:

  169. 

  170. 	KERNEL4x8_SUBI1

  171. 	KERNEL4x8_SUB1

  172. 	KERNEL4x8_SUB1

  173. 	KERNEL4x8_SUB1

  174. 

  175. 	KERNEL4x8_SUB1

  176. 	KERNEL4x8_SUB1

  177. 	KERNEL4x8_SUB1

  178. 	KERNEL4x8_SUB1

  179. 

  180. 	b		CGEMM_L4x8_SUB1

  181. 

  182. CGEMM_L4x8_SUB0:

  183. 

  184. 	andi.		L,	K,	7

  185. 

  186. 	KERNEL4x8_SUBI1

  187. 

  188. 	addic.		L,	L,	-1

  189. 	ble		CGEMM_L4x8_SAVE

  190. 	b		CGEMM_L4x8_SUB2

  191. 

  192. CGEMM_L4x8_SUB1:

  193. 

  194. 	andi.		L,	K,	7

  195. 	ble		CGEMM_L4x8_SAVE

  196. 

  197. CGEMM_L4x8_SUB2:

  198. 

  199. 	KERNEL4x8_SUB1

  200. 

  201. 	addic.		L,	L,	-1

  202. 	bgt		CGEMM_L4x8_SUB2

  203. 

  204. CGEMM_L4x8_SAVE:

  205. 

  206. 	SAVE4x8

  207. 

  208. 	addic.		I,	I,	-1

  209. 	bgt		CGEMM_L4x8_BEGIN

  210. 

  211. CGEMM_L4x8_END:

  212. 

  213. CGEMM_L4x4_BEGIN:

  214. 

  215. 	andi.		T2,	M,	7

  216. 	ble		CGEMM_L4x1_END

  217. 

  218. 	andi.		T1,	M,	4

  219. 	ble		CGEMM_L4x4_END

  220. 	mr		BO,	BBUFFER

  221. 	srawi.		L,	K,	3

  222. 	ble		CGEMM_L4x4_SUB0

  223. 	cmpwi		cr0,	L,	1

  224. 	ble		CGEMM_L4x4_SUB4

  225. 

  226. CGEMM_L4x4_LOOP_START:

  227. 

  228. 	LOAD4x4_1

  229. 	KERNEL4x4_I1

  230. 	KERNEL4x4_2

  231. 	KERNEL4x4_1

  232. 	KERNEL4x4_2

  233. 

  234. 	KERNEL4x4_1

  235. 	KERNEL4x4_2

  236. 	KERNEL4x4_1

  237. 	KERNEL4x4_2

  238. 

  239. 	addic.		L,	L,	-2

  240. 	ble		CGEMM_L4x4_LOOP_END

  241. 

  242. 	.align 5

  243. 

  244. CGEMM_L4x4_LOOP:

  245. 

  246. 	KERNEL4x4_1

  247. 	KERNEL4x4_2

  248. 	KERNEL4x4_1

  249. 	KERNEL4x4_2

  250. 

  251. 	KERNEL4x4_1

  252. 	KERNEL4x4_2

  253. 	KERNEL4x4_1

  254. 	KERNEL4x4_2

  255. 

  256. 	addic.		L,	L,	-1

  257. 	bgt		CGEMM_L4x4_LOOP

  258. 

  259. CGEMM_L4x4_LOOP_END:

  260. 

  261. 	KERNEL4x4_1

  262. 	KERNEL4x4_2

  263. 	KERNEL4x4_1

  264. 	KERNEL4x4_2

  265. 

  266. 	KERNEL4x4_1

  267. 	KERNEL4x4_2

  268. 	KERNEL4x4_1

  269. 	KERNEL4x4_E2

  270. 

  271. 	b		CGEMM_L4x4_SUB1

  272. 

  273. CGEMM_L4x4_SUB4:

  274. 

  275. 	KERNEL4x4_SUBI1

  276. 	KERNEL4x4_SUB1

  277. 	KERNEL4x4_SUB1

  278. 	KERNEL4x4_SUB1

  279. 

  280. 	KERNEL4x4_SUB1

  281. 	KERNEL4x4_SUB1

  282. 	KERNEL4x4_SUB1

  283. 	KERNEL4x4_SUB1

  284. 

  285. 	b		CGEMM_L4x4_SUB1

  286. 

  287. CGEMM_L4x4_SUB0:

  288. 

  289. 	andi.		L,	K,	7

  290. 

  291. 	KERNEL4x4_SUBI1

  292. 

  293. 	addic.		L,	L,	-1

  294. 	ble		CGEMM_L4x4_SAVE

  295. 	b		CGEMM_L4x4_SUB2

  296. 

  297. CGEMM_L4x4_SUB1:

  298. 

  299. 	andi.		L,	K,	7

  300. 	ble		CGEMM_L4x4_SAVE

  301. 

  302. CGEMM_L4x4_SUB2:

  303. 

  304. 	KERNEL4x4_SUB1

  305. 

  306. 	addic.		L,	L,	-1

  307. 	bgt		CGEMM_L4x4_SUB2

  308. 

  309. CGEMM_L4x4_SAVE:

  310. 

  311. 	SAVE4x4

  312. 

  313. CGEMM_L4x4_END:

  314. 

  315. CGEMM_L4x2_BEGIN:

  316. 

  317. 

  318. 	andi.		T1,	M,	2

  319. 	ble		CGEMM_L4x2_END

  320. 	mr		BO,	BBUFFER

  321. 	srawi.		L,	K,	3

  322. 	ble		CGEMM_L4x2_SUB0

  323. 	cmpwi		cr0,	L,	1

  324. 	ble		CGEMM_L4x2_SUB4

  325. 

  326. CGEMM_L4x2_LOOP_START:

  327. 

  328. 	LOAD4x2_1

  329. 	KERNEL4x2_I1

  330. 	KERNEL4x2_2

  331. 	KERNEL4x2_1

  332. 	KERNEL4x2_2

  333. 

  334. 	KERNEL4x2_1

  335. 	KERNEL4x2_2

  336. 	KERNEL4x2_1

  337. 	KERNEL4x2_2

  338. 

  339. 	addic.		L,	L,	-2

  340. 	ble		CGEMM_L4x2_LOOP_END

  341. 

  342. 	.align 5

  343. 

  344. CGEMM_L4x2_LOOP:

  345. 

  346. 	KERNEL4x2_1

  347. 	KERNEL4x2_2

  348. 	KERNEL4x2_1

  349. 	KERNEL4x2_2

  350. 

  351. 	KERNEL4x2_1

  352. 	KERNEL4x2_2

  353. 	KERNEL4x2_1

  354. 	KERNEL4x2_2

  355. 

  356. 	addic.		L,	L,	-1

  357. 	bgt		CGEMM_L4x2_LOOP

  358. 

  359. CGEMM_L4x2_LOOP_END:

  360. 

  361. 	KERNEL4x2_1

  362. 	KERNEL4x2_2

  363. 	KERNEL4x2_1

  364. 	KERNEL4x2_2

  365. 

  366. 	KERNEL4x2_1

  367. 	KERNEL4x2_2

  368. 	KERNEL4x2_1

  369. 	KERNEL4x2_E2

  370. 

  371. 	b		CGEMM_L4x2_SUB1

  372. 

  373. CGEMM_L4x2_SUB4:

  374. 

  375. 	KERNEL4x2_SUBI1

  376. 	KERNEL4x2_SUB1

  377. 	KERNEL4x2_SUB1

  378. 	KERNEL4x2_SUB1

  379. 

  380. 	KERNEL4x2_SUB1

  381. 	KERNEL4x2_SUB1

  382. 	KERNEL4x2_SUB1

  383. 	KERNEL4x2_SUB1

  384. 

  385. 	b		CGEMM_L4x2_SUB1

  386. 

  387. CGEMM_L4x2_SUB0:

  388. 

  389. 	andi.		L,	K,	7

  390. 

  391. 	KERNEL4x2_SUBI1

  392. 

  393. 	addic.		L,	L,	-1

  394. 	ble		CGEMM_L4x2_SAVE

  395. 	b		CGEMM_L4x2_SUB2

  396. 

  397. CGEMM_L4x2_SUB1:

  398. 

  399. 	andi.		L,	K,	7

  400. 	ble		CGEMM_L4x2_SAVE

  401. 

  402. CGEMM_L4x2_SUB2:

  403. 

  404. 	KERNEL4x2_SUB1

  405. 

  406. 	addic.		L,	L,	-1

  407. 	bgt		CGEMM_L4x2_SUB2

  408. 

  409. CGEMM_L4x2_SAVE:

  410. 

  411. 	SAVE4x2

  412. 

  413. CGEMM_L4x2_END:

  414. 

  415. CGEMM_L4x1_BEGIN:

  416. 

  417. 

  418. 	andi.		T1,	M,	1

  419. 	ble		CGEMM_L4x1_END

  420. 	mr		BO,	BBUFFER

  421. 	srawi.		L,	K,	3

  422. 	ble		CGEMM_L4x1_SUB0

  423. 	cmpwi		cr0,	L,	1

  424. 	ble		CGEMM_L4x1_SUB4

  425. 

  426. CGEMM_L4x1_LOOP_START:

  427. 

  428. 	LOAD4x1_1

  429. 	KERNEL4x1_I1

  430. 	KERNEL4x1_2

  431. 	KERNEL4x1_1

  432. 	KERNEL4x1_2

  433. 

  434. 	KERNEL4x1_1

  435. 	KERNEL4x1_2

  436. 	KERNEL4x1_1

  437. 	KERNEL4x1_2

  438. 

  439. 	addic.		L,	L,	-2

  440. 	ble		CGEMM_L4x1_LOOP_END

  441. 

  442. 	.align 5

  443. 

  444. CGEMM_L4x1_LOOP:

  445. 

  446. 	KERNEL4x1_1

  447. 	KERNEL4x1_2

  448. 	KERNEL4x1_1

  449. 	KERNEL4x1_2

  450. 

  451. 	KERNEL4x1_1

  452. 	KERNEL4x1_2

  453. 	KERNEL4x1_1

  454. 	KERNEL4x1_2

  455. 

  456. 	addic.		L,	L,	-1

  457. 	bgt		CGEMM_L4x1_LOOP

  458. 

  459. CGEMM_L4x1_LOOP_END:

  460. 

  461. 	KERNEL4x1_1

  462. 	KERNEL4x1_2

  463. 	KERNEL4x1_1

  464. 	KERNEL4x1_2

  465. 

  466. 	KERNEL4x1_1

  467. 	KERNEL4x1_2

  468. 	KERNEL4x1_1

  469. 	KERNEL4x1_E2

  470. 

  471. 	b		CGEMM_L4x1_SUB1

  472. 

  473. CGEMM_L4x1_SUB4:

  474. 

  475. 	KERNEL4x1_SUBI1

  476. 	KERNEL4x1_SUB1

  477. 	KERNEL4x1_SUB1

  478. 	KERNEL4x1_SUB1

  479. 

  480. 	KERNEL4x1_SUB1

  481. 	KERNEL4x1_SUB1

  482. 	KERNEL4x1_SUB1

  483. 	KERNEL4x1_SUB1

  484. 

  485. 	b		CGEMM_L4x1_SUB1

  486. 

  487. CGEMM_L4x1_SUB0:

  488. 

  489. 	andi.		L,	K,	7

  490. 

  491. 	KERNEL4x1_SUBI1

  492. 

  493. 	addic.		L,	L,	-1

  494. 	ble		CGEMM_L4x1_SAVE

  495. 	b		CGEMM_L4x1_SUB2

  496. 

  497. CGEMM_L4x1_SUB1:

  498. 

  499. 	andi.		L,	K,	7

  500. 	ble		CGEMM_L4x1_SAVE

  501. 

  502. CGEMM_L4x1_SUB2:

  503. 

  504. 	KERNEL4x1_SUB1

  505. 

  506. 	addic.		L,	L,	-1

  507. 	bgt		CGEMM_L4x1_SUB2

  508. 

  509. CGEMM_L4x1_SAVE:

  510. 

  511. 	SAVE4x1

  512. 

  513. CGEMM_L4x1_END:

  514. 

  515. 	slwi		T1,	K,	5

  516. 	add		B,	B,	T1

  517. 

  518. 	addic.		J,	J,	-1

  519. 	bgt		CGEMM_L4_BEGIN

  520. 

  521. 	andi.		T2,	N,	3

  522. 	ble		L999_H2

  523. 

  524. CGEMM_L4_END:

  525. 

  526. 	b		CGEMM_L2_BEGIN

  527. 

  528. L999_H1:

  529. 

  530. 	b		L999_H2

  531. 

  532. CGEMM_L2_BEGIN:

  533. 

  534. 	mr		BO,	B

  535. 	mr		BBO,	BBUFFER

  536. 	slwi		T1,	K,	2

  537. 

  538. CGEMM_L2_COPYB:

  539. 	dcbtst		BBO,	PRE

  540. 

  541. 	lxvw4x		vs3,	o0,	BO

  542. 	lxvw4x		vs11,	o16,	BO

  543. 	xxspltw		vs4,	vs3,	0

  544. 	xxspltw		vs5,	vs3,	1

  545. 	xxspltw		vs6,	vs3,	2

  546. 	xxspltw		vs7,	vs3,	3

  547. 	xxspltw		vs12,	vs11,	0

  548. 	xxspltw		vs13,	vs11,	1

  549. 	xxspltw		vs14,	vs11,	2

  550. 	xxspltw		vs15,	vs11,	3

  551. 	stxvw4x		vs4,	o0,	BBO

  552. 	stxvw4x		vs5,	o16,	BBO

  553. 	stxvw4x		vs6,	o32,	BBO

  554. 	stxvw4x		vs7,	o48,	BBO

  555. 	addi		BO,	BO,	32

  556. 	addi		BBO,	BBO,	64

  557. 	stxvw4x		vs12,	o0,	BBO

  558. 	stxvw4x		vs13,	o16,	BBO

  559. 	stxvw4x		vs14,	o32,	BBO

  560. 	stxvw4x		vs15,	o48,	BBO

  561. 	addic.		T1,	T1,	-8

  562. 	addi		BBO,	BBO,	64

  563. 

  564. 	bge		CGEMM_L2_COPYB

  565. 

  566. 

  567. 	andi.		T1,	N,	2

  568. 	ble		CGEMM_L2_END

  569. 	mr		CO,	C

  570. 	mr		AO,	A

  571. 	slwi		T1,	LDC	,	1

  572. 	add		C,	C,	T1

  573. 	srawi.		I,	M,	3

  574. 	ble		CGEMM_L2x8_END

  575. 

  576. CGEMM_L2x8_BEGIN:

  577. 

  578. 

  579. 	mr		BO,	BBUFFER

  580. 	srawi.		L,	K,	3

  581. 	ble		CGEMM_L2x8_SUB0

  582. 	cmpwi		cr0,	L,	1

  583. 	ble		CGEMM_L2x8_SUB4

  584. 

  585. CGEMM_L2x8_LOOP_START:

  586. 

  587. 	dcbt		AO,	PRE

  588. 	LOAD2x8_1

  589. 	KERNEL2x8_I1

  590. 	dcbt		AO,	PRE

  591. 	KERNEL2x8_2

  592. 	KERNEL2x8_1

  593. 	dcbt		AO,	PRE

  594. 	KERNEL2x8_2

  595. 

  596. 	KERNEL2x8_1

  597. 	dcbt		AO,	PRE

  598. 	KERNEL2x8_2

  599. 	KERNEL2x8_1

  600. 	dcbt		AO,	PRE

  601. 	KERNEL2x8_2

  602. 

  603. 	addic.		L,	L,	-2

  604. 	ble		CGEMM_L2x8_LOOP_END

  605. 

  606. 	.align 5

  607. 

  608. CGEMM_L2x8_LOOP:

  609. 

  610. 	KERNEL2x8_1

  611. 	dcbt		AO,	PRE

  612. 	KERNEL2x8_2

  613. 	KERNEL2x8_1

  614. 	dcbt		AO,	PRE

  615. 	KERNEL2x8_2

  616. 

  617. 	KERNEL2x8_1

  618. 	dcbt		AO,	PRE

  619. 	KERNEL2x8_2

  620. 	KERNEL2x8_1

  621. 	dcbt		AO,	PRE

  622. 	KERNEL2x8_2

  623. 

  624. 	addic.		L,	L,	-1

  625. 	bgt		CGEMM_L2x8_LOOP

  626. 

  627. CGEMM_L2x8_LOOP_END:

  628. 

  629. 	KERNEL2x8_1

  630. 	dcbt		AO,	PRE

  631. 	KERNEL2x8_2

  632. 	KERNEL2x8_1

  633. 	dcbt		AO,	PRE

  634. 	KERNEL2x8_2

  635. 

  636. 	KERNEL2x8_1

  637. 	dcbt		AO,	PRE

  638. 	KERNEL2x8_2

  639. 	KERNEL2x8_1

  640. 	KERNEL2x8_E2

  641. 

  642. 	b		CGEMM_L2x8_SUB1

  643. 

  644. CGEMM_L2x8_SUB4:

  645. 

  646. 	KERNEL2x8_SUBI1

  647. 	KERNEL2x8_SUB1

  648. 	KERNEL2x8_SUB1

  649. 	KERNEL2x8_SUB1

  650. 

  651. 	KERNEL2x8_SUB1

  652. 	KERNEL2x8_SUB1

  653. 	KERNEL2x8_SUB1

  654. 	KERNEL2x8_SUB1

  655. 

  656. 	b		CGEMM_L2x8_SUB1

  657. 

  658. CGEMM_L2x8_SUB0:

  659. 

  660. 	andi.		L,	K,	7

  661. 

  662. 	KERNEL2x8_SUBI1

  663. 

  664. 	addic.		L,	L,	-1

  665. 	ble		CGEMM_L2x8_SAVE

  666. 	b		CGEMM_L2x8_SUB2

  667. 

  668. CGEMM_L2x8_SUB1:

  669. 

  670. 	andi.		L,	K,	7

  671. 	ble		CGEMM_L2x8_SAVE

  672. 

  673. CGEMM_L2x8_SUB2:

  674. 

  675. 	KERNEL2x8_SUB1

  676. 

  677. 	addic.		L,	L,	-1

  678. 	bgt		CGEMM_L2x8_SUB2

  679. 

  680. CGEMM_L2x8_SAVE:

  681. 

  682. 	SAVE2x8

  683. 

  684. 	addic.		I,	I,	-1

  685. 	bgt		CGEMM_L2x8_BEGIN

  686. 

  687. CGEMM_L2x8_END:

  688. 

  689. CGEMM_L2x4_BEGIN:

  690. 

  691. 	andi.		T2,	M,	7

  692. 	ble		CGEMM_L2x1_END

  693. 

  694. 	andi.		T1,	M,	4

  695. 	ble		CGEMM_L2x4_END

  696. 	mr		BO,	BBUFFER

  697. 	srawi.		L,	K,	3

  698. 	ble		CGEMM_L2x4_SUB0

  699. 	cmpwi		cr0,	L,	1

  700. 	ble		CGEMM_L2x4_SUB4

  701. 

  702. CGEMM_L2x4_LOOP_START:

  703. 

  704. 	LOAD2x4_1

  705. 	KERNEL2x4_I1

  706. 	KERNEL2x4_2

  707. 	KERNEL2x4_1

  708. 	KERNEL2x4_2

  709. 

  710. 	KERNEL2x4_1

  711. 	KERNEL2x4_2

  712. 	KERNEL2x4_1

  713. 	KERNEL2x4_2

  714. 

  715. 	addic.		L,	L,	-2

  716. 	ble		CGEMM_L2x4_LOOP_END

  717. 

  718. 	.align 5

  719. 

  720. CGEMM_L2x4_LOOP:

  721. 

  722. 	KERNEL2x4_1

  723. 	KERNEL2x4_2

  724. 	KERNEL2x4_1

  725. 	KERNEL2x4_2

  726. 

  727. 	KERNEL2x4_1

  728. 	KERNEL2x4_2

  729. 	KERNEL2x4_1

  730. 	KERNEL2x4_2

  731. 

  732. 	addic.		L,	L,	-1

  733. 	bgt		CGEMM_L2x4_LOOP

  734. 

  735. CGEMM_L2x4_LOOP_END:

  736. 

  737. 	KERNEL2x4_1

  738. 	KERNEL2x4_2

  739. 	KERNEL2x4_1

  740. 	KERNEL2x4_2

  741. 

  742. 	KERNEL2x4_1

  743. 	KERNEL2x4_2

  744. 	KERNEL2x4_1

  745. 	KERNEL2x4_E2

  746. 

  747. 	b		CGEMM_L2x4_SUB1

  748. 

  749. CGEMM_L2x4_SUB4:

  750. 

  751. 	KERNEL2x4_SUBI1

  752. 	KERNEL2x4_SUB1

  753. 	KERNEL2x4_SUB1

  754. 	KERNEL2x4_SUB1

  755. 

  756. 	KERNEL2x4_SUB1

  757. 	KERNEL2x4_SUB1

  758. 	KERNEL2x4_SUB1

  759. 	KERNEL2x4_SUB1

  760. 

  761. 	b		CGEMM_L2x4_SUB1

  762. 

  763. CGEMM_L2x4_SUB0:

  764. 

  765. 	andi.		L,	K,	7

  766. 

  767. 	KERNEL2x4_SUBI1

  768. 

  769. 	addic.		L,	L,	-1

  770. 	ble		CGEMM_L2x4_SAVE

  771. 	b		CGEMM_L2x4_SUB2

  772. 

  773. CGEMM_L2x4_SUB1:

  774. 

  775. 	andi.		L,	K,	7

  776. 	ble		CGEMM_L2x4_SAVE

  777. 

  778. CGEMM_L2x4_SUB2:

  779. 

  780. 	KERNEL2x4_SUB1

  781. 

  782. 	addic.		L,	L,	-1

  783. 	bgt		CGEMM_L2x4_SUB2

  784. 

  785. CGEMM_L2x4_SAVE:

  786. 

  787. 	SAVE2x4

  788. 

  789. CGEMM_L2x4_END:

  790. 

  791. CGEMM_L2x2_BEGIN:

  792. 

  793. 

  794. 	andi.		T1,	M,	2

  795. 	ble		CGEMM_L2x2_END

  796. 	mr		BO,	BBUFFER

  797. 	srawi.		L,	K,	3

  798. 	ble		CGEMM_L2x2_SUB0

  799. 	cmpwi		cr0,	L,	1

  800. 	ble		CGEMM_L2x2_SUB4

  801. 

  802. CGEMM_L2x2_LOOP_START:

  803. 

  804. 	LOAD2x2_1

  805. 	KERNEL2x2_I1

  806. 	KERNEL2x2_2

  807. 	KERNEL2x2_1

  808. 	KERNEL2x2_2

  809. 

  810. 	KERNEL2x2_1

  811. 	KERNEL2x2_2

  812. 	KERNEL2x2_1

  813. 	KERNEL2x2_2

  814. 

  815. 	addic.		L,	L,	-2

  816. 	ble		CGEMM_L2x2_LOOP_END

  817. 

  818. 	.align 5

  819. 

  820. CGEMM_L2x2_LOOP:

  821. 

  822. 	KERNEL2x2_1

  823. 	KERNEL2x2_2

  824. 	KERNEL2x2_1

  825. 	KERNEL2x2_2

  826. 

  827. 	KERNEL2x2_1

  828. 	KERNEL2x2_2

  829. 	KERNEL2x2_1

  830. 	KERNEL2x2_2

  831. 

  832. 	addic.		L,	L,	-1

  833. 	bgt		CGEMM_L2x2_LOOP

  834. 

  835. CGEMM_L2x2_LOOP_END:

  836. 

  837. 	KERNEL2x2_1

  838. 	KERNEL2x2_2

  839. 	KERNEL2x2_1

  840. 	KERNEL2x2_2

  841. 

  842. 	KERNEL2x2_1

  843. 	KERNEL2x2_2

  844. 	KERNEL2x2_1

  845. 	KERNEL2x2_E2

  846. 

  847. 	b		CGEMM_L2x2_SUB1

  848. 

  849. CGEMM_L2x2_SUB4:

  850. 

  851. 	KERNEL2x2_SUBI1

  852. 	KERNEL2x2_SUB1

  853. 	KERNEL2x2_SUB1

  854. 	KERNEL2x2_SUB1

  855. 

  856. 	KERNEL2x2_SUB1

  857. 	KERNEL2x2_SUB1

  858. 	KERNEL2x2_SUB1

  859. 	KERNEL2x2_SUB1

  860. 

  861. 	b		CGEMM_L2x2_SUB1

  862. 

  863. CGEMM_L2x2_SUB0:

  864. 

  865. 	andi.		L,	K,	7

  866. 

  867. 	KERNEL2x2_SUBI1

  868. 

  869. 	addic.		L,	L,	-1

  870. 	ble		CGEMM_L2x2_SAVE

  871. 	b		CGEMM_L2x2_SUB2

  872. 

  873. CGEMM_L2x2_SUB1:

  874. 

  875. 	andi.		L,	K,	7

  876. 	ble		CGEMM_L2x2_SAVE

  877. 

  878. CGEMM_L2x2_SUB2:

  879. 

  880. 	KERNEL2x2_SUB1

  881. 

  882. 	addic.		L,	L,	-1

  883. 	bgt		CGEMM_L2x2_SUB2

  884. 

  885. CGEMM_L2x2_SAVE:

  886. 

  887. 	SAVE2x2

  888. 

  889. CGEMM_L2x2_END:

  890. 

  891. CGEMM_L2x1_BEGIN:

  892. 

  893. 

  894. 	andi.		T1,	M,	1

  895. 	ble		CGEMM_L2x1_END

  896. 	mr		BO,	BBUFFER

  897. 	srawi.		L,	K,	3

  898. 	ble		CGEMM_L2x1_SUB0

  899. 	cmpwi		cr0,	L,	1

  900. 	ble		CGEMM_L2x1_SUB4

  901. 

  902. CGEMM_L2x1_LOOP_START:

  903. 

  904. 	LOAD2x1_1

  905. 	KERNEL2x1_I1

  906. 	KERNEL2x1_2

  907. 	KERNEL2x1_1

  908. 	KERNEL2x1_2

  909. 

  910. 	KERNEL2x1_1

  911. 	KERNEL2x1_2

  912. 	KERNEL2x1_1

  913. 	KERNEL2x1_2

  914. 

  915. 	addic.		L,	L,	-2

  916. 	ble		CGEMM_L2x1_LOOP_END

  917. 

  918. 	.align 5

  919. 

  920. CGEMM_L2x1_LOOP:

  921. 

  922. 	KERNEL2x1_1

  923. 	KERNEL2x1_2

  924. 	KERNEL2x1_1

  925. 	KERNEL2x1_2

  926. 

  927. 	KERNEL2x1_1

  928. 	KERNEL2x1_2

  929. 	KERNEL2x1_1

  930. 	KERNEL2x1_2

  931. 

  932. 	addic.		L,	L,	-1

  933. 	bgt		CGEMM_L2x1_LOOP

  934. 

  935. CGEMM_L2x1_LOOP_END:

  936. 

  937. 	KERNEL2x1_1

  938. 	KERNEL2x1_2

  939. 	KERNEL2x1_1

  940. 	KERNEL2x1_2

  941. 

  942. 	KERNEL2x1_1

  943. 	KERNEL2x1_2

  944. 	KERNEL2x1_1

  945. 	KERNEL2x1_E2

  946. 

  947. 	b		CGEMM_L2x1_SUB1

  948. 

  949. CGEMM_L2x1_SUB4:

  950. 

  951. 	KERNEL2x1_SUBI1

  952. 	KERNEL2x1_SUB1

  953. 	KERNEL2x1_SUB1

  954. 	KERNEL2x1_SUB1

  955. 

  956. 	KERNEL2x1_SUB1

  957. 	KERNEL2x1_SUB1

  958. 	KERNEL2x1_SUB1

  959. 	KERNEL2x1_SUB1

  960. 

  961. 	b		CGEMM_L2x1_SUB1

  962. 

  963. CGEMM_L2x1_SUB0:

  964. 

  965. 	andi.		L,	K,	7

  966. 

  967. 	KERNEL2x1_SUBI1

  968. 

  969. 	addic.		L,	L,	-1

  970. 	ble		CGEMM_L2x1_SAVE

  971. 	b		CGEMM_L2x1_SUB2

  972. 

  973. CGEMM_L2x1_SUB1:

  974. 

  975. 	andi.		L,	K,	7

  976. 	ble		CGEMM_L2x1_SAVE

  977. 

  978. CGEMM_L2x1_SUB2:

  979. 

  980. 	KERNEL2x1_SUB1

  981. 

  982. 	addic.		L,	L,	-1

  983. 	bgt		CGEMM_L2x1_SUB2

  984. 

  985. CGEMM_L2x1_SAVE:

  986. 

  987. 	SAVE2x1

  988. 

  989. CGEMM_L2x1_END:

  990. 

  991. 	slwi		T1,	K,	4

  992. 	add		B,	B,	T1

  993. 

  994. CGEMM_L2_END:

  995. 

  996. 	b		CGEMM_L1_BEGIN

  997. 

  998. L999_H2:

  999. 

  1000. 	b		L999

  1001. 

  1002. CGEMM_L1_BEGIN:

  1003. 

  1004. 	mr		BO,	B

  1005. 	mr		BBO,	BBUFFER

  1006. 	slwi		T1,	K,	1

  1007. 

  1008. CGEMM_L1_COPYB:

  1009. 	dcbtst		BBO,	PRE

  1010. 

  1011. 	lxvw4x		vs3,	o0,	BO

  1012. 	lxvw4x		vs11,	o16,	BO

  1013. 	xxspltw		vs4,	vs3,	0

  1014. 	xxspltw		vs5,	vs3,	1

  1015. 	xxspltw		vs6,	vs3,	2

  1016. 	xxspltw		vs7,	vs3,	3

  1017. 	xxspltw		vs12,	vs11,	0

  1018. 	xxspltw		vs13,	vs11,	1

  1019. 	xxspltw		vs14,	vs11,	2

  1020. 	xxspltw		vs15,	vs11,	3

  1021. 	stxvw4x		vs4,	o0,	BBO

  1022. 	stxvw4x		vs5,	o16,	BBO

  1023. 	stxvw4x		vs6,	o32,	BBO

  1024. 	stxvw4x		vs7,	o48,	BBO

  1025. 	addi		BO,	BO,	32

  1026. 	addi		BBO,	BBO,	64

  1027. 	stxvw4x		vs12,	o0,	BBO

  1028. 	stxvw4x		vs13,	o16,	BBO

  1029. 	stxvw4x		vs14,	o32,	BBO

  1030. 	stxvw4x		vs15,	o48,	BBO

  1031. 	addic.		T1,	T1,	-8

  1032. 	addi		BBO,	BBO,	64

  1033. 

  1034. 	bge		CGEMM_L1_COPYB

  1035. 

  1036. 

  1037. 	andi.		T1,	N,	1

  1038. 	ble		CGEMM_L1_END

  1039. 	mr		CO,	C

  1040. 	mr		AO,	A

  1041. 	srawi.		I,	M,	3

  1042. 	ble		CGEMM_L1x8_END

  1043. 

  1044. CGEMM_L1x8_BEGIN:

  1045. 

  1046. 

  1047. 	mr		BO,	BBUFFER

  1048. 	srawi.		L,	K,	3

  1049. 	ble		CGEMM_L1x8_SUB0

  1050. 	cmpwi		cr0,	L,	1

  1051. 	ble		CGEMM_L1x8_SUB4

  1052. 

  1053. CGEMM_L1x8_LOOP_START:

  1054. 

  1055. 	dcbt		AO,	PRE

  1056. 	LOAD1x8_1

  1057. 	KERNEL1x8_I1

  1058. 	dcbt		AO,	PRE

  1059. 	KERNEL1x8_2

  1060. 	KERNEL1x8_1

  1061. 	dcbt		AO,	PRE

  1062. 	KERNEL1x8_2

  1063. 

  1064. 	KERNEL1x8_1

  1065. 	dcbt		AO,	PRE

  1066. 	KERNEL1x8_2

  1067. 	KERNEL1x8_1

  1068. 	dcbt		AO,	PRE

  1069. 	KERNEL1x8_2

  1070. 

  1071. 	addic.		L,	L,	-2

  1072. 	ble		CGEMM_L1x8_LOOP_END

  1073. 

  1074. 	.align 5

  1075. 

  1076. CGEMM_L1x8_LOOP:

  1077. 

  1078. 	KERNEL1x8_1

  1079. 	dcbt		AO,	PRE

  1080. 	KERNEL1x8_2

  1081. 	KERNEL1x8_1

  1082. 	dcbt		AO,	PRE

  1083. 	KERNEL1x8_2

  1084. 

  1085. 	KERNEL1x8_1

  1086. 	dcbt		AO,	PRE

  1087. 	KERNEL1x8_2

  1088. 	KERNEL1x8_1

  1089. 	dcbt		AO,	PRE

  1090. 	KERNEL1x8_2

  1091. 

  1092. 	addic.		L,	L,	-1

  1093. 	bgt		CGEMM_L1x8_LOOP

  1094. 

  1095. CGEMM_L1x8_LOOP_END:

  1096. 

  1097. 	KERNEL1x8_1

  1098. 	dcbt		AO,	PRE

  1099. 	KERNEL1x8_2

  1100. 	KERNEL1x8_1

  1101. 	dcbt		AO,	PRE

  1102. 	KERNEL1x8_2

  1103. 

  1104. 	KERNEL1x8_1

  1105. 	dcbt		AO,	PRE

  1106. 	KERNEL1x8_2

  1107. 	KERNEL1x8_1

  1108. 	KERNEL1x8_E2

  1109. 

  1110. 	b		CGEMM_L1x8_SUB1

  1111. 

  1112. CGEMM_L1x8_SUB4:

  1113. 

  1114. 	KERNEL1x8_SUBI1

  1115. 	KERNEL1x8_SUB1

  1116. 	KERNEL1x8_SUB1

  1117. 	KERNEL1x8_SUB1

  1118. 

  1119. 	KERNEL1x8_SUB1

  1120. 	KERNEL1x8_SUB1

  1121. 	KERNEL1x8_SUB1

  1122. 	KERNEL1x8_SUB1

  1123. 

  1124. 	b		CGEMM_L1x8_SUB1

  1125. 

  1126. CGEMM_L1x8_SUB0:

  1127. 

  1128. 	andi.		L,	K,	7

  1129. 

  1130. 	KERNEL1x8_SUBI1

  1131. 

  1132. 	addic.		L,	L,	-1

  1133. 	ble		CGEMM_L1x8_SAVE

  1134. 	b		CGEMM_L1x8_SUB2

  1135. 

  1136. CGEMM_L1x8_SUB1:

  1137. 

  1138. 	andi.		L,	K,	7

  1139. 	ble		CGEMM_L1x8_SAVE

  1140. 

  1141. CGEMM_L1x8_SUB2:

  1142. 

  1143. 	KERNEL1x8_SUB1

  1144. 

  1145. 	addic.		L,	L,	-1

  1146. 	bgt		CGEMM_L1x8_SUB2

  1147. 

  1148. CGEMM_L1x8_SAVE:

  1149. 

  1150. 	SAVE1x8

  1151. 

  1152. 	addic.		I,	I,	-1

  1153. 	bgt		CGEMM_L1x8_BEGIN

  1154. 

  1155. CGEMM_L1x8_END:

  1156. 

  1157. CGEMM_L1x4_BEGIN:

  1158. 

  1159. 	andi.		T2,	M,	7

  1160. 	ble		CGEMM_L1x1_END

  1161. 

  1162. 	andi.		T1,	M,	4

  1163. 	ble		CGEMM_L1x4_END

  1164. 	mr		BO,	BBUFFER

  1165. 	srawi.		L,	K,	3

  1166. 	ble		CGEMM_L1x4_SUB0

  1167. 	cmpwi		cr0,	L,	1

  1168. 	ble		CGEMM_L1x4_SUB4

  1169. 

  1170. CGEMM_L1x4_LOOP_START:

  1171. 

  1172. 	LOAD1x4_1

  1173. 	KERNEL1x4_I1

  1174. 	KERNEL1x4_2

  1175. 	KERNEL1x4_1

  1176. 	KERNEL1x4_2

  1177. 

  1178. 	KERNEL1x4_1

  1179. 	KERNEL1x4_2

  1180. 	KERNEL1x4_1

  1181. 	KERNEL1x4_2

  1182. 

  1183. 	addic.		L,	L,	-2

  1184. 	ble		CGEMM_L1x4_LOOP_END

  1185. 

  1186. 	.align 5

  1187. 

  1188. CGEMM_L1x4_LOOP:

  1189. 

  1190. 	KERNEL1x4_1

  1191. 	KERNEL1x4_2

  1192. 	KERNEL1x4_1

  1193. 	KERNEL1x4_2

  1194. 

  1195. 	KERNEL1x4_1

  1196. 	KERNEL1x4_2

  1197. 	KERNEL1x4_1

  1198. 	KERNEL1x4_2

  1199. 

  1200. 	addic.		L,	L,	-1

  1201. 	bgt		CGEMM_L1x4_LOOP

  1202. 

  1203. CGEMM_L1x4_LOOP_END:

  1204. 

  1205. 	KERNEL1x4_1

  1206. 	KERNEL1x4_2

  1207. 	KERNEL1x4_1

  1208. 	KERNEL1x4_2

  1209. 

  1210. 	KERNEL1x4_1

  1211. 	KERNEL1x4_2

  1212. 	KERNEL1x4_1

  1213. 	KERNEL1x4_E2

  1214. 

  1215. 	b		CGEMM_L1x4_SUB1

  1216. 

  1217. CGEMM_L1x4_SUB4:

  1218. 

  1219. 	KERNEL1x4_SUBI1

  1220. 	KERNEL1x4_SUB1

  1221. 	KERNEL1x4_SUB1

  1222. 	KERNEL1x4_SUB1

  1223. 

  1224. 	KERNEL1x4_SUB1

  1225. 	KERNEL1x4_SUB1

  1226. 	KERNEL1x4_SUB1

  1227. 	KERNEL1x4_SUB1

  1228. 

  1229. 	b		CGEMM_L1x4_SUB1

  1230. 

  1231. CGEMM_L1x4_SUB0:

  1232. 

  1233. 	andi.		L,	K,	7

  1234. 

  1235. 	KERNEL1x4_SUBI1

  1236. 

  1237. 	addic.		L,	L,	-1

  1238. 	ble		CGEMM_L1x4_SAVE

  1239. 	b		CGEMM_L1x4_SUB2

  1240. 

  1241. CGEMM_L1x4_SUB1:

  1242. 

  1243. 	andi.		L,	K,	7

  1244. 	ble		CGEMM_L1x4_SAVE

  1245. 

  1246. CGEMM_L1x4_SUB2:

  1247. 

  1248. 	KERNEL1x4_SUB1

  1249. 

  1250. 	addic.		L,	L,	-1

  1251. 	bgt		CGEMM_L1x4_SUB2

  1252. 

  1253. CGEMM_L1x4_SAVE:

  1254. 

  1255. 	SAVE1x4

  1256. 

  1257. CGEMM_L1x4_END:

  1258. 

  1259. CGEMM_L1x2_BEGIN:

  1260. 

  1261. 

  1262. 	andi.		T1,	M,	2

  1263. 	ble		CGEMM_L1x2_END

  1264. 	mr		BO,	BBUFFER

  1265. 	srawi.		L,	K,	3

  1266. 	ble		CGEMM_L1x2_SUB0

  1267. 	cmpwi		cr0,	L,	1

  1268. 	ble		CGEMM_L1x2_SUB4

  1269. 

  1270. CGEMM_L1x2_LOOP_START:

  1271. 

  1272. 	LOAD1x2_1

  1273. 	KERNEL1x2_I1

  1274. 	KERNEL1x2_2

  1275. 	KERNEL1x2_1

  1276. 	KERNEL1x2_2

  1277. 

  1278. 	KERNEL1x2_1

  1279. 	KERNEL1x2_2

  1280. 	KERNEL1x2_1

  1281. 	KERNEL1x2_2

  1282. 

  1283. 	addic.		L,	L,	-2

  1284. 	ble		CGEMM_L1x2_LOOP_END

  1285. 

  1286. 	.align 5

  1287. 

  1288. CGEMM_L1x2_LOOP:

  1289. 

  1290. 	KERNEL1x2_1

  1291. 	KERNEL1x2_2

  1292. 	KERNEL1x2_1

  1293. 	KERNEL1x2_2

  1294. 

  1295. 	KERNEL1x2_1

  1296. 	KERNEL1x2_2

  1297. 	KERNEL1x2_1

  1298. 	KERNEL1x2_2

  1299. 

  1300. 	addic.		L,	L,	-1

  1301. 	bgt		CGEMM_L1x2_LOOP

  1302. 

  1303. CGEMM_L1x2_LOOP_END:

  1304. 

  1305. 	KERNEL1x2_1

  1306. 	KERNEL1x2_2

  1307. 	KERNEL1x2_1

  1308. 	KERNEL1x2_2

  1309. 

  1310. 	KERNEL1x2_1

  1311. 	KERNEL1x2_2

  1312. 	KERNEL1x2_1

  1313. 	KERNEL1x2_E2

  1314. 

  1315. 	b		CGEMM_L1x2_SUB1

  1316. 

  1317. CGEMM_L1x2_SUB4:

  1318. 

  1319. 	KERNEL1x2_SUBI1

  1320. 	KERNEL1x2_SUB1

  1321. 	KERNEL1x2_SUB1

  1322. 	KERNEL1x2_SUB1

  1323. 

  1324. 	KERNEL1x2_SUB1

  1325. 	KERNEL1x2_SUB1

  1326. 	KERNEL1x2_SUB1

  1327. 	KERNEL1x2_SUB1

  1328. 

  1329. 	b		CGEMM_L1x2_SUB1

  1330. 

  1331. CGEMM_L1x2_SUB0:

  1332. 

  1333. 	andi.		L,	K,	7

  1334. 

  1335. 	KERNEL1x2_SUBI1

  1336. 

  1337. 	addic.		L,	L,	-1

  1338. 	ble		CGEMM_L1x2_SAVE

  1339. 	b		CGEMM_L1x2_SUB2

  1340. 

  1341. CGEMM_L1x2_SUB1:

  1342. 

  1343. 	andi.		L,	K,	7

  1344. 	ble		CGEMM_L1x2_SAVE

  1345. 

  1346. CGEMM_L1x2_SUB2:

  1347. 

  1348. 	KERNEL1x2_SUB1

  1349. 

  1350. 	addic.		L,	L,	-1

  1351. 	bgt		CGEMM_L1x2_SUB2

  1352. 

  1353. CGEMM_L1x2_SAVE:

  1354. 

  1355. 	SAVE1x2

  1356. 

  1357. CGEMM_L1x2_END:

  1358. 

  1359. CGEMM_L1x1_BEGIN:

  1360. 

  1361. 

  1362. 	andi.		T1,	M,	1

  1363. 	ble		CGEMM_L1x1_END

  1364. 	mr		BO,	BBUFFER

  1365. 	srawi.		L,	K,	3

  1366. 	ble		CGEMM_L1x1_SUB0

  1367. 	cmpwi		cr0,	L,	1

  1368. 	ble		CGEMM_L1x1_SUB4

  1369. 

  1370. CGEMM_L1x1_LOOP_START:

  1371. 

  1372. 	LOAD1x1_1

  1373. 	KERNEL1x1_I1

  1374. 	KERNEL1x1_2

  1375. 	KERNEL1x1_1

  1376. 	KERNEL1x1_2

  1377. 

  1378. 	KERNEL1x1_1

  1379. 	KERNEL1x1_2

  1380. 	KERNEL1x1_1

  1381. 	KERNEL1x1_2

  1382. 

  1383. 	addic.		L,	L,	-2

  1384. 	ble		CGEMM_L1x1_LOOP_END

  1385. 

  1386. 	.align 5

  1387. 

  1388. CGEMM_L1x1_LOOP:

  1389. 

  1390. 	KERNEL1x1_1

  1391. 	KERNEL1x1_2

  1392. 	KERNEL1x1_1

  1393. 	KERNEL1x1_2

  1394. 

  1395. 	KERNEL1x1_1

  1396. 	KERNEL1x1_2

  1397. 	KERNEL1x1_1

  1398. 	KERNEL1x1_2

  1399. 

  1400. 	addic.		L,	L,	-1

  1401. 	bgt		CGEMM_L1x1_LOOP

  1402. 

  1403. CGEMM_L1x1_LOOP_END:

  1404. 

  1405. 	KERNEL1x1_1

  1406. 	KERNEL1x1_2

  1407. 	KERNEL1x1_1

  1408. 	KERNEL1x1_2

  1409. 

  1410. 	KERNEL1x1_1

  1411. 	KERNEL1x1_2

  1412. 	KERNEL1x1_1

  1413. 	KERNEL1x1_E2

  1414. 

  1415. 	b		CGEMM_L1x1_SUB1

  1416. 

  1417. CGEMM_L1x1_SUB4:

  1418. 

  1419. 	KERNEL1x1_SUBI1

  1420. 	KERNEL1x1_SUB1

  1421. 	KERNEL1x1_SUB1

  1422. 	KERNEL1x1_SUB1

  1423. 

  1424. 	KERNEL1x1_SUB1

  1425. 	KERNEL1x1_SUB1

  1426. 	KERNEL1x1_SUB1

  1427. 	KERNEL1x1_SUB1

  1428. 

  1429. 	b		CGEMM_L1x1_SUB1

  1430. 

  1431. CGEMM_L1x1_SUB0:

  1432. 

  1433. 	andi.		L,	K,	7

  1434. 

  1435. 	KERNEL1x1_SUBI1

  1436. 

  1437. 	addic.		L,	L,	-1

  1438. 	ble		CGEMM_L1x1_SAVE

  1439. 	b		CGEMM_L1x1_SUB2

  1440. 

  1441. CGEMM_L1x1_SUB1:

  1442. 

  1443. 	andi.		L,	K,	7

  1444. 	ble		CGEMM_L1x1_SAVE

  1445. 

  1446. CGEMM_L1x1_SUB2:

  1447. 

  1448. 	KERNEL1x1_SUB1

  1449. 

  1450. 	addic.		L,	L,	-1

  1451. 	bgt		CGEMM_L1x1_SUB2

  1452. 

  1453. CGEMM_L1x1_SAVE:

  1454. 

  1455. 	SAVE1x1

  1456. 

  1457. CGEMM_L1x1_END:

  1458. 

  1459. CGEMM_L1_END: