OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: eebfeba768
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'eebfeba768'
${ noResults }
OpenBLAS/kernel/power/zgemm_kernel_g4.S

1638 lines
31 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef __64BIT__

  43. #define LOAD	lwz

  44. #else

  45. #define LOAD	ld

  46. #endif

  47. 

  48. #ifdef __64BIT__

  49. #define STACKSIZE 320

  50. #define ALPHA_R 296(SP)

  51. #define ALPHA_I 304(SP)

  52. #define FZERO	312(SP)

  53. #else

  54. #define STACKSIZE 256

  55. #define ALPHA_R 224(SP)

  56. #define ALPHA_I 232(SP)

  57. #define FZERO	240(SP)

  58. #endif

  59. 

  60. #define	M	r3

  61. #define	N	r4

  62. #define	K	r5

  63. 

  64. #if defined(linux) || defined(__FreeBSD__)

  65. #ifndef __64BIT__

  66. #define A	r6

  67. #define	B	r7

  68. #define	C	r8

  69. #define	LDC	r9

  70. #define OFFSET	r10

  71. #else

  72. #define A	r8

  73. #define	B	r9

  74. #define	C	r10

  75. #define	LDC	r6

  76. #define OFFSET	r7

  77. #endif

  78. #endif

  79. 

  80. #if defined(_AIX) || defined(__APPLE__)

  81. #if !defined(__64BIT__) && defined(DOUBLE)

  82. #define A	r10

  83. #define	B	r6

  84. #define	C	r7

  85. #define	LDC	r8

  86. #define OFFSET	r9

  87. #else

  88. #define A	r8

  89. #define	B	r9

  90. #define	C	r10

  91. #define	LDC	r6

  92. #define OFFSET	r7

  93. #endif

  94. #endif

  95. 

  96. #define TEMP	r22

  97. #define KK	r23

  98. #define	I	r24

  99. #define J	r25

  100. #define AO	r26

  101. #define	BO	r27

  102. #define	CO1	r28

  103. #define CO2	r29

  104. 

  105. #define PREA	r30

  106. #define PREC	r31

  107. 

  108. #define A1	f16

  109. #define A2	f17

  110. #define A3	f18

  111. #define A4	f19

  112. #define A5	f20

  113. #define A6	f21

  114. #define B1	f22

  115. #define B2	f23

  116. #define B3	f24

  117. #define B4	f25

  118. #define B5	f26

  119. #define B6	f27

  120. #define B7	f28

  121. #define B8	f29

  122. #define B9	f30

  123. #define B10	f31

  124. 

  125. #ifndef NEEDPARAM

  126. 

  127. 	PROLOGUE

  128. 	PROFCODE

  129. 

  130. 	addi	SP, SP, -STACKSIZE

  131. 	li	r0, 0

  132. 

  133. 	stfd	f14,    0(SP)

  134. 	stfd	f15,    8(SP)

  135. 	stfd	f16,   16(SP)

  136. 	stfd	f17,   24(SP)

  137. 

  138. 	stfd	f18,   32(SP)

  139. 	stfd	f19,   40(SP)

  140. 	stfd	f20,   48(SP)

  141. 	stfd	f21,   56(SP)

  142. 

  143. 	stfd	f22,   64(SP)

  144. 	stfd	f23,   72(SP)

  145. 	stfd	f24,   80(SP)

  146. 	stfd	f25,   88(SP)

  147. 

  148. 	stfd	f26,   96(SP)

  149. 	stfd	f27,  104(SP)

  150. 	stfd	f28,  112(SP)

  151. 	stfd	f29,  120(SP)

  152. 

  153. 	stfd	f30,  128(SP)

  154. 	stfd	f31,  136(SP)

  155. 

  156. #ifdef __64BIT__

  157. 	std	r31,  144(SP)

  158. 	std	r30,  152(SP)

  159. 	std	r29,  160(SP)

  160. 	std	r28,  168(SP)

  161. 	std	r27,  176(SP)

  162. 	std	r26,  184(SP)

  163. 	std	r25,  192(SP)

  164. 	std	r24,  200(SP)

  165. #ifdef TRMMKERNEL

  166. 	std	r23,  208(SP)

  167. 	std	r22,  216(SP)

  168. #endif

  169. #else

  170. 	stw	r31,  144(SP)

  171. 	stw	r30,  148(SP)

  172. 	stw	r29,  152(SP)

  173. 	stw	r28,  156(SP)

  174. 	stw	r27,  160(SP)

  175. 	stw	r26,  164(SP)

  176. 	stw	r25,  168(SP)

  177. 	stw	r24,  172(SP)

  178. #ifdef TRMMKERNEL

  179. 	stw	r23,  176(SP)

  180. 	stw	r22,  180(SP)

  181. #endif

  182. #endif

  183. 

  184. 	stfd	f1,  ALPHA_R

  185. 	stfd	f2,  ALPHA_I

  186. 	stw	r0,  FZERO

  187. 

  188. #if defined(linux) || defined(__FreeBSD__)

  189. #ifdef __64BIT__

  190. 	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  191. #endif

  192. #endif

  193. 

  194. #if defined(_AIX) || defined(__APPLE__)

  195. #ifdef __64BIT__

  196. 	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  197. #else

  198. #ifdef DOUBLE

  199. 	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)

  200. 	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)

  201. 	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)

  202. #else

  203. 	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  204. #endif

  205. #endif

  206. #endif

  207. 

  208. #ifdef TRMMKERNEL

  209. #if (defined(linux) || defined(__FreeBSD__)) && defined(__64BIT__)

  210. 	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  211. #endif

  212. 

  213. #if defined(_AIX) || defined(__APPLE__)

  214. #ifdef __64BIT__

  215. 	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  216. #else

  217. #ifdef DOUBLE

  218. 	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)

  219. #else

  220. 	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  221. #endif

  222. #endif

  223. #endif

  224. #if defined(TRMMKERNEL) && !defined(LEFT)

  225. 	neg	KK, OFFSET

  226. #endif

  227. #endif

  228. 

  229. 	slwi	LDC, LDC, ZBASE_SHIFT

  230. 	li	PREA, 8 * 8 * SIZE

  231. 	li	PREC, 3 * SIZE

  232. 

  233. 	cmpwi	cr0, M, 0

  234. 	ble	.L999

  235. 	cmpwi	cr0, N, 0

  236. 	ble	.L999

  237. 	cmpwi	cr0, K, 0

  238. 	ble	.L999

  239. 

  240. 	lfs	f0, FZERO

  241. 

  242. 	srawi.	J, N,  1

  243. 	ble	.L30

  244. 	.align 4

  245. 

  246. .L10:

  247.  	fmr	f1,  f0

  248. 	fmr	f2,  f0

  249. 	fmr	f3,  f0

  250. 	fmr	f4,  f0

  251. 	fmr	f5,  f0

  252. 	fmr	f6,  f0

  253. 	fmr	f7,  f0

  254. 	fmr	f8,  f0

  255. 	fmr	f9,  f0

  256. 	fmr	f10, f0

  257. 	fmr	f11, f0

  258. 	fmr	f12, f0

  259. 	fmr	f13, f0

  260. 	fmr	f14, f0

  261. 	fmr	f15, f0

  262. 

  263. 	mr	CO1, C

  264. 	add	CO2, C,  LDC

  265. 	add	C,  CO2, LDC

  266. 

  267. #if defined(TRMMKERNEL) && defined(LEFT)

  268. 	mr	KK, OFFSET

  269. #endif

  270. 

  271. 	srawi.	I, M,  1

  272. 	mr	AO, A

  273. 	ble	.L20

  274. 	.align 4

  275. 

  276. .L11:

  277. #ifndef TRMMKERNEL

  278. 	LFD	A1,  0 * SIZE(AO)

  279. 	LFD	A2,  1 * SIZE(AO)

  280. 	LFD	A3,  2 * SIZE(AO)

  281. 	LFDU	A5,  4 * SIZE(AO)

  282. 

  283. 	LFD	B1,  0 * SIZE(B)

  284. 	LFD	B2,  1 * SIZE(B)

  285. 	LFD	B3,  2 * SIZE(B)

  286. 	LFD	B4,  3 * SIZE(B)

  287. 

  288. 	dcbtst	CO1, PREC

  289. 	dcbtst	CO2, PREC

  290. 

  291. 	srawi.	r0,  K,  1

  292. 	mr	BO,  B

  293. 	mtspr	CTR, r0

  294. 	ble	.L15

  295. #else

  296. #if	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  297. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  298. 

  299. 	LFD	A1,  0 * SIZE(AO)

  300. 	LFD	A2,  1 * SIZE(AO)

  301. 	LFD	A3,  2 * SIZE(AO)

  302. 	LFDU	A5,  4 * SIZE(AO)

  303. 

  304. 	LFD	B1,  0 * SIZE(B)

  305. 	LFD	B2,  1 * SIZE(B)

  306. 	LFD	B3,  2 * SIZE(B)

  307. 	LFD	B4,  3 * SIZE(B)

  308. 	mr	BO,  B

  309. 

  310. #else

  311. 	slwi	r0, KK, 1 + ZBASE_SHIFT

  312. 	add	AO, AO, r0

  313. 	add	BO, B,  r0

  314. 

  315. 	LFD	A1,  0 * SIZE(AO)

  316. 	LFD	A2,  1 * SIZE(AO)

  317. 	LFD	A3,  2 * SIZE(AO)

  318. 	LFDU	A5,  4 * SIZE(AO)

  319. 

  320. 	LFD	B1,  0 * SIZE(BO)

  321. 	LFD	B2,  1 * SIZE(BO)

  322. 	LFD	B3,  2 * SIZE(BO)

  323. 	LFD	B4,  3 * SIZE(BO)

  324. 

  325. #endif

  326. 

  327. 	dcbtst	CO1, PREC

  328. 	dcbtst	CO2, PREC

  329. 

  330. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  331. 	sub	TEMP, K, KK

  332. #elif defined(LEFT)

  333. 	addi	TEMP, KK, 2

  334. #else

  335. 	addi	TEMP, KK, 2

  336. #endif

  337. 	srawi.	TEMP,  TEMP,  1

  338. 	mtspr	CTR, TEMP

  339. 	ble	.L15

  340. #endif

  341. 	.align 4

  342. 

  343. .L12:

  344. 	FMADD	f0,  A1, B1, f0

  345. 	dcbt	AO, PREA

  346. 	FMADD	f4,  A1, B2, f4

  347. 	LFDU	B5,  4 * SIZE(BO)

  348. 

  349. 	FMADD	f8,  A1, B3, f8

  350. 	dcbt	BO, PREA

  351. 	FMADD	f12, A1, B4, f12

  352. 	LFD	A4, -1 * SIZE(AO)

  353. 

  354. 	FMADD	f1,  A2, B1, f1

  355. 	nop

  356. 	FMADD	f5,  A2, B2, f5

  357. 	LFD	B6,  1 * SIZE(BO)

  358. 

  359. 	FMADD	f9,  A2, B3, f9

  360. 	LFDU	A1,  4 * SIZE(AO)

  361. 	FMADD	f13, A2, B4, f13

  362. 	nop

  363. 

  364. 	FMADD	f2,  A3, B1, f2

  365. 	nop

  366. 	FMADD	f6,  A3, B2, f6

  367. 	LFD	B7,  2 * SIZE(BO)

  368. 

  369. 	FMADD	f10, A3, B3, f10

  370. 	LFD	A2, -3 * SIZE(AO)

  371. 	FMADD	f14, A3, B4, f14

  372. 	nop

  373. 

  374. 	FMADD	f3,  A4, B1, f3

  375. 	nop

  376. 	FMADD	f7,  A4, B2, f7

  377. 	LFD	B8,  3 * SIZE(BO)

  378. 

  379. 	FMADD	f11, A4, B3, f11

  380. 	LFD	A3, -2 * SIZE(AO)

  381. 	FMADD	f15, A4, B4, f15

  382. 	nop

  383. 

  384. 	FMADD	f0,  A5, B5, f0

  385. #ifdef DOUBLE

  386. 	dcbt	AO, PREA

  387. #else

  388. 	nop

  389. #endif

  390. 	FMADD	f4,  A5, B6, f4

  391. 	LFDU	B1,  4 * SIZE(BO)

  392. 

  393. 	FMADD	f8,  A5, B7, f8

  394. #ifdef DOUBLE

  395. 	dcbt	BO, PREA

  396. #else

  397. 	nop

  398. #endif

  399. 	FMADD	f12, A5, B8, f12

  400. 	LFD	A4, -1 * SIZE(AO)

  401. 

  402. 	FMADD	f1,  A2, B5, f1

  403. 	nop

  404. 	FMADD	f5,  A2, B6, f5

  405. 	LFD	B2,  1 * SIZE(BO)

  406. 

  407. 	FMADD	f9,  A2, B7, f9

  408. 	LFDU	A5,  4 * SIZE(AO)

  409. 	FMADD	f13, A2, B8, f13

  410. 	nop

  411. 

  412. 	FMADD	f2,  A3, B5, f2

  413. 	nop

  414. 	FMADD	f6,  A3, B6, f6

  415. 	LFD	B3,  2 * SIZE(BO)

  416. 

  417. 	FMADD	f10, A3, B7, f10

  418. 	LFD	A2, -3 * SIZE(AO)

  419. 	FMADD	f14, A3, B8, f14

  420. 	nop

  421. 

  422. 	FMADD	f3,  A4, B5, f3

  423. 	nop

  424. 	FMADD	f7,  A4, B6, f7

  425. 	LFD	B4,  3 * SIZE(BO)

  426. 

  427. 	FMADD	f11, A4, B7, f11

  428. 	LFD	A3, -2 * SIZE(AO)

  429. 	FMADD	f15, A4, B8, f15

  430. 	bdnz	.L12

  431. 	.align 4

  432. 	.align 4

  433. 

  434. .L15:

  435. 	addi	AO, AO, -4 * SIZE

  436. 

  437. #ifndef TRMMKERNEL

  438. 	andi.	r0,  K,  1

  439. 	lfd	f30,  ALPHA_R

  440. 	lfd	f31,  ALPHA_I

  441. 	ble	.LKERNEL_MainFinish

  442. #else

  443. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  444. 	sub	TEMP, K, KK

  445. #elif defined(LEFT)

  446. 	addi	TEMP, KK, 2

  447. #else

  448. 	addi	TEMP, KK, 2

  449. #endif

  450. 	andi.	TEMP,  TEMP,  1

  451. 	lfd	f30,  ALPHA_R

  452. 	lfd	f31,  ALPHA_I

  453. 	ble	.LKERNEL_MainFinish

  454. #endif

  455. 	.align 4

  456. 

  457. .L16:

  458. 	FMADD	f0,  A1, B1, f0

  459. 	LFD	A4,  3 * SIZE(AO)

  460. 	FMADD	f4,  A1, B2, f4

  461. 	FMADD	f8,  A1, B3, f8

  462. 	FMADD	f12, A1, B4, f12

  463. 

  464. 	FMADD	f1,  A2, B1, f1

  465. 	FMADD	f5,  A2, B2, f5

  466. 	FMADD	f9,  A2, B3, f9

  467. 	FMADD	f13, A2, B4, f13

  468. 

  469. 	FMADD	f2,  A3, B1, f2

  470. 	FMADD	f6,  A3, B2, f6

  471. 	FMADD	f10, A3, B3, f10

  472. 	FMADD	f14, A3, B4, f14

  473. 

  474. 	FMADD	f3,  A4, B1, f3

  475. 	FMADD	f7,  A4, B2, f7

  476. 	FMADD	f11, A4, B3, f11

  477. 	addi	AO, AO, 4 * SIZE

  478. 	FMADD	f15, A4, B4, f15

  479. 	addi	BO, BO, 4 * SIZE

  480. 	.align 4

  481. 

  482. .LKERNEL_MainFinish:

  483. #ifndef TRMMKERNEL

  484. 	LFD	f16, 0 * SIZE(CO1)

  485. 	LFD	f17, 1 * SIZE(CO1)

  486. 	LFD	f18, 2 * SIZE(CO1)

  487. 	LFD	f19, 3 * SIZE(CO1)

  488. #endif

  489. 

  490. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  491.       defined(CC) || defined(CR) || defined(RC) || defined(RR)

  492. 

  493. 	FSUB	  f0,  f0,  f5

  494. 	FADD	  f1,  f1,  f4

  495. 	FSUB	  f2,  f2,  f7

  496. 	FADD	  f3,  f3,  f6

  497. 

  498. #ifndef TRMMKERNEL

  499. 	LFD	f20, 0 * SIZE(CO2)

  500. 	LFD	f21, 1 * SIZE(CO2)

  501. 	LFD	f22, 2 * SIZE(CO2)

  502. 	LFD	f23, 3 * SIZE(CO2)

  503. #endif

  504. 

  505. 	FSUB	  f8,  f8,  f13

  506. 	FADD	  f9,  f9,  f12

  507. 	FSUB	  f10, f10, f15

  508. 	FADD	  f11, f11, f14

  509. 

  510. #elif defined(CN) || defined(CT) || defined(RN) || defined(RT)

  511. 

  512. 	FADD	  f0,  f0,  f5

  513. 	FSUB	  f1,  f1,  f4

  514. 	FADD	  f2,  f2,  f7

  515. 	FSUB	  f3,  f3,  f6

  516. 

  517. #ifndef TRMMKERNEL

  518. 	LFD	f20, 0 * SIZE(CO2)

  519. 	LFD	f21, 1 * SIZE(CO2)

  520. 	LFD	f22, 2 * SIZE(CO2)

  521. 	LFD	f23, 3 * SIZE(CO2)

  522. #endif

  523. 

  524. 	FADD	  f8,  f8,  f13

  525. 	FSUB	  f9,  f9,  f12

  526. 	FADD	  f10, f10, f15

  527. 	FSUB	  f11, f11, f14

  528. 

  529. #else /* defined(NC) || defined(TC) || defined(NR) || defined(TR) */

  530. 

  531. 	FADD	  f0,  f0,  f5

  532. 	FSUB	  f1,  f4,  f1

  533. 	FADD	  f2,  f2,  f7

  534. 	FSUB	  f3,  f6,  f3

  535. 

  536. #ifndef TRMMKERNEL

  537. 	LFD	f20, 0 * SIZE(CO2)

  538. 	LFD	f21, 1 * SIZE(CO2)

  539. 	LFD	f22, 2 * SIZE(CO2)

  540. 	LFD	f23, 3 * SIZE(CO2)

  541. #endif

  542. 

  543. 	FADD	  f8,  f8,  f13

  544. 	FSUB	  f9,  f12, f9

  545. 	FADD	  f10, f10, f15

  546. 	FSUB	  f11, f14, f11

  547. 

  548. #endif

  549. 

  550. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  551. 

  552. #ifndef TRMMKERNEL

  553. 	FMADD	f16, f30, f0,  f16

  554. 	FMADD	f17, f30, f1,  f17

  555. 	FMADD	f18, f30, f2,  f18

  556. 	FMADD	f19, f30, f3,  f19

  557. 

  558. 	FMADD	f20, f30, f8,  f20

  559. 	FMADD	f21, f30, f9,  f21

  560. 	FMADD	f22, f30, f10, f22

  561. 	FMADD	f23, f30, f11, f23

  562. #else

  563. 	FMUL	f16, f30, f0

  564. 	FMUL	f17, f30, f1

  565. 	FMUL	f18, f30, f2

  566. 	FMUL	f19, f30, f3

  567. 

  568. 	FMUL	f20, f30, f8

  569. 	FMUL	f21, f30, f9

  570. 	FMUL	f22, f30, f10

  571. 	FMUL	f23, f30, f11

  572. #endif

  573. 

  574. 	FNMSUB	f16, f31, f1,  f16

  575. 	FMADD	f17, f31, f0,  f17

  576. 	FNMSUB	f18, f31, f3,  f18

  577. 	FMADD	f19, f31, f2,  f19

  578. 

  579. 	FNMSUB	f20, f31, f9,  f20

  580. 	FMADD	f21, f31, f8,  f21

  581. 	FNMSUB	f22, f31, f11, f22

  582. 	FMADD	f23, f31, f10, f23

  583. 

  584. #else /* defined(CN)||defined(CT)||defined(NC)||defined(TC)||defined(CC) */

  585.       /* defined(RN)||defined(RT)||defined(NR)||defined(TR)||defined(CR) */

  586.       /* defined(RC)|| defined(RR) */

  587. 

  588. #ifndef TRMMKERNEL

  589. 	FMADD	f16, f30, f0,  f16

  590. 	FNMSUB	f17, f30, f1,  f17

  591. 	FMADD	f18, f30, f2,  f18

  592. 	FNMSUB	f19, f30, f3,  f19

  593. 

  594. 	FMADD	f20, f30, f8,  f20

  595. 	FNMSUB	f21, f30, f9,  f21

  596. 	FMADD	f22, f30, f10, f22

  597. 	FNMSUB	f23, f30, f11, f23

  598. 

  599. 	FMADD	f16, f31, f1,  f16

  600. 	FMADD	f17, f31, f0,  f17

  601. 	FMADD	f18, f31, f3,  f18

  602. 	FMADD	f19, f31, f2,  f19

  603. 

  604. 	FMADD	f20, f31, f9,  f20

  605. 	FMADD	f21, f31, f8,  f21

  606. 	FMADD	f22, f31, f11, f22

  607. 	FMADD	f23, f31, f10, f23

  608. #else

  609. 	FMUL	f16, f30, f0

  610. 	FMUL	f17, f30, f1

  611. 	FMUL	f18, f30, f2

  612. 	FMUL	f19, f30, f3

  613. 

  614. 	FMUL	f20, f30, f8

  615. 	FMUL	f21, f30, f9

  616. 	FMUL	f22, f30, f10

  617. 	FMUL	f23, f30, f11

  618. 

  619. 	FMADD	f16, f31, f1,  f16

  620. 	FNMADD	f17, f31, f0,  f17

  621. 	FMADD	f18, f31, f3,  f18

  622. 	FNMADD	f19, f31, f2,  f19

  623. 

  624. 	FMADD	f20, f31, f9,  f20

  625. 	FNMADD	f21, f31, f8,  f21

  626. 	FMADD	f22, f31, f11, f22

  627. 	FNMADD	f23, f31, f10, f23

  628. #endif

  629. #endif

  630. 

  631. 	STFD	f16,  0 * SIZE(CO1)

  632. 	STFD	f17,  1 * SIZE(CO1)

  633. 	STFD	f18,  2 * SIZE(CO1)

  634. 	STFD	f19,  3 * SIZE(CO1)

  635. 

  636. 	lfs	f0,  FZERO

  637.  	fmr	f1,  f0

  638. 	fmr	f2,  f0

  639. 	fmr	f3,  f0

  640. 

  641. 	STFD	f20,  0 * SIZE(CO2)

  642. 	STFD	f21,  1 * SIZE(CO2)

  643. 	STFD	f22,  2 * SIZE(CO2)

  644. 	STFD	f23,  3 * SIZE(CO2)

  645. 

  646. 	fmr	f4,  f0

  647. 	fmr	f5,  f0

  648. 	fmr	f6,  f0

  649. 	fmr	f7,  f0

  650. 

  651. 	fmr	f8,  f0

  652. 	fmr	f9,  f0

  653. 	fmr	f10, f0

  654. 	fmr	f11, f0

  655. 

  656. 	fmr	f12, f0

  657. 	fmr	f13, f0

  658. 	fmr	f14, f0

  659. 	fmr	f15, f0

  660. 

  661. 	addi	CO1, CO1, 4 * SIZE

  662. 	addi	CO2, CO2, 4 * SIZE

  663. 

  664. #ifdef TRMMKERNEL

  665. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  666.     (!defined(LEFT) && !defined(TRANSA))

  667. 	sub	TEMP, K, KK

  668. #ifdef LEFT

  669. 	addi	TEMP, TEMP, -2

  670. #else

  671. 	addi	TEMP, TEMP, -2

  672. #endif

  673. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  674. 	add	AO, AO, TEMP

  675. 	add	BO, BO, TEMP

  676. #endif

  677. 

  678. #ifdef LEFT

  679. 	addi	KK, KK, 2

  680. #endif

  681. #endif

  682. 

  683. 	addic.	I, I, -1

  684. 	bgt	.L11

  685. 	.align 4

  686. 

  687. .L20:

  688. 	andi.	I,  M,  1

  689. 	ble	.L29

  690. 

  691. #ifndef TRMMKERNEL

  692. 	LFD	f16,  0 * SIZE(AO)

  693. 	LFD	f17,  1 * SIZE(AO)

  694. 	LFD	f18,  2 * SIZE(AO)

  695. 	LFD	f19,  3 * SIZE(AO)

  696. 

  697. 	LFD	f20,  0 * SIZE(B)

  698. 	LFD	f21,  1 * SIZE(B)

  699. 	LFD	f22,  2 * SIZE(B)

  700. 	LFD	f23,  3 * SIZE(B)

  701. 

  702. 	LFD	f24,  4 * SIZE(B)

  703. 	LFD	f25,  5 * SIZE(B)

  704. 	LFD	f26,  6 * SIZE(B)

  705. 	LFD	f27,  7 * SIZE(B)

  706. 

  707. 	lfs	f0, FZERO

  708. 	fmr	f1, f0

  709. 	fmr	f2, f0

  710. 	fmr	f3, f0

  711. 	fmr	f4, f0

  712. 	fmr	f5, f0

  713. 	fmr	f6, f0

  714. 	fmr	f7, f0

  715. 

  716. 	srawi.	r0,  K,  2

  717. 	mr	BO,  B

  718. 	mtspr	CTR, r0

  719. 	ble	.L25

  720. #else

  721. #if	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  722. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  723. 

  724. 	LFD	f16,  0 * SIZE(AO)

  725. 	LFD	f17,  1 * SIZE(AO)

  726. 	LFD	f18,  2 * SIZE(AO)

  727. 	LFD	f19,  3 * SIZE(AO)

  728. 

  729. 	LFD	f20,  0 * SIZE(B)

  730. 	LFD	f21,  1 * SIZE(B)

  731. 	LFD	f22,  2 * SIZE(B)

  732. 	LFD	f23,  3 * SIZE(B)

  733. 

  734. 	LFD	f24,  4 * SIZE(B)

  735. 	LFD	f25,  5 * SIZE(B)

  736. 	LFD	f26,  6 * SIZE(B)

  737. 	LFD	f27,  7 * SIZE(B)

  738. 

  739. 	mr	BO,  B

  740. #else

  741. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  742. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  743. 	add	AO, AO, r0

  744. 	add	BO, B,  TEMP

  745. 

  746. 	LFD	f16,  0 * SIZE(AO)

  747. 	LFD	f17,  1 * SIZE(AO)

  748. 	LFD	f18,  2 * SIZE(AO)

  749. 	LFD	f19,  3 * SIZE(AO)

  750. 

  751. 	LFD	f20,  0 * SIZE(BO)

  752. 	LFD	f21,  1 * SIZE(BO)

  753. 	LFD	f22,  2 * SIZE(BO)

  754. 	LFD	f23,  3 * SIZE(BO)

  755. 

  756. 	LFD	f24,  4 * SIZE(BO)

  757. 	LFD	f25,  5 * SIZE(BO)

  758. 	LFD	f26,  6 * SIZE(BO)

  759. 	LFD	f27,  7 * SIZE(BO)

  760. #endif

  761. 

  762. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  763. 	sub	TEMP, K, KK

  764. #elif defined(LEFT)

  765. 	addi	TEMP, KK, 1

  766. #else

  767. 	addi	TEMP, KK, 2

  768. #endif

  769. 	srawi.	TEMP,  TEMP,  2

  770. 	mtspr	CTR, TEMP

  771. 	ble	.L25

  772. #endif

  773. 	.align 4

  774. 

  775. .L22:

  776. 	fmadd	f0,  f16, f20, f0

  777. 	LFD	f27,  7 * SIZE(BO)

  778. 	fmadd	f1,  f16, f21, f1

  779. 	LFD	f19,  3 * SIZE(AO)

  780. 	fmadd	f2,  f16, f22, f2

  781. 	nop

  782. 	fmadd	f3,  f16, f23, f3

  783. 	LFD	f16,  4 * SIZE(AO)

  784. 

  785. 	fmadd	f4,  f17, f20, f4

  786.  	LFD	f20,  8 * SIZE(BO)

  787. 	fmadd	f5,  f17, f21, f5

  788. 	LFD	f21,  9 * SIZE(BO)

  789. 	fmadd	f6,  f17, f22, f6

  790. 	LFD	f22, 10 * SIZE(BO)

  791. 	fmadd	f7,  f17, f23, f7

  792. 	LFD	f23, 11 * SIZE(BO)

  793. 

  794. 	fmadd	f0,  f18, f24, f0

  795. 	LFD	f17,  5 * SIZE(AO)

  796. 	fmadd	f1,  f18, f25, f1

  797. 	nop

  798. 	fmadd	f2,  f18, f26, f2

  799. 	nop

  800. 	fmadd	f3,  f18, f27, f3

  801. 	LFD	f18,  6 * SIZE(AO)

  802. 

  803. 	fmadd	f4,  f19, f24, f4

  804.  	LFD	f24, 12 * SIZE(BO)

  805. 	fmadd	f5,  f19, f25, f5

  806. 	LFD	f25, 13 * SIZE(BO)

  807. 	fmadd	f6,  f19, f26, f6

  808. 	LFD	f26, 14 * SIZE(BO)

  809. 	fmadd	f7,  f19, f27, f7

  810. 	LFD	f27, 15 * SIZE(BO)

  811. 

  812. 	fmadd	f0,  f16, f20, f0

  813. 	LFD	f19,  7 * SIZE(AO)

  814. 	fmadd	f1,  f16, f21, f1

  815. 	nop

  816. 	fmadd	f2,  f16, f22, f2

  817. 	nop

  818. 	fmadd	f3,  f16, f23, f3

  819. 	LFDU	f16,  8 * SIZE(AO)

  820. 

  821. 	fmadd	f4,  f17, f20, f4

  822.  	LFDU	f20, 16 * SIZE(BO)

  823. 	fmadd	f5,  f17, f21, f5

  824. 	LFD	f21,  1 * SIZE(BO)

  825. 	fmadd	f6,  f17, f22, f6

  826. 	LFD	f22,  2 * SIZE(BO)

  827. 	fmadd	f7,  f17, f23, f7

  828. 	LFD	f23,  3 * SIZE(BO)

  829. 

  830. 	fmadd	f0,  f18, f24, f0

  831. 	LFD	f17,  1 * SIZE(AO)

  832. 	fmadd	f1,  f18, f25, f1

  833. 	nop

  834. 	fmadd	f2,  f18, f26, f2

  835. 	nop

  836. 	fmadd	f3,  f18, f27, f3

  837. 	LFD	f18,  2 * SIZE(AO)

  838. 

  839. 	fmadd	f4,  f19, f24, f4

  840.  	LFD	f24,  4 * SIZE(BO)

  841. 	fmadd	f5,  f19, f25, f5

  842. 	LFD	f25,  5 * SIZE(BO)

  843. 	fmadd	f6,  f19, f26, f6

  844. 	LFD	f26,  6 * SIZE(BO)

  845. 	fmadd	f7,  f19, f27, f7

  846. 	bdnz	.L22

  847. 	.align 4

  848. 

  849. .L25:

  850. #ifndef TRMMKERNEL

  851. 	andi.	r0,  K,  3

  852. 	lfd	f30, ALPHA_R

  853. 	lfd	f31, ALPHA_I

  854. 	mtspr	CTR, r0

  855. 	ble	.L27

  856. #else

  857. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  858. 	sub	TEMP, K, KK

  859. #elif defined(LEFT)

  860. 	addi	TEMP, KK, 1

  861. #else

  862. 	addi	TEMP, KK, 2

  863. #endif

  864. 	andi.	TEMP,  TEMP,  3

  865. 	lfd	f30, ALPHA_R

  866. 	lfd	f31, ALPHA_I

  867. 	mtspr	CTR, TEMP

  868. 	ble	.L27

  869. #endif

  870. 	.align 4

  871. 

  872. .L26:

  873. 	fmadd	f0,  f16, f20, f0

  874. 	fmadd	f1,  f16, f21, f1

  875. 	fmadd	f2,  f16, f22, f2

  876. 	fmadd	f3,  f16, f23, f3

  877. 	LFDU	f16,  2 * SIZE(AO)

  878. 

  879. 	fmadd	f4,  f17, f20, f4

  880.  	LFDU	f20,  4 * SIZE(BO)

  881. 	fmadd	f5,  f17, f21, f5

  882. 	LFD	f21,  1 * SIZE(BO)

  883. 	fmadd	f6,  f17, f22, f6

  884. 	LFD	f22,  2 * SIZE(BO)

  885. 	fmadd	f7,  f17, f23, f7

  886. 	LFD	f23,  3 * SIZE(BO)

  887. 	LFD	f17,  1 * SIZE(AO)

  888. 	bdnz	.L26

  889. 	.align 4

  890. 

  891. .L27:

  892. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  893.       defined(CC) || defined(CR) || defined(RC) || defined(RR)

  894. 

  895. 	FSUB	  f0,  f0,  f5

  896. 	FADD	  f1,  f1,  f4

  897. 	FSUB	  f2,  f2,  f7

  898. 	FADD	  f3,  f3,  f6

  899. 

  900. #elif defined(CN) || defined(CT) || defined(RN) || defined(RT)

  901. 

  902. 	FADD	  f0,  f0,  f5

  903. 	FSUB	  f1,  f4,  f1

  904. 	FADD	  f2,  f2,  f7

  905. 	FSUB	  f3,  f6,  f3

  906. 

  907. #else /* defined(NC) || defined(TC) || defined(NR) || defined(TR) */

  908. 

  909. 	FADD	  f0,  f0,  f5

  910. 	FSUB	  f1,  f1,  f4

  911. 	FADD	  f2,  f2,  f7

  912. 	FSUB	  f3,  f3,  f6

  913. 

  914. #endif

  915. 

  916. #ifndef TRMMKERNEL

  917. 	LFD	f16, 0 * SIZE(CO1)

  918. 	LFD	f17, 1 * SIZE(CO1)

  919. 

  920. 	LFD	f18, 0 * SIZE(CO2)

  921. 	LFD	f19, 1 * SIZE(CO2)

  922. #endif

  923. 

  924. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  925. 

  926. #ifndef TRMMKERNEL

  927. 	FMADD	f16, f30, f0,  f16

  928. 	FMADD	f17, f30, f1,  f17

  929. 	FMADD	f18, f30, f2,  f18

  930. 	FMADD	f19, f30, f3,  f19

  931. #else

  932. 	FMUL	f16, f30, f0

  933. 	FMUL	f17, f30, f1

  934. 	FMUL	f18, f30, f2

  935. 	FMUL	f19, f30, f3

  936. #endif

  937. 

  938. 	FNMSUB	f16, f31, f1,  f16

  939. 	FMADD	f17, f31, f0,  f17

  940. 	FNMSUB	f18, f31, f3,  f18

  941. 	FMADD	f19, f31, f2,  f19

  942. 

  943. 

  944. #else /* defined(CN)||defined(CT)||defined(NC)||defined(TC)||defined(CC) */

  945.       /* defined(RN)||defined(RT)||defined(NR)||defined(TR)||defined(CR) */

  946.       /* defined(RC)|| defined(RR) */

  947. 

  948. #ifndef TRMMKERNEL

  949. 	FMADD	f16, f30, f0,  f16

  950. 	FNMSUB	f17, f30, f1,  f17

  951. 	FMADD	f18, f30, f2,  f18

  952. 	FNMSUB	f19, f30, f3,  f19

  953. 

  954. 	FMADD	f16, f31, f1,  f16

  955. 	FMADD	f17, f31, f0,  f17

  956. 	FMADD	f18, f31, f3,  f18

  957. 	FMADD	f19, f31, f2,  f19

  958. #else

  959. 	FMUL	f16, f30, f0

  960. 	FMUL	f17, f30, f1

  961. 	FMUL	f18, f30, f2

  962. 	FMUL	f19, f30, f3

  963. 

  964. 	FMADD	f16, f31, f1,  f16

  965. 	FNMADD	f17, f31, f0,  f17

  966. 	FMADD	f18, f31, f3,  f18

  967. 	FNMADD	f19, f31, f2,  f19

  968. #endif

  969. #endif

  970. 

  971. 	STFD	f16, 0 * SIZE(CO1)

  972. 	STFD	f17, 1 * SIZE(CO1)

  973. 	STFD	f18, 0 * SIZE(CO2)

  974. 	STFD	f19, 1 * SIZE(CO2)

  975. 

  976. 	addi	CO1, CO1, 2 * SIZE

  977. 	addi	CO2, CO2, 2 * SIZE

  978. 

  979. #ifdef TRMMKERNEL

  980. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  981.     (!defined(LEFT) && !defined(TRANSA))

  982. 	sub	TEMP, K, KK

  983. #ifdef LEFT

  984. 	addi	TEMP, TEMP, -1

  985. #else

  986. 	addi	TEMP, TEMP, -2

  987. #endif

  988. 	slwi	r0,   TEMP, 0 + ZBASE_SHIFT

  989. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  990. 	add	AO, AO, r0

  991. 	add	BO, BO, TEMP

  992. #endif

  993. 

  994. #ifdef LEFT

  995. 	addi	KK, KK, 1

  996. #endif

  997. #endif

  998. 	.align 4

  999. 

  1000. .L29:

  1001. #if defined(TRMMKERNEL) && !defined(LEFT)

  1002. 	addi	KK, KK, 2

  1003. #endif

  1004. 

  1005. 	mr	B,  BO

  1006. 	addic.	J, J, -1

  1007. 	lfs	f0, FZERO

  1008. 	bgt	.L10

  1009. 	.align 4

  1010. 

  1011. .L30:

  1012. 	andi.	J, N,  1

  1013. 	ble	.L999

  1014. 

  1015. #if defined(TRMMKERNEL) && defined(LEFT)

  1016. 	mr	KK, OFFSET

  1017. #endif

  1018. 

  1019. 	srawi.	I,  M,  1

  1020. 	mr	CO1, C

  1021. 	add	C, C, LDC

  1022. 	mr	AO, A

  1023. 	ble	.L40

  1024. 	.align 4

  1025. 

  1026. .L31:

  1027. #ifndef TRMMKERNEL

  1028. 	LFD	f20,  0 * SIZE(AO)

  1029. 	LFD	f21,  1 * SIZE(AO)

  1030. 	LFD	f22,  2 * SIZE(AO)

  1031. 	LFD	f23,  3 * SIZE(AO)

  1032. 

  1033. 	LFD	f24,  4 * SIZE(AO)

  1034. 	LFD	f25,  5 * SIZE(AO)

  1035. 	LFD	f26,  6 * SIZE(AO)

  1036. 	LFD	f27,  7 * SIZE(AO)

  1037. 

  1038. 	LFD	f16, 0 * SIZE(B)

  1039. 	LFD	f17, 1 * SIZE(B)

  1040. 	LFD	f18, 2 * SIZE(B)

  1041. 	LFD	f19, 3 * SIZE(B)

  1042. 

  1043. 	lfs	f0, FZERO

  1044. 	fmr	f1, f0

  1045. 	fmr	f2, f0

  1046. 	fmr	f3, f0

  1047. 	fmr	f4, f0

  1048. 	fmr	f5, f0

  1049. 	fmr	f6, f0

  1050. 	fmr	f7, f0

  1051. 

  1052. 	srawi.	r0,  K,  2

  1053. 	mr	BO, B

  1054. 	mtspr	CTR, r0

  1055. 	ble	.L35

  1056. #else

  1057. #if	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1058. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1059. 

  1060. 	LFD	f20,  0 * SIZE(AO)

  1061. 	LFD	f21,  1 * SIZE(AO)

  1062. 	LFD	f22,  2 * SIZE(AO)

  1063. 	LFD	f23,  3 * SIZE(AO)

  1064. 

  1065. 	LFD	f24,  4 * SIZE(AO)

  1066. 	LFD	f25,  5 * SIZE(AO)

  1067. 	LFD	f26,  6 * SIZE(AO)

  1068. 	LFD	f27,  7 * SIZE(AO)

  1069. 

  1070. 	LFD	f16, 0 * SIZE(B)

  1071. 	LFD	f17, 1 * SIZE(B)

  1072. 	LFD	f18, 2 * SIZE(B)

  1073. 	LFD	f19, 3 * SIZE(B)

  1074. 

  1075. 	lfs	f0, FZERO

  1076. 	fmr	f1, f0

  1077. 	fmr	f2, f0

  1078. 	fmr	f3, f0

  1079. 	fmr	f4, f0

  1080. 	fmr	f5, f0

  1081. 	fmr	f6, f0

  1082. 	fmr	f7, f0

  1083. 

  1084. 	mr	BO,  B

  1085. #else

  1086. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  1087. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  1088. 	add	AO, AO, r0

  1089. 	add	BO, B,  TEMP

  1090. 

  1091. 	LFD	f20,  0 * SIZE(AO)

  1092. 	LFD	f21,  1 * SIZE(AO)

  1093. 	LFD	f22,  2 * SIZE(AO)

  1094. 	LFD	f23,  3 * SIZE(AO)

  1095. 

  1096. 	LFD	f24,  4 * SIZE(AO)

  1097. 	LFD	f25,  5 * SIZE(AO)

  1098. 	LFD	f26,  6 * SIZE(AO)

  1099. 	LFD	f27,  7 * SIZE(AO)

  1100. 

  1101. 	LFD	f16, 0 * SIZE(BO)

  1102. 	LFD	f17, 1 * SIZE(BO)

  1103. 	LFD	f18, 2 * SIZE(BO)

  1104. 	LFD	f19, 3 * SIZE(BO)

  1105. 

  1106. 	lfs	f0, FZERO

  1107. 	fmr	f1, f0

  1108. 	fmr	f2, f0

  1109. 	fmr	f3, f0

  1110. 	fmr	f4, f0

  1111. 	fmr	f5, f0

  1112. 	fmr	f6, f0

  1113. 	fmr	f7, f0

  1114. #endif

  1115. 

  1116. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1117. 	sub	TEMP, K, KK

  1118. #elif defined(LEFT)

  1119. 	addi	TEMP, KK, 2

  1120. #else

  1121. 	addi	TEMP, KK, 1

  1122. #endif

  1123. 	srawi.	TEMP,  TEMP,  2

  1124. 	mtspr	CTR, TEMP

  1125. 	ble	.L35

  1126. #endif

  1127. 	.align 4

  1128. 

  1129. .L32:

  1130. 	fmadd	f0,  f16, f20, f0

  1131. 	LFD	f27,  7 * SIZE(AO)

  1132. 	fmadd	f1,  f16, f21, f1

  1133. 	LFD	f19,  3 * SIZE(BO)

  1134. 	fmadd	f2,  f16, f22, f2

  1135. 	nop

  1136. 	fmadd	f3,  f16, f23, f3

  1137. 	LFD	f16,  4 * SIZE(BO)

  1138. 

  1139. 	fmadd	f4,  f17, f20, f4

  1140. 	LFD	f20,  8 * SIZE(AO)

  1141. 	fmadd	f5,  f17, f21, f5

  1142. 	LFD	f21,  9 * SIZE(AO)

  1143. 	fmadd	f6,  f17, f22, f6

  1144. 	LFD	f22, 10 * SIZE(AO)

  1145. 	fmadd	f7,  f17, f23, f7

  1146. 	LFD	f23, 11 * SIZE(AO)

  1147. 

  1148. 	fmadd	f0,  f18, f24, f0

  1149. 	LFD	f17,  5 * SIZE(BO)

  1150. 	fmadd	f1,  f18, f25, f1

  1151. 	nop

  1152. 	fmadd	f2,  f18, f26, f2

  1153. 	nop

  1154. 	fmadd	f3,  f18, f27, f3

  1155. 	LFD	f18,  6 * SIZE(BO)

  1156. 

  1157. 	fmadd	f4,  f19, f24, f4

  1158. 	LFD	f24, 12 * SIZE(AO)

  1159. 	fmadd	f5,  f19, f25, f5

  1160. 	LFD	f25, 13 * SIZE(AO)

  1161. 	fmadd	f6,  f19, f26, f6

  1162. 	LFD	f26, 14 * SIZE(AO)

  1163. 	fmadd	f7,  f19, f27, f7

  1164. 	LFD	f27, 15 * SIZE(AO)

  1165. 

  1166. 	fmadd	f0,  f16, f20, f0

  1167. 	LFD	f19,  7 * SIZE(BO)

  1168. 	fmadd	f1,  f16, f21, f1

  1169. 	nop

  1170. 	fmadd	f2,  f16, f22, f2

  1171. 	nop

  1172. 	fmadd	f3,  f16, f23, f3

  1173. 	LFDU	f16,  8 * SIZE(BO)

  1174. 

  1175. 	fmadd	f4,  f17, f20, f4

  1176. 	LFDU	f20, 16 * SIZE(AO)

  1177. 	fmadd	f5,  f17, f21, f5

  1178. 	LFD	f21,  1 * SIZE(AO)

  1179. 	fmadd	f6,  f17, f22, f6

  1180. 	LFD	f22,  2 * SIZE(AO)

  1181. 	fmadd	f7,  f17, f23, f7

  1182. 	LFD	f23,  3 * SIZE(AO)

  1183. 

  1184. 	fmadd	f0,  f18, f24, f0

  1185. 	LFD	f17,  1 * SIZE(BO)

  1186. 	fmadd	f1,  f18, f25, f1

  1187. 	nop

  1188. 	fmadd	f2,  f18, f26, f2

  1189. 	nop

  1190. 	fmadd	f3,  f18, f27, f3

  1191. 	LFD	f18,  2 * SIZE(BO)

  1192. 

  1193. 	fmadd	f4,  f19, f24, f4

  1194. 	LFD	f24,  4 * SIZE(AO)

  1195. 	fmadd	f5,  f19, f25, f5

  1196. 	LFD	f25,  5 * SIZE(AO)

  1197. 	fmadd	f6,  f19, f26, f6

  1198. 	LFD	f26,  6 * SIZE(AO)

  1199. 	fmadd	f7,  f19, f27, f7

  1200. 	bdnz	.L32

  1201. 	.align 4

  1202. 

  1203. .L35:

  1204. #ifndef TRMMKERNEL

  1205. 	andi.	r0,  K,  3

  1206. 	lfd	f30, ALPHA_R

  1207. 	lfd	f31, ALPHA_I

  1208. 	mtspr	CTR, r0

  1209. 	ble	.L37

  1210. #else

  1211. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1212. 	sub	TEMP, K, KK

  1213. #elif defined(LEFT)

  1214. 	addi	TEMP, KK, 2

  1215. #else

  1216. 	addi	TEMP, KK, 1

  1217. #endif

  1218. 	andi.	TEMP,  TEMP,  3

  1219. 	lfd	f30, ALPHA_R

  1220. 	lfd	f31, ALPHA_I

  1221. 	mtspr	CTR, TEMP

  1222. 	ble	.L37

  1223. #endif

  1224. 	.align 4

  1225. 

  1226. .L36:

  1227. 	fmadd	f0,  f16, f20, f0

  1228. 	fmadd	f1,  f16, f21, f1

  1229. 	fmadd	f2,  f16, f22, f2

  1230. 	fmadd	f3,  f16, f23, f3

  1231. 	LFDU	f16, 2 * SIZE(BO)

  1232. 

  1233. 	fmadd	f4,  f17, f20, f4

  1234. 	LFDU	f20, 4 * SIZE(AO)

  1235. 	fmadd	f5,  f17, f21, f5

  1236. 	LFD	f21, 1 * SIZE(AO)

  1237. 	fmadd	f6,  f17, f22, f6

  1238. 	LFD	f22, 2 * SIZE(AO)

  1239. 	fmadd	f7,  f17, f23, f7

  1240. 	LFD	f23, 3 * SIZE(AO)

  1241. 	LFD	f17, 1 * SIZE(BO)

  1242. 	bdnz	.L36

  1243. 	.align 4

  1244. 

  1245. .L37:

  1246. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1247.       defined(CC) || defined(CR) || defined(RC) || defined(RR)

  1248. 

  1249. 	FSUB	  f0,  f0,  f5

  1250. 	FADD	  f1,  f1,  f4

  1251. 	FSUB	  f2,  f2,  f7

  1252. 	FADD	  f3,  f3,  f6

  1253. 

  1254. #elif defined(CN) || defined(CT) || defined(RN) || defined(RT)

  1255. 

  1256. 	FADD	  f0,  f0,  f5

  1257. 	FSUB	  f1,  f1,  f4

  1258. 	FADD	  f2,  f2,  f7

  1259. 	FSUB	  f3,  f3,  f6

  1260. 

  1261. #else /* defined(NC) || defined(TC) || defined(NR) || defined(TR) */

  1262. 

  1263. 	FADD	  f0,  f0,  f5

  1264. 	FSUB	  f1,  f4,  f1

  1265. 	FADD	  f2,  f2,  f7

  1266. 	FSUB	  f3,  f6,  f3

  1267. 

  1268. #endif

  1269. 

  1270. #ifndef TRMMKERNEL

  1271. 	LFD	f16, 0 * SIZE(CO1)

  1272. 	LFD	f17, 1 * SIZE(CO1)

  1273. 	LFD	f18, 2 * SIZE(CO1)

  1274. 	LFD	f19, 3 * SIZE(CO1)

  1275. #endif

  1276. 

  1277. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  1278. 

  1279. #ifndef TRMMKERNEL

  1280. 	FMADD	f16, f30, f0,  f16

  1281. 	FMADD	f17, f30, f1,  f17

  1282. 	FMADD	f18, f30, f2,  f18

  1283. 	FMADD	f19, f30, f3,  f19

  1284. #else

  1285. 	FMUL	f16, f30, f0

  1286. 	FMUL	f17, f30, f1

  1287. 	FMUL	f18, f30, f2

  1288. 	FMUL	f19, f30, f3

  1289. #endif

  1290. 

  1291. 	FNMSUB	f16, f31, f1,  f16

  1292. 	FMADD	f17, f31, f0,  f17

  1293. 	FNMSUB	f18, f31, f3,  f18

  1294. 	FMADD	f19, f31, f2,  f19

  1295. 

  1296. #else /* defined(CN)||defined(CT)||defined(NC)||defined(TC)||defined(CC) */

  1297.       /* defined(RN)||defined(RT)||defined(NR)||defined(TR)||defined(CR) */

  1298.       /* defined(RC)|| defined(RR) */

  1299. 

  1300. #ifndef TRMMKERNEL

  1301. 	FMADD	f16, f30, f0,  f16

  1302. 	FNMSUB	f17, f30, f1,  f17

  1303. 	FMADD	f18, f30, f2,  f18

  1304. 	FNMSUB	f19, f30, f3,  f19

  1305. 

  1306. 	FMADD	f16, f31, f1,  f16

  1307. 	FMADD	f17, f31, f0,  f17

  1308. 	FMADD	f18, f31, f3,  f18

  1309. 	FMADD	f19, f31, f2,  f19

  1310. #else

  1311. 	FMUL	f16, f30, f0

  1312. 	FMUL	f17, f30, f1

  1313. 	FMUL	f18, f30, f2

  1314. 	FMUL	f19, f30, f3

  1315. 

  1316. 	FMADD	f16, f31, f1,  f16

  1317. 	FNMADD	f17, f31, f0,  f17

  1318. 	FMADD	f18, f31, f3,  f18

  1319. 	FNMADD	f19, f31, f2,  f19

  1320. #endif

  1321. 

  1322. #endif

  1323. 

  1324. 	STFD	f16, 0 * SIZE(CO1)

  1325. 	STFD	f17, 1 * SIZE(CO1)

  1326. 	STFD	f18, 2 * SIZE(CO1)

  1327. 	STFD	f19, 3 * SIZE(CO1)

  1328. 

  1329. 	addi	CO1, CO1, 4 * SIZE

  1330. 

  1331. #ifdef TRMMKERNEL

  1332. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  1333.     (!defined(LEFT) && !defined(TRANSA))

  1334. 	sub	TEMP, K, KK

  1335. #ifdef LEFT

  1336. 	addi	TEMP, TEMP, -2

  1337. #else

  1338. 	addi	TEMP, TEMP, -1

  1339. #endif

  1340. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  1341. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  1342. 	add	AO, AO, r0

  1343. 	add	BO, BO, TEMP

  1344. #endif

  1345. 

  1346. #ifdef LEFT

  1347. 	addi	KK, KK, 2

  1348. #endif

  1349. #endif

  1350. 

  1351. 	addic.	I, I, -1

  1352. 	bgt	.L31

  1353. 	.align 4

  1354. 

  1355. .L40:

  1356. 	andi.	I,  M,  1

  1357. 	ble	.L999

  1358. 

  1359. #ifndef TRMMKERNEL

  1360. 	LFD	f16, 0 * SIZE(AO)

  1361. 	LFD	f17, 1 * SIZE(AO)

  1362. 	LFD	f18, 2 * SIZE(AO)

  1363. 	LFD	f19, 3 * SIZE(AO)

  1364. 

  1365. 	LFD	f20, 0 * SIZE(B)

  1366. 	LFD	f21, 1 * SIZE(B)

  1367. 	LFD	f22, 2 * SIZE(B)

  1368. 	LFD	f23, 3 * SIZE(B)

  1369. 

  1370. 	lfs	f0, FZERO

  1371. 	fmr	f1, f0

  1372. 	fmr	f2, f0

  1373. 	fmr	f3, f0

  1374. 	fmr	f4, f0

  1375. 	fmr	f5, f0

  1376. 	fmr	f6, f0

  1377. 	fmr	f7, f0

  1378. 

  1379. 	srawi.	r0,  K,  2

  1380. 	mr	BO, B

  1381. 	mtspr	CTR, r0

  1382. 	ble	.L45

  1383. #else

  1384. #if	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1385. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1386. 

  1387. 	LFD	f16,  0 * SIZE(AO)

  1388. 	LFD	f17,  1 * SIZE(AO)

  1389. 	LFD	f18,  2 * SIZE(AO)

  1390. 	LFD	f19,  3 * SIZE(AO)

  1391. 

  1392. 	LFD	f20,  0 * SIZE(B)

  1393. 	LFD	f21,  1 * SIZE(B)

  1394. 	LFD	f22,  2 * SIZE(B)

  1395. 	LFD	f23,  3 * SIZE(B)

  1396. 

  1397. 	lfs	f0, FZERO

  1398. 	fmr	f1, f0

  1399. 	fmr	f2, f0

  1400. 	fmr	f3, f0

  1401. 	fmr	f4, f0

  1402. 	fmr	f5, f0

  1403. 	fmr	f6, f0

  1404. 	fmr	f7, f0

  1405. 

  1406. 	mr	BO,  B

  1407. #else

  1408. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  1409. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  1410. 	add	AO, AO, r0

  1411. 	add	BO, B,  TEMP

  1412. 

  1413. 	LFD	f16,  0 * SIZE(AO)

  1414. 	LFD	f17,  1 * SIZE(AO)

  1415. 	LFD	f18,  2 * SIZE(AO)

  1416. 	LFD	f19,  3 * SIZE(AO)

  1417. 

  1418. 	LFD	f20,  0 * SIZE(BO)

  1419. 	LFD	f21,  1 * SIZE(BO)

  1420. 	LFD	f22,  2 * SIZE(BO)

  1421. 	LFD	f23,  3 * SIZE(BO)

  1422. 

  1423. 	lfs	f0, FZERO

  1424. 	fmr	f1, f0

  1425. 	fmr	f2, f0

  1426. 	fmr	f3, f0

  1427. 	fmr	f4, f0

  1428. 	fmr	f5, f0

  1429. 	fmr	f6, f0

  1430. 	fmr	f7, f0

  1431. #endif

  1432. 

  1433. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1434. 	sub	TEMP, K, KK

  1435. #elif defined(LEFT)

  1436. 	addi	TEMP, KK, 1

  1437. #else

  1438. 	addi	TEMP, KK, 1

  1439. #endif

  1440. 	srawi.	TEMP,  TEMP,  2

  1441. 	mtspr	CTR, TEMP

  1442. 	ble	.L45

  1443. #endif

  1444. 	.align 4

  1445. 

  1446. .L42:

  1447. 	fmadd	f0,  f16,  f20,  f0

  1448. 	LFD	f23,  3 * SIZE(BO)

  1449. 	fmadd	f3,  f16,  f21,  f3

  1450. 	LFD	f16,  4 * SIZE(AO)

  1451. 	fmadd	f2,  f17,  f20,  f2

  1452. 	LFD	f20,  4 * SIZE(BO)

  1453. 	fmadd	f1,  f17,  f21,  f1

  1454. 	LFD	f17,  5 * SIZE(AO)

  1455. 

  1456. 	fmadd	f4,  f18,  f22,  f4

  1457. 	LFD	f21,  5 * SIZE(BO)

  1458. 	fmadd	f7,  f18,  f23,  f7

  1459. 	LFD	f18,  6 * SIZE(AO)

  1460. 	fmadd	f6,  f19,  f22,  f6

  1461. 	LFD	f22,  6 * SIZE(BO)

  1462. 	fmadd	f5,  f19,  f23,  f5

  1463. 	LFD	f19,  7 * SIZE(AO)

  1464. 

  1465. 	fmadd	f0,  f16,  f20,  f0

  1466. 	LFD	f23,  7 * SIZE(BO)

  1467. 	fmadd	f3,  f16,  f21,  f3

  1468. 	LFDU	f16,  8 * SIZE(AO)

  1469. 	fmadd	f2,  f17,  f20,  f2

  1470. 	LFDU	f20,  8 * SIZE(BO)

  1471. 	fmadd	f1,  f17,  f21,  f1

  1472. 	LFD	f17,  1 * SIZE(AO)

  1473. 

  1474. 	fmadd	f4,  f18,  f22,  f4

  1475. 	LFD	f21,  1 * SIZE(BO)

  1476. 	fmadd	f7,  f18,  f23,  f7

  1477. 	LFD	f18,  2 * SIZE(AO)

  1478. 	fmadd	f6,  f19,  f22,  f6

  1479. 	LFD	f22,  2 * SIZE(BO)

  1480. 	fmadd	f5,  f19,  f23,  f5

  1481. 	LFD	f19,  3 * SIZE(AO)

  1482. 	bdnz	.L42

  1483. 	.align 4

  1484. 

  1485. .L45:

  1486. 	fadd	f0, f0, f4

  1487. 	fadd	f1, f1, f5

  1488. 	fadd	f2, f2, f6

  1489. 	fadd	f3, f3, f7

  1490. 

  1491. #ifndef TRMMKERNEL

  1492. 	andi.	r0,  K,  3

  1493. 	lfd	f30, ALPHA_R

  1494. 	lfd	f31, ALPHA_I

  1495. 	mtspr	CTR,r0

  1496. 	ble	.L47

  1497. #else

  1498. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1499. 	sub	TEMP, K, KK

  1500. #elif defined(LEFT)

  1501. 	addi	TEMP, KK, 1

  1502. #else

  1503. 	addi	TEMP, KK, 1

  1504. #endif

  1505. 	andi.	TEMP,  TEMP,  3

  1506. 	lfd	f30, ALPHA_R

  1507. 	lfd	f31, ALPHA_I

  1508. 	mtspr	CTR,TEMP

  1509. 	ble	.L47

  1510. #endif

  1511. 	.align 4

  1512. 

  1513. .L46:

  1514. 	fmadd	f0,  f16,  f20,  f0

  1515. 	fmadd	f3,  f16,  f21,  f3

  1516. 	LFDU	f16, 2 * SIZE(AO)

  1517. 	fmadd	f2,  f17,  f20,  f2

  1518. 	LFDU	f20, 2 * SIZE(BO)

  1519. 	fmadd	f1,  f17,  f21,  f1

  1520. 	LFD	f17, 1 * SIZE(AO)

  1521. 	LFD	f21, 1 * SIZE(BO)

  1522. 	bdnz	.L46

  1523. 	.align 4

  1524. 

  1525. .L47:

  1526. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1527.       defined(CC) || defined(CR) || defined(RC) || defined(RR)

  1528. 	fsub	f0, f0, f1

  1529. 	fadd	f2, f2, f3

  1530. #elif defined(CN) || defined(CT) || defined(RN) || defined(RT)

  1531. 	fadd	f0, f0, f1

  1532. 	fsub	f2, f2, f3

  1533. #else

  1534. 	fadd	f0, f0, f1

  1535. 	fsub	f2, f3, f2

  1536. #endif

  1537. 

  1538. #ifndef TRMMKERNEL

  1539. 	LFD	f16,  0 * SIZE(CO1)

  1540. 	LFD	f17,  1 * SIZE(CO1)

  1541. #endif

  1542. 

  1543. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  1544. #ifndef TRMMKERNEL

  1545. 	FMADD	f16, f30, f0, f16

  1546. 	FMADD	f17, f30, f2, f17

  1547. #else

  1548. 	FMUL	f16, f30, f0

  1549. 	FMUL	f17, f30, f2

  1550. #endif

  1551. 

  1552. 	FNMSUB	f16, f31, f2, f16

  1553. 	FMADD	f17, f31, f0, f17

  1554. 

  1555. #else /* defined(CN)||defined(CT)||defined(NC)||defined(TC)||defined(CC) */

  1556.       /* defined(RN)||defined(RT)||defined(NR)||defined(TR)||defined(CR) */

  1557.       /* defined(RC) || defined(RR) */

  1558. 

  1559. #ifndef TRMMKERNEL

  1560. 	FMADD	f16, f30, f0, f16

  1561. 	FNMSUB	f17, f30, f2, f17

  1562. 

  1563. 	FMADD	f16, f31, f2, f16

  1564. 	FMADD	f17, f31, f0, f17

  1565. #else

  1566. 	FMUL	f16, f30, f0

  1567. 	FMUL	f17, f30, f2

  1568. 

  1569. 	FMADD	f16, f31, f2, f16

  1570. 	FNMADD	f17, f31, f0, f17

  1571. #endif

  1572. 

  1573. #endif

  1574. 	STFD	f16,  0 * SIZE(CO1)

  1575. 	STFD	f17,  1 * SIZE(CO1)

  1576. 	.align 4

  1577. 

  1578. .L999:

  1579. 	addi	r3, 0, 0

  1580. 

  1581. 	lfd	f14,    0(SP)

  1582. 	lfd	f15,    8(SP)

  1583. 	lfd	f16,   16(SP)

  1584. 	lfd	f17,   24(SP)

  1585. 

  1586. 	lfd	f18,   32(SP)

  1587. 	lfd	f19,   40(SP)

  1588. 	lfd	f20,   48(SP)

  1589. 	lfd	f21,   56(SP)

  1590. 

  1591. 	lfd	f22,   64(SP)

  1592. 	lfd	f23,   72(SP)

  1593. 	lfd	f24,   80(SP)

  1594. 	lfd	f25,   88(SP)

  1595. 

  1596. 	lfd	f26,   96(SP)

  1597. 	lfd	f27,  104(SP)

  1598. 	lfd	f28,  112(SP)

  1599. 	lfd	f29,  120(SP)

  1600. 

  1601. 	lfd	f30,  128(SP)

  1602. 	lfd	f31,  136(SP)

  1603. 

  1604. #ifdef __64BIT__

  1605. 	ld	r31,  144(SP)

  1606. 	ld	r30,  152(SP)

  1607. 	ld	r29,  160(SP)

  1608. 	ld	r28,  168(SP)

  1609. 	ld	r27,  176(SP)

  1610. 	ld	r26,  184(SP)

  1611. 	ld	r25,  192(SP)

  1612. 	ld	r24,  200(SP)

  1613. #ifdef TRMMKERNEL

  1614. 	ld	r23,  208(SP)

  1615. 	ld	r22,  216(SP)

  1616. #endif

  1617. #else

  1618. 	lwz	r31,  144(SP)

  1619. 	lwz	r30,  148(SP)

  1620. 	lwz	r29,  152(SP)

  1621. 	lwz	r28,  156(SP)

  1622. 	lwz	r27,  160(SP)

  1623. 	lwz	r26,  164(SP)

  1624. 	lwz	r25,  168(SP)

  1625. 	lwz	r24,  172(SP)

  1626. #ifdef TRMMKERNEL

  1627. 	lwz	r23,  176(SP)

  1628. 	lwz	r22,  180(SP)

  1629. #endif

  1630. #endif

  1631. 

  1632. 	addi	SP, SP, STACKSIZE

  1633. 

  1634. 	blr

  1635. 

  1636. 	EPILOGUE

  1637. #endif