OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 02634b549b
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '02634b549b'
${ noResults }
OpenBLAS/kernel/power/zgemm_kernel_hummer.S

4429 lines
79 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #undef ZERO

  43. 

  44. #define ALPHA    0

  45. #define FZERO	16

  46. 

  47. #define	M	r3

  48. #define	N	r4

  49. #define	K	r5

  50. 

  51. #ifdef linux

  52. #define A	r6

  53. #define	B	r7

  54. #define	C	r8

  55. #define	LDC	r9

  56. #define OFFSET	r10

  57. #endif

  58. 

  59. #define TEMP	r11

  60. #define KK	r14

  61. #define INCM1	r15

  62. #define INCM3	r16

  63. #define INCM5	r17

  64. #define INCM7	r18

  65. #define INC2	r19

  66. #define INC	r20

  67. #define INC4	r21

  68. 

  69. #define	I	r22

  70. #define J	r23

  71. #define AO	r24

  72. #define BO	r25

  73. #define AO2	r26

  74. #define	BO2	r27

  75. 

  76. #define	CO1	r28

  77. #define CO2	r29

  78. #define	ZERO	r31

  79. 

  80. #ifndef NEEDPARAM

  81. 

  82. #define A1	f16

  83. #define A2	f17

  84. #define A3	f18

  85. #define A4	f19

  86. #define A5	f20

  87. #define A6	f21

  88. #define A7	f22

  89. #define A8	f23

  90. #define A9	f24

  91. #define A10	f25

  92. 

  93. #define B1	f26

  94. #define B2	f27

  95. #define B3	f28

  96. #define B4	f29

  97. #define B5	f30

  98. #define B6	f31

  99. 

  100. #define AP	B6

  101. 

  102. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  103.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  104. #define FXCPMADD fxcpmadd

  105. #define FXCSMADD fxcxnpma

  106. #else

  107. #define FXCPMADD fxcpnsma

  108. #define FXCSMADD fxcxma

  109. #endif

  110. 

  111. 	PROLOGUE

  112. 	PROFCODE

  113. 

  114. 	li	r0, -16

  115. 

  116. 	stfpdux	f14, SP, r0

  117. 	stfpdux	f15, SP, r0

  118. 	stfpdux	f16, SP, r0

  119. 	stfpdux	f17, SP, r0

  120. 	stfpdux	f18, SP, r0

  121. 	stfpdux	f19, SP, r0

  122. 	stfpdux	f20, SP, r0

  123. 	stfpdux	f21, SP, r0

  124. 	stfpdux	f22, SP, r0

  125. 	stfpdux	f23, SP, r0

  126. 	stfpdux	f24, SP, r0

  127. 	stfpdux	f25, SP, r0

  128. 	stfpdux	f26, SP, r0

  129. 	stfpdux	f27, SP, r0

  130. 	stfpdux	f28, SP, r0

  131. 	stfpdux	f29, SP, r0

  132. 	stfpdux	f30, SP, r0

  133. 	stfpdux	f31, SP, r0

  134. 

  135. 	stwu	r31,  -4(SP)

  136. 	stwu	r30,  -4(SP)

  137. 	stwu	r29,  -4(SP)

  138. 	stwu	r28,  -4(SP)

  139. 

  140. 	stwu	r27,  -4(SP)

  141. 	stwu	r26,  -4(SP)

  142. 	stwu	r25,  -4(SP)

  143. 	stwu	r24,  -4(SP)

  144. 

  145. 	stwu	r23,  -4(SP)

  146. 	stwu	r22,  -4(SP)

  147. 	stwu	r21,  -4(SP)

  148. 	stwu	r20,  -4(SP)

  149. 

  150. 	stwu	r19,  -4(SP)

  151. 	stwu	r18,  -4(SP)

  152. 	stwu	r17,  -4(SP)

  153. 	stwu	r16,  -4(SP)

  154. 

  155. 	stwu	r15,  -4(SP)

  156. 	stwu	r14,  -4(SP)

  157. 

  158. 	li	r0,   0

  159. 	stwu	r0,   -4(SP)

  160. 	stwu	r0,   -4(SP)

  161. 

  162. 	stfdu	f2,   -8(SP)

  163. 	stfdu	f1,   -8(SP)

  164. 

  165. 	slwi	LDC, LDC, ZBASE_SHIFT

  166. 

  167. 	cmpwi	cr0, M, 0

  168. 	ble	.L999

  169. 	cmpwi	cr0, N, 0

  170. 	ble	.L999

  171. 	cmpwi	cr0, K, 0

  172. 	ble	.L999

  173. 

  174. #if defined(TRMMKERNEL) && !defined(LEFT)

  175. 	neg	KK, OFFSET

  176. #endif

  177. 

  178. 	andi.	r0, C,   2 * SIZE - 1

  179. 	bne	.L1000

  180. 

  181. 	li	INC,    1 * SIZE

  182. 	li	INC2,   2 * SIZE

  183. 	li	INC4,   4 * SIZE

  184. 	li	INCM1, -1 * SIZE

  185. 	li	INCM3, -2 * SIZE

  186. 	li	INCM5, -4 * SIZE

  187. 	li	INCM7, -6 * SIZE

  188. 

  189. 	addi	C, C, - 2 * SIZE

  190. 	srawi.	J, N,  1

  191. 	ble	.L50

  192. 	.align 4

  193. 

  194. .L10:

  195. 	mr	CO1, C

  196. 	add	CO2, C,   LDC

  197. 	add	C,   CO2, LDC

  198. 

  199. #if defined(TRMMKERNEL) &&  defined(LEFT)

  200. 	mr	KK, OFFSET

  201. #endif

  202. 

  203. 	addi	AO, A, -4 * SIZE

  204. 

  205. 	li	r0, FZERO

  206. 	lfpsx	f0, SP, r0

  207. 

  208. 	srawi.	I, M,  2

  209. 	ble	.L20

  210. 	.align 4

  211. 

  212. .L11:

  213. #if defined(TRMMKERNEL)

  214. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  215. 	addi	AO2, AO,   2 * SIZE

  216. 	fpmr	f4,  f0

  217. 	addi	BO,  B,  - 4 * SIZE

  218. 	fpmr	f8,  f0

  219. 	addi	BO2, B,  - 2 * SIZE

  220. 	fpmr	f12, f0

  221. #else

  222. 	slwi	TEMP, KK, 2 + ZBASE_SHIFT

  223. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  224. 	add	AO, AO, TEMP

  225. 	add	BO, B,  r0

  226. 

  227. 	addi	AO2, AO,   2 * SIZE

  228. 	fpmr	f4,  f0

  229. 	addi	BO,  BO, - 4 * SIZE

  230. 	fpmr	f8,  f0

  231. 	addi	BO2, BO,   2 * SIZE

  232. 	fpmr	f12, f0

  233. #endif

  234. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  235. 	sub	TEMP, K, KK

  236. #elif defined(LEFT)

  237. 	addi	TEMP, KK, 4

  238. #else

  239. 	addi	TEMP, KK, 2

  240. #endif

  241. 	srawi.	r0,  TEMP,  2

  242.  	fpmr	f1,  f0

  243. 	mtspr	CTR, r0

  244. 	ble	.L14

  245. #else

  246. 	addi	AO2, AO,   2 * SIZE

  247. 	fpmr	f4,  f0

  248. 	addi	BO,  B,  - 4 * SIZE

  249. 	fpmr	f8,  f0

  250. 	addi	BO2, B,  - 2 * SIZE

  251. 	fpmr	f12, f0

  252. 

  253. 	srawi.	r0,  K,  2

  254.  	fpmr	f1,  f0

  255. 	mtspr	CTR, r0

  256. 	ble	.L14

  257. #endif

  258. 

  259. 	LFPDUX	A1,  AO, INC4

  260. 	fpmr	f5,  f0

  261. 	LFPDUX	A3,  AO, INC4

  262. 	fpmr	f9,  f0

  263. 	LFPDUX	B1,  BO, INC4

  264. 	fpmr	f13, f0

  265. 

  266. 	LFPDUX	A5,  AO, INC4

  267. 	fpmr	f2,  f0

  268. 	LFPDUX	A6,  AO, INC4

  269. 	fpmr	f6,  f0

  270. 	LFPDUX	B3,  BO, INC4

  271. 	fpmr	f10, f0

  272. 	LFPDUX	A7,  AO, INC4

  273. 	fpmr	f14, f0

  274. 

  275. 	LFPDUX	A8,  AO, INC4

  276. 	fpmr	f3,  f0

  277. 	LFPDUX	B5,  BO, INC4

  278. 	fpmr	f7,  f0

  279. 	LFPDUX	A9,  AO, INC4

  280. 	fpmr	f11, f0

  281. 	LFPDUX	A2, AO2, INC4

  282. 	fpmr	f15, f0

  283. 	LFPDUX	B2, BO2, INC4

  284. 	bdz-	.L13

  285. 	.align 4

  286. 

  287. .L12:

  288. 

  289. ## 1 ##

  290. 	FXCPMADD	f0,  B1, A1, f0

  291. 	nop

  292. 	FXCSMADD	f4,  B1, A1, f4

  293. 	nop

  294. 	FXCPMADD	f8,  B2, A1, f8

  295. 	LFPDUX	B4, BO2, INC4

  296. 	FXCSMADD	f12, B2, A1, f12

  297. 	LFPDUX	B6,  BO, INC4

  298. 

  299. 	FXCPMADD	f1,  B1, A2, f1

  300. 	nop

  301. 	FXCSMADD	f5,  B1, A2, f5

  302. 	LFPDUX	A4, AO2, INC4

  303. 	FXCPMADD	f9,  B2, A2, f9

  304. 	LFPDUX	A10, AO, INC4

  305. 	FXCSMADD	f13, B2, A2, f13

  306. 	nop

  307. 

  308. 	FXCPMADD	f2,  B1, A3, f2

  309. 	nop

  310. 	FXCSMADD	f6,  B1, A3, f6

  311. 	nop

  312. 	FXCPMADD	f10, B2, A3, f10

  313. 	nop

  314. 	FXCSMADD	f14, B2, A3, f14

  315. 	nop

  316. 

  317. 	FXCPMADD	f3,  B1, A4, f3

  318. 	nop

  319. 	FXCSMADD	f7,  B1, A4, f7

  320. 	LFPDUX	A2, AO2, INC4

  321. 	FXCPMADD	f11, B2, A4, f11

  322. 	LFPDUX	A1,  AO, INC4

  323. 	FXCSMADD	f15, B2, A4, f15

  324. 	nop

  325. 

  326. ## 2 ##

  327. 

  328. 	FXCPMADD	f0,  B3, A5, f0

  329. 	nop

  330. 	FXCSMADD	f4,  B3, A5, f4

  331. 	nop

  332. 	FXCPMADD	f8,  B4, A5, f8

  333. 	LFPDUX	B2, BO2, INC4

  334. 	FXCSMADD	f12, B4, A5, f12

  335. 	LFPDUX	B1,  BO, INC4

  336. 

  337. 	FXCPMADD	f1,  B3, A2, f1

  338. 	nop

  339. 	FXCSMADD	f5,  B3, A2, f5

  340. 	LFPDUX	A4, AO2, INC4

  341. 	FXCPMADD	f9,  B4, A2, f9

  342. 	LFPDUX	A3,  AO, INC4

  343. 	FXCSMADD	f13, B4, A2, f13

  344. 	nop

  345. 

  346. 	FXCPMADD	f2,  B3, A6, f2

  347. 	nop

  348. 	FXCSMADD	f6,  B3, A6, f6

  349. 	nop

  350. 	FXCPMADD	f10, B4, A6, f10

  351. 	nop

  352. 	FXCSMADD	f14, B4, A6, f14

  353. 	nop

  354. 

  355. 	FXCPMADD	f3,  B3, A4, f3

  356. 	nop

  357. 	FXCSMADD	f7,  B3, A4, f7

  358. 	LFPDUX	A2, AO2, INC4

  359. 	FXCPMADD	f11, B4, A4, f11

  360. 	LFPDUX	A5,  AO, INC4

  361. 	FXCSMADD	f15, B4, A4, f15

  362. 	nop

  363. 

  364. ## 3 ##

  365. 

  366. 	FXCPMADD	f0,  B5, A7, f0

  367. 	nop

  368. 	FXCSMADD	f4,  B5, A7, f4

  369. 	nop

  370. 	FXCPMADD	f8,  B2, A7, f8

  371. 	LFPDUX	B4, BO2, INC4

  372. 	FXCSMADD	f12, B2, A7, f12

  373. 	LFPDUX	B3,  BO, INC4

  374. 

  375. 	FXCPMADD	f1,  B5, A2, f1

  376. 	nop

  377. 	FXCSMADD	f5,  B5, A2, f5

  378. 	LFPDUX	A4, AO2, INC4

  379. 	FXCPMADD	f9,  B2, A2, f9

  380. 	LFPDUX	A6,  AO, INC4

  381. 	FXCSMADD	f13, B2, A2, f13

  382. 	nop

  383. 

  384. 	FXCPMADD	f2,  B5, A8, f2

  385. 	nop

  386. 	FXCSMADD	f6,  B5, A8, f6

  387. 	nop

  388. 	FXCPMADD	f10, B2, A8, f10

  389. 	nop

  390. 	FXCSMADD	f14, B2, A8, f14

  391. 	nop

  392. 

  393. 	FXCPMADD	f3,  B5, A4, f3

  394. 	nop

  395. 	FXCSMADD	f7,  B5, A4, f7

  396. 	LFPDUX	A2, AO2, INC4

  397. 	FXCPMADD	f11, B2, A4, f11

  398. 	LFPDUX	A7,  AO, INC4

  399. 	FXCSMADD	f15, B2, A4, f15

  400. 	nop

  401. 

  402. ## 4 ##

  403. 	FXCPMADD	f0,  B6, A9, f0

  404. 	nop

  405. 	FXCSMADD	f4,  B6, A9, f4

  406. 	nop

  407. 	FXCPMADD	f8,  B4, A9, f8

  408. 	LFPDUX	B2, BO2, INC4

  409. 	FXCSMADD	f12, B4, A9, f12

  410. 	LFPDUX	B5,  BO, INC4

  411. 

  412. 	FXCPMADD	f1,  B6, A2, f1

  413. 	nop

  414. 	FXCSMADD	f5,  B6, A2, f5

  415. 	LFPDUX	A4, AO2, INC4

  416. 	FXCPMADD	f9,  B4, A2, f9

  417. 	LFPDUX	A8,  AO, INC4

  418. 	FXCSMADD	f13, B4, A2, f13

  419. 	nop

  420. 

  421. 	FXCPMADD	f2,  B6, A10, f2

  422. 	nop

  423. 	FXCSMADD	f6,  B6, A10, f6

  424. 	nop

  425. 	FXCPMADD	f10, B4, A10, f10

  426. 	nop

  427. 	FXCSMADD	f14, B4, A10, f14

  428. 	nop

  429. 

  430. 	FXCPMADD	f3,  B6, A4, f3

  431. 	LFPDUX	A2, AO2, INC4

  432. 	FXCSMADD	f7,  B6, A4, f7

  433. 	LFPDUX	A9,  AO, INC4

  434. 	FXCPMADD	f11, B4, A4, f11

  435. 	nop

  436. 	FXCSMADD	f15, B4, A4, f15

  437. 	bdnz+	.L12

  438. 	.align 4

  439. 

  440. .L13:

  441. ## 1 ##

  442. 

  443. 	FXCPMADD	f0,  B1, A1, f0

  444. 	nop

  445. 	FXCSMADD	f4,  B1, A1, f4

  446. 	nop

  447. 	FXCPMADD	f8,  B2, A1, f8

  448. 	LFPDUX	B4, BO2, INC4

  449. 	FXCSMADD	f12, B2, A1, f12

  450. 	LFPDUX	B6,  BO, INC4

  451. 

  452. 	FXCPMADD	f1,  B1, A2, f1

  453. 	nop

  454. 	FXCSMADD	f5,  B1, A2, f5

  455. 	LFPDUX	A4, AO2, INC4

  456. 	FXCPMADD	f9,  B2, A2, f9

  457. 	LFPDUX	A10, AO, INC4

  458. 	FXCSMADD	f13, B2, A2, f13

  459. 	nop

  460. 

  461. 	FXCPMADD	f2,  B1, A3, f2

  462. 	nop

  463. 	FXCSMADD	f6,  B1, A3, f6

  464. 	nop

  465. 	FXCPMADD	f10, B2, A3, f10

  466. 	nop

  467. 	FXCSMADD	f14, B2, A3, f14

  468. 	nop

  469. 

  470. 	FXCPMADD	f3,  B1, A4, f3

  471. 	nop

  472. 	FXCSMADD	f7,  B1, A4, f7

  473. 	LFPDUX	A2, AO2, INC4

  474. 	FXCPMADD	f11, B2, A4, f11

  475. #ifndef TRMMKERNEL

  476. 	LFPDUX	A1, CO1, INC2

  477. #else

  478. 	nop

  479. #endif

  480. 	FXCSMADD	f15, B2, A4, f15

  481. 	nop

  482. 

  483. ## 2 ##

  484. 

  485. 	FXCPMADD	f0,  B3, A5, f0

  486. 	nop

  487. 	FXCSMADD	f4,  B3, A5, f4

  488. 	nop

  489. 	FXCPMADD	f8,  B4, A5, f8

  490. 	LFPDUX	B2, BO2, INC4

  491. 	FXCSMADD	f12, B4, A5, f12

  492. #ifndef TRMMKERNEL

  493. 	LFPDUX	B1, CO1, INC2

  494. #else

  495. 	nop

  496. #endif

  497. 

  498. 	FXCPMADD	f1,  B3, A2, f1

  499. 	nop

  500. 	FXCSMADD	f5,  B3, A2, f5

  501. 	LFPDUX	A4, AO2, INC4

  502. 	FXCPMADD	f9,  B4, A2, f9

  503. #ifndef TRMMKERNEL

  504. 	LFPDUX	A3, CO1, INC2

  505. #else

  506. 	nop

  507. #endif

  508. 	FXCSMADD	f13, B4, A2, f13

  509. 	nop

  510. 

  511. 	FXCPMADD	f2,  B3, A6, f2

  512. 	nop

  513. 	FXCSMADD	f6,  B3, A6, f6

  514. 	nop

  515. 	FXCPMADD	f10, B4, A6, f10

  516. 	nop

  517. 	FXCSMADD	f14, B4, A6, f14

  518. 	nop

  519. 

  520. 	FXCPMADD	f3,  B3, A4, f3

  521. 	nop

  522. 	FXCSMADD	f7,  B3, A4, f7

  523. 	LFPDUX	A2, AO2, INC4

  524. 	FXCPMADD	f11, B4, A4, f11

  525. #ifndef TRMMKERNEL

  526.    	LFPDUX	A5, CO1, INC2

  527. #else

  528. 	nop

  529. #endif

  530. 	FXCSMADD	f15, B4, A4, f15

  531. 	nop

  532. 

  533. ## 3 ##

  534. 

  535. 	FXCPMADD	f0,  B5, A7, f0

  536. 	nop

  537. 	FXCSMADD	f4,  B5, A7, f4

  538. 	nop

  539. 	FXCPMADD	f8,  B2, A7, f8

  540. 	LFPDUX	B4, BO2, INC4

  541. 	FXCSMADD	f12, B2, A7, f12

  542. #ifndef TRMMKERNEL

  543. 	LFPDUX	B3, CO2, INC2

  544. #else

  545. 	nop

  546. #endif

  547. 

  548. 	FXCPMADD	f1,  B5, A2, f1

  549. 	nop

  550. 	FXCSMADD	f5,  B5, A2, f5

  551. 	LFPDUX	A4, AO2, INC4

  552. 	FXCPMADD	f9,  B2, A2, f9

  553. #ifndef TRMMKERNEL

  554. 	LFPDUX	A6, CO2, INC2

  555. #else

  556. 	nop

  557. #endif

  558. 	FXCSMADD	f13, B2, A2, f13

  559. 

  560. 	FXCPMADD	f2,  B5, A8, f2

  561. 	nop

  562. 	FXCSMADD	f6,  B5, A8, f6

  563. 	nop

  564. 	FXCPMADD	f10, B2, A8, f10

  565. 	nop

  566. 	FXCSMADD	f14, B2, A8, f14

  567. 	nop

  568. 

  569. 	FXCPMADD	f3,  B5, A4, f3

  570. 	nop

  571. 	FXCSMADD	f7,  B5, A4, f7

  572. 	LFPDUX	A2, AO2, INC4

  573. 	FXCPMADD	f11, B2, A4, f11

  574. #ifndef TRMMKERNEL

  575. 	LFPDUX	A7, CO2, INC2

  576. #else

  577. 	nop

  578. #endif

  579. 	FXCSMADD	f15, B2, A4, f15

  580. 	nop

  581. 

  582. ## 4 ##

  583. 

  584. 	FXCPMADD	f0,  B6, A9, f0

  585. 	nop

  586. 	FXCSMADD	f4,  B6, A9, f4

  587. 	nop

  588. 	FXCPMADD	f8,  B4, A9, f8

  589. #ifndef TRMMKERNEL

  590. 	LFPDUX	B2, CO2, INC2

  591. #else

  592. 	nop

  593. #endif

  594. 

  595. 	FXCSMADD	f12, B4, A9, f12

  596. 

  597. 	FXCPMADD	f1,  B6, A2, f1

  598. 	nop

  599. 	FXCSMADD	f5,  B6, A2, f5

  600. 	LFPDUX	A4, AO2, INC4

  601. 	FXCPMADD	f9,  B4, A2, f9

  602. 	nop

  603. 	FXCSMADD	f13, B4, A2, f13

  604. 	nop

  605. 

  606. 	FXCPMADD	f2,  B6, A10, f2

  607. 	FXCSMADD	f6,  B6, A10, f6

  608. 	FXCPMADD	f10, B4, A10, f10

  609. 	FXCSMADD	f14, B4, A10, f14

  610. 

  611. 	FXCPMADD	f3,  B6, A4, f3

  612. 	FXCSMADD	f7,  B6, A4, f7

  613. 	FXCPMADD	f11, B4, A4, f11

  614. 	FXCSMADD	f15, B4, A4, f15

  615. 	.align 4

  616. 

  617. .L14:

  618. 	li	r0, ALPHA

  619. 	lfpdx	AP,  SP, r0

  620. #ifdef TRMMKERNEL

  621. 	li	r0, FZERO

  622. 	lfpsx	f30, SP, r0

  623. #endif

  624. 

  625. #if defined(TRMMKERNEL)

  626. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  627. 	sub	TEMP, K, KK

  628. #elif defined(LEFT)

  629. 	addi	TEMP, KK, 4

  630. #else

  631. 	addi	TEMP, KK, 2

  632. #endif

  633. 	andi.	r0,  TEMP,  3

  634. 	mtspr	CTR, r0

  635. 	ble+	.L18

  636. 

  637. 	cmpwi	cr0, TEMP, 3

  638. 	bgt+	.L15

  639. #else

  640. 	andi.	r0,  K,  3

  641. 	mtspr	CTR, r0

  642. 	ble+	.L18

  643. 

  644. 	cmpwi	cr0, K, 3

  645. 	bgt+	.L15

  646. #endif

  647. 

  648. #ifndef TRMMKERNEL

  649. 	LFPDUX	A1, CO1, INC2

  650. 	fpmr	f5,  f0

  651. 	LFPDUX	B1, CO1, INC2

  652. 	fpmr	f9,  f0

  653. 	LFPDUX	A3, CO1, INC2

  654. 	fpmr	f13, f0

  655.    	LFPDUX	A5, CO1, INC2

  656. 	fpmr	f2,  f0

  657. 

  658. 	LFPDUX	B3, CO2, INC2

  659. 	fpmr	f6,  f0

  660. 	LFPDUX	A6, CO2, INC2

  661. 	fpmr	f10, f0

  662. 	LFPDUX	A7, CO2, INC2

  663. 	fpmr	f14, f0

  664. 	LFPDUX	B2, CO2, INC2

  665. 	fpmr	f3,  f0

  666. #else

  667. 	fpmr	f5,  f0

  668. 	fpmr	f9,  f0

  669. 	fpmr	f13, f0

  670. 	fpmr	f2,  f0

  671. 

  672. 	fpmr	f6,  f0

  673. 	fpmr	f10, f0

  674. 	fpmr	f14, f0

  675. 	fpmr	f3,  f0

  676. #endif

  677. 

  678. 	fpmr	f7,  f0

  679. 	fpmr	f11, f0

  680. 	fpmr	f15, f0

  681. 	.align 4

  682. 

  683. .L15:

  684. 	LFPDUX	A2,  AO,  INC4

  685. 	LFPDUX	A4,  AO2, INC4

  686. 	LFPDUX	A10, BO,  INC4

  687. 	LFPDUX	B4,  BO2, INC4

  688. 	bdz-	.L17

  689. 	.align 4

  690. 

  691. .L16:

  692. 	FXCPMADD	f0,  A10, A2, f0

  693. 	FXCSMADD	f4,  A10, A2, f4

  694. 	FXCPMADD	f8,  B4, A2, f8

  695. 	FXCSMADD	f12, B4, A2, f12

  696. 	LFPDUX	A2, AO,  INC4

  697. 

  698. 	FXCPMADD	f1,  A10, A4, f1

  699. 	FXCSMADD	f5,  A10, A4, f5

  700. 	FXCPMADD	f9,  B4, A4, f9

  701. 	FXCSMADD	f13, B4, A4, f13

  702. 	LFPDUX	A4, AO2, INC4

  703. 

  704. 	FXCPMADD	f2,  A10, A2, f2

  705. 	FXCSMADD	f6,  A10, A2, f6

  706. 	FXCPMADD	f10, B4, A2, f10

  707. 	FXCSMADD	f14, B4, A2, f14

  708. 	LFPDUX	A2, AO,  INC4

  709. 

  710. 	FXCPMADD	f3,  A10, A4, f3

  711. 	FXCSMADD	f7,  A10, A4, f7

  712. 	LFPDUX	A10, BO,  INC4

  713. 	FXCPMADD	f11, B4, A4, f11

  714. 	FXCSMADD	f15, B4, A4, f15

  715. 	LFPDUX	A4, AO2, INC4

  716. 	LFPDUX	B4, BO2, INC4

  717. 	bdnz+	.L16

  718. 	.align 4

  719. 

  720. .L17:

  721. 	FXCPMADD	f0,  A10, A2, f0

  722. 	FXCSMADD	f4,  A10, A2, f4

  723. 	FXCPMADD	f8,  B4, A2, f8

  724. 	FXCSMADD	f12, B4, A2, f12

  725. 	LFPDUX	A2, AO,  INC4

  726. 

  727. 	FXCPMADD	f1,  A10, A4, f1

  728. 	FXCSMADD	f5,  A10, A4, f5

  729. 	FXCPMADD	f9,  B4, A4, f9

  730. 	FXCSMADD	f13, B4, A4, f13

  731. 	LFPDUX	A4, AO2, INC4

  732. 

  733. 	FXCPMADD	f2,  A10, A2, f2

  734. 	FXCSMADD	f6,  A10, A2, f6

  735. 	FXCPMADD	f10, B4, A2, f10

  736. 	FXCSMADD	f14, B4, A2, f14

  737. 

  738. 	FXCPMADD	f3,  A10, A4, f3

  739. 	FXCSMADD	f7,  A10, A4, f7

  740. 	FXCPMADD	f11, B4, A4, f11

  741. 	FXCSMADD	f15, B4, A4, f15

  742. 	.align 4

  743. 

  744. .L18:

  745. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  746.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  747. 	fpadd	f0, f0, f4

  748. 	fpadd	f8, f8, f12

  749. 	fpadd	f1, f1, f5

  750. 	fpadd	f9, f9, f13

  751. 

  752. 	fpadd	f2,  f2,  f6

  753. 	fpadd	f10, f10, f14

  754. 	fpadd	f3,  f3,  f7

  755. 	fpadd	f11, f11, f15

  756. #else

  757. 	fpsub	f0, f0, f4

  758. 	fpsub	f8, f8, f12

  759. 	fpsub	f1, f1, f5

  760. 	fpsub	f9, f9, f13

  761. 

  762. 	fpsub	f2,  f2,  f6

  763. 	fpsub	f10, f10, f14

  764. 	fpsub	f3,  f3,  f7

  765. 	fpsub	f11, f11, f15

  766. #endif

  767. 

  768. #ifndef TRMMKERNEL

  769. 	fxcpmadd A1,  f0, AP,  A1

  770. 	fxcpmadd B1,  f1, AP,  B1

  771. 	fxcpmadd A3,  f2, AP,  A3

  772. 	fxcpmadd A5,  f3, AP,  A5

  773. 

  774. 	fxcxnpma f0,  f0, AP,  A1

  775. 	fxcpmadd B3,  f8,  AP,  B3

  776. 	fxcxnpma f1,  f1, AP,  B1

  777. 	fxcpmadd A6,  f9,  AP,  A6

  778. 	fxcxnpma f2,  f2, AP,  A3

  779. 	fxcpmadd A7,  f10, AP,  A7

  780. 

  781. 	fxcxnpma f3,  f3, AP,  A5

  782. 	fxcpmadd B2,  f11, AP,  B2

  783. 	fxcxnpma f8,  f8,  AP,  B3

  784. 	STFPDUX	f0,  CO1, INCM7

  785. 	fxcxnpma f9,  f9,  AP,  A6

  786. 	STFPDUX	f1,  CO1, INC2

  787. 	fxcxnpma f10, f10, AP,  A7

  788. 	STFPDUX	f2,  CO1, INC2

  789. 

  790. 	fxcxnpma f11, f11, AP,  B2

  791. 	STFPDUX	f3,  CO1, INC2

  792. 	STFPDUX	f8,  CO2, INCM7

  793. 	STFPDUX	f9,  CO2, INC2

  794. 	STFPDUX	f10, CO2, INC2

  795. 	STFPDUX	f11, CO2, INC2

  796. #else

  797. 	fxcpmadd f12, f0,  AP,  f30

  798. 	fxcpmadd f13, f1,  AP,  f30

  799. 	fxcpmadd f14, f2,  AP,  f30

  800. 	fxcpmadd f15, f3,  AP,  f30

  801. 

  802. 	fxcxnpma f0,  f0,  AP,  f12

  803. 	fxcxnpma f1,  f1,  AP,  f13

  804. 	fxcxnpma f2,  f2,  AP,  f14

  805. 	fxcxnpma f3,  f3,  AP,  f15

  806. 

  807. 	fxcpmadd f16, f8,  AP,  f30

  808. 	fxcpmadd f17, f9,  AP,  f30

  809. 	fxcpmadd f18, f10, AP,  f30

  810. 	fxcpmadd f19, f11, AP,  f30

  811. 

  812. 	fxcxnpma f8,  f8,  AP,  f16

  813. 	fxcxnpma f9,  f9,  AP,  f17

  814. 	fxcxnpma f10, f10, AP,  f18

  815. 	fxcxnpma f11, f11, AP,  f19

  816. 

  817. 	STFPDUX	f0,  CO1, INC2

  818. 	STFPDUX	f1,  CO1, INC2

  819. 	STFPDUX	f2,  CO1, INC2

  820. 	STFPDUX	f3,  CO1, INC2

  821. 

  822. 	STFPDUX	f8,  CO2, INC2

  823. 	STFPDUX	f9,  CO2, INC2

  824. 	STFPDUX	f10, CO2, INC2

  825. 	STFPDUX	f11, CO2, INC2

  826. 

  827. #endif

  828. 

  829. 

  830. #ifdef TRMMKERNEL

  831. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  832.     (!defined(LEFT) && !defined(TRANSA))

  833. 	sub	TEMP, K, KK

  834. #ifdef LEFT

  835. 	addi	TEMP, TEMP, -4

  836. #else

  837. 	addi	TEMP, TEMP, -2

  838. #endif

  839. 	slwi	r0,   TEMP, 2 + ZBASE_SHIFT

  840. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  841. 	add	AO, AO, r0

  842. 	add	BO, BO, TEMP

  843. #endif

  844. 

  845. #ifdef LEFT

  846. 	addi	KK, KK, 4

  847. #endif

  848. #endif

  849. 

  850. 	addic.	I, I, -1

  851. 	li	r0, FZERO

  852. 

  853. 	lfpsx	f0, SP, r0

  854. 	bgt+	.L11

  855. 	.align 4

  856. 

  857. .L20:

  858. 	andi.	I, M,  2

  859. 	beq	.L30

  860. 

  861. #if defined(TRMMKERNEL)

  862. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  863. 	addi	AO2, AO,   2 * SIZE

  864. 	fpmr	f4,  f0

  865. 	addi	BO,  B,  - 4 * SIZE

  866. 	fpmr	f8,  f0

  867. 	addi	BO2, B,  - 2 * SIZE

  868. 	fpmr	f12, f0

  869. #else

  870. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  871. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  872. 	add	AO, AO, TEMP

  873. 	add	BO, B,  r0

  874. 

  875. 	addi	AO2, AO,   2 * SIZE

  876. 	fpmr	f4,  f0

  877. 	addi	BO,  BO, - 4 * SIZE

  878. 	fpmr	f8,  f0

  879. 	addi	BO2, BO,   2 * SIZE

  880. 	fpmr	f12, f0

  881. 

  882. #endif

  883. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  884. 	sub	TEMP, K, KK

  885. #elif defined(LEFT)

  886. 	addi	TEMP, KK, 2

  887. #else

  888. 	addi	TEMP, KK, 2

  889. #endif

  890. 	srawi.	r0,  TEMP,  2

  891.  	fpmr	f1,  f0

  892. 	fpmr	f5,  f0

  893. 	fpmr	f9,  f0

  894. 	mtspr	CTR, r0

  895. 	fpmr	f13, f0

  896. 	ble	.L24

  897. #else

  898. 	addi	AO2, AO,   2 * SIZE

  899. 	fpmr	f4,  f0

  900. 	addi	BO,  B,  - 4 * SIZE

  901. 	fpmr	f8,  f0

  902. 	addi	BO2, B,  - 2 * SIZE

  903. 	fpmr	f12, f0

  904. 

  905. 	srawi.	r0,  K,  2

  906.  	fpmr	f1,  f0

  907. 	fpmr	f5,  f0

  908. 	fpmr	f9,  f0

  909. 	mtspr	CTR, r0

  910. 	fpmr	f13, f0

  911. 	ble	.L24

  912. #endif

  913. 

  914. 	LFPDUX	A1,   AO, INC4

  915. 	LFPDUX	B1,   BO, INC4

  916. 	LFPDUX	A2,  AO2, INC4

  917. 	LFPDUX	B2,  BO2, INC4

  918. 	LFPDUX	A3,   AO, INC4

  919. 	LFPDUX	B3,   BO, INC4

  920. 	LFPDUX	A4,  AO2, INC4

  921. 	LFPDUX	B4,  BO2, INC4

  922. 

  923. 	LFPDUX	A5,   AO, INC4

  924. 	LFPDUX	B5,   BO, INC4

  925. 	LFPDUX	A6,  AO2, INC4

  926. 	LFPDUX	B6,  BO2, INC4

  927. 	LFPDUX	A7,   AO, INC4

  928. 	LFPDUX	A9,   BO, INC4

  929. 	LFPDUX	A10, BO2, INC4

  930. 	bdz-	.L23

  931. 	.align 4

  932. 

  933. .L22:

  934. 	FXCPMADD	f0,  B1, A1, f0

  935. 	nop

  936. 	FXCSMADD	f4,  B1, A1, f4

  937. 	LFPDUX	A8,  AO2, INC4

  938. 	FXCPMADD	f8,  B2, A1, f8

  939. 	nop

  940. 	FXCSMADD	f12, B2, A1, f12

  941. 	LFPDUX	A1,   AO, INC4

  942. 

  943. 	FXCPMADD	f1,  B1, A2, f1

  944. 	nop

  945. 	FXCSMADD	f5,  B1, A2, f5

  946. 	LFPDUX	B1,   BO, INC4

  947. 	FXCPMADD	f9,  B2, A2, f9

  948. 	nop

  949. 	FXCSMADD	f13, B2, A2, f13

  950. 	LFPDUX	B2,  BO2, INC4

  951. 

  952. 	FXCPMADD	f0,  B3, A3, f0

  953. 	nop

  954. 	FXCSMADD	f4,  B3, A3, f4

  955. 	LFPDUX	A2,  AO2, INC4

  956. 	FXCPMADD	f8,  B4, A3, f8

  957. 	nop

  958. 	FXCSMADD	f12, B4, A3, f12

  959. 	LFPDUX	A3,   AO, INC4

  960. 

  961. 	FXCPMADD	f1,  B3, A4, f1

  962. 	nop

  963. 	FXCSMADD	f5,  B3, A4, f5

  964. 	LFPDUX	B3,   BO, INC4

  965. 	FXCPMADD	f9,  B4, A4, f9

  966. 	nop

  967. 	FXCSMADD	f13, B4, A4, f13

  968. 	LFPDUX	B4,  BO2, INC4

  969. 

  970. 	FXCPMADD	f0,  B5, A5, f0

  971. 	nop

  972. 	FXCSMADD	f4,  B5, A5, f4

  973. 	LFPDUX	A4,  AO2, INC4

  974. 	FXCPMADD	f8,  B6, A5, f8

  975. 	nop

  976. 	FXCSMADD	f12, B6, A5, f12

  977. 	LFPDUX	A5,   AO, INC4

  978. 

  979. 	FXCPMADD	f1,  B5, A6, f1

  980. 	nop

  981. 	FXCSMADD	f5,  B5, A6, f5

  982. 	LFPDUX	B5,   BO, INC4

  983. 	FXCPMADD	f9,  B6, A6, f9

  984. 	nop

  985. 	FXCSMADD	f13, B6, A6, f13

  986. 	LFPDUX	B6,  BO2, INC4

  987. 

  988. 	FXCPMADD	f0,  A9,  A7, f0

  989. 	nop

  990. 	FXCSMADD	f4,  A9,  A7, f4

  991. 	LFPDUX	A6,  AO2, INC4

  992. 	FXCPMADD	f8,  A10, A7, f8

  993. 	nop

  994. 	FXCSMADD	f12, A10, A7, f12

  995. 	LFPDUX	A7,   AO, INC4

  996. 

  997. 	FXCPMADD	f1,  A9,  A8, f1

  998. 	nop

  999. 	FXCSMADD	f5,  A9,  A8, f5

  1000. 	LFPDUX	A9,   BO, INC4

  1001. 	FXCPMADD	f9,  A10, A8, f9

  1002. 	nop

  1003. 	FXCSMADD	f13, A10, A8, f13

  1004. 	LFPDUX	A10, BO2, INC4

  1005. 	bdnz+	.L22

  1006. 	.align 4

  1007. 

  1008. .L23:

  1009. 	FXCPMADD	f0,  B1, A1, f0

  1010. 	FXCSMADD	f4,  B1, A1, f4

  1011. 	LFPDUX	A8,  AO2, INC4

  1012. 	FXCPMADD	f8,  B2, A1, f8

  1013. 	FXCSMADD	f12, B2, A1, f12

  1014. 

  1015. 	FXCPMADD	f1,  B1, A2, f1

  1016. 	FXCSMADD	f5,  B1, A2, f5

  1017. 	FXCPMADD	f9,  B2, A2, f9

  1018. 	FXCSMADD	f13, B2, A2, f13

  1019. 

  1020. 	FXCPMADD	f0,  B3, A3, f0

  1021. 	FXCSMADD	f4,  B3, A3, f4

  1022. 	FXCPMADD	f8,  B4, A3, f8

  1023. 	FXCSMADD	f12, B4, A3, f12

  1024. 

  1025. 	FXCPMADD	f1,  B3, A4, f1

  1026. 	FXCSMADD	f5,  B3, A4, f5

  1027. 	FXCPMADD	f9,  B4, A4, f9

  1028. 	FXCSMADD	f13, B4, A4, f13

  1029. 

  1030. 	FXCPMADD	f0,  B5, A5, f0

  1031. 	FXCSMADD	f4,  B5, A5, f4

  1032. 	FXCPMADD	f8,  B6, A5, f8

  1033. 	FXCSMADD	f12, B6, A5, f12

  1034. 

  1035. 	FXCPMADD	f1,  B5, A6, f1

  1036. 	FXCSMADD	f5,  B5, A6, f5

  1037. 	FXCPMADD	f9,  B6, A6, f9

  1038. 	FXCSMADD	f13, B6, A6, f13

  1039. 

  1040. 	FXCPMADD	f0,  A9, A7, f0

  1041. 	FXCSMADD	f4,  A9, A7, f4

  1042. 	FXCPMADD	f8,  A10, A7, f8

  1043. 	FXCSMADD	f12, A10, A7, f12

  1044. 

  1045. 	FXCPMADD	f1,  A9, A8, f1

  1046. 	FXCSMADD	f5,  A9, A8, f5

  1047. 	FXCPMADD	f9,  A10, A8, f9

  1048. 	FXCSMADD	f13, A10, A8, f13

  1049. 	.align 4

  1050. 

  1051. .L24:

  1052. 	li	r0, ALPHA

  1053. 	lfpdx	AP,  SP, r0

  1054. #ifdef TRMMKERNEL

  1055. 	li	r0, FZERO

  1056. 	lfpsx	f30, SP, r0

  1057. #endif

  1058. 

  1059. #if defined(TRMMKERNEL)

  1060. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1061. 	sub	TEMP, K, KK

  1062. #elif defined(LEFT)

  1063. 	addi	TEMP, KK, 2

  1064. #else

  1065. 	addi	TEMP, KK, 2

  1066. #endif

  1067. 	andi.	r0,  TEMP,  3

  1068. 	mtspr	CTR, r0

  1069. #else

  1070. 	andi.	r0,  K,  3

  1071. 	mtspr	CTR, r0

  1072. #endif

  1073. 	ble+	.L28

  1074. 

  1075. 	LFPDUX	A1,  AO,  INC4

  1076. 	LFPDUX	A2,  AO2, INC4

  1077. 	LFPDUX	B1,  BO,  INC4

  1078. 	LFPDUX	B2,  BO2, INC4

  1079. 	bdz-	.L27

  1080. 	.align 4

  1081. 

  1082. .L26:

  1083. 	FXCPMADD	f0,  B1, A1, f0

  1084. 	FXCSMADD	f4,  B1, A1, f4

  1085. 	FXCPMADD	f8,  B2, A1, f8

  1086. 	FXCSMADD	f12, B2, A1, f12

  1087. 	LFPDUX	A1,  AO,  INC4

  1088. 

  1089. 	FXCPMADD	f1,  B1, A2, f1

  1090. 	FXCSMADD	f5,  B1, A2, f5

  1091. 	LFPDUX	B1,  BO,  INC4

  1092. 	FXCPMADD	f9,  B2, A2, f9

  1093. 	FXCSMADD	f13, B2, A2, f13

  1094. 	LFPDUX	A2,  AO2, INC4

  1095. 	LFPDUX	B2,  BO2, INC4

  1096. 	bdnz+	.L26

  1097. 	.align 4

  1098. 

  1099. .L27:

  1100. 	FXCPMADD	f0,  B1, A1, f0

  1101. 	FXCSMADD	f4,  B1, A1, f4

  1102. 	FXCPMADD	f8,  B2, A1, f8

  1103. 	FXCSMADD	f12, B2, A1, f12

  1104. 

  1105. 	FXCPMADD	f1,  B1, A2, f1

  1106. 	FXCSMADD	f5,  B1, A2, f5

  1107. 	FXCPMADD	f9,  B2, A2, f9

  1108. 	FXCSMADD	f13, B2, A2, f13

  1109. 	.align 4

  1110. 

  1111. .L28:

  1112. #ifndef TRMMKERNEL

  1113. 	LFPDUX	A1, CO1, INC2

  1114. 	LFPDUX	A2, CO1, INC2

  1115. 	LFPDUX	A3, CO2, INC2

  1116. 	LFPDUX	A4, CO2, INC2

  1117. #endif

  1118. 

  1119. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1120.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  1121. 	fpadd	f0, f0, f4

  1122. 	fpadd	f8, f8, f12

  1123. 	fpadd	f1, f1, f5

  1124. 	fpadd	f9, f9, f13

  1125. #else

  1126. 	fpsub	f0, f0, f4

  1127. 	fpsub	f8, f8, f12

  1128. 	fpsub	f1, f1, f5

  1129. 	fpsub	f9, f9, f13

  1130. #endif

  1131. 

  1132. #ifndef TRMMKERNEL

  1133. 	fxcpmadd A1,  f0, AP,  A1

  1134. 	fxcpmadd A2,  f1, AP,  A2

  1135. 	fxcpmadd A3,  f8, AP,  A3

  1136. 	fxcpmadd A4,  f9, AP,  A4

  1137. 

  1138. 	fxcxnpma f0,  f0, AP,  A1

  1139. 	fxcxnpma f1,  f1, AP,  A2

  1140. 	fxcxnpma f8,  f8, AP,  A3

  1141. 	fxcxnpma f9,  f9, AP,  A4

  1142. 

  1143. 	STFPDUX	f0,  CO1, INCM3

  1144. 	STFPDUX	f1,  CO1, INC2

  1145. 

  1146. 	STFPDUX	f8,  CO2, INCM3

  1147. 	STFPDUX	f9,  CO2, INC2

  1148. #else

  1149. 	fxcpmadd f12,  f0, AP,  f30

  1150. 	fxcpmadd f13,  f1, AP,  f30

  1151. 	fxcpmadd f14,  f8, AP,  f30

  1152. 	fxcpmadd f15,  f9, AP,  f30

  1153. 

  1154. 	fxcxnpma f0,  f0, AP,  f12

  1155. 	fxcxnpma f1,  f1, AP,  f13

  1156. 	fxcxnpma f8,  f8, AP,  f14

  1157. 	fxcxnpma f9,  f9, AP,  f15

  1158. 

  1159. 	STFPDUX	f0,  CO1, INC2

  1160. 	STFPDUX	f1,  CO1, INC2

  1161. 

  1162. 	STFPDUX	f8,  CO2, INC2

  1163. 	STFPDUX	f9,  CO2, INC2

  1164. #endif

  1165. 

  1166. 

  1167. #ifdef TRMMKERNEL

  1168. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  1169.     (!defined(LEFT) && !defined(TRANSA))

  1170. 	sub	TEMP, K, KK

  1171. #ifdef LEFT

  1172. 	addi	TEMP, TEMP, -2

  1173. #else

  1174. 	addi	TEMP, TEMP, -2

  1175. #endif

  1176. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  1177. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  1178. 	add	AO, AO, r0

  1179. 	add	BO, BO, TEMP

  1180. #endif

  1181. 

  1182. #ifdef LEFT

  1183. 	addi	KK, KK, 2

  1184. #endif

  1185. #endif

  1186. 

  1187. 	li	r0, FZERO

  1188. 	lfpsx	f0, SP, r0

  1189. 	.align 4

  1190. 

  1191. .L30:

  1192. 	andi.	I, M,  1

  1193. 	beq	.L49

  1194. 

  1195. #if defined(TRMMKERNEL)

  1196. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1197. 	addi	AO2, AO,   2 * SIZE

  1198. 	fpmr	f1,  f0

  1199. 	addi	BO,  B,  - 4 * SIZE

  1200. 	fpmr	f2,  f0

  1201. 	addi	BO2, B,  - 2 * SIZE

  1202. 	fpmr	f3, f0

  1203. #else

  1204. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  1205. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  1206. 	add	AO, AO, TEMP

  1207. 	add	BO, B,  r0

  1208. 

  1209. 	addi	AO2, AO,   2 * SIZE

  1210. 	fpmr	f1,  f0

  1211. 	addi	BO,  BO,  - 4 * SIZE

  1212. 	fpmr	f2,  f0

  1213. 	addi	BO2, BO,    2 * SIZE

  1214. 	fpmr	f3, f0

  1215. #endif

  1216. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1217. 	sub	TEMP, K, KK

  1218. #elif defined(LEFT)

  1219. 	addi	TEMP, KK, 1

  1220. #else

  1221. 	addi	TEMP, KK, 2

  1222. #endif

  1223. 	srawi.	r0,  TEMP,  2

  1224. 	mtspr	CTR, r0

  1225. 	ble	.L34

  1226. #else

  1227. 	addi	AO2, AO,   2 * SIZE

  1228. 	fpmr	f1,  f0

  1229. 	addi	BO,  B,  - 4 * SIZE

  1230. 	fpmr	f2,  f0

  1231. 	addi	BO2, B,  - 2 * SIZE

  1232. 	fpmr	f3, f0

  1233. 

  1234. 	srawi.	r0,  K,  2

  1235. 	mtspr	CTR, r0

  1236. 	ble	.L34

  1237. #endif

  1238. 

  1239. 	LFPDUX	A1,  AO, INC4

  1240. 	LFPDUX	B1,  BO, INC4

  1241. 	LFPDUX	B2, BO2, INC4

  1242. 	LFPDUX	A2, AO2, INC4

  1243. 	LFPDUX	B3,  BO, INC4

  1244. 	LFPDUX	B4, BO2, INC4

  1245. 

  1246. 	LFPDUX	A3,  AO, INC4

  1247. 	LFPDUX	A5,  BO, INC4

  1248. 	LFPDUX	A6, BO2, INC4

  1249. 	LFPDUX	A4, AO2, INC4

  1250. 	LFPDUX	A7,  BO, INC4

  1251. 	LFPDUX	A8, BO2, INC4

  1252. 	bdz-	.L33

  1253. 	.align 4

  1254. 

  1255. .L32:

  1256. 	FXCPMADD	f0,  B1, A1, f0

  1257. 	FXCSMADD	f1,  B1, A1, f1

  1258. 	LFPDUX	B1,  BO, INC4

  1259. 	FXCPMADD	f2,  B2, A1, f2

  1260. 	FXCSMADD	f3,  B2, A1, f3

  1261. 	LFPDUX	B2, BO2, INC4

  1262. 	LFPDUX	A1,  AO, INC4

  1263. 

  1264. 	FXCPMADD	f0,  B3, A2, f0

  1265. 	FXCSMADD	f1,  B3, A2, f1

  1266. 	LFPDUX	B3,  BO, INC4

  1267. 	FXCPMADD	f2,  B4, A2, f2

  1268. 	FXCSMADD	f3,  B4, A2, f3

  1269. 	LFPDUX	B4, BO2, INC4

  1270. 	LFPDUX	A2, AO2, INC4

  1271. 

  1272. 	FXCPMADD	f0,  A5, A3, f0

  1273. 	FXCSMADD	f1,  A5, A3, f1

  1274. 	LFPDUX	A5,  BO, INC4

  1275. 	FXCPMADD	f2,  A6, A3, f2

  1276. 	FXCSMADD	f3,  A6, A3, f3

  1277. 	LFPDUX	A6, BO2, INC4

  1278. 	LFPDUX	A3,  AO, INC4

  1279. 

  1280. 	FXCPMADD	f0,  A7, A4, f0

  1281. 	FXCSMADD	f1,  A7, A4, f1

  1282. 	LFPDUX	A7,  BO, INC4

  1283. 	FXCPMADD	f2,  A8, A4, f2

  1284. 	FXCSMADD	f3,  A8, A4, f3

  1285. 	LFPDUX	A8, BO2, INC4

  1286. 	LFPDUX	A4, AO2, INC4

  1287. 	bdnz+	.L32

  1288. 	.align 4

  1289. 

  1290. .L33:

  1291. 	FXCPMADD	f0,  B1, A1, f0

  1292. 	FXCSMADD	f1,  B1, A1, f1

  1293. 	FXCPMADD	f2,  B2, A1, f2

  1294. 	FXCSMADD	f3,  B2, A1, f3

  1295. 

  1296. 	FXCPMADD	f0,  B3, A2, f0

  1297. 	FXCSMADD	f1,  B3, A2, f1

  1298. 	FXCPMADD	f2,  B4, A2, f2

  1299. 	FXCSMADD	f3,  B4, A2, f3

  1300. 

  1301. 	FXCPMADD	f0,  A5, A3, f0

  1302. 	FXCSMADD	f1,  A5, A3, f1

  1303. 	FXCPMADD	f2,  A6, A3, f2

  1304. 	FXCSMADD	f3,  A6, A3, f3

  1305. 

  1306. 	FXCPMADD	f0,  A7, A4, f0

  1307. 	FXCSMADD	f1,  A7, A4, f1

  1308. 	FXCPMADD	f2,  A8, A4, f2

  1309. 	FXCSMADD	f3,  A8, A4, f3

  1310. 	.align 4

  1311. 

  1312. .L34:

  1313. 	li	r0, ALPHA

  1314. 	lfpdx	AP,  SP, r0

  1315. #ifdef TRMMKERNEL

  1316. 	li	r0, FZERO

  1317. 	lfpsx	f30, SP, r0

  1318. #endif

  1319. 

  1320. #if defined(TRMMKERNEL)

  1321. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1322. 	sub	TEMP, K, KK

  1323. #elif defined(LEFT)

  1324. 	addi	TEMP, KK, 1

  1325. #else

  1326. 	addi	TEMP, KK, 2

  1327. #endif

  1328. 	andi.	r0,  TEMP,  3

  1329. 	mtspr	CTR, r0

  1330. #else

  1331. 	andi.	r0,  K,  3

  1332. 	mtspr	CTR, r0

  1333. #endif

  1334. 	ble+	.L38

  1335. 

  1336. 	LFPDX	A1,  AO,  INC4

  1337. 	LFPDUX	B1,  BO,  INC4

  1338. 	LFPDUX	B2,  BO2, INC4

  1339. 	add	AO, AO, INC2

  1340. 	bdz-	.L37

  1341. 	.align 4

  1342. 

  1343. .L36:

  1344. 	FXCPMADD	f0,  B1, A1, f0

  1345. 	FXCSMADD	f1,  B1, A1, f1

  1346. 	LFPDUX	B1,  BO,  INC4

  1347. 	FXCPMADD	f2,  B2, A1, f2

  1348. 	FXCSMADD	f3,  B2, A1, f3

  1349. 	LFPDX	A1,  AO,  INC4

  1350. 	LFPDUX	B2,  BO2, INC4

  1351. 	add	AO, AO, INC2

  1352. 	bdnz+	.L36

  1353. 	.align 4

  1354. 

  1355. .L37:

  1356. 	FXCPMADD	f0,  B1, A1, f0

  1357. 	FXCSMADD	f1,  B1, A1, f1

  1358. 	FXCPMADD	f2,  B2, A1, f2

  1359. 	FXCSMADD	f3,  B2, A1, f3

  1360. 	.align 4

  1361. 

  1362. .L38:

  1363. #ifndef TRMMKERNEL

  1364. 	LFPDX	A1, CO1, INC2

  1365. 	LFPDX	A2, CO2, INC2

  1366. #endif

  1367. 

  1368. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1369.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  1370. 	fpadd	f0, f0, f1

  1371. 	fpadd	f2, f2, f3

  1372. #else

  1373. 	fpsub	f0, f0, f1

  1374. 	fpsub	f2, f2, f3

  1375. #endif

  1376. 

  1377. #ifndef TRMMKERNEL

  1378. 	fxcpmadd A1,  f0, AP,  A1

  1379. 	fxcpmadd A2,  f2, AP,  A2

  1380. 	fxcxnpma f0,  f0, AP,  A1

  1381. 	fxcxnpma f2,  f2, AP,  A2

  1382. #else

  1383. 	fxcpmadd f12, f0, AP,  f30

  1384. 	fxcpmadd f13, f2, AP,  f30

  1385. 	fxcxnpma f0,  f0, AP,  f12

  1386. 	fxcxnpma f2,  f2, AP,  f13

  1387. #endif

  1388. 

  1389. 	STFPDUX	f0,  CO1, INC2

  1390. 	STFPDUX	f2,  CO2, INC2

  1391. 

  1392. #ifdef TRMMKERNEL

  1393. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  1394.     (!defined(LEFT) && !defined(TRANSA))

  1395. 	sub	TEMP, K, KK

  1396. #ifdef LEFT

  1397. 	addi	TEMP, TEMP, -1

  1398. #else

  1399. 	addi	TEMP, TEMP, -2

  1400. #endif

  1401. 	slwi	r0,   TEMP, 0 + ZBASE_SHIFT

  1402. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  1403. 	add	AO, AO, r0

  1404. 	add	BO, BO, TEMP

  1405. #endif

  1406. 

  1407. #ifdef LEFT

  1408. 	addi	KK, KK, 1

  1409. #endif

  1410. #endif

  1411. 

  1412. 	li	r0, FZERO

  1413. 	lfpsx	f0, SP, r0

  1414. 	.align 4

  1415. 

  1416. .L49:

  1417. #if defined(TRMMKERNEL) && !defined(LEFT)

  1418. 	addi	KK, KK, 2

  1419. #endif

  1420. 

  1421. 	addi	B,  BO, 4 * SIZE

  1422. 

  1423. 	addic.	J, J, -1

  1424. 	bgt+	.L10

  1425. 	.align 4

  1426. 

  1427. .L50:

  1428. 	andi.	J, N,  1

  1429. 	beq	.L999

  1430. 

  1431. 	mr	CO1, C

  1432. 

  1433. #if defined(TRMMKERNEL) &&  defined(LEFT)

  1434. 	mr	KK, OFFSET

  1435. #endif

  1436. 

  1437. 	addi	AO, A, -2 * SIZE

  1438. 

  1439. 	li	r0, FZERO

  1440. 	lfpsx	f0, SP, r0

  1441. 

  1442. 	srawi.	I, M,  2

  1443. 	ble	.L60

  1444. 	.align 4

  1445. 

  1446. .L51:

  1447. #if defined(TRMMKERNEL)

  1448. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1449. 	fpmr	f4,  f0

  1450. 	addi	BO,  B,  - 2 * SIZE

  1451.  	fpmr	f1,  f0

  1452. 	fpmr	f5,  f0

  1453. 	fpmr	f2,  f0

  1454. 	fpmr	f6,  f0

  1455. #else

  1456. 	slwi	TEMP, KK, 2 + ZBASE_SHIFT

  1457. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  1458. 	add	AO, AO, TEMP

  1459. 	add	BO, B,  r0

  1460. 

  1461. 	fpmr	f4,  f0

  1462. 	addi	BO,  BO,  - 2 * SIZE

  1463.  	fpmr	f1,  f0

  1464. 	fpmr	f5,  f0

  1465. 	fpmr	f2,  f0

  1466. 	fpmr	f6,  f0

  1467. #endif

  1468. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1469. 	sub	TEMP, K, KK

  1470. #elif defined(LEFT)

  1471. 	addi	TEMP, KK, 4

  1472. #else

  1473. 	addi	TEMP, KK, 1

  1474. #endif

  1475. 	srawi.	r0,  TEMP,  2

  1476. 	fpmr	f3,  f0

  1477. 	mtspr	CTR, r0

  1478. 	fpmr	f7,  f0

  1479. 	ble	.L54

  1480. #else

  1481. 	srawi.	r0,  K,  2

  1482. 	fpmr	f4,  f0

  1483. 	addi	BO,  B,  - 2 * SIZE

  1484.  	fpmr	f1,  f0

  1485. 	fpmr	f5,  f0

  1486. 	fpmr	f2,  f0

  1487. 	fpmr	f6,  f0

  1488. 	fpmr	f3,  f0

  1489. 	mtspr	CTR, r0

  1490. 	fpmr	f7,  f0

  1491. 	ble	.L54

  1492. #endif

  1493. 

  1494. 	LFPDUX	B1,  BO,  INC2

  1495. 	LFPDUX	A1,  AO,  INC2

  1496. 	LFPDUX	A2,  AO,  INC2

  1497. 	LFPDUX	B2,  BO,  INC2

  1498. 	LFPDUX	A3,  AO,  INC2

  1499. 	LFPDUX	A4,  AO,  INC2

  1500. 

  1501. 	LFPDUX	B3,  BO,  INC2

  1502. 	LFPDUX	A5,  AO,  INC2

  1503. 	LFPDUX	A6,  AO,  INC2

  1504. 	LFPDUX	A7,  AO,  INC2

  1505. 	LFPDUX	A8,  AO,  INC2

  1506. 	bdz-	.L53

  1507. 	.align 4

  1508. 

  1509. .L52:

  1510. 	FXCPMADD	f0,  B1, A1, f0

  1511. 	LFPDUX	B4,  BO,  INC2

  1512. 	FXCSMADD	f4,  B1, A1, f4

  1513. 	LFPDUX	A1,  AO,  INC2

  1514. 	FXCPMADD	f1,  B1, A2, f1

  1515. 	nop

  1516. 	FXCSMADD	f5,  B1, A2, f5

  1517. 	LFPDUX	A2,  AO,  INC2

  1518. 

  1519. 	FXCPMADD	f2,  B1, A3, f2

  1520. 	nop

  1521. 	FXCSMADD	f6,  B1, A3, f6

  1522. 	LFPDUX	A3,  AO,  INC2

  1523. 	FXCPMADD	f3,  B1, A4, f3

  1524. 	nop

  1525. 	FXCSMADD	f7,  B1, A4, f7

  1526. 	LFPDUX	A4,  AO,  INC2

  1527. 

  1528. 	FXCPMADD	f0,  B2, A5, f0

  1529. 	LFPDUX	B1,  BO,  INC2

  1530. 	FXCSMADD	f4,  B2, A5, f4

  1531. 	LFPDUX	A5,  AO,  INC2

  1532. 	FXCPMADD	f1,  B2, A6, f1

  1533. 	nop

  1534. 	FXCSMADD	f5,  B2, A6, f5

  1535. 	LFPDUX	A6,  AO,  INC2

  1536. 

  1537. 	FXCPMADD	f2,  B2, A7, f2

  1538. 	nop

  1539. 	FXCSMADD	f6,  B2, A7, f6

  1540. 	LFPDUX	A7,  AO,  INC2

  1541. 	FXCPMADD	f3,  B2, A8, f3

  1542. 	nop

  1543. 	FXCSMADD	f7,  B2, A8, f7

  1544. 	LFPDUX	A8,  AO,  INC2

  1545. 

  1546. 	FXCPMADD	f0,  B3, A1, f0

  1547. 	LFPDUX	B2,  BO,  INC2

  1548. 	FXCSMADD	f4,  B3, A1, f4

  1549. 	LFPDUX	A1,  AO,  INC2

  1550. 	FXCPMADD	f1,  B3, A2, f1

  1551. 	nop

  1552. 	FXCSMADD	f5,  B3, A2, f5

  1553. 	LFPDUX	A2,  AO,  INC2

  1554. 

  1555. 	FXCPMADD	f2,  B3, A3, f2

  1556. 	nop

  1557. 	FXCSMADD	f6,  B3, A3, f6

  1558. 	LFPDUX	A3,  AO,  INC2

  1559. 	FXCPMADD	f3,  B3, A4, f3

  1560. 	nop

  1561. 	FXCSMADD	f7,  B3, A4, f7

  1562. 	LFPDUX	A4,  AO,  INC2

  1563. 

  1564. 	FXCPMADD	f0,  B4, A5, f0

  1565. 	LFPDUX	B3,  BO,  INC2

  1566. 	FXCSMADD	f4,  B4, A5, f4

  1567. 	LFPDUX	A5,  AO,  INC2

  1568. 	FXCPMADD	f1,  B4, A6, f1

  1569. 	nop

  1570. 	FXCSMADD	f5,  B4, A6, f5

  1571. 	LFPDUX	A6,  AO,  INC2

  1572. 

  1573. 	FXCPMADD	f2,  B4, A7, f2

  1574. 	nop

  1575. 	FXCSMADD	f6,  B4, A7, f6

  1576. 	LFPDUX	A7,  AO,  INC2

  1577. 	FXCPMADD	f3,  B4, A8, f3

  1578. 	nop

  1579. 	FXCSMADD	f7,  B4, A8, f7

  1580. 	LFPDUX	A8,  AO,  INC2

  1581. 	bdnz+	.L52

  1582. 	.align 4

  1583. 

  1584. .L53:

  1585. 	FXCPMADD	f0,  B1, A1, f0

  1586. 	LFPDUX	B4,  BO,  INC2

  1587. 	FXCSMADD	f4,  B1, A1, f4

  1588. 	LFPDUX	A1,  AO,  INC2

  1589. 	FXCPMADD	f1,  B1, A2, f1

  1590. 	nop

  1591. 	FXCSMADD	f5,  B1, A2, f5

  1592. 	LFPDUX	A2,  AO,  INC2

  1593. 

  1594. 	FXCPMADD	f2,  B1, A3, f2

  1595. 	nop

  1596. 	FXCSMADD	f6,  B1, A3, f6

  1597. 	LFPDUX	A3,  AO,  INC2

  1598. 	FXCPMADD	f3,  B1, A4, f3

  1599. 	nop

  1600. 	FXCSMADD	f7,  B1, A4, f7

  1601. 	LFPDUX	A4,  AO,  INC2

  1602. 

  1603. 	FXCPMADD	f0,  B2, A5, f0

  1604. 	nop

  1605. 	FXCSMADD	f4,  B2, A5, f4

  1606. 	LFPDUX	A5,  AO,  INC2

  1607. 	FXCPMADD	f1,  B2, A6, f1

  1608. 	nop

  1609. 	FXCSMADD	f5,  B2, A6, f5

  1610. 	LFPDUX	A6,  AO,  INC2

  1611. 

  1612. 	FXCPMADD	f2,  B2, A7, f2

  1613. 	nop

  1614. 	FXCSMADD	f6,  B2, A7, f6

  1615. 	LFPDUX	A7,  AO,  INC2

  1616. 	FXCPMADD	f3,  B2, A8, f3

  1617. 	nop

  1618. 	FXCSMADD	f7,  B2, A8, f7

  1619. 	LFPDUX	A8,  AO,  INC2

  1620. 

  1621. 	FXCPMADD	f0,  B3, A1, f0

  1622. 	FXCSMADD	f4,  B3, A1, f4

  1623. 	FXCPMADD	f1,  B3, A2, f1

  1624. 	FXCSMADD	f5,  B3, A2, f5

  1625. 

  1626. 	FXCPMADD	f2,  B3, A3, f2

  1627. 	FXCSMADD	f6,  B3, A3, f6

  1628. 	FXCPMADD	f3,  B3, A4, f3

  1629. 	FXCSMADD	f7,  B3, A4, f7

  1630. 

  1631. 	FXCPMADD	f0,  B4, A5, f0

  1632. 	FXCSMADD	f4,  B4, A5, f4

  1633. 	FXCPMADD	f1,  B4, A6, f1

  1634. 	FXCSMADD	f5,  B4, A6, f5

  1635. 

  1636. 	FXCPMADD	f2,  B4, A7, f2

  1637. 	FXCSMADD	f6,  B4, A7, f6

  1638. 	FXCPMADD	f3,  B4, A8, f3

  1639. 	FXCSMADD	f7,  B4, A8, f7

  1640. 	.align 4

  1641. 

  1642. .L54:

  1643. 	li	r0, ALPHA

  1644. 	lfpdx	AP,  SP, r0

  1645. #ifdef TRMMKERNEL

  1646. 	li	r0, FZERO

  1647. 	lfpsx	f30, SP, r0

  1648. #endif

  1649. 

  1650. #if defined(TRMMKERNEL)

  1651. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1652. 	sub	TEMP, K, KK

  1653. #elif defined(LEFT)

  1654. 	addi	TEMP, KK, 4

  1655. #else

  1656. 	addi	TEMP, KK, 1

  1657. #endif

  1658. 	andi.	r0,  TEMP,  3

  1659. 	mtspr	CTR, r0

  1660. #else

  1661. 	andi.	r0,  K,  3

  1662. 	mtspr	CTR, r0

  1663. #endif

  1664. 	ble+	.L58

  1665. 

  1666. 	LFPDUX	A1,  AO,  INC2

  1667. 	LFPDUX	B1,  BO,  INC2

  1668. 	LFPDUX	A2,  AO,  INC2

  1669. 	LFPDUX	A3,  AO,  INC2

  1670. 	LFPDUX	A4,  AO,  INC2

  1671. 	bdz-	.L57

  1672. 	.align 4

  1673. 

  1674. .L56:

  1675. 	FXCPMADD	f0,  B1, A1, f0

  1676. 	FXCSMADD	f4,  B1, A1, f4

  1677. 	LFPDUX	A1,  AO,  INC2

  1678. 	FXCPMADD	f1,  B1, A2, f1

  1679. 	FXCSMADD	f5,  B1, A2, f5

  1680. 	LFPDUX	A2,  AO,  INC2

  1681. 

  1682. 	FXCPMADD	f2,  B1, A3, f2

  1683. 	FXCSMADD	f6,  B1, A3, f6

  1684. 	LFPDUX	A3,  AO,  INC2

  1685. 	FXCPMADD	f3,  B1, A4, f3

  1686. 	FXCSMADD	f7,  B1, A4, f7

  1687. 	LFPDUX	A4,  AO,  INC2

  1688. 	LFPDUX	B1,  BO,  INC2

  1689. 	bdnz+	.L56

  1690. 	.align 4

  1691. 

  1692. .L57:

  1693. 	FXCPMADD	f0,  B1, A1, f0

  1694. 	FXCSMADD	f4,  B1, A1, f4

  1695. 	FXCPMADD	f1,  B1, A2, f1

  1696. 	FXCSMADD	f5,  B1, A2, f5

  1697. 

  1698. 	FXCPMADD	f2,  B1, A3, f2

  1699. 	FXCSMADD	f6,  B1, A3, f6

  1700. 	FXCPMADD	f3,  B1, A4, f3

  1701. 	FXCSMADD	f7,  B1, A4, f7

  1702. 	.align 4

  1703. 

  1704. .L58:

  1705. #ifndef TRMMKERNEL

  1706. 	LFPDUX	A1, CO1, INC2

  1707. 	LFPDUX	A2, CO1, INC2

  1708. 	LFPDUX	A3, CO1, INC2

  1709.    	LFPDUX	A4, CO1, INC2

  1710. #endif

  1711. 

  1712. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1713.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  1714. 	fpadd	f0, f0, f4

  1715. 	fpadd	f1, f1, f5

  1716. 	fpadd	f2, f2, f6

  1717. 	fpadd	f3, f3, f7

  1718. #else

  1719. 	fpsub	f0, f0, f4

  1720. 	fpsub	f1, f1, f5

  1721. 	fpsub	f2, f2, f6

  1722. 	fpsub	f3, f3, f7

  1723. #endif

  1724. 

  1725. #ifndef TRMMKERNEL

  1726. 	fxcpmadd A1,  f0, AP,  A1

  1727. 	fxcpmadd A2,  f1, AP,  A2

  1728. 	fxcpmadd A3,  f2, AP,  A3

  1729. 	fxcpmadd A4,  f3, AP,  A4

  1730. 

  1731. 	fxcxnpma f0,  f0, AP,  A1

  1732. 	fxcxnpma f1,  f1, AP,  A2

  1733. 	fxcxnpma f2,  f2, AP,  A3

  1734. 	fxcxnpma f3,  f3, AP,  A4

  1735. 

  1736. 	STFPDUX	f0,  CO1, INCM7

  1737. 	STFPDUX	f1,  CO1, INC2

  1738. 	STFPDUX	f2,  CO1, INC2

  1739. 	STFPDUX	f3,  CO1, INC2

  1740. #else

  1741. 	fxcpmadd f12, f0, AP,  f30

  1742. 	fxcpmadd f13, f1, AP,  f30

  1743. 	fxcpmadd f14, f2, AP,  f30

  1744. 	fxcpmadd f15, f3, AP,  f30

  1745. 

  1746. 	fxcxnpma f0,  f0, AP,  f12

  1747. 	fxcxnpma f1,  f1, AP,  f13

  1748. 	fxcxnpma f2,  f2, AP,  f14

  1749. 	fxcxnpma f3,  f3, AP,  f15

  1750. 

  1751. 	STFPDUX	f0,  CO1, INC2

  1752. 	STFPDUX	f1,  CO1, INC2

  1753. 	STFPDUX	f2,  CO1, INC2

  1754. 	STFPDUX	f3,  CO1, INC2

  1755. #endif

  1756. 

  1757. 

  1758. #ifdef TRMMKERNEL

  1759. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  1760.     (!defined(LEFT) && !defined(TRANSA))

  1761. 	sub	TEMP, K, KK

  1762. #ifdef LEFT

  1763. 	addi	TEMP, TEMP, -4

  1764. #else

  1765. 	addi	TEMP, TEMP, -1

  1766. #endif

  1767. 	slwi	r0,   TEMP, 2 + ZBASE_SHIFT

  1768. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  1769. 	add	AO, AO, r0

  1770. 	add	BO, BO, TEMP

  1771. #endif

  1772. 

  1773. #ifdef LEFT

  1774. 	addi	KK, KK, 4

  1775. #endif

  1776. #endif

  1777. 

  1778. 	addic.	I, I, -1

  1779. 	li	r0, FZERO

  1780. 

  1781. 	lfpsx	f0, SP, r0

  1782. 	bgt+	.L51

  1783. 	.align 4

  1784. 

  1785. .L60:

  1786. 	andi.	I, M,  2

  1787. 	beq	.L70

  1788. 

  1789. #if defined(TRMMKERNEL)

  1790. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  1791. 	addi	BO,  B,  - 2 * SIZE

  1792.  	fpmr	f1,  f0

  1793. #else

  1794. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  1795. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  1796. 	add	AO, AO, TEMP

  1797. 	add	BO, B,  r0

  1798. 

  1799. 	addi	BO,  BO,  - 2 * SIZE

  1800.  	fpmr	f1,  f0

  1801. #endif

  1802. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1803. 	sub	TEMP, K, KK

  1804. #elif defined(LEFT)

  1805. 	addi	TEMP, KK, 2

  1806. #else

  1807. 	addi	TEMP, KK, 1

  1808. #endif

  1809. 

  1810. 	srawi.	r0,  TEMP,  2

  1811. 	fpmr	f2,  f0

  1812. 	mtspr	CTR, r0

  1813. 	fpmr	f3,  f0

  1814. 	ble	.L64

  1815. 

  1816. #else

  1817. 	srawi.	r0,  K,  2

  1818.  	fpmr	f1,  f0

  1819. 	addi	BO,  B,  - 2 * SIZE

  1820. 	fpmr	f2,  f0

  1821. 	mtspr	CTR, r0

  1822. 	fpmr	f3,  f0

  1823. 	ble	.L64

  1824. #endif

  1825. 

  1826. 	LFPDUX	B1,  BO, INC2

  1827. 	LFPDUX	A1,  AO, INC2

  1828. 	LFPDUX	A2,  AO, INC2

  1829. 	LFPDUX	B2,  BO, INC2

  1830. 	LFPDUX	A3,  AO, INC2

  1831. 	LFPDUX	A4,  AO, INC2

  1832. 

  1833. 	LFPDUX	B3,  BO, INC2

  1834. 	LFPDUX	A5,  AO, INC2

  1835. 	LFPDUX	A6,  AO, INC2

  1836. 	LFPDUX	B4,  BO, INC2

  1837. 	LFPDUX	A7,  AO, INC2

  1838. 	LFPDUX	A8,  AO, INC2

  1839. 	bdz-	.L63

  1840. 	.align 4

  1841. 

  1842. .L62:

  1843. 	FXCPMADD	f0,  B1, A1, f0

  1844. 	FXCSMADD	f2,  B1, A1, f2

  1845. 	LFPDUX	A1,  AO, INC2

  1846. 	FXCPMADD	f1,  B1, A2, f1

  1847. 	FXCSMADD	f3,  B1, A2, f3

  1848. 	LFPDUX	A2,  AO, INC2

  1849. 	LFPDUX	B1,  BO, INC2

  1850. 

  1851. 	FXCPMADD	f0,  B2, A3, f0

  1852. 	FXCSMADD	f2,  B2, A3, f2

  1853. 	LFPDUX	A3,  AO, INC2

  1854. 	FXCPMADD	f1,  B2, A4, f1

  1855. 	FXCSMADD	f3,  B2, A4, f3

  1856. 	LFPDUX	A4,  AO, INC2

  1857. 	LFPDUX	B2,  BO, INC2

  1858. 

  1859. 	FXCPMADD	f0,  B3, A5, f0

  1860. 	FXCSMADD	f2,  B3, A5, f2

  1861. 	LFPDUX	A5,  AO, INC2

  1862. 	FXCPMADD	f1,  B3, A6, f1

  1863. 	FXCSMADD	f3,  B3, A6, f3

  1864. 	LFPDUX	A6,  AO, INC2

  1865. 	LFPDUX	B3,  BO, INC2

  1866. 

  1867. 	FXCPMADD	f0,  B4, A7, f0

  1868. 	FXCSMADD	f2,  B4, A7, f2

  1869. 	LFPDUX	A7,  AO, INC2

  1870. 	FXCPMADD	f1,  B4, A8, f1

  1871. 	FXCSMADD	f3,  B4, A8, f3

  1872. 	LFPDUX	A8,  AO, INC2

  1873. 	LFPDUX	B4,  BO, INC2

  1874. 	bdnz+	.L62

  1875. 	.align 4

  1876. 

  1877. .L63:

  1878. 	FXCPMADD	f0,  B1, A1, f0

  1879. 	FXCSMADD	f2,  B1, A1, f2

  1880. 	FXCPMADD	f1,  B1, A2, f1

  1881. 	FXCSMADD	f3,  B1, A2, f3

  1882. 

  1883. 	FXCPMADD	f0,  B2, A3, f0

  1884. 	FXCSMADD	f2,  B2, A3, f2

  1885. 	FXCPMADD	f1,  B2, A4, f1

  1886. 	FXCSMADD	f3,  B2, A4, f3

  1887. 

  1888. 	FXCPMADD	f0,  B3, A5, f0

  1889. 	FXCSMADD	f2,  B3, A5, f2

  1890. 	FXCPMADD	f1,  B3, A6, f1

  1891. 	FXCSMADD	f3,  B3, A6, f3

  1892. 

  1893. 	FXCPMADD	f0,  B4, A7, f0

  1894. 	FXCSMADD	f2,  B4, A7, f2

  1895. 	FXCPMADD	f1,  B4, A8, f1

  1896. 	FXCSMADD	f3,  B4, A8, f3

  1897. 	.align 4

  1898. 

  1899. .L64:

  1900. 	li	r0, ALPHA

  1901. 	lfpdx	AP,  SP, r0

  1902. #ifdef TRMMKERNEL

  1903. 	li	r0, FZERO

  1904. 	lfpsx	f30, SP, r0

  1905. #endif

  1906. 

  1907. #if defined(TRMMKERNEL)

  1908. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1909. 	sub	TEMP, K, KK

  1910. #elif defined(LEFT)

  1911. 	addi	TEMP, KK, 2

  1912. #else

  1913. 	addi	TEMP, KK, 1

  1914. #endif

  1915. 	andi.	r0,  TEMP,  3

  1916. 	mtspr	CTR, r0

  1917. #else

  1918. 	andi.	r0,  K,  3

  1919. 	mtspr	CTR, r0

  1920. #endif

  1921. 	ble+	.L68

  1922. 

  1923. 	LFPDUX	A1,  AO,  INC2

  1924. 	LFPDUX	B1,  BO,  INC2

  1925. 	LFPDUX	A2,  AO,  INC2

  1926. 	bdz-	.L67

  1927. 	.align 4

  1928. 

  1929. .L66:

  1930. 	FXCPMADD	f0,  B1, A1, f0

  1931. 	FXCSMADD	f2,  B1, A1, f2

  1932. 	LFPDUX	A1,  AO,  INC2

  1933. 	FXCPMADD	f1,  B1, A2, f1

  1934. 	FXCSMADD	f3,  B1, A2, f3

  1935. 	LFPDUX	B1,  BO,  INC2

  1936. 	LFPDUX	A2,  AO,  INC2

  1937. 	bdnz+	.L66

  1938. 	.align 4

  1939. 

  1940. .L67:

  1941. 	FXCPMADD	f0,  B1, A1, f0

  1942. 	FXCSMADD	f2,  B1, A1, f2

  1943. 	FXCPMADD	f1,  B1, A2, f1

  1944. 	FXCSMADD	f3,  B1, A2, f3

  1945. 	.align 4

  1946. 

  1947. .L68:

  1948. #ifndef TRMMKERNEL

  1949. 	LFPDUX	A1, CO1, INC2

  1950. 	LFPDUX	A2, CO1, INC2

  1951. #endif

  1952. 

  1953. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1954.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  1955. 	fpadd	f0, f0, f2

  1956. 	fpadd	f1, f1, f3

  1957. #else

  1958. 	fpsub	f0, f0, f2

  1959. 	fpsub	f1, f1, f3

  1960. #endif

  1961. 

  1962. #ifndef TRMMKERNEL

  1963. 	fxcpmadd A1,  f0, AP,  A1

  1964. 	fxcpmadd A2,  f1, AP,  A2

  1965. 	fxcxnpma f0,  f0, AP,  A1

  1966. 	fxcxnpma f1,  f1, AP,  A2

  1967. 

  1968. 	STFPDUX	f0,  CO1, INCM3

  1969. 	STFPDUX	f1,  CO1, INC2

  1970. #else

  1971. 	fxcpmadd f12, f0, AP,  f30

  1972. 	fxcpmadd f13, f1, AP,  f30

  1973. 	fxcxnpma f0,  f0, AP,  f12

  1974. 	fxcxnpma f1,  f1, AP,  f13

  1975. 

  1976. 	STFPDUX	f0,  CO1, INC2

  1977. 	STFPDUX	f1,  CO1, INC2

  1978. #endif

  1979. 

  1980. #ifdef TRMMKERNEL

  1981. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  1982.     (!defined(LEFT) && !defined(TRANSA))

  1983. 	sub	TEMP, K, KK

  1984. #ifdef LEFT

  1985. 	addi	TEMP, TEMP, -2

  1986. #else

  1987. 	addi	TEMP, TEMP, -1

  1988. #endif

  1989. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  1990. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  1991. 	add	AO, AO, r0

  1992. 	add	BO, BO, TEMP

  1993. #endif

  1994. 

  1995. #ifdef LEFT

  1996. 	addi	KK, KK, 2

  1997. #endif

  1998. #endif

  1999. 

  2000. 	li	r0, FZERO

  2001. 	lfpsx	f0, SP, r0

  2002. 	.align 4

  2003. 

  2004. .L70:

  2005. 	andi.	I, M,  1

  2006. 	beq	.L89

  2007. 

  2008. #if defined(TRMMKERNEL)

  2009. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2010. 	addi	BO,  B,  - 2 * SIZE

  2011. 	fpmr	f1,  f0

  2012. #else

  2013. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  2014. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  2015. 	add	AO, AO, TEMP

  2016. 	add	BO, B,  r0

  2017. 

  2018. 	addi	BO,  BO,  - 2 * SIZE

  2019. 	fpmr	f1,  f0

  2020. #endif

  2021. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2022. 	sub	TEMP, K, KK

  2023. #elif defined(LEFT)

  2024. 	addi	TEMP, KK, 1

  2025. #else

  2026. 	addi	TEMP, KK, 1

  2027. #endif

  2028. 	srawi.	r0,  TEMP,  3

  2029. 	fpmr	f2,  f0

  2030. 	mtspr	CTR, r0

  2031. 	fpmr	f3,  f0

  2032. 	ble	.L74

  2033. #else

  2034. 	addi	BO,  B,  - 2 * SIZE

  2035. 	fpmr	f1,  f0

  2036. 	srawi.	r0,  K,  3

  2037. 	fpmr	f2,  f0

  2038. 	mtspr	CTR, r0

  2039. 	fpmr	f3,  f0

  2040. 	ble	.L74

  2041. #endif

  2042. 

  2043. 	LFPDUX	A1,  AO, INC2

  2044. 	LFPDUX	B1,  BO, INC2

  2045. 	LFPDUX	A2,  AO, INC2

  2046. 	LFPDUX	B2,  BO, INC2

  2047. 	LFPDUX	A3,  AO, INC2

  2048. 	LFPDUX	B3,  BO, INC2

  2049. 	LFPDUX	A4,  AO, INC2

  2050. 	LFPDUX	B4,  BO, INC2

  2051. 

  2052. 	LFPDUX	A5,  AO, INC2

  2053. 	LFPDUX	B5,  BO, INC2

  2054. 	LFPDUX	A6,  AO, INC2

  2055. 	LFPDUX	B6,  BO, INC2

  2056. 	LFPDUX	A7,  AO, INC2

  2057. 	LFPDUX	A9,  BO, INC2

  2058. 	LFPDUX	A8,  AO, INC2

  2059. 	LFPDUX	A10, BO, INC2

  2060. 	bdz-	.L73

  2061. 	.align 4

  2062. 

  2063. .L72:

  2064. 	FXCPMADD	f0,  B1, A1, f0

  2065. 	FXCSMADD	f1,  B1, A1, f1

  2066. 	LFPDUX	A1,  AO, INC2

  2067. 	LFPDUX	B1,  BO, INC2

  2068. 	FXCPMADD	f2,  B2, A2, f2

  2069. 	FXCSMADD	f3,  B2, A2, f3

  2070. 	LFPDUX	A2,  AO, INC2

  2071. 	LFPDUX	B2,  BO, INC2

  2072. 

  2073. 	FXCPMADD	f0,  B3, A3, f0

  2074. 	FXCSMADD	f1,  B3, A3, f1

  2075. 	LFPDUX	A3,  AO, INC2

  2076. 	LFPDUX	B3,  BO, INC2

  2077. 	FXCPMADD	f2,  B4, A4, f2

  2078. 	FXCSMADD	f3,  B4, A4, f3

  2079. 	LFPDUX	A4,  AO, INC2

  2080. 	LFPDUX	B4,  BO, INC2

  2081. 

  2082. 	FXCPMADD	f0,  B5, A5, f0

  2083. 	FXCSMADD	f1,  B5, A5, f1

  2084. 	LFPDUX	A5,  AO, INC2

  2085. 	LFPDUX	B5,  BO, INC2

  2086. 	FXCPMADD	f2,  B6, A6, f2

  2087. 	FXCSMADD	f3,  B6, A6, f3

  2088. 	LFPDUX	A6,  AO, INC2

  2089. 	LFPDUX	B6,  BO, INC2

  2090. 

  2091. 	FXCPMADD	f0,  A9,  A7, f0

  2092. 	FXCSMADD	f1,  A9,  A7, f1

  2093. 	LFPDUX	A7,  AO, INC2

  2094. 	LFPDUX	A9,  BO, INC2

  2095. 	FXCPMADD	f2,  A10, A8, f2

  2096. 	FXCSMADD	f3,  A10, A8, f3

  2097. 	LFPDUX	A8,  AO, INC2

  2098. 	LFPDUX	A10, BO, INC2

  2099. 

  2100. 	bdnz+	.L72

  2101. 	.align 4

  2102. 

  2103. .L73:

  2104. 	FXCPMADD	f0,  B1, A1, f0

  2105. 	FXCSMADD	f1,  B1, A1, f1

  2106. 	FXCPMADD	f2,  B2, A2, f2

  2107. 	FXCSMADD	f3,  B2, A2, f3

  2108. 

  2109. 	FXCPMADD	f0,  B3, A3, f0

  2110. 	FXCSMADD	f1,  B3, A3, f1

  2111. 	FXCPMADD	f2,  B4, A4, f2

  2112. 	FXCSMADD	f3,  B4, A4, f3

  2113. 

  2114. 	FXCPMADD	f0,  B5, A5, f0

  2115. 	FXCSMADD	f1,  B5, A5, f1

  2116. 	FXCPMADD	f2,  B6, A6, f2

  2117. 	FXCSMADD	f3,  B6, A6, f3

  2118. 

  2119. 	FXCPMADD	f0,  A9,  A7, f0

  2120. 	FXCSMADD	f1,  A9,  A7, f1

  2121. 	FXCPMADD	f2,  A10, A8, f2

  2122. 	FXCSMADD	f3,  A10, A8, f3

  2123. 	.align 4

  2124. 

  2125. .L74:

  2126. 	li	r0, ALPHA

  2127. 	lfpdx	AP,  SP, r0

  2128. #ifdef TRMMKERNEL

  2129. 	li	r0, FZERO

  2130. 	lfpsx	f30, SP, r0

  2131. #endif

  2132. 

  2133. #if defined(TRMMKERNEL)

  2134. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2135. 	sub	TEMP, K, KK

  2136. #elif defined(LEFT)

  2137. 	addi	TEMP, KK, 1

  2138. #else

  2139. 	addi	TEMP, KK, 1

  2140. #endif

  2141. 	andi.	r0,  TEMP,  7

  2142. 	mtspr	CTR, r0

  2143. #else

  2144. 	andi.	r0,  K,  7

  2145. 	mtspr	CTR, r0

  2146. #endif

  2147. 	ble+	.L78

  2148. 

  2149. 	LFPDUX	A1,  AO,  INC2

  2150. 	LFPDUX	B1,  BO,  INC2

  2151. 	bdz-	.L77

  2152. 	.align 4

  2153. 

  2154. .L76:

  2155. 	FXCPMADD	f0,  B1, A1, f0

  2156. 	FXCSMADD	f1,  B1, A1, f1

  2157. 	LFPDUX	A1,  AO,  INC2

  2158. 	LFPDUX	B1,  BO,  INC2

  2159. 	bdnz+	.L76

  2160. 	.align 4

  2161. 

  2162. .L77:

  2163. 	FXCPMADD	f0,  B1, A1, f0

  2164. 	FXCSMADD	f1,  B1, A1, f1

  2165. 	.align 4

  2166. 

  2167. .L78:

  2168. #ifndef TRMMKERNEL

  2169. 	LFPDX	A1, CO1, INC2

  2170. #endif

  2171. 

  2172. 	fpadd	f0, f0, f2

  2173. 	fpadd	f1, f1, f3

  2174. 

  2175. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  2176.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  2177. 	fpadd	f0, f0, f1

  2178. #else

  2179. 	fpsub	f0, f0, f1

  2180. #endif

  2181. 

  2182. #ifndef TRMMKERNEL

  2183. 	fxcpmadd A1,  f0, AP,  A1

  2184. 	fxcxnpma f0,  f0, AP,  A1

  2185. #else

  2186. 	fxcpmadd f12, f0, AP,  f30

  2187. 	fxcxnpma f0,  f0, AP,  f12

  2188. #endif

  2189. 

  2190. 	STFPDUX	f0,  CO1, INC2

  2191. 

  2192. 	li	r0, FZERO

  2193. 	lfpsx	f0, SP, r0

  2194. 	.align 4

  2195. 

  2196. .L89:

  2197. 	addi	B,  BO, 2 * SIZE

  2198. 	.align 4

  2199. 

  2200. .L999:

  2201. 	addi	SP, SP, 20

  2202. 

  2203. 	lwzu	r14,   4(SP)

  2204. 	lwzu	r15,   4(SP)

  2205. 

  2206. 	lwzu	r16,   4(SP)

  2207. 	lwzu	r17,   4(SP)

  2208. 	lwzu	r18,   4(SP)

  2209. 	lwzu	r19,   4(SP)

  2210. 

  2211. 	lwzu	r20,   4(SP)

  2212. 	lwzu	r21,   4(SP)

  2213. 	lwzu	r22,   4(SP)

  2214. 	lwzu	r23,   4(SP)

  2215. 

  2216. 	lwzu	r24,   4(SP)

  2217. 	lwzu	r25,   4(SP)

  2218. 	lwzu	r26,   4(SP)

  2219. 	lwzu	r27,   4(SP)

  2220. 

  2221. 	lwzu	r28,   4(SP)

  2222. 	lwzu	r29,   4(SP)

  2223. 	lwzu	r30,   4(SP)

  2224. 	lwzu	r31,   4(SP)

  2225. 

  2226. 	subi	SP, SP, 12

  2227. 	li	r0, 16

  2228. 

  2229. 	lfpdux	f31, SP, r0

  2230. 	lfpdux	f30, SP, r0

  2231. 	lfpdux	f29, SP, r0

  2232. 	lfpdux	f28, SP, r0

  2233. 	lfpdux	f27, SP, r0

  2234. 	lfpdux	f26, SP, r0

  2235. 	lfpdux	f25, SP, r0

  2236. 	lfpdux	f24, SP, r0

  2237. 	lfpdux	f23, SP, r0

  2238. 	lfpdux	f22, SP, r0

  2239. 	lfpdux	f21, SP, r0

  2240. 	lfpdux	f20, SP, r0

  2241. 	lfpdux	f19, SP, r0

  2242. 	lfpdux	f18, SP, r0

  2243. 	lfpdux	f17, SP, r0

  2244. 	lfpdux	f16, SP, r0

  2245. 	lfpdux	f15, SP, r0

  2246. 	lfpdux	f14, SP, r0

  2247. 	addi	SP, SP, 16

  2248. 	blr

  2249. 	.align 4

  2250. 

  2251. 

  2252. .L1000:

  2253. 	li	INC,    1 * SIZE

  2254. 	li	INC2,   2 * SIZE

  2255. 	li	INC4,   4 * SIZE

  2256. 	li	INCM1, -1 * SIZE

  2257. 	li	INCM3, -3 * SIZE

  2258. 	li	INCM5, -5 * SIZE

  2259. 	li	INCM7, -7 * SIZE

  2260. 

  2261. 	addi	C, C, - 1 * SIZE

  2262. 	srawi.	J, N,  1

  2263. 	ble	.L1050

  2264. 	.align 4

  2265. 

  2266. .L1010:

  2267. 	mr	CO1, C

  2268. 	add	CO2, C,   LDC

  2269. 	add	C,   CO2, LDC

  2270. 

  2271. #if defined(TRMMKERNEL) &&  defined(LEFT)

  2272. 	mr	KK, OFFSET

  2273. #endif

  2274. 

  2275. 	addi	AO, A, -4 * SIZE

  2276. 

  2277. 	li	r0, FZERO

  2278. 	lfpsx	f0, SP, r0

  2279. 

  2280. 	srawi.	I, M,  2

  2281. 	ble	.L1020

  2282. 	.align 4

  2283. 

  2284. .L1011:

  2285. #if defined(TRMMKERNEL)

  2286. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2287. 	addi	AO2, AO,   2 * SIZE

  2288. 	fpmr	f4,  f0

  2289. 	addi	BO,  B,  - 4 * SIZE

  2290. 	fpmr	f8,  f0

  2291. 	addi	BO2, B,  - 2 * SIZE

  2292. 	fpmr	f12, f0

  2293. #else

  2294. 	slwi	TEMP, KK, 2 + ZBASE_SHIFT

  2295. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  2296. 	add	AO, AO, TEMP

  2297. 	add	BO, B,  r0

  2298. 

  2299. 	addi	AO2, AO,   2 * SIZE

  2300. 	fpmr	f4,  f0

  2301. 	addi	BO,  BO, - 4 * SIZE

  2302. 	fpmr	f8,  f0

  2303. 	addi	BO2, BO,   2 * SIZE

  2304. 	fpmr	f12, f0

  2305. #endif

  2306. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2307. 	sub	TEMP, K, KK

  2308. #elif defined(LEFT)

  2309. 	addi	TEMP, KK, 4

  2310. #else

  2311. 	addi	TEMP, KK, 2

  2312. #endif

  2313. 	srawi.	r0,  TEMP,  2

  2314.  	fpmr	f1,  f0

  2315. 	mtspr	CTR, r0

  2316. 	ble	.L1014

  2317. #else

  2318. 	addi	AO2, AO,   2 * SIZE

  2319. 	fpmr	f4,  f0

  2320. 	addi	BO,  B,  - 4 * SIZE

  2321. 	fpmr	f8,  f0

  2322. 	addi	BO2, B,  - 2 * SIZE

  2323. 	fpmr	f12, f0

  2324. 

  2325. 	srawi.	r0,  K,  2

  2326.  	fpmr	f1,  f0

  2327. 	mtspr	CTR, r0

  2328. 	ble	.L1014

  2329. #endif

  2330. 

  2331. 	LFPDUX	A1,  AO, INC4

  2332. 	fpmr	f5,  f0

  2333. 	LFPDUX	A3,  AO, INC4

  2334. 	fpmr	f9,  f0

  2335. 	LFPDUX	B1,  BO, INC4

  2336. 	fpmr	f13, f0

  2337. 

  2338. 	LFPDUX	A5,  AO, INC4

  2339. 	fpmr	f2,  f0

  2340. 	LFPDUX	A6,  AO, INC4

  2341. 	fpmr	f6,  f0

  2342. 	LFPDUX	B3,  BO, INC4

  2343. 	fpmr	f10, f0

  2344. 	LFPDUX	A7,  AO, INC4

  2345. 	fpmr	f14, f0

  2346. 

  2347. 	LFPDUX	A8,  AO, INC4

  2348. 	fpmr	f3,  f0

  2349. 	LFPDUX	B5,  BO, INC4

  2350. 	fpmr	f7,  f0

  2351. 	LFPDUX	A9,  AO, INC4

  2352. 	fpmr	f11, f0

  2353. 	LFPDUX	A2, AO2, INC4

  2354. 	fpmr	f15, f0

  2355. 	LFPDUX	B2, BO2, INC4

  2356. 	bdz-	.L1013

  2357. 	.align 4

  2358. 

  2359. .L1012:

  2360. 

  2361. ## 1 ##

  2362. 	FXCPMADD	f0,  B1, A1, f0

  2363. 	nop

  2364. 	FXCSMADD	f4,  B1, A1, f4

  2365. 	nop

  2366. 	FXCPMADD	f8,  B2, A1, f8

  2367. 	LFPDUX	B4, BO2, INC4

  2368. 	FXCSMADD	f12, B2, A1, f12

  2369. 	LFPDUX	B6,  BO, INC4

  2370. 

  2371. 	FXCPMADD	f1,  B1, A2, f1

  2372. 	nop

  2373. 	FXCSMADD	f5,  B1, A2, f5

  2374. 	LFPDUX	A4, AO2, INC4

  2375. 	FXCPMADD	f9,  B2, A2, f9

  2376. 	LFPDUX	A10, AO, INC4

  2377. 	FXCSMADD	f13, B2, A2, f13

  2378. 	nop

  2379. 

  2380. 	FXCPMADD	f2,  B1, A3, f2

  2381. 	nop

  2382. 	FXCSMADD	f6,  B1, A3, f6

  2383. 	nop

  2384. 	FXCPMADD	f10, B2, A3, f10

  2385. 	nop

  2386. 	FXCSMADD	f14, B2, A3, f14

  2387. 	nop

  2388. 

  2389. 	FXCPMADD	f3,  B1, A4, f3

  2390. 	nop

  2391. 	FXCSMADD	f7,  B1, A4, f7

  2392. 	LFPDUX	A2, AO2, INC4

  2393. 	FXCPMADD	f11, B2, A4, f11

  2394. 	LFPDUX	A1,  AO, INC4

  2395. 	FXCSMADD	f15, B2, A4, f15

  2396. 	nop

  2397. 

  2398. ## 2 ##

  2399. 

  2400. 	FXCPMADD	f0,  B3, A5, f0

  2401. 	nop

  2402. 	FXCSMADD	f4,  B3, A5, f4

  2403. 	nop

  2404. 	FXCPMADD	f8,  B4, A5, f8

  2405. 	LFPDUX	B2, BO2, INC4

  2406. 	FXCSMADD	f12, B4, A5, f12

  2407. 	LFPDUX	B1,  BO, INC4

  2408. 

  2409. 	FXCPMADD	f1,  B3, A2, f1

  2410. 	nop

  2411. 	FXCSMADD	f5,  B3, A2, f5

  2412. 	LFPDUX	A4, AO2, INC4

  2413. 	FXCPMADD	f9,  B4, A2, f9

  2414. 	LFPDUX	A3,  AO, INC4

  2415. 	FXCSMADD	f13, B4, A2, f13

  2416. 	nop

  2417. 

  2418. 	FXCPMADD	f2,  B3, A6, f2

  2419. 	nop

  2420. 	FXCSMADD	f6,  B3, A6, f6

  2421. 	nop

  2422. 	FXCPMADD	f10, B4, A6, f10

  2423. 	nop

  2424. 	FXCSMADD	f14, B4, A6, f14

  2425. 	nop

  2426. 

  2427. 	FXCPMADD	f3,  B3, A4, f3

  2428. 	nop

  2429. 	FXCSMADD	f7,  B3, A4, f7

  2430. 	LFPDUX	A2, AO2, INC4

  2431. 	FXCPMADD	f11, B4, A4, f11

  2432. 	LFPDUX	A5,  AO, INC4

  2433. 	FXCSMADD	f15, B4, A4, f15

  2434. 	nop

  2435. 

  2436. ## 3 ##

  2437. 

  2438. 	FXCPMADD	f0,  B5, A7, f0

  2439. 	nop

  2440. 	FXCSMADD	f4,  B5, A7, f4

  2441. 	nop

  2442. 	FXCPMADD	f8,  B2, A7, f8

  2443. 	LFPDUX	B4, BO2, INC4

  2444. 	FXCSMADD	f12, B2, A7, f12

  2445. 	LFPDUX	B3,  BO, INC4

  2446. 

  2447. 	FXCPMADD	f1,  B5, A2, f1

  2448. 	nop

  2449. 	FXCSMADD	f5,  B5, A2, f5

  2450. 	LFPDUX	A4, AO2, INC4

  2451. 	FXCPMADD	f9,  B2, A2, f9

  2452. 	LFPDUX	A6,  AO, INC4

  2453. 	FXCSMADD	f13, B2, A2, f13

  2454. 	nop

  2455. 

  2456. 	FXCPMADD	f2,  B5, A8, f2

  2457. 	nop

  2458. 	FXCSMADD	f6,  B5, A8, f6

  2459. 	nop

  2460. 	FXCPMADD	f10, B2, A8, f10

  2461. 	nop

  2462. 	FXCSMADD	f14, B2, A8, f14

  2463. 	nop

  2464. 

  2465. 	FXCPMADD	f3,  B5, A4, f3

  2466. 	nop

  2467. 	FXCSMADD	f7,  B5, A4, f7

  2468. 	LFPDUX	A2, AO2, INC4

  2469. 	FXCPMADD	f11, B2, A4, f11

  2470. 	LFPDUX	A7,  AO, INC4

  2471. 	FXCSMADD	f15, B2, A4, f15

  2472. 	nop

  2473. 

  2474. ## 4 ##

  2475. 	FXCPMADD	f0,  B6, A9, f0

  2476. 	nop

  2477. 	FXCSMADD	f4,  B6, A9, f4

  2478. 	nop

  2479. 	FXCPMADD	f8,  B4, A9, f8

  2480. 	LFPDUX	B2, BO2, INC4

  2481. 	FXCSMADD	f12, B4, A9, f12

  2482. 	LFPDUX	B5,  BO, INC4

  2483. 

  2484. 	FXCPMADD	f1,  B6, A2, f1

  2485. 	nop

  2486. 	FXCSMADD	f5,  B6, A2, f5

  2487. 	LFPDUX	A4, AO2, INC4

  2488. 	FXCPMADD	f9,  B4, A2, f9

  2489. 	LFPDUX	A8,  AO, INC4

  2490. 	FXCSMADD	f13, B4, A2, f13

  2491. 	nop

  2492. 

  2493. 	FXCPMADD	f2,  B6, A10, f2

  2494. 	nop

  2495. 	FXCSMADD	f6,  B6, A10, f6

  2496. 	nop

  2497. 	FXCPMADD	f10, B4, A10, f10

  2498. 	nop

  2499. 	FXCSMADD	f14, B4, A10, f14

  2500. 	nop

  2501. 

  2502. 	FXCPMADD	f3,  B6, A4, f3

  2503. 	LFPDUX	A2, AO2, INC4

  2504. 	FXCSMADD	f7,  B6, A4, f7

  2505. 	LFPDUX	A9,  AO, INC4

  2506. 	FXCPMADD	f11, B4, A4, f11

  2507. 	nop

  2508. 	FXCSMADD	f15, B4, A4, f15

  2509. 	bdnz+	.L1012

  2510. 	.align 4

  2511. 

  2512. .L1013:

  2513. ## 1 ##

  2514. 

  2515. 	FXCPMADD	f0,  B1, A1, f0

  2516. 	nop

  2517. 	FXCSMADD	f4,  B1, A1, f4

  2518. 	nop

  2519. 	FXCPMADD	f8,  B2, A1, f8

  2520. 	LFPDUX	B4, BO2, INC4

  2521. 	FXCSMADD	f12, B2, A1, f12

  2522. 	LFPDUX	B6,  BO, INC4

  2523. 

  2524. 	FXCPMADD	f1,  B1, A2, f1

  2525. 	nop

  2526. 	FXCSMADD	f5,  B1, A2, f5

  2527. 	LFPDUX	A4, AO2, INC4

  2528. 	FXCPMADD	f9,  B2, A2, f9

  2529. 	LFPDUX	A10, AO, INC4

  2530. 	FXCSMADD	f13, B2, A2, f13

  2531. 	nop

  2532. 

  2533. 	FXCPMADD	f2,  B1, A3, f2

  2534. 	nop

  2535. 	FXCSMADD	f6,  B1, A3, f6

  2536. 	nop

  2537. 	FXCPMADD	f10, B2, A3, f10

  2538. 	nop

  2539. 	FXCSMADD	f14, B2, A3, f14

  2540. 	nop

  2541. 

  2542. 	FXCPMADD	f3,  B1, A4, f3

  2543. 	nop

  2544. 	FXCSMADD	f7,  B1, A4, f7

  2545. 	LFPDUX	A2, AO2, INC4

  2546. 	FXCPMADD	f11, B2, A4, f11

  2547. #ifndef TRMMKERNEL

  2548. 	LFDUX	A1, CO1, INC

  2549. #else

  2550. 	nop

  2551. #endif

  2552. 	FXCSMADD	f15, B2, A4, f15

  2553. 	nop

  2554. 

  2555. ## 2 ##

  2556. 

  2557. 	FXCPMADD	f0,  B3, A5, f0

  2558. 	nop

  2559. 	FXCSMADD	f4,  B3, A5, f4

  2560. 	nop

  2561. 	FXCPMADD	f8,  B4, A5, f8

  2562. 	LFPDUX	B2, BO2, INC4

  2563. 	FXCSMADD	f12, B4, A5, f12

  2564. #ifndef TRMMKERNEL

  2565. 	LFDUX	B1, CO1, INC2

  2566. #else

  2567. 	nop

  2568. #endif

  2569. 

  2570. 	FXCPMADD	f1,  B3, A2, f1

  2571. 	nop

  2572. 	FXCSMADD	f5,  B3, A2, f5

  2573. 	LFPDUX	A4, AO2, INC4

  2574. 	FXCPMADD	f9,  B4, A2, f9

  2575. #ifndef TRMMKERNEL

  2576. 	LFDUX	A3, CO1, INC2

  2577. #else

  2578. 	nop

  2579. #endif

  2580. 	FXCSMADD	f13, B4, A2, f13

  2581. 	nop

  2582. 

  2583. 	FXCPMADD	f2,  B3, A6, f2

  2584. 	nop

  2585. 	FXCSMADD	f6,  B3, A6, f6

  2586. 	nop

  2587. 	FXCPMADD	f10, B4, A6, f10

  2588. 	nop

  2589. 	FXCSMADD	f14, B4, A6, f14

  2590. 	nop

  2591. 

  2592. 	FXCPMADD	f3,  B3, A4, f3

  2593. 	nop

  2594. 	FXCSMADD	f7,  B3, A4, f7

  2595. 	LFPDUX	A2, AO2, INC4

  2596. 	FXCPMADD	f11, B4, A4, f11

  2597. #ifndef TRMMKERNEL

  2598.    	LFDUX	A5, CO1, INC2

  2599. #else

  2600. 	nop

  2601. #endif

  2602. 	FXCSMADD	f15, B4, A4, f15

  2603. 	nop

  2604. 

  2605. ## 3 ##

  2606. 

  2607. 	FXCPMADD	f0,  B5, A7, f0

  2608. 	nop

  2609. 	FXCSMADD	f4,  B5, A7, f4

  2610. 	nop

  2611. 	FXCPMADD	f8,  B2, A7, f8

  2612. 	LFPDUX	B4, BO2, INC4

  2613. 	FXCSMADD	f12, B2, A7, f12

  2614. #ifndef TRMMKERNEL

  2615. 	LFSDUX	A1, CO1, INCM5

  2616. #else

  2617. 	nop

  2618. #endif

  2619. 

  2620. 	FXCPMADD	f1,  B5, A2, f1

  2621. 	nop

  2622. 	FXCSMADD	f5,  B5, A2, f5

  2623. 	LFPDUX	A4, AO2, INC4

  2624. 	FXCPMADD	f9,  B2, A2, f9

  2625. #ifndef TRMMKERNEL

  2626. 	LFSDUX	B1, CO1, INC2

  2627. #else

  2628. 	nop

  2629. #endif

  2630. 	FXCSMADD	f13, B2, A2, f13

  2631. 	nop

  2632. 

  2633. 	FXCPMADD	f2,  B5, A8, f2

  2634. 	nop

  2635. 	FXCSMADD	f6,  B5, A8, f6

  2636. 	nop

  2637. 	FXCPMADD	f10, B2, A8, f10

  2638. 	nop

  2639. 	FXCSMADD	f14, B2, A8, f14

  2640. 	nop

  2641. 

  2642. 	FXCPMADD	f3,  B5, A4, f3

  2643. 	nop

  2644. 	FXCSMADD	f7,  B5, A4, f7

  2645. 	LFPDUX	A2, AO2, INC4

  2646. 	FXCPMADD	f11, B2, A4, f11

  2647. #ifndef TRMMKERNEL

  2648. 	LFSDUX	A3, CO1, INC2

  2649. #else

  2650. 	nop

  2651. #endif

  2652. 	FXCSMADD	f15, B2, A4, f15

  2653. 	nop

  2654. 

  2655. ## 4 ##

  2656. 

  2657. 	FXCPMADD	f0,  B6, A9, f0

  2658. 	nop

  2659. 	FXCSMADD	f4,  B6, A9, f4

  2660. 	nop

  2661. 	FXCPMADD	f8,  B4, A9, f8

  2662. #ifndef TRMMKERNEL

  2663. 	LFSDUX	A5, CO1, INC2

  2664. #else

  2665. 	nop

  2666. #endif

  2667. 	FXCSMADD	f12, B4, A9, f12

  2668. #ifndef TRMMKERNEL

  2669. 	LFDUX	B3, CO2, INC

  2670. #else

  2671. 	nop

  2672. #endif

  2673. 

  2674. 	FXCPMADD	f1,  B6, A2, f1

  2675. 	nop

  2676. 	FXCSMADD	f5,  B6, A2, f5

  2677. 	LFPDUX	A4, AO2, INC4

  2678. 	FXCPMADD	f9,  B4, A2, f9

  2679. #ifndef TRMMKERNEL

  2680. 	LFDUX	A6, CO2, INC2

  2681. #else

  2682. 	nop

  2683. #endif

  2684. 	FXCSMADD	f13, B4, A2, f13

  2685. 	nop

  2686. 

  2687. 	FXCPMADD	f2,  B6, A10, f2

  2688. 	nop

  2689. 	FXCSMADD	f6,  B6, A10, f6

  2690. 	nop

  2691. 	FXCPMADD	f10, B4, A10, f10

  2692. 	nop

  2693. 	FXCSMADD	f14, B4, A10, f14

  2694. #ifndef TRMMKERNEL

  2695. 	LFDUX	A7, CO2, INC2

  2696. #else

  2697. 	nop

  2698. #endif

  2699. 

  2700. 	FXCPMADD	f3,  B6, A4, f3

  2701. 	nop

  2702. 	FXCSMADD	f7,  B6, A4, f7

  2703. 	nop

  2704. 	FXCPMADD	f11, B4, A4, f11

  2705. 	nop

  2706. 	FXCSMADD	f15, B4, A4, f15

  2707. #ifndef TRMMKERNEL

  2708. 	LFDUX	B2, CO2, INC2

  2709. #else

  2710. 	nop

  2711. #endif

  2712. 	.align 4

  2713. 

  2714. .L1014:

  2715. 	li	r0, ALPHA

  2716. 	lfpdx	AP,  SP, r0

  2717. #ifdef TRMMKERNEL

  2718. 	li	r0, FZERO

  2719. 	lfpsx	f30, SP, r0

  2720. #endif

  2721. 

  2722. #if defined(TRMMKERNEL)

  2723. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2724. 	sub	TEMP, K, KK

  2725. #elif defined(LEFT)

  2726. 	addi	TEMP, KK, 4

  2727. #else

  2728. 	addi	TEMP, KK, 2

  2729. #endif

  2730. 	andi.	r0,  TEMP,  3

  2731. 	mtspr	CTR, r0

  2732. 	ble+	.L1018

  2733. 

  2734. 	cmpwi	cr0, TEMP, 3

  2735. 	bgt+	.L1015

  2736. #else

  2737. 	andi.	r0,  K,  3

  2738. 	mtspr	CTR, r0

  2739. 	ble+	.L1018

  2740. 

  2741. 	cmpwi	cr0, K, 3

  2742. 	bgt+	.L1015

  2743. #endif

  2744. 

  2745. #ifndef TRMMKERNEL

  2746. 	LFDUX	A1, CO1, INC

  2747. 	fpmr	f5,  f0

  2748. 	LFDUX	B1, CO1, INC2

  2749. 	fpmr	f9,  f0

  2750. 	LFDUX	A3, CO1, INC2

  2751. 	fpmr	f13, f0

  2752.    	LFDUX	A5, CO1, INC2

  2753. 	fpmr	f2,  f0

  2754. 

  2755. 	LFSDUX	A1, CO1, INCM5

  2756. 	fpmr	f6,  f0

  2757. 	LFSDUX	B1, CO1, INC2

  2758. 	fpmr	f10, f0

  2759. 	LFSDUX	A3, CO1, INC2

  2760. 	fpmr	f14, f0

  2761. 	LFSDUX	A5, CO1, INC2

  2762. 	fpmr	f3,  f0

  2763. 

  2764. 	LFDUX	B3, CO2, INC

  2765. 	fpmr	f7,  f0

  2766. 	LFDUX	A6, CO2, INC2

  2767. 	fpmr	f11, f0

  2768. 	LFDUX	A7, CO2, INC2

  2769. 	fpmr	f15, f0

  2770. 	LFDUX	B2, CO2, INC2

  2771. #else

  2772. 	fpmr	f5,  f0

  2773. 	fpmr	f9,  f0

  2774. 	fpmr	f13, f0

  2775. 	fpmr	f2,  f0

  2776. 

  2777. 	fpmr	f6,  f0

  2778. 	fpmr	f10, f0

  2779. 	fpmr	f14, f0

  2780. 	fpmr	f3,  f0

  2781. 

  2782. 	fpmr	f7,  f0

  2783. 	fpmr	f11, f0

  2784. 	fpmr	f15, f0

  2785. #endif

  2786. 	.align 4

  2787. 

  2788. .L1015:

  2789. 	LFPDUX	A2,  AO,  INC4

  2790. 	LFPDUX	A4,  AO2, INC4

  2791. 	LFPDUX	A10, BO,  INC4

  2792. 	LFPDUX	B4,  BO2, INC4

  2793. 	bdz-	.L1017

  2794. 	.align 4

  2795. 

  2796. .L1016:

  2797. 	FXCPMADD	f0,  A10, A2, f0

  2798. 	FXCSMADD	f4,  A10, A2, f4

  2799. 	FXCPMADD	f8,  B4, A2, f8

  2800. 	FXCSMADD	f12, B4, A2, f12

  2801. 	LFPDUX	A2, AO,  INC4

  2802. 

  2803. 	FXCPMADD	f1,  A10, A4, f1

  2804. 	FXCSMADD	f5,  A10, A4, f5

  2805. 	FXCPMADD	f9,  B4, A4, f9

  2806. 	FXCSMADD	f13, B4, A4, f13

  2807. 	LFPDUX	A4, AO2, INC4

  2808. 

  2809. 	FXCPMADD	f2,  A10, A2, f2

  2810. 	FXCSMADD	f6,  A10, A2, f6

  2811. 	FXCPMADD	f10, B4, A2, f10

  2812. 	FXCSMADD	f14, B4, A2, f14

  2813. 	LFPDUX	A2, AO,  INC4

  2814. 

  2815. 	FXCPMADD	f3,  A10, A4, f3

  2816. 	FXCSMADD	f7,  A10, A4, f7

  2817. 	LFPDUX	A10, BO,  INC4

  2818. 	FXCPMADD	f11, B4, A4, f11

  2819. 	FXCSMADD	f15, B4, A4, f15

  2820. 	LFPDUX	A4, AO2, INC4

  2821. 	LFPDUX	B4, BO2, INC4

  2822. 	bdnz+	.L1016

  2823. 	.align 4

  2824. 

  2825. .L1017:

  2826. 	FXCPMADD	f0,  A10, A2, f0

  2827. 	FXCSMADD	f4,  A10, A2, f4

  2828. 	FXCPMADD	f8,  B4, A2, f8

  2829. 	FXCSMADD	f12, B4, A2, f12

  2830. 	LFPDUX	A2, AO,  INC4

  2831. 

  2832. 	FXCPMADD	f1,  A10, A4, f1

  2833. 	FXCSMADD	f5,  A10, A4, f5

  2834. 	FXCPMADD	f9,  B4, A4, f9

  2835. 	FXCSMADD	f13, B4, A4, f13

  2836. 	LFPDUX	A4, AO2, INC4

  2837. 

  2838. 	FXCPMADD	f2,  A10, A2, f2

  2839. 	FXCSMADD	f6,  A10, A2, f6

  2840. 	FXCPMADD	f10, B4, A2, f10

  2841. 	FXCSMADD	f14, B4, A2, f14

  2842. 

  2843. 	FXCPMADD	f3,  A10, A4, f3

  2844. 	FXCSMADD	f7,  A10, A4, f7

  2845. 	FXCPMADD	f11, B4, A4, f11

  2846. 	FXCSMADD	f15, B4, A4, f15

  2847. 	.align 4

  2848. 

  2849. .L1018:

  2850. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  2851.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  2852. 	fpadd	f0, f0, f4

  2853. 	fpadd	f8, f8, f12

  2854. 	fpadd	f1, f1, f5

  2855. 	fpadd	f9, f9, f13

  2856. 

  2857. 	fpadd	f2,  f2,  f6

  2858. 	fpadd	f10, f10, f14

  2859. 	fpadd	f3,  f3,  f7

  2860. 	fpadd	f11, f11, f15

  2861. #else

  2862. 	fpsub	f0, f0, f4

  2863. 	fpsub	f8, f8, f12

  2864. 	fpsub	f1, f1, f5

  2865. 	fpsub	f9, f9, f13

  2866. 

  2867. 	fpsub	f2,  f2,  f6

  2868. 	fpsub	f10, f10, f14

  2869. 	fpsub	f3,  f3,  f7

  2870. 	fpsub	f11, f11, f15

  2871. #endif

  2872. 

  2873. #ifndef TRMMKERNEL

  2874. 	fxcpmadd A1,  f0, AP,  A1

  2875. 	LFSDUX	B3, CO2, INCM5

  2876. 	fxcpmadd B1,  f1, AP,  B1

  2877. 	LFSDUX	A6, CO2, INC2

  2878. 	fxcpmadd A3,  f2, AP,  A3

  2879. 	LFSDUX	A7, CO2, INC2

  2880. 	fxcpmadd A5,  f3, AP,  A5

  2881. 	LFSDUX	B2, CO2, INC2

  2882. 

  2883. 	fxcxnpma f0,  f0, AP,  A1

  2884. 	fxcpmadd B3,  f8,  AP,  B3

  2885. 	fxcxnpma f1,  f1, AP,  B1

  2886. 	fxcpmadd A6,  f9,  AP,  A6

  2887. 	fxcxnpma f2,  f2, AP,  A3

  2888. 	fxcpmadd A7,  f10, AP,  A7

  2889. 

  2890. 	fxcxnpma f3,  f3, AP,  A5

  2891. 	STFDUX	f0,  CO1, INCM7

  2892. 	fxcpmadd B2,  f11, AP,  B2

  2893. 	STFSDUX	f0,  CO1, INC

  2894. 	fxcxnpma f8,  f8,  AP,  B3

  2895. 	STFDUX	f1,  CO1, INC

  2896. 	STFSDUX	f1,  CO1, INC

  2897. 	fxcxnpma f9,  f9,  AP,  A6

  2898. 	STFDUX	f2,  CO1, INC

  2899. 	STFSDUX	f2,  CO1, INC

  2900. 	fxcxnpma f10, f10, AP,  A7

  2901. 	STFDUX	f3,  CO1, INC

  2902. 	STFSDUX	f3,  CO1, INC

  2903. 

  2904. 	fxcxnpma f11, f11, AP,  B2

  2905. 	STFDUX	f8,  CO2, INCM7

  2906. #else

  2907. 	fxcpmadd f12, f0, AP,  f30

  2908. 	fxcpmadd f13, f1, AP,  f30

  2909. 	fxcpmadd f14, f2, AP,  f30

  2910. 	fxcpmadd f15, f3, AP,  f30

  2911. 

  2912. 	fxcxnpma f0,  f0, AP,  f12

  2913. 	fxcxnpma f1,  f1, AP,  f13

  2914. 	fxcxnpma f2,  f2, AP,  f14

  2915. 	fxcxnpma f3,  f3, AP,  f15

  2916. 

  2917. 	fxcpmadd f16, f8,  AP, f30

  2918. 	fxcpmadd f17, f9,  AP, f30

  2919. 	fxcpmadd f18, f10, AP, f30

  2920. 	fxcpmadd f19, f11, AP, f30

  2921. 

  2922. 	fxcxnpma f8,  f8,  AP, f16

  2923. 	fxcxnpma f9,  f9,  AP, f17

  2924. 	fxcxnpma f10, f10, AP, f18

  2925. 	fxcxnpma f11, f11, AP, f19

  2926. 

  2927. 	STFDUX	f0,  CO1, INC

  2928. 	STFSDUX	f0,  CO1, INC

  2929. 	STFDUX	f1,  CO1, INC

  2930. 	STFSDUX	f1,  CO1, INC

  2931. 	STFDUX	f2,  CO1, INC

  2932. 	STFSDUX	f2,  CO1, INC

  2933. 	STFDUX	f3,  CO1, INC

  2934. 	STFSDUX	f3,  CO1, INC

  2935. 	STFDUX	f8,  CO2, INC

  2936. #endif

  2937. 	STFSDUX	f8,  CO2, INC

  2938. 	STFDUX	f9,  CO2, INC

  2939. 	STFSDUX	f9,  CO2, INC

  2940. 	STFDUX	f10, CO2, INC

  2941. 	STFSDUX	f10, CO2, INC

  2942. 

  2943. 	STFDUX	f11, CO2, INC

  2944. 	STFSDUX	f11, CO2, INC

  2945. 

  2946. 

  2947. 

  2948. #ifdef TRMMKERNEL

  2949. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  2950.     (!defined(LEFT) && !defined(TRANSA))

  2951. 	sub	TEMP, K, KK

  2952. #ifdef LEFT

  2953. 	addi	TEMP, TEMP, -4

  2954. #else

  2955. 	addi	TEMP, TEMP, -2

  2956. #endif

  2957. 	slwi	r0,   TEMP, 2 + ZBASE_SHIFT

  2958. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  2959. 	add	AO, AO, r0

  2960. 	add	BO, BO, TEMP

  2961. #endif

  2962. 

  2963. #ifdef LEFT

  2964. 	addi	KK, KK, 4

  2965. #endif

  2966. #endif

  2967. 

  2968. 	addic.	I, I, -1

  2969. 	li	r0, FZERO

  2970. 

  2971. 	lfpsx	f0, SP, r0

  2972. 	bgt+	.L1011

  2973. 	.align 4

  2974. 

  2975. .L1020:

  2976. 	andi.	I, M,  2

  2977. 	beq	.L1030

  2978. 

  2979. #if defined(TRMMKERNEL)

  2980. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  2981. 	addi	AO2, AO,   2 * SIZE

  2982. 	fpmr	f4,  f0

  2983. 	addi	BO,  B,  - 4 * SIZE

  2984. 	fpmr	f8,  f0

  2985. 	addi	BO2, B,  - 2 * SIZE

  2986. 	fpmr	f12, f0

  2987. #else

  2988. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  2989. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  2990. 	add	AO, AO, TEMP

  2991. 	add	BO, B,  r0

  2992. 

  2993. 	addi	AO2, AO,   2 * SIZE

  2994. 	fpmr	f4,  f0

  2995. 	addi	BO,  BO, - 4 * SIZE

  2996. 	fpmr	f8,  f0

  2997. 	addi	BO2, BO,   2 * SIZE

  2998. 	fpmr	f12, f0

  2999. 

  3000. #endif

  3001. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3002. 	sub	TEMP, K, KK

  3003. #elif defined(LEFT)

  3004. 	addi	TEMP, KK, 2

  3005. #else

  3006. 	addi	TEMP, KK, 2

  3007. #endif

  3008. 	srawi.	r0,  TEMP,  2

  3009.  	fpmr	f1,  f0

  3010. 	fpmr	f5,  f0

  3011. 	fpmr	f9,  f0

  3012. 	mtspr	CTR, r0

  3013. 	fpmr	f13, f0

  3014. 	ble	.L1024

  3015. #else

  3016. 	addi	AO2, AO,   2 * SIZE

  3017. 	fpmr	f4,  f0

  3018. 	addi	BO,  B,  - 4 * SIZE

  3019. 	fpmr	f8,  f0

  3020. 	addi	BO2, B,  - 2 * SIZE

  3021. 	fpmr	f12, f0

  3022. 

  3023. 	srawi.	r0,  K,  2

  3024.  	fpmr	f1,  f0

  3025. 	fpmr	f5,  f0

  3026. 	fpmr	f9,  f0

  3027. 	mtspr	CTR, r0

  3028. 	fpmr	f13, f0

  3029. 	ble	.L1024

  3030. #endif

  3031. 

  3032. 	LFPDUX	A1,   AO, INC4

  3033. 	LFPDUX	B1,   BO, INC4

  3034. 	LFPDUX	A2,  AO2, INC4

  3035. 	LFPDUX	B2,  BO2, INC4

  3036. 	LFPDUX	A3,   AO, INC4

  3037. 	LFPDUX	B3,   BO, INC4

  3038. 	LFPDUX	A4,  AO2, INC4

  3039. 	LFPDUX	B4,  BO2, INC4

  3040. 

  3041. 	LFPDUX	A5,   AO, INC4

  3042. 	LFPDUX	B5,   BO, INC4

  3043. 	LFPDUX	A6,  AO2, INC4

  3044. 	LFPDUX	B6,  BO2, INC4

  3045. 	LFPDUX	A7,   AO, INC4

  3046. 	LFPDUX	A9,   BO, INC4

  3047. 	LFPDUX	A10, BO2, INC4

  3048. 	bdz-	.L1023

  3049. 	.align 4

  3050. 

  3051. .L1022:

  3052. 	FXCPMADD	f0,  B1, A1, f0

  3053. 	nop

  3054. 	FXCSMADD	f4,  B1, A1, f4

  3055. 	LFPDUX	A8,  AO2, INC4

  3056. 	FXCPMADD	f8,  B2, A1, f8

  3057. 	nop

  3058. 	FXCSMADD	f12, B2, A1, f12

  3059. 	LFPDUX	A1,   AO, INC4

  3060. 

  3061. 	FXCPMADD	f1,  B1, A2, f1

  3062. 	nop

  3063. 	FXCSMADD	f5,  B1, A2, f5

  3064. 	LFPDUX	B1,   BO, INC4

  3065. 	FXCPMADD	f9,  B2, A2, f9

  3066. 	nop

  3067. 	FXCSMADD	f13, B2, A2, f13

  3068. 	LFPDUX	B2,  BO2, INC4

  3069. 

  3070. 	FXCPMADD	f0,  B3, A3, f0

  3071. 	nop

  3072. 	FXCSMADD	f4,  B3, A3, f4

  3073. 	LFPDUX	A2,  AO2, INC4

  3074. 	FXCPMADD	f8,  B4, A3, f8

  3075. 	nop

  3076. 	FXCSMADD	f12, B4, A3, f12

  3077. 	LFPDUX	A3,   AO, INC4

  3078. 

  3079. 	FXCPMADD	f1,  B3, A4, f1

  3080. 	nop

  3081. 	FXCSMADD	f5,  B3, A4, f5

  3082. 	LFPDUX	B3,   BO, INC4

  3083. 	FXCPMADD	f9,  B4, A4, f9

  3084. 	nop

  3085. 	FXCSMADD	f13, B4, A4, f13

  3086. 	LFPDUX	B4,  BO2, INC4

  3087. 

  3088. 	FXCPMADD	f0,  B5, A5, f0

  3089. 	nop

  3090. 	FXCSMADD	f4,  B5, A5, f4

  3091. 	LFPDUX	A4,  AO2, INC4

  3092. 	FXCPMADD	f8,  B6, A5, f8

  3093. 	nop

  3094. 	FXCSMADD	f12, B6, A5, f12

  3095. 	LFPDUX	A5,   AO, INC4

  3096. 

  3097. 	FXCPMADD	f1,  B5, A6, f1

  3098. 	nop

  3099. 	FXCSMADD	f5,  B5, A6, f5

  3100. 	LFPDUX	B5,   BO, INC4

  3101. 	FXCPMADD	f9,  B6, A6, f9

  3102. 	nop

  3103. 	FXCSMADD	f13, B6, A6, f13

  3104. 	LFPDUX	B6,  BO2, INC4

  3105. 

  3106. 	FXCPMADD	f0,  A9,  A7, f0

  3107. 	nop

  3108. 	FXCSMADD	f4,  A9,  A7, f4

  3109. 	LFPDUX	A6,  AO2, INC4

  3110. 	FXCPMADD	f8,  A10, A7, f8

  3111. 	nop

  3112. 	FXCSMADD	f12, A10, A7, f12

  3113. 	LFPDUX	A7,   AO, INC4

  3114. 

  3115. 	FXCPMADD	f1,  A9,  A8, f1

  3116. 	nop

  3117. 	FXCSMADD	f5,  A9,  A8, f5

  3118. 	LFPDUX	A9,   BO, INC4

  3119. 	FXCPMADD	f9,  A10, A8, f9

  3120. 	nop

  3121. 	FXCSMADD	f13, A10, A8, f13

  3122. 	LFPDUX	A10, BO2, INC4

  3123. 	bdnz+	.L1022

  3124. 	.align 4

  3125. 

  3126. .L1023:

  3127. 	FXCPMADD	f0,  B1, A1, f0

  3128. 	FXCSMADD	f4,  B1, A1, f4

  3129. 	LFPDUX	A8,  AO2, INC4

  3130. 	FXCPMADD	f8,  B2, A1, f8

  3131. 	FXCSMADD	f12, B2, A1, f12

  3132. 

  3133. 	FXCPMADD	f1,  B1, A2, f1

  3134. 	FXCSMADD	f5,  B1, A2, f5

  3135. 	FXCPMADD	f9,  B2, A2, f9

  3136. 	FXCSMADD	f13, B2, A2, f13

  3137. 

  3138. 	FXCPMADD	f0,  B3, A3, f0

  3139. 	FXCSMADD	f4,  B3, A3, f4

  3140. 	FXCPMADD	f8,  B4, A3, f8

  3141. 	FXCSMADD	f12, B4, A3, f12

  3142. 

  3143. 	FXCPMADD	f1,  B3, A4, f1

  3144. 	FXCSMADD	f5,  B3, A4, f5

  3145. 	FXCPMADD	f9,  B4, A4, f9

  3146. 	FXCSMADD	f13, B4, A4, f13

  3147. 

  3148. 	FXCPMADD	f0,  B5, A5, f0

  3149. 	FXCSMADD	f4,  B5, A5, f4

  3150. 	FXCPMADD	f8,  B6, A5, f8

  3151. 	FXCSMADD	f12, B6, A5, f12

  3152. 

  3153. 	FXCPMADD	f1,  B5, A6, f1

  3154. 	FXCSMADD	f5,  B5, A6, f5

  3155. 	FXCPMADD	f9,  B6, A6, f9

  3156. 	FXCSMADD	f13, B6, A6, f13

  3157. 

  3158. 	FXCPMADD	f0,  A9, A7, f0

  3159. 	FXCSMADD	f4,  A9, A7, f4

  3160. 	FXCPMADD	f8,  A10, A7, f8

  3161. 	FXCSMADD	f12, A10, A7, f12

  3162. 

  3163. 	FXCPMADD	f1,  A9, A8, f1

  3164. 	FXCSMADD	f5,  A9, A8, f5

  3165. 	FXCPMADD	f9,  A10, A8, f9

  3166. 	FXCSMADD	f13, A10, A8, f13

  3167. 	.align 4

  3168. 

  3169. .L1024:

  3170. 	li	r0, ALPHA

  3171. 	lfpdx	AP,  SP, r0

  3172. #ifdef TRMMKERNEL

  3173. 	li	r0, FZERO

  3174. 	lfpsx	f30, SP, r0

  3175. #endif

  3176. 

  3177. #if defined(TRMMKERNEL)

  3178. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3179. 	sub	TEMP, K, KK

  3180. #elif defined(LEFT)

  3181. 	addi	TEMP, KK, 2

  3182. #else

  3183. 	addi	TEMP, KK, 2

  3184. #endif

  3185. 	andi.	r0,  TEMP,  3

  3186. 	mtspr	CTR, r0

  3187. #else

  3188. 	andi.	r0,  K,  3

  3189. 	mtspr	CTR, r0

  3190. #endif

  3191. 	ble+	.L1028

  3192. 

  3193. 	LFPDUX	A1,  AO,  INC4

  3194. 	LFPDUX	A2,  AO2, INC4

  3195. 	LFPDUX	B1,  BO,  INC4

  3196. 	LFPDUX	B2,  BO2, INC4

  3197. 	bdz-	.L1027

  3198. 	.align 4

  3199. 

  3200. .L1026:

  3201. 	FXCPMADD	f0,  B1, A1, f0

  3202. 	FXCSMADD	f4,  B1, A1, f4

  3203. 	FXCPMADD	f8,  B2, A1, f8

  3204. 	FXCSMADD	f12, B2, A1, f12

  3205. 	LFPDUX	A1,  AO,  INC4

  3206. 

  3207. 	FXCPMADD	f1,  B1, A2, f1

  3208. 	FXCSMADD	f5,  B1, A2, f5

  3209. 	LFPDUX	B1,  BO,  INC4

  3210. 	FXCPMADD	f9,  B2, A2, f9

  3211. 	FXCSMADD	f13, B2, A2, f13

  3212. 	LFPDUX	A2,  AO2, INC4

  3213. 	LFPDUX	B2,  BO2, INC4

  3214. 	bdnz+	.L1026

  3215. 	.align 4

  3216. 

  3217. .L1027:

  3218. 	FXCPMADD	f0,  B1, A1, f0

  3219. 	FXCSMADD	f4,  B1, A1, f4

  3220. 	FXCPMADD	f8,  B2, A1, f8

  3221. 	FXCSMADD	f12, B2, A1, f12

  3222. 

  3223. 	FXCPMADD	f1,  B1, A2, f1

  3224. 	FXCSMADD	f5,  B1, A2, f5

  3225. 	FXCPMADD	f9,  B2, A2, f9

  3226. 	FXCSMADD	f13, B2, A2, f13

  3227. 	.align 4

  3228. 

  3229. .L1028:

  3230. #ifndef TRMMKERNEL

  3231. 	LFDUX	A1, CO1, INC

  3232. 	LFDUX	A2, CO1, INC2

  3233. 	LFDUX	A3, CO2, INC

  3234. 	LFDUX	A4, CO2, INC2

  3235. 

  3236. 	LFSDUX	A1, CO1, INCM1

  3237. 	LFSDUX	A2, CO1, INC2

  3238. 	LFSDUX	A3, CO2, INCM1

  3239. 	LFSDUX	A4, CO2, INC2

  3240. #endif

  3241. 

  3242. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  3243.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  3244. 	fpadd	f0, f0, f4

  3245. 	fpadd	f8, f8, f12

  3246. 	fpadd	f1, f1, f5

  3247. 	fpadd	f9, f9, f13

  3248. #else

  3249. 	fpsub	f0, f0, f4

  3250. 	fpsub	f8, f8, f12

  3251. 	fpsub	f1, f1, f5

  3252. 	fpsub	f9, f9, f13

  3253. #endif

  3254. 

  3255. #ifndef TRMMKERNEL

  3256. 	fxcpmadd A1,  f0, AP,  A1

  3257. 	fxcpmadd A2,  f1, AP,  A2

  3258. 	fxcpmadd A3,  f8, AP,  A3

  3259. 	fxcpmadd A4,  f9, AP,  A4

  3260. 

  3261. 	fxcxnpma f0,  f0, AP,  A1

  3262. 	fxcxnpma f1,  f1, AP,  A2

  3263. 	fxcxnpma f8,  f8, AP,  A3

  3264. 	fxcxnpma f9,  f9, AP,  A4

  3265. 

  3266. 	STFDUX	f0,  CO1, INCM3

  3267. 	STFSDUX	f0,  CO1, INC

  3268. 	STFDUX	f1,  CO1, INC

  3269. 	STFSDUX	f1,  CO1, INC

  3270. 

  3271. 	STFDUX	f8,  CO2, INCM3

  3272. 	STFSDUX	f8,  CO2, INC

  3273. 	STFDUX	f9,  CO2, INC

  3274. 	STFSDUX	f9,  CO2, INC

  3275. #else

  3276. 	fxcpmadd f12, f0, AP,  f30

  3277. 	fxcpmadd f13, f1, AP,  f30

  3278. 	fxcpmadd f14, f8, AP,  f30

  3279. 	fxcpmadd f15, f9, AP,  f30

  3280. 

  3281. 	fxcxnpma f0,  f0, AP,  f12

  3282. 	fxcxnpma f1,  f1, AP,  f13

  3283. 	fxcxnpma f8,  f8, AP,  f14

  3284. 	fxcxnpma f9,  f9, AP,  f15

  3285. 

  3286. 	STFDUX	f0,  CO1, INC

  3287. 	STFSDUX	f0,  CO1, INC

  3288. 	STFDUX	f1,  CO1, INC

  3289. 	STFSDUX	f1,  CO1, INC

  3290. 

  3291. 	STFDUX	f8,  CO2, INC

  3292. 	STFSDUX	f8,  CO2, INC

  3293. 	STFDUX	f9,  CO2, INC

  3294. 	STFSDUX	f9,  CO2, INC

  3295. #endif

  3296. 

  3297. #ifdef TRMMKERNEL

  3298. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  3299.     (!defined(LEFT) && !defined(TRANSA))

  3300. 	sub	TEMP, K, KK

  3301. #ifdef LEFT

  3302. 	addi	TEMP, TEMP, -2

  3303. #else

  3304. 	addi	TEMP, TEMP, -2

  3305. #endif

  3306. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  3307. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  3308. 	add	AO, AO, r0

  3309. 	add	BO, BO, TEMP

  3310. #endif

  3311. 

  3312. #ifdef LEFT

  3313. 	addi	KK, KK, 2

  3314. #endif

  3315. #endif

  3316. 

  3317. 	li	r0, FZERO

  3318. 	lfpsx	f0, SP, r0

  3319. 	.align 4

  3320. 

  3321. .L1030:

  3322. 	andi.	I, M,  1

  3323. 	beq	.L1049

  3324. 

  3325. #if defined(TRMMKERNEL)

  3326. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  3327. 	addi	AO2, AO,   2 * SIZE

  3328. 	fpmr	f1,  f0

  3329. 	addi	BO,  B,  - 4 * SIZE

  3330. 	fpmr	f2,  f0

  3331. 	addi	BO2, B,  - 2 * SIZE

  3332. 	fpmr	f3, f0

  3333. #else

  3334. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  3335. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  3336. 	add	AO, AO, TEMP

  3337. 	add	BO, B,  r0

  3338. 

  3339. 	addi	AO2, AO,   2 * SIZE

  3340. 	fpmr	f1,  f0

  3341. 	addi	BO,  BO,  - 4 * SIZE

  3342. 	fpmr	f2,  f0

  3343. 	addi	BO2, BO,    2 * SIZE

  3344. 	fpmr	f3, f0

  3345. #endif

  3346. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3347. 	sub	TEMP, K, KK

  3348. #elif defined(LEFT)

  3349. 	addi	TEMP, KK, 1

  3350. #else

  3351. 	addi	TEMP, KK, 2

  3352. #endif

  3353. 	srawi.	r0,  TEMP,  2

  3354. 	mtspr	CTR, r0

  3355. 	ble	.L1034

  3356. #else

  3357. 	addi	AO2, AO,   2 * SIZE

  3358. 	fpmr	f1,  f0

  3359. 	addi	BO,  B,  - 4 * SIZE

  3360. 	fpmr	f2,  f0

  3361. 	addi	BO2, B,  - 2 * SIZE

  3362. 	fpmr	f3, f0

  3363. 

  3364. 	srawi.	r0,  K,  2

  3365. 	mtspr	CTR, r0

  3366. 	ble	.L1034

  3367. #endif

  3368. 

  3369. 	LFPDUX	A1,  AO, INC4

  3370. 	LFPDUX	B1,  BO, INC4

  3371. 	LFPDUX	B2, BO2, INC4

  3372. 	LFPDUX	A2, AO2, INC4

  3373. 	LFPDUX	B3,  BO, INC4

  3374. 	LFPDUX	B4, BO2, INC4

  3375. 

  3376. 	LFPDUX	A3,  AO, INC4

  3377. 	LFPDUX	A5,  BO, INC4

  3378. 	LFPDUX	A6, BO2, INC4

  3379. 	LFPDUX	A4, AO2, INC4

  3380. 	LFPDUX	A7,  BO, INC4

  3381. 	LFPDUX	A8, BO2, INC4

  3382. 	bdz-	.L1033

  3383. 	.align 4

  3384. 

  3385. .L1032:

  3386. 	FXCPMADD	f0,  B1, A1, f0

  3387. 	FXCSMADD	f1,  B1, A1, f1

  3388. 	LFPDUX	B1,  BO, INC4

  3389. 	FXCPMADD	f2,  B2, A1, f2

  3390. 	FXCSMADD	f3,  B2, A1, f3

  3391. 	LFPDUX	B2, BO2, INC4

  3392. 	LFPDUX	A1,  AO, INC4

  3393. 

  3394. 	FXCPMADD	f0,  B3, A2, f0

  3395. 	FXCSMADD	f1,  B3, A2, f1

  3396. 	LFPDUX	B3,  BO, INC4

  3397. 	FXCPMADD	f2,  B4, A2, f2

  3398. 	FXCSMADD	f3,  B4, A2, f3

  3399. 	LFPDUX	B4, BO2, INC4

  3400. 	LFPDUX	A2, AO2, INC4

  3401. 

  3402. 	FXCPMADD	f0,  A5, A3, f0

  3403. 	FXCSMADD	f1,  A5, A3, f1

  3404. 	LFPDUX	A5,  BO, INC4

  3405. 	FXCPMADD	f2,  A6, A3, f2

  3406. 	FXCSMADD	f3,  A6, A3, f3

  3407. 	LFPDUX	A6, BO2, INC4

  3408. 	LFPDUX	A3,  AO, INC4

  3409. 

  3410. 	FXCPMADD	f0,  A7, A4, f0

  3411. 	FXCSMADD	f1,  A7, A4, f1

  3412. 	LFPDUX	A7,  BO, INC4

  3413. 	FXCPMADD	f2,  A8, A4, f2

  3414. 	FXCSMADD	f3,  A8, A4, f3

  3415. 	LFPDUX	A8, BO2, INC4

  3416. 	LFPDUX	A4, AO2, INC4

  3417. 	bdnz+	.L1032

  3418. 	.align 4

  3419. 

  3420. .L1033:

  3421. 	FXCPMADD	f0,  B1, A1, f0

  3422. 	FXCSMADD	f1,  B1, A1, f1

  3423. 	FXCPMADD	f2,  B2, A1, f2

  3424. 	FXCSMADD	f3,  B2, A1, f3

  3425. 

  3426. 	FXCPMADD	f0,  B3, A2, f0

  3427. 	FXCSMADD	f1,  B3, A2, f1

  3428. 	FXCPMADD	f2,  B4, A2, f2

  3429. 	FXCSMADD	f3,  B4, A2, f3

  3430. 

  3431. 	FXCPMADD	f0,  A5, A3, f0

  3432. 	FXCSMADD	f1,  A5, A3, f1

  3433. 	FXCPMADD	f2,  A6, A3, f2

  3434. 	FXCSMADD	f3,  A6, A3, f3

  3435. 

  3436. 	FXCPMADD	f0,  A7, A4, f0

  3437. 	FXCSMADD	f1,  A7, A4, f1

  3438. 	FXCPMADD	f2,  A8, A4, f2

  3439. 	FXCSMADD	f3,  A8, A4, f3

  3440. 	.align 4

  3441. 

  3442. .L1034:

  3443. 	li	r0, ALPHA

  3444. 	lfpdx	AP,  SP, r0

  3445. #ifdef TRMMKERNEL

  3446. 	li	r0, FZERO

  3447. 	lfpsx	f30, SP, r0

  3448. #endif

  3449. 

  3450. #if defined(TRMMKERNEL)

  3451. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3452. 	sub	TEMP, K, KK

  3453. #elif defined(LEFT)

  3454. 	addi	TEMP, KK, 1

  3455. #else

  3456. 	addi	TEMP, KK, 2

  3457. #endif

  3458. 	andi.	r0,  TEMP,  3

  3459. 	mtspr	CTR, r0

  3460. #else

  3461. 	andi.	r0,  K,  3

  3462. 	mtspr	CTR, r0

  3463. #endif

  3464. 	ble+	.L1038

  3465. 

  3466. 	LFPDX	A1,  AO,  INC4

  3467. 	LFPDUX	B1,  BO,  INC4

  3468. 	LFPDUX	B2,  BO2, INC4

  3469. 	add	AO, AO, INC2

  3470. 	bdz-	.L1037

  3471. 	.align 4

  3472. 

  3473. .L1036:

  3474. 	FXCPMADD	f0,  B1, A1, f0

  3475. 	FXCSMADD	f1,  B1, A1, f1

  3476. 	LFPDUX	B1,  BO,  INC4

  3477. 	FXCPMADD	f2,  B2, A1, f2

  3478. 	FXCSMADD	f3,  B2, A1, f3

  3479. 	LFPDX	A1,  AO,  INC4

  3480. 	LFPDUX	B2,  BO2, INC4

  3481. 	add	AO, AO, INC2

  3482. 	bdnz+	.L1036

  3483. 	.align 4

  3484. 

  3485. .L1037:

  3486. 	FXCPMADD	f0,  B1, A1, f0

  3487. 	FXCSMADD	f1,  B1, A1, f1

  3488. 	FXCPMADD	f2,  B2, A1, f2

  3489. 	FXCSMADD	f3,  B2, A1, f3

  3490. 	.align 4

  3491. 

  3492. .L1038:

  3493. #ifndef TRMMKERNEL

  3494. 	LFDUX	A1, CO1, INC

  3495. 	LFDUX	A2, CO2, INC

  3496. 	LFSDUX	A1, CO1, INC

  3497. 	LFSDUX	A2, CO2, INC

  3498. #endif

  3499. 

  3500. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  3501.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  3502. 	fpadd	f0, f0, f1

  3503. 	fpadd	f2, f2, f3

  3504. #else

  3505. 	fpsub	f0, f0, f1

  3506. 	fpsub	f2, f2, f3

  3507. #endif

  3508. 

  3509. #ifndef TRMMKERNEL

  3510. 	fxcpmadd A1,  f0, AP,  A1

  3511. 	fxcpmadd A2,  f2, AP,  A2

  3512. 	fxcxnpma f0,  f0, AP,  A1

  3513. 	fxcxnpma f2,  f2, AP,  A2

  3514. 

  3515. 	STFDUX	f0,  CO1, INCM1

  3516. 	STFSDUX	f0,  CO1, INC

  3517. 

  3518. 	STFDUX	f2,  CO2, INCM1

  3519. 	STFSDUX	f2,  CO2, INC

  3520. #else

  3521. 	fxcpmadd f12, f0, AP,  f30

  3522. 	fxcpmadd f13, f2, AP,  f30

  3523. 	fxcxnpma f0,  f0, AP,  f12

  3524. 	fxcxnpma f2,  f2, AP,  f13

  3525. 

  3526. 	STFDUX	f0,  CO1, INC

  3527. 	STFSDUX	f0,  CO1, INC

  3528. 

  3529. 	STFDUX	f2,  CO2, INC

  3530. 	STFSDUX	f2,  CO2, INC

  3531. #endif

  3532. 

  3533. #ifdef TRMMKERNEL

  3534. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  3535.     (!defined(LEFT) && !defined(TRANSA))

  3536. 	sub	TEMP, K, KK

  3537. #ifdef LEFT

  3538. 	addi	TEMP, TEMP, -1

  3539. #else

  3540. 	addi	TEMP, TEMP, -2

  3541. #endif

  3542. 	slwi	r0,   TEMP, 0 + ZBASE_SHIFT

  3543. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  3544. 	add	AO, AO, r0

  3545. 	add	BO, BO, TEMP

  3546. #endif

  3547. 

  3548. #ifdef LEFT

  3549. 	addi	KK, KK, 1

  3550. #endif

  3551. #endif

  3552. 

  3553. 	li	r0, FZERO

  3554. 	lfpsx	f0, SP, r0

  3555. 	.align 4

  3556. 

  3557. .L1049:

  3558. #if defined(TRMMKERNEL) && !defined(LEFT)

  3559. 	addi	KK, KK, 2

  3560. #endif

  3561. 

  3562. 	addi	B,  BO, 4 * SIZE

  3563. 

  3564. 	addic.	J, J, -1

  3565. 	bgt+	.L1010

  3566. 	.align 4

  3567. 

  3568. .L1050:

  3569. 	andi.	J, N,  1

  3570. 	beq	.L10999

  3571. 

  3572. 	mr	CO1, C

  3573. 

  3574. #if defined(TRMMKERNEL) &&  defined(LEFT)

  3575. 	mr	KK, OFFSET

  3576. #endif

  3577. 

  3578. 	addi	AO, A, -2 * SIZE

  3579. 

  3580. 	li	r0, FZERO

  3581. 	lfpsx	f0, SP, r0

  3582. 

  3583. 	srawi.	I, M,  2

  3584. 	ble	.L1060

  3585. 	.align 4

  3586. 

  3587. .L1051:

  3588. #if defined(TRMMKERNEL)

  3589. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  3590. 	fpmr	f4,  f0

  3591. 	addi	BO,  B,  - 2 * SIZE

  3592.  	fpmr	f1,  f0

  3593. 	fpmr	f5,  f0

  3594. 	fpmr	f2,  f0

  3595. 	fpmr	f6,  f0

  3596. #else

  3597. 	slwi	TEMP, KK, 2 + ZBASE_SHIFT

  3598. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  3599. 	add	AO, AO, TEMP

  3600. 	add	BO, B,  r0

  3601. 

  3602. 	fpmr	f4,  f0

  3603. 	addi	BO,  BO,  - 2 * SIZE

  3604.  	fpmr	f1,  f0

  3605. 	fpmr	f5,  f0

  3606. 	fpmr	f2,  f0

  3607. 	fpmr	f6,  f0

  3608. #endif

  3609. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3610. 	sub	TEMP, K, KK

  3611. #elif defined(LEFT)

  3612. 	addi	TEMP, KK, 4

  3613. #else

  3614. 	addi	TEMP, KK, 1

  3615. #endif

  3616. 	srawi.	r0,  TEMP,  2

  3617. 	fpmr	f3,  f0

  3618. 	mtspr	CTR, r0

  3619. 	fpmr	f7,  f0

  3620. 	ble	.L1054

  3621. #else

  3622. 	srawi.	r0,  K,  2

  3623. 	fpmr	f4,  f0

  3624. 	addi	BO,  B,  - 2 * SIZE

  3625.  	fpmr	f1,  f0

  3626. 	fpmr	f5,  f0

  3627. 	fpmr	f2,  f0

  3628. 	fpmr	f6,  f0

  3629. 	fpmr	f3,  f0

  3630. 	mtspr	CTR, r0

  3631. 	fpmr	f7,  f0

  3632. 	ble	.L1054

  3633. #endif

  3634. 

  3635. 	LFPDUX	B1,  BO,  INC2

  3636. 	LFPDUX	A1,  AO,  INC2

  3637. 	LFPDUX	A2,  AO,  INC2

  3638. 	LFPDUX	B2,  BO,  INC2

  3639. 	LFPDUX	A3,  AO,  INC2

  3640. 	LFPDUX	A4,  AO,  INC2

  3641. 

  3642. 	LFPDUX	B3,  BO,  INC2

  3643. 	LFPDUX	A5,  AO,  INC2

  3644. 	LFPDUX	A6,  AO,  INC2

  3645. 	LFPDUX	A7,  AO,  INC2

  3646. 	LFPDUX	A8,  AO,  INC2

  3647. 	bdz-	.L1053

  3648. 	.align 4

  3649. 

  3650. .L1052:

  3651. 	FXCPMADD	f0,  B1, A1, f0

  3652. 	LFPDUX	B4,  BO,  INC2

  3653. 	FXCSMADD	f4,  B1, A1, f4

  3654. 	LFPDUX	A1,  AO,  INC2

  3655. 	FXCPMADD	f1,  B1, A2, f1

  3656. 	nop

  3657. 	FXCSMADD	f5,  B1, A2, f5

  3658. 	LFPDUX	A2,  AO,  INC2

  3659. 

  3660. 	FXCPMADD	f2,  B1, A3, f2

  3661. 	nop

  3662. 	FXCSMADD	f6,  B1, A3, f6

  3663. 	LFPDUX	A3,  AO,  INC2

  3664. 	FXCPMADD	f3,  B1, A4, f3

  3665. 	nop

  3666. 	FXCSMADD	f7,  B1, A4, f7

  3667. 	LFPDUX	A4,  AO,  INC2

  3668. 

  3669. 	FXCPMADD	f0,  B2, A5, f0

  3670. 	LFPDUX	B1,  BO,  INC2

  3671. 	FXCSMADD	f4,  B2, A5, f4

  3672. 	LFPDUX	A5,  AO,  INC2

  3673. 	FXCPMADD	f1,  B2, A6, f1

  3674. 	nop

  3675. 	FXCSMADD	f5,  B2, A6, f5

  3676. 	LFPDUX	A6,  AO,  INC2

  3677. 

  3678. 	FXCPMADD	f2,  B2, A7, f2

  3679. 	nop

  3680. 	FXCSMADD	f6,  B2, A7, f6

  3681. 	LFPDUX	A7,  AO,  INC2

  3682. 	FXCPMADD	f3,  B2, A8, f3

  3683. 	nop

  3684. 	FXCSMADD	f7,  B2, A8, f7

  3685. 	LFPDUX	A8,  AO,  INC2

  3686. 

  3687. 	FXCPMADD	f0,  B3, A1, f0

  3688. 	LFPDUX	B2,  BO,  INC2

  3689. 	FXCSMADD	f4,  B3, A1, f4

  3690. 	LFPDUX	A1,  AO,  INC2

  3691. 	FXCPMADD	f1,  B3, A2, f1

  3692. 	nop

  3693. 	FXCSMADD	f5,  B3, A2, f5

  3694. 	LFPDUX	A2,  AO,  INC2

  3695. 

  3696. 	FXCPMADD	f2,  B3, A3, f2

  3697. 	nop

  3698. 	FXCSMADD	f6,  B3, A3, f6

  3699. 	LFPDUX	A3,  AO,  INC2

  3700. 	FXCPMADD	f3,  B3, A4, f3

  3701. 	nop

  3702. 	FXCSMADD	f7,  B3, A4, f7

  3703. 	LFPDUX	A4,  AO,  INC2

  3704. 

  3705. 	FXCPMADD	f0,  B4, A5, f0

  3706. 	LFPDUX	B3,  BO,  INC2

  3707. 	FXCSMADD	f4,  B4, A5, f4

  3708. 	LFPDUX	A5,  AO,  INC2

  3709. 	FXCPMADD	f1,  B4, A6, f1

  3710. 	nop

  3711. 	FXCSMADD	f5,  B4, A6, f5

  3712. 	LFPDUX	A6,  AO,  INC2

  3713. 

  3714. 	FXCPMADD	f2,  B4, A7, f2

  3715. 	nop

  3716. 	FXCSMADD	f6,  B4, A7, f6

  3717. 	LFPDUX	A7,  AO,  INC2

  3718. 	FXCPMADD	f3,  B4, A8, f3

  3719. 	nop

  3720. 	FXCSMADD	f7,  B4, A8, f7

  3721. 	LFPDUX	A8,  AO,  INC2

  3722. 	bdnz+	.L1052

  3723. 	.align 4

  3724. 

  3725. .L1053:

  3726. 	FXCPMADD	f0,  B1, A1, f0

  3727. 	LFPDUX	B4,  BO,  INC2

  3728. 	FXCSMADD	f4,  B1, A1, f4

  3729. 	LFPDUX	A1,  AO,  INC2

  3730. 	FXCPMADD	f1,  B1, A2, f1

  3731. 	nop

  3732. 	FXCSMADD	f5,  B1, A2, f5

  3733. 	LFPDUX	A2,  AO,  INC2

  3734. 

  3735. 	FXCPMADD	f2,  B1, A3, f2

  3736. 	nop

  3737. 	FXCSMADD	f6,  B1, A3, f6

  3738. 	LFPDUX	A3,  AO,  INC2

  3739. 	FXCPMADD	f3,  B1, A4, f3

  3740. 	nop

  3741. 	FXCSMADD	f7,  B1, A4, f7

  3742. 	LFPDUX	A4,  AO,  INC2

  3743. 

  3744. 	FXCPMADD	f0,  B2, A5, f0

  3745. 	nop

  3746. 	FXCSMADD	f4,  B2, A5, f4

  3747. 	LFPDUX	A5,  AO,  INC2

  3748. 	FXCPMADD	f1,  B2, A6, f1

  3749. 	nop

  3750. 	FXCSMADD	f5,  B2, A6, f5

  3751. 	LFPDUX	A6,  AO,  INC2

  3752. 

  3753. 	FXCPMADD	f2,  B2, A7, f2

  3754. 	nop

  3755. 	FXCSMADD	f6,  B2, A7, f6

  3756. 	LFPDUX	A7,  AO,  INC2

  3757. 	FXCPMADD	f3,  B2, A8, f3

  3758. 	nop

  3759. 	FXCSMADD	f7,  B2, A8, f7

  3760. 	LFPDUX	A8,  AO,  INC2

  3761. 

  3762. 	FXCPMADD	f0,  B3, A1, f0

  3763. 	FXCSMADD	f4,  B3, A1, f4

  3764. 	FXCPMADD	f1,  B3, A2, f1

  3765. 	FXCSMADD	f5,  B3, A2, f5

  3766. 

  3767. 	FXCPMADD	f2,  B3, A3, f2

  3768. 	FXCSMADD	f6,  B3, A3, f6

  3769. 	FXCPMADD	f3,  B3, A4, f3

  3770. 	FXCSMADD	f7,  B3, A4, f7

  3771. 

  3772. 	FXCPMADD	f0,  B4, A5, f0

  3773. 	FXCSMADD	f4,  B4, A5, f4

  3774. 	FXCPMADD	f1,  B4, A6, f1

  3775. 	FXCSMADD	f5,  B4, A6, f5

  3776. 

  3777. 	FXCPMADD	f2,  B4, A7, f2

  3778. 	FXCSMADD	f6,  B4, A7, f6

  3779. 	FXCPMADD	f3,  B4, A8, f3

  3780. 	FXCSMADD	f7,  B4, A8, f7

  3781. 	.align 4

  3782. 

  3783. .L1054:

  3784. 	li	r0, ALPHA

  3785. 	lfpdx	AP,  SP, r0

  3786. #ifdef TRMMKERNEL

  3787. 	li	r0, FZERO

  3788. 	lfpsx	f30, SP, r0

  3789. #endif

  3790. 

  3791. #if defined(TRMMKERNEL)

  3792. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3793. 	sub	TEMP, K, KK

  3794. #elif defined(LEFT)

  3795. 	addi	TEMP, KK, 4

  3796. #else

  3797. 	addi	TEMP, KK, 1

  3798. #endif

  3799. 	andi.	r0,  TEMP,  3

  3800. 	mtspr	CTR, r0

  3801. #else

  3802. 	andi.	r0,  K,  3

  3803. 	mtspr	CTR, r0

  3804. #endif

  3805. 	ble+	.L1058

  3806. 

  3807. 	LFPDUX	A1,  AO,  INC2

  3808. 	LFPDUX	B1,  BO,  INC2

  3809. 	LFPDUX	A2,  AO,  INC2

  3810. 	LFPDUX	A3,  AO,  INC2

  3811. 	LFPDUX	A4,  AO,  INC2

  3812. 	bdz-	.L1057

  3813. 	.align 4

  3814. 

  3815. .L1056:

  3816. 	FXCPMADD	f0,  B1, A1, f0

  3817. 	FXCSMADD	f4,  B1, A1, f4

  3818. 	LFPDUX	A1,  AO,  INC2

  3819. 	FXCPMADD	f1,  B1, A2, f1

  3820. 	FXCSMADD	f5,  B1, A2, f5

  3821. 	LFPDUX	A2,  AO,  INC2

  3822. 

  3823. 	FXCPMADD	f2,  B1, A3, f2

  3824. 	FXCSMADD	f6,  B1, A3, f6

  3825. 	LFPDUX	A3,  AO,  INC2

  3826. 	FXCPMADD	f3,  B1, A4, f3

  3827. 	FXCSMADD	f7,  B1, A4, f7

  3828. 	LFPDUX	A4,  AO,  INC2

  3829. 	LFPDUX	B1,  BO,  INC2

  3830. 	bdnz+	.L1056

  3831. 	.align 4

  3832. 

  3833. .L1057:

  3834. 	FXCPMADD	f0,  B1, A1, f0

  3835. 	FXCSMADD	f4,  B1, A1, f4

  3836. 	FXCPMADD	f1,  B1, A2, f1

  3837. 	FXCSMADD	f5,  B1, A2, f5

  3838. 

  3839. 	FXCPMADD	f2,  B1, A3, f2

  3840. 	FXCSMADD	f6,  B1, A3, f6

  3841. 	FXCPMADD	f3,  B1, A4, f3

  3842. 	FXCSMADD	f7,  B1, A4, f7

  3843. 	.align 4

  3844. 

  3845. .L1058:

  3846. #ifndef TRMMKERNEL

  3847. 	LFDUX	A1, CO1, INC

  3848. 	LFDUX	A2, CO1, INC2

  3849. 	LFDUX	A3, CO1, INC2

  3850.    	LFDUX	A4, CO1, INC2

  3851. 

  3852. 	LFSDUX	A1, CO1, INCM5

  3853. 	LFSDUX	A2, CO1, INC2

  3854. 	LFSDUX	A3, CO1, INC2

  3855. 	LFSDUX	A4, CO1, INC2

  3856. #endif

  3857. 

  3858. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  3859.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  3860. 	fpadd	f0, f0, f4

  3861. 	fpadd	f1, f1, f5

  3862. 	fpadd	f2, f2, f6

  3863. 	fpadd	f3, f3, f7

  3864. #else

  3865. 	fpsub	f0, f0, f4

  3866. 	fpsub	f1, f1, f5

  3867. 	fpsub	f2, f2, f6

  3868. 	fpsub	f3, f3, f7

  3869. #endif

  3870. 

  3871. #ifndef TRMMKERNEL

  3872. 	fxcpmadd A1,  f0, AP,  A1

  3873. 	fxcpmadd A2,  f1, AP,  A2

  3874. 	fxcpmadd A3,  f2, AP,  A3

  3875. 	fxcpmadd A4,  f3, AP,  A4

  3876. 

  3877. 	fxcxnpma f0,  f0, AP,  A1

  3878. 	fxcxnpma f1,  f1, AP,  A2

  3879. 	fxcxnpma f2,  f2, AP,  A3

  3880. 	fxcxnpma f3,  f3, AP,  A4

  3881. 

  3882. 	STFDUX	f0,  CO1, INCM7

  3883. 	STFSDUX	f0,  CO1, INC

  3884. 

  3885. 	STFDUX	f1,  CO1, INC

  3886. 	STFSDUX	f1,  CO1, INC

  3887. 

  3888. 	STFDUX	f2,  CO1, INC

  3889. 	STFSDUX	f2,  CO1, INC

  3890. 

  3891. 	STFDUX	f3,  CO1, INC

  3892. 	STFSDUX	f3,  CO1, INC

  3893. #else

  3894. 	fxcpmadd f12, f0, AP,  f30

  3895. 	fxcpmadd f13, f1, AP,  f30

  3896. 	fxcpmadd f14, f2, AP,  f30

  3897. 	fxcpmadd f15, f3, AP,  f30

  3898. 

  3899. 	fxcxnpma f0,  f0, AP,  f12

  3900. 	fxcxnpma f1,  f1, AP,  f13

  3901. 	fxcxnpma f2,  f2, AP,  f14

  3902. 	fxcxnpma f3,  f3, AP,  f15

  3903. 

  3904. 	STFDUX	f0,  CO1, INC

  3905. 	STFSDUX	f0,  CO1, INC

  3906. 

  3907. 	STFDUX	f1,  CO1, INC

  3908. 	STFSDUX	f1,  CO1, INC

  3909. 

  3910. 	STFDUX	f2,  CO1, INC

  3911. 	STFSDUX	f2,  CO1, INC

  3912. 

  3913. 	STFDUX	f3,  CO1, INC

  3914. 	STFSDUX	f3,  CO1, INC

  3915. #endif

  3916. 

  3917. 

  3918. #ifdef TRMMKERNEL

  3919. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  3920.     (!defined(LEFT) && !defined(TRANSA))

  3921. 	sub	TEMP, K, KK

  3922. #ifdef LEFT

  3923. 	addi	TEMP, TEMP, -4

  3924. #else

  3925. 	addi	TEMP, TEMP, -1

  3926. #endif

  3927. 	slwi	r0,   TEMP, 2 + ZBASE_SHIFT

  3928. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  3929. 	add	AO, AO, r0

  3930. 	add	BO, BO, TEMP

  3931. #endif

  3932. 

  3933. #ifdef LEFT

  3934. 	addi	KK, KK, 4

  3935. #endif

  3936. #endif

  3937. 

  3938. 	addic.	I, I, -1

  3939. 	li	r0, FZERO

  3940. 

  3941. 	lfpsx	f0, SP, r0

  3942. 	bgt+	.L1051

  3943. 	.align 4

  3944. 

  3945. .L1060:

  3946. 	andi.	I, M,  2

  3947. 	beq	.L1070

  3948. 

  3949. #if defined(TRMMKERNEL)

  3950. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  3951. 	addi	BO,  B,  - 2 * SIZE

  3952.  	fpmr	f1,  f0

  3953. #else

  3954. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  3955. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  3956. 	add	AO, AO, TEMP

  3957. 	add	BO, B,  r0

  3958. 

  3959. 	addi	BO,  BO,  - 2 * SIZE

  3960.  	fpmr	f1,  f0

  3961. #endif

  3962. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3963. 	sub	TEMP, K, KK

  3964. #elif defined(LEFT)

  3965. 	addi	TEMP, KK, 2

  3966. #else

  3967. 	addi	TEMP, KK, 1

  3968. #endif

  3969. 

  3970. 	srawi.	r0,  TEMP,  2

  3971. 	fpmr	f2,  f0

  3972. 	mtspr	CTR, r0

  3973. 	fpmr	f3,  f0

  3974. 	ble	.L1064

  3975. 

  3976. #else

  3977. 	srawi.	r0,  K,  2

  3978.  	fpmr	f1,  f0

  3979. 	addi	BO,  B,  - 2 * SIZE

  3980. 	fpmr	f2,  f0

  3981. 	mtspr	CTR, r0

  3982. 	fpmr	f3,  f0

  3983. 	ble	.L1064

  3984. #endif

  3985. 

  3986. 	LFPDUX	B1,  BO, INC2

  3987. 	LFPDUX	A1,  AO, INC2

  3988. 	LFPDUX	A2,  AO, INC2

  3989. 	LFPDUX	B2,  BO, INC2

  3990. 	LFPDUX	A3,  AO, INC2

  3991. 	LFPDUX	A4,  AO, INC2

  3992. 

  3993. 	LFPDUX	B3,  BO, INC2

  3994. 	LFPDUX	A5,  AO, INC2

  3995. 	LFPDUX	A6,  AO, INC2

  3996. 	LFPDUX	B4,  BO, INC2

  3997. 	LFPDUX	A7,  AO, INC2

  3998. 	LFPDUX	A8,  AO, INC2

  3999. 	bdz-	.L1063

  4000. 	.align 4

  4001. 

  4002. .L1062:

  4003. 	FXCPMADD	f0,  B1, A1, f0

  4004. 	FXCSMADD	f2,  B1, A1, f2

  4005. 	LFPDUX	A1,  AO, INC2

  4006. 	FXCPMADD	f1,  B1, A2, f1

  4007. 	FXCSMADD	f3,  B1, A2, f3

  4008. 	LFPDUX	A2,  AO, INC2

  4009. 	LFPDUX	B1,  BO, INC2

  4010. 

  4011. 	FXCPMADD	f0,  B2, A3, f0

  4012. 	FXCSMADD	f2,  B2, A3, f2

  4013. 	LFPDUX	A3,  AO, INC2

  4014. 	FXCPMADD	f1,  B2, A4, f1

  4015. 	FXCSMADD	f3,  B2, A4, f3

  4016. 	LFPDUX	A4,  AO, INC2

  4017. 	LFPDUX	B2,  BO, INC2

  4018. 

  4019. 	FXCPMADD	f0,  B3, A5, f0

  4020. 	FXCSMADD	f2,  B3, A5, f2

  4021. 	LFPDUX	A5,  AO, INC2

  4022. 	FXCPMADD	f1,  B3, A6, f1

  4023. 	FXCSMADD	f3,  B3, A6, f3

  4024. 	LFPDUX	A6,  AO, INC2

  4025. 	LFPDUX	B3,  BO, INC2

  4026. 

  4027. 	FXCPMADD	f0,  B4, A7, f0

  4028. 	FXCSMADD	f2,  B4, A7, f2

  4029. 	LFPDUX	A7,  AO, INC2

  4030. 	FXCPMADD	f1,  B4, A8, f1

  4031. 	FXCSMADD	f3,  B4, A8, f3

  4032. 	LFPDUX	A8,  AO, INC2

  4033. 	LFPDUX	B4,  BO, INC2

  4034. 	bdnz+	.L1062

  4035. 	.align 4

  4036. 

  4037. .L1063:

  4038. 	FXCPMADD	f0,  B1, A1, f0

  4039. 	FXCSMADD	f2,  B1, A1, f2

  4040. 	FXCPMADD	f1,  B1, A2, f1

  4041. 	FXCSMADD	f3,  B1, A2, f3

  4042. 

  4043. 	FXCPMADD	f0,  B2, A3, f0

  4044. 	FXCSMADD	f2,  B2, A3, f2

  4045. 	FXCPMADD	f1,  B2, A4, f1

  4046. 	FXCSMADD	f3,  B2, A4, f3

  4047. 

  4048. 	FXCPMADD	f0,  B3, A5, f0

  4049. 	FXCSMADD	f2,  B3, A5, f2

  4050. 	FXCPMADD	f1,  B3, A6, f1

  4051. 	FXCSMADD	f3,  B3, A6, f3

  4052. 

  4053. 	FXCPMADD	f0,  B4, A7, f0

  4054. 	FXCSMADD	f2,  B4, A7, f2

  4055. 	FXCPMADD	f1,  B4, A8, f1

  4056. 	FXCSMADD	f3,  B4, A8, f3

  4057. 	.align 4

  4058. 

  4059. .L1064:

  4060. 	li	r0, ALPHA

  4061. 	lfpdx	AP,  SP, r0

  4062. #ifdef TRMMKERNEL

  4063. 	li	r0, FZERO

  4064. 	lfpsx	f30, SP, r0

  4065. #endif

  4066. 

  4067. #if defined(TRMMKERNEL)

  4068. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4069. 	sub	TEMP, K, KK

  4070. #elif defined(LEFT)

  4071. 	addi	TEMP, KK, 2

  4072. #else

  4073. 	addi	TEMP, KK, 1

  4074. #endif

  4075. 	andi.	r0,  TEMP,  3

  4076. 	mtspr	CTR, r0

  4077. #else

  4078. 	andi.	r0,  K,  3

  4079. 	mtspr	CTR, r0

  4080. #endif

  4081. 	ble+	.L1068

  4082. 

  4083. 	LFPDUX	A1,  AO,  INC2

  4084. 	LFPDUX	B1,  BO,  INC2

  4085. 	LFPDUX	A2,  AO,  INC2

  4086. 	bdz-	.L1067

  4087. 	.align 4

  4088. 

  4089. .L1066:

  4090. 	FXCPMADD	f0,  B1, A1, f0

  4091. 	FXCSMADD	f2,  B1, A1, f2

  4092. 	LFPDUX	A1,  AO,  INC2

  4093. 	FXCPMADD	f1,  B1, A2, f1

  4094. 	FXCSMADD	f3,  B1, A2, f3

  4095. 	LFPDUX	B1,  BO,  INC2

  4096. 	LFPDUX	A2,  AO,  INC2

  4097. 	bdnz+	.L1066

  4098. 	.align 4

  4099. 

  4100. .L1067:

  4101. 	FXCPMADD	f0,  B1, A1, f0

  4102. 	FXCSMADD	f2,  B1, A1, f2

  4103. 	FXCPMADD	f1,  B1, A2, f1

  4104. 	FXCSMADD	f3,  B1, A2, f3

  4105. 	.align 4

  4106. 

  4107. .L1068:

  4108. #ifndef TRMMKERNEL

  4109. 	LFDUX	A1, CO1, INC

  4110. 	LFDUX	A2, CO1, INC2

  4111. 	LFSDUX	A1, CO1, INCM1

  4112. 	LFSDUX	A2, CO1, INC2

  4113. #endif

  4114. 

  4115. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  4116.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  4117. 	fpadd	f0, f0, f2

  4118. 	fpadd	f1, f1, f3

  4119. #else

  4120. 	fpsub	f0, f0, f2

  4121. 	fpsub	f1, f1, f3

  4122. #endif

  4123. 

  4124. #ifndef TRMMKERNEL

  4125. 	fxcpmadd A1,  f0, AP,  A1

  4126. 	fxcpmadd A2,  f1, AP,  A2

  4127. 	fxcxnpma f0,  f0, AP,  A1

  4128. 	fxcxnpma f1,  f1, AP,  A2

  4129. 

  4130. 	STFDUX	f0,  CO1, INCM3

  4131. 	STFSDUX	f0,  CO1, INC

  4132. 

  4133. 	STFDUX	f1,  CO1, INC

  4134. 	STFSDUX	f1,  CO1, INC

  4135. #else

  4136. 	fxcpmadd f12, f0, AP,  f30

  4137. 	fxcpmadd f13, f1, AP,  f30

  4138. 	fxcxnpma f0,  f0, AP,  f12

  4139. 	fxcxnpma f1,  f1, AP,  f13

  4140. 

  4141. 	STFDUX	f0,  CO1, INC

  4142. 	STFSDUX	f0,  CO1, INC

  4143. 

  4144. 	STFDUX	f1,  CO1, INC

  4145. 	STFSDUX	f1,  CO1, INC

  4146. #endif

  4147. 

  4148. #ifdef TRMMKERNEL

  4149. #if ( defined(LEFT) &&  defined(TRANSA)) || \

  4150.     (!defined(LEFT) && !defined(TRANSA))

  4151. 	sub	TEMP, K, KK

  4152. #ifdef LEFT

  4153. 	addi	TEMP, TEMP, -2

  4154. #else

  4155. 	addi	TEMP, TEMP, -1

  4156. #endif

  4157. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  4158. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  4159. 	add	AO, AO, r0

  4160. 	add	BO, BO, TEMP

  4161. #endif

  4162. 

  4163. #ifdef LEFT

  4164. 	addi	KK, KK, 2

  4165. #endif

  4166. #endif

  4167. 

  4168. 	li	r0, FZERO

  4169. 	lfpsx	f0, SP, r0

  4170. 	.align 4

  4171. 

  4172. .L1070:

  4173. 	andi.	I, M,  1

  4174. 	beq	.L1089

  4175. 

  4176. #if defined(TRMMKERNEL)

  4177. #if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  4178. 	addi	BO,  B,  - 2 * SIZE

  4179. 	fpmr	f1,  f0

  4180. #else

  4181. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  4182. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  4183. 	add	AO, AO, TEMP

  4184. 	add	BO, B,  r0

  4185. 

  4186. 	addi	BO,  BO,  - 2 * SIZE

  4187. 	fpmr	f1,  f0

  4188. #endif

  4189. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4190. 	sub	TEMP, K, KK

  4191. #elif defined(LEFT)

  4192. 	addi	TEMP, KK, 1

  4193. #else

  4194. 	addi	TEMP, KK, 1

  4195. #endif

  4196. 	srawi.	r0,  TEMP,  3

  4197. 	fpmr	f2,  f0

  4198. 	mtspr	CTR, r0

  4199. 	fpmr	f3,  f0

  4200. 	ble	.L1074

  4201. #else

  4202. 	addi	BO,  B,  - 2 * SIZE

  4203. 	fpmr	f1,  f0

  4204. 	srawi.	r0,  K,  3

  4205. 	fpmr	f2,  f0

  4206. 	mtspr	CTR, r0

  4207. 	fpmr	f3,  f0

  4208. 	ble	.L1074

  4209. #endif

  4210. 

  4211. 	LFPDUX	A1,  AO, INC2

  4212. 	LFPDUX	B1,  BO, INC2

  4213. 	LFPDUX	A2,  AO, INC2

  4214. 	LFPDUX	B2,  BO, INC2

  4215. 	LFPDUX	A3,  AO, INC2

  4216. 	LFPDUX	B3,  BO, INC2

  4217. 	LFPDUX	A4,  AO, INC2

  4218. 	LFPDUX	B4,  BO, INC2

  4219. 

  4220. 	LFPDUX	A5,  AO, INC2

  4221. 	LFPDUX	B5,  BO, INC2

  4222. 	LFPDUX	A6,  AO, INC2

  4223. 	LFPDUX	B6,  BO, INC2

  4224. 	LFPDUX	A7,  AO, INC2

  4225. 	LFPDUX	A9,  BO, INC2

  4226. 	LFPDUX	A8,  AO, INC2

  4227. 	LFPDUX	A10, BO, INC2

  4228. 	bdz-	.L1073

  4229. 	.align 4

  4230. 

  4231. .L1072:

  4232. 	FXCPMADD	f0,  B1, A1, f0

  4233. 	FXCSMADD	f1,  B1, A1, f1

  4234. 	LFPDUX	A1,  AO, INC2

  4235. 	LFPDUX	B1,  BO, INC2

  4236. 	FXCPMADD	f2,  B2, A2, f2

  4237. 	FXCSMADD	f3,  B2, A2, f3

  4238. 	LFPDUX	A2,  AO, INC2

  4239. 	LFPDUX	B2,  BO, INC2

  4240. 

  4241. 	FXCPMADD	f0,  B3, A3, f0

  4242. 	FXCSMADD	f1,  B3, A3, f1

  4243. 	LFPDUX	A3,  AO, INC2

  4244. 	LFPDUX	B3,  BO, INC2

  4245. 	FXCPMADD	f2,  B4, A4, f2

  4246. 	FXCSMADD	f3,  B4, A4, f3

  4247. 	LFPDUX	A4,  AO, INC2

  4248. 	LFPDUX	B4,  BO, INC2

  4249. 

  4250. 	FXCPMADD	f0,  B5, A5, f0

  4251. 	FXCSMADD	f1,  B5, A5, f1

  4252. 	LFPDUX	A5,  AO, INC2

  4253. 	LFPDUX	B5,  BO, INC2

  4254. 	FXCPMADD	f2,  B6, A6, f2

  4255. 	FXCSMADD	f3,  B6, A6, f3

  4256. 	LFPDUX	A6,  AO, INC2

  4257. 	LFPDUX	B6,  BO, INC2

  4258. 

  4259. 	FXCPMADD	f0,  A9,  A7, f0

  4260. 	FXCSMADD	f1,  A9,  A7, f1

  4261. 	LFPDUX	A7,  AO, INC2

  4262. 	LFPDUX	A9,  BO, INC2

  4263. 	FXCPMADD	f2,  A10, A8, f2

  4264. 	FXCSMADD	f3,  A10, A8, f3

  4265. 	LFPDUX	A8,  AO, INC2

  4266. 	LFPDUX	A10, BO, INC2

  4267. 

  4268. 	bdnz+	.L1072

  4269. 	.align 4

  4270. 

  4271. .L1073:

  4272. 	FXCPMADD	f0,  B1, A1, f0

  4273. 	FXCSMADD	f1,  B1, A1, f1

  4274. 	FXCPMADD	f2,  B2, A2, f2

  4275. 	FXCSMADD	f3,  B2, A2, f3

  4276. 

  4277. 	FXCPMADD	f0,  B3, A3, f0

  4278. 	FXCSMADD	f1,  B3, A3, f1

  4279. 	FXCPMADD	f2,  B4, A4, f2

  4280. 	FXCSMADD	f3,  B4, A4, f3

  4281. 

  4282. 	FXCPMADD	f0,  B5, A5, f0

  4283. 	FXCSMADD	f1,  B5, A5, f1

  4284. 	FXCPMADD	f2,  B6, A6, f2

  4285. 	FXCSMADD	f3,  B6, A6, f3

  4286. 

  4287. 	FXCPMADD	f0,  A9,  A7, f0

  4288. 	FXCSMADD	f1,  A9,  A7, f1

  4289. 	FXCPMADD	f2,  A10, A8, f2

  4290. 	FXCSMADD	f3,  A10, A8, f3

  4291. 	.align 4

  4292. 

  4293. .L1074:

  4294. 	li	r0, ALPHA

  4295. 	lfpdx	AP,  SP, r0

  4296. #ifdef TRMMKERNEL

  4297. 	li	r0, FZERO

  4298. 	lfpsx	f30, SP, r0

  4299. #endif

  4300. 

  4301. #if defined(TRMMKERNEL)

  4302. #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4303. 	sub	TEMP, K, KK

  4304. #elif defined(LEFT)

  4305. 	addi	TEMP, KK, 1

  4306. #else

  4307. 	addi	TEMP, KK, 1

  4308. #endif

  4309. 	andi.	r0,  TEMP,  7

  4310. 	mtspr	CTR, r0

  4311. #else

  4312. 	andi.	r0,  K,  7

  4313. 	mtspr	CTR, r0

  4314. #endif

  4315. 	ble+	.L1078

  4316. 

  4317. 	LFPDUX	A1,  AO,  INC2

  4318. 	LFPDUX	B1,  BO,  INC2

  4319. 	bdz-	.L1077

  4320. 	.align 4

  4321. 

  4322. .L1076:

  4323. 	FXCPMADD	f0,  B1, A1, f0

  4324. 	FXCSMADD	f1,  B1, A1, f1

  4325. 	LFPDUX	A1,  AO,  INC2

  4326. 	LFPDUX	B1,  BO,  INC2

  4327. 	bdnz+	.L1076

  4328. 	.align 4

  4329. 

  4330. .L1077:

  4331. 	FXCPMADD	f0,  B1, A1, f0

  4332. 	FXCSMADD	f1,  B1, A1, f1

  4333. 	.align 4

  4334. 

  4335. .L1078:

  4336. #ifndef TRMMKERNEL

  4337. 	LFDUX	A1, CO1, INC

  4338. 	LFDUX	A2, CO1, INC

  4339. #endif

  4340. 

  4341. 	fpadd	f0, f0, f2

  4342. 	fpadd	f1, f1, f3

  4343. 

  4344. 	fsmfp	A1, A2

  4345. 

  4346. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  4347.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  4348. 	fpadd	f0, f0, f1

  4349. #else

  4350. 	fpsub	f0, f0, f1

  4351. #endif

  4352. 

  4353. #ifndef TRMMKERNEL

  4354. 	fxcpmadd A1,  f0, AP,  A1

  4355. 	fxcxnpma f0,  f0, AP,  A1

  4356. 

  4357. 	STFDUX	f0,  CO1, INCM1

  4358. 	STFSDUX	f0,  CO1, INC

  4359. #else

  4360. 	fxcpmadd f12, f0, AP,  f30

  4361. 	fxcxnpma f0,  f0, AP,  f12

  4362. 

  4363. 	STFDUX	f0,  CO1, INC

  4364. 	STFSDUX	f0,  CO1, INC

  4365. #endif

  4366. 

  4367. 	li	r0, FZERO

  4368. 	lfpsx	f0, SP, r0

  4369. 	.align 4

  4370. 

  4371. .L1089:

  4372. 	addi	B,  BO, 2 * SIZE

  4373. 	.align 4

  4374. 

  4375. .L10999:

  4376. 	addi	SP, SP, 20

  4377. 

  4378. 	lwzu	r14,   4(SP)

  4379. 	lwzu	r15,   4(SP)

  4380. 

  4381. 	lwzu	r16,   4(SP)

  4382. 	lwzu	r17,   4(SP)

  4383. 	lwzu	r18,   4(SP)

  4384. 	lwzu	r19,   4(SP)

  4385. 

  4386. 	lwzu	r20,   4(SP)

  4387. 	lwzu	r21,   4(SP)

  4388. 	lwzu	r22,   4(SP)

  4389. 	lwzu	r23,   4(SP)

  4390. 

  4391. 	lwzu	r24,   4(SP)

  4392. 	lwzu	r25,   4(SP)

  4393. 	lwzu	r26,   4(SP)

  4394. 	lwzu	r27,   4(SP)

  4395. 

  4396. 	lwzu	r28,   4(SP)

  4397. 	lwzu	r29,   4(SP)

  4398. 	lwzu	r30,   4(SP)

  4399. 	lwzu	r31,   4(SP)

  4400. 

  4401. 	subi	SP, SP, 12

  4402. 	li	r0, 16

  4403. 

  4404. 	lfpdux	f31, SP, r0

  4405. 	lfpdux	f30, SP, r0

  4406. 	lfpdux	f29, SP, r0

  4407. 	lfpdux	f28, SP, r0

  4408. 	lfpdux	f27, SP, r0

  4409. 	lfpdux	f26, SP, r0

  4410. 	lfpdux	f25, SP, r0

  4411. 	lfpdux	f24, SP, r0

  4412. 	lfpdux	f23, SP, r0

  4413. 	lfpdux	f22, SP, r0

  4414. 	lfpdux	f21, SP, r0

  4415. 	lfpdux	f20, SP, r0

  4416. 	lfpdux	f19, SP, r0

  4417. 	lfpdux	f18, SP, r0

  4418. 	lfpdux	f17, SP, r0

  4419. 	lfpdux	f16, SP, r0

  4420. 	lfpdux	f15, SP, r0

  4421. 	lfpdux	f14, SP, r0

  4422. 	addi	SP, SP, 16

  4423. 	blr

  4424. 	.align 4

  4425. 

  4426. 

  4427. 	EPILOGUE

  4428. #endif