OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 886cbaf4e4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '886cbaf4e4'
${ noResults }
OpenBLAS/kernel/power/trsm_kernel_hummer_RT.S

5697 lines
98 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 		

  42. #define ALPHA    0

  43. #define FZERO	 8

  44. 

  45. #define	M	r3

  46. #define	N	r4

  47. #define	K	r5

  48. 

  49. #ifdef linux

  50. #define A	r6

  51. #define	B	r7

  52. #define	C	r8

  53. #define	LDC	r9

  54. #define OFFSET	r10

  55. #endif

  56. 

  57. #define TEMP	r11

  58. #define AORIG	r12

  59. #define KK	r14

  60. #define INCM1	r15

  61. #define INCM4	r16

  62. #define INCM2	r17

  63. #define INC2	r19

  64. #define INC	r20

  65. #define INC4	r21

  66. 

  67. #define	I	r22

  68. #define J	r23

  69. #define AO	r24

  70. #define BO	r25

  71. #define AO2	r26

  72. #define	BO2	r27

  73. 	

  74. #define	CO1	r28

  75. #define CO2	r29

  76. #define	CO3	r30

  77. #define	CO4	r31

  78. 

  79. #ifndef NEEDPARAM

  80. 

  81. #define A1	f16

  82. #define A2	f17

  83. #define A3	f18

  84. #define A4	f19

  85. #define A5	f20

  86. #define A6	f21

  87. #define A7	f22

  88. #define A8	f23

  89. #define A9	f24

  90. #define A10	f25

  91. 

  92. #define B1	f26

  93. #define B2	f27

  94. #define B3	f28

  95. #define B4	f29

  96. #define B5	f30

  97. #define B6	f31

  98. 

  99. #define AP	B6

  100. 

  101. 

  102. 	PROLOGUE

  103. 	PROFCODE

  104. 

  105. 	li	r0, -16

  106. 

  107. 	stfpdux	f14, SP, r0

  108. 	stfpdux	f15, SP, r0

  109. 	stfpdux	f16, SP, r0

  110. 	stfpdux	f17, SP, r0

  111. 	stfpdux	f18, SP, r0

  112. 	stfpdux	f19, SP, r0

  113. 	stfpdux	f20, SP, r0

  114. 	stfpdux	f21, SP, r0

  115. 	stfpdux	f22, SP, r0

  116. 	stfpdux	f23, SP, r0

  117. 	stfpdux	f24, SP, r0

  118. 	stfpdux	f25, SP, r0

  119. 	stfpdux	f26, SP, r0

  120. 	stfpdux	f27, SP, r0

  121. 	stfpdux	f28, SP, r0

  122. 	stfpdux	f29, SP, r0

  123. 	stfpdux	f30, SP, r0

  124. 	stfpdux	f31, SP, r0

  125. 	

  126. 	stwu	r31,  -4(SP)

  127. 	stwu	r30,  -4(SP)

  128. 	stwu	r29,  -4(SP)

  129. 	stwu	r28,  -4(SP)

  130. 

  131. 	stwu	r27,  -4(SP)

  132. 	stwu	r26,  -4(SP)

  133. 	stwu	r25,  -4(SP)

  134. 	stwu	r24,  -4(SP)

  135. 

  136. 	stwu	r23,  -4(SP)

  137. 	stwu	r22,  -4(SP)

  138. 	stwu	r21,  -4(SP)

  139. 	stwu	r20,  -4(SP)

  140. 

  141. 	stwu	r19,  -4(SP)

  142. 	stwu	r18,  -4(SP)

  143. 	stwu	r17,  -4(SP)

  144. 	stwu	r16,  -4(SP)

  145. 

  146. 	stwu	r15,  -4(SP)

  147. 	stwu	r14,  -4(SP)	# dummy

  148. 

  149. 	li	r0,   0

  150. 

  151. 	stwu	r0,   -4(SP)

  152. 	stwu	r0,   -4(SP)

  153. 	stfdu	f1,   -8(SP)

  154. 

  155. 	slwi	LDC, LDC, BASE_SHIFT

  156. 

  157. 	cmpwi	cr0, M, 0

  158. 	ble	.L999

  159. 	cmpwi	cr0, N, 0

  160. 	ble	.L999

  161. 	cmpwi	cr0, K, 0

  162. 	ble	.L999

  163. 

  164. 	li	INC,    1 * SIZE

  165. 	li	INC2,   2 * SIZE

  166. 	li	INC4,   4 * SIZE

  167. 

  168. 	li	INCM1, -1 * SIZE

  169. 	li	INCM2, -2 * SIZE

  170. 	li	INCM4, -4 * SIZE

  171. 

  172. 	addi	C, C, - 1 * SIZE

  173. 

  174. #ifdef LN

  175. 	mullw	r0, M, K

  176. 	slwi	r0, r0, BASE_SHIFT

  177. 	add	A, A, r0

  178. 

  179. 	slwi	r0, M, BASE_SHIFT

  180. 	add	C, C, r0

  181. #endif

  182. 

  183. #ifdef RN

  184. 	neg	KK, OFFSET

  185. #endif

  186. 

  187. #ifdef RT

  188. 	mullw	r0, N, K

  189. 	slwi	r0, r0, BASE_SHIFT

  190. 	add	B, B, r0

  191. 

  192. 	mullw	r0, N, LDC

  193. 	add	C, C, r0

  194. 

  195. 	sub	KK, N, OFFSET

  196. #endif

  197. 

  198. 	andi.	J, N,  1

  199. 	beq	.L50

  200. 

  201. #ifdef RT

  202. 	slwi	r0, K, 0 + BASE_SHIFT

  203. 	sub	B, B, r0

  204. 

  205. 	sub	C, C, LDC

  206. #endif

  207. 

  208. 	mr	CO1, C

  209. 

  210. #ifdef LN

  211. 	add	KK, M, OFFSET

  212. #endif

  213. 

  214. #ifdef LT

  215. 	mr	KK, OFFSET

  216. #endif

  217. 

  218. #if defined(LN) || defined(RT)

  219. 	addi	AORIG, A, -2 * SIZE

  220. #else

  221. 	addi	AO, A, -2 * SIZE

  222. #endif

  223. #ifndef RT

  224. 	add	C,  CO1, LDC

  225. #endif

  226. 

  227. 	li	r0, FZERO

  228. 	lfpsx	f0, SP, r0

  229. 

  230. 	srawi.	I, M,  3

  231. 	ble	.L100

  232. 	.align 4

  233. 

  234. .L91:

  235. #if defined(LT) || defined(RN)

  236.  	fpmr	f1,  f0

  237. 	addi	BO,  B,  - 2 * SIZE

  238. 	fpmr	f2,  f0

  239. 	fpmr	f3,  f0

  240. 

  241. 	srawi.	r0,  KK,  2

  242. 	mtspr	CTR, r0

  243. 	ble	.L94

  244. #else

  245. 

  246. #ifdef LN

  247. 	slwi	r0,   K,  3 + BASE_SHIFT

  248. 	sub	AORIG, AORIG, r0

  249. #endif

  250. 

  251. 	slwi	r0  , KK, 3 + BASE_SHIFT

  252. 	slwi	TEMP, KK, 0 + BASE_SHIFT

  253. 	add	AO, AORIG, r0

  254. 	add	BO, B,     TEMP

  255. 

  256. 	sub	TEMP, K, KK

  257. 

  258.  	fpmr	f1,  f0

  259. 	addi	BO,  BO,  - 2 * SIZE

  260. 	fpmr	f2,  f0

  261. 	fpmr	f3,  f0

  262. 

  263. 	srawi.	r0,  TEMP,  2

  264. 	mtspr	CTR, r0

  265. 	ble	.L94

  266. #endif

  267. 

  268. 	LFPDUX	B1,  BO,  INC2

  269. 	LFPDUX	A1,  AO,  INC2

  270. 	LFPDUX	A2,  AO,  INC2

  271. 	LFPDUX	A3,  AO,  INC2

  272. 	LFPDUX	A4,  AO,  INC2

  273. 	LFPDUX	B2,  BO,  INC2

  274. 	LFPDUX	A5,  AO,  INC2

  275. 	LFPDUX	A6,  AO,  INC2

  276. 	LFPDUX	A7,  AO,  INC2

  277. 	LFPDUX	A8,  AO,  INC2

  278. 	bdz-	.L93

  279. 	.align 4

  280. 

  281. .L92:

  282. 	fxcpmadd	f0,  B1, A1, f0

  283. 	LFPDUX	A1,  AO,  INC2

  284. 	fxcpmadd	f1,  B1, A2, f1

  285. 	LFPDUX	A2,  AO,  INC2

  286. 	fxcpmadd	f2,  B1, A3, f2

  287. 	LFPDUX	A3,  AO,  INC2

  288. 	fxcpmadd	f3,  B1, A4, f3

  289. 	LFPDUX	A4,  AO,  INC2

  290. 

  291. 	fxcsmadd	f0,  B1, A5, f0

  292. 	LFPDUX	A5,  AO,  INC2

  293. 	fxcsmadd	f1,  B1, A6, f1

  294. 	LFPDUX	A6,  AO,  INC2

  295. 	fxcsmadd	f2,  B1, A7, f2

  296. 	LFPDUX	A7,  AO,  INC2

  297. 	fxcsmadd	f3,  B1, A8, f3

  298. 	LFPDUX	A8,  AO,  INC2

  299. 	LFPDUX	B1,  BO,  INC2

  300. 

  301. 	fxcpmadd	f0,  B2, A1, f0

  302. 	LFPDUX	A1,  AO,  INC2

  303. 	fxcpmadd	f1,  B2, A2, f1

  304. 	LFPDUX	A2,  AO,  INC2

  305. 	fxcpmadd	f2,  B2, A3, f2

  306. 	LFPDUX	A3,  AO,  INC2

  307. 	fxcpmadd	f3,  B2, A4, f3

  308. 	LFPDUX	A4,  AO,  INC2

  309. 

  310. 	fxcsmadd	f0,  B2, A5, f0

  311. 	LFPDUX	A5,  AO,  INC2

  312. 	fxcsmadd	f1,  B2, A6, f1

  313. 	LFPDUX	A6,  AO,  INC2

  314. 	fxcsmadd	f2,  B2, A7, f2

  315. 	LFPDUX	A7,  AO,  INC2

  316. 	fxcsmadd	f3,  B2, A8, f3

  317. 	LFPDUX	A8,  AO,  INC2

  318. 	LFPDUX	B2,  BO,  INC2

  319. 	bdnz+	.L92

  320. 	.align 4

  321. 

  322. .L93:

  323. 	fxcpmadd	f0,  B1, A1, f0

  324. 	LFPDUX	A1,  AO,  INC2

  325. 	fxcpmadd	f1,  B1, A2, f1

  326. 	LFPDUX	A2,  AO,  INC2

  327. 	fxcpmadd	f2,  B1, A3, f2

  328. 	LFPDUX	A3,  AO,  INC2

  329. 	fxcpmadd	f3,  B1, A4, f3

  330. 	LFPDUX	A4,  AO,  INC2

  331. 

  332. 	fxcsmadd	f0,  B1, A5, f0

  333. 	LFPDUX	A5,  AO,  INC2

  334. 	fxcsmadd	f1,  B1, A6, f1

  335. 	LFPDUX	A6,  AO,  INC2

  336. 	fxcsmadd	f2,  B1, A7, f2

  337. 	LFPDUX	A7,  AO,  INC2

  338. 	fxcsmadd	f3,  B1, A8, f3

  339. 	LFPDUX	A8,  AO,  INC2

  340. 

  341. 	fxcpmadd	f0,  B2, A1, f0

  342. 	fxcpmadd	f1,  B2, A2, f1

  343. 	fxcpmadd	f2,  B2, A3, f2

  344. 	fxcpmadd	f3,  B2, A4, f3

  345. 

  346. 	fxcsmadd	f0,  B2, A5, f0

  347. 	fxcsmadd	f1,  B2, A6, f1

  348. 	fxcsmadd	f2,  B2, A7, f2

  349. 	fxcsmadd	f3,  B2, A8, f3

  350. 	.align 4

  351. 

  352. .L94:

  353. #if defined(LT) || defined(RN)

  354. 	andi.	r0,  KK,  3

  355. 	mtspr	CTR, r0

  356. 	ble+	.L98

  357. #else

  358. 	andi.	r0, TEMP, 3

  359. 	mtspr	CTR, r0

  360. 	ble+	.L98

  361. #endif

  362. 

  363. 	LFDX	B1,  BO,  INC2

  364. 	LFPDUX	A1,  AO,  INC2

  365. 	LFPDUX	A2,  AO,  INC2

  366. 	LFPDUX	A3,  AO,  INC2

  367. 	LFPDUX	A4,  AO,  INC2

  368. 	add	BO, BO, INC

  369. 	bdz-	.L97

  370. 	.align 4

  371. 

  372. .L96:

  373. 	fxcpmadd	f0,  B1, A1, f0

  374. 	LFPDUX	A1,  AO,  INC2

  375. 	fxcpmadd	f1,  B1, A2, f1

  376. 	LFPDUX	A2,  AO,  INC2

  377. 	fxcpmadd	f2,  B1, A3, f2

  378. 	LFPDUX	A3,  AO,  INC2

  379. 	fxcpmadd	f3,  B1, A4, f3

  380. 	LFDX	B1,  BO,  INC2

  381. 	LFPDUX	A4,  AO,  INC2

  382. 	add	BO, BO, INC

  383. 	bdnz+	.L96

  384. 	.align 4

  385. 

  386. .L97:

  387. 	fxcpmadd	f0,  B1, A1, f0

  388. 	fxcpmadd	f1,  B1, A2, f1

  389. 	fxcpmadd	f2,  B1, A3, f2

  390. 	fxcpmadd	f3,  B1, A4, f3

  391. 	.align 4

  392. 

  393. .L98:

  394. #if defined(LN) || defined(RT)

  395. #ifdef LN

  396. 	subi	r0, KK, 8

  397. #else

  398. 	subi	r0, KK, 1

  399. #endif

  400. 	slwi	TEMP, r0, 3 + BASE_SHIFT

  401. 	slwi	r0,   r0, 0 + BASE_SHIFT

  402. 	add	AO, AORIG, TEMP

  403. 	add	BO,  B,     r0

  404. 	addi	BO,  BO, - 2 * SIZE

  405. #endif

  406. 

  407. #if defined(LN) || defined(LT)

  408. 	LFPDUX	f16, BO,  INC2

  409. 	LFPDUX	f17, BO,  INC2

  410. 	LFPDUX	f18, BO,  INC2

  411. 	LFPDUX	f19, BO,  INC2

  412. 

  413. 	subi	BO,  BO,   8 * SIZE

  414. 

  415. 	fpsub	f0,  f16,  f0

  416. 	fpsub	f1,  f17,  f1

  417. 	fpsub	f2,  f18,  f2

  418. 	fpsub	f3,  f19,  f3

  419. #else

  420. 	LFPDUX	f16, AO,  INC2

  421. 	LFPDUX	f17, AO,  INC2

  422. 	LFPDUX	f18, AO,  INC2

  423. 	LFPDUX	f19, AO,  INC2

  424. 

  425. 	subi	AO,  AO,   8 * SIZE

  426. 

  427. 	fpsub	f0,  f16,  f0

  428. 	fpsub	f1,  f17,  f1

  429. 	fpsub	f2,  f18,  f2

  430. 	fpsub	f3,  f19,  f3

  431. #endif

  432. 

  433. #ifdef LN

  434. 	fsmtp	f4, f0

  435. 	fsmtp	f5, f1

  436. 	fsmtp	f6, f2

  437. 	fsmtp	f7, f3

  438. 

  439. 	LFD	A1, (2 + 63) * SIZE(AO)

  440. 	LFD	A2, (2 + 62) * SIZE(AO)

  441. 	LFD	A3, (2 + 61) * SIZE(AO)

  442. 	LFD	A4, (2 + 60) * SIZE(AO)

  443. 	LFD	A5, (2 + 59) * SIZE(AO)

  444. 	LFD	A6, (2 + 58) * SIZE(AO)

  445. 	LFD	A7, (2 + 57) * SIZE(AO)

  446. 	LFD	A8, (2 + 56) * SIZE(AO)

  447. 

  448. 	fmul	f7, A1, f7

  449. 	fnmsub	f3, A2, f7, f3

  450. 	fnmsub	f6, A3, f7, f6

  451. 	fnmsub	f2, A4, f7, f2

  452. 	fnmsub	f5, A5, f7, f5

  453. 	fnmsub	f1, A6, f7, f1

  454. 	fnmsub	f4, A7, f7, f4

  455. 	fnmsub	f0, A8, f7, f0

  456. 

  457. 	LFD	A1, (2 + 54) * SIZE(AO)

  458. 	LFD	A2, (2 + 53) * SIZE(AO)

  459. 	LFD	A3, (2 + 52) * SIZE(AO)

  460. 	LFD	A4, (2 + 51) * SIZE(AO)

  461. 	LFD	A5, (2 + 50) * SIZE(AO)

  462. 	LFD	A6, (2 + 49) * SIZE(AO)

  463. 	LFD	A7, (2 + 48) * SIZE(AO)

  464. 

  465. 	fmul	f3, A1, f3

  466. 	fnmsub	f6, A2, f3, f6

  467. 	fnmsub	f2, A3, f3, f2

  468. 	fnmsub	f5, A4, f3, f5

  469. 	fnmsub	f1, A5, f3, f1

  470. 	fnmsub	f4, A6, f3, f4

  471. 	fnmsub	f0, A7, f3, f0

  472. 

  473. 	LFD	A1, (2 + 45) * SIZE(AO)

  474. 	LFD	A2, (2 + 44) * SIZE(AO)

  475. 	LFD	A3, (2 + 43) * SIZE(AO)

  476. 	LFD	A4, (2 + 42) * SIZE(AO)

  477. 	LFD	A5, (2 + 41) * SIZE(AO)

  478. 	LFD	A6, (2 + 40) * SIZE(AO)

  479. 

  480. 	fmul	f6, A1, f6

  481. 	fnmsub	f2, A2, f6, f2

  482. 	fnmsub	f5, A3, f6, f5

  483. 	fnmsub	f1, A4, f6, f1

  484. 	fnmsub	f4, A5, f6, f4

  485. 	fnmsub	f0, A6, f6, f0

  486. 

  487. 	LFD	A1, (2 + 36) * SIZE(AO)

  488. 	LFD	A2, (2 + 35) * SIZE(AO)

  489. 	LFD	A3, (2 + 34) * SIZE(AO)

  490. 	LFD	A4, (2 + 33) * SIZE(AO)

  491. 	LFD	A5, (2 + 32) * SIZE(AO)

  492. 

  493. 	fmul	f2, A1, f2

  494. 	fnmsub	f5, A2, f2, f5

  495. 	fnmsub	f1, A3, f2, f1

  496. 	fnmsub	f4, A4, f2, f4

  497. 	fnmsub	f0, A5, f2, f0

  498. 

  499. 	LFD	A1, (2 + 27) * SIZE(AO)

  500. 	LFD	A2, (2 + 26) * SIZE(AO)

  501. 	LFD	A3, (2 + 25) * SIZE(AO)

  502. 	LFD	A4, (2 + 24) * SIZE(AO)

  503. 

  504. 	fmul	f5, A1, f5

  505. 	fnmsub	f1, A2, f5, f1

  506. 	fnmsub	f4, A3, f5, f4

  507. 	fnmsub	f0, A4, f5, f0

  508. 

  509. 	LFD	A1, (2 + 18) * SIZE(AO)

  510. 	LFD	A2, (2 + 17) * SIZE(AO)

  511. 	LFD	A3, (2 + 16) * SIZE(AO)

  512. 

  513. 	fmul	f1, A1, f1

  514. 	fnmsub	f4, A2, f1, f4

  515. 	fnmsub	f0, A3, f1, f0

  516. 

  517. 	LFD	A1, (2 +  9) * SIZE(AO)

  518. 	LFD	A2, (2 +  8) * SIZE(AO)

  519. 

  520. 	fmul	f4, A1, f4

  521. 	fnmsub	f0, A2, f4, f0

  522. 

  523. 	LFD	A1, (2 +  0) * SIZE(AO)

  524. 

  525. 	fmul	f0, A1, f0

  526. 

  527. 	fsmfp	f0, f4

  528. 	fsmfp	f1, f5

  529. 	fsmfp	f2, f6

  530. 	fsmfp	f3, f7	

  531. #endif

  532. 

  533. #ifdef LT

  534. 	fsmtp	f4, f0

  535. 	fsmtp	f5, f1

  536. 	fsmtp	f6, f2

  537. 	fsmtp	f7, f3

  538. 

  539. 	LFD	A1, (2 +  0) * SIZE(AO)

  540. 	LFD	A2, (2 +  1) * SIZE(AO)

  541. 	LFD	A3, (2 +  2) * SIZE(AO)

  542. 	LFD	A4, (2 +  3) * SIZE(AO)

  543. 	LFD	A5, (2 +  4) * SIZE(AO)

  544. 	LFD	A6, (2 +  5) * SIZE(AO)

  545. 	LFD	A7, (2 +  6) * SIZE(AO)

  546. 	LFD	A8, (2 +  7) * SIZE(AO)

  547. 

  548. 	fmul	f0, A1, f0

  549. 	fnmsub	f4, A2, f0, f4

  550. 	fnmsub	f1, A3, f0, f1

  551. 	fnmsub	f5, A4, f0, f5

  552. 	fnmsub	f2, A5, f0, f2

  553. 	fnmsub	f6, A6, f0, f6

  554. 	fnmsub	f3, A7, f0, f3

  555. 	fnmsub	f7, A8, f0, f7

  556. 

  557. 	LFD	A1, (2 +  9) * SIZE(AO)

  558. 	LFD	A2, (2 + 10) * SIZE(AO)

  559. 	LFD	A3, (2 + 11) * SIZE(AO)

  560. 	LFD	A4, (2 + 12) * SIZE(AO)

  561. 	LFD	A5, (2 + 13) * SIZE(AO)

  562. 	LFD	A6, (2 + 14) * SIZE(AO)

  563. 	LFD	A7, (2 + 15) * SIZE(AO)

  564. 

  565. 	fmul	f4, A1, f4

  566. 	fnmsub	f1, A2, f4, f1

  567. 	fnmsub	f5, A3, f4, f5

  568. 	fnmsub	f2, A4, f4, f2

  569. 	fnmsub	f6, A5, f4, f6

  570. 	fnmsub	f3, A6, f4, f3

  571. 	fnmsub	f7, A7, f4, f7

  572. 

  573. 	LFD	A1, (2 + 18) * SIZE(AO)

  574. 	LFD	A2, (2 + 19) * SIZE(AO)

  575. 	LFD	A3, (2 + 20) * SIZE(AO)

  576. 	LFD	A4, (2 + 21) * SIZE(AO)

  577. 	LFD	A5, (2 + 22) * SIZE(AO)

  578. 	LFD	A6, (2 + 23) * SIZE(AO)

  579. 

  580. 	fmul	f1, A1, f1

  581. 	fnmsub	f5, A2, f1, f5

  582. 	fnmsub	f2, A3, f1, f2

  583. 	fnmsub	f6, A4, f1, f6

  584. 	fnmsub	f3, A5, f1, f3

  585. 	fnmsub	f7, A6, f1, f7

  586. 

  587. 	LFD	A1, (2 + 27) * SIZE(AO)

  588. 	LFD	A2, (2 + 28) * SIZE(AO)

  589. 	LFD	A3, (2 + 29) * SIZE(AO)

  590. 	LFD	A4, (2 + 30) * SIZE(AO)

  591. 	LFD	A5, (2 + 31) * SIZE(AO)

  592. 

  593. 	fmul	f5, A1, f5

  594. 	fnmsub	f2, A2, f5, f2

  595. 	fnmsub	f6, A3, f5, f6

  596. 	fnmsub	f3, A4, f5, f3

  597. 	fnmsub	f7, A5, f5, f7

  598. 

  599. 	LFD	A1, (2 + 36) * SIZE(AO)

  600. 	LFD	A2, (2 + 37) * SIZE(AO)

  601. 	LFD	A3, (2 + 38) * SIZE(AO)

  602. 	LFD	A4, (2 + 39) * SIZE(AO)

  603. 

  604. 	fmul	f2, A1, f2

  605. 	fnmsub	f6, A2, f2, f6

  606. 	fnmsub	f3, A3, f2, f3

  607. 	fnmsub	f7, A4, f2, f7

  608. 

  609. 	LFD	A1, (2 + 45) * SIZE(AO)

  610. 	LFD	A2, (2 + 46) * SIZE(AO)

  611. 	LFD	A3, (2 + 47) * SIZE(AO)

  612. 

  613. 	fmul	f6, A1, f6

  614. 	fnmsub	f3, A2, f6, f3

  615. 	fnmsub	f7, A3, f6, f7

  616. 

  617. 	LFD	A1, (2 + 54) * SIZE(AO)

  618. 	LFD	A2, (2 + 55) * SIZE(AO)

  619. 

  620. 	fmul	f3, A1, f3

  621. 	fnmsub	f7, A2, f3, f7

  622. 

  623. 	LFD	A1, (2 + 63) * SIZE(AO)

  624. 

  625. 	fmul	f7, A1, f7

  626. 

  627. 	fsmfp	f0, f4

  628. 	fsmfp	f1, f5

  629. 	fsmfp	f2, f6

  630. 	fsmfp	f3, f7	

  631. #endif

  632. 

  633. #ifdef RN

  634. 	LFPDX	A1,  BO,  INC2

  635. 

  636. 	fxpmul	f0,  A1,  f0

  637. 	fxpmul	f1,  A1,  f1

  638. 	fxpmul	f2,  A1,  f2

  639. 	fxpmul	f3,  A1,  f3

  640. #endif

  641. 

  642. #ifdef RT

  643. 	LFPDX	A1,  BO,  INC2

  644. 

  645. 	fxpmul	f0,  A1,  f0

  646. 	fxpmul	f1,  A1,  f1

  647. 	fxpmul	f2,  A1,  f2

  648. 	fxpmul	f3,  A1,  f3

  649. 

  650. #endif

  651. 

  652. #ifdef LN

  653. 	subi	CO1, CO1, 8 * SIZE

  654. #endif

  655. 

  656. #if defined(LN) || defined(LT)

  657. 	STFPDUX	f0,  BO,  INC2

  658. 	STFPDUX	f1,  BO,  INC2

  659. 	STFPDUX	f2,  BO,  INC2

  660. 	STFPDUX	f3,  BO,  INC2

  661. 

  662. 	subi	BO,  BO,   8 * SIZE

  663. 

  664. 	STFDUX	f0,  CO1, INC

  665. 	STFSDUX	f0,  CO1, INC

  666. 	STFDUX	f1,  CO1, INC

  667. 	STFSDUX	f1,  CO1, INC

  668. 	STFDUX	f2,  CO1, INC

  669. 	STFSDUX	f2,  CO1, INC

  670. 	STFDUX	f3,  CO1, INC

  671. 	STFSDUX	f3,  CO1, INC

  672. #else

  673. 	STFPDUX	f0,  AO,  INC2

  674. 	STFPDUX	f1,  AO,  INC2

  675. 	STFPDUX	f2,  AO,  INC2

  676. 	STFPDUX	f3,  AO,  INC2

  677. 

  678. 	subi	AO,  AO,   8 * SIZE

  679. 

  680. 	STFDUX	f0,  CO1, INC

  681. 	STFSDUX	f0,  CO1, INC

  682. 	STFDUX	f1,  CO1, INC

  683. 	STFSDUX	f1,  CO1, INC

  684. 	STFDUX	f2,  CO1, INC

  685. 	STFSDUX	f2,  CO1, INC

  686. 	STFDUX	f3,  CO1, INC

  687. 	STFSDUX	f3,  CO1, INC

  688. #endif

  689. 

  690. #ifdef LN

  691. 	subi	CO1, CO1, 8 * SIZE

  692. #endif

  693. 

  694. #ifdef RT

  695. 	slwi	r0, K, 3 + BASE_SHIFT

  696. 	add	AORIG, AORIG, r0

  697. #endif

  698. 

  699. #if defined(LT) || defined(RN)

  700. 	sub	TEMP, K, KK

  701. 	slwi	r0,   TEMP, 3 + BASE_SHIFT

  702. 	slwi	TEMP, TEMP, 0 + BASE_SHIFT

  703. 	add	AO, AO, r0

  704. 	add	BO, BO, TEMP

  705. #endif

  706. 

  707. #ifdef LT

  708. 	addi	KK, KK, 8

  709. #endif

  710. 

  711. #ifdef LN

  712. 	subi	KK, KK, 8

  713. #endif

  714. 

  715. 	addic.	I, I, -1

  716. 	li	r0, FZERO

  717. 

  718. 	lfpsx	f0, SP, r0

  719. 	bgt+	.L91

  720. 	.align 4

  721. 

  722. .L100:

  723. 	andi.	I, M,  4

  724. 	beq	.L110

  725. 

  726. #if defined(LT) || defined(RN)

  727. 	addi	BO,  B,  - 2 * SIZE

  728. 	fpmr	f1,  f0

  729. 	fpmr	f2,  f0

  730. 	fpmr	f3, f0

  731. 

  732. 	srawi.	r0,  KK,  3

  733. 	mtspr	CTR, r0

  734. 	ble	.L104

  735. #else

  736. 

  737. #ifdef LN

  738. 	slwi	r0,   K,  2 + BASE_SHIFT

  739. 	sub	AORIG, AORIG, r0

  740. #endif

  741. 

  742. 	slwi	r0  , KK, 2 + BASE_SHIFT

  743. 	slwi	TEMP, KK, 0 + BASE_SHIFT

  744. 	add	AO, AORIG, r0

  745. 	add	BO, B,     TEMP

  746. 

  747. 	sub	TEMP, K, KK

  748. 

  749. 	addi	BO,  BO,  - 2 * SIZE

  750. 	fpmr	f1,  f0

  751. 	fpmr	f2,  f0

  752. 	fpmr	f3, f0

  753. 

  754. 	srawi.	r0,  TEMP,  3

  755. 	mtspr	CTR, r0

  756. 	ble	.L104

  757. #endif

  758. 

  759. 	LFPDUX	B1,  BO,  INC2

  760. 	LFPDUX	A1,  AO,  INC2

  761. 	LFPDUX	A2,  AO,  INC2

  762. 	LFPDUX	A3,  AO,  INC2

  763. 	LFPDUX	A4,  AO,  INC2

  764. 	LFPDUX	B2,  BO,  INC2

  765. 	LFPDUX	A5,  AO,  INC2

  766. 	LFPDUX	A6,  AO,  INC2

  767. 	LFPDUX	A7,  AO,  INC2

  768. 	LFPDUX	A8,  AO,  INC2

  769. 	LFPDUX	B3,  BO,  INC2

  770. 	LFPDUX	B4,  BO,  INC2

  771. 

  772. 	bdz-	.L103

  773. 	.align 4

  774. 

  775. .L102:

  776. 	fxcpmadd	f0,  B1, A1, f0

  777. 	LFPDUX	A1,  AO,  INC2

  778. 	fxcpmadd	f1,  B1, A2, f1

  779. 	LFPDUX	A2,  AO,  INC2

  780. 	fxcsmadd	f2,  B1, A3, f2

  781. 	LFPDUX	A3,  AO,  INC2

  782. 	fxcsmadd	f3,  B1, A4, f3

  783. 	LFPDUX	A4,  AO,  INC2

  784. 	LFPDUX	B1,  BO,  INC2

  785. 

  786. 	fxcpmadd	f0,  B2, A5, f0

  787. 	LFPDUX	A5,  AO,  INC2

  788. 	fxcpmadd	f1,  B2, A6, f1

  789. 	LFPDUX	A6,  AO,  INC2

  790. 	fxcsmadd	f2,  B2, A7, f2

  791. 	LFPDUX	A7,  AO,  INC2

  792. 	fxcsmadd	f3,  B2, A8, f3

  793. 	LFPDUX	A8,  AO,  INC2

  794. 	LFPDUX	B2,  BO,  INC2

  795. 

  796. 	fxcpmadd	f0,  B3, A1, f0

  797. 	LFPDUX	A1,  AO,  INC2

  798. 	fxcpmadd	f1,  B3, A2, f1

  799. 	LFPDUX	A2,  AO,  INC2

  800. 	fxcsmadd	f2,  B3, A3, f2

  801. 	LFPDUX	A3,  AO,  INC2

  802. 	fxcsmadd	f3,  B3, A4, f3

  803. 	LFPDUX	A4,  AO,  INC2

  804. 	LFPDUX	B3,  BO,  INC2

  805. 

  806. 	fxcpmadd	f0,  B4, A5, f0

  807. 	LFPDUX	A5,  AO,  INC2

  808. 	fxcpmadd	f1,  B4, A6, f1

  809. 	LFPDUX	A6,  AO,  INC2

  810. 	fxcsmadd	f2,  B4, A7, f2

  811. 	LFPDUX	A7,  AO,  INC2

  812. 	fxcsmadd	f3,  B4, A8, f3

  813. 	LFPDUX	A8,  AO,  INC2

  814. 	LFPDUX	B4,  BO,  INC2

  815. 	bdnz+	.L102

  816. 	.align 4

  817. 

  818. .L103:

  819. 	fxcpmadd	f0,  B1, A1, f0

  820. 	LFPDUX	A1,  AO,  INC2

  821. 	fxcpmadd	f1,  B1, A2, f1

  822. 	LFPDUX	A2,  AO,  INC2

  823. 	fxcsmadd	f2,  B1, A3, f2

  824. 	LFPDUX	A3,  AO,  INC2

  825. 	fxcsmadd	f3,  B1, A4, f3

  826. 	LFPDUX	A4,  AO,  INC2

  827. 

  828. 	fxcpmadd	f0,  B2, A5, f0

  829. 	LFPDUX	A5,  AO,  INC2

  830. 	fxcpmadd	f1,  B2, A6, f1

  831. 	LFPDUX	A6,  AO,  INC2

  832. 	fxcsmadd	f2,  B2, A7, f2

  833. 	LFPDUX	A7,  AO,  INC2

  834. 	fxcsmadd	f3,  B2, A8, f3

  835. 	LFPDUX	A8,  AO,  INC2

  836. 

  837. 	fxcpmadd	f0,  B3, A1, f0

  838. 	fxcpmadd	f1,  B3, A2, f1

  839. 	fxcsmadd	f2,  B3, A3, f2

  840. 	fxcsmadd	f3,  B3, A4, f3

  841. 

  842. 	fxcpmadd	f0,  B4, A5, f0

  843. 	fxcpmadd	f1,  B4, A6, f1

  844. 	fxcsmadd	f2,  B4, A7, f2

  845. 	fxcsmadd	f3,  B4, A8, f3

  846. 	.align 4

  847. 

  848. .L104:

  849. #if defined(LT) || defined(RN)

  850. 	andi.	r0,  KK,  7

  851. 	mtspr	CTR, r0

  852. 	ble+	.L108

  853. #else

  854. 	andi.	r0, TEMP, 7

  855. 	mtspr	CTR, r0

  856. 	ble+	.L108

  857. #endif

  858. 

  859. 	LFPDUX	A1,  AO,  INC2

  860. 	LFDX	B1,  BO,  INC2

  861. 	LFPDUX	A2,  AO,  INC2

  862. 	add	BO, BO, INC

  863. 	bdz-	.L107

  864. 	.align 4

  865. 

  866. .L106:

  867. 	fxcpmadd	f0,  B1, A1, f0

  868. 	LFPDUX	A1,  AO,  INC2

  869. 	fxcpmadd	f1,  B1, A2, f1

  870. 	LFDX	B1,  BO,  INC2

  871. 	LFPDUX	A2,  AO,  INC2

  872. 	add	BO, BO, INC

  873. 	bdnz+	.L106

  874. 	.align 4

  875. 

  876. .L107:

  877. 	fxcpmadd	f0,  B1, A1, f0

  878. 	fxcpmadd	f1,  B1, A2, f1

  879. 	.align 4

  880. 

  881. .L108:

  882. 	fpadd	f0, f0, f2

  883. 	fpadd	f1, f1, f3

  884. 

  885. #if defined(LN) || defined(RT)

  886. #ifdef LN

  887. 	subi	r0, KK, 4

  888. #else

  889. 	subi	r0, KK, 1

  890. #endif

  891. 	slwi	TEMP, r0, 2 + BASE_SHIFT

  892. 	slwi	r0,   r0, 0 + BASE_SHIFT

  893. 	add	AO, AORIG, TEMP

  894. 	add	BO,  B,     r0

  895. 	addi	BO,  BO, - 2 * SIZE

  896. #endif

  897. 

  898. #if defined(LN) || defined(LT)

  899. 	LFPDUX	f16, BO,  INC2

  900. 	LFPDUX	f17, BO,  INC2

  901. 

  902. 	subi	BO,  BO,   4 * SIZE

  903. 

  904. 	fpsub	f0,  f16,  f0

  905. 	fpsub	f1,  f17,  f1

  906. #else

  907. 	LFPDUX	f16, AO,  INC2

  908. 	LFPDUX	f17, AO,  INC2

  909. 

  910. 	subi	AO,  AO,   4 * SIZE

  911. 

  912. 	fpsub	f0,  f16,  f0

  913. 	fpsub	f1,  f17,  f1

  914. #endif

  915. 

  916. #ifdef LN

  917. 	fsmtp	f4, f0

  918. 	fsmtp	f5, f1

  919. 

  920. 	LFD	A1, (2 + 15) * SIZE(AO)

  921. 	LFD	A2, (2 + 14) * SIZE(AO)

  922. 	LFD	A3, (2 + 13) * SIZE(AO)

  923. 	LFD	A4, (2 + 12) * SIZE(AO)

  924. 

  925. 	fmul	f5, A1, f5

  926. 	fnmsub	f1, A2, f5, f1

  927. 	fnmsub	f4, A3, f5, f4

  928. 	fnmsub	f0, A4, f5, f0

  929. 

  930. 	LFD	A1, (2 + 10) * SIZE(AO)

  931. 	LFD	A2, (2 +  9) * SIZE(AO)

  932. 	LFD	A3, (2 +  8) * SIZE(AO)

  933. 

  934. 	fmul	f1, A1, f1

  935. 	fnmsub	f4, A2, f1, f4

  936. 	fnmsub	f0, A3, f1, f0

  937. 

  938. 	LFD	A1, (2 +  5) * SIZE(AO)

  939. 	LFD	A2, (2 +  4) * SIZE(AO)

  940. 

  941. 	fmul	f4, A1, f4

  942. 	fnmsub	f0, A2, f4, f0

  943. 

  944. 	LFD	A1, (2 +  0) * SIZE(AO)

  945. 

  946. 	fmul	f0, A1, f0

  947. 

  948. 	fsmfp	f0, f4

  949. 	fsmfp	f1, f5

  950. #endif

  951. 

  952. #ifdef LT

  953. 	fsmtp	f4, f0

  954. 	fsmtp	f5, f1

  955. 

  956. 	LFD	A1, (2 +  0) * SIZE(AO)

  957. 	LFD	A2, (2 +  1) * SIZE(AO)

  958. 	LFD	A3, (2 +  2) * SIZE(AO)

  959. 	LFD	A4, (2 +  3) * SIZE(AO)

  960. 

  961. 	fmul	f0, A1, f0

  962. 	fnmsub	f4, A2, f0, f4

  963. 	fnmsub	f1, A3, f0, f1

  964. 	fnmsub	f5, A4, f0, f5

  965. 

  966. 	LFD	A1, (2 +  5) * SIZE(AO)

  967. 	LFD	A2, (2 +  6) * SIZE(AO)

  968. 	LFD	A3, (2 +  7) * SIZE(AO)

  969. 

  970. 	fmul	f4, A1, f4

  971. 	fnmsub	f1, A2, f4, f1

  972. 	fnmsub	f5, A3, f4, f5

  973. 

  974. 	LFD	A1, (2 + 10) * SIZE(AO)

  975. 	LFD	A2, (2 + 11) * SIZE(AO)

  976. 

  977. 	fmul	f1, A1, f1

  978. 	fnmsub	f5, A2, f1, f5

  979. 

  980. 	LFD	A1, (2 + 15) * SIZE(AO)

  981. 

  982. 	fmul	f5, A1, f5

  983. 

  984. 	fsmfp	f0, f4

  985. 	fsmfp	f1, f5

  986. #endif

  987. 

  988. #ifdef RN

  989. 	LFPDX	A1,  BO,  INC2

  990. 

  991. 	fxpmul	f0,  A1,  f0

  992. 	fxpmul	f1,  A1,  f1

  993. #endif

  994. 

  995. #ifdef RT

  996. 	LFPDX	A1,  BO,  INC2

  997. 

  998. 	fxpmul	f0,  A1,  f0

  999. 	fxpmul	f1,  A1,  f1

  1000. #endif

  1001. 

  1002. #ifdef LN

  1003. 	subi	CO1, CO1, 4 * SIZE

  1004. #endif

  1005. 

  1006. #if defined(LN) || defined(LT)

  1007. 	STFPDUX	f0,  BO,  INC2

  1008. 	STFPDUX	f1,  BO,  INC2

  1009. 

  1010. 	subi	BO,  BO,   4 * SIZE

  1011. 

  1012. 	STFDUX	f0,  CO1, INC

  1013. 	STFSDUX	f0,  CO1, INC

  1014. 	STFDUX	f1,  CO1, INC

  1015. 	STFSDUX	f1,  CO1, INC

  1016. #else

  1017. 	STFPDUX	f0,  AO,  INC2

  1018. 	STFPDUX	f1,  AO,  INC2

  1019. 

  1020. 	subi	AO,  AO,   4 * SIZE

  1021. 

  1022. 	STFDUX	f0,  CO1, INC

  1023. 	STFSDUX	f0,  CO1, INC

  1024. 	STFDUX	f1,  CO1, INC

  1025. 	STFSDUX	f1,  CO1, INC

  1026. #endif

  1027. 

  1028. #ifdef LN

  1029. 	subi	CO1, CO1, 4 * SIZE

  1030. #endif

  1031. 

  1032. #ifdef RT

  1033. 	slwi	r0, K, 2 + BASE_SHIFT

  1034. 	add	AORIG, AORIG, r0

  1035. #endif

  1036. 

  1037. #if defined(LT) || defined(RN)

  1038. 	sub	TEMP, K, KK

  1039. 	slwi	r0,   TEMP, 2 + BASE_SHIFT

  1040. 	slwi	TEMP, TEMP, 0 + BASE_SHIFT

  1041. 	add	AO, AO, r0

  1042. 	add	BO, BO, TEMP

  1043. #endif

  1044. 

  1045. #ifdef LT

  1046. 	addi	KK, KK, 4

  1047. #endif

  1048. 

  1049. #ifdef LN

  1050. 	subi	KK, KK, 4

  1051. #endif

  1052. 

  1053. 	li	r0, FZERO

  1054. 	lfpsx	f0, SP, r0

  1055. 	.align 4

  1056. 

  1057. .L110:

  1058. 	andi.	I, M,  2

  1059. 	beq	.L120

  1060. 

  1061. #if defined(LT) || defined(RN)

  1062. 	addi	BO,  B,  - 2 * SIZE

  1063. 	fpmr	f1,  f0

  1064. 	fpmr	f2,  f0

  1065. 	fpmr	f3,  f0

  1066. 

  1067. 	srawi.	r0,  KK,  3

  1068. 	mtspr	CTR, r0

  1069. 	ble	.L114

  1070. #else

  1071. 

  1072. #ifdef LN

  1073. 	slwi	r0,   K,  1 + BASE_SHIFT

  1074. 	sub	AORIG, AORIG, r0

  1075. #endif

  1076. 

  1077. 	slwi	r0  , KK, 1 + BASE_SHIFT

  1078. 	slwi	TEMP, KK, 0 + BASE_SHIFT

  1079. 	add	AO, AORIG, r0

  1080. 	add	BO, B,     TEMP

  1081. 

  1082. 	sub	TEMP, K, KK

  1083. 

  1084. 	addi	BO,  BO,  - 2 * SIZE

  1085. 	fpmr	f1,  f0

  1086. 	fpmr	f2,  f0

  1087. 	fpmr	f3,  f0

  1088. 

  1089. 	srawi.	r0, TEMP,  3

  1090. 	mtspr	CTR, r0

  1091. 	ble	.L114

  1092. #endif

  1093. 	

  1094. 	LFPDUX	A1,  AO,  INC2

  1095. 	LFPDUX	A2,  AO,  INC2

  1096. 	LFPDUX	B1,  BO,  INC2

  1097. 

  1098. 	LFPDUX	A3,  AO,  INC2

  1099. 	LFPDUX	A4,  AO,  INC2

  1100. 	LFPDUX	B2,  BO,  INC2

  1101. 

  1102. 	LFPDUX	A5,  AO,  INC2

  1103. 	LFPDUX	A6,  AO,  INC2

  1104. 	LFPDUX	B3,  BO,  INC2

  1105. 

  1106. 	LFPDUX	A7,  AO,  INC2

  1107. 	LFPDUX	A8,  AO,  INC2

  1108. 	LFPDUX	B4,  BO,  INC2

  1109. 	bdz-	.L113

  1110. 	.align 4

  1111. 

  1112. .L112:

  1113. 	fxcpmadd	f0,  B1, A1, f0

  1114. 	LFPDUX	A1,  AO,  INC2

  1115. 	fxcsmadd	f1,  B1, A2, f1

  1116. 	LFPDUX	A2,  AO,  INC2

  1117. 	LFPDUX	B1,  BO,  INC2

  1118. 	fxcpmadd	f2,  B2, A3, f2

  1119. 	LFPDUX	A3,  AO,  INC2

  1120. 	fxcsmadd	f3,  B2, A4, f3

  1121. 	LFPDUX	A4,  AO,  INC2

  1122. 	LFPDUX	B2,  BO,  INC2

  1123. 	fxcpmadd	f0,  B3, A5, f0

  1124. 	LFPDUX	A5,  AO,  INC2

  1125. 	fxcsmadd	f1,  B3, A6, f1

  1126. 	LFPDUX	A6,  AO,  INC2

  1127. 	LFPDUX	B3,  BO,  INC2

  1128. 	fxcpmadd	f2,  B4, A7, f2

  1129. 	LFPDUX	A7,  AO,  INC2

  1130. 	fxcsmadd	f3,  B4, A8, f3

  1131. 	LFPDUX	A8,  AO,  INC2

  1132. 	LFPDUX	B4,  BO,  INC2

  1133. 	bdnz+	.L112

  1134. 	.align 4

  1135. 

  1136. .L113:

  1137. 	fxcpmadd	f0,  B1, A1, f0

  1138. 	fxcsmadd	f1,  B1, A2, f1

  1139. 	fxcpmadd	f2,  B2, A3, f2

  1140. 	fxcsmadd	f3,  B2, A4, f3

  1141. 	fxcpmadd	f0,  B3, A5, f0

  1142. 	fxcsmadd	f1,  B3, A6, f1

  1143. 	fxcpmadd	f2,  B4, A7, f2

  1144. 	fxcsmadd	f3,  B4, A8, f3

  1145. 	.align 4

  1146. 

  1147. .L114:

  1148. #if defined(LT) || defined(RN)

  1149. 	andi.	r0,  KK,  7

  1150. 	mtspr	CTR, r0

  1151. 	ble+	.L118

  1152. #else

  1153. 	andi.	r0, TEMP, 7

  1154. 	mtspr	CTR, r0

  1155. 	ble+	.L118

  1156. #endif

  1157. 

  1158. 	LFPDUX	A1,  AO,  INC2

  1159. 	LFDX	B1,  BO,  INC2

  1160. 	add	BO, BO, INC

  1161. 	bdz-	.L117

  1162. 	.align 4

  1163. 

  1164. .L116:

  1165. 	fxcpmadd	f0,  B1, A1, f0

  1166. 	LFPDUX	A1,  AO,  INC2

  1167. 	LFDX	B1,  BO,  INC2

  1168. 	add	BO, BO, INC

  1169. 	bdnz+	.L116

  1170. 	.align 4

  1171. 

  1172. .L117:

  1173. 	fxcpmadd	f0,  B1, A1, f0

  1174. 	.align 4

  1175. 

  1176. .L118:

  1177. 	fpadd	f0, f0, f1

  1178. 	fpadd	f2, f3, f2

  1179. 	fpadd	f0, f0, f2

  1180. 

  1181. #if defined(LN) || defined(RT)

  1182. #ifdef LN

  1183. 	subi	r0, KK, 2

  1184. #else

  1185. 	subi	r0, KK, 1

  1186. #endif

  1187. 	slwi	TEMP, r0, 1 + BASE_SHIFT

  1188. 	slwi	r0,   r0, 0 + BASE_SHIFT

  1189. 	add	AO, AORIG, TEMP

  1190. 	add	BO,  B,     r0

  1191. 	addi	BO,  BO, - 2 * SIZE

  1192. #endif

  1193. 

  1194. #if defined(LN) || defined(LT)

  1195. 	LFPDX	f16, BO,  INC2

  1196. 

  1197. 	fpsub	f0,  f16,  f0

  1198. #else

  1199. 	LFPDX	f16, AO,  INC2

  1200. 

  1201. 	fpsub	f0,  f16,  f0

  1202. #endif

  1203. 

  1204. #ifdef LN

  1205. 	fsmtp	f4, f0

  1206. 

  1207. 	LFD	A1, (2 +  3) * SIZE(AO)

  1208. 	LFD	A2, (2 +  2) * SIZE(AO)

  1209. 	LFD	A3, (2 +  0) * SIZE(AO)

  1210. 

  1211. 	fmul	f4, A1, f4

  1212. 	fnmsub	f0, A2, f4, f0

  1213. 	fmul	f0, A3, f0

  1214. 	fsmfp	f0, f4

  1215. #endif

  1216. 

  1217. #ifdef LT

  1218. 	fsmtp	f4, f0

  1219. 

  1220. 	LFD	A1, (2 +  0) * SIZE(AO)

  1221. 	LFD	A2, (2 +  1) * SIZE(AO)

  1222. 	LFD	A3, (2 +  3) * SIZE(AO)

  1223. 

  1224. 	fmul	f0, A1, f0

  1225. 	fnmsub	f4, A2, f0, f4

  1226. 	fmul	f4, A3, f4

  1227. 

  1228. 	fsmfp	f0, f4

  1229. #endif

  1230. 

  1231. #ifdef RN

  1232. 	LFPDX	A1,  BO,  INC2

  1233. 

  1234. 	fxpmul	f0,  A1,  f0

  1235. #endif

  1236. 

  1237. #ifdef RT

  1238. 	LFPDX	A1,  BO,  INC2

  1239. 

  1240. 	fxpmul	f0,  A1,  f0

  1241. #endif

  1242. 

  1243. #ifdef LN

  1244. 	subi	CO1, CO1, 2 * SIZE

  1245. #endif

  1246. 

  1247. #if defined(LN) || defined(LT)

  1248. 	STFPDX	f0,  BO,  INC2

  1249. 

  1250. 	STFDUX	f0,  CO1, INC

  1251. 	STFSDUX	f0,  CO1, INC

  1252. #else

  1253. 	STFPDX	f0,  AO,  INC2

  1254. 

  1255. 	STFDUX	f0,  CO1, INC

  1256. 	STFSDUX	f0,  CO1, INC

  1257. #endif

  1258. 

  1259. #ifdef LN

  1260. 	subi	CO1, CO1, 2 * SIZE

  1261. #endif

  1262. 

  1263. #ifdef RT

  1264. 	slwi	r0, K, 1 + BASE_SHIFT

  1265. 	add	AORIG, AORIG, r0

  1266. #endif

  1267. 

  1268. #if defined(LT) || defined(RN)

  1269. 	sub	TEMP, K, KK

  1270. 	slwi	r0,   TEMP, 1 + BASE_SHIFT

  1271. 	slwi	TEMP, TEMP, 0 + BASE_SHIFT

  1272. 	add	AO, AO, r0

  1273. 	add	BO, BO, TEMP

  1274. #endif

  1275. 

  1276. #ifdef LT

  1277. 	addi	KK, KK, 2

  1278. #endif

  1279. 

  1280. #ifdef LN

  1281. 	subi	KK, KK, 2

  1282. #endif

  1283. 

  1284. 	li	r0, FZERO

  1285. 	lfpsx	f0, SP, r0

  1286. 	.align 4

  1287. 

  1288. .L120:

  1289. 	andi.	I, M,  1

  1290. 	beq	.L129

  1291. 

  1292. #if defined(LT) || defined(RN)

  1293. 	addi	BO,  B,  - 2 * SIZE

  1294. 	fpmr	f1,  f0

  1295. 	fpmr	f2,  f0

  1296. 	fpmr	f3,  f0

  1297. 

  1298. 	srawi.	r0,  KK,  3

  1299. 	mtspr	CTR, r0

  1300. 	ble	.L124

  1301. #else

  1302. 

  1303. #ifdef LN

  1304. 	slwi	r0,   K,  0 + BASE_SHIFT

  1305. 	sub	AORIG, AORIG, r0

  1306. #endif

  1307. 

  1308. 	slwi	r0  , KK, 0 + BASE_SHIFT

  1309. 	slwi	TEMP, KK, 0 + BASE_SHIFT

  1310. 	add	AO, AORIG, r0

  1311. 	add	BO, B,     TEMP

  1312. 

  1313. 	sub	TEMP, K, KK

  1314. 

  1315. 	addi	BO,  BO,  - 2 * SIZE

  1316. 	fpmr	f1,  f0

  1317. 	fpmr	f2,  f0

  1318. 	fpmr	f3,  f0

  1319. 

  1320. 	srawi.	r0,  TEMP,  3

  1321. 	mtspr	CTR, r0

  1322. 	ble	.L124

  1323. #endif

  1324. 

  1325. 	LFPDUX	A1,  AO,  INC2

  1326. 	LFPDUX	B1,  BO,  INC2

  1327. 	LFPDUX	A2,  AO,  INC2

  1328. 	LFPDUX	B2,  BO,  INC2

  1329. 	LFPDUX	A3,  AO,  INC2

  1330. 	LFPDUX	B3,  BO,  INC2

  1331. 	LFPDUX	A4,  AO,  INC2

  1332. 	LFPDUX	B4,  BO,  INC2

  1333. 	bdz-	.L123

  1334. 	.align 4

  1335. 

  1336. .L122:

  1337. 	fpmadd	f0,  A1, B1, f0

  1338. 	LFPDUX	A1,  AO,  INC2

  1339. 	LFPDUX	B1,  BO,  INC2

  1340. 	fpmadd	f1,  A2, B2, f1

  1341. 	LFPDUX	A2,  AO,  INC2

  1342. 	LFPDUX	B2,  BO,  INC2

  1343. 	fpmadd	f2,  A3, B3, f2

  1344. 	LFPDUX	A3,  AO,  INC2

  1345. 	LFPDUX	B3,  BO,  INC2

  1346. 	fpmadd	f3,  A4, B4, f3

  1347. 	LFPDUX	A4,  AO,  INC2

  1348. 	LFPDUX	B4,  BO,  INC2

  1349. 	bdnz+	.L122

  1350. 	.align 4

  1351. 

  1352. .L123:

  1353. 	fpmadd	f0,  A1, B1, f0

  1354. 	fpmadd	f1,  A2, B2, f1

  1355. 	fpmadd	f2,  A3, B3, f2

  1356. 	fpmadd	f3,  A4, B4, f3

  1357. 	.align 4

  1358. 

  1359. .L124:

  1360. #if defined(LT) || defined(RN)

  1361. 	andi.	r0,  KK,  7

  1362. 	mtspr	CTR, r0

  1363. 	ble+	.L128

  1364. #else

  1365. 	andi.	r0, TEMP, 7

  1366. 	mtspr	CTR, r0

  1367. 	ble+	.L128

  1368. #endif

  1369. 

  1370. 	LFDX	A1,  AO,  INC2

  1371. 	LFDX	B1,  BO,  INC2

  1372. 	add	AO, AO, INC

  1373. 	add	BO, BO, INC

  1374. 	bdz-	.L127

  1375. 	.align 4

  1376. 

  1377. .L126:

  1378. 	fmadd	f0,  A1, B1, f0

  1379. 	LFDX	A1,  AO,  INC2

  1380. 	LFDX	B1,  BO,  INC2

  1381. 	add	AO, AO, INC

  1382. 	add	BO, BO, INC

  1383. 	bdnz+	.L126

  1384. 	.align 4

  1385. 

  1386. .L127:

  1387. 	fmadd	f0,  A1, B1, f0

  1388. 	.align 4

  1389. 

  1390. .L128:

  1391. 	fpadd	f0, f0, f1

  1392. 	fpadd	f2, f2, f3

  1393. 	fpadd	f0, f0, f2

  1394. 	fsmtp	f1, f0

  1395. 

  1396. 	fadd	f0, f0, f1

  1397. 

  1398. #if defined(LN) || defined(RT)

  1399. #ifdef LN

  1400. 	subi	r0, KK, 1

  1401. #else

  1402. 	subi	r0, KK, 1

  1403. #endif

  1404. 	slwi	TEMP, r0, 0 + BASE_SHIFT

  1405. 	slwi	r0,   r0, 0 + BASE_SHIFT

  1406. 	add	AO, AORIG, TEMP

  1407. 	add	BO,  B,     r0

  1408. 	addi	BO,  BO, - 2 * SIZE

  1409. #endif

  1410. 

  1411. #if defined(LN) || defined(LT)

  1412. 	LFDX	f16, BO,  INC2

  1413. 

  1414. 	fsub	f0,  f16,  f0

  1415. #else

  1416. 	LFDX	f16, AO,  INC2

  1417. 

  1418. 	fsub	f0,  f16,  f0

  1419. #endif

  1420. 

  1421. #ifdef LN

  1422. 	LFD	A1, (2 +  0) * SIZE(AO)

  1423. 

  1424. 	fmul	f0, A1, f0

  1425. #endif

  1426. 

  1427. #ifdef LT

  1428. 	LFD	A1, (2 +  0) * SIZE(AO)

  1429. 

  1430. 	fmul	f0, A1, f0

  1431. #endif

  1432. 

  1433. #ifdef RN

  1434. 	LFDX	A1,  BO,  INC2

  1435. 

  1436. 	fmul	f0,  A1,  f0

  1437. #endif

  1438. 

  1439. #ifdef RT

  1440. 	LFDX	A1,  BO,  INC2

  1441. 

  1442. 	fmul	f0,  A1,  f0

  1443. #endif

  1444. 

  1445. #ifdef LN

  1446. 	subi	CO1, CO1, 1 * SIZE

  1447. #endif

  1448. 

  1449. #if defined(LN) || defined(LT)

  1450. 	STFDX	f0,  BO,  INC2

  1451. 

  1452. 	STFDUX	f0,  CO1, INC

  1453. #else

  1454. 	STFDX	f0,  AO,  INC2

  1455. 

  1456. 	STFDUX	f0,  CO1, INC

  1457. #endif

  1458. 

  1459. #ifdef LN

  1460. 	subi	CO1, CO1, 1 * SIZE

  1461. #endif

  1462. 

  1463. #ifdef RT

  1464. 	slwi	r0, K, 0 + BASE_SHIFT

  1465. 	add	AORIG, AORIG, r0

  1466. #endif

  1467. 

  1468. #if defined(LT) || defined(RN)

  1469. 	sub	TEMP, K, KK

  1470. 	slwi	r0,   TEMP, 0 + BASE_SHIFT

  1471. 	slwi	TEMP, TEMP, 0 + BASE_SHIFT

  1472. 	add	AO, AO, r0

  1473. 	add	BO, BO, TEMP

  1474. #endif

  1475. 

  1476. #ifdef LT

  1477. 	addi	KK, KK, 1

  1478. #endif

  1479. 

  1480. #ifdef LN

  1481. 	subi	KK, KK, 1

  1482. #endif

  1483. 	.align 4

  1484. 

  1485. .L129:

  1486. #ifdef LN

  1487. 	slwi	r0, K, 0 + BASE_SHIFT

  1488. 	add	B, B, r0

  1489. #endif

  1490. 

  1491. #if defined(LT) || defined(RN)

  1492. 	addi	B,  BO, 2 * SIZE

  1493. #endif

  1494. 

  1495. #ifdef RN

  1496. 	addi	KK, KK, 1

  1497. #endif

  1498. 

  1499. #ifdef RT

  1500. 	subi	KK, KK, 1

  1501. #endif

  1502. 	.align 4

  1503. 

  1504. .L50:

  1505. 	andi.	J, N,  2

  1506. 	beq	.L90

  1507. 

  1508. #ifdef RT

  1509. 	slwi	r0, K, 1 + BASE_SHIFT

  1510. 	sub	B, B, r0

  1511. 

  1512. 	slwi	r0, LDC, 1

  1513. 	sub	C, C, r0

  1514. #endif

  1515. 

  1516. 	mr	CO1, C

  1517. 	add	CO2, C,   LDC

  1518. 

  1519. #ifdef LN

  1520. 	add	KK, M, OFFSET

  1521. #endif

  1522. 

  1523. #ifdef LT

  1524. 	mr	KK, OFFSET

  1525. #endif

  1526. 

  1527. 

  1528. #if defined(LN) || defined(RT)

  1529. 	addi	AORIG, A, -2 * SIZE

  1530. #else

  1531. 	addi	AO, A, -2 * SIZE

  1532. #endif

  1533. #ifndef RT

  1534. 	add	C,  CO2, LDC

  1535. #endif

  1536. 

  1537. 	li	r0, FZERO

  1538. 	lfpsx	f0, SP, r0

  1539. 

  1540. 	srawi.	I, M,  3

  1541. 	ble	.L60

  1542. 	.align 4

  1543. 

  1544. .L51:

  1545. #if defined(LT) || defined(RN)

  1546. 	fpmr	f4,  f0

  1547. 	addi	BO,   B,  - 2 * SIZE

  1548.  	fpmr	f1,  f0

  1549. 	fpmr	f5,  f0

  1550. 	fpmr	f2,  f0

  1551. 	fpmr	f6,  f0

  1552. 

  1553. 	srawi.	r0,  KK,  2

  1554. 	fpmr	f3,  f0

  1555. 	mtspr	CTR, r0

  1556. 	fpmr	f7,  f0

  1557. 	ble	.L54

  1558. #else

  1559. 

  1560. #ifdef LN

  1561. 	slwi	r0,   K,  3 + BASE_SHIFT

  1562. 	sub	AORIG, AORIG, r0

  1563. #endif

  1564. 

  1565. 	slwi	r0  , KK, 3 + BASE_SHIFT

  1566. 	slwi	TEMP, KK, 1 + BASE_SHIFT

  1567. 	add	AO, AORIG, r0

  1568. 	add	BO, B,     TEMP

  1569. 

  1570. 	sub	TEMP, K, KK

  1571. 

  1572. 	fpmr	f4,  f0

  1573. 	addi	BO,  BO,  - 2 * SIZE

  1574.  	fpmr	f1,  f0

  1575. 	fpmr	f5,  f0

  1576. 	fpmr	f2,  f0

  1577. 	fpmr	f6,  f0

  1578. 

  1579. 	srawi.	r0,  TEMP,  2

  1580. 	fpmr	f3,  f0

  1581. 	mtspr	CTR, r0

  1582. 	fpmr	f7,  f0

  1583. 	ble	.L54

  1584. #endif

  1585. 

  1586. 	LFPDUX	B1,  BO,  INC2

  1587. 	LFPDUX	A1,  AO,  INC2

  1588. 	LFPDUX	A2,  AO,  INC2

  1589. 	LFPDUX	B2,  BO,  INC2

  1590. 	LFPDUX	A3,  AO,  INC2

  1591. 	LFPDUX	A4,  AO,  INC2

  1592. 

  1593. 	LFPDUX	B3,  BO,  INC2

  1594. 	LFPDUX	A5,  AO,  INC2

  1595. 	LFPDUX	A6,  AO,  INC2

  1596. 	LFPDUX	A7,  AO,  INC2

  1597. 	LFPDUX	A8,  AO,  INC2

  1598. 	bdz-	.L53

  1599. 	.align 4

  1600. 

  1601. .L52:

  1602. 	fxcpmadd	f0,  B1, A1, f0

  1603. 	LFPDUX	B4,  BO,  INC2

  1604. 	fxcsmadd	f4,  B1, A1, f4

  1605. 	LFPDUX	A1,  AO,  INC2

  1606. 	fxcpmadd	f1,  B1, A2, f1

  1607. 	nop

  1608. 	fxcsmadd	f5,  B1, A2, f5

  1609. 	LFPDUX	A2,  AO,  INC2

  1610. 

  1611. 	fxcpmadd	f2,  B1, A3, f2

  1612. 	nop

  1613. 	fxcsmadd	f6,  B1, A3, f6

  1614. 	LFPDUX	A3,  AO,  INC2

  1615. 	fxcpmadd	f3,  B1, A4, f3

  1616. 	nop

  1617. 	fxcsmadd	f7,  B1, A4, f7

  1618. 	LFPDUX	A4,  AO,  INC2

  1619. 

  1620. 	fxcpmadd	f0,  B2, A5, f0

  1621. 	LFPDUX	B1,  BO,  INC2

  1622. 	fxcsmadd	f4,  B2, A5, f4

  1623. 	LFPDUX	A5,  AO,  INC2

  1624. 	fxcpmadd	f1,  B2, A6, f1

  1625. 	nop

  1626. 	fxcsmadd	f5,  B2, A6, f5

  1627. 	LFPDUX	A6,  AO,  INC2

  1628. 

  1629. 	fxcpmadd	f2,  B2, A7, f2

  1630. 	nop

  1631. 	fxcsmadd	f6,  B2, A7, f6

  1632. 	LFPDUX	A7,  AO,  INC2

  1633. 	fxcpmadd	f3,  B2, A8, f3

  1634. 	nop

  1635. 	fxcsmadd	f7,  B2, A8, f7

  1636. 	LFPDUX	A8,  AO,  INC2

  1637. 

  1638. 	fxcpmadd	f0,  B3, A1, f0

  1639. 	LFPDUX	B2,  BO,  INC2

  1640. 	fxcsmadd	f4,  B3, A1, f4

  1641. 	LFPDUX	A1,  AO,  INC2

  1642. 	fxcpmadd	f1,  B3, A2, f1

  1643. 	nop

  1644. 	fxcsmadd	f5,  B3, A2, f5

  1645. 	LFPDUX	A2,  AO,  INC2

  1646. 

  1647. 	fxcpmadd	f2,  B3, A3, f2

  1648. 	nop

  1649. 	fxcsmadd	f6,  B3, A3, f6

  1650. 	LFPDUX	A3,  AO,  INC2

  1651. 	fxcpmadd	f3,  B3, A4, f3

  1652. 	nop

  1653. 	fxcsmadd	f7,  B3, A4, f7

  1654. 	LFPDUX	A4,  AO,  INC2

  1655. 

  1656. 	fxcpmadd	f0,  B4, A5, f0

  1657. 	LFPDUX	B3,  BO,  INC2

  1658. 	fxcsmadd	f4,  B4, A5, f4

  1659. 	LFPDUX	A5,  AO,  INC2

  1660. 	fxcpmadd	f1,  B4, A6, f1

  1661. 	nop

  1662. 	fxcsmadd	f5,  B4, A6, f5

  1663. 	LFPDUX	A6,  AO,  INC2

  1664. 

  1665. 	fxcpmadd	f2,  B4, A7, f2

  1666. 	nop

  1667. 	fxcsmadd	f6,  B4, A7, f6

  1668. 	LFPDUX	A7,  AO,  INC2

  1669. 	fxcpmadd	f3,  B4, A8, f3

  1670. 	nop

  1671. 	fxcsmadd	f7,  B4, A8, f7

  1672. 	LFPDUX	A8,  AO,  INC2

  1673. 	bdnz+	.L52

  1674. 	.align 4

  1675. 

  1676. .L53:

  1677. 	fxcpmadd	f0,  B1, A1, f0

  1678. 	LFPDUX	B4,  BO,  INC2

  1679. 	fxcsmadd	f4,  B1, A1, f4

  1680. 	LFPDUX	A1,  AO,  INC2

  1681. 	fxcpmadd	f1,  B1, A2, f1

  1682. 	nop

  1683. 	fxcsmadd	f5,  B1, A2, f5

  1684. 	LFPDUX	A2,  AO,  INC2

  1685. 

  1686. 	fxcpmadd	f2,  B1, A3, f2

  1687. 	nop

  1688. 	fxcsmadd	f6,  B1, A3, f6

  1689. 	LFPDUX	A3,  AO,  INC2

  1690. 	fxcpmadd	f3,  B1, A4, f3

  1691. 	nop

  1692. 	fxcsmadd	f7,  B1, A4, f7

  1693. 	LFPDUX	A4,  AO,  INC2

  1694. 

  1695. 	fxcpmadd	f0,  B2, A5, f0

  1696. 	nop

  1697. 	fxcsmadd	f4,  B2, A5, f4

  1698. 	LFPDUX	A5,  AO,  INC2

  1699. 	fxcpmadd	f1,  B2, A6, f1

  1700. 	nop

  1701. 	fxcsmadd	f5,  B2, A6, f5

  1702. 	LFPDUX	A6,  AO,  INC2

  1703. 

  1704. 	fxcpmadd	f2,  B2, A7, f2

  1705. 	nop

  1706. 	fxcsmadd	f6,  B2, A7, f6

  1707. 	LFPDUX	A7,  AO,  INC2

  1708. 	fxcpmadd	f3,  B2, A8, f3

  1709. 	nop

  1710. 	fxcsmadd	f7,  B2, A8, f7

  1711. 	LFPDUX	A8,  AO,  INC2

  1712. 

  1713. 	fxcpmadd	f0,  B3, A1, f0

  1714. 	fxcsmadd	f4,  B3, A1, f4

  1715. 	fxcpmadd	f1,  B3, A2, f1

  1716. 	fxcsmadd	f5,  B3, A2, f5

  1717. 

  1718. 	fxcpmadd	f2,  B3, A3, f2

  1719. 	fxcsmadd	f6,  B3, A3, f6

  1720. 	fxcpmadd	f3,  B3, A4, f3

  1721. 	fxcsmadd	f7,  B3, A4, f7

  1722. 

  1723. 	fxcpmadd	f0,  B4, A5, f0

  1724. 	fxcsmadd	f4,  B4, A5, f4

  1725. 	fxcpmadd	f1,  B4, A6, f1

  1726. 	fxcsmadd	f5,  B4, A6, f5

  1727. 

  1728. 	fxcpmadd	f2,  B4, A7, f2

  1729. 	fxcsmadd	f6,  B4, A7, f6

  1730. 	fxcpmadd	f3,  B4, A8, f3

  1731. 	fxcsmadd	f7,  B4, A8, f7

  1732. 	.align 4

  1733. 

  1734. .L54:

  1735. #if defined(LT) || defined(RN)

  1736. 	andi.	r0,  KK,  3

  1737. 	mtspr	CTR, r0

  1738. 	ble+	.L58

  1739. #else

  1740. 	andi.	r0, TEMP, 3

  1741. 	mtspr	CTR, r0

  1742. 	ble+	.L58

  1743. #endif

  1744. 

  1745. 	LFPDUX	A1,  AO,  INC2

  1746. 	LFPDUX	B1,  BO,  INC2

  1747. 	LFPDUX	A2,  AO,  INC2

  1748. 	LFPDUX	A3,  AO,  INC2

  1749. 	LFPDUX	A4,  AO,  INC2

  1750. 	bdz-	.L57

  1751. 	.align 4

  1752. 

  1753. .L56:

  1754. 	fxcpmadd	f0,  B1, A1, f0

  1755. 	fxcsmadd	f4,  B1, A1, f4

  1756. 	LFPDUX	A1,  AO,  INC2

  1757. 	fxcpmadd	f1,  B1, A2, f1

  1758. 	fxcsmadd	f5,  B1, A2, f5

  1759. 	LFPDUX	A2,  AO,  INC2

  1760. 

  1761. 	fxcpmadd	f2,  B1, A3, f2

  1762. 	fxcsmadd	f6,  B1, A3, f6

  1763. 	LFPDUX	A3,  AO,  INC2

  1764. 	fxcpmadd	f3,  B1, A4, f3

  1765. 	fxcsmadd	f7,  B1, A4, f7

  1766. 	LFPDUX	A4,  AO,  INC2

  1767. 	LFPDUX	B1,  BO,  INC2

  1768. 	bdnz+	.L56

  1769. 	.align 4

  1770. 

  1771. .L57:

  1772. 	fxcpmadd	f0,  B1, A1, f0

  1773. 	fxcsmadd	f4,  B1, A1, f4

  1774. 	fxcpmadd	f1,  B1, A2, f1

  1775. 	fxcsmadd	f5,  B1, A2, f5

  1776. 

  1777. 	fxcpmadd	f2,  B1, A3, f2

  1778. 	fxcsmadd	f6,  B1, A3, f6

  1779. 	fxcpmadd	f3,  B1, A4, f3

  1780. 	fxcsmadd	f7,  B1, A4, f7

  1781. 	.align 4

  1782. 

  1783. .L58:

  1784. #if defined(LN) || defined(RT)

  1785. #ifdef LN

  1786. 	subi	r0, KK, 8

  1787. #else

  1788. 	subi	r0, KK, 2

  1789. #endif

  1790. 	slwi	TEMP, r0, 3 + BASE_SHIFT

  1791. 	slwi	r0,   r0, 1 + BASE_SHIFT

  1792. 	add	AO, AORIG, TEMP

  1793. 	add	BO,  B,     r0

  1794. 	addi	BO,  BO, - 2 * SIZE

  1795. #endif

  1796. 

  1797. #if defined(LN) || defined(LT)

  1798. 	fpmr	f24, f0

  1799. 	fpmr	f25, f1

  1800. 	fpmr	f26, f2

  1801. 	fpmr	f27, f3

  1802. 

  1803. 	fsmfp	f0,  f4

  1804. 	fsmfp	f1,  f5

  1805. 	fsmfp	f2,  f6

  1806. 	fsmfp	f3,  f7

  1807. 

  1808. 	fsmtp	f4,  f24

  1809. 	fsmtp	f5,  f25

  1810. 	fsmtp	f6,  f26

  1811. 	fsmtp	f7,  f27

  1812. 

  1813. 	LFPDUX	f16, BO,  INC2

  1814. 	LFPDUX	f17, BO,  INC2

  1815. 	LFPDUX	f18, BO,  INC2

  1816. 	LFPDUX	f19, BO,  INC2

  1817. 

  1818.  	LFPDUX	f20, BO,  INC2

  1819. 	LFPDUX	f21, BO,  INC2

  1820. 	LFPDUX	f22, BO,  INC2

  1821. 	LFPDUX	f23, BO,  INC2

  1822. 

  1823. 	subi	BO,  BO,  16 * SIZE

  1824. 

  1825. 	fpsub	f0,  f16,  f0

  1826. 	fpsub	f4,  f17,  f4

  1827. 	fpsub	f1,  f18,  f1

  1828. 	fpsub	f5,  f19,  f5

  1829. 

  1830. 	fpsub	f2,  f20,  f2

  1831. 	fpsub	f6,  f21,  f6

  1832. 	fpsub	f3,  f22,  f3

  1833. 	fpsub	f7,  f23,  f7

  1834. 

  1835. #else

  1836. 	LFPDUX	f16, AO,  INC2

  1837. 	LFPDUX	f17, AO,  INC2

  1838. 	LFPDUX	f18, AO,  INC2

  1839. 	LFPDUX	f19, AO,  INC2

  1840. 

  1841. 	LFPDUX	f20, AO,  INC2

  1842. 	LFPDUX	f21, AO,  INC2

  1843. 	LFPDUX	f22, AO,  INC2

  1844. 	LFPDUX	f23, AO,  INC2

  1845. 

  1846. 	subi	AO,  AO,  16 * SIZE

  1847. 

  1848. 	fpsub	f0,  f16,  f0

  1849. 	fpsub	f1,  f17,  f1

  1850. 	fpsub	f2,  f18,  f2

  1851. 	fpsub	f3,  f19,  f3

  1852. 	fpsub	f4,  f20,  f4

  1853. 	fpsub	f5,  f21,  f5

  1854. 	fpsub	f6,  f22,  f6

  1855. 	fpsub	f7,  f23,  f7

  1856. #endif

  1857. 

  1858. #ifdef LN

  1859.        addi	AO,  AO,  66 * SIZE

  1860. 

  1861. 	LFPDUX	A1,  AO,  INCM2

  1862. 	LFPDUX	A2,  AO,  INCM2

  1863. 	LFPDUX	A3,  AO,  INCM2

  1864. 	LFPDUX	A4,  AO,  INCM2

  1865. 	LFPDUX	A5,  AO,  INCM2

  1866. 	LFPDUX	A6,  AO,  INCM2

  1867. 	LFPDUX	A7,  AO,  INCM2

  1868. 	LFPDUX	A8,  AO,  INCM2

  1869. 

  1870. 	fxsmul	   f7,  A1, f7

  1871. 	fxcpnmsub  f3,  A1, f7,  f3

  1872. 	fxcsnmsub  f6,  A2, f7,  f6

  1873. 	fxcpnmsub  f2,  A2, f7,  f2

  1874. 

  1875. 	fxcsnmsub  f5,  A3, f7,  f5

  1876. 	fxcpnmsub  f1,  A3, f7,  f1

  1877. 	fxcsnmsub  f4,  A4, f7,  f4

  1878. 	fxcpnmsub  f0,  A4, f7,  f0

  1879. 

  1880. 	fxpmul	   f3,  A5, f3

  1881. 	fxcsnmsub  f6,  A6, f3,  f6

  1882. 	fxcpnmsub  f2,  A6, f3,  f2

  1883. 

  1884. 	fxcsnmsub  f5,  A7, f3,  f5

  1885. 	fxcpnmsub  f1,  A7, f3,  f1

  1886. 	fxcsnmsub  f4,  A8, f3,  f4

  1887. 	fxcpnmsub  f0,  A8, f3,  f0

  1888. 

  1889. 	add	AO,  AO,  INCM2

  1890. 	LFPDUX	A1,  AO,  INCM2

  1891. 	LFPDUX	A2,  AO,  INCM2

  1892. 	LFPDUX	A3,  AO,  INCM2

  1893. 

  1894. 	add	AO,  AO,  INCM2

  1895. 	LFPDUX	A4,  AO,  INCM2

  1896. 	LFPDUX	A5,  AO,  INCM2

  1897. 	LFPDUX	A6,  AO,  INCM2

  1898. 

  1899. 	add	AO,  AO,  INCM2

  1900. 	add	AO,  AO,  INCM2

  1901. 	LFPDUX	A7,  AO,  INCM2

  1902. 	LFPDUX	A8,  AO,  INCM2

  1903. 

  1904. 	fxsmul	   f6,  A1, f6

  1905. 	fxcpnmsub  f2,  A1, f6,  f2

  1906. 	fxcsnmsub  f5,  A2, f6,  f5

  1907. 	fxcpnmsub  f1,  A2, f6,  f1

  1908. 	fxcsnmsub  f4,  A3, f6,  f4

  1909. 	fxcpnmsub  f0,  A3, f6,  f0

  1910. 

  1911. 	fxpmul	   f2,  A4, f2

  1912. 	fxcsnmsub  f5,  A5, f2,  f5

  1913. 	fxcpnmsub  f1,  A5, f2,  f1

  1914. 	fxcsnmsub  f4,  A6, f2,  f4

  1915. 	fxcpnmsub  f0,  A6, f2,  f0

  1916. 

  1917. 	fxsmul	   f5,  A7, f5

  1918. 	fxcpnmsub  f1,  A7, f5,  f1

  1919. 	fxcsnmsub  f4,  A8, f5,  f4

  1920. 	fxcpnmsub  f0,  A8, f5,  f0

  1921. 

  1922. 	add	AO,  AO,  INCM2

  1923. 	add	AO,  AO,  INCM2

  1924. 	LFPDUX	A1,  AO,  INCM2

  1925. 	LFPDUX	A2,  AO,  INCM2

  1926. 

  1927. 	subi	AO,  AO, 6 * SIZE

  1928. 	LFPDUX	A3,  AO,  INCM2

  1929. 	subi	AO,  AO, 6 * SIZE

  1930. 	LFPDUX	A4,  AO,  INCM2

  1931. 

  1932. 	addi	AO,  AO,  -2 * SIZE

  1933. 

  1934. 	fxpmul	   f1,  A1, f1

  1935. 	fxcsnmsub  f4,  A2, f1,  f4

  1936. 	fxcpnmsub  f0,  A2, f1,  f0

  1937. 

  1938. 	fxsmul	   f4,  A3, f4

  1939. 	fxcpnmsub  f0,  A3, f4,  f0

  1940. 

  1941. 	fxpmul	f0,  A4, f0

  1942. #endif

  1943. 

  1944. #ifdef LT

  1945. 	LFPDUX	A1,  AO,  INC2

  1946. 	LFPDUX	A2,  AO,  INC2

  1947. 	LFPDUX	A3,  AO,  INC2

  1948. 	LFPDUX	A4,  AO,  INC2

  1949. 

  1950. 	LFPDUX	A5,  AO,  INC2

  1951. 	LFPDUX	A6,  AO,  INC2

  1952. 	LFPDUX	A7,  AO,  INC2

  1953. 	LFPDUX	A8,  AO,  INC2

  1954. 

  1955. 	fxpmul	   f0,  A1,  f0

  1956. 	fxcsnmsub  f4,  A1, f0, f4

  1957. 	fxcpnmsub  f1,  A2, f0, f1

  1958. 	fxcsnmsub  f5,  A2, f0, f5

  1959. 	fxcpnmsub  f2,  A3, f0, f2

  1960. 	fxcsnmsub  f6,  A3, f0, f6

  1961. 	fxcpnmsub  f3,  A4, f0, f3

  1962. 	fxcsnmsub  f7,  A4, f0, f7

  1963. 

  1964. 	fxsmul	   f4,  A5,  f4

  1965. 	fxcpnmsub  f1,  A6, f4,  f1

  1966. 	fxcsnmsub  f5,  A6, f4,  f5

  1967. 	fxcpnmsub  f2,  A7, f4,  f2

  1968. 	fxcsnmsub  f6,  A7, f4,  f6

  1969. 	fxcpnmsub  f3,  A8, f4,  f3

  1970. 	fxcsnmsub  f7,  A8, f4,  f7

  1971. 

  1972. 	add	AO,  AO,  INC2

  1973. 	LFPDUX	A1,  AO,  INC2

  1974. 	LFPDUX	A2,  AO,  INC2

  1975. 	LFPDUX	A3,  AO,  INC2

  1976. 

  1977. 	add	AO,  AO,  INC2

  1978. 	LFPDUX	A4,  AO,  INC2

  1979. 	LFPDUX	A5,  AO,  INC2

  1980. 	LFPDUX	A6,  AO,  INC2

  1981. 

  1982. 	add	AO,  AO,  INC2

  1983. 	add	AO,  AO,  INC2

  1984. 	LFPDUX	A7,  AO,  INC2

  1985. 	LFPDUX	A8,  AO,  INC2

  1986. 

  1987. 	fxpmul	   f1,  A1,  f1

  1988. 	fxcsnmsub  f5,  A1, f1, f5

  1989. 	fxcpnmsub  f2,  A2, f1, f2

  1990. 	fxcsnmsub  f6,  A2, f1, f6

  1991. 	fxcpnmsub  f3,  A3, f1, f3

  1992. 	fxcsnmsub  f7,  A3, f1, f7

  1993. 

  1994. 	fxsmul	   f5,  A4,  f5

  1995. 	fxcpnmsub  f2,  A5, f5,  f2

  1996. 	fxcsnmsub  f6,  A5, f5,  f6

  1997. 	fxcpnmsub  f3,  A6, f5,  f3

  1998. 	fxcsnmsub  f7,  A6, f5,  f7

  1999. 

  2000. 	fxpmul	   f2,  A7,  f2

  2001. 	fxcsnmsub  f6,  A7, f2,  f6

  2002. 	fxcpnmsub  f3,  A8, f2,  f3

  2003. 	fxcsnmsub  f7,  A8, f2,  f7

  2004. 

  2005. 	add	AO,  AO,  INC2

  2006. 	add	AO,  AO,  INC2

  2007. 	LFPDUX	A1,  AO,  INC2

  2008. 	LFPDUX	A2,  AO,  INC2

  2009. 

  2010. 	addi	AO,  AO,  6 * SIZE

  2011. 	LFPDUX	A3,  AO,  INC2

  2012. 	addi	AO,  AO,  6 * SIZE

  2013. 	LFPDUX	A4,  AO,  INC2

  2014. 

  2015. 	subi	AO,  AO,  64 * SIZE

  2016. 

  2017. 	fxsmul	   f6,  A1,  f6

  2018. 	fxcpnmsub  f3,  A2, f6,  f3

  2019. 	fxcsnmsub  f7,  A2, f6,  f7

  2020. 

  2021. 	fxpmul	   f3,  A3,  f3

  2022. 	fxcsnmsub  f7,  A3, f3,  f7

  2023. 

  2024. 	fxsmul	   f7,  A4,  f7

  2025. #endif

  2026. 

  2027. #ifdef RN

  2028. 	LFPDUX	A1,  BO,  INC2

  2029. 	LFPDUX	A2,  BO,  INC2

  2030. 

  2031. 	subi	BO, BO, 4 * SIZE

  2032. 

  2033. 	fxpmul	f0,  A1,  f0

  2034. 	fxpmul	f1,  A1,  f1

  2035. 	fxpmul	f2,  A1,  f2

  2036. 	fxpmul	f3,  A1,  f3

  2037. 

  2038. 	fxcsnmsub    f4,  A1, f0, f4

  2039. 	fxcsnmsub    f5,  A1, f1, f5

  2040. 	fxcsnmsub    f6,  A1, f2, f6

  2041. 	fxcsnmsub    f7,  A1, f3, f7

  2042. 

  2043. 	fxsmul	f4,  A2,  f4

  2044. 	fxsmul	f5,  A2,  f5

  2045. 	fxsmul	f6,  A2,  f6

  2046. 	fxsmul	f7,  A2,  f7

  2047. #endif

  2048. 

  2049. #ifdef RT

  2050. 	LFPDUX	A2,  BO,  INC2

  2051. 	LFPDUX	A1,  BO,  INC2

  2052. 

  2053. 	subi	BO, BO, 4 * SIZE

  2054. 

  2055. 	fxsmul	f4,  A1,  f4

  2056. 	fxsmul	f5,  A1,  f5

  2057. 	fxsmul	f6,  A1,  f6

  2058. 	fxsmul	f7,  A1,  f7

  2059. 

  2060. 	fxcpnmsub    f0,  A1, f4,  f0

  2061. 	fxcpnmsub    f1,  A1, f5,  f1

  2062. 	fxcpnmsub    f2,  A1, f6,  f2

  2063. 	fxcpnmsub    f3,  A1, f7,  f3

  2064. 

  2065. 	fxpmul	f0,  A2,  f0

  2066. 	fxpmul	f1,  A2,  f1

  2067. 	fxpmul	f2,  A2,  f2

  2068. 	fxpmul	f3,  A2,  f3

  2069. 

  2070. #endif

  2071. 

  2072. #ifdef LN

  2073. 	subi	CO1, CO1, 8 * SIZE

  2074. 	subi	CO2, CO2, 8 * SIZE

  2075. #endif

  2076. 

  2077. #if defined(LN) || defined(LT)

  2078. 	STFPDUX	f0,  BO,  INC2

  2079. 	STFPDUX	f4,  BO,  INC2

  2080. 	STFPDUX	f1,  BO,  INC2

  2081. 	STFPDUX	f5,  BO,  INC2

  2082. 	STFPDUX	f2,  BO,  INC2

  2083. 	STFPDUX	f6,  BO,  INC2

  2084. 	STFPDUX	f3,  BO,  INC2

  2085. 	STFPDUX	f7,  BO,  INC2

  2086. 

  2087. 	subi	BO,  BO,  16 * SIZE

  2088. 

  2089. 	STFDUX	f0,  CO1, INC

  2090. 	STFDUX	f4,  CO1, INC

  2091. 	STFDUX	f1,  CO1, INC

  2092. 	STFDUX	f5,  CO1, INC

  2093. 	STFDUX	f2,  CO1, INC

  2094. 	STFDUX	f6,  CO1, INC

  2095. 	STFDUX	f3,  CO1, INC

  2096. 	STFDUX	f7,  CO1, INC

  2097. 

  2098. 	STFSDUX	f0,  CO2, INC

  2099. 	STFSDUX	f4,  CO2, INC

  2100. 	STFSDUX	f1,  CO2, INC

  2101. 	STFSDUX	f5,  CO2, INC

  2102. 	STFSDUX	f2,  CO2, INC

  2103. 	STFSDUX	f6,  CO2, INC

  2104. 	STFSDUX	f3,  CO2, INC

  2105. 	STFSDUX	f7,  CO2, INC

  2106. #else

  2107. 	STFPDUX	f0,  AO,  INC2

  2108. 	STFPDUX	f1,  AO,  INC2

  2109. 	STFPDUX	f2,  AO,  INC2

  2110. 	STFPDUX	f3,  AO,  INC2

  2111. 	STFPDUX	f4,  AO,  INC2

  2112. 	STFPDUX	f5,  AO,  INC2

  2113. 	STFPDUX	f6,  AO,  INC2

  2114. 	STFPDUX	f7,  AO,  INC2

  2115. 

  2116. 	subi	AO,  AO,  16 * SIZE

  2117. 

  2118. 	STFDUX	f0,  CO1, INC

  2119. 	STFSDUX	f0,  CO1, INC

  2120. 	STFDUX	f1,  CO1, INC

  2121. 	STFSDUX	f1,  CO1, INC

  2122. 	STFDUX	f2,  CO1, INC

  2123. 	STFSDUX	f2,  CO1, INC

  2124. 	STFDUX	f3,  CO1, INC

  2125. 	STFSDUX	f3,  CO1, INC

  2126. 

  2127. 	STFDUX	f4,  CO2, INC

  2128. 	STFSDUX	f4,  CO2, INC

  2129. 	STFDUX	f5,  CO2, INC

  2130. 	STFSDUX	f5,  CO2, INC

  2131. 	STFDUX	f6,  CO2, INC

  2132. 	STFSDUX	f6,  CO2, INC

  2133. 	STFDUX	f7,  CO2, INC

  2134. 	STFSDUX	f7,  CO2, INC

  2135. #endif

  2136. 

  2137. #ifdef LN

  2138. 	subi	CO1, CO1, 8 * SIZE

  2139. 	subi	CO2, CO2, 8 * SIZE

  2140. #endif

  2141. 

  2142. #ifdef RT

  2143. 	slwi	r0, K, 3 + BASE_SHIFT

  2144. 	add	AORIG, AORIG, r0

  2145. #endif

  2146. 

  2147. #if defined(LT) || defined(RN)

  2148. 	sub	TEMP, K, KK

  2149. 	slwi	r0,   TEMP, 3 + BASE_SHIFT

  2150. 	slwi	TEMP, TEMP, 1 + BASE_SHIFT

  2151. 	add	AO, AO, r0

  2152. 	add	BO, BO, TEMP

  2153. #endif

  2154. 

  2155. #ifdef LT

  2156. 	addi	KK, KK, 8

  2157. #endif

  2158. 

  2159. #ifdef LN

  2160. 	subi	KK, KK, 8

  2161. #endif

  2162. 

  2163. 	addic.	I, I, -1

  2164. 	li	r0, FZERO

  2165. 

  2166. 	lfpsx	f0, SP, r0

  2167. 	bgt+	.L51

  2168. 	.align 4

  2169. 

  2170. .L60:

  2171. 	andi.	I, M,  4

  2172. 	beq	.L70

  2173. 

  2174. #if defined(LT) || defined(RN)

  2175.  	fpmr	f1,  f0

  2176. 	addi	BO,  B,  - 2 * SIZE

  2177. 	fpmr	f2,  f0

  2178. 	fpmr	f3,  f0

  2179. 

  2180. 	srawi.	r0,  KK,  2

  2181. 	mtspr	CTR, r0

  2182. 	ble	.L64

  2183. #else

  2184. 

  2185. #ifdef LN

  2186. 	slwi	r0,   K,  2 + BASE_SHIFT

  2187. 	sub	AORIG, AORIG, r0

  2188. #endif

  2189. 

  2190. 	slwi	r0  , KK, 2 + BASE_SHIFT

  2191. 	slwi	TEMP, KK, 1 + BASE_SHIFT

  2192. 	add	AO, AORIG, r0

  2193. 	add	BO, B,     TEMP

  2194. 

  2195. 	sub	TEMP, K, KK

  2196. 

  2197.  	fpmr	f1,  f0

  2198. 	addi	BO,  BO,  - 2 * SIZE

  2199. 	fpmr	f2,  f0

  2200. 	fpmr	f3,  f0

  2201. 

  2202. 	srawi.	r0,  TEMP,  2

  2203. 	mtspr	CTR, r0

  2204. 	ble	.L64

  2205. #endif

  2206. 

  2207. 	LFPDUX	B1,  BO, INC2

  2208. 	LFPDUX	A1,  AO, INC2

  2209. 	LFPDUX	A2,  AO, INC2

  2210. 	LFPDUX	B2,  BO, INC2

  2211. 	LFPDUX	A3,  AO, INC2

  2212. 	LFPDUX	A4,  AO, INC2

  2213. 

  2214. 	LFPDUX	B3,  BO, INC2

  2215. 	LFPDUX	A5,  AO, INC2

  2216. 	LFPDUX	A6,  AO, INC2

  2217. 	LFPDUX	B4,  BO, INC2

  2218. 	LFPDUX	A7,  AO, INC2

  2219. 	LFPDUX	A8,  AO, INC2

  2220. 	bdz-	.L63

  2221. 	.align 4

  2222. 

  2223. .L62:

  2224. 	fxcpmadd	f0,  B1, A1, f0

  2225. 	fxcsmadd	f2,  B1, A1, f2

  2226. 	LFPDUX	A1,  AO, INC2

  2227. 	fxcpmadd	f1,  B1, A2, f1

  2228. 	fxcsmadd	f3,  B1, A2, f3

  2229. 	LFPDUX	A2,  AO, INC2

  2230. 	LFPDUX	B1,  BO, INC2

  2231. 

  2232. 	fxcpmadd	f0,  B2, A3, f0

  2233. 	fxcsmadd	f2,  B2, A3, f2

  2234. 	LFPDUX	A3,  AO, INC2

  2235. 	fxcpmadd	f1,  B2, A4, f1

  2236. 	fxcsmadd	f3,  B2, A4, f3

  2237. 	LFPDUX	A4,  AO, INC2

  2238. 	LFPDUX	B2,  BO, INC2

  2239. 

  2240. 	fxcpmadd	f0,  B3, A5, f0

  2241. 	fxcsmadd	f2,  B3, A5, f2

  2242. 	LFPDUX	A5,  AO, INC2

  2243. 	fxcpmadd	f1,  B3, A6, f1

  2244. 	fxcsmadd	f3,  B3, A6, f3

  2245. 	LFPDUX	A6,  AO, INC2

  2246. 	LFPDUX	B3,  BO, INC2

  2247. 

  2248. 	fxcpmadd	f0,  B4, A7, f0

  2249. 	fxcsmadd	f2,  B4, A7, f2

  2250. 	LFPDUX	A7,  AO, INC2

  2251. 	fxcpmadd	f1,  B4, A8, f1

  2252. 	fxcsmadd	f3,  B4, A8, f3

  2253. 	LFPDUX	A8,  AO, INC2

  2254. 	LFPDUX	B4,  BO, INC2

  2255. 	bdnz+	.L62

  2256. 	.align 4

  2257. 

  2258. .L63:

  2259. 	fxcpmadd	f0,  B1, A1, f0

  2260. 	fxcsmadd	f2,  B1, A1, f2

  2261. 	fxcpmadd	f1,  B1, A2, f1

  2262. 	fxcsmadd	f3,  B1, A2, f3

  2263. 

  2264. 	fxcpmadd	f0,  B2, A3, f0

  2265. 	fxcsmadd	f2,  B2, A3, f2

  2266. 	fxcpmadd	f1,  B2, A4, f1

  2267. 	fxcsmadd	f3,  B2, A4, f3

  2268. 

  2269. 	fxcpmadd	f0,  B3, A5, f0

  2270. 	fxcsmadd	f2,  B3, A5, f2

  2271. 	fxcpmadd	f1,  B3, A6, f1

  2272. 	fxcsmadd	f3,  B3, A6, f3

  2273. 

  2274. 	fxcpmadd	f0,  B4, A7, f0

  2275. 	fxcsmadd	f2,  B4, A7, f2

  2276. 	fxcpmadd	f1,  B4, A8, f1

  2277. 	fxcsmadd	f3,  B4, A8, f3

  2278. 	.align 4

  2279. 

  2280. .L64:

  2281. #if defined(LT) || defined(RN)

  2282. 	andi.	r0,  KK,  3

  2283. 	mtspr	CTR, r0

  2284. 	ble+	.L68

  2285. #else

  2286. 	andi.	r0, TEMP, 3

  2287. 	mtspr	CTR, r0

  2288. 	ble+	.L68

  2289. #endif

  2290. 

  2291. 	LFPDUX	A1,  AO,  INC2

  2292. 	LFPDUX	B1,  BO,  INC2

  2293. 	LFPDUX	A2,  AO,  INC2

  2294. 	bdz-	.L67

  2295. 	.align 4

  2296. 

  2297. .L66:

  2298. 	fxcpmadd	f0,  B1, A1, f0

  2299. 	fxcsmadd	f2,  B1, A1, f2

  2300. 	LFPDUX	A1,  AO,  INC2

  2301. 	fxcpmadd	f1,  B1, A2, f1

  2302. 	fxcsmadd	f3,  B1, A2, f3

  2303. 	LFPDUX	B1,  BO,  INC2

  2304. 	LFPDUX	A2,  AO,  INC2

  2305. 	bdnz+	.L66

  2306. 	.align 4

  2307. 

  2308. .L67:

  2309. 	fxcpmadd	f0,  B1, A1, f0

  2310. 	fxcsmadd	f2,  B1, A1, f2

  2311. 	fxcpmadd	f1,  B1, A2, f1

  2312. 	fxcsmadd	f3,  B1, A2, f3

  2313. 	.align 4

  2314. 

  2315. .L68:

  2316. #if defined(LN) || defined(RT)

  2317. #ifdef LN

  2318. 	subi	r0, KK, 4

  2319. #else

  2320. 	subi	r0, KK, 2

  2321. #endif

  2322. 	slwi	TEMP, r0, 2 + BASE_SHIFT

  2323. 	slwi	r0,   r0, 1 + BASE_SHIFT

  2324. 	add	AO, AORIG, TEMP

  2325. 	add	BO,  B,     r0

  2326. 	addi	BO,  BO, - 2 * SIZE

  2327. #endif

  2328. 

  2329. #if defined(LN) || defined(LT)

  2330. 	fpmr	f24, f0

  2331. 	fpmr	f25, f1

  2332. 

  2333. 	fsmfp	f0,  f2

  2334. 	fsmfp	f1,  f3

  2335. 	fsmtp	f2,  f24

  2336. 	fsmtp	f3,  f25

  2337. 

  2338. 	LFPDUX	f16, BO,  INC2

  2339. 	LFPDUX	f17, BO,  INC2

  2340. 	LFPDUX	f18, BO,  INC2

  2341. 	LFPDUX	f19, BO,  INC2

  2342. 

  2343. 	subi	BO,  BO,   8 * SIZE

  2344. 

  2345. 	fpsub	f0,  f16,  f0

  2346. 	fpsub	f2,  f17,  f2

  2347. 	fpsub	f1,  f18,  f1

  2348. 	fpsub	f3,  f19,  f3

  2349. #else

  2350. 	LFPDUX	f16, AO,  INC2

  2351. 	LFPDUX	f17, AO,  INC2

  2352. 	LFPDUX	f18, AO,  INC2

  2353. 	LFPDUX	f19, AO,  INC2

  2354. 

  2355. 	subi	AO,  AO,   8 * SIZE

  2356. 

  2357. 	fpsub	f0,  f16,  f0

  2358. 	fpsub	f1,  f17,  f1

  2359. 	fpsub	f2,  f18,  f2

  2360. 	fpsub	f3,  f19,  f3

  2361. #endif

  2362. 

  2363. #ifdef LN

  2364. 	addi	AO,  AO,  18 * SIZE

  2365. 

  2366. 	LFPDUX	A1,  AO,  INCM2

  2367. 	LFPDUX	A2,  AO,  INCM2

  2368. 	LFPDUX	A3,  AO,  INCM2

  2369. 	LFPDUX	A4,  AO,  INCM2

  2370. 	add	AO,  AO,  INCM2

  2371. 	LFPDUX	A5,  AO,  INCM2

  2372. 	add	AO,  AO,  INCM2

  2373. 	LFPDUX	A6,  AO,  INCM2

  2374. 

  2375. 	subi	AO,  AO,  2 * SIZE

  2376. 

  2377. 	fxsmul	   f3,  A1, f3

  2378. 	fxcpnmsub  f1,  A1, f3,  f1

  2379. 	fxcsnmsub  f2,  A2, f3,  f2

  2380. 	fxcpnmsub  f0,  A2, f3,  f0

  2381. 

  2382. 	fxpmul	   f1,  A3, f1

  2383. 	fxcsnmsub  f2,  A4, f1,  f2

  2384. 	fxcpnmsub  f0,  A4, f1,  f0

  2385. 

  2386. 	fxsmul	   f2,  A5, f2

  2387. 	fxcpnmsub  f0,  A5, f2,  f0

  2388. 

  2389. 	fxpmul	   f0,  A6, f0

  2390. #endif

  2391. 

  2392. #ifdef LT

  2393. 	LFPDUX	A1,  AO,  INC2

  2394. 	LFPDUX	A2,  AO,  INC2

  2395. 	LFPDUX	A3,  AO,  INC2

  2396. 	LFPDUX	A4,  AO,  INC2

  2397. 

  2398. 	add	AO,  AO,  INC2

  2399. 	LFPDUX	A5,  AO,  INC2

  2400. 	add	AO,  AO,  INC2

  2401. 	LFPDUX	A6,  AO,  INC2

  2402. 

  2403. 	subi	AO,  AO,  16 * SIZE

  2404. 

  2405. 	fxpmul	   f0,  A1,  f0

  2406. 	fxcsnmsub  f2,  A1, f0, f2

  2407. 	fxcpnmsub  f1,  A2, f0, f1

  2408. 	fxcsnmsub  f3,  A2, f0, f3

  2409. 

  2410. 	fxsmul	   f2,  A3,  f2

  2411. 	fxcpnmsub  f1,  A4,  f2,  f1

  2412. 	fxcsnmsub  f3,  A4,  f2,  f3

  2413. 

  2414. 	fxpmul	   f1,  A5,  f1

  2415. 	fxcsnmsub  f3,  A5,  f1, f3

  2416. 

  2417. 	fxsmul	   f3,  A6,  f3

  2418. #endif

  2419. 

  2420. #ifdef RN

  2421. 	LFPDUX	A1,  BO,  INC2

  2422. 	LFPDUX	A2,  BO,  INC2

  2423. 

  2424. 	subi	BO, BO, 4 * SIZE

  2425. 

  2426. 	fxpmul	f0,  A1,  f0

  2427. 	fxpmul	f1,  A1,  f1

  2428. 

  2429. 	fxcsnmsub    f2,  A1, f0, f2

  2430. 	fxcsnmsub    f3,  A1, f1, f3

  2431. 

  2432. 	fxsmul	f2,  A2,  f2

  2433. 	fxsmul	f3,  A2,  f3

  2434. #endif

  2435. 

  2436. #ifdef RT

  2437. 	LFPDUX	A2,  BO,  INC2

  2438. 	LFPDUX	A1,  BO,  INC2

  2439. 

  2440. 	subi	BO, BO, 4 * SIZE

  2441. 

  2442. 	fxsmul	f2,  A1,  f2

  2443. 	fxsmul	f3,  A1,  f3

  2444. 

  2445. 	fxcpnmsub    f0,  A1, f2,  f0

  2446. 	fxcpnmsub    f1,  A1, f3,  f1

  2447. 

  2448. 	fxpmul	f0,  A2,  f0

  2449. 	fxpmul	f1,  A2,  f1

  2450. #endif

  2451. 

  2452. #ifdef LN

  2453. 	subi	CO1, CO1, 4 * SIZE

  2454. 	subi	CO2, CO2, 4 * SIZE

  2455. #endif

  2456. 

  2457. #if defined(LN) || defined(LT)

  2458. 	STFPDUX	f0,  BO,  INC2

  2459. 	STFPDUX	f2,  BO,  INC2

  2460. 	STFPDUX	f1,  BO,  INC2

  2461. 	STFPDUX	f3,  BO,  INC2

  2462. 

  2463. 	subi	BO,  BO,   8 * SIZE

  2464. 

  2465. 	STFDUX	f0,  CO1, INC

  2466. 	STFDUX	f2,  CO1, INC

  2467. 	STFDUX	f1,  CO1, INC

  2468. 	STFDUX	f3,  CO1, INC

  2469. 

  2470. 	STFSDUX	f0,  CO2, INC

  2471. 	STFSDUX	f2,  CO2, INC

  2472. 	STFSDUX	f1,  CO2, INC

  2473. 	STFSDUX	f3,  CO2, INC

  2474. #else

  2475. 	STFPDUX	f0,  AO,  INC2

  2476. 	STFPDUX	f1,  AO,  INC2

  2477. 	STFPDUX	f2,  AO,  INC2

  2478. 	STFPDUX	f3,  AO,  INC2

  2479. 

  2480. 	subi	AO,  AO,   8 * SIZE

  2481. 

  2482. 	STFDUX	f0,  CO1, INC

  2483. 	STFSDUX	f0,  CO1, INC

  2484. 	STFDUX	f1,  CO1, INC

  2485. 	STFSDUX	f1,  CO1, INC

  2486. 

  2487. 	STFDUX	f2,  CO2, INC

  2488. 	STFSDUX	f2,  CO2, INC

  2489. 	STFDUX	f3,  CO2, INC

  2490. 	STFSDUX	f3,  CO2, INC

  2491. #endif

  2492. 

  2493. #ifdef LN

  2494. 	subi	CO1, CO1, 4 * SIZE

  2495. 	subi	CO2, CO2, 4 * SIZE

  2496. #endif

  2497. 

  2498. #ifdef RT

  2499. 	slwi	r0, K, 2 + BASE_SHIFT

  2500. 	add	AORIG, AORIG, r0

  2501. #endif

  2502. 

  2503. #if defined(LT) || defined(RN)

  2504. 	sub	TEMP, K, KK

  2505. 	slwi	r0,   TEMP, 2 + BASE_SHIFT

  2506. 	slwi	TEMP, TEMP, 1 + BASE_SHIFT

  2507. 	add	AO, AO, r0

  2508. 	add	BO, BO, TEMP

  2509. #endif

  2510. 

  2511. #ifdef LT

  2512. 	addi	KK, KK, 4

  2513. #endif

  2514. 

  2515. #ifdef LN

  2516. 	subi	KK, KK, 4

  2517. #endif

  2518. 

  2519. 	li	r0, FZERO

  2520. 	lfpsx	f0, SP, r0

  2521. 	.align 4

  2522. 

  2523. .L70:

  2524. 	andi.	I, M,  2

  2525. 	beq	.L80

  2526. 

  2527. #if defined(LT) || defined(RN)

  2528. 	addi	BO,  B,  - 2 * SIZE

  2529. 	fpmr	f1,  f0

  2530. 

  2531. 	fpmr	f2,  f0

  2532. 	fpmr	f3, f0

  2533. 

  2534. 	srawi.	r0,  KK,  3

  2535. 	mtspr	CTR, r0

  2536. 	ble	.L74

  2537. #else

  2538. 

  2539. #ifdef LN

  2540. 	slwi	r0,   K,  1 + BASE_SHIFT

  2541. 	sub	AORIG, AORIG, r0

  2542. #endif

  2543. 

  2544. 	slwi	r0  , KK, 1 + BASE_SHIFT

  2545. 	slwi	TEMP, KK, 1 + BASE_SHIFT

  2546. 	add	AO, AORIG, r0

  2547. 	add	BO, B,     TEMP

  2548. 

  2549. 	sub	TEMP, K, KK

  2550. 

  2551. 	addi	BO,  BO,  - 2 * SIZE

  2552. 	fpmr	f1,  f0

  2553. 

  2554. 	fpmr	f2,  f0

  2555. 	fpmr	f3, f0

  2556. 

  2557. 	srawi.	r0,  TEMP,  3

  2558. 	mtspr	CTR, r0

  2559. 	ble	.L74

  2560. #endif

  2561. 

  2562. 	LFPDUX	A1,  AO, INC2

  2563. 	LFPDUX	B1,  BO, INC2

  2564. 	LFPDUX	A2,  AO, INC2

  2565. 	LFPDUX	B2,  BO, INC2

  2566. 	LFPDUX	A3,  AO, INC2

  2567. 	LFPDUX	B3,  BO, INC2

  2568. 	LFPDUX	A4,  AO, INC2

  2569. 	LFPDUX	B4,  BO, INC2

  2570. 

  2571. 	LFPDUX	A5,  AO, INC2

  2572. 	LFPDUX	B5,  BO, INC2

  2573. 	LFPDUX	A6,  AO, INC2

  2574. 	LFPDUX	B6,  BO, INC2

  2575. 	LFPDUX	A7,  AO, INC2

  2576. 	LFPDUX	A9,  BO, INC2

  2577. 	LFPDUX	A8,  AO, INC2

  2578. 	LFPDUX	A10, BO, INC2

  2579. 	bdz-	.L73

  2580. 	.align 4

  2581. 

  2582. .L72:

  2583. 	fxcpmadd	f0,  B1, A1, f0

  2584. 	fxcsmadd	f1,  B1, A1, f1

  2585. 	LFPDUX	A1,  AO, INC2

  2586. 	LFPDUX	B1,  BO, INC2

  2587. 	fxcpmadd	f2,  B2, A2, f2

  2588. 	fxcsmadd	f3,  B2, A2, f3

  2589. 	LFPDUX	A2,  AO, INC2

  2590. 	LFPDUX	B2,  BO, INC2

  2591. 

  2592. 	fxcpmadd	f0,  B3, A3, f0

  2593. 	fxcsmadd	f1,  B3, A3, f1

  2594. 	LFPDUX	A3,  AO, INC2

  2595. 	LFPDUX	B3,  BO, INC2

  2596. 	fxcpmadd	f2,  B4, A4, f2

  2597. 	fxcsmadd	f3,  B4, A4, f3

  2598. 	LFPDUX	A4,  AO, INC2

  2599. 	LFPDUX	B4,  BO, INC2

  2600. 

  2601. 	fxcpmadd	f0,  B5, A5, f0

  2602. 	fxcsmadd	f1,  B5, A5, f1

  2603. 	LFPDUX	A5,  AO, INC2

  2604. 	LFPDUX	B5,  BO, INC2

  2605. 	fxcpmadd	f2,  B6, A6, f2

  2606. 	fxcsmadd	f3,  B6, A6, f3

  2607. 	LFPDUX	A6,  AO, INC2

  2608. 	LFPDUX	B6,  BO, INC2

  2609. 

  2610. 	fxcpmadd	f0,  A9,  A7, f0

  2611. 	fxcsmadd	f1,  A9,  A7, f1

  2612. 	LFPDUX	A7,  AO, INC2

  2613. 	LFPDUX	A9,  BO, INC2

  2614. 	fxcpmadd	f2,  A10, A8, f2

  2615. 	fxcsmadd	f3,  A10, A8, f3

  2616. 	LFPDUX	A8,  AO, INC2

  2617. 	LFPDUX	A10, BO, INC2

  2618. 	bdnz+	.L72

  2619. 	.align 4

  2620. 

  2621. .L73:

  2622. 	fxcpmadd	f0,  B1, A1, f0

  2623. 	fxcsmadd	f1,  B1, A1, f1

  2624. 	fxcpmadd	f2,  B2, A2, f2

  2625. 	fxcsmadd	f3,  B2, A2, f3

  2626. 

  2627. 	fxcpmadd	f0,  B3, A3, f0

  2628. 	fxcsmadd	f1,  B3, A3, f1

  2629. 	fxcpmadd	f2,  B4, A4, f2

  2630. 	fxcsmadd	f3,  B4, A4, f3

  2631. 

  2632. 	fxcpmadd	f0,  B5, A5, f0

  2633. 	fxcsmadd	f1,  B5, A5, f1

  2634. 	fxcpmadd	f2,  B6, A6, f2

  2635. 	fxcsmadd	f3,  B6, A6, f3

  2636. 

  2637. 	fxcpmadd	f0,  A9,  A7, f0

  2638. 	fxcsmadd	f1,  A9,  A7, f1

  2639. 	fxcpmadd	f2,  A10, A8, f2

  2640. 	fxcsmadd	f3,  A10, A8, f3

  2641. 	.align 4

  2642. 

  2643. .L74:

  2644. #if defined(LT) || defined(RN)

  2645. 	andi.	r0,  KK,  7

  2646. 	mtspr	CTR, r0

  2647. 	ble+	.L78

  2648. #else

  2649. 	andi.	r0, TEMP, 7

  2650. 	mtspr	CTR, r0

  2651. 	ble+	.L78

  2652. #endif

  2653. 

  2654. 	LFPDUX	A1,  AO,  INC2

  2655. 	LFPDUX	B1,  BO,  INC2

  2656. 	bdz-	.L77

  2657. 	.align 4

  2658. 

  2659. .L76:

  2660. 	fxcpmadd	f0,  B1, A1, f0

  2661. 	fxcsmadd	f1,  B1, A1, f1

  2662. 	LFPDUX	A1,  AO,  INC2

  2663. 	LFPDUX	B1,  BO,  INC2

  2664. 	bdnz+	.L76

  2665. 	.align 4

  2666. 

  2667. .L77:

  2668. 	fxcpmadd	f0,  B1, A1, f0

  2669. 	fxcsmadd	f1,  B1, A1, f1

  2670. 	.align 4

  2671. 

  2672. .L78:

  2673. 	fpadd	f0, f0, f2

  2674. 	fpadd	f1, f1, f3

  2675. 

  2676. #if defined(LN) || defined(RT)

  2677. #ifdef LN

  2678. 	subi	r0, KK, 2

  2679. #else

  2680. 	subi	r0, KK, 2

  2681. #endif

  2682. 	slwi	TEMP, r0, 1 + BASE_SHIFT

  2683. 	slwi	r0,   r0, 1 + BASE_SHIFT

  2684. 	add	AO, AORIG, TEMP

  2685. 	add	BO,  B,     r0

  2686. 	addi	BO,  BO, - 2 * SIZE

  2687. #endif

  2688. 

  2689. #if defined(LN) || defined(LT)

  2690. 	fpmr	f24, f0

  2691. 	fsmfp	f0,  f1

  2692. 	fsmtp	f1,  f24

  2693. 

  2694. 	LFPDUX	f16, BO,  INC2

  2695. 	LFPDUX	f17, BO,  INC2

  2696. 

  2697. 	subi	BO,  BO,   4 * SIZE

  2698. 

  2699. 	fpsub	f0,  f16,  f0

  2700. 	fpsub	f1,  f17,  f1

  2701. #else

  2702. 	LFPDUX	f16, AO,  INC2

  2703. 	LFPDUX	f17, AO,  INC2

  2704. 

  2705. 	subi	AO,  AO,   4 * SIZE

  2706. 

  2707. 	fpsub	f0,  f16,  f0

  2708. 	fpsub	f1,  f17,  f1

  2709. #endif

  2710. 

  2711. #ifdef LN

  2712. 	LFPDUX	A1,  AO,  INC2

  2713. 	LFPDUX	A2,  AO,  INC2

  2714. 

  2715. 	addi	AO,  AO,  -4 * SIZE

  2716. 

  2717. 	fxsmul	   f1,  A2, f1

  2718. 	fxcpnmsub  f0,  A2, f1,  f0

  2719. 	fxpmul	   f0,  A1, f0

  2720. #endif

  2721. 

  2722. #ifdef LT

  2723. 	LFPDUX	A1,  AO,  INC2

  2724. 	LFPDUX	A2,  AO,  INC2

  2725. 

  2726. 	addi	AO,  AO,  -4 * SIZE

  2727. 

  2728. 	fxpmul	   f0,  A1, f0

  2729. 	fxcsnmsub  f1,  A1, f0, f1

  2730. 

  2731. 	fxsmul	   f1,  A2,  f1

  2732. #endif

  2733. 

  2734. #ifdef RN

  2735. 	LFPDUX	A1,  BO,  INC2

  2736. 	LFPDUX	A2,  BO,  INC2

  2737. 

  2738. 	subi	BO, BO, 4 * SIZE

  2739. 

  2740. 	fxpmul	     f0,  A1, f0

  2741. 	fxcsnmsub    f1,  A1, f0, f1

  2742. 

  2743. 	fxsmul	     f1,  A2,  f1

  2744. #endif

  2745. 

  2746. #ifdef RT

  2747. 	LFPDUX	A2,  BO,  INC2

  2748. 	LFPDUX	A1,  BO,  INC2

  2749. 

  2750. 	subi	BO, BO, 4 * SIZE

  2751. 

  2752. 	fxsmul	     f1,  A1, f1

  2753. 	fxcpnmsub    f0,  A1, f1,  f0

  2754. 	fxpmul	     f0,  A2,  f0

  2755. #endif

  2756. 

  2757. #ifdef LN

  2758. 	subi	CO1, CO1, 2 * SIZE

  2759. 	subi	CO2, CO2, 2 * SIZE

  2760. #endif

  2761. 

  2762. #if defined(LN) || defined(LT)

  2763. 	STFPDUX	f0,  BO,  INC2

  2764. 	STFPDUX	f1,  BO,  INC2

  2765. 

  2766. 	subi	BO,  BO,   4 * SIZE

  2767. 

  2768. 	STFDUX	f0,  CO1, INC

  2769. 	STFDUX	f1,  CO1, INC

  2770. 

  2771. 	STFSDUX	f0,  CO2, INC

  2772. 	STFSDUX	f1,  CO2, INC

  2773. #else

  2774. 	STFPDUX	f0,  AO,  INC2

  2775. 	STFPDUX	f1,  AO,  INC2

  2776. 

  2777. 	subi	AO,  AO,   4 * SIZE

  2778. 

  2779. 	STFDUX	f0,  CO1, INC

  2780. 	STFSDUX	f0,  CO1, INC

  2781. 

  2782. 	STFDUX	f1,  CO2, INC

  2783. 	STFSDUX	f1,  CO2, INC

  2784. #endif

  2785. 

  2786. #ifdef LN

  2787. 	subi	CO1, CO1, 2 * SIZE

  2788. 	subi	CO2, CO2, 2 * SIZE

  2789. #endif

  2790. 

  2791. #ifdef RT

  2792. 	slwi	r0, K, 1 + BASE_SHIFT

  2793. 	add	AORIG, AORIG, r0

  2794. #endif

  2795. 

  2796. #if defined(LT) || defined(RN)

  2797. 	sub	TEMP, K, KK

  2798. 	slwi	r0,   TEMP, 1 + BASE_SHIFT

  2799. 	slwi	TEMP, TEMP, 1 + BASE_SHIFT

  2800. 	add	AO, AO, r0

  2801. 	add	BO, BO, TEMP

  2802. #endif

  2803. 

  2804. #ifdef LT

  2805. 	addi	KK, KK, 2

  2806. #endif

  2807. 

  2808. #ifdef LN

  2809. 	subi	KK, KK, 2

  2810. #endif

  2811. 

  2812. 	li	r0, FZERO

  2813. 	lfpsx	f0, SP, r0

  2814. 	.align 4

  2815. 

  2816. .L80:

  2817. 	andi.	I, M,  1

  2818. 	beq	.L89

  2819. 

  2820. #if defined(LT) || defined(RN)

  2821. 	addi	BO,  B,  - 2 * SIZE

  2822. 	fpmr	f1,  f0

  2823. 	fpmr	f2,  f0

  2824. 	fpmr	f3,  f0

  2825. 

  2826. 	srawi.	r0,  KK,  3

  2827. 	mtspr	CTR, r0

  2828. 	ble	.L84

  2829. #else

  2830. 

  2831. #ifdef LN

  2832. 	slwi	r0,   K,  0 + BASE_SHIFT

  2833. 	sub	AORIG, AORIG, r0

  2834. #endif

  2835. 

  2836. 	slwi	r0  , KK, 0 + BASE_SHIFT

  2837. 	slwi	TEMP, KK, 1 + BASE_SHIFT

  2838. 	add	AO, AORIG, r0

  2839. 	add	BO, B,     TEMP

  2840. 

  2841. 	sub	TEMP, K, KK

  2842. 

  2843. 	addi	BO,  BO,  - 2 * SIZE

  2844. 	fpmr	f1,  f0

  2845. 	fpmr	f2,  f0

  2846. 	fpmr	f3,  f0

  2847. 

  2848. 	srawi.	r0,  TEMP,  3

  2849. 	mtspr	CTR, r0

  2850. 	ble	.L84

  2851. 

  2852. #endif

  2853. 

  2854. 	LFPDUX	B1,  BO,  INC2

  2855. 	LFPDUX	A1,  AO,  INC2

  2856. 	LFPDUX	A2,  AO,  INC2

  2857. 

  2858. 	LFPDUX	B2,  BO,  INC2

  2859. 	LFPDUX	A3,  AO,  INC2

  2860. 	LFPDUX	A4,  AO,  INC2

  2861. 

  2862. 	LFPDUX	B3,  BO,  INC2

  2863. 	LFPDUX	B4,  BO,  INC2

  2864. 	bdz-	.L83

  2865. 	.align 4

  2866. 

  2867. .L82:

  2868. 	fxcpmadd	f0,  A1, B1, f0

  2869. 	LFPDUX	B1,  BO,  INC2

  2870. 	fxcsmadd	f1,  A1, B2, f1

  2871. 	LFPDUX	B2,  BO,  INC2

  2872. 	LFPDUX	A1,  AO,  INC2

  2873. 	fxcpmadd	f2,  A2, B3, f2

  2874. 	LFPDUX	B3,  BO,  INC2

  2875. 	fxcsmadd	f3,  A2, B4, f3

  2876. 	LFPDUX	B4,  BO,  INC2

  2877. 	LFPDUX	A2,  AO,  INC2

  2878. 

  2879. 	fxcpmadd	f0,  A3, B1, f0

  2880. 	LFPDUX	B1,  BO,  INC2

  2881. 	fxcsmadd	f1,  A3, B2, f1

  2882. 	LFPDUX	B2,  BO,  INC2

  2883. 	LFPDUX	A3,  AO,  INC2

  2884. 	fxcpmadd	f2,  A4, B3, f2

  2885. 	LFPDUX	B3,  BO,  INC2

  2886. 	fxcsmadd	f3,  A4, B4, f3

  2887. 	LFPDUX	B4,  BO,  INC2

  2888. 	LFPDUX	A4,  AO,  INC2

  2889. 	bdnz+	.L82

  2890. 	.align 4

  2891. 

  2892. .L83:

  2893. 	fxcpmadd	f0,  A1, B1, f0

  2894. 	LFPDUX	B1,  BO,  INC2

  2895. 	fxcsmadd	f1,  A1, B2, f1

  2896. 	LFPDUX	B2,  BO,  INC2

  2897. 	fxcpmadd	f2,  A2, B3, f2

  2898. 	LFPDUX	B3,  BO,  INC2

  2899. 	fxcsmadd	f3,  A2, B4, f3

  2900. 	LFPDUX	B4,  BO,  INC2

  2901. 

  2902. 	fxcpmadd	f0,  A3, B1, f0

  2903. 	fxcsmadd	f1,  A3, B2, f1

  2904. 	fxcpmadd	f2,  A4, B3, f2

  2905. 	fxcsmadd	f3,  A4, B4, f3

  2906. 	.align 4

  2907. 

  2908. .L84:

  2909. #if defined(LT) || defined(RN)

  2910. 	andi.	r0,  KK,  7

  2911. 	mtspr	CTR, r0

  2912. 	ble+	.L88

  2913. #else

  2914. 	andi.	r0, TEMP, 7

  2915. 	mtspr	CTR, r0

  2916. 	ble+	.L88

  2917. #endif

  2918. 

  2919. 	LFDX	A1,  AO,  INC2

  2920. 	LFPDUX	B1,  BO,  INC2

  2921. 	add	AO, AO, INC

  2922. 	bdz-	.L87

  2923. 	.align 4

  2924. 

  2925. .L86:

  2926. 	fxcpmadd	f0,  A1, B1, f0

  2927. 	LFDX	A1,  AO,  INC2

  2928. 	LFPDUX	B1,  BO,  INC2

  2929. 	add	AO, AO, INC

  2930. 	bdnz+	.L86

  2931. 	.align 4

  2932. 

  2933. .L87:

  2934. 	fxcpmadd	f0,  A1, B1, f0

  2935. 	.align 4

  2936. 

  2937. .L88:

  2938. 	fpadd	f0, f0, f1

  2939. 	fpadd	f2, f2, f3

  2940. 	fpadd	f0, f0, f2

  2941. 

  2942. #if defined(LN) || defined(RT)

  2943. #ifdef LN

  2944. 	subi	r0, KK, 1

  2945. #else

  2946. 	subi	r0, KK, 2

  2947. #endif

  2948. 	slwi	TEMP, r0, 0 + BASE_SHIFT

  2949. 	slwi	r0,   r0, 1 + BASE_SHIFT

  2950. 	add	AO, AORIG, TEMP

  2951. 	add	BO,  B,     r0

  2952. 	addi	BO,  BO, - 2 * SIZE

  2953. #endif

  2954. 

  2955. #if defined(LN) || defined(LT)

  2956. 	LFPDX	f16, BO,  INC2

  2957. 

  2958. 	fpsub	f0,  f16,  f0

  2959. #else

  2960. 	LFPDX	f16, AO,  INC2

  2961. 

  2962. 	fpsub	f0,  f16,  f0

  2963. #endif

  2964. 

  2965. #ifdef LN

  2966. 	LFPDX	A1,  AO,  INC2

  2967. 

  2968. 	fxpmul	   f0,  A1, f0

  2969. #endif

  2970. 

  2971. #ifdef LT

  2972. 	LFPDX	A1,  AO,  INC2

  2973. 

  2974. 	fxpmul	   f0,  A1, f0

  2975. #endif

  2976. 

  2977. #ifdef RN

  2978. 	LFD	A1,  (2 + 0) * SIZE(BO)

  2979. 	LFD	A2,  (2 + 1) * SIZE(BO)

  2980. 	LFD	A3,  (2 + 3) * SIZE(BO)

  2981. 

  2982. 	fsmtp	     f1, f0

  2983. 

  2984. 	fmul	     f0,  A1, f0

  2985. 	fnmsub	     f1,  A2, f0, f1

  2986. 

  2987. 	fmul	     f1,  A3, f1

  2988. 	fsmfp	     f0, f1

  2989. #endif

  2990. 

  2991. #ifdef RT

  2992. 	LFD	A1,  (2 + 3) * SIZE(BO)

  2993. 	LFD	A2,  (2 + 2) * SIZE(BO)

  2994. 	LFD	A3,  (2 + 0) * SIZE(BO)

  2995. 

  2996. 	fsmtp	     f1, f0

  2997. 

  2998. 	fmul	     f1,  A1, f1

  2999. 	fnmsub	     f0,  A2, f1, f0

  3000. 

  3001. 	fmul	     f0,  A3, f0

  3002. 	fsmfp	     f0, f1

  3003. #endif

  3004. 

  3005. #ifdef LN

  3006. 	subi	CO1, CO1, 1 * SIZE

  3007. 	subi	CO2, CO2, 1 * SIZE

  3008. #endif

  3009. 

  3010. #if defined(LN) || defined(LT)

  3011. 	STFPDX	f0,  BO,  INC2

  3012. 

  3013. 	STFDUX	f0,  CO1, INC

  3014. 	STFSDUX	f0,  CO2, INC

  3015. #else

  3016. 	STFPDX	f0,  AO,  INC2

  3017. 

  3018. 	STFDUX	f0,  CO1, INC

  3019. 	STFDUX	f1,  CO2, INC

  3020. #endif

  3021. 

  3022. #ifdef LN

  3023. 	subi	CO1, CO1, 1 * SIZE

  3024. 	subi	CO2, CO2, 1 * SIZE

  3025. #endif

  3026. 

  3027. #ifdef RT

  3028. 	slwi	r0, K, 0 + BASE_SHIFT

  3029. 	add	AORIG, AORIG, r0

  3030. #endif

  3031. 

  3032. #if defined(LT) || defined(RN)

  3033. 	sub	TEMP, K, KK

  3034. 	slwi	r0,   TEMP, 0 + BASE_SHIFT

  3035. 	slwi	TEMP, TEMP, 1 + BASE_SHIFT

  3036. 	add	AO, AO, r0

  3037. 	add	BO, BO, TEMP

  3038. #endif

  3039. 

  3040. #ifdef LT

  3041. 	addi	KK, KK, 1

  3042. #endif

  3043. 

  3044. #ifdef LN

  3045. 	subi	KK, KK, 1

  3046. #endif

  3047. 	.align 4

  3048. 

  3049. .L89:

  3050. #ifdef LN

  3051. 	slwi	r0, K, 1 + BASE_SHIFT

  3052. 	add	B, B, r0

  3053. #endif

  3054. 

  3055. #if defined(LT) || defined(RN)

  3056. 	addi	B,  BO, 2 * SIZE

  3057. #endif

  3058. 

  3059. #ifdef RN

  3060. 	addi	KK, KK, 2

  3061. #endif

  3062. 

  3063. #ifdef RT

  3064. 	subi	KK, KK, 2

  3065. #endif

  3066. 	.align 4

  3067. 

  3068. .L90:

  3069. 	srawi.	J, N,  2

  3070. 	ble	.L999

  3071. 	.align 4

  3072. 

  3073. .L10:

  3074. #ifdef RT

  3075. 	slwi	r0, K, 2 + BASE_SHIFT

  3076. 	sub	B, B, r0

  3077. 

  3078. 	slwi	r0, LDC, 2

  3079. 	sub	C, C, r0

  3080. #endif

  3081. 

  3082. 	mr	CO1, C

  3083. 	add	CO2, C,   LDC

  3084. 	add	CO3, CO2, LDC

  3085. 	add	CO4, CO3, LDC

  3086. 

  3087. #ifdef LN

  3088. 	add	KK, M, OFFSET

  3089. #endif

  3090. 

  3091. #ifdef LT

  3092. 	mr	KK, OFFSET

  3093. #endif

  3094. 

  3095. 

  3096. #if defined(LN) || defined(RT)

  3097. 	addi	AORIG, A, -4 * SIZE

  3098. #else

  3099. 	addi	AO, A, -4 * SIZE

  3100. #endif

  3101. #ifndef RT

  3102. 	add	C,  CO4, LDC

  3103. #endif

  3104. 

  3105. 	li	r0, FZERO

  3106. 	lfpsx	f0, SP, r0

  3107. 

  3108. 	srawi.	I, M,  3

  3109. 	ble	.L20

  3110. 	.align 4

  3111. 

  3112. .L11:

  3113. #if defined(LT) || defined(RN)

  3114. 	addi	AO2, AO,   2 * SIZE

  3115. 	fpmr	f4,  f0

  3116. 	addi	BO,  B,  - 4 * SIZE

  3117. 	fpmr	f8,  f0

  3118. 	addi	BO2, B,  - 2 * SIZE

  3119. 	fpmr	f12, f0

  3120. 

  3121. 	fpmr	f5,  f0

  3122. 	fpmr	f9,  f0

  3123. 	fpmr	f13, f0

  3124. 	fpmr	f2,  f0

  3125. 

  3126. 	fpmr	f6,  f0

  3127. 	fpmr	f10, f0

  3128. 	fpmr	f14, f0

  3129. 	fpmr	f3,  f0

  3130. 

  3131. 	fpmr	f7,  f0

  3132. 	fpmr	f11, f0

  3133. 	fpmr	f15, f0

  3134. 	nop

  3135. 

  3136. 	srawi.	r0,  KK,  2

  3137.  	fpmr	f1,  f0

  3138. 	mtspr	CTR, r0

  3139. 	ble	.L14

  3140. #else

  3141. 

  3142. #ifdef LN

  3143. 	slwi	r0,   K,  3 + BASE_SHIFT

  3144. 	sub	AORIG, AORIG, r0

  3145. #endif

  3146. 

  3147. 	slwi	r0  , KK, 3 + BASE_SHIFT

  3148. 	slwi	TEMP, KK, 2 + BASE_SHIFT

  3149. 	add	AO, AORIG, r0

  3150. 	add	BO, B,     TEMP

  3151. 

  3152. 	sub	TEMP, K, KK

  3153. 

  3154. 	addi	AO2, AO,   2 * SIZE

  3155. 	fpmr	f4,  f0

  3156. 	addi	BO,  BO, - 4 * SIZE

  3157. 	fpmr	f8,  f0

  3158. 	addi	BO2, BO,   2 * SIZE

  3159. 	fpmr	f12, f0

  3160. 

  3161. 	fpmr	f5,  f0

  3162. 	fpmr	f9,  f0

  3163. 	fpmr	f13, f0

  3164. 	fpmr	f2,  f0

  3165. 

  3166. 	fpmr	f6,  f0

  3167. 	fpmr	f10, f0

  3168. 	fpmr	f14, f0

  3169. 	fpmr	f3,  f0

  3170. 

  3171. 	fpmr	f7,  f0

  3172. 	fpmr	f11, f0

  3173. 	fpmr	f15, f0

  3174. 	nop

  3175. 

  3176. 	srawi.	r0,  TEMP,  2

  3177.  	fpmr	f1,  f0

  3178. 	mtspr	CTR, r0

  3179. 	ble	.L14

  3180. #endif

  3181. 

  3182. 	LFPDUX	A1,  AO, INC4

  3183. 	fpmr	f5,  f0

  3184. 	LFPDUX	A3,  AO, INC4

  3185. 	fpmr	f9,  f0

  3186. 	LFPDUX	B1,  BO, INC4

  3187. 	fpmr	f13, f0

  3188. 

  3189. 	LFPDUX	A5,  AO, INC4

  3190. 	fpmr	f2,  f0

  3191. 	LFPDUX	A6,  AO, INC4

  3192. 	fpmr	f6,  f0

  3193. 	LFPDUX	B3,  BO, INC4

  3194. 	fpmr	f10, f0

  3195. 	LFPDUX	A7,  AO, INC4

  3196. 	fpmr	f14, f0

  3197. 

  3198. 	LFPDUX	A8,  AO, INC4

  3199. 	fpmr	f3,  f0

  3200. 	LFPDUX	B5,  BO, INC4

  3201. 	fpmr	f7,  f0

  3202. 	LFPDUX	A9,  AO, INC4

  3203. 	fpmr	f11, f0

  3204. 	LFPDUX	A2, AO2, INC4

  3205. 	fpmr	f15, f0

  3206. 	LFPDUX	B2, BO2, INC4

  3207. 	bdz-	.L13

  3208. 	.align 4

  3209. 

  3210. .L12:

  3211. 

  3212. ## 1 ##

  3213. 	fxcpmadd	f0,  B1, A1, f0

  3214. 	nop

  3215. 	fxcsmadd	f4,  B1, A1, f4

  3216. 	nop

  3217. 	fxcpmadd	f8,  B2, A1, f8

  3218. 	LFPDUX	B4, BO2, INC4

  3219. 	fxcsmadd	f12, B2, A1, f12

  3220. 	LFPDUX	B6,  BO, INC4

  3221. 

  3222. 	fxcpmadd	f1,  B1, A2, f1

  3223. 	nop

  3224. 	fxcsmadd	f5,  B1, A2, f5

  3225. 	LFPDUX	A4, AO2, INC4

  3226. 	fxcpmadd	f9,  B2, A2, f9

  3227. 	LFPDUX	A10, AO, INC4

  3228. 	fxcsmadd	f13, B2, A2, f13

  3229. 	nop

  3230. 

  3231. 	fxcpmadd	f2,  B1, A3, f2

  3232. 	nop

  3233. 	fxcsmadd	f6,  B1, A3, f6

  3234. 	nop

  3235. 	fxcpmadd	f10, B2, A3, f10

  3236. 	nop

  3237. 	fxcsmadd	f14, B2, A3, f14

  3238. 	nop

  3239. 

  3240. 	fxcpmadd	f3,  B1, A4, f3

  3241. 	nop

  3242. 	fxcsmadd	f7,  B1, A4, f7

  3243. 	LFPDUX	A2, AO2, INC4

  3244. 	fxcpmadd	f11, B2, A4, f11

  3245. 	LFPDUX	A1,  AO, INC4

  3246. 	fxcsmadd	f15, B2, A4, f15

  3247. 	nop

  3248. 

  3249. ## 2 ##

  3250. 

  3251. 	fxcpmadd	f0,  B3, A5, f0

  3252. 	nop

  3253. 	fxcsmadd	f4,  B3, A5, f4

  3254. 	nop

  3255. 	fxcpmadd	f8,  B4, A5, f8

  3256. 	LFPDUX	B2, BO2, INC4

  3257. 	fxcsmadd	f12, B4, A5, f12

  3258. 	LFPDUX	B1,  BO, INC4

  3259. 

  3260. 	fxcpmadd	f1,  B3, A2, f1

  3261. 	nop

  3262. 	fxcsmadd	f5,  B3, A2, f5

  3263. 	LFPDUX	A4, AO2, INC4

  3264. 	fxcpmadd	f9,  B4, A2, f9

  3265. 	LFPDUX	A3,  AO, INC4

  3266. 	fxcsmadd	f13, B4, A2, f13

  3267. 	nop

  3268. 

  3269. 	fxcpmadd	f2,  B3, A6, f2

  3270. 	nop

  3271. 	fxcsmadd	f6,  B3, A6, f6

  3272. 	nop

  3273. 	fxcpmadd	f10, B4, A6, f10

  3274. 	nop

  3275. 	fxcsmadd	f14, B4, A6, f14

  3276. 	nop

  3277. 

  3278. 	fxcpmadd	f3,  B3, A4, f3

  3279. 	nop

  3280. 	fxcsmadd	f7,  B3, A4, f7

  3281. 	LFPDUX	A2, AO2, INC4

  3282. 	fxcpmadd	f11, B4, A4, f11

  3283. 	LFPDUX	A5,  AO, INC4

  3284. 	fxcsmadd	f15, B4, A4, f15

  3285. 	nop

  3286. 

  3287. ## 3 ##

  3288. 

  3289. 	fxcpmadd	f0,  B5, A7, f0

  3290. 	nop

  3291. 	fxcsmadd	f4,  B5, A7, f4

  3292. 	nop

  3293. 	fxcpmadd	f8,  B2, A7, f8

  3294. 	LFPDUX	B4, BO2, INC4

  3295. 	fxcsmadd	f12, B2, A7, f12

  3296. 	LFPDUX	B3,  BO, INC4

  3297. 

  3298. 	fxcpmadd	f1,  B5, A2, f1

  3299. 	nop

  3300. 	fxcsmadd	f5,  B5, A2, f5

  3301. 	LFPDUX	A4, AO2, INC4

  3302. 	fxcpmadd	f9,  B2, A2, f9

  3303. 	LFPDUX	A6,  AO, INC4

  3304. 	fxcsmadd	f13, B2, A2, f13

  3305. 	nop

  3306. 

  3307. 	fxcpmadd	f2,  B5, A8, f2

  3308. 	nop

  3309. 	fxcsmadd	f6,  B5, A8, f6

  3310. 	nop

  3311. 	fxcpmadd	f10, B2, A8, f10

  3312. 	nop

  3313. 	fxcsmadd	f14, B2, A8, f14

  3314. 	nop

  3315. 

  3316. 	fxcpmadd	f3,  B5, A4, f3

  3317. 	nop

  3318. 	fxcsmadd	f7,  B5, A4, f7

  3319. 	LFPDUX	A2, AO2, INC4

  3320. 	fxcpmadd	f11, B2, A4, f11

  3321. 	LFPDUX	A7,  AO, INC4

  3322. 	fxcsmadd	f15, B2, A4, f15

  3323. 	nop

  3324. 

  3325. ## 4 ##

  3326. 	fxcpmadd	f0,  B6, A9, f0

  3327. 	nop

  3328. 	fxcsmadd	f4,  B6, A9, f4

  3329. 	nop

  3330. 	fxcpmadd	f8,  B4, A9, f8

  3331. 	LFPDUX	B2, BO2, INC4

  3332. 	fxcsmadd	f12, B4, A9, f12

  3333. 	LFPDUX	B5,  BO, INC4

  3334. 

  3335. 	fxcpmadd	f1,  B6, A2, f1

  3336. 	nop

  3337. 	fxcsmadd	f5,  B6, A2, f5

  3338. 	LFPDUX	A4, AO2, INC4

  3339. 	fxcpmadd	f9,  B4, A2, f9

  3340. 	LFPDUX	A8,  AO, INC4

  3341. 	fxcsmadd	f13, B4, A2, f13

  3342. 	nop

  3343. 

  3344. 	fxcpmadd	f2,  B6, A10, f2

  3345. 	nop

  3346. 	fxcsmadd	f6,  B6, A10, f6

  3347. 	nop

  3348. 	fxcpmadd	f10, B4, A10, f10

  3349. 	nop

  3350. 	fxcsmadd	f14, B4, A10, f14

  3351. 	nop

  3352. 

  3353. 	fxcpmadd	f3,  B6, A4, f3

  3354. 	LFPDUX	A2, AO2, INC4

  3355. 	fxcsmadd	f7,  B6, A4, f7

  3356. 	LFPDUX	A9,  AO, INC4

  3357. 	fxcpmadd	f11, B4, A4, f11

  3358. 	nop	

  3359. 	fxcsmadd	f15, B4, A4, f15

  3360. 	bdnz+	.L12

  3361. 	.align 4

  3362. 

  3363. .L13:

  3364. ## 1 ##

  3365. 

  3366. 	fxcpmadd	f0,  B1, A1, f0

  3367. 	nop

  3368. 	fxcsmadd	f4,  B1, A1, f4

  3369. 	nop

  3370. 	fxcpmadd	f8,  B2, A1, f8

  3371. 	LFPDUX	B4, BO2, INC4

  3372. 	fxcsmadd	f12, B2, A1, f12

  3373. 	LFPDUX	B6,  BO, INC4

  3374. 

  3375. 	fxcpmadd	f1,  B1, A2, f1

  3376. 	nop

  3377. 	fxcsmadd	f5,  B1, A2, f5

  3378. 	LFPDUX	A4, AO2, INC4

  3379. 	fxcpmadd	f9,  B2, A2, f9

  3380. 	LFPDUX	A10, AO, INC4

  3381. 	fxcsmadd	f13, B2, A2, f13

  3382. 	nop

  3383. 

  3384. 	fxcpmadd	f2,  B1, A3, f2

  3385. 	nop

  3386. 	fxcsmadd	f6,  B1, A3, f6

  3387. 	nop

  3388. 	fxcpmadd	f10, B2, A3, f10

  3389. 	nop

  3390. 	fxcsmadd	f14, B2, A3, f14

  3391. 	nop

  3392. 

  3393. 	fxcpmadd	f3,  B1, A4, f3

  3394. 	nop

  3395. 	fxcsmadd	f7,  B1, A4, f7

  3396. 	LFPDUX	A2, AO2, INC4

  3397. 	fxcpmadd	f11, B2, A4, f11

  3398. 	nop

  3399. 	fxcsmadd	f15, B2, A4, f15

  3400. 	nop

  3401. 

  3402. ## 2 ##

  3403. 

  3404. 	fxcpmadd	f0,  B3, A5, f0

  3405. 	nop

  3406. 	fxcsmadd	f4,  B3, A5, f4

  3407. 	nop

  3408. 	fxcpmadd	f8,  B4, A5, f8

  3409. 	LFPDUX	B2, BO2, INC4

  3410. 	fxcsmadd	f12, B4, A5, f12

  3411. 	nop

  3412. 

  3413. 	fxcpmadd	f1,  B3, A2, f1

  3414. 	nop

  3415. 	fxcsmadd	f5,  B3, A2, f5

  3416. 	LFPDUX	A4, AO2, INC4

  3417. 	fxcpmadd	f9,  B4, A2, f9

  3418. 	nop

  3419. 	fxcsmadd	f13, B4, A2, f13

  3420. 	nop

  3421. 

  3422. 	fxcpmadd	f2,  B3, A6, f2

  3423. 	nop

  3424. 	fxcsmadd	f6,  B3, A6, f6

  3425. 	nop

  3426. 	fxcpmadd	f10, B4, A6, f10

  3427. 	nop

  3428. 	fxcsmadd	f14, B4, A6, f14

  3429. 	nop

  3430. 

  3431. 	fxcpmadd	f3,  B3, A4, f3

  3432. 	nop

  3433. 	fxcsmadd	f7,  B3, A4, f7

  3434. 	LFPDUX	A2, AO2, INC4

  3435. 	fxcpmadd	f11, B4, A4, f11

  3436. 	nop

  3437. 	fxcsmadd	f15, B4, A4, f15

  3438. 	nop

  3439. 

  3440. ## 3 ##

  3441. 

  3442. 	fxcpmadd	f0,  B5, A7, f0

  3443. 	nop

  3444. 	fxcsmadd	f4,  B5, A7, f4

  3445. 	nop

  3446. 	fxcpmadd	f8,  B2, A7, f8

  3447. 	LFPDUX	B4, BO2, INC4

  3448. 	fxcsmadd	f12, B2, A7, f12

  3449. 	nop

  3450. 

  3451. 	fxcpmadd	f1,  B5, A2, f1

  3452. 	nop

  3453. 	fxcsmadd	f5,  B5, A2, f5

  3454. 	LFPDUX	A4, AO2, INC4

  3455. 	fxcpmadd	f9,  B2, A2, f9

  3456. 	nop

  3457. 

  3458. 	fxcsmadd	f13, B2, A2, f13

  3459. 

  3460. 	fxcpmadd	f2,  B5, A8, f2

  3461. 	nop

  3462. 	fxcsmadd	f6,  B5, A8, f6

  3463. 	nop

  3464. 	fxcpmadd	f10, B2, A8, f10

  3465. 	nop

  3466. 	fxcsmadd	f14, B2, A8, f14

  3467. 	nop

  3468. 

  3469. 	fxcpmadd	f3,  B5, A4, f3

  3470. 	nop

  3471. 	fxcsmadd	f7,  B5, A4, f7

  3472. 	LFPDUX	A2, AO2, INC4

  3473. 	fxcpmadd	f11, B2, A4, f11

  3474. 	nop

  3475. 	fxcsmadd	f15, B2, A4, f15

  3476. 	nop

  3477. 

  3478. ## 4 ##

  3479. 

  3480. 	fxcpmadd	f0,  B6, A9, f0

  3481. 	nop

  3482. 	fxcsmadd	f4,  B6, A9, f4

  3483. 	nop

  3484. 	fxcpmadd	f8,  B4, A9, f8

  3485. 	nop

  3486. 	fxcsmadd	f12, B4, A9, f12

  3487. 	nop

  3488. 

  3489. 	fxcpmadd	f1,  B6, A2, f1

  3490. 	nop

  3491. 	fxcsmadd	f5,  B6, A2, f5

  3492. 	LFPDUX	A4, AO2, INC4

  3493. 	fxcpmadd	f9,  B4, A2, f9

  3494. 	nop

  3495. 	fxcsmadd	f13, B4, A2, f13

  3496. 	nop

  3497. 

  3498. 	fxcpmadd	f2,  B6, A10, f2

  3499. 	nop

  3500. 	fxcsmadd	f6,  B6, A10, f6

  3501. 	nop

  3502. 	fxcpmadd	f10, B4, A10, f10

  3503. 	nop

  3504. 	fxcsmadd	f14, B4, A10, f14

  3505. 	nop

  3506. 

  3507. 	fxcpmadd	f3,  B6, A4, f3

  3508. 	nop

  3509. 	fxcsmadd	f7,  B6, A4, f7

  3510. 	nop

  3511. 	fxcpmadd	f11, B4, A4, f11

  3512. 	nop

  3513. 	fxcsmadd	f15, B4, A4, f15

  3514. 	nop

  3515. 	.align 4

  3516. 

  3517. .L14:

  3518. #if defined(LT) || defined(RN)

  3519. 	andi.	r0,  KK,  3

  3520. 	mtspr	CTR, r0

  3521. 	ble+	.L18

  3522. #else

  3523. 	andi.	r0, TEMP, 3

  3524. 	mtspr	CTR, r0

  3525. 	ble+	.L18

  3526. #endif

  3527. 	.align 4

  3528. 

  3529. .L15:

  3530. 	LFPDUX	A2,  AO,  INC4

  3531. 	LFPDUX	A4,  AO2, INC4

  3532. 	LFPDUX	A10, BO,  INC4

  3533. 	LFPDUX	B4,  BO2, INC4

  3534. 	bdz-	.L17

  3535. 	.align 4

  3536. 

  3537. .L16:

  3538. 	fxcpmadd	f0,  A10, A2, f0

  3539. 	fxcsmadd	f4,  A10, A2, f4

  3540. 	fxcpmadd	f8,  B4, A2, f8

  3541. 	fxcsmadd	f12, B4, A2, f12

  3542. 	LFPDUX	A2, AO,  INC4

  3543. 

  3544. 	fxcpmadd	f1,  A10, A4, f1

  3545. 	fxcsmadd	f5,  A10, A4, f5

  3546. 	fxcpmadd	f9,  B4, A4, f9

  3547. 	fxcsmadd	f13, B4, A4, f13

  3548. 	LFPDUX	A4, AO2, INC4

  3549. 

  3550. 	fxcpmadd	f2,  A10, A2, f2

  3551. 	fxcsmadd	f6,  A10, A2, f6

  3552. 	fxcpmadd	f10, B4, A2, f10

  3553. 	fxcsmadd	f14, B4, A2, f14

  3554. 	LFPDUX	A2, AO,  INC4

  3555. 

  3556. 	fxcpmadd	f3,  A10, A4, f3

  3557. 	fxcsmadd	f7,  A10, A4, f7

  3558. 	LFPDUX	A10, BO,  INC4

  3559. 	fxcpmadd	f11, B4, A4, f11

  3560. 	fxcsmadd	f15, B4, A4, f15

  3561. 	LFPDUX	A4, AO2, INC4

  3562. 	LFPDUX	B4, BO2, INC4

  3563. 	bdnz+	.L16

  3564. 	.align 4

  3565. 

  3566. .L17:

  3567. 	fxcpmadd	f0,  A10, A2, f0

  3568. 	fxcsmadd	f4,  A10, A2, f4

  3569. 	fxcpmadd	f8,  B4, A2, f8

  3570. 	fxcsmadd	f12, B4, A2, f12

  3571. 	LFPDUX	A2, AO,  INC4

  3572. 

  3573. 	fxcpmadd	f1,  A10, A4, f1

  3574. 	fxcsmadd	f5,  A10, A4, f5

  3575. 	fxcpmadd	f9,  B4, A4, f9

  3576. 	fxcsmadd	f13, B4, A4, f13

  3577. 	LFPDUX	A4, AO2, INC4

  3578. 

  3579. 	fxcpmadd	f2,  A10, A2, f2

  3580. 	fxcsmadd	f6,  A10, A2, f6

  3581. 	fxcpmadd	f10, B4, A2, f10

  3582. 	fxcsmadd	f14, B4, A2, f14

  3583. 

  3584. 	fxcpmadd	f3,  A10, A4, f3

  3585. 	fxcsmadd	f7,  A10, A4, f7

  3586. 	fxcpmadd	f11, B4, A4, f11

  3587. 	fxcsmadd	f15, B4, A4, f15

  3588. 	.align 4

  3589. 

  3590. .L18:

  3591. #if defined(LN) || defined(RT)

  3592. #ifdef LN

  3593. 	subi	r0, KK, 8

  3594. #else

  3595. 	subi	r0, KK, 4

  3596. #endif

  3597. 	slwi	TEMP, r0, 3 + BASE_SHIFT

  3598. 	slwi	r0,   r0, 2 + BASE_SHIFT

  3599. 	add	AO, AORIG, TEMP

  3600. 	add	BO, B,     r0

  3601. 	addi	AO2, AO,   2 * SIZE

  3602. 	addi	BO,  BO, - 4 * SIZE

  3603. 	addi	BO2, BO,   2 * SIZE

  3604. #endif

  3605. 

  3606. #if defined(LN) || defined(LT)

  3607. 	fpmr	f24, f0

  3608. 	LFPDUX	f16, BO,  INC4

  3609. 	fpmr	f25, f1

  3610. 	nop

  3611. 	fpmr	f26, f2

  3612. 	LFPDUX	f17, BO2, INC4

  3613. 	fpmr	f27, f3

  3614. 	nop

  3615. 

  3616. 	fpmr	f28, f8

  3617. 	LFPDUX	f18, BO,  INC4

  3618. 	fpmr	f29, f9

  3619. 	nop

  3620. 	fpmr	f30, f10

  3621. 	LFPDUX	f19, BO2, INC4

  3622. 	fpmr	f31, f11

  3623. 	nop

  3624. 

  3625. 	fsmfp	f0,  f4

  3626.  	LFPDUX	f20, BO,  INC4

  3627. 	fsmfp	f1,  f5

  3628. 	nop

  3629. 	fsmfp	f2,  f6

  3630. 	LFPDUX	f21, BO2, INC4

  3631. 	fsmfp	f3,  f7

  3632. 	nop

  3633. 

  3634. 	fsmfp	f8,  f12

  3635. 	LFPDUX	f22, BO,  INC4

  3636. 	fsmfp	f9,  f13

  3637. 	nop

  3638. 	fsmfp	f10, f14

  3639. 	LFPDUX	f23, BO2, INC4

  3640. 	fsmfp	f11, f15

  3641. 	nop

  3642. 

  3643. 	fsmtp	f4,  f24

  3644. 	LFPDUX	f24, BO,  INC4

  3645. 	fsmtp	f5,  f25

  3646. 	nop

  3647. 	fsmtp	f6,  f26

  3648. 	LFPDUX	f25, BO2, INC4

  3649. 	fsmtp	f7,  f27

  3650. 	nop

  3651. 

  3652. 	fsmtp	f12, f28

  3653. 	LFPDUX	f26, BO,  INC4

  3654. 	fsmtp	f13, f29

  3655. 	nop

  3656. 	fsmtp	f14, f30

  3657. 	LFPDUX	f27, BO2, INC4

  3658. 	fsmtp	f15, f31

  3659. 	nop

  3660. 

  3661. 	fpsub	f0,  f16,  f0

  3662. 	LFPDUX	f28, BO,  INC4

  3663. 	fpsub	f8,  f17,  f8

  3664. 	nop

  3665. 	fpsub	f4,  f18,  f4

  3666. 	LFPDUX	f29, BO2, INC4

  3667. 	fpsub	f12, f19,  f12

  3668. 	nop

  3669. 

  3670. 	fpsub	f1,  f20,  f1

  3671. 	LFPDUX	f30, BO,  INC4

  3672. 	fpsub	f9,  f21,  f9

  3673. 	subi	BO,  BO,  32 * SIZE

  3674. 	fpsub	f5,  f22,  f5

  3675. 	LFPDUX	f31, BO2, INC4

  3676. 	fpsub	f13, f23,  f13

  3677. 	subi	BO2, BO2, 32 * SIZE

  3678. 

  3679. 	fpsub	f2,  f24,  f2

  3680. 	fpsub	f10, f25,  f10

  3681. 	fpsub	f6,  f26,  f6

  3682. 	fpsub	f14, f27,  f14

  3683. 	fpsub	f3,  f28,  f3

  3684. 	fpsub	f11, f29,  f11

  3685. 	fpsub	f7,  f30,  f7

  3686. 	fpsub	f15, f31,  f15

  3687. 

  3688. #else

  3689. 	LFPDUX	f16, AO,  INC4

  3690. 	LFPDUX	f17, AO2, INC4

  3691. 	LFPDUX	f18, AO,  INC4

  3692. 	LFPDUX	f19, AO2, INC4

  3693. 	LFPDUX	f20, AO,  INC4

  3694. 	LFPDUX	f21, AO2, INC4

  3695. 	LFPDUX	f22, AO,  INC4

  3696. 	LFPDUX	f23, AO2, INC4

  3697. 

  3698. 	fpsub	f0,  f16,  f0

  3699. 	LFPDUX	f24, AO,  INC4

  3700. 	fpsub	f1,  f17,  f1

  3701. 	LFPDUX	f25, AO2, INC4

  3702. 	fpsub	f2,  f18,  f2

  3703. 	LFPDUX	f26, AO,  INC4

  3704. 	fpsub	f3,  f19,  f3

  3705. 	LFPDUX	f27, AO2, INC4

  3706. 	fpsub	f4,  f20,  f4

  3707. 	LFPDUX	f28, AO,  INC4

  3708. 	fpsub	f5,  f21,  f5

  3709. 	LFPDUX	f29, AO2, INC4

  3710. 	fpsub	f6,  f22,  f6

  3711. 	LFPDUX	f30, AO,  INC4

  3712. 	fpsub	f7,  f23,  f7

  3713. 	LFPDUX	f31, AO2, INC4

  3714. 

  3715. 	fpsub	f8,  f24,  f8

  3716. 	subi	AO,  AO,  32 * SIZE

  3717. 	fpsub	f9,  f25,  f9

  3718. 	subi	AO2, AO2, 32 * SIZE

  3719. 	fpsub	f10, f26,  f10

  3720. 	fpsub	f11, f27,  f11

  3721. 	fpsub	f12, f28,  f12

  3722. 	fpsub	f13, f29,  f13

  3723. 	fpsub	f14, f30,  f14

  3724. 	fpsub	f15, f31,  f15

  3725. #endif

  3726. 

  3727. #ifdef LN

  3728.        addi	AO,  AO,  68 * SIZE

  3729.        addi	AO2, AO2, 68 * SIZE

  3730. 

  3731. 	LFPDUX	A1,  AO2, INCM4

  3732. 	LFPDUX	A2,  AO,  INCM4

  3733. 	LFPDUX	A3,  AO2, INCM4

  3734. 	LFPDUX	A4,  AO,  INCM4

  3735. 	LFPDUX	A5,  AO2, INCM4

  3736. 	LFPDUX	A6,  AO,  INCM4

  3737. 	LFPDUX	A7,  AO2, INCM4

  3738. 	LFPDUX	A8,  AO,  INCM4

  3739. 

  3740. 	fxsmul	f7,  A1, f7

  3741. 	fxsmul	f15, A1, f15

  3742. 

  3743. 	fxcpnmsub  f3,  A1, f7,  f3

  3744. 	fxcpnmsub  f11, A1, f15, f11

  3745. 

  3746. 	fxcsnmsub  f6,  A2, f7,  f6

  3747. 	fxcsnmsub  f14, A2, f15, f14

  3748. 

  3749. 	fxcpnmsub  f2,  A2, f7,  f2

  3750. 	fxcpnmsub  f10, A2, f15, f10

  3751. 

  3752. 	fxcsnmsub  f5,  A3, f7,  f5

  3753. 	fxcsnmsub  f13, A3, f15, f13

  3754. 

  3755. 	fxcpnmsub  f1,  A3, f7,  f1

  3756. 	fxcpnmsub  f9,  A3, f15, f9

  3757. 

  3758. 	fxcsnmsub  f4,  A4, f7,  f4

  3759. 	fxcsnmsub  f12, A4, f15, f12

  3760. 

  3761. 	fxcpnmsub  f0,  A4, f7,  f0

  3762. 	fxcpnmsub  f8,  A4, f15, f8

  3763. 

  3764. 	fxpmul	f3,  A5, f3

  3765. 	fxpmul	f11, A5, f11

  3766. 

  3767. 	fxcsnmsub  f6,  A6, f3,  f6

  3768. 	fxcsnmsub  f14, A6, f11, f14

  3769. 

  3770. 	fxcpnmsub  f2,  A6, f3,  f2

  3771. 	fxcpnmsub  f10, A6, f11, f10

  3772. 

  3773. 	fxcsnmsub  f5,  A7, f3,  f5

  3774. 	fxcsnmsub  f13, A7, f11, f13

  3775. 

  3776. 	fxcpnmsub  f1,  A7, f3,  f1

  3777. 	fxcpnmsub  f9,  A7, f11, f9

  3778. 

  3779. 	fxcsnmsub  f4,  A8, f3,  f4

  3780. 	fxcsnmsub  f12, A8, f11, f12

  3781. 

  3782. 	fxcpnmsub  f0,  A8, f3,  f0

  3783. 	fxcpnmsub  f8,  A8, f11, f8

  3784. 

  3785. 	add	AO2, AO2, INCM4

  3786. 	LFPDUX	A1,  AO,  INCM4

  3787. 	LFPDUX	A2,  AO2, INCM4

  3788. 	LFPDUX	A3,  AO,  INCM4

  3789. 

  3790. 	add	AO2, AO2, INCM4

  3791. 	LFPDUX	A4,  AO,  INCM4

  3792. 	LFPDUX	A5,  AO2, INCM4

  3793. 	LFPDUX	A6,  AO,  INCM4

  3794. 

  3795. 	add	AO2, AO2, INCM4

  3796. 	add	AO,  AO,  INCM4

  3797. 	LFPDUX	A7,  AO2, INCM4

  3798. 	LFPDUX	A8,  AO,  INCM4

  3799. 

  3800. 

  3801. 	fxsmul	f6,  A1, f6

  3802. 	fxsmul	f14, A1, f14

  3803. 

  3804. 	fxcpnmsub  f2,  A1, f6,  f2

  3805. 	fxcpnmsub  f10, A1, f14, f10

  3806. 

  3807. 	fxcsnmsub  f5,  A2, f6,  f5

  3808. 	fxcsnmsub  f13, A2, f14, f13

  3809. 

  3810. 	fxcpnmsub  f1,  A2, f6,  f1

  3811. 	fxcpnmsub  f9,  A2, f14, f9

  3812. 

  3813. 	fxcsnmsub  f4,  A3, f6,  f4

  3814. 	fxcsnmsub  f12, A3, f14, f12

  3815. 

  3816. 	fxcpnmsub  f0,  A3, f6,  f0

  3817. 	fxcpnmsub  f8,  A3, f14, f8

  3818. 

  3819. 	fxpmul	f2,  A4, f2

  3820. 	fxpmul	f10, A4, f10

  3821. 

  3822. 	fxcsnmsub  f5,  A5, f2,  f5

  3823. 	fxcsnmsub  f13, A5, f10, f13

  3824. 

  3825. 	fxcpnmsub  f1,  A5, f2,  f1

  3826. 	fxcpnmsub  f9,  A5, f10, f9

  3827. 

  3828. 	fxcsnmsub  f4,  A6, f2,  f4

  3829. 	fxcsnmsub  f12, A6, f10, f12

  3830. 

  3831. 	fxcpnmsub  f0,  A6, f2,  f0

  3832. 	fxcpnmsub  f8,  A6, f10, f8

  3833. 

  3834. 	fxsmul	f5,  A7, f5

  3835. 	fxsmul	f13, A7, f13

  3836. 

  3837. 	fxcpnmsub  f1,  A7, f5,  f1

  3838. 	fxcpnmsub  f9,  A7, f13, f9

  3839. 

  3840. 	fxcsnmsub  f4,  A8, f5,  f4

  3841. 	fxcsnmsub  f12, A8, f13, f12

  3842. 

  3843. 	fxcpnmsub  f0,  A8, f5,  f0

  3844. 	fxcpnmsub  f8,  A8, f13, f8

  3845. 

  3846. 	add	AO2, AO2, INCM4

  3847. 	add	AO,  AO,  INCM4

  3848. 	LFPDUX	A1,  AO2, INCM4

  3849. 	LFPDUX	A2,  AO,  INCM4

  3850. 

  3851. 	subi	AO2, AO2, 8 * SIZE

  3852. 	add	AO,  AO,  INCM4

  3853. 	LFPDUX	A3,  AO,  INCM4

  3854. 

  3855. 	subi	AO2, AO2, 8 * SIZE

  3856. 	add	AO,  AO,  INCM4

  3857. 	LFPDUX	A4,  AO,  INCM4

  3858. 

  3859. 	addi	AO,  AO,  -4 * SIZE

  3860. 	addi	AO2, AO2, -4 * SIZE

  3861. 

  3862. 	fxpmul	f1,  A1, f1

  3863. 	fxpmul	f9,  A1, f9

  3864. 

  3865. 	fxcsnmsub  f4,  A2, f1,  f4

  3866. 	fxcsnmsub  f12, A2, f9,  f12

  3867. 

  3868. 	fxcpnmsub  f0,  A2, f1,  f0

  3869. 	fxcpnmsub  f8,  A2, f9,  f8

  3870. 

  3871. 	fxsmul	f4,  A3, f4

  3872. 	fxsmul	f12, A3, f12

  3873. 

  3874. 	fxcpnmsub  f0,  A3, f4,  f0

  3875. 	fxcpnmsub  f8,  A3, f12, f8

  3876. 

  3877. 	fxpmul	f0,  A4, f0

  3878. 	fxpmul	f8,  A4, f8

  3879. 

  3880. #endif

  3881. 

  3882. #ifdef LT

  3883. 	LFPDUX	A1,  AO,  INC4

  3884. 	LFPDUX	A2,  AO2, INC4

  3885. 	LFPDUX	A3,  AO,  INC4

  3886. 	LFPDUX	A4,  AO2, INC4

  3887. 

  3888. 	LFPDUX	A5,  AO,  INC4

  3889. 	LFPDUX	A6,  AO2, INC4

  3890. 	LFPDUX	A7,  AO,  INC4

  3891. 	LFPDUX	A8,  AO2, INC4

  3892. 

  3893. 	fxpmul	f0,  A1,  f0

  3894. 	fxpmul	f8,  A1,  f8

  3895. 

  3896. 	fxcsnmsub  f4,  A1, f0, f4

  3897. 	fxcsnmsub  f12, A1, f8, f12

  3898. 

  3899. 	fxcpnmsub  f1,  A2, f0, f1

  3900. 	fxcpnmsub  f9,  A2, f8, f9

  3901. 

  3902. 	fxcsnmsub  f5,  A2, f0, f5

  3903. 	fxcsnmsub  f13, A2, f8, f13

  3904. 

  3905. 	fxcpnmsub  f2,  A3, f0, f2

  3906. 	fxcpnmsub  f10, A3, f8, f10

  3907. 

  3908. 	fxcsnmsub  f6,  A3, f0, f6

  3909. 	fxcsnmsub  f14, A3, f8, f14

  3910. 

  3911. 	fxcpnmsub  f3,  A4, f0, f3

  3912. 	fxcpnmsub  f11, A4, f8, f11

  3913. 

  3914. 	fxcsnmsub  f7,  A4, f0, f7

  3915. 	fxcsnmsub  f15, A4, f8, f15

  3916. 

  3917. 	fxsmul	f4,  A5,  f4

  3918. 	fxsmul	f12, A5,  f12

  3919. 

  3920. 	fxcpnmsub  f1,  A6, f4,  f1

  3921. 	fxcpnmsub  f9,  A6, f12, f9

  3922. 

  3923. 	fxcsnmsub  f5,  A6, f4,  f5

  3924. 	fxcsnmsub  f13, A6, f12, f13

  3925. 

  3926. 	fxcpnmsub  f2,  A7, f4,  f2

  3927. 	fxcpnmsub  f10, A7, f12, f10

  3928. 

  3929. 	fxcsnmsub  f6,  A7, f4,  f6

  3930. 	fxcsnmsub  f14, A7, f12, f14

  3931. 

  3932. 	fxcpnmsub  f3,  A8, f4,  f3

  3933. 	fxcpnmsub  f11, A8, f12, f11

  3934. 

  3935. 	fxcsnmsub  f7,  A8, f4,  f7

  3936. 	fxcsnmsub  f15, A8, f12, f15

  3937. 

  3938. 	add	AO,  AO,  INC4

  3939. 	LFPDUX	A1,  AO2, INC4

  3940. 	LFPDUX	A2,  AO,  INC4

  3941. 	LFPDUX	A3,  AO2, INC4

  3942. 

  3943. 	add	AO,  AO,  INC4

  3944. 	LFPDUX	A4,  AO2, INC4

  3945. 	LFPDUX	A5,  AO,  INC4

  3946. 	LFPDUX	A6,  AO2, INC4

  3947. 

  3948. 	add	AO,  AO,  INC4

  3949. 	add	AO2, AO2, INC4

  3950. 	LFPDUX	A7,  AO,  INC4

  3951. 	LFPDUX	A8,  AO2, INC4

  3952. 

  3953. 	fxpmul	f1,  A1,  f1

  3954. 	fxpmul	f9,  A1,  f9

  3955. 

  3956. 	fxcsnmsub  f5,  A1, f1, f5

  3957. 	fxcsnmsub  f13, A1, f9, f13

  3958. 

  3959. 	fxcpnmsub  f2,  A2, f1, f2

  3960. 	fxcpnmsub  f10, A2, f9, f10

  3961. 

  3962. 	fxcsnmsub  f6,  A2, f1, f6

  3963. 	fxcsnmsub  f14, A2, f9, f14

  3964. 

  3965. 	fxcpnmsub  f3,  A3, f1, f3

  3966. 	fxcpnmsub  f11, A3, f9, f11

  3967. 

  3968. 	fxcsnmsub  f7,  A3, f1, f7

  3969. 	fxcsnmsub  f15, A3, f9, f15

  3970. 

  3971. 	fxsmul	f5,  A4,  f5

  3972. 	fxsmul	f13, A4,  f13

  3973. 

  3974. 	fxcpnmsub  f2,  A5, f5,  f2

  3975. 	fxcpnmsub  f10, A5, f13, f10

  3976. 

  3977. 	fxcsnmsub  f6,  A5, f5,  f6

  3978. 	fxcsnmsub  f14, A5, f13, f14

  3979. 

  3980. 	fxcpnmsub  f3,  A6, f5,  f3

  3981. 	fxcpnmsub  f11, A6, f13, f11

  3982. 

  3983. 	fxcsnmsub  f7,  A6, f5,  f7

  3984. 	fxcsnmsub  f15, A6, f13, f15

  3985. 

  3986. 	fxpmul	f2,  A7,  f2

  3987. 	fxpmul	f10, A7,  f10

  3988. 

  3989. 	fxcsnmsub  f6,  A7, f2,  f6

  3990. 	fxcsnmsub  f14, A7, f10, f14

  3991. 

  3992. 	fxcpnmsub  f3,  A8, f2,  f3

  3993. 	fxcpnmsub  f11, A8, f10, f11

  3994. 

  3995. 	fxcsnmsub  f7,  A8, f2,  f7

  3996. 	fxcsnmsub  f15, A8, f10, f15

  3997. 

  3998. 	add	AO,  AO,  INC4

  3999. 	add	AO2, AO2, INC4

  4000. 	LFPDUX	A1,  AO,  INC4

  4001. 	LFPDUX	A2,  AO2, INC4

  4002. 

  4003. 	addi	AO,  AO,  8 * SIZE

  4004. 	addi	AO2, AO2, 4 * SIZE

  4005. 	LFPDUX	A3,  AO2, INC4

  4006. 

  4007. 	addi	AO,  AO,  8 * SIZE

  4008. 	addi	AO2, AO2, 4 * SIZE

  4009. 	LFPDUX	A4,  AO2, INC4

  4010. 

  4011. 	subi	AO,  AO,  64 * SIZE

  4012. 	subi	AO2, AO2, 64 * SIZE

  4013. 

  4014. 	fxsmul	f6,  A1,  f6

  4015. 	fxsmul	f14, A1,  f14

  4016. 

  4017. 	fxcpnmsub  f3,  A2, f6,  f3

  4018. 	fxcpnmsub  f11, A2, f14, f11

  4019. 

  4020. 	fxcsnmsub  f7,  A2, f6,  f7

  4021. 	fxcsnmsub  f15, A2, f14, f15

  4022. 

  4023. 	fxpmul	f3,  A3,  f3

  4024. 	fxpmul	f11, A3,  f11

  4025. 

  4026. 	fxcsnmsub  f7,  A3, f3,  f7

  4027. 	fxcsnmsub  f15, A3, f11, f15

  4028. 

  4029. 	fxsmul	f7,  A4,  f7

  4030. 	fxsmul	f15, A4,  f15

  4031. #endif

  4032. 

  4033. #ifdef RN

  4034. 	LFPDUX	A1,  BO,  INC4

  4035. 	LFPDUX	A2,  BO2, INC4

  4036. 	LFPDUX	A3,  BO,  INC4

  4037. 	LFPDUX	A4,  BO2, INC4

  4038. 

  4039. 	add	BO,  BO,  INC4

  4040. 	LFPDUX	A5,  BO2, INC4

  4041. 

  4042. 	add	BO,  BO,  INC4

  4043. 	LFPDUX	A6,  BO2, INC4

  4044. 	subi	BO,  BO,  16 * SIZE

  4045. 	subi	BO2, BO2, 16 * SIZE

  4046. 

  4047. 	fxpmul	f0,  A1,  f0

  4048. 	fxpmul	f1,  A1,  f1

  4049. 	fxpmul	f2,  A1,  f2

  4050. 	fxpmul	f3,  A1,  f3

  4051. 

  4052. 	fxcsnmsub    f4,  A1, f0, f4

  4053. 	fxcsnmsub    f5,  A1, f1, f5

  4054. 	fxcsnmsub    f6,  A1, f2, f6

  4055. 	fxcsnmsub    f7,  A1, f3, f7

  4056. 

  4057. 	fxcpnmsub    f8,  A2, f0, f8

  4058. 	fxcpnmsub    f9,  A2, f1, f9

  4059. 	fxcpnmsub    f10, A2, f2, f10

  4060. 	fxcpnmsub    f11, A2, f3, f11

  4061. 

  4062. 	fxcsnmsub    f12, A2, f0, f12

  4063. 	fxcsnmsub    f13, A2, f1, f13

  4064. 	fxcsnmsub    f14, A2, f2, f14

  4065. 	fxcsnmsub    f15, A2, f3, f15

  4066. 

  4067. 	fxsmul	f4,  A3,  f4

  4068. 	fxsmul	f5,  A3,  f5

  4069. 	fxsmul	f6,  A3,  f6

  4070. 	fxsmul	f7,  A3,  f7

  4071. 

  4072. 	fxcpnmsub    f8,  A4, f4, f8

  4073. 	fxcpnmsub    f9,  A4, f5, f9

  4074. 	fxcpnmsub    f10, A4, f6, f10

  4075. 	fxcpnmsub    f11, A4, f7, f11

  4076. 

  4077. 	fxcsnmsub    f12, A4, f4, f12

  4078. 	fxcsnmsub    f13, A4, f5, f13

  4079. 	fxcsnmsub    f14, A4, f6, f14

  4080. 	fxcsnmsub    f15, A4, f7, f15

  4081. 

  4082. 	fxpmul	f8,  A5,  f8

  4083. 	fxpmul	f9,  A5,  f9

  4084. 	fxpmul	f10, A5,  f10

  4085. 	fxpmul	f11, A5,  f11

  4086. 

  4087. 	fxcsnmsub    f12, A5, f8,  f12

  4088. 	fxcsnmsub    f13, A5, f9,  f13

  4089. 	fxcsnmsub    f14, A5, f10, f14

  4090. 	fxcsnmsub    f15, A5, f11, f15

  4091. 

  4092. 	fxsmul	f12,  A6,  f12

  4093. 	fxsmul	f13,  A6,  f13

  4094. 	fxsmul	f14,  A6,  f14

  4095. 	fxsmul	f15,  A6,  f15

  4096. 

  4097. #endif

  4098. 

  4099. #ifdef RT

  4100. 	addi	BO,  BO,  20 * SIZE

  4101. 	addi	BO2, BO2, 20 * SIZE

  4102. 

  4103. 	LFPDUX	A1,  BO2, INCM4

  4104. 	LFPDUX	A2,  BO,  INCM4

  4105. 

  4106. 	LFPDUX	A3,  BO2, INCM4

  4107. 	LFPDUX	A4,  BO,  INCM4

  4108. 

  4109. 	add	BO2, BO2, INCM4

  4110. 	LFPDUX	A5,  BO,  INCM4

  4111. 

  4112. 	add	BO2, BO2, INCM4

  4113. 	LFPDUX	A6,  BO,  INCM4

  4114. 	subi	BO,  BO,  4 * SIZE

  4115. 	subi	BO2, BO2, 4 * SIZE

  4116. 

  4117. 	fxsmul	f12, A1,  f12

  4118. 	fxsmul	f13, A1,  f13

  4119. 	fxsmul	f14, A1,  f14

  4120. 	fxsmul	f15, A1,  f15

  4121. 

  4122. 	fxcpnmsub    f8,  A1, f12, f8

  4123. 	fxcpnmsub    f9,  A1, f13, f9

  4124. 	fxcpnmsub    f10, A1, f14, f10

  4125. 	fxcpnmsub    f11, A1, f15, f11

  4126. 

  4127. 	fxcsnmsub    f4,  A2, f12, f4

  4128. 	fxcsnmsub    f5,  A2, f13, f5

  4129. 	fxcsnmsub    f6,  A2, f14, f6

  4130. 	fxcsnmsub    f7,  A2, f15, f7

  4131. 

  4132. 	fxcpnmsub    f0,  A2, f12, f0

  4133. 	fxcpnmsub    f1,  A2, f13, f1

  4134. 	fxcpnmsub    f2,  A2, f14, f2

  4135. 	fxcpnmsub    f3,  A2, f15, f3

  4136. 

  4137. 	fxpmul	f8,  A3,  f8

  4138. 	fxpmul	f9,  A3,  f9

  4139. 	fxpmul	f10, A3,  f10

  4140. 	fxpmul	f11, A3,  f11

  4141. 

  4142. 	fxcsnmsub    f4,  A4, f8,  f4

  4143. 	fxcsnmsub    f5,  A4, f9,  f5

  4144. 	fxcsnmsub    f6,  A4, f10, f6

  4145. 	fxcsnmsub    f7,  A4, f11, f7

  4146. 

  4147. 	fxcpnmsub    f0,  A4, f8,  f0

  4148. 	fxcpnmsub    f1,  A4, f9,  f1

  4149. 	fxcpnmsub    f2,  A4, f10, f2

  4150. 	fxcpnmsub    f3,  A4, f11, f3

  4151. 

  4152. 	fxsmul	f4,  A5,  f4

  4153. 	fxsmul	f5,  A5,  f5

  4154. 	fxsmul	f6,  A5,  f6

  4155. 	fxsmul	f7,  A5,  f7

  4156. 

  4157. 	fxcpnmsub    f0,  A5, f4,  f0

  4158. 	fxcpnmsub    f1,  A5, f5,  f1

  4159. 	fxcpnmsub    f2,  A5, f6,  f2

  4160. 	fxcpnmsub    f3,  A5, f7,  f3

  4161. 

  4162. 	fxpmul	f0,  A6,  f0

  4163. 	fxpmul	f1,  A6,  f1

  4164. 	fxpmul	f2,  A6,  f2

  4165. 	fxpmul	f3,  A6,  f3

  4166. 

  4167. #endif

  4168. 

  4169. #ifdef LN

  4170. 	subi	CO1, CO1, 8 * SIZE

  4171. 	subi	CO2, CO2, 8 * SIZE

  4172. 	subi	CO3, CO3, 8 * SIZE

  4173. 	subi	CO4, CO4, 8 * SIZE

  4174. #endif

  4175. 

  4176. #if defined(LN) || defined(LT)

  4177. 	STFPDUX	f0,  BO,  INC4

  4178. 	STFPDUX	f8,  BO2, INC4

  4179. 	STFPDUX	f4,  BO,  INC4

  4180. 	STFPDUX	f12, BO2, INC4

  4181. 	STFPDUX	f1,  BO,  INC4

  4182. 	STFPDUX	f9,  BO2, INC4

  4183. 	STFPDUX	f5,  BO,  INC4

  4184. 	STFPDUX	f13, BO2, INC4

  4185. 	STFPDUX	f2,  BO,  INC4

  4186. 	STFPDUX	f10, BO2, INC4

  4187. 	STFPDUX	f6,  BO,  INC4

  4188. 	STFPDUX	f14, BO2, INC4

  4189. 	STFPDUX	f3,  BO,  INC4

  4190. 	STFPDUX	f11, BO2, INC4

  4191. 	STFPDUX	f7,  BO,  INC4

  4192. 	STFPDUX	f15, BO2, INC4

  4193. 

  4194. 	subi	BO,  BO,  32 * SIZE

  4195. 	subi	BO2, BO2, 32 * SIZE

  4196. 

  4197. 	STFDUX	f0,  CO1, INC

  4198. 	STFDUX	f4,  CO1, INC

  4199. 	STFDUX	f1,  CO1, INC

  4200. 	STFDUX	f5,  CO1, INC

  4201. 	STFDUX	f2,  CO1, INC

  4202. 	STFDUX	f6,  CO1, INC

  4203. 	STFDUX	f3,  CO1, INC

  4204. 	STFDUX	f7,  CO1, INC

  4205. 

  4206. 	STFSDUX	f0,  CO2, INC

  4207. 	STFSDUX	f4,  CO2, INC

  4208. 	STFSDUX	f1,  CO2, INC

  4209. 	STFSDUX	f5,  CO2, INC

  4210. 	STFSDUX	f2,  CO2, INC

  4211. 	STFSDUX	f6,  CO2, INC

  4212. 	STFSDUX	f3,  CO2, INC

  4213. 	STFSDUX	f7,  CO2, INC

  4214. 

  4215. 	STFDUX	f8,  CO3, INC

  4216. 	STFDUX	f12, CO3, INC

  4217. 	STFDUX	f9,  CO3, INC

  4218. 	STFDUX	f13, CO3, INC

  4219. 	STFDUX	f10, CO3, INC

  4220. 	STFDUX	f14, CO3, INC

  4221. 	STFDUX	f11, CO3, INC

  4222. 	STFDUX	f15, CO3, INC

  4223. 

  4224. 	STFSDUX	f8,  CO4, INC

  4225. 	STFSDUX	f12, CO4, INC

  4226. 	STFSDUX	f9,  CO4, INC

  4227. 	STFSDUX	f13, CO4, INC

  4228. 	STFSDUX	f10, CO4, INC

  4229. 	STFSDUX	f14, CO4, INC

  4230. 	STFSDUX	f11, CO4, INC

  4231. 	STFSDUX	f15, CO4, INC

  4232. 

  4233. #else

  4234. 	STFPDUX	f0,  AO,  INC4

  4235. 	STFPDUX	f1,  AO2, INC4

  4236. 	STFPDUX	f2,  AO,  INC4

  4237. 	STFPDUX	f3,  AO2, INC4

  4238. 	STFPDUX	f4,  AO,  INC4

  4239. 	STFPDUX	f5,  AO2, INC4

  4240. 	STFPDUX	f6,  AO,  INC4

  4241. 	STFPDUX	f7,  AO2, INC4

  4242. 	STFPDUX	f8,  AO,  INC4

  4243. 	STFPDUX	f9,  AO2, INC4

  4244. 	STFPDUX	f10, AO,  INC4

  4245. 	STFPDUX	f11, AO2, INC4

  4246. 	STFPDUX	f12, AO,  INC4

  4247. 	STFPDUX	f13, AO2, INC4

  4248. 	STFPDUX	f14, AO,  INC4

  4249. 	STFPDUX	f15, AO2, INC4

  4250. 

  4251. 	subi	AO,  AO,  32 * SIZE

  4252. 	subi	AO2, AO2, 32 * SIZE

  4253. 

  4254. 	STFDUX	f0,  CO1, INC

  4255. 	STFSDUX	f0,  CO1, INC

  4256. 	STFDUX	f1,  CO1, INC

  4257. 	STFSDUX	f1,  CO1, INC

  4258. 	STFDUX	f2,  CO1, INC

  4259. 	STFSDUX	f2,  CO1, INC

  4260. 	STFDUX	f3,  CO1, INC

  4261. 	STFSDUX	f3,  CO1, INC

  4262. 

  4263. 	STFDUX	f4,  CO2, INC

  4264. 	STFSDUX	f4,  CO2, INC

  4265. 	STFDUX	f5,  CO2, INC

  4266. 	STFSDUX	f5,  CO2, INC

  4267. 	STFDUX	f6,  CO2, INC

  4268. 	STFSDUX	f6,  CO2, INC

  4269. 	STFDUX	f7,  CO2, INC

  4270. 	STFSDUX	f7,  CO2, INC

  4271. 

  4272. 	STFDUX	f8,  CO3, INC

  4273. 	STFSDUX	f8,  CO3, INC

  4274. 	STFDUX	f9,  CO3, INC

  4275. 	STFSDUX	f9,  CO3, INC

  4276. 	STFDUX	f10, CO3, INC

  4277. 	STFSDUX	f10, CO3, INC

  4278. 	STFDUX	f11, CO3, INC

  4279. 	STFSDUX	f11, CO3, INC

  4280. 

  4281. 	STFDUX	f12, CO4, INC

  4282. 	STFSDUX	f12, CO4, INC

  4283. 	STFDUX	f13, CO4, INC

  4284. 	STFSDUX	f13, CO4, INC

  4285. 	STFDUX	f14, CO4, INC

  4286. 	STFSDUX	f14, CO4, INC

  4287. 	STFDUX	f15, CO4, INC

  4288. 	STFSDUX	f15, CO4, INC

  4289. #endif

  4290. 

  4291. #ifdef LN

  4292. 	subi	CO1, CO1, 8 * SIZE

  4293. 	subi	CO2, CO2, 8 * SIZE

  4294. 	subi	CO3, CO3, 8 * SIZE

  4295. 	subi	CO4, CO4, 8 * SIZE

  4296. #endif

  4297. 

  4298. #ifdef RT

  4299. 	slwi	r0, K, 3 + BASE_SHIFT

  4300. 	add	AORIG, AORIG, r0

  4301. #endif

  4302. 

  4303. #if defined(LT) || defined(RN)

  4304. 	sub	TEMP, K, KK

  4305. 	slwi	r0,   TEMP, 3 + BASE_SHIFT

  4306. 	slwi	TEMP, TEMP, 2 + BASE_SHIFT

  4307. 	add	AO, AO, r0

  4308. 	add	BO, BO, TEMP

  4309. #endif

  4310. 

  4311. #ifdef LT

  4312. 	addi	KK, KK, 8

  4313. #endif

  4314. 

  4315. #ifdef LN

  4316. 	subi	KK, KK, 8

  4317. #endif

  4318. 

  4319. 	addic.	I, I, -1

  4320. 	li	r0, FZERO

  4321. 

  4322. 	lfpsx	f0, SP, r0

  4323. 	bgt+	.L11

  4324. 	.align 4

  4325. 

  4326. .L20:

  4327. 	andi.	I, M,  4

  4328. 	beq	.L30

  4329. 

  4330. #if defined(LT) || defined(RN)

  4331. 	addi	AO2, AO,   2 * SIZE

  4332. 	fpmr	f4,  f0

  4333. 	addi	BO,  B,  - 4 * SIZE

  4334. 	fpmr	f8,  f0

  4335. 	addi	BO2, B,  - 2 * SIZE

  4336. 	fpmr	f12, f0

  4337. 

  4338. 	srawi.	r0,  KK,  2

  4339.  	fpmr	f1,  f0

  4340. 	fpmr	f5,  f0

  4341. 	fpmr	f9,  f0

  4342. 	mtspr	CTR, r0

  4343. 	fpmr	f13, f0

  4344. 	ble	.L24

  4345. #else

  4346. 

  4347. #ifdef LN

  4348. 	slwi	r0,   K,  2 + BASE_SHIFT

  4349. 	sub	AORIG, AORIG, r0

  4350. #endif

  4351. 

  4352. 	slwi	r0  , KK, 2 + BASE_SHIFT

  4353. 	slwi	TEMP, KK, 2 + BASE_SHIFT

  4354. 	add	AO, AORIG, r0

  4355. 	add	BO, B,     TEMP

  4356. 

  4357. 	sub	TEMP, K, KK

  4358. 

  4359. 	addi	AO2, AO,   2 * SIZE

  4360. 	fpmr	f4,  f0

  4361. 	addi	BO,  BO,  - 4 * SIZE

  4362. 	fpmr	f8,  f0

  4363. 	addi	BO2, BO,    2 * SIZE

  4364. 	fpmr	f12, f0

  4365. 

  4366. 	srawi.	r0,  TEMP,  2

  4367.  	fpmr	f1,  f0

  4368. 	fpmr	f5,  f0

  4369. 	fpmr	f9,  f0

  4370. 	mtspr	CTR, r0

  4371. 	fpmr	f13, f0

  4372. 	ble	.L24

  4373. #endif

  4374. 

  4375. 	LFPDUX	A1,   AO, INC4

  4376. 	LFPDUX	B1,   BO, INC4

  4377. 	LFPDUX	A2,  AO2, INC4

  4378. 	LFPDUX	B2,  BO2, INC4

  4379. 	LFPDUX	A3,   AO, INC4

  4380. 	LFPDUX	B3,   BO, INC4

  4381. 	LFPDUX	A4,  AO2, INC4

  4382. 	LFPDUX	B4,  BO2, INC4

  4383. 

  4384. 	LFPDUX	A5,   AO, INC4

  4385. 	LFPDUX	B5,   BO, INC4

  4386. 	LFPDUX	A6,  AO2, INC4

  4387. 	LFPDUX	B6,  BO2, INC4

  4388. 	LFPDUX	A7,   AO, INC4

  4389. 	LFPDUX	A9,   BO, INC4

  4390. 	LFPDUX	A10, BO2, INC4

  4391. 	bdz-	.L23

  4392. 	.align 4

  4393. 

  4394. .L22:

  4395. 	fxcpmadd	f0,  B1, A1, f0

  4396. 	nop

  4397. 	fxcsmadd	f4,  B1, A1, f4

  4398. 	LFPDUX	A8,  AO2, INC4

  4399. 	fxcpmadd	f8,  B2, A1, f8

  4400. 	nop

  4401. 	fxcsmadd	f12, B2, A1, f12

  4402. 	LFPDUX	A1,   AO, INC4

  4403. 

  4404. 	fxcpmadd	f1,  B1, A2, f1

  4405. 	nop

  4406. 	fxcsmadd	f5,  B1, A2, f5

  4407. 	LFPDUX	B1,   BO, INC4

  4408. 	fxcpmadd	f9,  B2, A2, f9

  4409. 	nop

  4410. 	fxcsmadd	f13, B2, A2, f13

  4411. 	LFPDUX	B2,  BO2, INC4

  4412. 

  4413. 	fxcpmadd	f0,  B3, A3, f0

  4414. 	nop

  4415. 	fxcsmadd	f4,  B3, A3, f4

  4416. 	LFPDUX	A2,  AO2, INC4

  4417. 	fxcpmadd	f8,  B4, A3, f8

  4418. 	nop

  4419. 	fxcsmadd	f12, B4, A3, f12

  4420. 	LFPDUX	A3,   AO, INC4

  4421. 

  4422. 	fxcpmadd	f1,  B3, A4, f1

  4423. 	nop

  4424. 	fxcsmadd	f5,  B3, A4, f5

  4425. 	LFPDUX	B3,   BO, INC4

  4426. 	fxcpmadd	f9,  B4, A4, f9

  4427. 	nop

  4428. 	fxcsmadd	f13, B4, A4, f13

  4429. 	LFPDUX	B4,  BO2, INC4

  4430. 

  4431. 	fxcpmadd	f0,  B5, A5, f0

  4432. 	nop

  4433. 	fxcsmadd	f4,  B5, A5, f4

  4434. 	LFPDUX	A4,  AO2, INC4

  4435. 	fxcpmadd	f8,  B6, A5, f8

  4436. 	nop

  4437. 	fxcsmadd	f12, B6, A5, f12

  4438. 	LFPDUX	A5,   AO, INC4

  4439. 

  4440. 	fxcpmadd	f1,  B5, A6, f1

  4441. 	nop

  4442. 	fxcsmadd	f5,  B5, A6, f5

  4443. 	LFPDUX	B5,   BO, INC4

  4444. 	fxcpmadd	f9,  B6, A6, f9

  4445. 	nop

  4446. 	fxcsmadd	f13, B6, A6, f13

  4447. 	LFPDUX	B6,  BO2, INC4

  4448. 

  4449. 	fxcpmadd	f0,  A9,  A7, f0

  4450. 	nop

  4451. 	fxcsmadd	f4,  A9,  A7, f4

  4452. 	LFPDUX	A6,  AO2, INC4

  4453. 	fxcpmadd	f8,  A10, A7, f8

  4454. 	nop

  4455. 	fxcsmadd	f12, A10, A7, f12

  4456. 	LFPDUX	A7,   AO, INC4

  4457. 

  4458. 	fxcpmadd	f1,  A9,  A8, f1

  4459. 	nop

  4460. 	fxcsmadd	f5,  A9,  A8, f5

  4461. 	LFPDUX	A9,   BO, INC4

  4462. 	fxcpmadd	f9,  A10, A8, f9

  4463. 	nop

  4464. 	fxcsmadd	f13, A10, A8, f13

  4465. 	LFPDUX	A10, BO2, INC4

  4466. 	bdnz+	.L22

  4467. 	.align 4

  4468. 

  4469. .L23:

  4470. 	fxcpmadd	f0,  B1, A1, f0

  4471. 	fxcsmadd	f4,  B1, A1, f4

  4472. 	LFPDUX	A8,  AO2, INC4

  4473. 	fxcpmadd	f8,  B2, A1, f8

  4474. 	fxcsmadd	f12, B2, A1, f12

  4475. 

  4476. 	fxcpmadd	f1,  B1, A2, f1

  4477. 	fxcsmadd	f5,  B1, A2, f5

  4478. 	fxcpmadd	f9,  B2, A2, f9

  4479. 	fxcsmadd	f13, B2, A2, f13

  4480. 

  4481. 	fxcpmadd	f0,  B3, A3, f0

  4482. 	fxcsmadd	f4,  B3, A3, f4

  4483. 	fxcpmadd	f8,  B4, A3, f8

  4484. 	fxcsmadd	f12, B4, A3, f12

  4485. 

  4486. 	fxcpmadd	f1,  B3, A4, f1

  4487. 	fxcsmadd	f5,  B3, A4, f5

  4488. 	fxcpmadd	f9,  B4, A4, f9

  4489. 	fxcsmadd	f13, B4, A4, f13

  4490. 

  4491. 	fxcpmadd	f0,  B5, A5, f0

  4492. 	fxcsmadd	f4,  B5, A5, f4

  4493. 	fxcpmadd	f8,  B6, A5, f8

  4494. 	fxcsmadd	f12, B6, A5, f12

  4495. 

  4496. 	fxcpmadd	f1,  B5, A6, f1

  4497. 	fxcsmadd	f5,  B5, A6, f5

  4498. 	fxcpmadd	f9,  B6, A6, f9

  4499. 	fxcsmadd	f13, B6, A6, f13

  4500. 

  4501. 	fxcpmadd	f0,  A9, A7, f0

  4502. 	fxcsmadd	f4,  A9, A7, f4

  4503. 	fxcpmadd	f8,  A10, A7, f8

  4504. 	fxcsmadd	f12, A10, A7, f12

  4505. 

  4506. 	fxcpmadd	f1,  A9, A8, f1

  4507. 	fxcsmadd	f5,  A9, A8, f5

  4508. 	fxcpmadd	f9,  A10, A8, f9

  4509. 	fxcsmadd	f13, A10, A8, f13

  4510. 	.align 4

  4511. 

  4512. .L24:

  4513. #if defined(LT) || defined(RN)

  4514. 	andi.	r0,  KK,  3

  4515. 	mtspr	CTR, r0

  4516. 	ble+	.L28

  4517. #else

  4518. 	andi.	r0, TEMP, 3

  4519. 	mtspr	CTR, r0

  4520. 	ble+	.L28

  4521. #endif

  4522. 

  4523. 	LFPDUX	A1,  AO,  INC4

  4524. 	LFPDUX	A2,  AO2, INC4

  4525. 	LFPDUX	B1,  BO,  INC4

  4526. 	LFPDUX	B2,  BO2, INC4

  4527. 	bdz-	.L27

  4528. 	.align 4

  4529. 

  4530. .L26:

  4531. 	fxcpmadd	f0,  B1, A1, f0

  4532. 	fxcsmadd	f4,  B1, A1, f4

  4533. 	fxcpmadd	f8,  B2, A1, f8

  4534. 	fxcsmadd	f12, B2, A1, f12

  4535. 	LFPDUX	A1,  AO,  INC4

  4536. 

  4537. 	fxcpmadd	f1,  B1, A2, f1

  4538. 	fxcsmadd	f5,  B1, A2, f5

  4539. 	LFPDUX	B1,  BO,  INC4

  4540. 	fxcpmadd	f9,  B2, A2, f9

  4541. 	fxcsmadd	f13, B2, A2, f13

  4542. 	LFPDUX	A2,  AO2, INC4

  4543. 	LFPDUX	B2,  BO2, INC4

  4544. 	bdnz+	.L26

  4545. 	.align 4

  4546. 

  4547. .L27:

  4548. 	fxcpmadd	f0,  B1, A1, f0

  4549. 	fxcsmadd	f4,  B1, A1, f4

  4550. 	fxcpmadd	f8,  B2, A1, f8

  4551. 	fxcsmadd	f12, B2, A1, f12

  4552. 

  4553. 	fxcpmadd	f1,  B1, A2, f1

  4554. 	fxcsmadd	f5,  B1, A2, f5

  4555. 	fxcpmadd	f9,  B2, A2, f9

  4556. 	fxcsmadd	f13, B2, A2, f13

  4557. 	.align 4

  4558. 

  4559. .L28:

  4560. #if defined(LN) || defined(RT)

  4561. #ifdef LN

  4562. 	subi	r0, KK, 4

  4563. #else

  4564. 	subi	r0, KK, 4

  4565. #endif

  4566. 	slwi	TEMP, r0, 2 + BASE_SHIFT

  4567. 	slwi	r0,   r0, 2 + BASE_SHIFT

  4568. 	add	AO, AORIG, TEMP

  4569. 	add	BO, B,     r0

  4570. 	addi	AO2, AO,   2 * SIZE

  4571. 	addi	BO,  BO, - 4 * SIZE

  4572. 	addi	BO2, BO,   2 * SIZE

  4573. #endif

  4574. 

  4575. #if defined(LN) || defined(LT)

  4576. 	fpmr	f24, f0

  4577. 	fpmr	f25, f1

  4578. 	fpmr	f28, f8

  4579. 	fpmr	f29, f9

  4580. 

  4581. 	fsmfp	f0,  f4

  4582. 	fsmfp	f1,  f5

  4583. 	fsmfp	f8,  f12

  4584. 	fsmfp	f9,  f13

  4585. 

  4586. 	fsmtp	f4,  f24

  4587. 	fsmtp	f5,  f25

  4588. 	fsmtp	f12, f28

  4589. 	fsmtp	f13, f29

  4590. 

  4591. 	LFPDUX	f16, BO,  INC4

  4592. 	LFPDUX	f17, BO2, INC4

  4593. 	LFPDUX	f18, BO,  INC4

  4594. 	LFPDUX	f19, BO2, INC4

  4595. 

  4596.  	LFPDUX	f20, BO,  INC4

  4597. 	LFPDUX	f21, BO2, INC4

  4598. 	LFPDUX	f22, BO,  INC4

  4599. 	LFPDUX	f23, BO2, INC4

  4600. 

  4601. 	subi	BO,  BO,  16 * SIZE

  4602. 	subi	BO2, BO2, 16 * SIZE

  4603. 

  4604. 	fpsub	f0,  f16,  f0

  4605. 	fpsub	f8,  f17,  f8

  4606. 	fpsub	f4,  f18,  f4

  4607. 	fpsub	f12, f19,  f12

  4608. 

  4609. 	fpsub	f1,  f20,  f1

  4610. 	fpsub	f9,  f21,  f9

  4611. 	fpsub	f5,  f22,  f5

  4612. 	fpsub	f13, f23,  f13

  4613. #else

  4614. 	LFPDUX	f16, AO,  INC4

  4615. 	LFPDUX	f17, AO2, INC4

  4616. 	LFPDUX	f18, AO,  INC4

  4617. 	LFPDUX	f19, AO2, INC4

  4618. 	LFPDUX	f20, AO,  INC4

  4619. 	LFPDUX	f21, AO2, INC4

  4620. 	LFPDUX	f22, AO,  INC4

  4621. 	LFPDUX	f23, AO2, INC4

  4622. 

  4623. 	subi	AO,  AO,  16 * SIZE

  4624. 	subi	AO2, AO2, 16 * SIZE

  4625. 

  4626. 	fpsub	f0,  f16,  f0

  4627. 	fpsub	f1,  f17,  f1

  4628. 	fpsub	f4,  f18,  f4

  4629. 	fpsub	f5,  f19,  f5

  4630. 

  4631. 	fpsub	f8,  f20,  f8

  4632. 	fpsub	f9,  f21,  f9

  4633. 	fpsub	f12, f22,  f12

  4634. 	fpsub	f13, f23,  f13

  4635. #endif

  4636. 

  4637. #ifdef LN

  4638.        addi	AO,  AO,  20 * SIZE

  4639.        addi	AO2, AO2, 20 * SIZE

  4640. 

  4641. 	LFPDUX	A1,  AO2, INCM4

  4642. 	LFPDUX	A2,  AO,  INCM4

  4643. 	LFPDUX	A3,  AO2, INCM4

  4644. 	LFPDUX	A4,  AO,  INCM4

  4645. 

  4646. 	add	AO2, AO2, INCM4

  4647. 	LFPDUX	A5,  AO,  INCM4

  4648. 	add	AO2, AO2, INCM4

  4649. 	LFPDUX	A6,  AO,  INCM4

  4650. 

  4651. 	addi	AO,  AO,  -4 * SIZE

  4652. 	addi	AO2, AO2, -4 * SIZE

  4653. 

  4654. 	fxsmul	f5,  A1, f5

  4655. 	fxsmul	f13, A1, f13

  4656. 

  4657. 	fxcpnmsub  f1,  A1, f5,  f1

  4658. 	fxcpnmsub  f9,  A1, f13, f9

  4659. 

  4660. 	fxcsnmsub  f4,  A2, f5,  f4

  4661. 	fxcsnmsub  f12, A2, f13, f12

  4662. 

  4663. 	fxcpnmsub  f0,  A2, f5,  f0

  4664. 	fxcpnmsub  f8,  A2, f13, f8

  4665. 

  4666. 	fxpmul	f1,  A3, f1

  4667. 	fxpmul	f9,  A3, f9

  4668. 

  4669. 	fxcsnmsub  f4,  A4, f1,  f4

  4670. 	fxcsnmsub  f12, A4, f9,  f12

  4671. 

  4672. 	fxcpnmsub  f0,  A4, f1,  f0

  4673. 	fxcpnmsub  f8,  A4, f9,  f8

  4674. 

  4675. 	fxsmul	f4,  A5, f4

  4676. 	fxsmul	f12, A5, f12

  4677. 

  4678. 	fxcpnmsub  f0,  A5, f4,  f0

  4679. 	fxcpnmsub  f8,  A5, f12, f8

  4680. 

  4681. 	fxpmul	f0,  A6, f0

  4682. 	fxpmul	f8,  A6, f8

  4683. #endif

  4684. 

  4685. #ifdef LT

  4686. 	LFPDUX	A1,  AO,  INC4

  4687. 	LFPDUX	A2,  AO2, INC4

  4688. 	LFPDUX	A3,  AO,  INC4

  4689. 	LFPDUX	A4,  AO2, INC4

  4690. 

  4691. 	add	AO,  AO,  INC4

  4692. 	LFPDUX	A5,  AO2, INC4

  4693. 	add	AO,  AO,  INC4

  4694. 	LFPDUX	A6,  AO2, INC4

  4695. 

  4696. 	subi	AO,  AO,  16 * SIZE

  4697. 	subi	AO2, AO2, 16 * SIZE

  4698. 

  4699. 	fxpmul	f0,  A1,  f0

  4700. 	fxpmul	f8,  A1,  f8

  4701. 

  4702. 	fxcsnmsub  f4,  A1, f0, f4

  4703. 	fxcsnmsub  f12, A1, f8, f12

  4704. 

  4705. 	fxcpnmsub  f1,  A2, f0, f1

  4706. 	fxcpnmsub  f9,  A2, f8, f9

  4707. 

  4708. 	fxcsnmsub  f5,  A2, f0, f5

  4709. 	fxcsnmsub  f13, A2, f8, f13

  4710. 

  4711. 	fxsmul	f4,  A3,  f4

  4712. 	fxsmul	f12, A3,  f12

  4713. 

  4714. 	fxcpnmsub  f1,  A4, f4,  f1

  4715. 	fxcpnmsub  f9,  A4, f12, f9

  4716. 

  4717. 	fxcsnmsub  f5,  A4, f4,  f5

  4718. 	fxcsnmsub  f13, A4, f12, f13

  4719. 

  4720. 	fxpmul	f1,  A5,  f1

  4721. 	fxpmul	f9,  A5,  f9

  4722. 

  4723. 	fxcsnmsub  f5,  A5, f1, f5

  4724. 	fxcsnmsub  f13, A5, f9, f13

  4725. 

  4726. 	fxsmul	f5,  A6,  f5

  4727. 	fxsmul	f13, A6,  f13

  4728. #endif

  4729. 

  4730. #ifdef RN

  4731. 	LFPDUX	A1,  BO,  INC4

  4732. 	LFPDUX	A2,  BO2, INC4

  4733. 	LFPDUX	A3,  BO,  INC4

  4734. 	LFPDUX	A4,  BO2, INC4

  4735. 

  4736. 	add	BO,  BO,  INC4

  4737. 	LFPDUX	A5,  BO2, INC4

  4738. 

  4739. 	add	BO,  BO,  INC4

  4740. 	LFPDUX	A6,  BO2, INC4

  4741. 

  4742. 	subi	BO,  BO,  16 * SIZE

  4743. 	subi	BO2, BO2, 16 * SIZE

  4744. 

  4745. 	fxpmul	f0,  A1,  f0

  4746. 	fxpmul	f1,  A1,  f1

  4747. 	fxcsnmsub    f4,  A1, f0, f4

  4748. 	fxcsnmsub    f5,  A1, f1, f5

  4749. 

  4750. 	fxcpnmsub    f8,  A2, f0, f8

  4751. 	fxcpnmsub    f9,  A2, f1, f9

  4752. 	fxcsnmsub    f12, A2, f0, f12

  4753. 	fxcsnmsub    f13, A2, f1, f13

  4754. 

  4755. 	fxsmul	f4,  A3,  f4

  4756. 	fxsmul	f5,  A3,  f5

  4757. 	fxcpnmsub    f8,  A4, f4, f8

  4758. 	fxcpnmsub    f9,  A4, f5, f9

  4759. 

  4760. 	fxcsnmsub    f12, A4, f4, f12

  4761. 	fxcsnmsub    f13, A4, f5, f13

  4762. 

  4763. 	fxpmul	f8,  A5,  f8

  4764. 	fxpmul	f9,  A5,  f9

  4765. 	fxcsnmsub    f12, A5, f8,  f12

  4766. 	fxcsnmsub    f13, A5, f9,  f13

  4767. 

  4768. 	fxsmul	f12,  A6,  f12

  4769. 	fxsmul	f13,  A6,  f13

  4770. #endif

  4771. 

  4772. #ifdef RT

  4773. 	addi	BO,  BO,  20 * SIZE

  4774. 	addi	BO2, BO2, 20 * SIZE

  4775. 

  4776. 	LFPDUX	A1,  BO2, INCM4

  4777. 	LFPDUX	A2,  BO,  INCM4

  4778. 

  4779. 	LFPDUX	A3,  BO2, INCM4

  4780. 	LFPDUX	A4,  BO,  INCM4

  4781. 

  4782. 	add	BO2, BO2, INCM4

  4783. 	LFPDUX	A5,  BO,  INCM4

  4784. 

  4785. 	add	BO2, BO2, INCM4

  4786. 	LFPDUX	A6,  BO,  INCM4

  4787. 	subi	BO,  BO,  4 * SIZE

  4788. 	subi	BO2, BO2, 4 * SIZE

  4789. 

  4790. 	fxsmul	f12, A1,  f12

  4791. 	fxsmul	f13, A1,  f13

  4792. 	fxcpnmsub    f8,  A1, f12, f8

  4793. 	fxcpnmsub    f9,  A1, f13, f9

  4794. 

  4795. 	fxcsnmsub    f4,  A2, f12, f4

  4796. 	fxcsnmsub    f5,  A2, f13, f5

  4797. 	fxcpnmsub    f0,  A2, f12, f0

  4798. 	fxcpnmsub    f1,  A2, f13, f1

  4799. 

  4800. 	fxpmul	f8,  A3,  f8

  4801. 	fxpmul	f9,  A3,  f9

  4802. 	fxcsnmsub    f4,  A4, f8,  f4

  4803. 	fxcsnmsub    f5,  A4, f9,  f5

  4804. 

  4805. 	fxcpnmsub    f0,  A4, f8,  f0

  4806. 	fxcpnmsub    f1,  A4, f9,  f1

  4807. 

  4808. 	fxsmul	f4,  A5,  f4

  4809. 	fxsmul	f5,  A5,  f5

  4810. 	fxcpnmsub    f0,  A5, f4,  f0

  4811. 	fxcpnmsub    f1,  A5, f5,  f1

  4812. 

  4813. 	fxpmul	f0,  A6,  f0

  4814. 	fxpmul	f1,  A6,  f1

  4815. #endif

  4816. 

  4817. #ifdef LN

  4818. 	subi	CO1, CO1, 4 * SIZE

  4819. 	subi	CO2, CO2, 4 * SIZE

  4820. 	subi	CO3, CO3, 4 * SIZE

  4821. 	subi	CO4, CO4, 4 * SIZE

  4822. #endif

  4823. 

  4824. #if defined(LN) || defined(LT)

  4825. 	STFPDUX	f0,  BO,  INC4

  4826. 	STFPDUX	f8,  BO2, INC4

  4827. 	STFPDUX	f4,  BO,  INC4

  4828. 	STFPDUX	f12, BO2, INC4

  4829. 	STFPDUX	f1,  BO,  INC4

  4830. 	STFPDUX	f9,  BO2, INC4

  4831. 	STFPDUX	f5,  BO,  INC4

  4832. 	STFPDUX	f13, BO2, INC4

  4833. 

  4834. 	subi	BO,  BO,  16 * SIZE

  4835. 	subi	BO2, BO2, 16 * SIZE

  4836. 

  4837. 	STFDUX	f0,  CO1, INC

  4838. 	STFDUX	f4,  CO1, INC

  4839. 	STFDUX	f1,  CO1, INC

  4840. 	STFDUX	f5,  CO1, INC

  4841. 

  4842. 	STFSDUX	f0,  CO2, INC

  4843. 	STFSDUX	f4,  CO2, INC

  4844. 	STFSDUX	f1,  CO2, INC

  4845. 	STFSDUX	f5,  CO2, INC

  4846. 

  4847. 	STFDUX	f8,  CO3, INC

  4848. 	STFDUX	f12, CO3, INC

  4849. 	STFDUX	f9,  CO3, INC

  4850. 	STFDUX	f13, CO3, INC

  4851. 

  4852. 	STFSDUX	f8,  CO4, INC

  4853. 	STFSDUX	f12, CO4, INC

  4854. 	STFSDUX	f9,  CO4, INC

  4855. 	STFSDUX	f13, CO4, INC

  4856. #else

  4857. 	STFPDUX	f0,  AO,  INC4

  4858. 	STFPDUX	f1,  AO2, INC4

  4859. 	STFPDUX	f4,  AO,  INC4

  4860. 	STFPDUX	f5,  AO2, INC4

  4861. 	STFPDUX	f8,  AO,  INC4

  4862. 	STFPDUX	f9,  AO2, INC4

  4863. 	STFPDUX	f12, AO,  INC4

  4864. 	STFPDUX	f13, AO2, INC4

  4865. 

  4866. 	subi	AO,  AO,  16 * SIZE

  4867. 	subi	AO2, AO2, 16 * SIZE

  4868. 

  4869. 	STFDUX	f0,  CO1, INC

  4870. 	STFSDUX	f0,  CO1, INC

  4871. 	STFDUX	f1,  CO1, INC

  4872. 	STFSDUX	f1,  CO1, INC

  4873. 	STFDUX	f4,  CO2, INC

  4874. 	STFSDUX	f4,  CO2, INC

  4875. 	STFDUX	f5,  CO2, INC

  4876. 	STFSDUX	f5,  CO2, INC

  4877. 

  4878. 	STFDUX	f8,  CO3, INC

  4879. 	STFSDUX	f8,  CO3, INC

  4880. 	STFDUX	f9,  CO3, INC

  4881. 	STFSDUX	f9,  CO3, INC

  4882. 	STFDUX	f12, CO4, INC

  4883. 	STFSDUX	f12, CO4, INC

  4884. 	STFDUX	f13, CO4, INC

  4885. 	STFSDUX	f13, CO4, INC

  4886. #endif

  4887. 

  4888. #ifdef LN

  4889. 	subi	CO1, CO1, 4 * SIZE

  4890. 	subi	CO2, CO2, 4 * SIZE

  4891. 	subi	CO3, CO3, 4 * SIZE

  4892. 	subi	CO4, CO4, 4 * SIZE

  4893. #endif

  4894. 

  4895. #ifdef RT

  4896. 	slwi	r0, K, 2 + BASE_SHIFT

  4897. 	add	AORIG, AORIG, r0

  4898. #endif

  4899. 

  4900. #if defined(LT) || defined(RN)

  4901. 	sub	TEMP, K, KK

  4902. 	slwi	r0,   TEMP, 2 + BASE_SHIFT

  4903. 	slwi	TEMP, TEMP, 2 + BASE_SHIFT

  4904. 	add	AO, AO, r0

  4905. 	add	BO, BO, TEMP

  4906. #endif

  4907. 

  4908. #ifdef LT

  4909. 	addi	KK, KK, 4

  4910. #endif

  4911. 

  4912. #ifdef LN

  4913. 	subi	KK, KK, 4

  4914. #endif

  4915. 

  4916. 	li	r0, FZERO

  4917. 	lfpsx	f0, SP, r0

  4918. 	.align 4

  4919. 

  4920. .L30:

  4921. 	andi.	I, M,  2

  4922. 	beq	.L40

  4923. 

  4924. #if defined(LT) || defined(RN)

  4925. 	addi	AO2, AO,   2 * SIZE

  4926. 	fpmr	f4,  f0

  4927. 	addi	BO,  B,  - 4 * SIZE

  4928. 	fpmr	f8,  f0

  4929. 	addi	BO2, B,  - 2 * SIZE

  4930. 	fpmr	f12, f0

  4931. 

  4932. 	srawi.	r0,  KK,  2

  4933. 	mtspr	CTR, r0

  4934. 	ble	.L34

  4935. #else

  4936. 

  4937. #ifdef LN

  4938. 	slwi	r0,   K,  1 + BASE_SHIFT

  4939. 	sub	AORIG, AORIG, r0

  4940. #endif

  4941. 

  4942. 	slwi	r0  , KK, 1 + BASE_SHIFT

  4943. 	slwi	TEMP, KK, 2 + BASE_SHIFT

  4944. 	add	AO, AORIG, r0

  4945. 	add	BO, B,     TEMP

  4946. 

  4947. 	sub	TEMP, K, KK

  4948. 

  4949. 	addi	AO2, AO,    2 * SIZE

  4950. 	fpmr	f4,  f0

  4951. 	addi	BO,  BO,  - 4 * SIZE

  4952. 	fpmr	f8,  f0

  4953. 	addi	BO2, BO,    2 * SIZE

  4954. 	fpmr	f12, f0

  4955. 

  4956. 	srawi.	r0,  TEMP,  2

  4957. 	mtspr	CTR, r0

  4958. 	ble	.L34

  4959. #endif

  4960. 

  4961. 	LFPDUX	A1,  AO, INC4

  4962. 	LFPDUX	B1,  BO, INC4

  4963. 	LFPDUX	B2, BO2, INC4

  4964. 	LFPDUX	A2, AO2, INC4

  4965. 	LFPDUX	B3,  BO, INC4

  4966. 	LFPDUX	B4, BO2, INC4

  4967. 

  4968. 	LFPDUX	A3,  AO, INC4

  4969. 	LFPDUX	A5,  BO, INC4

  4970. 	LFPDUX	A6, BO2, INC4

  4971. 	LFPDUX	A4, AO2, INC4

  4972. 	LFPDUX	A7,  BO, INC4

  4973. 	LFPDUX	A8, BO2, INC4

  4974. 	bdz-	.L33

  4975. 	.align 4

  4976. 

  4977. .L32:

  4978. 	fxcpmadd	f0,  B1, A1, f0

  4979. 	fxcsmadd	f4,  B1, A1, f4

  4980. 	LFPDUX	B1,  BO, INC4

  4981. 	fxcpmadd	f8,  B2, A1, f8

  4982. 	fxcsmadd	f12, B2, A1, f12

  4983. 	LFPDUX	B2, BO2, INC4

  4984. 	LFPDUX	A1,  AO, INC4

  4985. 

  4986. 	fxcpmadd	f0,  B3, A2, f0

  4987. 	fxcsmadd	f4,  B3, A2, f4

  4988. 	LFPDUX	B3,  BO, INC4

  4989. 	fxcpmadd	f8,  B4, A2, f8

  4990. 	fxcsmadd	f12, B4, A2, f12

  4991. 	LFPDUX	B4, BO2, INC4

  4992. 	LFPDUX	A2, AO2, INC4

  4993. 

  4994. 	fxcpmadd	f0,  A5, A3, f0

  4995. 	fxcsmadd	f4,  A5, A3, f4

  4996. 	LFPDUX	A5,  BO, INC4

  4997. 	fxcpmadd	f8,  A6, A3, f8

  4998. 	fxcsmadd	f12, A6, A3, f12

  4999. 	LFPDUX	A6, BO2, INC4

  5000. 	LFPDUX	A3,  AO, INC4

  5001. 

  5002. 	fxcpmadd	f0,  A7, A4, f0

  5003. 	fxcsmadd	f4,  A7, A4, f4

  5004. 	LFPDUX	A7,  BO, INC4

  5005. 	fxcpmadd	f8,  A8, A4, f8

  5006. 	fxcsmadd	f12, A8, A4, f12

  5007. 	LFPDUX	A8, BO2, INC4

  5008. 	LFPDUX	A4, AO2, INC4

  5009. 	bdnz+	.L32

  5010. 	.align 4

  5011. 

  5012. .L33:

  5013. 	fxcpmadd	f0,  B1, A1, f0

  5014. 	fxcsmadd	f4,  B1, A1, f4

  5015. 	fxcpmadd	f8,  B2, A1, f8

  5016. 	fxcsmadd	f12, B2, A1, f12

  5017. 

  5018. 	fxcpmadd	f0,  B3, A2, f0

  5019. 	fxcsmadd	f4,  B3, A2, f4

  5020. 	fxcpmadd	f8,  B4, A2, f8

  5021. 	fxcsmadd	f12, B4, A2, f12

  5022. 

  5023. 	fxcpmadd	f0,  A5, A3, f0

  5024. 	fxcsmadd	f4,  A5, A3, f4

  5025. 	fxcpmadd	f8,  A6, A3, f8

  5026. 	fxcsmadd	f12, A6, A3, f12

  5027. 

  5028. 	fxcpmadd	f0,  A7, A4, f0

  5029. 	fxcsmadd	f4,  A7, A4, f4

  5030. 	fxcpmadd	f8,  A8, A4, f8

  5031. 	fxcsmadd	f12, A8, A4, f12

  5032. 	.align 4

  5033. 

  5034. .L34:

  5035. #if defined(LT) || defined(RN)

  5036. 	andi.	r0,  KK,  3

  5037. 	mtspr	CTR, r0

  5038. 	ble+	.L38

  5039. #else

  5040. 	andi.	r0, TEMP, 3

  5041. 	mtspr	CTR, r0

  5042. 	ble+	.L38

  5043. #endif

  5044. 

  5045. 	LFPDX	A1,  AO,  INC4

  5046. 	LFPDUX	B1,  BO,  INC4

  5047. 	LFPDUX	B2,  BO2, INC4

  5048. 	add	AO, AO, INC2

  5049. 	bdz-	.L37

  5050. 	.align 4

  5051. 

  5052. .L36:

  5053. 	fxcpmadd	f0,  B1, A1, f0

  5054. 	fxcsmadd	f4,  B1, A1, f4

  5055. 	LFPDUX	B1,  BO,  INC4

  5056. 	fxcpmadd	f8,  B2, A1, f8

  5057. 	fxcsmadd	f12, B2, A1, f12

  5058. 	LFPDX	A1,  AO,  INC4

  5059. 	LFPDUX	B2,  BO2, INC4

  5060. 	add	AO, AO, INC2

  5061. 	bdnz+	.L36

  5062. 	.align 4

  5063. 

  5064. .L37:

  5065. 	fxcpmadd	f0,  B1, A1, f0

  5066. 	fxcsmadd	f4,  B1, A1, f4

  5067. 	fxcpmadd	f8,  B2, A1, f8

  5068. 	fxcsmadd	f12, B2, A1, f12

  5069. 	.align 4

  5070. 

  5071. .L38:

  5072. #if defined(LN) || defined(RT)

  5073. #ifdef LN

  5074. 	subi	r0, KK, 2

  5075. #else

  5076. 	subi	r0, KK, 4

  5077. #endif

  5078. 	slwi	TEMP, r0, 1 + BASE_SHIFT

  5079. 	slwi	r0,   r0, 2 + BASE_SHIFT

  5080. 	add	AO, AORIG, TEMP

  5081. 	add	BO, B,     r0

  5082. 	addi	AO2, AO,   2 * SIZE

  5083. 	addi	BO,  BO, - 4 * SIZE

  5084. 	addi	BO2, BO,   2 * SIZE

  5085. #endif

  5086. 

  5087. #if defined(LN) || defined(LT)

  5088. 	fpmr	f24, f0

  5089. 	fpmr	f28, f8

  5090. 

  5091. 	fsmfp	f0,  f4

  5092. 	fsmfp	f8,  f12

  5093. 	fsmtp	f4,  f24

  5094. 	fsmtp	f12, f28

  5095. 

  5096. 	LFPDUX	f16, BO,  INC4

  5097. 	LFPDUX	f17, BO2, INC4

  5098. 	LFPDUX	f18, BO,  INC4

  5099. 	LFPDUX	f19, BO2, INC4

  5100. 

  5101. 	subi	BO,  BO,   8 * SIZE

  5102. 	subi	BO2, BO2,  8 * SIZE

  5103. 

  5104. 	fpsub	f0,  f16,  f0

  5105. 	fpsub	f8,  f17,  f8

  5106. 	fpsub	f4,  f18,  f4

  5107. 	fpsub	f12, f19,  f12

  5108. #else

  5109. 	LFPDUX	f16, AO,  INC4

  5110. 	LFPDUX	f17, AO2, INC4

  5111. 	LFPDUX	f18, AO,  INC4

  5112. 	LFPDUX	f19, AO2, INC4

  5113. 

  5114. 	subi	AO,  AO,   8 * SIZE

  5115. 	subi	AO2, AO2,  8 * SIZE

  5116. 

  5117. 	fpsub	f0,  f16,  f0

  5118. 	fpsub	f4,  f17,  f4

  5119. 	fpsub	f8,  f18,  f8

  5120. 	fpsub	f12, f19,  f12

  5121. #endif

  5122. 

  5123. #ifdef LN

  5124. 	addi	AO,  AO,   8 * SIZE

  5125. 	addi	AO2, AO2,  8 * SIZE

  5126. 

  5127. 	LFPDUX	A1,  AO2, INCM4

  5128. 	LFPDUX	A2,  AO,  INCM4

  5129. 

  5130. 	addi	AO,  AO,  -4 * SIZE

  5131. 	addi	AO2, AO2, -4 * SIZE

  5132. 

  5133. 	fxsmul	f4,  A1, f4

  5134. 	fxsmul	f12, A1, f12

  5135. 

  5136. 	fxcpnmsub  f0,  A1, f4,  f0

  5137. 	fxcpnmsub  f8,  A1, f12, f8

  5138. 

  5139. 	fxpmul	f0,  A2, f0

  5140. 	fxpmul	f8,  A2, f8

  5141. #endif

  5142. 

  5143. #ifdef LT

  5144. 	LFPDUX	A1,  AO,  INC4

  5145. 	LFPDUX	A2,  AO2, INC4

  5146. 

  5147. 	subi	AO,  AO,   4 * SIZE

  5148. 	subi	AO2, AO2,  4 * SIZE

  5149. 

  5150. 	fxpmul	f0,  A1,  f0

  5151. 	fxpmul	f8,  A1,  f8

  5152. 

  5153. 	fxcsnmsub  f4,  A1, f0, f4

  5154. 	fxcsnmsub  f12, A1, f8, f12

  5155. 

  5156. 	fxsmul	f4,  A2,  f4

  5157. 	fxsmul	f12, A2,  f12

  5158. #endif

  5159. 

  5160. #ifdef RN

  5161. 	LFPDUX	A1,  BO,  INC4

  5162. 	LFPDUX	A2,  BO2, INC4

  5163. 	LFPDUX	A3,  BO,  INC4

  5164. 	LFPDUX	A4,  BO2, INC4

  5165. 

  5166. 	add	BO,  BO,  INC4

  5167. 	LFPDUX	A5,  BO2, INC4

  5168. 

  5169. 	add	BO,  BO,  INC4

  5170. 	LFPDUX	A6,  BO2, INC4

  5171. 

  5172. 	subi	BO,  BO,  16 * SIZE

  5173. 	subi	BO2, BO2, 16 * SIZE

  5174. 

  5175. 	fxpmul	     f0,  A1,  f0

  5176. 	fxcsnmsub    f4,  A1, f0, f4

  5177. 	fxcpnmsub    f8,  A2, f0, f8

  5178. 	fxcsnmsub    f12, A2, f0, f12

  5179. 

  5180. 	fxsmul	     f4,  A3,  f4

  5181. 	fxcpnmsub    f8,  A4, f4, f8

  5182. 	fxcsnmsub    f12, A4, f4, f12

  5183. 

  5184. 	fxpmul	     f8,  A5,  f8

  5185. 	fxcsnmsub    f12, A5, f8,  f12

  5186. 	fxsmul	     f12, A6,  f12

  5187. #endif

  5188. 

  5189. #ifdef RT

  5190. 	addi	BO,  BO,  20 * SIZE

  5191. 	addi	BO2, BO2, 20 * SIZE

  5192. 

  5193. 	LFPDUX	A1,  BO2, INCM4

  5194. 	LFPDUX	A2,  BO,  INCM4

  5195. 

  5196. 	LFPDUX	A3,  BO2, INCM4

  5197. 	LFPDUX	A4,  BO,  INCM4

  5198. 

  5199. 	add	BO2, BO2, INCM4

  5200. 	LFPDUX	A5,  BO,  INCM4

  5201. 

  5202. 	add	BO2, BO2, INCM4

  5203. 	LFPDUX	A6,  BO,  INCM4

  5204. 	subi	BO,  BO,  4 * SIZE

  5205. 	subi	BO2, BO2, 4 * SIZE

  5206. 

  5207. 	fxsmul	     f12, A1,  f12

  5208. 	fxcpnmsub    f8,  A1, f12, f8

  5209. 	fxcsnmsub    f4,  A2, f12, f4

  5210. 	fxcpnmsub    f0,  A2, f12, f0

  5211. 

  5212. 	fxpmul	     f8,  A3,  f8

  5213. 	fxcsnmsub    f4,  A4, f8,  f4

  5214. 	fxcpnmsub    f0,  A4, f8,  f0

  5215. 

  5216. 	fxsmul	     f4,  A5,  f4

  5217. 	fxcpnmsub    f0,  A5, f4,  f0

  5218. 	fxpmul	     f0,  A6,  f0

  5219. #endif

  5220. 

  5221. #ifdef LN

  5222. 	subi	CO1, CO1, 2 * SIZE

  5223. 	subi	CO2, CO2, 2 * SIZE

  5224. 	subi	CO3, CO3, 2 * SIZE

  5225. 	subi	CO4, CO4, 2 * SIZE

  5226. #endif

  5227. 

  5228. #if defined(LN) || defined(LT)

  5229. 	STFPDUX	f0,  BO,  INC4

  5230. 	STFPDUX	f8,  BO2, INC4

  5231. 	STFPDUX	f4,  BO,  INC4

  5232. 	STFPDUX	f12, BO2, INC4

  5233. 

  5234. 	subi	BO,  BO,   8 * SIZE

  5235. 	subi	BO2, BO2,  8 * SIZE

  5236. 

  5237. 	STFDUX	f0,  CO1, INC

  5238. 	STFDUX	f4,  CO1, INC

  5239. 	STFSDUX	f0,  CO2, INC

  5240. 	STFSDUX	f4,  CO2, INC

  5241. 

  5242. 	STFDUX	f8,  CO3, INC

  5243. 	STFDUX	f12, CO3, INC

  5244. 	STFSDUX	f8,  CO4, INC

  5245. 	STFSDUX	f12, CO4, INC

  5246. 

  5247. #else

  5248. 	STFPDUX	f0,  AO,  INC4

  5249. 	STFPDUX	f4,  AO2, INC4

  5250. 	STFPDUX	f8,  AO,  INC4

  5251. 	STFPDUX	f12, AO2, INC4

  5252. 

  5253. 	subi	AO,  AO,   8 * SIZE

  5254. 	subi	AO2, AO2,  8 * SIZE

  5255. 

  5256. 	STFDUX	f0,  CO1, INC

  5257. 	STFSDUX	f0,  CO1, INC

  5258. 	STFDUX	f4,  CO2, INC

  5259. 	STFSDUX	f4,  CO2, INC

  5260. 

  5261. 	STFDUX	f8,  CO3, INC

  5262. 	STFSDUX	f8,  CO3, INC

  5263. 	STFDUX	f12, CO4, INC

  5264. 	STFSDUX	f12, CO4, INC

  5265. #endif

  5266. 

  5267. #ifdef LN

  5268. 	subi	CO1, CO1, 2 * SIZE

  5269. 	subi	CO2, CO2, 2 * SIZE

  5270. 	subi	CO3, CO3, 2 * SIZE

  5271. 	subi	CO4, CO4, 2 * SIZE

  5272. #endif

  5273. 

  5274. #ifdef RT

  5275. 	slwi	r0, K, 1 + BASE_SHIFT

  5276. 	add	AORIG, AORIG, r0

  5277. #endif

  5278. 

  5279. #if defined(LT) || defined(RN)

  5280. 	sub	TEMP, K, KK

  5281. 	slwi	r0,   TEMP, 1 + BASE_SHIFT

  5282. 	slwi	TEMP, TEMP, 2 + BASE_SHIFT

  5283. 	add	AO, AO, r0

  5284. 	add	BO, BO, TEMP

  5285. #endif

  5286. 

  5287. #ifdef LT

  5288. 	addi	KK, KK, 2

  5289. #endif

  5290. 

  5291. #ifdef LN

  5292. 	subi	KK, KK, 2

  5293. #endif

  5294. 

  5295. 	li	r0, FZERO

  5296. 	lfpsx	f0, SP, r0

  5297. 	.align 4

  5298. 

  5299. .L40:

  5300. 	andi.	I, M,  1

  5301. 	beq	.L49

  5302. 

  5303. #if defined(LT) || defined(RN)

  5304. 	addi	AO2, AO,   2 * SIZE

  5305. 	fpmr	f1,  f0

  5306. 	addi	BO,  B,  - 4 * SIZE

  5307. 	fpmr	f2,  f0

  5308. 	addi	BO2, B,  - 2 * SIZE

  5309. 	fpmr	f3,  f0

  5310. 

  5311. 	srawi.	r0,  KK,  3

  5312. 	mtspr	CTR, r0

  5313. 	ble	.L44

  5314. #else

  5315. 

  5316. #ifdef LN

  5317. 	slwi	r0,   K,  0 + BASE_SHIFT

  5318. 	sub	AORIG, AORIG, r0

  5319. #endif

  5320. 

  5321. 	slwi	r0  , KK, 0 + BASE_SHIFT

  5322. 	slwi	TEMP, KK, 2 + BASE_SHIFT

  5323. 	add	AO, AORIG, r0

  5324. 	add	BO, B,     TEMP

  5325. 

  5326. 	sub	TEMP, K, KK

  5327. 

  5328. 	addi	AO2, AO,   2 * SIZE

  5329. 	fpmr	f1,  f0

  5330. 	addi	BO,  BO, - 4 * SIZE

  5331. 	fpmr	f2,  f0

  5332. 	addi	BO2, BO,   2 * SIZE

  5333. 	fpmr	f3,  f0

  5334. 

  5335. 	srawi.	r0,  TEMP,  3

  5336. 	mtspr	CTR, r0

  5337. 	ble	.L44

  5338. #endif

  5339. 	

  5340. 	LFPDUX	A1,  AO,  INC4

  5341. 	LFPDUX	B1,  BO,  INC4

  5342. 	LFPDUX	B2,  BO2, INC4

  5343. 	LFPDUX	A2, AO2,  INC4

  5344. 	LFPDUX	B3,  BO,  INC4

  5345. 	LFPDUX	B4,  BO2, INC4

  5346. 

  5347. 	LFPDUX	A3,  AO,  INC4

  5348. 	LFPDUX	A5,  BO,  INC4

  5349. 	LFPDUX	A6,  BO2, INC4

  5350. 	LFPDUX	A4, AO2,  INC4

  5351. 	LFPDUX	A7,  BO,  INC4

  5352. 	LFPDUX	A8,  BO2, INC4

  5353. 	bdz-	.L43

  5354. 	.align 4

  5355. 

  5356. .L42:

  5357. 	fxcpmadd	f0,  A1, B1, f0

  5358. 	LFPDUX	B1,  BO,  INC4

  5359. 	fxcpmadd	f1,  A1, B2, f1

  5360. 	LFPDUX	B2,  BO2, INC4

  5361. 	fxcsmadd	f2,  A1, B3, f2

  5362. 	LFPDUX	B3,  BO,  INC4

  5363. 	fxcsmadd	f3,  A1, B4, f3

  5364. 	LFPDUX	B4,  BO2, INC4

  5365. 	LFPDUX	A1,  AO,  INC4

  5366. 

  5367. 	fxcpmadd	f0,  A2, A5, f0

  5368. 	LFPDUX	A5,  BO,  INC4

  5369. 	fxcpmadd	f1,  A2, A6, f1

  5370. 	LFPDUX	A6,  BO2, INC4

  5371. 	fxcsmadd	f2,  A2, A7, f2

  5372. 	LFPDUX	A7,  BO,  INC4

  5373. 	fxcsmadd	f3,  A2, A8, f3

  5374. 	LFPDUX	A8,  BO2, INC4

  5375. 	LFPDUX	A2, AO2,  INC4

  5376. 

  5377. 	fxcpmadd	f0,  A3, B1, f0

  5378. 	LFPDUX	B1,  BO,  INC4

  5379. 	fxcpmadd	f1,  A3, B2, f1

  5380. 	LFPDUX	B2,  BO2, INC4

  5381. 	fxcsmadd	f2,  A3, B3, f2

  5382. 	LFPDUX	B3,  BO,  INC4

  5383. 	fxcsmadd	f3,  A3, B4, f3

  5384. 	LFPDUX	B4,  BO2, INC4

  5385. 	LFPDUX	A3,  AO,  INC4

  5386. 

  5387. 	fxcpmadd	f0,  A4, A5, f0

  5388. 	LFPDUX	A5,  BO,  INC4

  5389. 	fxcpmadd	f1,  A4, A6, f1

  5390. 	LFPDUX	A6,  BO2, INC4

  5391. 	fxcsmadd	f2,  A4, A7, f2

  5392. 	LFPDUX	A7,  BO,  INC4

  5393. 	fxcsmadd	f3,  A4, A8, f3

  5394. 	LFPDUX	A8,  BO2, INC4

  5395. 	LFPDUX	A4, AO2,  INC4

  5396. 	bdnz+	.L42

  5397. 	.align 4

  5398. 

  5399. .L43:

  5400. 	fxcpmadd	f0,  A1, B1, f0

  5401. 	LFPDUX	B1,  BO,  INC4

  5402. 	fxcpmadd	f1,  A1, B2, f1

  5403. 	LFPDUX	B2,  BO2, INC4

  5404. 	fxcsmadd	f2,  A1, B3, f2

  5405. 	LFPDUX	B3,  BO,  INC4

  5406. 	fxcsmadd	f3,  A1, B4, f3

  5407. 	LFPDUX	B4,  BO2, INC4

  5408. 

  5409. 	fxcpmadd	f0,  A2, A5, f0

  5410. 	LFPDUX	A5,  BO,  INC4

  5411. 	fxcpmadd	f1,  A2, A6, f1

  5412. 	LFPDUX	A6,  BO2, INC4

  5413. 	fxcsmadd	f2,  A2, A7, f2

  5414. 	LFPDUX	A7,  BO,  INC4

  5415. 	fxcsmadd	f3,  A2, A8, f3

  5416. 	LFPDUX	A8,  BO2, INC4

  5417. 

  5418. 	fxcpmadd	f0,  A3, B1, f0

  5419. 	fxcpmadd	f1,  A3, B2, f1

  5420. 	fxcsmadd	f2,  A3, B3, f2

  5421. 	fxcsmadd	f3,  A3, B4, f3

  5422. 

  5423. 	fxcpmadd	f0,  A4, A5, f0

  5424. 	fxcpmadd	f1,  A4, A6, f1

  5425. 	fxcsmadd	f2,  A4, A7, f2

  5426. 	fxcsmadd	f3,  A4, A8, f3

  5427. 	.align 4

  5428. 

  5429. .L44:

  5430. #if defined(LT) || defined(RN)

  5431. 	andi.	r0,  KK,  7

  5432. 	mtspr	CTR, r0

  5433. 	ble+	.L48

  5434. #else

  5435. 	andi.	r0, TEMP, 7

  5436. 	mtspr	CTR, r0

  5437. 	ble+	.L48

  5438. #endif

  5439. 

  5440. 	LFDX	A1,  AO,  INC4

  5441. 	LFPDUX	B1,  BO,  INC4

  5442. 	LFPDUX	B2,  BO2, INC4

  5443. 	add	AO, AO, INC

  5444. 	bdz-	.L47

  5445. 	.align 4

  5446. 

  5447. .L46:

  5448. 	fxcpmadd	f0,  A1, B1, f0

  5449. 	LFPDUX	B1,  BO,  INC4

  5450. 	fxcpmadd	f1,  A1, B2, f1

  5451. 	LFDX	A1,  AO,  INC4

  5452. 	LFPDUX	B2,  BO2, INC4

  5453. 	add	AO, AO, INC

  5454. 	bdnz+	.L46

  5455. 	.align 4

  5456. 

  5457. .L47:

  5458. 	fxcpmadd	f0,  A1, B1, f0

  5459. 	fxcpmadd	f1,  A1, B2, f1

  5460. 	addi	AO2, AO,   2 * SIZE

  5461. 	.align 4

  5462. 

  5463. .L48:

  5464. 	fpadd	f0, f0, f2

  5465. 	fpadd	f1, f1, f3

  5466. 

  5467. #if defined(LN) || defined(RT)

  5468. #ifdef LN

  5469. 	subi	r0, KK, 1

  5470. #else

  5471. 	subi	r0, KK, 4

  5472. #endif

  5473. 	slwi	TEMP, r0, 0 + BASE_SHIFT

  5474. 	slwi	r0,   r0, 2 + BASE_SHIFT

  5475. 	add	AO, AORIG, TEMP

  5476. 	add	BO, B,     r0

  5477. 	addi	AO2, AO,   2 * SIZE

  5478. 	addi	BO,  BO, - 4 * SIZE

  5479. 	addi	BO2, BO,   2 * SIZE

  5480. #endif

  5481. 

  5482. #if defined(LN) || defined(LT)

  5483. 	LFPDX	f16, BO,  INC4

  5484. 	LFPDX	f17, BO2, INC4

  5485. 

  5486. 	fpsub	f0,  f16,  f0

  5487. 	fpsub	f1,  f17,  f1

  5488. #else

  5489. 	LFPDX	f16, AO,  INC4

  5490. 	LFPDX	f17, AO2, INC4

  5491. 

  5492. 	fpsub	f0,  f16,  f0

  5493. 	fpsub	f1,  f17,  f1

  5494. #endif

  5495. 

  5496. #if defined(LN) || defined(LT)

  5497. 	LFPDX	A1,  AO, INC4

  5498. 

  5499. 	fxpmul	f0,  A1, f0

  5500. 	fxpmul	f1,  A1, f1

  5501. #endif

  5502. 

  5503. #ifdef RN

  5504. 	LFD	A1,  (4 +  0) * SIZE(BO)

  5505. 	LFD	A2,  (4 +  1) * SIZE(BO)

  5506. 	LFD	A3,  (4 +  2) * SIZE(BO)

  5507. 	LFD	A4,  (4 +  3) * SIZE(BO)

  5508. 

  5509. 	LFD	A5,  (4 +  5) * SIZE(BO)

  5510. 	LFD	A6,  (4 +  6) * SIZE(BO)

  5511. 	LFD	A7,  (4 +  7) * SIZE(BO)

  5512. 	LFD	A8,  (4 + 10) * SIZE(BO)

  5513. 

  5514. 	LFD	A9,  (4 + 11) * SIZE(BO)

  5515. 	LFD	A10, (4 + 15) * SIZE(BO)

  5516. 

  5517. 	fsmtp	     f2, f0

  5518. 	fsmtp	     f3, f1

  5519. 

  5520. 	fmul	     f0,  A1, f0

  5521. 	fnmsub	     f2,  A2, f0, f2

  5522. 	fnmsub	     f1,  A3, f0, f1

  5523. 	fnmsub	     f3,  A4, f0, f3

  5524. 

  5525. 	fmul	     f2,  A5, f2

  5526. 	fnmsub	     f1,  A6, f2, f1

  5527. 	fnmsub	     f3,  A7, f2, f3

  5528. 

  5529. 	fmul	     f1,  A8, f1

  5530. 	fnmsub	     f3,  A9, f1, f3

  5531. 

  5532. 	fmul	     f3,  A10, f3

  5533. 

  5534. 	fsmfp	     f0, f2

  5535. 	fsmfp	     f1, f3

  5536. #endif

  5537. 

  5538. #ifdef RT

  5539. 	LFD	A1,  (4 + 15) * SIZE(BO)

  5540. 	LFD	A2,  (4 + 14) * SIZE(BO)

  5541. 	LFD	A3,  (4 + 13) * SIZE(BO)

  5542. 	LFD	A4,  (4 + 12) * SIZE(BO)

  5543. 

  5544. 	LFD	A5,  (4 + 10) * SIZE(BO)

  5545. 	LFD	A6,  (4 +  9) * SIZE(BO)

  5546. 	LFD	A7,  (4 +  8) * SIZE(BO)

  5547. 	LFD	A8,  (4 +  5) * SIZE(BO)

  5548. 

  5549. 	LFD	A9,  (4 +  4) * SIZE(BO)

  5550. 	LFD	A10, (4 +  0) * SIZE(BO)

  5551. 

  5552. 	fsmtp	     f2, f0

  5553. 	fsmtp	     f3, f1

  5554. 

  5555. 	fmul	     f3,  A1, f3

  5556. 	fnmsub	     f1,  A2, f3, f1

  5557. 	fnmsub	     f2,  A3, f3, f2

  5558. 	fnmsub	     f0,  A4, f3, f0

  5559. 

  5560. 	fmul	     f1,  A5, f1

  5561. 	fnmsub	     f2,  A6, f1, f2

  5562. 	fnmsub	     f0,  A7, f1, f0

  5563. 

  5564. 	fmul	     f2,  A8, f2

  5565. 	fnmsub	     f0,  A9, f2, f0

  5566. 

  5567. 	fmul	     f0,  A10, f0

  5568. 

  5569. 	fsmfp	     f0, f2

  5570. 	fsmfp	     f1, f3

  5571. #endif

  5572. 

  5573. #if defined(LN) || defined(LT)

  5574. 	STFPDX	f0,  BO,  INC4

  5575. 	STFPDX	f1,  BO2, INC4

  5576. #else

  5577. 	STFPDX	f0,  AO,  INC4

  5578. 	STFPDX	f1,  AO2, INC4

  5579. #endif

  5580. 

  5581. #ifdef LN

  5582. 	subi	CO1, CO1, 1 * SIZE

  5583. 	subi	CO2, CO2, 1 * SIZE

  5584. 	subi	CO3, CO3, 1 * SIZE

  5585. 	subi	CO4, CO4, 1 * SIZE

  5586. #endif

  5587. 

  5588. 	STFDUX	f0,  CO1, INC

  5589. 	STFSDUX	f0,  CO2, INC

  5590. 	STFDUX	f1,  CO3, INC

  5591. 	STFSDUX	f1,  CO4, INC

  5592. 

  5593. #ifdef LN

  5594. 	subi	CO1, CO1, 1 * SIZE

  5595. 	subi	CO2, CO2, 1 * SIZE

  5596. 	subi	CO3, CO3, 1 * SIZE

  5597. 	subi	CO4, CO4, 1 * SIZE

  5598. #endif

  5599. 

  5600. #ifdef RT

  5601. 	slwi	r0, K, 0 + BASE_SHIFT

  5602. 	add	AORIG, AORIG, r0

  5603. #endif

  5604. 

  5605. #if defined(LT) || defined(RN)

  5606. 	sub	TEMP, K, KK

  5607. 	slwi	r0,   TEMP, 0 + BASE_SHIFT

  5608. 	slwi	TEMP, TEMP, 2 + BASE_SHIFT

  5609. 	add	AO, AO, r0

  5610. 	add	BO, BO, TEMP

  5611. #endif

  5612. 

  5613. #ifdef LT

  5614. 	addi	KK, KK, 1

  5615. #endif

  5616. 

  5617. #ifdef LN

  5618. 	subi	KK, KK, 1

  5619. #endif

  5620. 	.align 4

  5621. 

  5622. .L49:

  5623. #ifdef LN

  5624. 	slwi	r0, K, 2 + BASE_SHIFT

  5625. 	add	B, B, r0

  5626. #endif

  5627. 

  5628. #if defined(LT) || defined(RN)

  5629. 	addi	B,  BO, 4 * SIZE

  5630. #endif

  5631. 

  5632. #ifdef RN

  5633. 	addi	KK, KK, 4

  5634. #endif

  5635. 

  5636. #ifdef RT

  5637. 	subi	KK, KK, 4

  5638. #endif

  5639. 

  5640. 	addic.	J, J, -1

  5641. 	bgt+	.L10

  5642. 	.align 4

  5643. 

  5644. .L999:

  5645. 	addi	SP, SP, 12

  5646. 

  5647. 	lwzu	r14,   4(SP)

  5648. 	lwzu	r15,   4(SP)

  5649. 

  5650. 	lwzu	r16,   4(SP)

  5651. 	lwzu	r17,   4(SP)

  5652. 	lwzu	r18,   4(SP)

  5653. 	lwzu	r19,   4(SP)

  5654. 

  5655. 	lwzu	r20,   4(SP)

  5656. 	lwzu	r21,   4(SP)

  5657. 	lwzu	r22,   4(SP)

  5658. 	lwzu	r23,   4(SP)

  5659. 

  5660. 	lwzu	r24,   4(SP)

  5661. 	lwzu	r25,   4(SP)

  5662. 	lwzu	r26,   4(SP)

  5663. 	lwzu	r27,   4(SP)

  5664. 

  5665. 	lwzu	r28,   4(SP)

  5666. 	lwzu	r29,   4(SP)

  5667. 	lwzu	r30,   4(SP)

  5668. 	lwzu	r31,   4(SP)

  5669. 

  5670. 	subi	SP, SP, 12

  5671. 	li	r0, 16

  5672. 

  5673. 	lfpdux	f31, SP, r0

  5674. 	lfpdux	f30, SP, r0

  5675. 	lfpdux	f29, SP, r0

  5676. 	lfpdux	f28, SP, r0

  5677. 	lfpdux	f27, SP, r0

  5678. 	lfpdux	f26, SP, r0

  5679. 	lfpdux	f25, SP, r0

  5680. 	lfpdux	f24, SP, r0

  5681. 	lfpdux	f23, SP, r0

  5682. 	lfpdux	f22, SP, r0

  5683. 	lfpdux	f21, SP, r0

  5684. 	lfpdux	f20, SP, r0

  5685. 	lfpdux	f19, SP, r0

  5686. 	lfpdux	f18, SP, r0

  5687. 	lfpdux	f17, SP, r0

  5688. 	lfpdux	f16, SP, r0

  5689. 	lfpdux	f15, SP, r0

  5690. 	lfpdux	f14, SP, r0

  5691. 	addi	SP, SP, 16

  5692. 	blr

  5693. 

  5694. 

  5695. 	EPILOGUE

  5696. #endif