OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: eebfeba768
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'eebfeba768'
${ noResults }
OpenBLAS/kernel/power/zsymv_L.S

1674 lines
33 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(linux) || defined(__FreeBSD__)

  43. #ifndef __64BIT__

  44. #define M	r3

  45. #define N	r4

  46. #define A	r5

  47. #define LDA	r6

  48. #define X	r7

  49. #define	INCX	r8

  50. #define	Y	r9

  51. #define	INCY	r10

  52. #define BUFFER	r14

  53. #else

  54. #define M	r3

  55. #define N	r4

  56. #define A	r7

  57. #define LDA	r8

  58. #define X	r9

  59. #define	INCX	r10

  60. #define	Y	r5

  61. #define	INCY	r6

  62. #define BUFFER	r14

  63. #endif

  64. #endif

  65. 

  66. #if defined(_AIX) || defined(__APPLE__)

  67. #if !defined(__64BIT__) && defined(DOUBLE)

  68. #define M	r3

  69. #define N	r4

  70. #define A	r9

  71. #define LDA	r10

  72. #define X	r5

  73. #define	INCX	r6

  74. #define	Y	r7

  75. #define	INCY	r8

  76. #define BUFFER	r14

  77. #else

  78. #define M	r3

  79. #define N	r4

  80. #define A	r7

  81. #define LDA	r8

  82. #define X	r9

  83. #define	INCX	r10

  84. #define	Y	r5

  85. #define	INCY	r6

  86. #define BUFFER	r14

  87. #endif

  88. #endif

  89. 

  90. #define I	r11

  91. #define	J	r12

  92. 

  93. #define AO1	r15

  94. #define AO2	r16

  95. #define AO3	r17

  96. #define AO4	r18

  97. #define XX	r19

  98. #define YY	r20

  99. #define	NEW_Y	r21

  100. #define TEMP	r22

  101. #define	PREA	r24

  102. #define IS	r25

  103. 

  104. #define y01 f0

  105. #define y02 f1

  106. #define y03 f2

  107. #define y04 f3

  108. #define y05 f4

  109. #define y06 f5

  110. #define y07 f6

  111. #define y08 f7

  112. 

  113. #define xtemp1 f8

  114. #define xtemp2 f9

  115. #define xtemp3 f10

  116. #define xtemp4 f11

  117. #define xtemp5 f12

  118. #define xtemp6 f13

  119. #define xtemp7 f14

  120. #define xtemp8 f15

  121. 

  122. #define atemp1 f16

  123. #define atemp2 f17

  124. #define atemp3 f18

  125. #define atemp4 f19

  126. 

  127. #define xsum1  f20

  128. #define xsum2  f21

  129. #define xsum3  f22

  130. #define xsum4  f23

  131. 

  132. #define a1     f24

  133. #define a2     f25

  134. #define a3     f26

  135. #define a4     f27

  136. #define a5     f28

  137. #define a6     f29

  138. #define a7     f30

  139. #define a8     f31

  140. 

  141. #define alpha_r  f1

  142. #define alpha_i  f2

  143. 

  144. #if defined(PPCG4)

  145. #define PREFETCHSIZE_A  24

  146. #endif

  147. 

  148. #if defined(PPC440) || defined(PPC440FP2)

  149. #define PREFETCHSIZE_A  24

  150. #endif

  151. 

  152. #ifdef PPC970

  153. #define PREFETCHSIZE_A  32

  154. #endif

  155. 

  156. #ifdef CELL

  157. #define PREFETCHSIZE_A  72

  158. #endif

  159. 

  160. #ifdef POWER4

  161. #define PREFETCHSIZE_A  16

  162. #endif

  163. 

  164. #ifdef POWER5

  165. #define PREFETCHSIZE_A  96

  166. #endif

  167. 

  168. #ifdef POWER6

  169. #define PREFETCHSIZE_A  112

  170. #endif

  171. 

  172. #if defined(POWER4) || defined(POWER5) || defined(POWER6) || defined(PPC970)

  173. #define NOP1

  174. #define NOP2

  175. #else

  176. #define NOP1   mr	LDA, LDA

  177. #define NOP2   mr	INCX, INCX

  178. #endif

  179. 

  180. #ifndef NEEDPARAM

  181. 

  182. #ifndef __64BIT__

  183. #define STACKSIZE 224

  184. #define ALPHA_R     200(SP)

  185. #define ALPHA_I     208(SP)

  186. #define	FZERO	    216(SP)

  187. #else

  188. #define STACKSIZE 280

  189. #define ALPHA_R     256(SP)

  190. #define ALPHA_I     264(SP)

  191. #define FZERO	    272(SP)

  192. #endif

  193. 

  194. #ifndef HEMV

  195. #define FMADD1	FNMSUB

  196. #define FMADD2	FMADD

  197. #else

  198. #define FMADD1	FMADD

  199. #define FMADD2	FNMSUB

  200. #endif

  201. 

  202. 	PROLOGUE

  203. 	PROFCODE

  204. 

  205. 	addi	SP,   SP, -STACKSIZE

  206. 	li	r0,   0

  207. 

  208. 	stfd	f14,     0(SP)

  209. 	stfd	f15,     8(SP)

  210. 	stfd	f16,    16(SP)

  211. 	stfd	f17,    24(SP)

  212. 	stfd	f18,    32(SP)

  213. 	stfd	f19,    40(SP)

  214. 	stfd	f20,    48(SP)

  215. 	stfd	f21,    56(SP)

  216. 	stfd	f22,    64(SP)

  217. 	stfd	f23,    72(SP)

  218. 	stfd	f24,    80(SP)

  219. 	stfd	f25,    88(SP)

  220. 	stfd	f26,    96(SP)

  221. 	stfd	f27,   104(SP)

  222. 	stfd	f28,   112(SP)

  223. 	stfd	f29,   120(SP)

  224. 	stfd	f30,   128(SP)

  225. 	stfd	f31,   136(SP)

  226. 

  227. #ifdef __64BIT__

  228. 	std	r0,    FZERO

  229. 	std	r14,   144(SP)

  230. 	std	r15,   152(SP)

  231. 	std	r16,   160(SP)

  232. 	std	r17,   168(SP)

  233. 	std	r18,   176(SP)

  234. 	std	r19,   184(SP)

  235. 	std	r20,   192(SP)

  236. 	std	r21,   200(SP)

  237. 	std	r22,   208(SP)

  238. 	std	r23,   216(SP)

  239. 	std	r24,   224(SP)

  240. 	std	r25,   232(SP)

  241. 	std	r26,   240(SP)

  242. 	std	r27,   248(SP)

  243. #else

  244. 	stw	r0,    0 + FZERO

  245. 	stw	r0,    4 + FZERO

  246. 	stw	r14,   144(SP)

  247. 	stw	r15,   148(SP)

  248. 	stw	r16,   152(SP)

  249. 	stw	r17,   156(SP)

  250. 	stw	r18,   160(SP)

  251. 	stw	r19,   164(SP)

  252. 	stw	r20,   168(SP)

  253. 	stw	r21,   172(SP)

  254. 	stw	r22,   176(SP)

  255. 	stw	r23,   180(SP)

  256. 	stw	r24,   184(SP)

  257. 	stw	r25,   188(SP)

  258. 	stw	r26,   192(SP)

  259. 	stw	r27,   196(SP)

  260. #endif

  261. 

  262. #if defined(linux) || defined(__FreeBSD__)

  263. #ifndef __64BIT__

  264. 	lwz	BUFFER,  FRAMESLOT(0) + STACKSIZE(SP)

  265. #else

  266. 	ld	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  267. 	ld	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  268. 	ld	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  269. #endif

  270. #endif

  271. 

  272. #if defined(_AIX) || defined(__APPLE__)

  273. #ifndef __64BIT__

  274. #ifdef DOUBLE

  275. 	lwz	X,       FRAMESLOT(0) + STACKSIZE(SP)

  276. 	lwz	INCX,    FRAMESLOT(1) + STACKSIZE(SP)

  277. 	lwz	Y,       FRAMESLOT(2) + STACKSIZE(SP)

  278. 	lwz	INCY,    FRAMESLOT(3) + STACKSIZE(SP)

  279. 	lwz	BUFFER,  FRAMESLOT(4) + STACKSIZE(SP)

  280. #else

  281. 	lwz	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  282. 	lwz	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  283. 	lwz	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  284. #endif

  285. #else

  286. 	ld	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  287. 	ld	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  288. 	ld	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  289. #endif

  290. #endif

  291. 

  292. 	STFD	alpha_r, ALPHA_R

  293. 	STFD	alpha_i, ALPHA_I

  294. 

  295. 	slwi	LDA,  LDA,  ZBASE_SHIFT

  296. 	slwi	INCX, INCX, ZBASE_SHIFT

  297. 	slwi	INCY, INCY, ZBASE_SHIFT

  298. 

  299. 	li	PREA, PREFETCHSIZE_A * SIZE

  300. 

  301. 	cmpwi	cr0, M, 0

  302. 	ble-	LL(999)

  303. 

  304. 	cmpwi	cr0, INCX, 2 * SIZE

  305. 	beq	LL(05)

  306. 

  307. 	mr	XX, X

  308. 	mr	X, BUFFER

  309. 

  310. 	srawi.	r0, M, 2

  311. 	mtspr	CTR, r0

  312. 	ble	LL(03)

  313. 	.align 4

  314. 

  315. LL(01):

  316. 	LFD	a1, 0 * SIZE(XX)

  317. 	LFD	a2, 1 * SIZE(XX)

  318. 	add	XX, XX, INCX

  319. 	LFD	a3, 0 * SIZE(XX)

  320. 	LFD	a4, 1 * SIZE(XX)

  321. 	add	XX, XX, INCX

  322. 	LFD	a5, 0 * SIZE(XX)

  323. 	LFD	a6, 1 * SIZE(XX)

  324. 	add	XX, XX, INCX

  325. 	LFD	a7, 0 * SIZE(XX)

  326. 	LFD	a8, 1 * SIZE(XX)

  327. 	add	XX, XX, INCX

  328. 

  329. 	dcbt	XX, PREA

  330. 	dcbtst	BUFFER, PREA

  331. 

  332. 	STFD	a1, 0 * SIZE(BUFFER)

  333. 	STFD	a2, 1 * SIZE(BUFFER)

  334. 	STFD	a3, 2 * SIZE(BUFFER)

  335. 	STFD	a4, 3 * SIZE(BUFFER)

  336. 	STFD	a5, 4 * SIZE(BUFFER)

  337. 	STFD	a6, 5 * SIZE(BUFFER)

  338. 	STFD	a7, 6 * SIZE(BUFFER)

  339. 	STFD	a8, 7 * SIZE(BUFFER)

  340. 

  341. 	addi	BUFFER, BUFFER, 8 * SIZE

  342. 	bdnz	LL(01)

  343. 	.align 4

  344. 

  345. LL(03):

  346. 	andi.	r0, M, 3

  347. 	mtspr	CTR, r0

  348. 	ble	LL(05)

  349. 	.align 4

  350. 

  351. LL(04):

  352. 	LFD	a1, 0 * SIZE(XX)

  353. 	LFD	a2, 1 * SIZE(XX)

  354. 	add	XX, XX, INCX

  355. 

  356. 	STFD	a1, 0 * SIZE(BUFFER)

  357. 	STFD	a2, 1 * SIZE(BUFFER)

  358. 

  359. 	addi	BUFFER, BUFFER, 2 * SIZE

  360. 	bdnz	LL(04)

  361. 	.align 4

  362. 

  363. LL(05):

  364. 	mr	NEW_Y, Y

  365. 	lfd	f0, FZERO

  366. 

  367. 	cmpwi	cr0, INCY, 2 * SIZE

  368. 	beq	LL(10)

  369. 

  370. 	mr	NEW_Y, BUFFER

  371. 

  372. 	addi	r0, M,  3

  373. 	srawi.	r0, r0, 2

  374. 	mtspr	CTR, r0

  375. 	.align 4

  376. 

  377. LL(06):

  378. 	STFD	f0, 0 * SIZE(BUFFER)

  379. 	STFD	f0, 1 * SIZE(BUFFER)

  380. 	STFD	f0, 2 * SIZE(BUFFER)

  381. 	STFD	f0, 3 * SIZE(BUFFER)

  382. 	STFD	f0, 4 * SIZE(BUFFER)

  383. 	STFD	f0, 5 * SIZE(BUFFER)

  384. 	STFD	f0, 6 * SIZE(BUFFER)

  385. 	STFD	f0, 7 * SIZE(BUFFER)

  386. 	addi	BUFFER, BUFFER, 8 * SIZE

  387. 	bdnz	LL(06)

  388. 	.align 4

  389. 

  390. LL(10):

  391. 	li	IS, 0

  392. 

  393. 	cmpwi	cr0, N, 2

  394. 	blt	LL(20)

  395. 	.align 4

  396. 

  397. LL(11):

  398. 	mr	AO1, A

  399. 	slwi	TEMP,  IS,  ZBASE_SHIFT

  400. 	add	AO2, A,   LDA

  401. 	add	XX, X,     TEMP

  402. 

  403. 	add	A,   AO2, LDA

  404. 	add	YY, NEW_Y, TEMP

  405. 	addi	A,   A,   4 * SIZE

  406. 	NOP2

  407. 

  408. 	LFD	y05, ALPHA_R

  409. 	LFD	y06, ALPHA_I

  410. 

  411. 	LFD	atemp1, 0 * SIZE(XX)

  412. 	LFD	atemp2, 1 * SIZE(XX)

  413. 	LFD	atemp3, 2 * SIZE(XX)

  414. 	LFD	atemp4, 3 * SIZE(XX)

  415. 

  416. 	LFD	a1,  0 * SIZE(AO1)

  417. 	LFD	a2,  1 * SIZE(AO1)

  418. 	LFD	a3,  2 * SIZE(AO1)

  419. 	LFD	a4,  3 * SIZE(AO1)

  420. 

  421. 	LFD	a7,  2 * SIZE(AO2)

  422. 	LFD	a8,  3 * SIZE(AO2)

  423. 

  424. 	FMUL	xsum1, atemp1, a1

  425. 	addi	AO2, AO2, 4 * SIZE

  426. 	FMUL	xsum2, atemp2, a1

  427. 	LFD	a1,  4 * SIZE(AO1)

  428. 	FMUL	xsum3, atemp1, a3

  429. 	addi	AO1, AO1, 4 * SIZE

  430. 	FMUL	xsum4, atemp2, a3

  431. 	LFD	a5,  0 * SIZE(AO2)

  432. 

  433. #ifndef HEMV

  434. 	FNMSUB	xsum1, atemp2, a2, xsum1

  435. #endif

  436. 	addi	XX,  XX,  4 * SIZE

  437. #ifndef HEMV

  438. 	FMADD	xsum2, atemp1, a2, xsum2

  439. #endif

  440. 	LFD	a2,  1 * SIZE(AO1)

  441. 	FNMSUB	xsum3, atemp2, a4, xsum3

  442. 	addi	YY,  YY,  4 * SIZE

  443. 	FMADD	xsum4, atemp1, a4, xsum4

  444. 	LFD	a6,  1 * SIZE(AO2)

  445. 

  446. 	FMADD	xsum1, atemp3, a3, xsum1

  447. 	sub	TEMP, M, IS

  448. 	FMADD	xsum2, atemp4, a3, xsum2

  449. 	LFD	a3,  2 * SIZE(AO1)

  450. 	FMADD	xsum3, atemp3, a7, xsum3

  451. 	addi	TEMP, TEMP, -2

  452. 	FMADD	xsum4, atemp4, a7, xsum4

  453. 	LFD	a7,  2 * SIZE(AO2)

  454. 

  455. 	FMADD1	xsum1, atemp4, a4, xsum1

  456. 	srawi.	r0,  TEMP, 3

  457. 	FMADD2	xsum2, atemp3, a4, xsum2

  458. 	LFD	a4,  3 * SIZE(AO1)

  459. #ifndef HEMV

  460. 	FMADD1	xsum3, atemp4, a8, xsum3

  461. #endif

  462. 	mtspr	CTR, r0

  463. #ifndef HEMV

  464. 	FMADD2	xsum4, atemp3, a8, xsum4

  465. #endif

  466. 	LFD	a8,  3 * SIZE(AO2)

  467. 

  468. 	FMUL	xtemp1, y05, atemp1

  469. 	LFD	y01,  0 * SIZE(YY)

  470. 	FMUL	xtemp2, y06, atemp1

  471. 	LFD	y02,  1 * SIZE(YY)

  472. 	FMUL	xtemp3, y05, atemp3

  473. 	LFD	y03,  2 * SIZE(YY)

  474. 	FMUL	xtemp4, y06, atemp3

  475. 	LFD	y04,  3 * SIZE(YY)

  476. 

  477. 	FNMSUB	atemp1, y06, atemp2, xtemp1

  478. 	LFD	xtemp1,  0 * SIZE(XX)

  479. 	FMADD	atemp2, y05, atemp2, xtemp2

  480. 	LFD	xtemp2,  1 * SIZE(XX)

  481. 	FNMSUB	atemp3, y06, atemp4, xtemp3

  482. 	LFD	xtemp3,  2 * SIZE(XX)

  483. 	FMADD	atemp4, y05, atemp4, xtemp4

  484. 	LFD	xtemp4,  3 * SIZE(XX)

  485. 

  486. 	NOP1

  487. 	ble	LL(15)

  488. 

  489. 	FMADD	xsum1, xtemp1, a1,  xsum1

  490. 	DCBT(AO1, PREA)

  491. 	FMADD	y01, atemp1, a1, y01

  492. 	NOP2

  493. 

  494. 	FMADD	xsum2, xtemp2, a1,  xsum2

  495. 	NOP1

  496. 	FMADD	y02, atemp2, a1, y02

  497. 	LFD	a1,  4 * SIZE(AO1)

  498. 

  499. 	FMADD	xsum3, xtemp1, a5,  xsum3

  500. 	NOP1

  501. 	FMADD	y03, atemp1, a3, y03

  502. 	NOP2

  503. 

  504. 	FMADD	xsum4, xtemp2, a5,  xsum4

  505. 	NOP1

  506. 	FMADD	y04, atemp2, a3, y04

  507. 	NOP2

  508. 

  509. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  510. 	LFD	y05,  4 * SIZE(YY)

  511. 	FNMSUB	y01, atemp2, a2, y01

  512. 	NOP2

  513. 

  514. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  515. 	LFD	y06,  5 * SIZE(YY)

  516. 	FMADD	y02, atemp1, a2, y02

  517. 	LFD	a2,  5 * SIZE(AO1)

  518. 

  519. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  520. 	LFD	xtemp2,  5 * SIZE(XX)

  521. 	FNMSUB	y03, atemp2, a4, y03

  522. 	NOP2

  523. 

  524. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  525. 	LFD	xtemp1,  4 * SIZE(XX)

  526. 	FMADD	y04, atemp1, a4, y04

  527. 	NOP2

  528. 

  529. 	FMADD	xsum1, xtemp3, a3,  xsum1

  530. 	LFD	y07,  6 * SIZE(YY)

  531. 	FMADD	y01, atemp3, a5, y01

  532. 	NOP2

  533. 

  534. 	FMADD	xsum2, xtemp4, a3,  xsum2

  535. 	LFD	a3,  6 * SIZE(AO1)

  536. 	FMADD	y02, atemp4, a5, y02

  537. 	LFD	a5,  4 * SIZE(AO2)

  538. 

  539. 	FMADD	xsum3, xtemp3, a7,  xsum3

  540. 	LFD	y08,  7 * SIZE(YY)

  541. 	FMADD	y03, atemp3, a7, y03

  542. 	NOP2

  543. 

  544. 	FMADD	xsum4, xtemp4, a7,  xsum4

  545. 	NOP1

  546. 	FMADD	y04, atemp4, a7, y04

  547. 	LFD	a7,  6 * SIZE(AO2)

  548. 

  549. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  550. 	NOP1

  551. 	FNMSUB	y01, atemp4, a6, y01

  552. #	DCBT(X, PREX)

  553. 	NOP2

  554. 

  555. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  556. 	LFD	a4,  7 * SIZE(AO1)

  557. 	FMADD	y02, atemp3, a6, y02

  558. 	LFD	a6,  5 * SIZE(AO2)

  559. 

  560. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  561. 	LFD	xtemp4,  7 * SIZE(XX)

  562. 	FNMSUB	y03, atemp4, a8, y03

  563. 	NOP2

  564. 

  565. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  566. 	LFD	xtemp3,  6 * SIZE(XX)

  567. 	FMADD	y04, atemp3, a8, y04

  568. 	LFD	a8,  7 * SIZE(AO2)

  569. 

  570. 	FMADD	xsum1, xtemp1, a1,  xsum1

  571. 	STFD	y01,  0 * SIZE(YY)

  572. 	FMADD	y05, atemp1, a1, y05

  573. 	NOP2

  574. 

  575. 	FMADD	xsum2, xtemp2, a1,  xsum2

  576. 	STFD	y02,  1 * SIZE(YY)

  577. 	FMADD	y06, atemp2, a1, y06

  578. 	LFD	a1,  8 * SIZE(AO1)

  579. 

  580. 	FMADD	xsum3, xtemp1, a5,  xsum3

  581. 	STFD	y03,  2 * SIZE(YY)

  582. 	FMADD	y07, atemp1, a3, y07

  583. 	NOP2

  584. 

  585. 	FMADD	xsum4, xtemp2, a5,  xsum4

  586. 	STFD	y04,  3 * SIZE(YY)

  587. 	FMADD	y08, atemp2, a3, y08

  588. 	NOP2

  589. 

  590. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  591. 	LFD	y01,  8 * SIZE(YY)

  592. 	FNMSUB	y05, atemp2, a2, y05

  593. 	NOP2

  594. 

  595. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  596. 	LFD	y02,  9 * SIZE(YY)

  597. 	FMADD	y06, atemp1, a2, y06

  598. 	LFD	a2,  9 * SIZE(AO1)

  599. 

  600. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  601. 	LFD	xtemp2,  9 * SIZE(XX)

  602. 	FNMSUB	y07, atemp2, a4, y07

  603. 	NOP2

  604. 

  605. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  606. 	LFD	xtemp1,  8 * SIZE(XX)

  607. 	FMADD	y08, atemp1, a4, y08

  608. 	NOP2

  609. 

  610. 	FMADD	xsum1, xtemp3, a3,  xsum1

  611. 	LFD	y03, 10 * SIZE(YY)

  612. 	FMADD	y05, atemp3, a5, y05

  613. 	NOP2

  614. 

  615. 	FMADD	xsum2, xtemp4, a3,  xsum2

  616. 	LFD	a3, 10 * SIZE(AO1)

  617. 	FMADD	y06, atemp4, a5, y06

  618. 	LFD	a5,  8 * SIZE(AO2)

  619. 

  620. 	FMADD	xsum3, xtemp3, a7,  xsum3

  621. 	LFD	y04, 11 * SIZE(YY)

  622. 	FMADD	y07, atemp3, a7, y07

  623. 	NOP2

  624. 

  625. 	FMADD	xsum4, xtemp4, a7,  xsum4

  626. 	NOP1

  627. 	FMADD	y08, atemp4, a7, y08

  628. 	LFD	a7, 10 * SIZE(AO2)

  629. 

  630. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  631. 	NOP1

  632. 	FNMSUB	y05, atemp4, a6, y05

  633. 	NOP2

  634. 

  635. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  636. 	LFD	a4, 11 * SIZE(AO1)

  637. 	FMADD	y06, atemp3, a6, y06

  638. 	LFD	a6,  9 * SIZE(AO2)

  639. 

  640. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  641. 	LFD	xtemp4, 11 * SIZE(XX)

  642. 	FNMSUB	y07, atemp4, a8, y07

  643. 	bdz	LL(13)

  644. 	.align 4

  645. 

  646. LL(12):

  647. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  648. 	LFD	xtemp3, 10 * SIZE(XX)

  649. 	FMADD	y08, atemp3, a8, y08

  650. 	LFD	a8, 11 * SIZE(AO2)

  651. 

  652. 	FMADD	xsum1, xtemp1, a1,  xsum1

  653. 	STFD	y05,  4 * SIZE(YY)

  654. 	FMADD	y01, atemp1, a1, y01

  655. 	DCBT(AO2, PREA)

  656. 

  657. 	FMADD	xsum2, xtemp2, a1,  xsum2

  658. 	STFD	y06,  5 * SIZE(YY)

  659. 	FMADD	y02, atemp2, a1, y02

  660. 	LFD	a1, 12 * SIZE(AO1)

  661. 

  662. 	FMADD	xsum3, xtemp1, a5,  xsum3

  663. 	STFD	y07,  6 * SIZE(YY)

  664. 	FMADD	y03, atemp1, a3, y03

  665. 	NOP2

  666. 

  667. 	FMADD	xsum4, xtemp2, a5,  xsum4

  668. 	STFD	y08,  7 * SIZE(YY)

  669. 	FMADD	y04, atemp2, a3, y04

  670. 	NOP2

  671. 

  672. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  673. 	LFD	y05, 12 * SIZE(YY)

  674. 	FNMSUB	y01, atemp2, a2, y01

  675. 	NOP2

  676. 

  677. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  678. 	LFD	y06, 13 * SIZE(YY)

  679. 	FMADD	y02, atemp1, a2, y02

  680. 	LFD	a2, 13 * SIZE(AO1)

  681. 

  682. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  683. 	LFD	xtemp2, 13 * SIZE(XX)

  684. 	FNMSUB	y03, atemp2, a4, y03

  685. 	NOP2

  686. 

  687. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  688. 	LFD	xtemp1, 12 * SIZE(XX)

  689. 	FMADD	y04, atemp1, a4, y04

  690. 	NOP2

  691. 

  692. 	FMADD	xsum1, xtemp3, a3,  xsum1

  693. 	LFD	y07, 14 * SIZE(YY)

  694. 	FMADD	y01, atemp3, a5, y01

  695. 	NOP2

  696. 

  697. 	FMADD	xsum2, xtemp4, a3,  xsum2

  698. 	LFD	a3, 14 * SIZE(AO1)

  699. 	FMADD	y02, atemp4, a5, y02

  700. 	LFD	a5, 12 * SIZE(AO2)

  701. 

  702. 	FMADD	xsum3, xtemp3, a7,  xsum3

  703. 	LFD	y08, 15 * SIZE(YY)

  704. 	FMADD	y03, atemp3, a7, y03

  705. 	NOP2

  706. 

  707. 	FMADD	xsum4, xtemp4, a7,  xsum4

  708. 	NOP1

  709. 	FMADD	y04, atemp4, a7, y04

  710. 	LFD	a7, 14 * SIZE(AO2)

  711. 

  712. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  713. 	NOP1

  714. 	FNMSUB	y01, atemp4, a6, y01

  715. #	DCBT(Y1, PREY)

  716. 	NOP2

  717. 

  718. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  719. 	LFD	a4, 15 * SIZE(AO1)

  720. 	FMADD	y02, atemp3, a6, y02

  721. 	LFD	a6, 13 * SIZE(AO2)

  722. 

  723. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  724. 	LFD	xtemp4, 15 * SIZE(XX)

  725. 	FNMSUB	y03, atemp4, a8, y03

  726. 	NOP2

  727. 

  728. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  729. 	LFD	xtemp3, 14 * SIZE(XX)

  730. 	FMADD	y04, atemp3, a8, y04

  731. 	LFD	a8, 15 * SIZE(AO2)

  732. 

  733. 	FMADD	xsum1, xtemp1, a1,  xsum1

  734. 	STFD	y01,  8 * SIZE(YY)

  735. 	FMADD	y05, atemp1, a1, y05

  736. 	NOP2

  737. 

  738. 	FMADD	xsum2, xtemp2, a1,  xsum2

  739. 	STFD	y02,  9 * SIZE(YY)

  740. 	FMADD	y06, atemp2, a1, y06

  741. 	LFD	a1, 16 * SIZE(AO1)

  742. 

  743. 	FMADD	xsum3, xtemp1, a5,  xsum3

  744. 	STFD	y03, 10 * SIZE(YY)

  745. 	FMADD	y07, atemp1, a3, y07

  746. 	NOP2

  747. 

  748. 	FMADD	xsum4, xtemp2, a5,  xsum4

  749. 	STFD	y04, 11 * SIZE(YY)

  750. 	FMADD	y08, atemp2, a3, y08

  751. 	NOP2

  752. 

  753. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  754. 	LFD	y01, 16 * SIZE(YY)

  755. 	FNMSUB	y05, atemp2, a2, y05

  756. 	NOP2

  757. 

  758. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  759. 	LFD	y02, 17 * SIZE(YY)

  760. 	FMADD	y06, atemp1, a2, y06

  761. 	LFD	a2, 17 * SIZE(AO1)

  762. 

  763. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  764. 	LFD	xtemp2, 17 * SIZE(XX)

  765. 	FNMSUB	y07, atemp2, a4, y07

  766. 	NOP2

  767. 

  768. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  769. 	LFD	xtemp1, 16 * SIZE(XX)

  770. 	FMADD	y08, atemp1, a4, y08

  771. 	addi	AO2, AO2, 16 * SIZE

  772. 

  773. 	FMADD	xsum1, xtemp3, a3,  xsum1

  774. 	LFD	y03, 18 * SIZE(YY)

  775. 	FMADD	y05, atemp3, a5, y05

  776. 	addi	XX, XX, 16 * SIZE

  777. 

  778. 	FMADD	xsum2, xtemp4, a3,  xsum2

  779. 	LFD	a3, 18 * SIZE(AO1)

  780. 	FMADD	y06, atemp4, a5, y06

  781. 	LFD	a5,  0 * SIZE(AO2)

  782. 

  783. 	FMADD	xsum3, xtemp3, a7,  xsum3

  784. 	LFD	y04, 19 * SIZE(YY)

  785. 	FMADD	y07, atemp3, a7, y07

  786. 	NOP2

  787. 

  788. 	FMADD	xsum4, xtemp4, a7,  xsum4

  789. 	addi	AO1, AO1, 16 * SIZE

  790. 	FMADD	y08, atemp4, a7, y08

  791. 	LFD	a7,  2 * SIZE(AO2)

  792. 

  793. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  794. 	addi	YY, YY, 16 * SIZE

  795. 	FNMSUB	y05, atemp4, a6, y05

  796. 	NOP2

  797. 

  798. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  799. 	LFD	a4,  3 * SIZE(AO1)

  800. 	FMADD	y06, atemp3, a6, y06

  801. 	LFD	a6,  1 * SIZE(AO2)

  802. 

  803. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  804. 	LFD	xtemp4,  3 * SIZE(XX)

  805. 	FNMSUB	y07, atemp4, a8, y07

  806. 	NOP2

  807. 

  808. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  809. 	LFD	xtemp3,  2 * SIZE(XX)

  810. 	FMADD	y08, atemp3, a8, y08

  811. 	LFD	a8,  3 * SIZE(AO2)

  812. 

  813. 	FMADD	xsum1, xtemp1, a1,  xsum1

  814. 	STFD	y05, -4 * SIZE(YY)

  815. 	FMADD	y01, atemp1, a1, y01

  816. 	DCBT(AO1, PREA)

  817. 

  818. 	FMADD	xsum2, xtemp2, a1,  xsum2

  819. 	STFD	y06, -3 * SIZE(YY)

  820. 	FMADD	y02, atemp2, a1, y02

  821. 	LFD	a1,  4 * SIZE(AO1)

  822. 

  823. 	FMADD	xsum3, xtemp1, a5,  xsum3

  824. 	STFD	y07, -2 * SIZE(YY)

  825. 	FMADD	y03, atemp1, a3, y03

  826. 	NOP2

  827. 

  828. 	FMADD	xsum4, xtemp2, a5,  xsum4

  829. 	STFD	y08, -1 * SIZE(YY)

  830. 	FMADD	y04, atemp2, a3, y04

  831. 	NOP2

  832. 

  833. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  834. 	LFD	y05,  4 * SIZE(YY)

  835. 	FNMSUB	y01, atemp2, a2, y01

  836. 	NOP2

  837. 

  838. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  839. 	LFD	y06,  5 * SIZE(YY)

  840. 	FMADD	y02, atemp1, a2, y02

  841. 	LFD	a2,  5 * SIZE(AO1)

  842. 

  843. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  844. 	LFD	xtemp2,  5 * SIZE(XX)

  845. 	FNMSUB	y03, atemp2, a4, y03

  846. 	NOP2

  847. 

  848. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  849. 	LFD	xtemp1,  4 * SIZE(XX)

  850. 	FMADD	y04, atemp1, a4, y04

  851. 	NOP2

  852. 

  853. 	FMADD	xsum1, xtemp3, a3,  xsum1

  854. 	LFD	y07,  6 * SIZE(YY)

  855. 	FMADD	y01, atemp3, a5, y01

  856. 	NOP2

  857. 

  858. 	FMADD	xsum2, xtemp4, a3,  xsum2

  859. 	LFD	a3,  6 * SIZE(AO1)

  860. 	FMADD	y02, atemp4, a5, y02

  861. 	LFD	a5,  4 * SIZE(AO2)

  862. 

  863. 	FMADD	xsum3, xtemp3, a7,  xsum3

  864. 	LFD	y08,  7 * SIZE(YY)

  865. 	FMADD	y03, atemp3, a7, y03

  866. 	NOP2

  867. 

  868. 	FMADD	xsum4, xtemp4, a7,  xsum4

  869. 	NOP1

  870. 	FMADD	y04, atemp4, a7, y04

  871. 	LFD	a7,  6 * SIZE(AO2)

  872. 

  873. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  874. 	NOP1

  875. 	FNMSUB	y01, atemp4, a6, y01

  876. #	DCBT(X, PREX)

  877. 	NOP2

  878. 

  879. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  880. 	LFD	a4,  7 * SIZE(AO1)

  881. 	FMADD	y02, atemp3, a6, y02

  882. 	LFD	a6,  5 * SIZE(AO2)

  883. 

  884. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  885. 	LFD	xtemp4,  7 * SIZE(XX)

  886. 	FNMSUB	y03, atemp4, a8, y03

  887. 	NOP2

  888. 

  889. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  890. 	LFD	xtemp3,  6 * SIZE(XX)

  891. 	FMADD	y04, atemp3, a8, y04

  892. 	LFD	a8,  7 * SIZE(AO2)

  893. 

  894. 	FMADD	xsum1, xtemp1, a1,  xsum1

  895. 	STFD	y01,  0 * SIZE(YY)

  896. 	FMADD	y05, atemp1, a1, y05

  897. 	NOP2

  898. 

  899. 	FMADD	xsum2, xtemp2, a1,  xsum2

  900. 	STFD	y02,  1 * SIZE(YY)

  901. 	FMADD	y06, atemp2, a1, y06

  902. 	LFD	a1,  8 * SIZE(AO1)

  903. 

  904. 	FMADD	xsum3, xtemp1, a5,  xsum3

  905. 	STFD	y03,  2 * SIZE(YY)

  906. 	FMADD	y07, atemp1, a3, y07

  907. 	NOP2

  908. 

  909. 	FMADD	xsum4, xtemp2, a5,  xsum4

  910. 	STFD	y04,  3 * SIZE(YY)

  911. 	FMADD	y08, atemp2, a3, y08

  912. 	NOP2

  913. 

  914. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  915. 	LFD	y01,  8 * SIZE(YY)

  916. 	FNMSUB	y05, atemp2, a2, y05

  917. 	NOP2

  918. 

  919. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  920. 	LFD	y02,  9 * SIZE(YY)

  921. 	FMADD	y06, atemp1, a2, y06

  922. 	LFD	a2,  9 * SIZE(AO1)

  923. 

  924. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  925. 	LFD	xtemp2,  9 * SIZE(XX)

  926. 	FNMSUB	y07, atemp2, a4, y07

  927. 	NOP2

  928. 

  929. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  930. 	LFD	xtemp1,  8 * SIZE(XX)

  931. 	FMADD	y08, atemp1, a4, y08

  932. 	NOP2

  933. 

  934. 	FMADD	xsum1, xtemp3, a3,  xsum1

  935. 	LFD	y03, 10 * SIZE(YY)

  936. 	FMADD	y05, atemp3, a5, y05

  937. 	NOP2

  938. 

  939. 	FMADD	xsum2, xtemp4, a3,  xsum2

  940. 	LFD	a3, 10 * SIZE(AO1)

  941. 	FMADD	y06, atemp4, a5, y06

  942. 	LFD	a5,  8 * SIZE(AO2)

  943. 

  944. 	FMADD	xsum3, xtemp3, a7,  xsum3

  945. 	LFD	y04, 11 * SIZE(YY)

  946. 	FMADD	y07, atemp3, a7, y07

  947. 	NOP2

  948. 

  949. 	FMADD	xsum4, xtemp4, a7,  xsum4

  950. 	NOP1

  951. 	FMADD	y08, atemp4, a7, y08

  952. 	LFD	a7, 10 * SIZE(AO2)

  953. 

  954. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  955. 	NOP1

  956. 	FNMSUB	y05, atemp4, a6, y05

  957. 	NOP2

  958. 

  959. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  960. 	LFD	a4, 11 * SIZE(AO1)

  961. 	FMADD	y06, atemp3, a6, y06

  962. 	LFD	a6,  9 * SIZE(AO2)

  963. 

  964. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  965. 	LFD	xtemp4, 11 * SIZE(XX)

  966. 	FNMSUB	y07, atemp4, a8, y07

  967. 	bdnz	LL(12)

  968. 	.align 4

  969. 

  970. LL(13):

  971. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  972. 	LFD	xtemp3, 10 * SIZE(XX)

  973. 	FMADD	y08, atemp3, a8, y08

  974. 	LFD	a8, 11 * SIZE(AO2)

  975. 

  976. 	FMADD	xsum1, xtemp1, a1,  xsum1

  977. 	STFD	y05,  4 * SIZE(YY)

  978. 	FMADD	y01, atemp1, a1, y01

  979. 	NOP2

  980. 

  981. 	FMADD	xsum2, xtemp2, a1,  xsum2

  982. 	STFD	y06,  5 * SIZE(YY)

  983. 	FMADD	y02, atemp2, a1, y02

  984. 	LFD	a1, 12 * SIZE(AO1)

  985. 

  986. 	FMADD	xsum3, xtemp1, a5,  xsum3

  987. 	STFD	y07,  6 * SIZE(YY)

  988. 	FMADD	y03, atemp1, a3, y03

  989. 	NOP2

  990. 

  991. 	FMADD	xsum4, xtemp2, a5,  xsum4

  992. 	STFD	y08,  7 * SIZE(YY)

  993. 	FMADD	y04, atemp2, a3, y04

  994. 	NOP2

  995. 

  996. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  997. 	LFD	y05, 12 * SIZE(YY)

  998. 	FNMSUB	y01, atemp2, a2, y01

  999. 	NOP2

  1000. 

  1001. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1002. 	LFD	y06, 13 * SIZE(YY)

  1003. 	FMADD	y02, atemp1, a2, y02

  1004. 	LFD	a2, 13 * SIZE(AO1)

  1005. 

  1006. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1007. 	LFD	xtemp2, 13 * SIZE(XX)

  1008. 	FNMSUB	y03, atemp2, a4, y03

  1009. 	NOP2

  1010. 

  1011. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1012. 	LFD	xtemp1, 12 * SIZE(XX)

  1013. 	FMADD	y04, atemp1, a4, y04

  1014. 	NOP2

  1015. 

  1016. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1017. 	LFD	y07, 14 * SIZE(YY)

  1018. 	FMADD	y01, atemp3, a5, y01

  1019. 	NOP2

  1020. 

  1021. 	FMADD	xsum2, xtemp4, a3,  xsum2

  1022. 	LFD	a3, 14 * SIZE(AO1)

  1023. 	FMADD	y02, atemp4, a5, y02

  1024. 	LFD	a5, 12 * SIZE(AO2)

  1025. 

  1026. 	FMADD	xsum3, xtemp3, a7,  xsum3

  1027. 	LFD	y08, 15 * SIZE(YY)

  1028. 	FMADD	y03, atemp3, a7, y03

  1029. 	NOP2

  1030. 

  1031. 	FMADD	xsum4, xtemp4, a7,  xsum4

  1032. 	NOP1

  1033. 	FMADD	y04, atemp4, a7, y04

  1034. 	LFD	a7, 14 * SIZE(AO2)

  1035. 

  1036. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  1037. 	NOP1

  1038. 	FNMSUB	y01, atemp4, a6, y01

  1039. 	NOP2

  1040. 

  1041. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  1042. 	LFD	a4, 15 * SIZE(AO1)

  1043. 	FMADD	y02, atemp3, a6, y02

  1044. 	LFD	a6, 13 * SIZE(AO2)

  1045. 

  1046. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  1047. 	LFD	xtemp4, 15 * SIZE(XX)

  1048. 	FNMSUB	y03, atemp4, a8, y03

  1049. 	NOP2

  1050. 

  1051. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  1052. 	LFD	xtemp3, 14 * SIZE(XX)

  1053. 	FMADD	y04, atemp3, a8, y04

  1054. 	LFD	a8, 15 * SIZE(AO2)

  1055. 

  1056. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1057. 	STFD	y01,  8 * SIZE(YY)

  1058. 	FMADD	y05, atemp1, a1, y05

  1059. 	NOP2

  1060. 

  1061. 	FMADD	xsum2, xtemp2, a1,  xsum2

  1062. 	STFD	y02,  9 * SIZE(YY)

  1063. 	FMADD	y06, atemp2, a1, y06

  1064. 	LFD	a1, 16 * SIZE(AO1)

  1065. 

  1066. 	FMADD	xsum3, xtemp1, a5,  xsum3

  1067. 	STFD	y03, 10 * SIZE(YY)

  1068. 	FMADD	y07, atemp1, a3, y07

  1069. 	NOP2

  1070. 

  1071. 	FMADD	xsum4, xtemp2, a5,  xsum4

  1072. 	STFD	y04, 11 * SIZE(YY)

  1073. 	FMADD	y08, atemp2, a3, y08

  1074. 	NOP2

  1075. 

  1076. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  1077. 	LFD	y01, 16 * SIZE(YY)

  1078. 	FNMSUB	y05, atemp2, a2, y05

  1079. 	NOP2

  1080. 

  1081. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1082. 	LFD	y02, 17 * SIZE(YY)

  1083. 	FMADD	y06, atemp1, a2, y06

  1084. 	LFD	a2, 17 * SIZE(AO1)

  1085. 

  1086. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1087. 	LFD	xtemp2, 17 * SIZE(XX)

  1088. 	FNMSUB	y07, atemp2, a4, y07

  1089. 	NOP2

  1090. 

  1091. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1092. 	LFD	xtemp1, 16 * SIZE(XX)

  1093. 	FMADD	y08, atemp1, a4, y08

  1094. 	addi	AO2, AO2, 16 * SIZE

  1095. 

  1096. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1097. 	LFD	y03, 18 * SIZE(YY)

  1098. 	FMADD	y05, atemp3, a5, y05

  1099. 	addi	XX, XX, 16 * SIZE

  1100. 

  1101. 	FMADD	xsum2, xtemp4, a3,  xsum2

  1102. 	LFD	a3, 18 * SIZE(AO1)

  1103. 	FMADD	y06, atemp4, a5, y06

  1104. 	LFD	a5,  0 * SIZE(AO2)

  1105. 

  1106. 	FMADD	xsum3, xtemp3, a7,  xsum3

  1107. 	LFD	y04, 19 * SIZE(YY)

  1108. 	FMADD	y07, atemp3, a7, y07

  1109. 	NOP2

  1110. 

  1111. 	FMADD	xsum4, xtemp4, a7,  xsum4

  1112. 	addi	AO1, AO1, 16 * SIZE

  1113. 	FMADD	y08, atemp4, a7, y08

  1114. 	LFD	a7,  2 * SIZE(AO2)

  1115. 

  1116. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  1117. 	addi	YY, YY, 16 * SIZE

  1118. 	FNMSUB	y05, atemp4, a6, y05

  1119. 	NOP2

  1120. 

  1121. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  1122. 	LFD	a4,  3 * SIZE(AO1)

  1123. 	FMADD	y06, atemp3, a6, y06

  1124. 	LFD	a6,  1 * SIZE(AO2)

  1125. 

  1126. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  1127. 	LFD	xtemp4,  3 * SIZE(XX)

  1128. 	FNMSUB	y07, atemp4, a8, y07

  1129. 	NOP2

  1130. 

  1131. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  1132. 	LFD	xtemp3,  2 * SIZE(XX)

  1133. 	FMADD	y08, atemp3, a8, y08

  1134. 	LFD	a8,  3 * SIZE(AO2)

  1135. 

  1136. 	STFD	y05, -4 * SIZE(YY)

  1137. 	STFD	y06, -3 * SIZE(YY)

  1138. 	STFD	y07, -2 * SIZE(YY)

  1139. 	STFD	y08, -1 * SIZE(YY)

  1140. 	.align 4

  1141. 

  1142. LL(15):

  1143. 	andi.	r0,  TEMP, 4

  1144. 	ble	LL(16)

  1145. 

  1146. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1147. 	NOP1

  1148. 	FMADD	y01, atemp1, a1, y01

  1149. 	NOP2

  1150. 

  1151. 	FMADD	xsum2, xtemp2, a1,  xsum2

  1152. 	NOP1

  1153. 	FMADD	y02, atemp2, a1, y02

  1154. 	LFD	a1,  4 * SIZE(AO1)

  1155. 

  1156. 	FMADD	xsum3, xtemp1, a5,  xsum3

  1157. 	NOP1

  1158. 	FMADD	y03, atemp1, a3, y03

  1159. 	NOP2

  1160. 

  1161. 	FMADD	xsum4, xtemp2, a5,  xsum4

  1162. 	NOP1

  1163. 	FMADD	y04, atemp2, a3, y04

  1164. 	NOP2

  1165. 

  1166. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  1167. 	LFD	y05,  4 * SIZE(YY)

  1168. 	FNMSUB	y01, atemp2, a2, y01

  1169. 	NOP2

  1170. 

  1171. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1172. 	LFD	y06,  5 * SIZE(YY)

  1173. 	FMADD	y02, atemp1, a2, y02

  1174. 	LFD	a2,  5 * SIZE(AO1)

  1175. 

  1176. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1177. 	LFD	xtemp2,  5 * SIZE(XX)

  1178. 	FNMSUB	y03, atemp2, a4, y03

  1179. 	NOP2

  1180. 

  1181. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1182. 	LFD	xtemp1,  4 * SIZE(XX)

  1183. 	FMADD	y04, atemp1, a4, y04

  1184. 	NOP2

  1185. 

  1186. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1187. 	LFD	y07,  6 * SIZE(YY)

  1188. 	FMADD	y01, atemp3, a5, y01

  1189. 	NOP2

  1190. 

  1191. 	FMADD	xsum2, xtemp4, a3,  xsum2

  1192. 	LFD	a3,  6 * SIZE(AO1)

  1193. 	FMADD	y02, atemp4, a5, y02

  1194. 	LFD	a5,  4 * SIZE(AO2)

  1195. 

  1196. 	FMADD	xsum3, xtemp3, a7,  xsum3

  1197. 	LFD	y08,  7 * SIZE(YY)

  1198. 	FMADD	y03, atemp3, a7, y03

  1199. 	NOP2

  1200. 

  1201. 	FMADD	xsum4, xtemp4, a7,  xsum4

  1202. 	NOP1

  1203. 	FMADD	y04, atemp4, a7, y04

  1204. 	LFD	a7,  6 * SIZE(AO2)

  1205. 

  1206. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  1207. 	NOP1

  1208. 	FNMSUB	y01, atemp4, a6, y01

  1209. 	NOP2

  1210. 

  1211. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  1212. 	LFD	a4,  7 * SIZE(AO1)

  1213. 	FMADD	y02, atemp3, a6, y02

  1214. 	LFD	a6,  5 * SIZE(AO2)

  1215. 

  1216. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  1217. 	LFD	xtemp4,  7 * SIZE(XX)

  1218. 	FNMSUB	y03, atemp4, a8, y03

  1219. 	NOP2

  1220. 

  1221. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  1222. 	LFD	xtemp3,  6 * SIZE(XX)

  1223. 	FMADD	y04, atemp3, a8, y04

  1224. 	LFD	a8,  7 * SIZE(AO2)

  1225. 

  1226. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1227. 	STFD	y01,  0 * SIZE(YY)

  1228. 	FMADD	y05, atemp1, a1, y05

  1229. 	NOP2

  1230. 

  1231. 	FMADD	xsum2, xtemp2, a1,  xsum2

  1232. 	STFD	y02,  1 * SIZE(YY)

  1233. 	FMADD	y06, atemp2, a1, y06

  1234. 	LFD	a1,  8 * SIZE(AO1)

  1235. 

  1236. 	FMADD	xsum3, xtemp1, a5,  xsum3

  1237. 	STFD	y03,  2 * SIZE(YY)

  1238. 	FMADD	y07, atemp1, a3, y07

  1239. 	NOP2

  1240. 

  1241. 	FMADD	xsum4, xtemp2, a5,  xsum4

  1242. 	STFD	y04,  3 * SIZE(YY)

  1243. 	FMADD	y08, atemp2, a3, y08

  1244. 	NOP2

  1245. 

  1246. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  1247. 	LFD	y01,  8 * SIZE(YY)

  1248. 	FNMSUB	y05, atemp2, a2, y05

  1249. 	NOP2

  1250. 

  1251. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1252. 	LFD	y02,  9 * SIZE(YY)

  1253. 	FMADD	y06, atemp1, a2, y06

  1254. 	LFD	a2,  9 * SIZE(AO1)

  1255. 

  1256. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1257. 	LFD	xtemp2,  9 * SIZE(XX)

  1258. 	FNMSUB	y07, atemp2, a4, y07

  1259. 	NOP2

  1260. 

  1261. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1262. 	LFD	xtemp1,  8 * SIZE(XX)

  1263. 	FMADD	y08, atemp1, a4, y08

  1264. 	NOP2

  1265. 

  1266. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1267. 	LFD	y03, 10 * SIZE(YY)

  1268. 	FMADD	y05, atemp3, a5, y05

  1269. 	NOP2

  1270. 

  1271. 	FMADD	xsum2, xtemp4, a3,  xsum2

  1272. 	LFD	a3, 10 * SIZE(AO1)

  1273. 	FMADD	y06, atemp4, a5, y06

  1274. 	LFD	a5,  8 * SIZE(AO2)

  1275. 

  1276. 	FMADD	xsum3, xtemp3, a7,  xsum3

  1277. 	LFD	y04, 11 * SIZE(YY)

  1278. 	FMADD	y07, atemp3, a7, y07

  1279. 	NOP2

  1280. 

  1281. 	FMADD	xsum4, xtemp4, a7,  xsum4

  1282. 	addi	YY, YY, 8 * SIZE

  1283. 	FMADD	y08, atemp4, a7, y08

  1284. 	LFD	a7, 10 * SIZE(AO2)

  1285. 

  1286. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  1287. 	addi	AO2, AO2, 8 * SIZE

  1288. 	FNMSUB	y05, atemp4, a6, y05

  1289. 	addi	XX, XX, 8 * SIZE

  1290. 

  1291. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  1292. 	LFD	a4, 11 * SIZE(AO1)

  1293. 	FMADD	y06, atemp3, a6, y06

  1294. 	LFD	a6,  1 * SIZE(AO2)

  1295. 

  1296. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  1297. 	LFD	xtemp4,  3 * SIZE(XX)

  1298. 	FNMSUB	y07, atemp4, a8, y07

  1299. 	addi	AO1, AO1, 8 * SIZE

  1300. 

  1301. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  1302. 	LFD	xtemp3,  2 * SIZE(XX)

  1303. 	FMADD	y08, atemp3, a8, y08

  1304. 	LFD	a8,  3 * SIZE(AO2)

  1305. 

  1306. 	STFD	y05, -4 * SIZE(YY)

  1307. 	STFD	y06, -3 * SIZE(YY)

  1308. 	STFD	y07, -2 * SIZE(YY)

  1309. 	STFD	y08, -1 * SIZE(YY)

  1310. 	.align 4

  1311. 

  1312. LL(16):

  1313. 	andi.	r0,  TEMP, 2

  1314. 	ble	LL(17)

  1315. 

  1316. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1317. 	NOP1

  1318. 	FMADD	y01, atemp1, a1, y01

  1319. 	NOP2

  1320. 

  1321. 	FMADD	xsum2, xtemp2, a1,  xsum2

  1322. 	NOP1

  1323. 	FMADD	y02, atemp2, a1, y02

  1324. 	LFD	a1,  4 * SIZE(AO1)

  1325. 

  1326. 	FMADD	xsum3, xtemp1, a5,  xsum3

  1327. 	FMADD	y03, atemp1, a3, y03

  1328. 	FMADD	xsum4, xtemp2, a5,  xsum4

  1329. 	FMADD	y04, atemp2, a3, y04

  1330. 

  1331. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  1332. 	NOP1

  1333. 	FNMSUB	y01, atemp2, a2, y01

  1334. 	NOP2

  1335. 

  1336. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1337. 	NOP1

  1338. 	FMADD	y02, atemp1, a2, y02

  1339. 	LFD	a2,  5 * SIZE(AO1)

  1340. 

  1341. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1342. 	LFD	xtemp2,  5 * SIZE(XX)

  1343. 	FNMSUB	y03, atemp2, a4, y03

  1344. 	NOP2

  1345. 

  1346. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1347. 	LFD	xtemp1,  4 * SIZE(XX)

  1348. 	FMADD	y04, atemp1, a4, y04

  1349. 	NOP2

  1350. 

  1351. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1352. 	NOP1

  1353. 	FMADD	y01, atemp3, a5, y01

  1354. 	NOP2

  1355. 

  1356. 	FMADD	xsum2, xtemp4, a3,  xsum2

  1357. 	NOP1

  1358. 	FMADD	y02, atemp4, a5, y02

  1359. 	LFD	a5,  4 * SIZE(AO2)

  1360. 

  1361. 	FMADD	xsum3, xtemp3, a7,  xsum3

  1362. 	FMADD	y03, atemp3, a7, y03

  1363. 	FMADD	xsum4, xtemp4, a7,  xsum4

  1364. 	FMADD	y04, atemp4, a7, y04

  1365. 

  1366. 	FMADD1	xsum1, xtemp4, a4,  xsum1

  1367. 	NOP1

  1368. 	FNMSUB	y01, atemp4, a6, y01

  1369. 	NOP2

  1370. 

  1371. 	FMADD2	xsum2, xtemp3, a4,  xsum2

  1372. 	NOP1

  1373. 	FMADD	y02, atemp3, a6, y02

  1374. 	LFD	a6,  5 * SIZE(AO2)

  1375. 

  1376. 	FMADD1	xsum3, xtemp4, a8,  xsum3

  1377. 	addi	AO1, AO1, 4 * SIZE

  1378. 	FNMSUB	y03, atemp4, a8, y03

  1379. 	addi	AO2, AO2, 4 * SIZE

  1380. 	FMADD2	xsum4, xtemp3, a8,  xsum4

  1381. 	addi	YY,  YY, 4 * SIZE

  1382. 	FMADD	y04, atemp3, a8, y04

  1383. 	NOP2

  1384. 

  1385. 	STFD	y01, -4 * SIZE(YY)

  1386. 	LFD	y01,  0 * SIZE(YY)

  1387. 	STFD	y02, -3 * SIZE(YY)

  1388. 	LFD	y02,  1 * SIZE(YY)

  1389. 

  1390. 	STFD	y03, -2 * SIZE(YY)

  1391. 	STFD	y04, -1 * SIZE(YY)

  1392. 	.align 4

  1393. 

  1394. LL(17):

  1395. 	andi.	r0,  M, 1

  1396. 	ble	LL(18)

  1397. 

  1398. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1399. 	FMADD	y01, atemp1, a1, y01

  1400. 	FMADD	xsum2, xtemp2, a1,  xsum2

  1401. 	FMADD	y02, atemp2, a1, y02

  1402. 	FMADD	xsum3, xtemp1, a5,  xsum3

  1403. 	FNMSUB	y01, atemp2, a2, y01

  1404. 	FMADD	xsum4, xtemp2, a5,  xsum4

  1405. 	FMADD	y02, atemp1, a2, y02

  1406. 

  1407. 	FMADD1	xsum1, xtemp2, a2,  xsum1

  1408. 	FMADD	y01, atemp3, a5, y01

  1409. 	FMADD2	xsum2, xtemp1, a2,  xsum2

  1410. 	FMADD	y02, atemp4, a5, y02

  1411. 	FMADD1	xsum3, xtemp2, a6,  xsum3

  1412. 	FNMSUB	y01, atemp4, a6, y01

  1413. 	FMADD2	xsum4, xtemp1, a6,  xsum4

  1414. 	FMADD	y02, atemp3, a6, y02

  1415. 

  1416. 	STFD	y01,  0 * SIZE(YY)

  1417. 	STFD	y02,  1 * SIZE(YY)

  1418. 	STFD	y03,  2 * SIZE(YY)

  1419. 	STFD	y04,  3 * SIZE(YY)

  1420. 	.align 4

  1421. 

  1422. LL(18):

  1423. 	LFD	y05, ALPHA_R

  1424. 	LFD	y06, ALPHA_I

  1425. 

  1426. 	slwi	TEMP,  IS,  ZBASE_SHIFT

  1427. 	add	YY, NEW_Y, TEMP

  1428. 

  1429. 	LFD	y01,  0 * SIZE(YY)

  1430. 	LFD	y02,  1 * SIZE(YY)

  1431. 	LFD	y03,  2 * SIZE(YY)

  1432. 	LFD	y04,  3 * SIZE(YY)

  1433. 

  1434. 	FMUL	xtemp1, y05, xsum1

  1435. 	FMUL	xtemp2, y06, xsum1

  1436. 	FMUL	xtemp3, y05, xsum3

  1437. 	FMUL	xtemp4, y06, xsum3

  1438. 

  1439. 	FNMSUB	xsum1, y06, xsum2, xtemp1

  1440. 	FMADD	xsum2, y05, xsum2, xtemp2

  1441. 	FNMSUB	xsum3, y06, xsum4, xtemp3

  1442. 	FMADD	xsum4, y05, xsum4, xtemp4

  1443. 

  1444. 	FADD	y01, y01, xsum1

  1445. 	FADD	y02, y02, xsum2

  1446. 	FADD	y03, y03, xsum3

  1447. 	FADD	y04, y04, xsum4

  1448. 

  1449. 	STFD	y01,  0 * SIZE(YY)

  1450. 	addi	TEMP, IS, 4

  1451. 	STFD	y02,  1 * SIZE(YY)

  1452. 	addi	IS,   IS, 2

  1453. 	STFD	y03,  2 * SIZE(YY)

  1454. 	cmpw	cr0, TEMP, N

  1455. 	STFD	y04,  3 * SIZE(YY)

  1456. 	ble	LL(11)

  1457. 	.align 4

  1458. 

  1459. LL(20):

  1460. 	andi.	TEMP, N, 1

  1461. 	ble	LL(990)

  1462. 

  1463. 	slwi	TEMP,  IS,  ZBASE_SHIFT

  1464. 	add	XX, X,     TEMP

  1465. 	add	YY, NEW_Y, TEMP

  1466. 

  1467. 	LFD	y05, ALPHA_R

  1468. 	LFD	y06, ALPHA_I

  1469. 

  1470. 	LFD	atemp1, 0 * SIZE(XX)

  1471. 	LFD	atemp2, 1 * SIZE(XX)

  1472. 

  1473. 	LFD	a1,  0 * SIZE(A)

  1474. 	LFD	a2,  1 * SIZE(A)

  1475. 

  1476. 	FMUL	xsum1, atemp1, a1

  1477. 	FMUL	xsum2, atemp2, a1

  1478. 

  1479. #ifndef HEMV

  1480. 	FNMSUB	xsum1, atemp2, a2, xsum1

  1481. 	FMADD	xsum2, atemp1, a2, xsum2

  1482. #endif

  1483. 

  1484. 	FMUL	xtemp1, y05, atemp1

  1485. 	FMUL	xtemp2, y06, atemp1

  1486. 

  1487. 	FNMSUB	atemp1, y06, atemp2, xtemp1

  1488. 	FMADD	atemp2, y05, atemp2, xtemp2

  1489. 

  1490. 	LFD	y05, ALPHA_R

  1491. 	LFD	y06, ALPHA_I

  1492. 

  1493. 	LFD	y01,  0 * SIZE(YY)

  1494. 	LFD	y02,  1 * SIZE(YY)

  1495. 

  1496. 	FMUL	xtemp1, y05, xsum1

  1497. 	FMUL	xtemp2, y06, xsum1

  1498. 

  1499. 	FNMSUB	xsum1, y06, xsum2, xtemp1

  1500. 	FMADD	xsum2, y05, xsum2, xtemp2

  1501. 

  1502. 	FADD	y01, y01, xsum1

  1503. 	FADD	y02, y02, xsum2

  1504. 

  1505. 	STFD	y01,  0 * SIZE(YY)

  1506. 	STFD	y02,  1 * SIZE(YY)

  1507. 	.align 4

  1508. 

  1509. LL(990):

  1510. 	cmpwi	cr0, INCY, 2 * SIZE

  1511. 	beq	LL(999)

  1512. 

  1513. 	mr	YY, Y

  1514. 

  1515. 	srawi.	r0, M, 2

  1516. 	mtspr	CTR, r0

  1517. 	ble	LL(995)

  1518. 	.align 4

  1519. 

  1520. LL(991):

  1521. 	LFD	f0,  0 * SIZE(Y)

  1522. 	LFD	f1,  1 * SIZE(Y)

  1523. 	add	Y, Y, INCY

  1524. 	LFD	f2,  0 * SIZE(Y)

  1525. 	LFD	f3,  1 * SIZE(Y)

  1526. 	add	Y, Y, INCY

  1527. 	LFD	f4,  0 * SIZE(Y)

  1528. 	LFD	f5,  1 * SIZE(Y)

  1529. 	add	Y, Y, INCY

  1530. 	LFD	f6,  0 * SIZE(Y)

  1531. 	LFD	f7,  1 * SIZE(Y)

  1532. 	add	Y, Y, INCY

  1533. 

  1534. 	LFD	f8,   0 * SIZE(NEW_Y)

  1535. 	LFD	f9,   1 * SIZE(NEW_Y)

  1536. 	LFD	f10,  2 * SIZE(NEW_Y)

  1537. 	LFD	f11,  3 * SIZE(NEW_Y)

  1538. 	LFD	f12,  4 * SIZE(NEW_Y)

  1539. 	LFD	f13,  5 * SIZE(NEW_Y)

  1540. 	LFD	f14,  6 * SIZE(NEW_Y)

  1541. 	LFD	f15,  7 * SIZE(NEW_Y)

  1542. 	addi	NEW_Y, NEW_Y, 8 * SIZE

  1543. 

  1544. 	FADD	f8,  f8,  f0

  1545. 	FADD	f9,  f9,  f1

  1546. 	FADD	f10, f10, f2

  1547. 	FADD	f11, f11, f3

  1548. 	FADD	f12, f12, f4

  1549. 	FADD	f13, f13, f5

  1550. 	FADD	f14, f14, f6

  1551. 	FADD	f15, f15, f7

  1552. 

  1553. 	STFD	f8,  0 * SIZE(YY)

  1554. 	STFD	f9,  1 * SIZE(YY)

  1555. 	add	YY, YY, INCY

  1556. 	STFD	f10, 0 * SIZE(YY)

  1557. 	STFD	f11, 1 * SIZE(YY)

  1558. 	add	YY, YY, INCY

  1559. 	STFD	f12, 0 * SIZE(YY)

  1560. 	STFD	f13, 1 * SIZE(YY)

  1561. 	add	YY, YY, INCY

  1562. 	STFD	f14, 0 * SIZE(YY)

  1563. 	STFD	f15, 1 * SIZE(YY)

  1564. 	add	YY, YY, INCY

  1565. 	bdnz	LL(991)

  1566. 	.align 4

  1567. 

  1568. LL(995):

  1569. 	andi.	J, M, 2

  1570. 	ble	LL(996)

  1571. 

  1572. 	LFD	f0,  0 * SIZE(Y)

  1573. 	LFD	f1,  1 * SIZE(Y)

  1574. 	add	Y, Y, INCY

  1575. 	LFD	f2,  0 * SIZE(Y)

  1576. 	LFD	f3,  1 * SIZE(Y)

  1577. 	add	Y, Y, INCY

  1578. 

  1579. 	LFD	f8,   0 * SIZE(NEW_Y)

  1580. 	LFD	f9,   1 * SIZE(NEW_Y)

  1581. 	LFD	f10,  2 * SIZE(NEW_Y)

  1582. 	LFD	f11,  3 * SIZE(NEW_Y)

  1583. 	addi	NEW_Y, NEW_Y, 4 * SIZE

  1584. 

  1585. 	FADD	f8,  f8,  f0

  1586. 	FADD	f9,  f9,  f1

  1587. 	FADD	f10, f10, f2

  1588. 	FADD	f11, f11, f3

  1589. 

  1590. 	STFD	f8,  0 * SIZE(YY)

  1591. 	STFD	f9,  1 * SIZE(YY)

  1592. 	add	YY, YY, INCY

  1593. 	STFD	f10, 0 * SIZE(YY)

  1594. 	STFD	f11, 1 * SIZE(YY)

  1595. 	add	YY, YY, INCY

  1596. 	.align 4

  1597. 

  1598. LL(996):

  1599. 	andi.	J, M, 1

  1600. 	ble	LL(999)

  1601. 

  1602. 	LFD	f0,  0 * SIZE(Y)

  1603. 	LFD	f1,  1 * SIZE(Y)

  1604. 

  1605. 	LFD	f8,   0 * SIZE(NEW_Y)

  1606. 	LFD	f9,   1 * SIZE(NEW_Y)

  1607. 

  1608. 	FADD	f8,  f8,  f0

  1609. 	FADD	f9,  f9,  f1

  1610. 

  1611. 	STFD	f8,  0 * SIZE(YY)

  1612. 	STFD	f9,  1 * SIZE(YY)

  1613. 	.align 4

  1614. 

  1615. LL(999):

  1616. 	li	r3, 0

  1617. 

  1618. 	lfd	f14,     0(SP)

  1619. 	lfd	f15,     8(SP)

  1620. 	lfd	f16,    16(SP)

  1621. 	lfd	f17,    24(SP)

  1622. 	lfd	f18,    32(SP)

  1623. 	lfd	f19,    40(SP)

  1624. 	lfd	f20,    48(SP)

  1625. 	lfd	f21,    56(SP)

  1626. 	lfd	f22,    64(SP)

  1627. 	lfd	f23,    72(SP)

  1628. 	lfd	f24,    80(SP)

  1629. 	lfd	f25,    88(SP)

  1630. 	lfd	f26,    96(SP)

  1631. 	lfd	f27,   104(SP)

  1632. 	lfd	f28,   112(SP)

  1633. 	lfd	f29,   120(SP)

  1634. 	lfd	f30,   128(SP)

  1635. 	lfd	f31,   136(SP)

  1636. 

  1637. #ifdef __64BIT__

  1638. 	ld	r14,   144(SP)

  1639. 	ld	r15,   152(SP)

  1640. 	ld	r16,   160(SP)

  1641. 	ld	r17,   168(SP)

  1642. 	ld	r18,   176(SP)

  1643. 	ld	r19,   184(SP)

  1644. 	ld	r20,   192(SP)

  1645. 	ld	r21,   200(SP)

  1646. 	ld	r22,   208(SP)

  1647. 	ld	r23,   216(SP)

  1648. 	ld	r24,   224(SP)

  1649. 	ld	r25,   232(SP)

  1650. 	ld	r26,   240(SP)

  1651. 	ld	r27,   248(SP)

  1652. #else

  1653. 	lwz	r14,   144(SP)

  1654. 	lwz	r15,   148(SP)

  1655. 	lwz	r16,   152(SP)

  1656. 	lwz	r17,   156(SP)

  1657. 	lwz	r18,   160(SP)

  1658. 	lwz	r19,   164(SP)

  1659. 	lwz	r20,   168(SP)

  1660. 	lwz	r21,   172(SP)

  1661. 	lwz	r22,   176(SP)

  1662. 	lwz	r23,   180(SP)

  1663. 	lwz	r24,   184(SP)

  1664. 	lwz	r25,   188(SP)

  1665. 	lwz	r26,   192(SP)

  1666. 	lwz	r27,   196(SP)

  1667. #endif

  1668. 

  1669. 	addi	SP, SP, STACKSIZE

  1670. 	blr

  1671. 

  1672. 	EPILOGUE

  1673. #endif