OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/symv_L.S

1522 lines
30 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(linux) || defined(__FreeBSD__)

  43. #ifndef __64BIT__

  44. #define M	r3

  45. #define N	r4

  46. #define A	r5

  47. #define LDA	r6

  48. #define X	r7

  49. #define	INCX	r8

  50. #define	Y	r9

  51. #define	INCY	r10

  52. #define BUFFER	r14

  53. #else

  54. #define M	r3

  55. #define N	r4

  56. #define A	r6

  57. #define LDA	r7

  58. #define X	r8

  59. #define	INCX	r9

  60. #define	Y	r10

  61. #define	INCY	r5

  62. #define BUFFER	r14

  63. #endif

  64. #endif

  65. 

  66. #if defined(_AIX) || defined(__APPLE__)

  67. #if !defined(__64BIT__) && defined(DOUBLE)

  68. #define M	r3

  69. #define N	r4

  70. #define A	r7

  71. #define LDA	r8

  72. #define X	r9

  73. #define	INCX	r10

  74. #define	Y	r5

  75. #define	INCY	r6

  76. #define BUFFER	r14

  77. #else

  78. #define M	r3

  79. #define N	r4

  80. #define A	r6

  81. #define LDA	r7

  82. #define X	r8

  83. #define	INCX	r9

  84. #define	Y	r10

  85. #define	INCY	r5

  86. #define BUFFER	r14

  87. #endif

  88. #endif

  89. 

  90. #define I	r11

  91. #define	J	r12

  92. 

  93. #define AO1	r15

  94. #define AO2	r16

  95. #define AO3	r17

  96. #define AO4	r18

  97. #define XX	r19

  98. #define YY	r20

  99. #define	NEW_Y	r21

  100. #define TEMP	r22

  101. #define	PREA	r24

  102. #define IS	r25

  103. 

  104. #define y01 f0

  105. #define y02 f1

  106. #define y03 f2

  107. #define y04 f3

  108. 

  109. #define atemp1 f4

  110. #define atemp2 f5

  111. #define atemp3 f6

  112. #define atemp4 f7

  113. 

  114. #define xtemp1 f8

  115. #define xtemp2 f9

  116. #define xtemp3 f10

  117. #define xtemp4 f11

  118. 

  119. #define xsum1  f12

  120. #define xsum2  f13

  121. #define xsum3  f14

  122. #define xsum4  f15

  123. 

  124. #define a1     f16

  125. #define a2     f17

  126. #define a3     f18

  127. #define a4     f19

  128. #define a5     f20

  129. #define a6     f21

  130. #define a7     f22

  131. #define a8     f23

  132. #define a9     f24

  133. #define a10    f25

  134. #define a11    f26

  135. #define a12    f27

  136. #define a13    f28

  137. #define a14    f29

  138. #define a15    f30

  139. #define a16    f31

  140. 

  141. #define alpha  f1

  142. 

  143. #if defined(PPCG4)

  144. #define PREFETCHSIZE_A  24

  145. #endif

  146. 

  147. #if defined(PPC440) || defined(PPC440FP2)

  148. #define PREFETCHSIZE_A  24

  149. #endif

  150. 

  151. #ifdef PPC970

  152. #define PREFETCHSIZE_A  64

  153. #endif

  154. 

  155. #ifdef CELL

  156. #define PREFETCHSIZE_A  72

  157. #endif

  158. 

  159. #ifdef POWER4

  160. #define PREFETCHSIZE_A  16

  161. #endif

  162. 

  163. #ifdef POWER5

  164. #define PREFETCHSIZE_A  96

  165. #endif

  166. 

  167. #ifdef POWER6

  168. #define PREFETCHSIZE_A  40

  169. #endif

  170. 

  171. #if defined(POWER4) || defined(POWER5) || defined(POWER6) || defined(PPC970)

  172. #define NOP1

  173. #define NOP2

  174. #else

  175. #define NOP1   mr	LDA, LDA

  176. #define NOP2   mr	INCX, INCX

  177. #endif

  178. 

  179. #ifndef NEEDPARAM

  180. 

  181. #ifndef __64BIT__

  182. #define STACKSIZE 224

  183. #define ALPHA     200(SP)

  184. #define	FZERO	  208(SP)

  185. #else

  186. #define STACKSIZE 280

  187. #define ALPHA     256(SP)

  188. #define FZERO	  264(SP)

  189. #endif

  190. 

  191. 	PROLOGUE

  192. 	PROFCODE

  193. 

  194. 	addi	SP,   SP, -STACKSIZE

  195. 	li	r0,   0

  196. 

  197. 	stfd	f14,     0(SP)

  198. 	stfd	f15,     8(SP)

  199. 	stfd	f16,    16(SP)

  200. 	stfd	f17,    24(SP)

  201. 	stfd	f18,    32(SP)

  202. 	stfd	f19,    40(SP)

  203. 	stfd	f20,    48(SP)

  204. 	stfd	f21,    56(SP)

  205. 	stfd	f22,    64(SP)

  206. 	stfd	f23,    72(SP)

  207. 	stfd	f24,    80(SP)

  208. 	stfd	f25,    88(SP)

  209. 	stfd	f26,    96(SP)

  210. 	stfd	f27,   104(SP)

  211. 	stfd	f28,   112(SP)

  212. 	stfd	f29,   120(SP)

  213. 	stfd	f30,   128(SP)

  214. 	stfd	f31,   136(SP)

  215. 

  216. #ifdef __64BIT__

  217. 	std	r0,    FZERO

  218. 	std	r14,   144(SP)

  219. 	std	r15,   152(SP)

  220. 	std	r16,   160(SP)

  221. 	std	r17,   168(SP)

  222. 	std	r18,   176(SP)

  223. 	std	r19,   184(SP)

  224. 	std	r20,   192(SP)

  225. 	std	r21,   200(SP)

  226. 	std	r22,   208(SP)

  227. 	std	r23,   216(SP)

  228. 	std	r24,   224(SP)

  229. 	std	r25,   232(SP)

  230. 	std	r26,   240(SP)

  231. 	std	r27,   248(SP)

  232. #else

  233. 	stw	r0,    0 + FZERO

  234. 	stw	r0,    4 + FZERO

  235. 	stw	r14,   144(SP)

  236. 	stw	r15,   148(SP)

  237. 	stw	r16,   152(SP)

  238. 	stw	r17,   156(SP)

  239. 	stw	r18,   160(SP)

  240. 	stw	r19,   164(SP)

  241. 	stw	r20,   168(SP)

  242. 	stw	r21,   172(SP)

  243. 	stw	r22,   176(SP)

  244. 	stw	r23,   180(SP)

  245. 	stw	r24,   184(SP)

  246. 	stw	r25,   188(SP)

  247. 	stw	r26,   192(SP)

  248. 	stw	r27,   196(SP)

  249. #endif

  250. 

  251. #if defined(linux) || defined(__FreeBSD__)

  252. #ifndef __64BIT__

  253. 	lwz	BUFFER,  FRAMESLOT(0) + STACKSIZE(SP)

  254. #else

  255. 	ld	INCY,    FRAMESLOT(0) + STACKSIZE(SP)

  256. 	ld	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  257. #endif

  258. #endif

  259. 

  260. #if defined(_AIX) || defined(__APPLE__)

  261. #ifndef __64BIT__

  262. #ifdef DOUBLE

  263. 	lwz	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  264. 	lwz	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  265. 	lwz	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  266. #else

  267. 	lwz	  INCY,  FRAMESLOT(0) + STACKSIZE(SP)

  268. 	lwz	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  269. #endif

  270. #else

  271. 	ld	INCY,    FRAMESLOT(0) + STACKSIZE(SP)

  272. 	ld	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  273. #endif

  274. #endif

  275. 

  276. 	STFD	alpha, ALPHA

  277. 

  278. 	slwi	LDA,  LDA,  BASE_SHIFT

  279. 	slwi	INCX, INCX, BASE_SHIFT

  280. 	slwi	INCY, INCY, BASE_SHIFT

  281. 

  282. 	li	PREA, PREFETCHSIZE_A * SIZE

  283. 

  284. 	cmpwi	cr0, M, 0

  285. 	ble-	LL(999)

  286. 

  287. 	cmpwi	cr0, INCX, SIZE

  288. 	beq	LL(05)

  289. 

  290. 	mr	XX, X

  291. 	mr	X, BUFFER

  292. 

  293. 	srawi.	r0, M, 3

  294. 	mtspr	CTR, r0

  295. 	ble	LL(03)

  296. 	.align 4

  297. 

  298. LL(01):

  299. 	LFD	a1, 0 * SIZE(XX)

  300. 	add	XX, XX, INCX

  301. 	LFD	a2, 0 * SIZE(XX)

  302. 	add	XX, XX, INCX

  303. 	LFD	a3, 0 * SIZE(XX)

  304. 	add	XX, XX, INCX

  305. 	LFD	a4, 0 * SIZE(XX)

  306. 	add	XX, XX, INCX

  307. 	LFD	a5, 0 * SIZE(XX)

  308. 	add	XX, XX, INCX

  309. 	LFD	a6, 0 * SIZE(XX)

  310. 	add	XX, XX, INCX

  311. 	LFD	a7, 0 * SIZE(XX)

  312. 	add	XX, XX, INCX

  313. 	LFD	a8, 0 * SIZE(XX)

  314. 	add	XX, XX, INCX

  315. 

  316. 	dcbt	XX, PREA

  317. 	dcbtst	BUFFER, PREA

  318. 

  319. 	STFD	a1, 0 * SIZE(BUFFER)

  320. 	STFD	a2, 1 * SIZE(BUFFER)

  321. 	STFD	a3, 2 * SIZE(BUFFER)

  322. 	STFD	a4, 3 * SIZE(BUFFER)

  323. 	STFD	a5, 4 * SIZE(BUFFER)

  324. 	STFD	a6, 5 * SIZE(BUFFER)

  325. 	STFD	a7, 6 * SIZE(BUFFER)

  326. 	STFD	a8, 7 * SIZE(BUFFER)

  327. 

  328. 	addi	BUFFER, BUFFER, 8 * SIZE

  329. 	bdnz	LL(01)

  330. 	.align 4

  331. 

  332. LL(03):

  333. 	andi.	r0, M, 7

  334. 	mtspr	CTR, r0

  335. 	ble	LL(05)

  336. 	.align 4

  337. 

  338. LL(04):

  339. 	LFD	a1, 0 * SIZE(XX)

  340. 	add	XX, XX, INCX

  341. 

  342. 	STFD	a1, 0 * SIZE(BUFFER)

  343. 	addi	BUFFER, BUFFER, 1 * SIZE

  344. 	bdnz	LL(04)

  345. 	.align 4

  346. 

  347. LL(05):

  348. 	mr	NEW_Y, Y

  349. 	lfd	f0, FZERO

  350. 

  351. 	cmpwi	cr0, INCY, SIZE

  352. 	beq	LL(10)

  353. 

  354. 	mr	NEW_Y, BUFFER

  355. 

  356. 	addi	r0, M,  7

  357. 	srawi.	r0, r0, 3

  358. 	mtspr	CTR, r0

  359. 	.align 4

  360. 

  361. LL(06):

  362. 	STFD	f0, 0 * SIZE(BUFFER)

  363. 	STFD	f0, 1 * SIZE(BUFFER)

  364. 	STFD	f0, 2 * SIZE(BUFFER)

  365. 	STFD	f0, 3 * SIZE(BUFFER)

  366. 	STFD	f0, 4 * SIZE(BUFFER)

  367. 	STFD	f0, 5 * SIZE(BUFFER)

  368. 	STFD	f0, 6 * SIZE(BUFFER)

  369. 	STFD	f0, 7 * SIZE(BUFFER)

  370. 	addi	BUFFER, BUFFER, 8 * SIZE

  371. 	bdnz	LL(06)

  372. 	.align 4

  373. 

  374. LL(10):

  375. 	li	IS, 0

  376. 

  377. 	cmpwi	cr0, N, 4

  378. 	blt	LL(20)

  379. 	.align 4

  380. 

  381. LL(11):

  382. 	mr	AO1, A

  383. 	add	AO2, A,   LDA

  384. 	add	AO3, AO2, LDA

  385. 	add	AO4, AO3, LDA

  386. 	add	A,   AO4, LDA

  387. 	addi	A, A, 4 * SIZE

  388. 

  389. 	slwi	TEMP,  IS,  BASE_SHIFT

  390. 	add	XX, X,     TEMP

  391. 	add	YY, NEW_Y, TEMP

  392. 

  393. 	LFD	atemp1, 0 * SIZE(XX)

  394. 	LFD	atemp2, 1 * SIZE(XX)

  395. 	LFD	atemp3, 2 * SIZE(XX)

  396. 	LFD	atemp4, 3 * SIZE(XX)

  397. 

  398. 	LFD	a1,  0 * SIZE(AO1)

  399. 	LFD	a2,  1 * SIZE(AO1)

  400. 	LFD	a3,  2 * SIZE(AO1)

  401. 	LFD	a4,  3 * SIZE(AO1)

  402. 

  403. 	LFD	a6,  1 * SIZE(AO2)

  404. 	LFD	a7,  2 * SIZE(AO2)

  405. 	LFD	a8,  3 * SIZE(AO2)

  406. 

  407. 	LFD	a11, 2 * SIZE(AO3)

  408. 	LFD	a12, 3 * SIZE(AO3)

  409. 

  410. 	LFD	a16, 3 * SIZE(AO4)

  411. 

  412. 	LFD	a5, ALPHA

  413. 

  414. 	FMUL	xsum1, atemp1, a1

  415. 	FMUL	xsum2, atemp1, a2

  416. 	FMUL	xsum3, atemp1, a3

  417. 	FMUL	xsum4, atemp1, a4

  418. 

  419. 	FMADD	xsum1, atemp2, a2,  xsum1

  420. 	FMADD	xsum2, atemp2, a6,  xsum2

  421. 	FMADD	xsum3, atemp2, a7,  xsum3

  422. 	FMADD	xsum4, atemp2, a8,  xsum4

  423. 

  424. 	FMADD	xsum1, atemp3, a3,  xsum1

  425. 	FMADD	xsum2, atemp3, a7,  xsum2

  426. 	FMADD	xsum3, atemp3, a11, xsum3

  427. 	FMADD	xsum4, atemp3, a12, xsum4

  428. 

  429. 	FMADD	xsum1, atemp4, a4,  xsum1

  430. 	FMADD	xsum2, atemp4, a8,  xsum2

  431. 	FMADD	xsum3, atemp4, a12, xsum3

  432. 	FMADD	xsum4, atemp4, a16, xsum4

  433. 

  434. 	FMUL	atemp1, a5, atemp1

  435. 	FMUL	atemp2, a5, atemp2

  436. 	FMUL	atemp3, a5, atemp3

  437. 	FMUL	atemp4, a5, atemp4

  438. 

  439. 	LFD	xtemp1,  4 * SIZE(XX)

  440. 	LFD	xtemp2,  5 * SIZE(XX)

  441. 	LFD	xtemp3,  6 * SIZE(XX)

  442. 	LFD	xtemp4,  7 * SIZE(XX)

  443. 

  444. 	LFD	y01,  4 * SIZE(YY)

  445. 	LFD	y02,  5 * SIZE(YY)

  446. 	LFD	y03,  6 * SIZE(YY)

  447. 	LFD	y04,  7 * SIZE(YY)

  448. 

  449. 	LFD	a1,  4 * SIZE(AO1)

  450. 	LFD	a2,  5 * SIZE(AO1)

  451. 	LFD	a3,  6 * SIZE(AO1)

  452. 	LFD	a4,  7 * SIZE(AO1)

  453. 

  454. 	LFD	a5,  4 * SIZE(AO2)

  455. 	LFD	a6,  5 * SIZE(AO2)

  456. 	LFD	a7,  6 * SIZE(AO2)

  457. 	LFD	a8,  7 * SIZE(AO2)

  458. 

  459. 	LFD	a9,  4 * SIZE(AO3)

  460. 	LFD	a10, 5 * SIZE(AO3)

  461. 	LFD	a11, 6 * SIZE(AO3)

  462. 	LFD	a12, 7 * SIZE(AO3)

  463. 

  464. 	LFD	a13, 4 * SIZE(AO4)

  465. 	LFD	a14, 5 * SIZE(AO4)

  466. 	LFD	a15, 6 * SIZE(AO4)

  467. 	LFD	a16, 7 * SIZE(AO4)

  468. 

  469. 	addi	AO1, AO1, 4 * SIZE

  470. 	addi	AO2, AO2, 4 * SIZE

  471. 	addi	AO3, AO3, 4 * SIZE

  472. 	addi	AO4, AO4, 4 * SIZE

  473. 

  474. 	addi	XX, XX, 4 * SIZE

  475. 	addi	YY, YY, 4 * SIZE

  476. 

  477. 	sub	TEMP, M, IS

  478. 	addi	TEMP, TEMP, -4

  479. 	srawi.	r0,  TEMP, 4

  480. 	mtspr	CTR, r0

  481. 	ble	LL(14)

  482. 	.align 4

  483. 

  484. LL(12):

  485. 	FMADD	xsum1, xtemp1, a1,  xsum1

  486. 	DCBT(AO1, PREA)

  487. 	FMADD	y01, atemp1, a1,  y01

  488. 	LFD	a1,  4 * SIZE(AO1)

  489. 

  490. 	FMADD	xsum2, xtemp1, a5,  xsum2

  491. 	NOP1

  492. 	FMADD	y02, atemp1, a2,  y02

  493. 	NOP2

  494. 

  495. 	FMADD	xsum3, xtemp1, a9,  xsum3

  496. 	NOP1

  497. 	FMADD	y03, atemp1, a3,  y03

  498. 	NOP2

  499. 

  500. 	FMADD	xsum4, xtemp1, a13, xsum4

  501. 	LFD	xtemp1,  4 * SIZE(XX)

  502. 	FMADD	y04, atemp1, a4,  y04

  503. 	NOP2

  504. 

  505. 	FMADD	xsum1, xtemp2, a2,  xsum1

  506. 	LFD	a2,  5 * SIZE(AO1)

  507. 	FMADD	y01, atemp2, a5,  y01

  508. 	LFD	a5,  4 * SIZE(AO2)

  509. 

  510. 	FMADD	xsum2, xtemp2, a6,  xsum2

  511. 	NOP1

  512. 	FMADD	y02, atemp2, a6,  y02

  513. 	LFD	a6,  5 * SIZE(AO2)

  514. 

  515. 	FMADD	xsum3, xtemp2, a10, xsum3

  516. 	NOP1

  517. 	FMADD	y03, atemp2, a7,  y03

  518. 	NOP2

  519. 

  520. 	FMADD	xsum4, xtemp2, a14, xsum4

  521. 	LFD	xtemp2,  5 * SIZE(XX)

  522. 	FMADD	y04, atemp2, a8,  y04

  523. #	DCBT(X, PREX)

  524. 	NOP2

  525. 

  526. 	FMADD	xsum1, xtemp3, a3,  xsum1

  527. 	LFD	a3,  6 * SIZE(AO1)

  528. 	FMADD	y01, atemp3, a9,  y01

  529. 	LFD	a9,  4 * SIZE(AO3)

  530. 

  531. 	FMADD	xsum2, xtemp3, a7,  xsum2

  532. 	LFD	a7,  6 * SIZE(AO2)

  533. 	FMADD	y02, atemp3, a10, y02

  534. 	LFD	a10, 5 * SIZE(AO3)

  535. 

  536. 	FMADD	xsum3, xtemp3, a11, xsum3

  537. 	NOP1

  538. 	FMADD	y03, atemp3, a11, y03

  539. 	LFD	a11, 6 * SIZE(AO3)

  540. 

  541. 	FMADD	xsum4, xtemp3, a15, xsum4

  542. 	LFD	xtemp3,  6 * SIZE(XX)

  543. 	FMADD	y04, atemp3, a12, y04

  544. 	NOP2

  545. 

  546. 	FMADD	xsum1, xtemp4, a4,  xsum1

  547. 	LFD	a4,  7 * SIZE(AO1)

  548. 	FMADD	y01, atemp4, a13, y01

  549. 	LFD	a13, 4 * SIZE(AO4)

  550. 

  551. 	FMADD	xsum2, xtemp4, a8,  xsum2

  552. 	LFD	a8,  7 * SIZE(AO2)

  553. 	FMADD	y02, atemp4, a14, y02

  554. 	LFD	a14, 5 * SIZE(AO4)

  555. 

  556. 	FMADD	xsum3, xtemp4, a12, xsum3

  557. 	LFD	a12, 7 * SIZE(AO3)

  558. 	FMADD	y03, atemp4, a15, y03

  559. 	LFD	a15, 6 * SIZE(AO4)

  560. 

  561. 	FMADD	xsum4, xtemp4, a16, xsum4

  562. 	LFD	xtemp4,  7 * SIZE(XX)

  563. 	FMADD	y04, atemp4, a16, y04

  564. 	LFD	a16, 7 * SIZE(AO4)

  565. 

  566. 	STFD	y01,  0 * SIZE(YY)

  567. 	LFD	y01,  4 * SIZE(YY)

  568. 	STFD	y02,  1 * SIZE(YY)

  569. 	LFD	y02,  5 * SIZE(YY)

  570. 

  571. 	STFD	y03,  2 * SIZE(YY)

  572. 	LFD	y03,  6 * SIZE(YY)

  573. 	STFD	y04,  3 * SIZE(YY)

  574. 	LFD	y04,  7 * SIZE(YY)

  575. 

  576. 	FMADD	xsum1, xtemp1, a1,  xsum1

  577. 	DCBT(AO2, PREA)

  578. 	FMADD	y01, atemp1, a1,  y01

  579. 	LFD	a1,  8 * SIZE(AO1)

  580. 

  581. 	FMADD	xsum2, xtemp1, a5,  xsum2

  582. 	NOP1

  583. 	FMADD	y02, atemp1, a2,  y02

  584. 	NOP2

  585. 

  586. 	FMADD	xsum3, xtemp1, a9,  xsum3

  587. 	NOP1

  588. 	FMADD	y03, atemp1, a3,  y03

  589. 	NOP2

  590. 

  591. 	FMADD	xsum4, xtemp1, a13, xsum4

  592. 	LFD	xtemp1,  8 * SIZE(XX)

  593. 	FMADD	y04, atemp1, a4,  y04

  594. 	NOP2

  595. 

  596. 	FMADD	xsum1, xtemp2, a2,  xsum1

  597. 	LFD	a2,  9 * SIZE(AO1)

  598. 	FMADD	y01, atemp2, a5,  y01

  599. 	LFD	a5,  8 * SIZE(AO2)

  600. 

  601. 	FMADD	xsum2, xtemp2, a6,  xsum2

  602. 	NOP1

  603. 	FMADD	y02, atemp2, a6,  y02

  604. 	LFD	a6,  9 * SIZE(AO2)

  605. 

  606. 	FMADD	xsum3, xtemp2, a10, xsum3

  607. 	NOP1

  608. 	FMADD	y03, atemp2, a7,  y03

  609. 	NOP2

  610. 

  611. 	FMADD	xsum4, xtemp2, a14, xsum4

  612. 	LFD	xtemp2,  9 * SIZE(XX)

  613. 	FMADD	y04, atemp2, a8,  y04

  614. 	NOP2

  615. 

  616. 	FMADD	xsum1, xtemp3, a3,  xsum1

  617. 	LFD	a3, 10 * SIZE(AO1)

  618. 	FMADD	y01, atemp3, a9,  y01

  619. 	LFD	a9,  8 * SIZE(AO3)

  620. 

  621. 	FMADD	xsum2, xtemp3, a7,  xsum2

  622. 	LFD	a7, 10 * SIZE(AO2)

  623. 	FMADD	y02, atemp3, a10, y02

  624. 	LFD	a10, 9 * SIZE(AO3)

  625. 

  626. 	FMADD	xsum3, xtemp3, a11, xsum3

  627. 	NOP1

  628. 	FMADD	y03, atemp3, a11, y03

  629. 	LFD	a11, 10 * SIZE(AO3)

  630. 

  631. 	FMADD	xsum4, xtemp3, a15, xsum4

  632. 	LFD	xtemp3, 10 * SIZE(XX)

  633. 	FMADD	y04, atemp3, a12, y04

  634. 	NOP2

  635. 

  636. 	FMADD	xsum1, xtemp4, a4,  xsum1

  637. 	LFD	a4, 11 * SIZE(AO1)

  638. 	FMADD	y01, atemp4, a13, y01

  639. 	LFD	a13, 8 * SIZE(AO4)

  640. 

  641. 	FMADD	xsum2, xtemp4, a8,  xsum2

  642. 	LFD	a8, 11 * SIZE(AO2)

  643. 	FMADD	y02, atemp4, a14, y02

  644. 	LFD	a14, 9 * SIZE(AO4)

  645. 

  646. 	FMADD	xsum3, xtemp4, a12, xsum3

  647. 	LFD	a12, 11 * SIZE(AO3)

  648. 	FMADD	y03, atemp4, a15, y03

  649. 	LFD	a15, 10 * SIZE(AO4)

  650. 

  651. 	FMADD	xsum4, xtemp4, a16, xsum4

  652. 	LFD	xtemp4, 11 * SIZE(XX)

  653. 	FMADD	y04, atemp4, a16, y04

  654. 	LFD	a16, 11 * SIZE(AO4)

  655. 

  656. 	STFD	y01,  4 * SIZE(YY)

  657. 	LFD	y01,  8 * SIZE(YY)

  658. 	STFD	y02,  5 * SIZE(YY)

  659. 	LFD	y02,  9 * SIZE(YY)

  660. 

  661. 	STFD	y03,  6 * SIZE(YY)

  662. 	LFD	y03, 10 * SIZE(YY)

  663. 	STFD	y04,  7 * SIZE(YY)

  664. 	LFD	y04, 11 * SIZE(YY)

  665. 

  666. 

  667. 	FMADD	xsum1, xtemp1, a1,  xsum1

  668. 	DCBT(AO3, PREA)

  669. 	FMADD	y01, atemp1, a1,  y01

  670. 	LFD	a1, 12 * SIZE(AO1)

  671. 

  672. 	FMADD	xsum2, xtemp1, a5,  xsum2

  673. 	NOP1

  674. 	FMADD	y02, atemp1, a2,  y02

  675. 	NOP2

  676. 

  677. 	FMADD	xsum3, xtemp1, a9,  xsum3

  678. 	NOP1

  679. 	FMADD	y03, atemp1, a3,  y03

  680. 	NOP2

  681. 

  682. 	FMADD	xsum4, xtemp1, a13, xsum4

  683. 	LFD	xtemp1, 12 * SIZE(XX)

  684. 	FMADD	y04, atemp1, a4,  y04

  685. 	NOP2

  686. 

  687. 	FMADD	xsum1, xtemp2, a2,  xsum1

  688. 	LFD	a2, 13 * SIZE(AO1)

  689. 	FMADD	y01, atemp2, a5,  y01

  690. 	LFD	a5, 12 * SIZE(AO2)

  691. 

  692. 	FMADD	xsum2, xtemp2, a6,  xsum2

  693. 	NOP1

  694. 	FMADD	y02, atemp2, a6,  y02

  695. 	LFD	a6, 13 * SIZE(AO2)

  696. 

  697. 	FMADD	xsum3, xtemp2, a10, xsum3

  698. 	NOP1

  699. 	FMADD	y03, atemp2, a7,  y03

  700. #	DCBT(Y1, PREY)

  701. 	NOP2

  702. 

  703. 	FMADD	xsum4, xtemp2, a14, xsum4

  704. 	LFD	xtemp2, 13 * SIZE(XX)

  705. 	FMADD	y04, atemp2, a8,  y04

  706. 	NOP2

  707. 

  708. 	FMADD	xsum1, xtemp3, a3,  xsum1

  709. 	LFD	a3, 14 * SIZE(AO1)

  710. 	FMADD	y01, atemp3, a9,  y01

  711. 	LFD	a9, 12 * SIZE(AO3)

  712. 

  713. 	FMADD	xsum2, xtemp3, a7,  xsum2

  714. 	LFD	a7, 14 * SIZE(AO2)

  715. 	FMADD	y02, atemp3, a10, y02

  716. 	LFD	a10,13 * SIZE(AO3)

  717. 

  718. 	FMADD	xsum3, xtemp3, a11, xsum3

  719. 	NOP1

  720. 	FMADD	y03, atemp3, a11, y03

  721. 	LFD	a11, 14 * SIZE(AO3)

  722. 

  723. 	FMADD	xsum4, xtemp3, a15, xsum4

  724. 	LFD	xtemp3, 14 * SIZE(XX)

  725. 	FMADD	y04, atemp3, a12, y04

  726. 	NOP2

  727. 

  728. 	FMADD	xsum1, xtemp4, a4,  xsum1

  729. 	LFD	a4, 15 * SIZE(AO1)

  730. 	FMADD	y01, atemp4, a13, y01

  731. 	LFD	a13,12 * SIZE(AO4)

  732. 

  733. 	FMADD	xsum2, xtemp4, a8,  xsum2

  734. 	LFD	a8, 15 * SIZE(AO2)

  735. 	FMADD	y02, atemp4, a14, y02

  736. 	LFD	a14, 13 * SIZE(AO4)

  737. 

  738. 	FMADD	xsum3, xtemp4, a12, xsum3

  739. 	LFD	a12, 15 * SIZE(AO3)

  740. 	FMADD	y03, atemp4, a15, y03

  741. 	LFD	a15, 14 * SIZE(AO4)

  742. 

  743. 	FMADD	xsum4, xtemp4, a16, xsum4

  744. 	LFD	xtemp4,  15 * SIZE(XX)

  745. 	FMADD	y04, atemp4, a16, y04

  746. 	LFD	a16, 15 * SIZE(AO4)

  747. 

  748. 	STFD	y01,  8 * SIZE(YY)

  749. 	LFD	y01, 12 * SIZE(YY)

  750. 	STFD	y02,  9 * SIZE(YY)

  751. 	LFD	y02, 13 * SIZE(YY)

  752. 

  753. 	STFD	y03, 10 * SIZE(YY)

  754. 	LFD	y03, 14 * SIZE(YY)

  755. 	STFD	y04, 11 * SIZE(YY)

  756. 	LFD	y04, 15 * SIZE(YY)

  757. 

  758. 	FMADD	xsum1, xtemp1, a1,  xsum1

  759. 	DCBT(AO4, PREA)

  760. 	FMADD	y01, atemp1, a1,  y01

  761. 	LFD	a1, 16 * SIZE(AO1)

  762. 

  763. 	FMADD	xsum2, xtemp1, a5,  xsum2

  764. 	NOP1

  765. 	FMADD	y02, atemp1, a2,  y02

  766. 	NOP2

  767. 

  768. 	FMADD	xsum3, xtemp1, a9,  xsum3

  769. 	NOP1

  770. 	FMADD	y03, atemp1, a3,  y03

  771. 	NOP2

  772. 

  773. 	FMADD	xsum4, xtemp1, a13, xsum4

  774. 	LFD	xtemp1, 16 * SIZE(XX)

  775. 	FMADD	y04, atemp1, a4,  y04

  776. 	addi	YY, YY, 16 * SIZE

  777. 

  778. 	FMADD	xsum1, xtemp2, a2,  xsum1

  779. 	LFD	a2, 17 * SIZE(AO1)

  780. 	FMADD	y01, atemp2, a5,  y01

  781. 	LFD	a5, 16 * SIZE(AO2)

  782. 

  783. 	FMADD	xsum2, xtemp2, a6,  xsum2

  784. 	addi	AO3, AO3, 16 * SIZE

  785. 	FMADD	y02, atemp2, a6,  y02

  786. 	LFD	a6, 17 * SIZE(AO2)

  787. 

  788. 	FMADD	xsum3, xtemp2, a10, xsum3

  789. 	addi	AO1, AO1, 16 * SIZE

  790. 	FMADD	y03, atemp2, a7,  y03

  791. 	addi	AO2, AO2, 16 * SIZE

  792. 

  793. 	FMADD	xsum4, xtemp2, a14, xsum4

  794. 	LFD	xtemp2, 17 * SIZE(XX)

  795. 	FMADD	y04, atemp2, a8,  y04

  796. 	addi	AO4, AO4, 16 * SIZE

  797. 

  798. 	FMADD	xsum1, xtemp3, a3,  xsum1

  799. 	LFD	a3,  2 * SIZE(AO1)

  800. 	FMADD	y01, atemp3, a9,  y01

  801. 	LFD	a9,  0 * SIZE(AO3)

  802. 

  803. 	FMADD	xsum2, xtemp3, a7,  xsum2

  804. 	LFD	a7,  2 * SIZE(AO2)

  805. 	FMADD	y02, atemp3, a10, y02

  806. 	LFD	a10,  1 * SIZE(AO3)

  807. 

  808. 	FMADD	xsum3, xtemp3, a11, xsum3

  809. 	NOP1

  810. 	FMADD	y03, atemp3, a11, y03

  811. 	LFD	a11,  2 * SIZE(AO3)

  812. 

  813. 	FMADD	xsum4, xtemp3, a15, xsum4

  814. 	LFD	xtemp3, 18 * SIZE(XX)

  815. 	FMADD	y04, atemp3, a12, y04

  816. 	addi	XX, XX, 16 * SIZE

  817. 

  818. 	FMADD	xsum1, xtemp4, a4,  xsum1

  819. 	LFD	a4,  3 * SIZE(AO1)

  820. 	FMADD	y01, atemp4, a13, y01

  821. 	LFD	a13,  0 * SIZE(AO4)

  822. 

  823. 	FMADD	xsum2, xtemp4, a8,  xsum2

  824. 	LFD	a8,  3 * SIZE(AO2)

  825. 	FMADD	y02, atemp4, a14, y02

  826. 	LFD	a14,  1 * SIZE(AO4)

  827. 

  828. 	FMADD	xsum3, xtemp4, a12, xsum3

  829. 	LFD	a12,  3 * SIZE(AO3)

  830. 	FMADD	y03, atemp4, a15, y03

  831. 	LFD	a15,  2 * SIZE(AO4)

  832. 

  833. 	FMADD	xsum4, xtemp4, a16, xsum4

  834. 	LFD	xtemp4,  3 * SIZE(XX)

  835. 	FMADD	y04, atemp4, a16, y04

  836. 	LFD	a16,  3 * SIZE(AO4)

  837. 

  838. 	STFD	y01, -4 * SIZE(YY)

  839. 	LFD	y01,  0 * SIZE(YY)

  840. 	STFD	y02, -3 * SIZE(YY)

  841. 	LFD	y02,  1 * SIZE(YY)

  842. 

  843. 	STFD	y03, -2 * SIZE(YY)

  844. 	LFD	y03,  2 * SIZE(YY)

  845. 	STFD	y04, -1 * SIZE(YY)

  846. 	LFD	y04,  3 * SIZE(YY)

  847. 	bdnz	LL(12)

  848. 	.align 4

  849. 

  850. LL(14):

  851. 	sub	TEMP, M, IS

  852. 	addi	TEMP, TEMP, -4

  853. 	andi.	r0,  TEMP, 8

  854. 	ble	LL(15)

  855. 

  856. 	FMADD	xsum1, xtemp1, a1,  xsum1

  857. 	NOP1

  858. 	FMADD	y01, atemp1, a1,  y01

  859. 	LFD	a1,  4 * SIZE(AO1)

  860. 

  861. 	FMADD	xsum2, xtemp1, a5,  xsum2

  862. 	NOP1

  863. 	FMADD	y02, atemp1, a2,  y02

  864. 	NOP2

  865. 

  866. 	FMADD	xsum3, xtemp1, a9,  xsum3

  867. 	NOP1

  868. 	FMADD	y03, atemp1, a3,  y03

  869. 	NOP2

  870. 

  871. 	FMADD	xsum4, xtemp1, a13, xsum4

  872. 	LFD	xtemp1,  4 * SIZE(XX)

  873. 	FMADD	y04, atemp1, a4,  y04

  874. 	NOP2

  875. 

  876. 	FMADD	xsum1, xtemp2, a2,  xsum1

  877. 	LFD	a2,  5 * SIZE(AO1)

  878. 	FMADD	y01, atemp2, a5,  y01

  879. 	LFD	a5,  4 * SIZE(AO2)

  880. 

  881. 	FMADD	xsum2, xtemp2, a6,  xsum2

  882. 	NOP1

  883. 	FMADD	y02, atemp2, a6,  y02

  884. 	LFD	a6,  5 * SIZE(AO2)

  885. 

  886. 	FMADD	xsum3, xtemp2, a10, xsum3

  887. 	NOP1

  888. 	FMADD	y03, atemp2, a7,  y03

  889. 	NOP2

  890. 

  891. 	FMADD	xsum4, xtemp2, a14, xsum4

  892. 	LFD	xtemp2,  5 * SIZE(XX)

  893. 	FMADD	y04, atemp2, a8,  y04

  894. 	NOP2

  895. 

  896. 	FMADD	xsum1, xtemp3, a3,  xsum1

  897. 	LFD	a3,  6 * SIZE(AO1)

  898. 	FMADD	y01, atemp3, a9,  y01

  899. 	LFD	a9,  4 * SIZE(AO3)

  900. 

  901. 	FMADD	xsum2, xtemp3, a7,  xsum2

  902. 	LFD	a7,  6 * SIZE(AO2)

  903. 	FMADD	y02, atemp3, a10, y02

  904. 	LFD	a10, 5 * SIZE(AO3)

  905. 

  906. 	FMADD	xsum3, xtemp3, a11, xsum3

  907. 	NOP1

  908. 	FMADD	y03, atemp3, a11, y03

  909. 	LFD	a11, 6 * SIZE(AO3)

  910. 

  911. 	FMADD	xsum4, xtemp3, a15, xsum4

  912. 	LFD	xtemp3,  6 * SIZE(XX)

  913. 	FMADD	y04, atemp3, a12, y04

  914. 	NOP2

  915. 

  916. 	FMADD	xsum1, xtemp4, a4,  xsum1

  917. 	LFD	a4,  7 * SIZE(AO1)

  918. 	FMADD	y01, atemp4, a13, y01

  919. 	LFD	a13, 4 * SIZE(AO4)

  920. 

  921. 	FMADD	xsum2, xtemp4, a8,  xsum2

  922. 	LFD	a8,  7 * SIZE(AO2)

  923. 	FMADD	y02, atemp4, a14, y02

  924. 	LFD	a14, 5 * SIZE(AO4)

  925. 

  926. 	FMADD	xsum3, xtemp4, a12, xsum3

  927. 	LFD	a12, 7 * SIZE(AO3)

  928. 	FMADD	y03, atemp4, a15, y03

  929. 	LFD	a15, 6 * SIZE(AO4)

  930. 

  931. 	FMADD	xsum4, xtemp4, a16, xsum4

  932. 	LFD	xtemp4,  7 * SIZE(XX)

  933. 	FMADD	y04, atemp4, a16, y04

  934. 	LFD	a16, 7 * SIZE(AO4)

  935. 

  936. 	STFD	y01,  0 * SIZE(YY)

  937. 	LFD	y01,  4 * SIZE(YY)

  938. 	STFD	y02,  1 * SIZE(YY)

  939. 	LFD	y02,  5 * SIZE(YY)

  940. 

  941. 	STFD	y03,  2 * SIZE(YY)

  942. 	LFD	y03,  6 * SIZE(YY)

  943. 	STFD	y04,  3 * SIZE(YY)

  944. 	LFD	y04,  7 * SIZE(YY)

  945. 

  946. 	FMADD	xsum1, xtemp1, a1,  xsum1

  947. 	NOP1

  948. 	FMADD	y01, atemp1, a1,  y01

  949. 	LFD	a1,  8 * SIZE(AO1)

  950. 

  951. 	FMADD	xsum2, xtemp1, a5,  xsum2

  952. 	NOP1

  953. 	FMADD	y02, atemp1, a2,  y02

  954. 	NOP2

  955. 

  956. 	FMADD	xsum3, xtemp1, a9,  xsum3

  957. 	NOP1

  958. 	FMADD	y03, atemp1, a3,  y03

  959. 	NOP2

  960. 

  961. 	FMADD	xsum4, xtemp1, a13, xsum4

  962. 	LFD	xtemp1,  8 * SIZE(XX)

  963. 	FMADD	y04, atemp1, a4,  y04

  964. 	NOP2

  965. 

  966. 	FMADD	xsum1, xtemp2, a2,  xsum1

  967. 	LFD	a2,  9 * SIZE(AO1)

  968. 	FMADD	y01, atemp2, a5,  y01

  969. 	LFD	a5,  8 * SIZE(AO2)

  970. 

  971. 	FMADD	xsum2, xtemp2, a6,  xsum2

  972. 	NOP1

  973. 	FMADD	y02, atemp2, a6,  y02

  974. 	LFD	a6,  9 * SIZE(AO2)

  975. 

  976. 	FMADD	xsum3, xtemp2, a10, xsum3

  977. 	NOP1

  978. 	FMADD	y03, atemp2, a7,  y03

  979. 	NOP2

  980. 

  981. 	FMADD	xsum4, xtemp2, a14, xsum4

  982. 	LFD	xtemp2,  9 * SIZE(XX)

  983. 	FMADD	y04, atemp2, a8,  y04

  984. 	NOP2

  985. 

  986. 	FMADD	xsum1, xtemp3, a3,  xsum1

  987. 	LFD	a3, 10 * SIZE(AO1)

  988. 	FMADD	y01, atemp3, a9,  y01

  989. 	LFD	a9,  8 * SIZE(AO3)

  990. 

  991. 	FMADD	xsum2, xtemp3, a7,  xsum2

  992. 	LFD	a7, 10 * SIZE(AO2)

  993. 	FMADD	y02, atemp3, a10, y02

  994. 	LFD	a10, 9 * SIZE(AO3)

  995. 

  996. 	FMADD	xsum3, xtemp3, a11, xsum3

  997. 	NOP1

  998. 	FMADD	y03, atemp3, a11, y03

  999. 	LFD	a11, 10 * SIZE(AO3)

  1000. 

  1001. 	FMADD	xsum4, xtemp3, a15, xsum4

  1002. 	LFD	xtemp3, 10 * SIZE(XX)

  1003. 	FMADD	y04, atemp3, a12, y04

  1004. 	NOP2

  1005. 

  1006. 	FMADD	xsum1, xtemp4, a4,  xsum1

  1007. 	LFD	a4, 11 * SIZE(AO1)

  1008. 	FMADD	y01, atemp4, a13, y01

  1009. 	LFD	a13, 8 * SIZE(AO4)

  1010. 

  1011. 	FMADD	xsum2, xtemp4, a8,  xsum2

  1012. 	LFD	a8, 11 * SIZE(AO2)

  1013. 	FMADD	y02, atemp4, a14, y02

  1014. 	LFD	a14, 9 * SIZE(AO4)

  1015. 

  1016. 	FMADD	xsum3, xtemp4, a12, xsum3

  1017. 	LFD	a12, 11 * SIZE(AO3)

  1018. 	FMADD	y03, atemp4, a15, y03

  1019. 	LFD	a15, 10 * SIZE(AO4)

  1020. 

  1021. 	FMADD	xsum4, xtemp4, a16, xsum4

  1022. 	LFD	xtemp4, 11 * SIZE(XX)

  1023. 	FMADD	y04, atemp4, a16, y04

  1024. 	LFD	a16, 11 * SIZE(AO4)

  1025. 

  1026. 	addi	AO1, AO1, 8 * SIZE

  1027. 	addi	AO2, AO2, 8 * SIZE

  1028. 	addi	AO3, AO3, 8 * SIZE

  1029. 	addi	AO4, AO4, 8 * SIZE

  1030. 

  1031. 	STFD	y01,  4 * SIZE(YY)

  1032. 	LFD	y01,  8 * SIZE(YY)

  1033. 	STFD	y02,  5 * SIZE(YY)

  1034. 	LFD	y02,  9 * SIZE(YY)

  1035. 

  1036. 	STFD	y03,  6 * SIZE(YY)

  1037. 	LFD	y03, 10 * SIZE(YY)

  1038. 	STFD	y04,  7 * SIZE(YY)

  1039. 	LFD	y04, 11 * SIZE(YY)

  1040. 

  1041. 	addi	XX, XX, 8 * SIZE

  1042. 	addi	YY, YY, 8 * SIZE

  1043. 	.align 4

  1044. 

  1045. LL(15):

  1046. 	sub	TEMP, M, IS

  1047. 	addi	TEMP, TEMP, -4

  1048. 	andi.	r0,  TEMP, 4

  1049. 	ble	LL(16)

  1050. 

  1051. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1052. 	NOP1

  1053. 	FMADD	y01, atemp1, a1,  y01

  1054. 	LFD	a1,  4 * SIZE(AO1)

  1055. 

  1056. 	FMADD	xsum2, xtemp1, a5,  xsum2

  1057. 	NOP1

  1058. 	FMADD	y02, atemp1, a2,  y02

  1059. 	NOP2

  1060. 

  1061. 	FMADD	xsum3, xtemp1, a9,  xsum3

  1062. 	NOP1

  1063. 	FMADD	y03, atemp1, a3,  y03

  1064. 	NOP2

  1065. 

  1066. 	FMADD	xsum4, xtemp1, a13, xsum4

  1067. 	LFD	xtemp1,  4 * SIZE(XX)

  1068. 	FMADD	y04, atemp1, a4,  y04

  1069. 	NOP2

  1070. 

  1071. 	FMADD	xsum1, xtemp2, a2,  xsum1

  1072. 	LFD	a2,  5 * SIZE(AO1)

  1073. 	FMADD	y01, atemp2, a5,  y01

  1074. 	LFD	a5,  4 * SIZE(AO2)

  1075. 

  1076. 	FMADD	xsum2, xtemp2, a6,  xsum2

  1077. 	NOP1

  1078. 	FMADD	y02, atemp2, a6,  y02

  1079. 	LFD	a6,  5 * SIZE(AO2)

  1080. 

  1081. 	FMADD	xsum3, xtemp2, a10, xsum3

  1082. 	NOP1

  1083. 	FMADD	y03, atemp2, a7,  y03

  1084. 	NOP2

  1085. 

  1086. 	FMADD	xsum4, xtemp2, a14, xsum4

  1087. 	LFD	xtemp2,  5 * SIZE(XX)

  1088. 	FMADD	y04, atemp2, a8,  y04

  1089. 	NOP2

  1090. 

  1091. 	FMADD	xsum1, xtemp3, a3,  xsum1

  1092. 	LFD	a3,  6 * SIZE(AO1)

  1093. 	FMADD	y01, atemp3, a9,  y01

  1094. 	LFD	a9,  4 * SIZE(AO3)

  1095. 

  1096. 	FMADD	xsum2, xtemp3, a7,  xsum2

  1097. 	LFD	a7,  6 * SIZE(AO2)

  1098. 	FMADD	y02, atemp3, a10, y02

  1099. 	LFD	a10, 5 * SIZE(AO3)

  1100. 

  1101. 	FMADD	xsum3, xtemp3, a11, xsum3

  1102. 	NOP1

  1103. 	FMADD	y03, atemp3, a11, y03

  1104. 	LFD	a11, 6 * SIZE(AO3)

  1105. 

  1106. 	FMADD	xsum4, xtemp3, a15, xsum4

  1107. 	LFD	xtemp3,  6 * SIZE(XX)

  1108. 	FMADD	y04, atemp3, a12, y04

  1109. 	NOP2

  1110. 

  1111. 	FMADD	xsum1, xtemp4, a4,  xsum1

  1112. 	LFD	a4,  7 * SIZE(AO1)

  1113. 	FMADD	y01, atemp4, a13, y01

  1114. 	LFD	a13, 4 * SIZE(AO4)

  1115. 

  1116. 	FMADD	xsum2, xtemp4, a8,  xsum2

  1117. 	LFD	a8,  7 * SIZE(AO2)

  1118. 	FMADD	y02, atemp4, a14, y02

  1119. 	LFD	a14, 5 * SIZE(AO4)

  1120. 

  1121. 	FMADD	xsum3, xtemp4, a12, xsum3

  1122. 	LFD	a12, 7 * SIZE(AO3)

  1123. 	FMADD	y03, atemp4, a15, y03

  1124. 	LFD	a15, 6 * SIZE(AO4)

  1125. 

  1126. 	FMADD	xsum4, xtemp4, a16, xsum4

  1127. 	LFD	xtemp4,  7 * SIZE(XX)

  1128. 	FMADD	y04, atemp4, a16, y04

  1129. 	LFD	a16, 7 * SIZE(AO4)

  1130. 

  1131. 	addi	AO1, AO1, 4 * SIZE

  1132. 	addi	AO2, AO2, 4 * SIZE

  1133. 	addi	AO3, AO3, 4 * SIZE

  1134. 	addi	AO4, AO4, 4 * SIZE

  1135. 

  1136. 	STFD	y01,  0 * SIZE(YY)

  1137. 	LFD	y01,  4 * SIZE(YY)

  1138. 	STFD	y02,  1 * SIZE(YY)

  1139. 	LFD	y02,  5 * SIZE(YY)

  1140. 

  1141. 	STFD	y03,  2 * SIZE(YY)

  1142. 	LFD	y03,  6 * SIZE(YY)

  1143. 	STFD	y04,  3 * SIZE(YY)

  1144. 	LFD	y04,  7 * SIZE(YY)

  1145. 

  1146. 	addi	XX, XX, 4 * SIZE

  1147. 	addi	YY, YY, 4 * SIZE

  1148. 	.align 4

  1149. 

  1150. LL(16):

  1151. 	andi.	r0,  M, 2

  1152. 	ble	LL(17)

  1153. 

  1154. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1155. 	FMADD	y01, atemp1, a1,  y01

  1156. 	LFD	a1,  2 * SIZE(AO1)

  1157. 

  1158. 	FMADD	xsum2, xtemp1, a5,  xsum2

  1159. 	FMADD	y02, atemp1, a2,  y02

  1160. 

  1161. 	FMADD	xsum3, xtemp1, a9,  xsum3

  1162. 	FMADD	y01, atemp2, a5,  y01

  1163. 	LFD	a5,  2 * SIZE(AO2)

  1164. 

  1165. 	FMADD	xsum4, xtemp1, a13, xsum4

  1166. 	LFD	xtemp1,  2 * SIZE(XX)

  1167. 	FMADD	y02, atemp2, a6,  y02

  1168. 

  1169. 	FMADD	xsum1, xtemp2, a2,  xsum1

  1170. 	FMADD	y01, atemp3, a9,  y01

  1171. 	LFD	a9,  2 * SIZE(AO3)

  1172. 

  1173. 	FMADD	xsum2, xtemp2, a6,  xsum2

  1174. 	FMADD	y02, atemp3, a10, y02

  1175. 

  1176. 	FMADD	xsum3, xtemp2, a10, xsum3

  1177. 	FMADD	y01, atemp4, a13, y01

  1178. 	LFD	a13, 2 * SIZE(AO4)

  1179. 

  1180. 	FMADD	xsum4, xtemp2, a14, xsum4

  1181. 	FMADD	y02, atemp4, a14, y02

  1182. 

  1183. 	STFD	y01,  0 * SIZE(YY)

  1184. 	LFD	y01,  2 * SIZE(YY)

  1185. 	STFD	y02,  1 * SIZE(YY)

  1186. 	addi	YY, YY, 2 * SIZE

  1187. 	.align 4

  1188. 

  1189. LL(17):

  1190. 	andi.	r0,  M, 1

  1191. 	ble	LL(18)

  1192. 

  1193. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1194. 	FMADD	y01, atemp1, a1,  y01

  1195. 	FMADD	xsum2, xtemp1, a5,  xsum2

  1196. 	FMADD	y01, atemp2, a5,  y01

  1197. 	FMADD	xsum3, xtemp1, a9,  xsum3

  1198. 	FMADD	y01, atemp3, a9,  y01

  1199. 	FMADD	xsum4, xtemp1, a13, xsum4

  1200. 	FMADD	y01, atemp4, a13, y01

  1201. 

  1202. 	STFD	y01,  0 * SIZE(YY)

  1203. 	.align 4

  1204. 

  1205. LL(18):

  1206. 	slwi	TEMP,  IS,  BASE_SHIFT

  1207. 	add	YY, NEW_Y, TEMP

  1208. 

  1209. 	LFD	y01,  0 * SIZE(YY)

  1210. 	LFD	y02,  1 * SIZE(YY)

  1211. 	LFD	y03,  2 * SIZE(YY)

  1212. 	LFD	y04,  3 * SIZE(YY)

  1213. 

  1214. 	LFD	xtemp1, ALPHA

  1215. 

  1216. 	FMUL	xsum1, xtemp1, xsum1

  1217. 	FMUL	xsum2, xtemp1, xsum2

  1218. 	FMUL	xsum3, xtemp1, xsum3

  1219. 	FMUL	xsum4, xtemp1, xsum4

  1220. 

  1221. 	FADD	y01, y01, xsum1

  1222. 	FADD	y02, y02, xsum2

  1223. 	FADD	y03, y03, xsum3

  1224. 	FADD	y04, y04, xsum4

  1225. 

  1226. 	STFD	y01,  0 * SIZE(YY)

  1227. 	STFD	y02,  1 * SIZE(YY)

  1228. 	STFD	y03,  2 * SIZE(YY)

  1229. 	STFD	y04,  3 * SIZE(YY)

  1230. 

  1231. 	addi	TEMP, IS, 8

  1232. 	addi	IS,   IS, 4

  1233. 	cmpw	cr0, TEMP, N

  1234. 	ble	LL(11)

  1235. 	.align 4

  1236. 

  1237. LL(20):

  1238. 	andi.	TEMP, N, 2

  1239. 	ble	LL(30)

  1240. 

  1241. 	mr	AO1, A

  1242. 	add	AO2, A,   LDA

  1243. 	add	A,   AO2, LDA

  1244. 	addi	A, A, 2 * SIZE

  1245. 

  1246. 	slwi	TEMP,  IS,  BASE_SHIFT

  1247. 	add	XX, X,     TEMP

  1248. 	add	YY, NEW_Y, TEMP

  1249. 

  1250. 	LFD	atemp1, 0 * SIZE(XX)

  1251. 	LFD	atemp2, 1 * SIZE(XX)

  1252. 

  1253. 	LFD	a1,  0 * SIZE(AO1)

  1254. 	LFD	a2,  1 * SIZE(AO1)

  1255. 	LFD	a6,  1 * SIZE(AO2)

  1256. 

  1257. 	LFD	a5, ALPHA

  1258. 

  1259. 	FMUL	xsum1, atemp1, a1

  1260. 	FMUL	xsum2, atemp1, a2

  1261. 

  1262. 	FMADD	xsum1, atemp2, a2,  xsum1

  1263. 	FMADD	xsum2, atemp2, a6,  xsum2

  1264. 

  1265. 	FMUL	atemp1, a5, atemp1

  1266. 	FMUL	atemp2, a5, atemp2

  1267. 

  1268. 	LFD	xtemp1,  2 * SIZE(XX)

  1269. 	LFD	y01,     2 * SIZE(YY)

  1270. 	LFD	a1,      2 * SIZE(AO1)

  1271. 	LFD	a5,      2 * SIZE(AO2)

  1272. 

  1273. 	andi.	r0,  M, 1

  1274. 	ble	LL(28)

  1275. 

  1276. 	FMADD	xsum1, xtemp1, a1,  xsum1

  1277. 	FMADD	y01, atemp1, a1,  y01

  1278. 	FMADD	xsum2, xtemp1, a5,  xsum2

  1279. 	FMADD	y01, atemp2, a5,  y01

  1280. 

  1281. 	STFD	y01,  2 * SIZE(YY)

  1282. 	.align 4

  1283. 

  1284. LL(28):

  1285. 	slwi	TEMP,  IS,  BASE_SHIFT

  1286. 	add	YY, NEW_Y, TEMP

  1287. 

  1288. 	LFD	y01,  0 * SIZE(YY)

  1289. 	LFD	y02,  1 * SIZE(YY)

  1290. 

  1291. 	LFD	xtemp1, ALPHA

  1292. 

  1293. 	FMUL	xsum1, xtemp1, xsum1

  1294. 	FMUL	xsum2, xtemp1, xsum2

  1295. 

  1296. 	FADD	y01, y01, xsum1

  1297. 	FADD	y02, y02, xsum2

  1298. 

  1299. 	STFD	y01,  0 * SIZE(YY)

  1300. 	STFD	y02,  1 * SIZE(YY)

  1301. 

  1302. 	addi	IS,   IS, 2

  1303. 	.align 4

  1304. 

  1305. LL(30):

  1306. 	andi.	TEMP, N, 1

  1307. 	ble	LL(990)

  1308. 

  1309. 	mr	AO1, A

  1310. 

  1311. 	slwi	TEMP,  IS,  BASE_SHIFT

  1312. 	add	XX, X,     TEMP

  1313. 	add	YY, NEW_Y, TEMP

  1314. 

  1315. 	LFD	atemp1, 0 * SIZE(XX)

  1316. 	LFD	a1,     0 * SIZE(AO1)

  1317. 	LFD	xtemp1, ALPHA

  1318. 	LFD	y01,    0 * SIZE(YY)

  1319. 

  1320. 	FMUL	xsum1, atemp1, a1

  1321. 	FMUL	xsum1, xtemp1, xsum1

  1322. 

  1323. 	FADD	y01, y01, xsum1

  1324. 

  1325. 	STFD	y01,  0 * SIZE(YY)

  1326. 	.align 4

  1327. 

  1328. LL(990):

  1329. 	cmpwi	cr0, INCY, SIZE

  1330. 	beq	LL(999)

  1331. 

  1332. 	mr	YY, Y

  1333. 

  1334. 	srawi.	r0, M, 3

  1335. 	mtspr	CTR, r0

  1336. 	ble	LL(995)

  1337. 	.align 4

  1338. 

  1339. LL(991):

  1340. 	LFD	f0,  0 * SIZE(Y)

  1341. 	add	Y, Y, INCY

  1342. 	LFD	f1,  0 * SIZE(Y)

  1343. 	add	Y, Y, INCY

  1344. 	LFD	f2,  0 * SIZE(Y)

  1345. 	add	Y, Y, INCY

  1346. 	LFD	f3,  0 * SIZE(Y)

  1347. 	add	Y, Y, INCY

  1348. 	LFD	f4,  0 * SIZE(Y)

  1349. 	add	Y, Y, INCY

  1350. 	LFD	f5,  0 * SIZE(Y)

  1351. 	add	Y, Y, INCY

  1352. 	LFD	f6,  0 * SIZE(Y)

  1353. 	add	Y, Y, INCY

  1354. 	LFD	f7,  0 * SIZE(Y)

  1355. 	add	Y, Y, INCY

  1356. 

  1357. 	LFD	f8,   0 * SIZE(NEW_Y)

  1358. 	LFD	f9,   1 * SIZE(NEW_Y)

  1359. 	LFD	f10,  2 * SIZE(NEW_Y)

  1360. 	LFD	f11,  3 * SIZE(NEW_Y)

  1361. 	LFD	f12,  4 * SIZE(NEW_Y)

  1362. 	LFD	f13,  5 * SIZE(NEW_Y)

  1363. 	LFD	f14,  6 * SIZE(NEW_Y)

  1364. 	LFD	f15,  7 * SIZE(NEW_Y)

  1365. 	addi	NEW_Y, NEW_Y, 8 * SIZE

  1366. 

  1367. 	FADD	f8,  f8,  f0

  1368. 	FADD	f9,  f9,  f1

  1369. 	FADD	f10, f10, f2

  1370. 	FADD	f11, f11, f3

  1371. 	FADD	f12, f12, f4

  1372. 	FADD	f13, f13, f5

  1373. 	FADD	f14, f14, f6

  1374. 	FADD	f15, f15, f7

  1375. 

  1376. 	STFD	f8,  0 * SIZE(YY)

  1377. 	add	YY, YY, INCY

  1378. 	STFD	f9,  0 * SIZE(YY)

  1379. 	add	YY, YY, INCY

  1380. 	STFD	f10, 0 * SIZE(YY)

  1381. 	add	YY, YY, INCY

  1382. 	STFD	f11, 0 * SIZE(YY)

  1383. 	add	YY, YY, INCY

  1384. 	STFD	f12, 0 * SIZE(YY)

  1385. 	add	YY, YY, INCY

  1386. 	STFD	f13, 0 * SIZE(YY)

  1387. 	add	YY, YY, INCY

  1388. 	STFD	f14, 0 * SIZE(YY)

  1389. 	add	YY, YY, INCY

  1390. 	STFD	f15, 0 * SIZE(YY)

  1391. 	add	YY, YY, INCY

  1392. 	bdnz	LL(991)

  1393. 	.align 4

  1394. 

  1395. LL(995):

  1396. 	andi.	J, M, 4

  1397. 	ble	LL(996)

  1398. 

  1399. 	LFD	f0,  0 * SIZE(Y)

  1400. 	add	Y, Y, INCY

  1401. 	LFD	f1,  0 * SIZE(Y)

  1402. 	add	Y, Y, INCY

  1403. 	LFD	f2,  0 * SIZE(Y)

  1404. 	add	Y, Y, INCY

  1405. 	LFD	f3,  0 * SIZE(Y)

  1406. 	add	Y, Y, INCY

  1407. 

  1408. 	LFD	f8,   0 * SIZE(NEW_Y)

  1409. 	LFD	f9,   1 * SIZE(NEW_Y)

  1410. 	LFD	f10,  2 * SIZE(NEW_Y)

  1411. 	LFD	f11,  3 * SIZE(NEW_Y)

  1412. 	addi	NEW_Y, NEW_Y, 4 * SIZE

  1413. 

  1414. 	FADD	f8,  f8,  f0

  1415. 	FADD	f9,  f9,  f1

  1416. 	FADD	f10, f10, f2

  1417. 	FADD	f11, f11, f3

  1418. 

  1419. 	STFD	f8,  0 * SIZE(YY)

  1420. 	add	YY, YY, INCY

  1421. 	STFD	f9,  0 * SIZE(YY)

  1422. 	add	YY, YY, INCY

  1423. 	STFD	f10, 0 * SIZE(YY)

  1424. 	add	YY, YY, INCY

  1425. 	STFD	f11, 0 * SIZE(YY)

  1426. 	add	YY, YY, INCY

  1427. 	.align 4

  1428. 

  1429. LL(996):

  1430. 	andi.	J, M, 2

  1431. 	ble	LL(997)

  1432. 

  1433. 	LFD	f0,  0 * SIZE(Y)

  1434. 	add	Y, Y, INCY

  1435. 	LFD	f1,  0 * SIZE(Y)

  1436. 	add	Y, Y, INCY

  1437. 

  1438. 	LFD	f8,   0 * SIZE(NEW_Y)

  1439. 	LFD	f9,   1 * SIZE(NEW_Y)

  1440. 	addi	NEW_Y, NEW_Y, 2 * SIZE

  1441. 

  1442. 	FADD	f8,  f8,  f0

  1443. 	FADD	f9,  f9,  f1

  1444. 

  1445. 	STFD	f8,  0 * SIZE(YY)

  1446. 	add	YY, YY, INCY

  1447. 	STFD	f9,  0 * SIZE(YY)

  1448. 	add	YY, YY, INCY

  1449. 	.align 4

  1450. 

  1451. LL(997):

  1452. 	andi.	J, M, 1

  1453. 	ble	LL(999)

  1454. 

  1455. 	LFD	f0,  0 * SIZE(Y)

  1456. 	LFD	f8,   0 * SIZE(NEW_Y)

  1457. 

  1458. 	FADD	f8,  f8,  f0

  1459. 

  1460. 	STFD	f8,  0 * SIZE(YY)

  1461. 	.align 4

  1462. 

  1463. LL(999):

  1464. 	li	r3, 0

  1465. 

  1466. 	lfd	f14,     0(SP)

  1467. 	lfd	f15,     8(SP)

  1468. 	lfd	f16,    16(SP)

  1469. 	lfd	f17,    24(SP)

  1470. 	lfd	f18,    32(SP)

  1471. 	lfd	f19,    40(SP)

  1472. 	lfd	f20,    48(SP)

  1473. 	lfd	f21,    56(SP)

  1474. 	lfd	f22,    64(SP)

  1475. 	lfd	f23,    72(SP)

  1476. 	lfd	f24,    80(SP)

  1477. 	lfd	f25,    88(SP)

  1478. 	lfd	f26,    96(SP)

  1479. 	lfd	f27,   104(SP)

  1480. 	lfd	f28,   112(SP)

  1481. 	lfd	f29,   120(SP)

  1482. 	lfd	f30,   128(SP)

  1483. 	lfd	f31,   136(SP)

  1484. 

  1485. #ifdef __64BIT__

  1486. 	ld	r14,   144(SP)

  1487. 	ld	r15,   152(SP)

  1488. 	ld	r16,   160(SP)

  1489. 	ld	r17,   168(SP)

  1490. 	ld	r18,   176(SP)

  1491. 	ld	r19,   184(SP)

  1492. 	ld	r20,   192(SP)

  1493. 	ld	r21,   200(SP)

  1494. 	ld	r22,   208(SP)

  1495. 	ld	r23,   216(SP)

  1496. 	ld	r24,   224(SP)

  1497. 	ld	r25,   232(SP)

  1498. 	ld	r26,   240(SP)

  1499. 	ld	r27,   248(SP)

  1500. #else

  1501. 	lwz	r14,   144(SP)

  1502. 	lwz	r15,   148(SP)

  1503. 	lwz	r16,   152(SP)

  1504. 	lwz	r17,   156(SP)

  1505. 	lwz	r18,   160(SP)

  1506. 	lwz	r19,   164(SP)

  1507. 	lwz	r20,   168(SP)

  1508. 	lwz	r21,   172(SP)

  1509. 	lwz	r22,   176(SP)

  1510. 	lwz	r23,   180(SP)

  1511. 	lwz	r24,   184(SP)

  1512. 	lwz	r25,   188(SP)

  1513. 	lwz	r26,   192(SP)

  1514. 	lwz	r27,   196(SP)

  1515. #endif

  1516. 

  1517. 	addi	SP, SP, STACKSIZE

  1518. 	blr

  1519. 

  1520. 	EPILOGUE

  1521. #endif