OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 66d9185ebe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '66d9185ebe'
${ noResults }
OpenBLAS/kernel/power/zger.S

1358 lines
29 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef NEEDPARAM

  43. #ifndef DOUBLE

  44. #include "cparam.h"

  45. #else

  46. #include "zparam.h"

  47. #endif

  48. #endif

  49. 

  50. #if defined(linux) || defined(__FreeBSD__)

  51. #ifndef __64BIT__

  52. #define M	r3

  53. #define	N	r4

  54. #define X	r6

  55. #define INCX	r7

  56. #define Y	r8

  57. #define	INCY	r9

  58. #define	A	r10

  59. #define	LDA	r5

  60. #else

  61. #define M	r3

  62. #define	N	r4

  63. #define X	r8

  64. #define INCX	r9

  65. #define Y	r10

  66. #define	INCY	r5

  67. #define	A	r6

  68. #define	LDA	r7

  69. #endif

  70. #endif

  71. 

  72. #if defined(_AIX) || defined(__APPLE__)

  73. #if !defined(__64BIT__) && defined(DOUBLE)

  74. #define M	r3

  75. #define	N	r4

  76. #define X	r10

  77. #define INCX	r5

  78. #define Y	r6

  79. #define	INCY	r7

  80. #define	A	r8

  81. #define	LDA	r9

  82. #else

  83. #define M	r3

  84. #define	N	r4

  85. #define X	r8

  86. #define INCX	r9

  87. #define Y	r10

  88. #define	INCY	r5

  89. #define	A	r6

  90. #define	LDA	r7

  91. #endif

  92. #endif

  93. 

  94. #define I	r11

  95. #define	J	r12

  96. 

  97. #define AO1	r14

  98. #define AO2	r15

  99. #define AO3	r16

  100. #define AO4	r17

  101. #define AO5	r18

  102. #define AO6	r19

  103. #define AO7	r20

  104. #define AO8	r21

  105. 

  106. #define	X1	r22

  107. #define	PREA	r23

  108. #define	PREC	r24

  109. #define XX	r25

  110. #define BUFFER	r26

  111. 

  112. #define y01 f0

  113. #define y02 f1

  114. #define y03 f2

  115. #define y04 f3

  116. #define y05 f4

  117. #define y06 f5

  118. #define y07 f6

  119. #define y08 f7

  120. 

  121. #define alpha1_r f8

  122. #define alpha1_i f9

  123. #define alpha2_r f10

  124. #define alpha2_i f11

  125. 

  126. #define a1     f12

  127. #define a2     f13

  128. #define a3     f14

  129. #define a4     f15

  130. #define a5     f16

  131. #define a6     f17

  132. #define a7     f18

  133. #define a8     f19

  134. #define a9     f20

  135. #define a10    f21

  136. #define a11    f22

  137. #define a12    f23

  138. #define a13    f24

  139. #define a14    f25

  140. #define a15    f26

  141. #define a16    f27

  142. 

  143. #define alpha_r  f30

  144. #define alpha_i  f31

  145. 

  146. #ifndef CONJ

  147. #define FMA1	FNMSUB

  148. #define FMA2	FMADD

  149. #else

  150. #define FMA1	FMADD

  151. #define FMA2	FNMSUB

  152. #endif

  153. 

  154. #if defined(PPC440) || defined(PPC440FP2)

  155. #define PREFETCHSIZE_A  24

  156. #define PREFETCHSIZE_C  16

  157. #endif

  158. 

  159. #ifdef PPC970

  160. #define PREFETCHSIZE_A  16

  161. #define PREFETCHSIZE_C  16

  162. #endif

  163. 

  164. #ifdef POWER4

  165. #define PREFETCHSIZE_A  16

  166. #define PREFETCHSIZE_C  16

  167. #endif

  168. 

  169. #ifdef POWER5

  170. #define PREFETCHSIZE_A  16

  171. #define PREFETCHSIZE_C  16

  172. #endif

  173. 

  174. #ifndef NEEDPARAM

  175. 

  176. #ifndef __64BIT__

  177. #define STACKSIZE 224

  178. #else

  179. #define STACKSIZE 280

  180. #endif

  181. 

  182. 	PROLOGUE

  183. 	PROFCODE

  184. 

  185. 	addi	SP,   SP, -STACKSIZE

  186. 

  187. 	stfd	f14,     0(SP)

  188. 	stfd	f15,     8(SP)

  189. 	stfd	f16,    16(SP)

  190. 	stfd	f17,    24(SP)

  191. 	stfd	f18,    32(SP)

  192. 	stfd	f19,    40(SP)

  193. 	stfd	f20,    48(SP)

  194. 	stfd	f21,    56(SP)

  195. 	stfd	f22,    64(SP)

  196. 	stfd	f23,    72(SP)

  197. 	stfd	f24,    80(SP)

  198. 	stfd	f25,    88(SP)

  199. 	stfd	f26,    96(SP)

  200. 	stfd	f27,   104(SP)

  201. 	stfd	f28,   112(SP)

  202. 	stfd	f29,   120(SP)

  203. 	stfd	f30,   128(SP)

  204. 	stfd	f31,   136(SP)

  205. 

  206. #ifdef __64BIT__

  207. 	std	r14,   144(SP)

  208. 	std	r15,   152(SP)

  209. 	std	r16,   160(SP)

  210. 	std	r17,   168(SP)

  211. 	std	r18,   176(SP)

  212. 	std	r19,   184(SP)

  213. 	std	r20,   192(SP)

  214. 	std	r21,   200(SP)

  215. 	std	r22,   208(SP)

  216. 	std	r23,   216(SP)

  217. 	std	r24,   224(SP)

  218. 	std	r25,   232(SP)

  219. 	std	r26,   240(SP)

  220. 	std	r27,   248(SP)

  221. #else

  222. 	stw	r14,   144(SP)

  223. 	stw	r15,   148(SP)

  224. 	stw	r16,   152(SP)

  225. 	stw	r17,   156(SP)

  226. 	stw	r18,   160(SP)

  227. 	stw	r19,   164(SP)

  228. 	stw	r20,   168(SP)

  229. 	stw	r21,   172(SP)

  230. 	stw	r22,   176(SP)

  231. 	stw	r23,   180(SP)

  232. 	stw	r24,   184(SP)

  233. 	stw	r25,   188(SP)

  234. 	stw	r26,   192(SP)

  235. 	stw	r27,   196(SP)

  236. #endif

  237. 

  238. #if defined(linux) || defined(__FreeBSD__)

  239. #ifndef __64BIT__

  240. 	lwz	LDA,     FRAMESLOT(0) + STACKSIZE(SP)

  241. 	lwz	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  242. #else

  243. 	ld	INCY,    FRAMESLOT(0) + STACKSIZE(SP)

  244. 	ld	A,       FRAMESLOT(1) + STACKSIZE(SP)

  245. 	ld	LDA,     FRAMESLOT(2) + STACKSIZE(SP)

  246. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  247. #endif

  248. #endif

  249. 

  250. #if defined(_AIX) || defined(__APPLE__)

  251. #ifndef __64BIT__

  252. #ifdef DOUBLE

  253. 	lwz	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  254. 	lwz	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  255. 	lwz	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  256. 	lwz	A,       FRAMESLOT(3) + STACKSIZE(SP)

  257. 	lwz	LDA,     FRAMESLOT(4) + STACKSIZE(SP)

  258. 	lwz	BUFFER,  FRAMESLOT(5) + STACKSIZE(SP)

  259. #else

  260. 	lwz	INCY,    FRAMESLOT(0) + STACKSIZE(SP)

  261. 	lwz	A,       FRAMESLOT(1) + STACKSIZE(SP)

  262. 	lwz	LDA,     FRAMESLOT(2) + STACKSIZE(SP)

  263. 	lwz	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  264. #endif

  265. #else

  266. 	ld	INCY,    FRAMESLOT(0) + STACKSIZE(SP)

  267. 	ld	A,       FRAMESLOT(1) + STACKSIZE(SP)

  268. 	ld	LDA,     FRAMESLOT(2) + STACKSIZE(SP)

  269. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  270. #endif

  271. #endif

  272. 

  273. 	fmr	alpha_r, f1

  274. 	fmr	alpha_i, f2

  275. 

  276. 	slwi	LDA,  LDA,  ZBASE_SHIFT

  277. 	slwi	INCX, INCX, ZBASE_SHIFT

  278. 	slwi	INCY, INCY, ZBASE_SHIFT

  279. 

  280. 	li	PREA, PREFETCHSIZE_A * SIZE

  281. 	li	PREC, PREFETCHSIZE_C * SIZE

  282. 

  283. 	cmpwi	cr0, M, 0

  284. 	ble-	LL(999)

  285. 

  286. 	cmpwi	cr0, N, 0

  287. 	ble-	LL(999)

  288. 

  289. 	mr	XX, X

  290. 

  291. 	cmpi	cr0, 0, INCX, 2 * SIZE

  292. 	beq	LL(10)

  293. 

  294. 	mr	XX, BUFFER

  295. 	mr	X1, BUFFER

  296. 

  297. 	srawi.	r0,  M, 2

  298. 	mtspr	CTR, r0

  299. 	ble	LL(05)

  300. 	.align 4

  301. 

  302. LL(01):

  303. 	LFD	a1, 0 * SIZE(X)

  304. 	LFD	a2, 1 * SIZE(X)

  305. 	add	X, X, INCX

  306. 	LFD	a3, 0 * SIZE(X)

  307. 	LFD	a4, 1 * SIZE(X)

  308. 	add	X, X, INCX

  309. 	LFD	a5, 0 * SIZE(X)

  310. 	LFD	a6, 1 * SIZE(X)

  311. 	add	X, X, INCX

  312. 	LFD	a7, 0 * SIZE(X)

  313. 	LFD	a8, 1 * SIZE(X)

  314. 	add	X, X, INCX

  315. 

  316. 	STFD	a1, 0 * SIZE(X1)

  317. 	STFD	a2, 1 * SIZE(X1)

  318. 	STFD	a3, 2 * SIZE(X1)

  319. 	STFD	a4, 3 * SIZE(X1)

  320. 	STFD	a5, 4 * SIZE(X1)

  321. 	STFD	a6, 5 * SIZE(X1)

  322. 	STFD	a7, 6 * SIZE(X1)

  323. 	STFD	a8, 7 * SIZE(X1)

  324. 

  325. 	addi	X1, X1, 8 * SIZE

  326. 	bdnz+	LL(01)

  327. 	.align 4

  328. 

  329. LL(05):

  330. 	andi.	r0, M, 7

  331. 	mtspr	CTR, r0

  332. 	ble	LL(10)

  333. 	.align 4

  334. 

  335. LL(06):

  336. 	LFD	a1, 0 * SIZE(X)

  337. 	LFD	a2, 1 * SIZE(X)

  338. 	STFD	a1, 0 * SIZE(X1)

  339. 	STFD	a2, 1 * SIZE(X1)

  340. 

  341. 	add	X, X, INCX

  342. 	addi	X1, X1, 2 * SIZE

  343. 	bdnz+	LL(06)

  344. 	.align 4

  345. 

  346. LL(10):

  347. 	srawi.	J, N, 1

  348. 	ble	LL(20)

  349. 	.align 4

  350. 

  351. LL(11):

  352. 	LFD	alpha1_r, 0 * SIZE(Y)

  353. 	LFD	alpha1_i, 1 * SIZE(Y)

  354. 	add	Y, Y, INCY

  355. 	LFD	alpha2_r, 0 * SIZE(Y)

  356. 	LFD	alpha2_i, 1 * SIZE(Y)

  357. 	add	Y, Y, INCY

  358. 

  359. 	FMUL	a1, alpha_r, alpha1_r

  360. 	FMUL	a2, alpha_i, alpha1_r

  361. 	FMUL	a3, alpha_r, alpha2_r

  362. 	FMUL	a4, alpha_i, alpha2_r

  363. 

  364. 	FMA1	alpha1_r, alpha_i, alpha1_i, a1

  365. 	FMA2	alpha1_i, alpha_r, alpha1_i, a2

  366. 	FMA1	alpha2_r, alpha_i, alpha2_i, a3

  367. 	FMA2	alpha2_i, alpha_r, alpha2_i, a4

  368. 

  369. 	mr	AO1, A

  370. 	add	AO2, A,   LDA

  371. 	add	A,   AO2, LDA

  372. 

  373. 	mr	X1, XX

  374. 

  375. 	srawi.	r0,  M, 3

  376. 	mtspr	CTR, r0

  377. 	ble	LL(15)

  378. 

  379. 	LFD	a1,  0 * SIZE(AO1)

  380. 	LFD	a2,  1 * SIZE(AO1)

  381. 	LFD	a3,  2 * SIZE(AO1)

  382. 	LFD	a4,  3 * SIZE(AO1)

  383. 	LFD	a5,  4 * SIZE(AO1)

  384. 	LFD	a6,  5 * SIZE(AO1)

  385. 	LFD	a7,  6 * SIZE(AO1)

  386. 	LFD	a8,  7 * SIZE(AO1)

  387. 

  388.  	LFD	y01, 0 * SIZE(X1)

  389. 	LFD	y02, 1 * SIZE(X1)

  390.  	LFD	y03, 2 * SIZE(X1)

  391. 	LFD	y04, 3 * SIZE(X1)

  392.  	LFD	y05, 4 * SIZE(X1)

  393. 	LFD	y06, 5 * SIZE(X1)

  394.  	LFD	y07, 6 * SIZE(X1)

  395. 	LFD	y08, 7 * SIZE(X1)

  396. 

  397. 	LFD	a9,  0 * SIZE(AO2)

  398. 	LFD	a10, 1 * SIZE(AO2)

  399. 	LFD	a11, 2 * SIZE(AO2)

  400. 	LFD	a12, 3 * SIZE(AO2)

  401. 	LFD	a13, 4 * SIZE(AO2)

  402. 	LFD	a14, 5 * SIZE(AO2)

  403. 	LFD	a15, 6 * SIZE(AO2)

  404. 	LFD	a16, 7 * SIZE(AO2)

  405. 

  406. 	bdz	LL(13)

  407. 	.align 4

  408. 

  409. LL(12):

  410. 	FMADD	a1,  alpha1_r, y01, a1

  411. 	FMADD	a2,  alpha1_r, y02, a2

  412. 	FMADD	a3,  alpha1_r, y03, a3

  413. 	FMADD	a4,  alpha1_r, y04, a4

  414. 

  415. 	FMADD	a5,  alpha1_r, y05, a5

  416. 	FMADD	a6,  alpha1_r, y06, a6

  417. 	FMADD	a7,  alpha1_r, y07, a7

  418. 	FMADD	a8,  alpha1_r, y08, a8

  419. 

  420. 	FMADD	a9,  alpha2_r, y01, a9

  421. 	FMADD	a10, alpha2_r, y02, a10

  422. 	FMADD	a11, alpha2_r, y03, a11

  423. 	FMADD	a12, alpha2_r, y04, a12

  424. 

  425. 	FMADD	a13, alpha2_r, y05, a13

  426. 	FMADD	a14, alpha2_r, y06, a14

  427. 	FMADD	a15, alpha2_r, y07, a15

  428. 	FMADD	a16, alpha2_r, y08, a16

  429. 

  430. 	FNMSUB	a1, alpha1_i, y02, a1

  431. 	FMADD	a2, alpha1_i, y01, a2

  432. 	FNMSUB	a3, alpha1_i, y04, a3

  433. 	FMADD	a4, alpha1_i, y03, a4

  434. 

  435. 	STFD	a1,  0 * SIZE(AO1)

  436. 	STFD	a2,  1 * SIZE(AO1)

  437. 	STFD	a3,  2 * SIZE(AO1)

  438. 	STFD	a4,  3 * SIZE(AO1)

  439. 

  440. 	LFD	a1,   8 * SIZE(AO1)

  441. 	LFD	a2,   9 * SIZE(AO1)

  442. 	LFD	a3,  10 * SIZE(AO1)

  443. 	LFD	a4,  11 * SIZE(AO1)

  444. 

  445. 	FNMSUB	a5, alpha1_i, y06, a5

  446. 	FMADD	a6, alpha1_i, y05, a6

  447. 	FNMSUB	a7, alpha1_i, y08, a7

  448. 	FMADD	a8, alpha1_i, y07, a8

  449. 

  450. 	STFD	a5,  4 * SIZE(AO1)

  451. 	STFD	a6,  5 * SIZE(AO1)

  452. 	STFD	a7,  6 * SIZE(AO1)

  453. 	STFD	a8,  7 * SIZE(AO1)

  454. 

  455. 	LFD	a5,  12 * SIZE(AO1)

  456. 	LFD	a6,  13 * SIZE(AO1)

  457. 	LFD	a7,  14 * SIZE(AO1)

  458. 	LFD	a8,  15 * SIZE(AO1)

  459. 

  460. 	FNMSUB	a9,  alpha2_i, y02, a9

  461. 	FMADD	a10, alpha2_i, y01, a10

  462. 	FNMSUB	a11, alpha2_i, y04, a11

  463. 	FMADD	a12, alpha2_i, y03, a12

  464. 

  465.  	LFD	y01,  8 * SIZE(X1)

  466. 	LFD	y02,  9 * SIZE(X1)

  467.  	LFD	y03, 10 * SIZE(X1)

  468. 	LFD	y04, 11 * SIZE(X1)

  469. 

  470. 	STFD	a9,  0 * SIZE(AO2)

  471. 	STFD	a10, 1 * SIZE(AO2)

  472. 	STFD	a11, 2 * SIZE(AO2)

  473. 	STFD	a12, 3 * SIZE(AO2)

  474. 

  475. 	LFD	a9,   8 * SIZE(AO2)

  476. 	LFD	a10,  9 * SIZE(AO2)

  477. 	LFD	a11, 10 * SIZE(AO2)

  478. 	LFD	a12, 11 * SIZE(AO2)

  479. 

  480. 	FNMSUB	a13, alpha2_i, y06, a13

  481. 	FMADD	a14, alpha2_i, y05, a14

  482. 	FNMSUB	a15, alpha2_i, y08, a15

  483. 	FMADD	a16, alpha2_i, y07, a16

  484. 

  485.  	LFD	y05, 12 * SIZE(X1)

  486. 	LFD	y06, 13 * SIZE(X1)

  487.  	LFD	y07, 14 * SIZE(X1)

  488. 	LFD	y08, 15 * SIZE(X1)

  489. 

  490. 	STFD	a13, 4 * SIZE(AO2)

  491. 	STFD	a14, 5 * SIZE(AO2)

  492. 	STFD	a15, 6 * SIZE(AO2)

  493. 	STFD	a16, 7 * SIZE(AO2)

  494. 

  495. 	LFD	a13, 12 * SIZE(AO2)

  496. 	LFD	a14, 13 * SIZE(AO2)

  497. 	LFD	a15, 14 * SIZE(AO2)

  498. 	LFD	a16, 15 * SIZE(AO2)

  499. 

  500. 	FMADD	a1,  alpha1_r, y01, a1

  501. 	FMADD	a2,  alpha1_r, y02, a2

  502. 	FMADD	a3,  alpha1_r, y03, a3

  503. 	FMADD	a4,  alpha1_r, y04, a4

  504. 

  505. 	FMADD	a5,  alpha1_r, y05, a5

  506. 	FMADD	a6,  alpha1_r, y06, a6

  507. 	FMADD	a7,  alpha1_r, y07, a7

  508. 	FMADD	a8,  alpha1_r, y08, a8

  509. 

  510. 	FMADD	a9,  alpha2_r, y01, a9

  511. 	FMADD	a10, alpha2_r, y02, a10

  512. 	FMADD	a11, alpha2_r, y03, a11

  513. 	FMADD	a12, alpha2_r, y04, a12

  514. 

  515. 	FMADD	a13, alpha2_r, y05, a13

  516. 	FMADD	a14, alpha2_r, y06, a14

  517. 	FMADD	a15, alpha2_r, y07, a15

  518. 	FMADD	a16, alpha2_r, y08, a16

  519. 

  520. 	FNMSUB	a1, alpha1_i, y02, a1

  521. 	FMADD	a2, alpha1_i, y01, a2

  522. 	FNMSUB	a3, alpha1_i, y04, a3

  523. 	FMADD	a4, alpha1_i, y03, a4

  524. 

  525. 	STFD	a1,   8 * SIZE(AO1)

  526. 	STFD	a2,   9 * SIZE(AO1)

  527. 	STFD	a3,  10 * SIZE(AO1)

  528. 	STFD	a4,  11 * SIZE(AO1)

  529. 

  530. 	LFD	a1,  16 * SIZE(AO1)

  531. 	LFD	a2,  17 * SIZE(AO1)

  532. 	LFD	a3,  18 * SIZE(AO1)

  533. 	LFD	a4,  19 * SIZE(AO1)

  534. 

  535. 	FNMSUB	a5, alpha1_i, y06, a5

  536. 	FMADD	a6, alpha1_i, y05, a6

  537. 	FNMSUB	a7, alpha1_i, y08, a7

  538. 	FMADD	a8, alpha1_i, y07, a8

  539. 

  540. 	STFD	a5,  12 * SIZE(AO1)

  541. 	STFD	a6,  13 * SIZE(AO1)

  542. 	STFD	a7,  14 * SIZE(AO1)

  543. 	STFD	a8,  15 * SIZE(AO1)

  544. 

  545. 	LFD	a5,  20 * SIZE(AO1)

  546. 	LFD	a6,  21 * SIZE(AO1)

  547. 	LFD	a7,  22 * SIZE(AO1)

  548. 	LFD	a8,  23 * SIZE(AO1)

  549. 

  550. 	FNMSUB	a9,  alpha2_i, y02, a9

  551. 	FMADD	a10, alpha2_i, y01, a10

  552. 	FNMSUB	a11, alpha2_i, y04, a11

  553. 	FMADD	a12, alpha2_i, y03, a12

  554. 

  555.  	LFD	y01, 16 * SIZE(X1)

  556. 	LFD	y02, 17 * SIZE(X1)

  557.  	LFD	y03, 18 * SIZE(X1)

  558. 	LFD	y04, 19 * SIZE(X1)

  559. 

  560. 	STFD	a9,   8 * SIZE(AO2)

  561. 	STFD	a10,  9 * SIZE(AO2)

  562. 	STFD	a11, 10 * SIZE(AO2)

  563. 	STFD	a12, 11 * SIZE(AO2)

  564. 

  565. 	LFD	a9,  16 * SIZE(AO2)

  566. 	LFD	a10, 17 * SIZE(AO2)

  567. 	LFD	a11, 18 * SIZE(AO2)

  568. 	LFD	a12, 19 * SIZE(AO2)

  569. 

  570. 	FNMSUB	a13, alpha2_i, y06, a13

  571. 	FMADD	a14, alpha2_i, y05, a14

  572. 	FNMSUB	a15, alpha2_i, y08, a15

  573. 	FMADD	a16, alpha2_i, y07, a16

  574. 

  575.  	LFD	y05, 20 * SIZE(X1)

  576. 	LFD	y06, 21 * SIZE(X1)

  577.  	LFD	y07, 22 * SIZE(X1)

  578. 	LFD	y08, 23 * SIZE(X1)

  579. 

  580. 	STFD	a13, 12 * SIZE(AO2)

  581. 	STFD	a14, 13 * SIZE(AO2)

  582. 	STFD	a15, 14 * SIZE(AO2)

  583. 	STFD	a16, 15 * SIZE(AO2)

  584. 

  585. 	LFD	a13, 20 * SIZE(AO2)

  586. 	LFD	a14, 21 * SIZE(AO2)

  587. 	LFD	a15, 22 * SIZE(AO2)

  588. 	LFD	a16, 23 * SIZE(AO2)

  589. 

  590. 	addi	AO1, AO1, 16 * SIZE

  591. 	addi	AO2, AO2, 16 * SIZE

  592. 	addi	X1,  X1,  16 * SIZE

  593. 

  594. 	DCBT(AO1, PREA)

  595. 	DCBT(AO2, PREA)

  596. 	DCBT(Y1, PREY)

  597. 

  598. 	bdnz+	LL(12)

  599. 	.align 4

  600. 

  601. LL(13):

  602. 	FMADD	a1,  alpha1_r, y01, a1

  603. 	FMADD	a2,  alpha1_r, y02, a2

  604. 	FMADD	a3,  alpha1_r, y03, a3

  605. 	FMADD	a4,  alpha1_r, y04, a4

  606. 

  607. 	FMADD	a5,  alpha1_r, y05, a5

  608. 	FMADD	a6,  alpha1_r, y06, a6

  609. 	FMADD	a7,  alpha1_r, y07, a7

  610. 	FMADD	a8,  alpha1_r, y08, a8

  611. 

  612. 	FMADD	a9,  alpha2_r, y01, a9

  613. 	FMADD	a10, alpha2_r, y02, a10

  614. 	FMADD	a11, alpha2_r, y03, a11

  615. 	FMADD	a12, alpha2_r, y04, a12

  616. 

  617. 	FMADD	a13, alpha2_r, y05, a13

  618. 	FMADD	a14, alpha2_r, y06, a14

  619. 	FMADD	a15, alpha2_r, y07, a15

  620. 	FMADD	a16, alpha2_r, y08, a16

  621. 

  622. 	FNMSUB	a1, alpha1_i, y02, a1

  623. 	FMADD	a2, alpha1_i, y01, a2

  624. 	FNMSUB	a3, alpha1_i, y04, a3

  625. 	FMADD	a4, alpha1_i, y03, a4

  626. 

  627. 	STFD	a1,  0 * SIZE(AO1)

  628. 	STFD	a2,  1 * SIZE(AO1)

  629. 	STFD	a3,  2 * SIZE(AO1)

  630. 	STFD	a4,  3 * SIZE(AO1)

  631. 

  632. 	LFD	a1,   8 * SIZE(AO1)

  633. 	LFD	a2,   9 * SIZE(AO1)

  634. 	LFD	a3,  10 * SIZE(AO1)

  635. 	LFD	a4,  11 * SIZE(AO1)

  636. 

  637. 	FNMSUB	a5, alpha1_i, y06, a5

  638. 	FMADD	a6, alpha1_i, y05, a6

  639. 	FNMSUB	a7, alpha1_i, y08, a7

  640. 	FMADD	a8, alpha1_i, y07, a8

  641. 

  642. 	STFD	a5,  4 * SIZE(AO1)

  643. 	STFD	a6,  5 * SIZE(AO1)

  644. 	STFD	a7,  6 * SIZE(AO1)

  645. 	STFD	a8,  7 * SIZE(AO1)

  646. 

  647. 	LFD	a5,  12 * SIZE(AO1)

  648. 	LFD	a6,  13 * SIZE(AO1)

  649. 	LFD	a7,  14 * SIZE(AO1)

  650. 	LFD	a8,  15 * SIZE(AO1)

  651. 

  652. 	FNMSUB	a9,  alpha2_i, y02, a9

  653. 	FMADD	a10, alpha2_i, y01, a10

  654. 	FNMSUB	a11, alpha2_i, y04, a11

  655. 	FMADD	a12, alpha2_i, y03, a12

  656. 

  657.  	LFD	y01,  8 * SIZE(X1)

  658. 	LFD	y02,  9 * SIZE(X1)

  659.  	LFD	y03, 10 * SIZE(X1)

  660. 	LFD	y04, 11 * SIZE(X1)

  661. 

  662. 	STFD	a9,  0 * SIZE(AO2)

  663. 	STFD	a10, 1 * SIZE(AO2)

  664. 	STFD	a11, 2 * SIZE(AO2)

  665. 	STFD	a12, 3 * SIZE(AO2)

  666. 

  667. 	LFD	a9,   8 * SIZE(AO2)

  668. 	LFD	a10,  9 * SIZE(AO2)

  669. 	LFD	a11, 10 * SIZE(AO2)

  670. 	LFD	a12, 11 * SIZE(AO2)

  671. 

  672. 	FNMSUB	a13, alpha2_i, y06, a13

  673. 	FMADD	a14, alpha2_i, y05, a14

  674. 	FNMSUB	a15, alpha2_i, y08, a15

  675. 	FMADD	a16, alpha2_i, y07, a16

  676. 

  677.  	LFD	y05, 12 * SIZE(X1)

  678. 	LFD	y06, 13 * SIZE(X1)

  679.  	LFD	y07, 14 * SIZE(X1)

  680. 	LFD	y08, 15 * SIZE(X1)

  681. 

  682. 	STFD	a13, 4 * SIZE(AO2)

  683. 	STFD	a14, 5 * SIZE(AO2)

  684. 	STFD	a15, 6 * SIZE(AO2)

  685. 	STFD	a16, 7 * SIZE(AO2)

  686. 

  687. 	LFD	a13, 12 * SIZE(AO2)

  688. 	LFD	a14, 13 * SIZE(AO2)

  689. 	LFD	a15, 14 * SIZE(AO2)

  690. 	LFD	a16, 15 * SIZE(AO2)

  691. 

  692. 	FMADD	a1,  alpha1_r, y01, a1

  693. 	FMADD	a2,  alpha1_r, y02, a2

  694. 	FMADD	a3,  alpha1_r, y03, a3

  695. 	FMADD	a4,  alpha1_r, y04, a4

  696. 

  697. 	FMADD	a5,  alpha1_r, y05, a5

  698. 	FMADD	a6,  alpha1_r, y06, a6

  699. 	FMADD	a7,  alpha1_r, y07, a7

  700. 	FMADD	a8,  alpha1_r, y08, a8

  701. 

  702. 	FMADD	a9,  alpha2_r, y01, a9

  703. 	FMADD	a10, alpha2_r, y02, a10

  704. 	FMADD	a11, alpha2_r, y03, a11

  705. 	FMADD	a12, alpha2_r, y04, a12

  706. 

  707. 	FMADD	a13, alpha2_r, y05, a13

  708. 	FMADD	a14, alpha2_r, y06, a14

  709. 	FMADD	a15, alpha2_r, y07, a15

  710. 	FMADD	a16, alpha2_r, y08, a16

  711. 

  712. 	FNMSUB	a1, alpha1_i, y02, a1

  713. 	FMADD	a2, alpha1_i, y01, a2

  714. 	FNMSUB	a3, alpha1_i, y04, a3

  715. 	FMADD	a4, alpha1_i, y03, a4

  716. 

  717. 	STFD	a1,   8 * SIZE(AO1)

  718. 	STFD	a2,   9 * SIZE(AO1)

  719. 	STFD	a3,  10 * SIZE(AO1)

  720. 	STFD	a4,  11 * SIZE(AO1)

  721. 

  722. 	FNMSUB	a5, alpha1_i, y06, a5

  723. 	FMADD	a6, alpha1_i, y05, a6

  724. 	FNMSUB	a7, alpha1_i, y08, a7

  725. 	FMADD	a8, alpha1_i, y07, a8

  726. 

  727. 	STFD	a5,  12 * SIZE(AO1)

  728. 	STFD	a6,  13 * SIZE(AO1)

  729. 	STFD	a7,  14 * SIZE(AO1)

  730. 	STFD	a8,  15 * SIZE(AO1)

  731. 

  732. 	FNMSUB	a9,  alpha2_i, y02, a9

  733. 	FMADD	a10, alpha2_i, y01, a10

  734. 	FNMSUB	a11, alpha2_i, y04, a11

  735. 	FMADD	a12, alpha2_i, y03, a12

  736. 

  737. 	STFD	a9,   8 * SIZE(AO2)

  738. 	STFD	a10,  9 * SIZE(AO2)

  739. 	STFD	a11, 10 * SIZE(AO2)

  740. 	STFD	a12, 11 * SIZE(AO2)

  741. 

  742. 	FNMSUB	a13, alpha2_i, y06, a13

  743. 	FMADD	a14, alpha2_i, y05, a14

  744. 	FNMSUB	a15, alpha2_i, y08, a15

  745. 	FMADD	a16, alpha2_i, y07, a16

  746. 

  747. 	STFD	a13, 12 * SIZE(AO2)

  748. 	STFD	a14, 13 * SIZE(AO2)

  749. 	STFD	a15, 14 * SIZE(AO2)

  750. 	STFD	a16, 15 * SIZE(AO2)

  751. 

  752. 	addi	AO1, AO1, 16 * SIZE

  753. 	addi	AO2, AO2, 16 * SIZE

  754. 	addi	X1,  X1,  16 * SIZE

  755. 	.align 4

  756. 

  757. LL(15):

  758. 	andi.	r0, M, 7

  759. 	ble	LL(19)

  760. 

  761. 	andi.	r0, M, 4

  762. 	ble	LL(17)

  763. 

  764. 	LFD	a1,  0 * SIZE(AO1)

  765. 	LFD	a2,  1 * SIZE(AO1)

  766. 	LFD	a3,  2 * SIZE(AO1)

  767. 	LFD	a4,  3 * SIZE(AO1)

  768. 	LFD	a5,  4 * SIZE(AO1)

  769. 	LFD	a6,  5 * SIZE(AO1)

  770. 	LFD	a7,  6 * SIZE(AO1)

  771. 	LFD	a8,  7 * SIZE(AO1)

  772. 

  773.  	LFD	y01, 0 * SIZE(X1)

  774. 	LFD	y02, 1 * SIZE(X1)

  775.  	LFD	y03, 2 * SIZE(X1)

  776. 	LFD	y04, 3 * SIZE(X1)

  777.  	LFD	y05, 4 * SIZE(X1)

  778. 	LFD	y06, 5 * SIZE(X1)

  779.  	LFD	y07, 6 * SIZE(X1)

  780. 	LFD	y08, 7 * SIZE(X1)

  781. 

  782. 	LFD	a9,  0 * SIZE(AO2)

  783. 	LFD	a10, 1 * SIZE(AO2)

  784. 	LFD	a11, 2 * SIZE(AO2)

  785. 	LFD	a12, 3 * SIZE(AO2)

  786. 	LFD	a13, 4 * SIZE(AO2)

  787. 	LFD	a14, 5 * SIZE(AO2)

  788. 	LFD	a15, 6 * SIZE(AO2)

  789. 	LFD	a16, 7 * SIZE(AO2)

  790. 

  791. 	FMADD	a1,  alpha1_r, y01, a1

  792. 	FMADD	a2,  alpha1_r, y02, a2

  793. 	FMADD	a3,  alpha1_r, y03, a3

  794. 	FMADD	a4,  alpha1_r, y04, a4

  795. 

  796. 	FMADD	a5,  alpha1_r, y05, a5

  797. 	FMADD	a6,  alpha1_r, y06, a6

  798. 	FMADD	a7,  alpha1_r, y07, a7

  799. 	FMADD	a8,  alpha1_r, y08, a8

  800. 

  801. 	FMADD	a9,  alpha2_r, y01, a9

  802. 	FMADD	a10, alpha2_r, y02, a10

  803. 	FMADD	a11, alpha2_r, y03, a11

  804. 	FMADD	a12, alpha2_r, y04, a12

  805. 

  806. 	FMADD	a13, alpha2_r, y05, a13

  807. 	FMADD	a14, alpha2_r, y06, a14

  808. 	FMADD	a15, alpha2_r, y07, a15

  809. 	FMADD	a16, alpha2_r, y08, a16

  810. 

  811. 	FNMSUB	a1, alpha1_i, y02, a1

  812. 	FMADD	a2, alpha1_i, y01, a2

  813. 	FNMSUB	a3, alpha1_i, y04, a3

  814. 	FMADD	a4, alpha1_i, y03, a4

  815. 

  816. 	FNMSUB	a5, alpha1_i, y06, a5

  817. 	FMADD	a6, alpha1_i, y05, a6

  818. 	FNMSUB	a7, alpha1_i, y08, a7

  819. 	FMADD	a8, alpha1_i, y07, a8

  820. 

  821. 	FNMSUB	a9,  alpha2_i, y02, a9

  822. 	FMADD	a10, alpha2_i, y01, a10

  823. 	FNMSUB	a11, alpha2_i, y04, a11

  824. 	FMADD	a12, alpha2_i, y03, a12

  825. 

  826. 	FNMSUB	a13, alpha2_i, y06, a13

  827. 	FMADD	a14, alpha2_i, y05, a14

  828. 	FNMSUB	a15, alpha2_i, y08, a15

  829. 	FMADD	a16, alpha2_i, y07, a16

  830. 

  831. 	STFD	a1,  0 * SIZE(AO1)

  832. 	STFD	a2,  1 * SIZE(AO1)

  833. 	STFD	a3,  2 * SIZE(AO1)

  834. 	STFD	a4,  3 * SIZE(AO1)

  835. 	STFD	a5,  4 * SIZE(AO1)

  836. 	STFD	a6,  5 * SIZE(AO1)

  837. 	STFD	a7,  6 * SIZE(AO1)

  838. 	STFD	a8,  7 * SIZE(AO1)

  839. 

  840. 	STFD	a9,  0 * SIZE(AO2)

  841. 	STFD	a10, 1 * SIZE(AO2)

  842. 	STFD	a11, 2 * SIZE(AO2)

  843. 	STFD	a12, 3 * SIZE(AO2)

  844. 	STFD	a13, 4 * SIZE(AO2)

  845. 	STFD	a14, 5 * SIZE(AO2)

  846. 	STFD	a15, 6 * SIZE(AO2)

  847. 	STFD	a16, 7 * SIZE(AO2)

  848. 

  849. 	addi	AO1, AO1, 8 * SIZE

  850. 	addi	AO2, AO2, 8 * SIZE

  851. 	addi	X1,  X1,  8 * SIZE

  852. 	.align 4

  853. 

  854. LL(17):

  855. 	andi.	r0, M, 2

  856. 	ble	LL(18)

  857. 

  858. 	LFD	a1,  0 * SIZE(AO1)

  859. 	LFD	a2,  1 * SIZE(AO1)

  860. 	LFD	a3,  2 * SIZE(AO1)

  861. 	LFD	a4,  3 * SIZE(AO1)

  862. 

  863.  	LFD	y01, 0 * SIZE(X1)

  864. 	LFD	y02, 1 * SIZE(X1)

  865.  	LFD	y03, 2 * SIZE(X1)

  866. 	LFD	y04, 3 * SIZE(X1)

  867. 

  868. 	LFD	a5,  0 * SIZE(AO2)

  869. 	LFD	a6,  1 * SIZE(AO2)

  870. 	LFD	a7,  2 * SIZE(AO2)

  871. 	LFD	a8,  3 * SIZE(AO2)

  872. 

  873. 	FMADD	a1, alpha1_r, y01, a1

  874. 	FMADD	a2, alpha1_r, y02, a2

  875. 	FMADD	a3, alpha1_r, y03, a3

  876. 	FMADD	a4, alpha1_r, y04, a4

  877. 

  878. 	FMADD	a5, alpha2_r, y01, a5

  879. 	FMADD	a6, alpha2_r, y02, a6

  880. 	FMADD	a7, alpha2_r, y03, a7

  881. 	FMADD	a8, alpha2_r, y04, a8

  882. 

  883. 	FNMSUB	a1, alpha1_i, y02, a1

  884. 	FMADD	a2, alpha1_i, y01, a2

  885. 	FNMSUB	a3, alpha1_i, y04, a3

  886. 	FMADD	a4, alpha1_i, y03, a4

  887. 

  888. 	FNMSUB	a5, alpha2_i, y02, a5

  889. 	FMADD	a6, alpha2_i, y01, a6

  890. 	FNMSUB	a7, alpha2_i, y04, a7

  891. 	FMADD	a8, alpha2_i, y03, a8

  892. 

  893. 	STFD	a1,  0 * SIZE(AO1)

  894. 	STFD	a2,  1 * SIZE(AO1)

  895. 	STFD	a3,  2 * SIZE(AO1)

  896. 	STFD	a4,  3 * SIZE(AO1)

  897. 	STFD	a5,  0 * SIZE(AO2)

  898. 	STFD	a6,  1 * SIZE(AO2)

  899. 	STFD	a7,  2 * SIZE(AO2)

  900. 	STFD	a8,  3 * SIZE(AO2)

  901. 

  902. 	addi	AO1, AO1, 4 * SIZE

  903. 	addi	AO2, AO2, 4 * SIZE

  904. 	addi	X1,  X1,  4 * SIZE

  905. 	.align 4

  906. 

  907. LL(18):

  908. 	andi.	r0, M, 1

  909. 	ble	LL(19)

  910. 

  911. 	LFD	a1,  0 * SIZE(AO1)

  912. 	LFD	a2,  1 * SIZE(AO1)

  913. 	LFD	a3,  0 * SIZE(AO2)

  914. 	LFD	a4,  1 * SIZE(AO2)

  915. 

  916. 	LFD	y01, 0 * SIZE(X1)

  917. 	LFD	y02, 1 * SIZE(X1)

  918. 

  919. 	FMADD	a1, alpha1_r, y01, a1

  920. 	FMADD	a2, alpha1_r, y02, a2

  921. 	FMADD	a3, alpha2_r, y01, a3

  922. 	FMADD	a4, alpha2_r, y02, a4

  923. 

  924. 	FNMSUB	a1, alpha1_i, y02, a1

  925. 	FMADD	a2, alpha1_i, y01, a2

  926. 	FNMSUB	a3, alpha2_i, y02, a3

  927. 	FMADD	a4, alpha2_i, y01, a4

  928. 

  929. 	STFD	a1,  0 * SIZE(AO1)

  930. 	STFD	a2,  1 * SIZE(AO1)

  931. 	STFD	a3,  0 * SIZE(AO2)

  932. 	STFD	a4,  1 * SIZE(AO2)

  933. 	.align 4

  934. 

  935. LL(19):

  936. 	addi	J, J, -1

  937. 	cmpi	cr0, 0, J, 0

  938. 	bgt	LL(11)

  939. 	.align 4

  940. 

  941. LL(20):

  942. 	andi.	J, N, 1

  943. 	ble	LL(999)

  944. 

  945. 	LFD	alpha1_r, 0 * SIZE(Y)

  946. 	LFD	alpha1_i, 1 * SIZE(Y)

  947. 

  948. 	FMUL	a1, alpha_r, alpha1_r

  949. 	FMUL	a2, alpha_i, alpha1_r

  950. 

  951. 	FMA1	alpha1_r, alpha_i, alpha1_i, a1

  952. 	FMA2	alpha1_i, alpha_r, alpha1_i, a2

  953. 

  954. 	mr	AO1, A

  955. 

  956. 	mr	X1, XX

  957. 

  958. 	srawi.	r0,  M, 3

  959. 	mtspr	CTR, r0

  960. 	ble	LL(25)

  961. 

  962. 	LFD	a1,  0 * SIZE(AO1)

  963. 	LFD	a2,  1 * SIZE(AO1)

  964. 	LFD	a3,  2 * SIZE(AO1)

  965. 	LFD	a4,  3 * SIZE(AO1)

  966. 	LFD	a5,  4 * SIZE(AO1)

  967. 	LFD	a6,  5 * SIZE(AO1)

  968. 	LFD	a7,  6 * SIZE(AO1)

  969. 	LFD	a8,  7 * SIZE(AO1)

  970. 

  971.  	LFD	y01, 0 * SIZE(X1)

  972. 	LFD	y02, 1 * SIZE(X1)

  973.  	LFD	y03, 2 * SIZE(X1)

  974. 	LFD	y04, 3 * SIZE(X1)

  975.  	LFD	y05, 4 * SIZE(X1)

  976. 	LFD	y06, 5 * SIZE(X1)

  977.  	LFD	y07, 6 * SIZE(X1)

  978. 	LFD	y08, 7 * SIZE(X1)

  979. 

  980. 	bdz	LL(23)

  981. 	.align 4

  982. 

  983. LL(22):

  984. 	FMADD	a1,  alpha1_r, y01, a1

  985. 	FMADD	a2,  alpha1_r, y02, a2

  986. 	FMADD	a3,  alpha1_r, y03, a3

  987. 	FMADD	a4,  alpha1_r, y04, a4

  988. 

  989. 	FMADD	a5,  alpha1_r, y05, a5

  990. 	FMADD	a6,  alpha1_r, y06, a6

  991. 	FMADD	a7,  alpha1_r, y07, a7

  992. 	FMADD	a8,  alpha1_r, y08, a8

  993. 

  994. 	FNMSUB	a1, alpha1_i, y02, a1

  995. 	FMADD	a2, alpha1_i, y01, a2

  996. 	FNMSUB	a3, alpha1_i, y04, a3

  997. 	FMADD	a4, alpha1_i, y03, a4

  998. 

  999. 	STFD	a1,  0 * SIZE(AO1)

  1000. 	STFD	a2,  1 * SIZE(AO1)

  1001. 	STFD	a3,  2 * SIZE(AO1)

  1002. 	STFD	a4,  3 * SIZE(AO1)

  1003. 

  1004. 	LFD	a1,   8 * SIZE(AO1)

  1005. 	LFD	a2,   9 * SIZE(AO1)

  1006. 	LFD	a3,  10 * SIZE(AO1)

  1007. 	LFD	a4,  11 * SIZE(AO1)

  1008. 

  1009. 	FNMSUB	a5, alpha1_i, y06, a5

  1010. 	FMADD	a6, alpha1_i, y05, a6

  1011. 	FNMSUB	a7, alpha1_i, y08, a7

  1012. 	FMADD	a8, alpha1_i, y07, a8

  1013. 

  1014. 	STFD	a5,  4 * SIZE(AO1)

  1015. 	STFD	a6,  5 * SIZE(AO1)

  1016. 	STFD	a7,  6 * SIZE(AO1)

  1017. 	STFD	a8,  7 * SIZE(AO1)

  1018. 

  1019. 	LFD	a5,  12 * SIZE(AO1)

  1020. 	LFD	a6,  13 * SIZE(AO1)

  1021. 	LFD	a7,  14 * SIZE(AO1)

  1022. 	LFD	a8,  15 * SIZE(AO1)

  1023. 

  1024.  	LFD	y01,  8 * SIZE(X1)

  1025. 	LFD	y02,  9 * SIZE(X1)

  1026.  	LFD	y03, 10 * SIZE(X1)

  1027. 	LFD	y04, 11 * SIZE(X1)

  1028. 

  1029.  	LFD	y05, 12 * SIZE(X1)

  1030. 	LFD	y06, 13 * SIZE(X1)

  1031.  	LFD	y07, 14 * SIZE(X1)

  1032. 	LFD	y08, 15 * SIZE(X1)

  1033. 

  1034. 	FMADD	a1,  alpha1_r, y01, a1

  1035. 	FMADD	a2,  alpha1_r, y02, a2

  1036. 	FMADD	a3,  alpha1_r, y03, a3

  1037. 	FMADD	a4,  alpha1_r, y04, a4

  1038. 

  1039. 	FMADD	a5,  alpha1_r, y05, a5

  1040. 	FMADD	a6,  alpha1_r, y06, a6

  1041. 	FMADD	a7,  alpha1_r, y07, a7

  1042. 	FMADD	a8,  alpha1_r, y08, a8

  1043. 

  1044. 	FMADD	a9,  alpha2_r, y01, a9

  1045. 	FMADD	a10, alpha2_r, y02, a10

  1046. 	FMADD	a11, alpha2_r, y03, a11

  1047. 	FMADD	a12, alpha2_r, y04, a12

  1048. 

  1049. 	FMADD	a13, alpha2_r, y05, a13

  1050. 	FMADD	a14, alpha2_r, y06, a14

  1051. 	FMADD	a15, alpha2_r, y07, a15

  1052. 	FMADD	a16, alpha2_r, y08, a16

  1053. 

  1054. 	FNMSUB	a1, alpha1_i, y02, a1

  1055. 	FMADD	a2, alpha1_i, y01, a2

  1056. 	FNMSUB	a3, alpha1_i, y04, a3

  1057. 	FMADD	a4, alpha1_i, y03, a4

  1058. 

  1059. 	STFD	a1,   8 * SIZE(AO1)

  1060. 	STFD	a2,   9 * SIZE(AO1)

  1061. 	STFD	a3,  10 * SIZE(AO1)

  1062. 	STFD	a4,  11 * SIZE(AO1)

  1063. 

  1064. 	LFD	a1,  16 * SIZE(AO1)

  1065. 	LFD	a2,  17 * SIZE(AO1)

  1066. 	LFD	a3,  18 * SIZE(AO1)

  1067. 	LFD	a4,  19 * SIZE(AO1)

  1068. 

  1069. 	FNMSUB	a5, alpha1_i, y06, a5

  1070. 	FMADD	a6, alpha1_i, y05, a6

  1071. 	FNMSUB	a7, alpha1_i, y08, a7

  1072. 	FMADD	a8, alpha1_i, y07, a8

  1073. 

  1074. 	STFD	a5,  12 * SIZE(AO1)

  1075. 	STFD	a6,  13 * SIZE(AO1)

  1076. 	STFD	a7,  14 * SIZE(AO1)

  1077. 	STFD	a8,  15 * SIZE(AO1)

  1078. 

  1079. 	LFD	a5,  20 * SIZE(AO1)

  1080. 	LFD	a6,  21 * SIZE(AO1)

  1081. 	LFD	a7,  22 * SIZE(AO1)

  1082. 	LFD	a8,  23 * SIZE(AO1)

  1083. 

  1084.  	LFD	y01, 16 * SIZE(X1)

  1085. 	LFD	y02, 17 * SIZE(X1)

  1086.  	LFD	y03, 18 * SIZE(X1)

  1087. 	LFD	y04, 19 * SIZE(X1)

  1088. 

  1089.  	LFD	y05, 20 * SIZE(X1)

  1090. 	LFD	y06, 21 * SIZE(X1)

  1091.  	LFD	y07, 22 * SIZE(X1)

  1092. 	LFD	y08, 23 * SIZE(X1)

  1093. 

  1094. 	addi	AO1, AO1, 16 * SIZE

  1095. 	addi	X1,  X1,  16 * SIZE

  1096. 

  1097. 	DCBT(AO1, PREA)

  1098. 	DCBT(Y1, PREY)

  1099. 

  1100. 	bdnz+	LL(22)

  1101. 	.align 4

  1102. 

  1103. LL(23):

  1104. 	FMADD	a1,  alpha1_r, y01, a1

  1105. 	FMADD	a2,  alpha1_r, y02, a2

  1106. 	FMADD	a3,  alpha1_r, y03, a3

  1107. 	FMADD	a4,  alpha1_r, y04, a4

  1108. 

  1109. 	FMADD	a5,  alpha1_r, y05, a5

  1110. 	FMADD	a6,  alpha1_r, y06, a6

  1111. 	FMADD	a7,  alpha1_r, y07, a7

  1112. 	FMADD	a8,  alpha1_r, y08, a8

  1113. 

  1114. 	FNMSUB	a1, alpha1_i, y02, a1

  1115. 	FMADD	a2, alpha1_i, y01, a2

  1116. 	FNMSUB	a3, alpha1_i, y04, a3

  1117. 	FMADD	a4, alpha1_i, y03, a4

  1118. 

  1119. 	STFD	a1,  0 * SIZE(AO1)

  1120. 	STFD	a2,  1 * SIZE(AO1)

  1121. 	STFD	a3,  2 * SIZE(AO1)

  1122. 	STFD	a4,  3 * SIZE(AO1)

  1123. 

  1124. 	LFD	a1,   8 * SIZE(AO1)

  1125. 	LFD	a2,   9 * SIZE(AO1)

  1126. 	LFD	a3,  10 * SIZE(AO1)

  1127. 	LFD	a4,  11 * SIZE(AO1)

  1128. 

  1129. 	FNMSUB	a5, alpha1_i, y06, a5

  1130. 	FMADD	a6, alpha1_i, y05, a6

  1131. 	FNMSUB	a7, alpha1_i, y08, a7

  1132. 	FMADD	a8, alpha1_i, y07, a8

  1133. 

  1134. 	STFD	a5,  4 * SIZE(AO1)

  1135. 	STFD	a6,  5 * SIZE(AO1)

  1136. 	STFD	a7,  6 * SIZE(AO1)

  1137. 	STFD	a8,  7 * SIZE(AO1)

  1138. 

  1139. 	LFD	a5,  12 * SIZE(AO1)

  1140. 	LFD	a6,  13 * SIZE(AO1)

  1141. 	LFD	a7,  14 * SIZE(AO1)

  1142. 	LFD	a8,  15 * SIZE(AO1)

  1143. 

  1144.  	LFD	y01,  8 * SIZE(X1)

  1145. 	LFD	y02,  9 * SIZE(X1)

  1146.  	LFD	y03, 10 * SIZE(X1)

  1147. 	LFD	y04, 11 * SIZE(X1)

  1148. 

  1149.  	LFD	y05, 12 * SIZE(X1)

  1150. 	LFD	y06, 13 * SIZE(X1)

  1151.  	LFD	y07, 14 * SIZE(X1)

  1152. 	LFD	y08, 15 * SIZE(X1)

  1153. 

  1154. 	FMADD	a1,  alpha1_r, y01, a1

  1155. 	FMADD	a2,  alpha1_r, y02, a2

  1156. 	FMADD	a3,  alpha1_r, y03, a3

  1157. 	FMADD	a4,  alpha1_r, y04, a4

  1158. 

  1159. 	FMADD	a5,  alpha1_r, y05, a5

  1160. 	FMADD	a6,  alpha1_r, y06, a6

  1161. 	FMADD	a7,  alpha1_r, y07, a7

  1162. 	FMADD	a8,  alpha1_r, y08, a8

  1163. 

  1164. 	FNMSUB	a1, alpha1_i, y02, a1

  1165. 	FMADD	a2, alpha1_i, y01, a2

  1166. 	FNMSUB	a3, alpha1_i, y04, a3

  1167. 	FMADD	a4, alpha1_i, y03, a4

  1168. 

  1169. 	STFD	a1,   8 * SIZE(AO1)

  1170. 	STFD	a2,   9 * SIZE(AO1)

  1171. 	STFD	a3,  10 * SIZE(AO1)

  1172. 	STFD	a4,  11 * SIZE(AO1)

  1173. 

  1174. 	FNMSUB	a5, alpha1_i, y06, a5

  1175. 	FMADD	a6, alpha1_i, y05, a6

  1176. 	FNMSUB	a7, alpha1_i, y08, a7

  1177. 	FMADD	a8, alpha1_i, y07, a8

  1178. 

  1179. 	STFD	a5,  12 * SIZE(AO1)

  1180. 	STFD	a6,  13 * SIZE(AO1)

  1181. 	STFD	a7,  14 * SIZE(AO1)

  1182. 	STFD	a8,  15 * SIZE(AO1)

  1183. 

  1184. 	addi	AO1, AO1, 16 * SIZE

  1185. 	addi	X1,  X1,  16 * SIZE

  1186. 	.align 4

  1187. 

  1188. LL(25):

  1189. 	andi.	r0, M, 7

  1190. 	ble	LL(999)

  1191. 

  1192. 	andi.	r0, M, 4

  1193. 	ble	LL(27)

  1194. 

  1195. 	LFD	a1,  0 * SIZE(AO1)

  1196. 	LFD	a2,  1 * SIZE(AO1)

  1197. 	LFD	a3,  2 * SIZE(AO1)

  1198. 	LFD	a4,  3 * SIZE(AO1)

  1199. 	LFD	a5,  4 * SIZE(AO1)

  1200. 	LFD	a6,  5 * SIZE(AO1)

  1201. 	LFD	a7,  6 * SIZE(AO1)

  1202. 	LFD	a8,  7 * SIZE(AO1)

  1203. 

  1204.  	LFD	y01, 0 * SIZE(X1)

  1205. 	LFD	y02, 1 * SIZE(X1)

  1206.  	LFD	y03, 2 * SIZE(X1)

  1207. 	LFD	y04, 3 * SIZE(X1)

  1208.  	LFD	y05, 4 * SIZE(X1)

  1209. 	LFD	y06, 5 * SIZE(X1)

  1210.  	LFD	y07, 6 * SIZE(X1)

  1211. 	LFD	y08, 7 * SIZE(X1)

  1212. 

  1213. 	FMADD	a1,  alpha1_r, y01, a1

  1214. 	FMADD	a2,  alpha1_r, y02, a2

  1215. 	FMADD	a3,  alpha1_r, y03, a3

  1216. 	FMADD	a4,  alpha1_r, y04, a4

  1217. 

  1218. 	FMADD	a5,  alpha1_r, y05, a5

  1219. 	FMADD	a6,  alpha1_r, y06, a6

  1220. 	FMADD	a7,  alpha1_r, y07, a7

  1221. 	FMADD	a8,  alpha1_r, y08, a8

  1222. 

  1223. 	FNMSUB	a1, alpha1_i, y02, a1

  1224. 	FMADD	a2, alpha1_i, y01, a2

  1225. 	FNMSUB	a3, alpha1_i, y04, a3

  1226. 	FMADD	a4, alpha1_i, y03, a4

  1227. 

  1228. 	FNMSUB	a5, alpha1_i, y06, a5

  1229. 	FMADD	a6, alpha1_i, y05, a6

  1230. 	FNMSUB	a7, alpha1_i, y08, a7

  1231. 	FMADD	a8, alpha1_i, y07, a8

  1232. 

  1233. 	STFD	a1,  0 * SIZE(AO1)

  1234. 	STFD	a2,  1 * SIZE(AO1)

  1235. 	STFD	a3,  2 * SIZE(AO1)

  1236. 	STFD	a4,  3 * SIZE(AO1)

  1237. 	STFD	a5,  4 * SIZE(AO1)

  1238. 	STFD	a6,  5 * SIZE(AO1)

  1239. 	STFD	a7,  6 * SIZE(AO1)

  1240. 	STFD	a8,  7 * SIZE(AO1)

  1241. 

  1242. 	addi	AO1, AO1, 8 * SIZE

  1243. 	addi	X1,  X1,  8 * SIZE

  1244. 	.align 4

  1245. 

  1246. LL(27):

  1247. 	andi.	r0, M, 2

  1248. 	ble	LL(28)

  1249. 

  1250. 	LFD	a1,  0 * SIZE(AO1)

  1251. 	LFD	a2,  1 * SIZE(AO1)

  1252. 	LFD	a3,  2 * SIZE(AO1)

  1253. 	LFD	a4,  3 * SIZE(AO1)

  1254. 

  1255.  	LFD	y01, 0 * SIZE(X1)

  1256. 	LFD	y02, 1 * SIZE(X1)

  1257.  	LFD	y03, 2 * SIZE(X1)

  1258. 	LFD	y04, 3 * SIZE(X1)

  1259. 

  1260. 	FMADD	a1, alpha1_r, y01, a1

  1261. 	FMADD	a2, alpha1_r, y02, a2

  1262. 	FMADD	a3, alpha1_r, y03, a3

  1263. 	FMADD	a4, alpha1_r, y04, a4

  1264. 

  1265. 	FNMSUB	a1, alpha1_i, y02, a1

  1266. 	FMADD	a2, alpha1_i, y01, a2

  1267. 	FNMSUB	a3, alpha1_i, y04, a3

  1268. 	FMADD	a4, alpha1_i, y03, a4

  1269. 

  1270. 	STFD	a1,  0 * SIZE(AO1)

  1271. 	STFD	a2,  1 * SIZE(AO1)

  1272. 	STFD	a3,  2 * SIZE(AO1)

  1273. 	STFD	a4,  3 * SIZE(AO1)

  1274. 

  1275. 	addi	AO1, AO1, 4 * SIZE

  1276. 	addi	X1,  X1,  4 * SIZE

  1277. 	.align 4

  1278. 

  1279. LL(28):

  1280. 	andi.	r0, M, 1

  1281. 	ble	LL(999)

  1282. 

  1283. 	LFD	a1,  0 * SIZE(AO1)

  1284. 	LFD	a2,  1 * SIZE(AO1)

  1285. 

  1286. 	LFD	y01, 0 * SIZE(X1)

  1287. 	LFD	y02, 1 * SIZE(X1)

  1288. 

  1289. 	FMADD	a1, alpha1_r, y01, a1

  1290. 	FMADD	a2, alpha1_r, y02, a2

  1291. 

  1292. 	FNMSUB	a1, alpha1_i, y02, a1

  1293. 	FMADD	a2, alpha1_i, y01, a2

  1294. 

  1295. 	STFD	a1,  0 * SIZE(AO1)

  1296. 	STFD	a2,  1 * SIZE(AO1)

  1297. 	.align 4

  1298. 

  1299. LL(999):

  1300. 	li	r3, 0

  1301. 

  1302. 	lfd	f14,     0(SP)

  1303. 	lfd	f15,     8(SP)

  1304. 	lfd	f16,    16(SP)

  1305. 	lfd	f17,    24(SP)

  1306. 	lfd	f18,    32(SP)

  1307. 	lfd	f19,    40(SP)

  1308. 	lfd	f20,    48(SP)

  1309. 	lfd	f21,    56(SP)

  1310. 	lfd	f22,    64(SP)

  1311. 	lfd	f23,    72(SP)

  1312. 	lfd	f24,    80(SP)

  1313. 	lfd	f25,    88(SP)

  1314. 	lfd	f26,    96(SP)

  1315. 	lfd	f27,   104(SP)

  1316. 	lfd	f28,   112(SP)

  1317. 	lfd	f29,   120(SP)

  1318. 	lfd	f30,   128(SP)

  1319. 	lfd	f31,   136(SP)

  1320. 

  1321. #ifdef __64BIT__

  1322. 	ld	r14,   144(SP)

  1323. 	ld	r15,   152(SP)

  1324. 	ld	r16,   160(SP)

  1325. 	ld	r17,   168(SP)

  1326. 	ld	r18,   176(SP)

  1327. 	ld	r19,   184(SP)

  1328. 	ld	r20,   192(SP)

  1329. 	ld	r21,   200(SP)

  1330. 	ld	r22,   208(SP)

  1331. 	ld	r23,   216(SP)

  1332. 	ld	r24,   224(SP)

  1333. 	ld	r25,   232(SP)

  1334. 	ld	r26,   240(SP)

  1335. 	ld	r27,   248(SP)

  1336. #else

  1337. 	lwz	r14,   144(SP)

  1338. 	lwz	r15,   148(SP)

  1339. 	lwz	r16,   152(SP)

  1340. 	lwz	r17,   156(SP)

  1341. 	lwz	r18,   160(SP)

  1342. 	lwz	r19,   164(SP)

  1343. 	lwz	r20,   168(SP)

  1344. 	lwz	r21,   172(SP)

  1345. 	lwz	r22,   176(SP)

  1346. 	lwz	r23,   180(SP)

  1347. 	lwz	r24,   184(SP)

  1348. 	lwz	r25,   188(SP)

  1349. 	lwz	r26,   192(SP)

  1350. 	lwz	r27,   196(SP)

  1351. #endif

  1352. 

  1353. 	addi	SP, SP, STACKSIZE

  1354. 	blr

  1355. 

  1356. 	EPILOGUE

  1357. #endif