OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/zgemv_n.S

4301 lines
87 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(linux) || defined(__FreeBSD__)

  43. #ifndef __64BIT__

  44. #define M	r3

  45. #define	N	r4

  46. #define A	r6

  47. #define LDA	r7

  48. #define X	r8

  49. #define	INCX	r9

  50. #define	Y	r10

  51. #define	INCY	r5

  52. #else

  53. #define M	r3

  54. #define	N	r4

  55. #define A	r8

  56. #define LDA	r9

  57. #define X	r10

  58. #define	INCX	r5

  59. #define	Y	r6

  60. #define	INCY	r7

  61. #endif

  62. #endif

  63. 

  64. #if defined(_AIX) || defined(__APPLE__)

  65. #if !defined(__64BIT__) && defined(DOUBLE)

  66. #define M	r3

  67. #define	N	r4

  68. #define A	r10

  69. #define LDA	r5

  70. #define X	r6

  71. #define	INCX	r7

  72. #define	Y	r8

  73. #define	INCY	r9

  74. #else

  75. #define M	r3

  76. #define	N	r4

  77. #define A	r8

  78. #define LDA	r9

  79. #define X	r10

  80. #define	INCX	r5

  81. #define	Y	r6

  82. #define	INCY	r7

  83. #endif

  84. #endif

  85. 

  86. #define I	r11

  87. #define	J	r12

  88. 

  89. #define AO1	r14

  90. #define AO2	r15

  91. #define AO3	r16

  92. #define AO4	r17

  93. #define LDA4	r18

  94. 

  95. #define	Y1	r19

  96. #define Y2	r20

  97. #define	PREA	r21

  98. #define	PREC	r22

  99. 

  100. #define y01 f0

  101. #define y02 f1

  102. #define y03 f2

  103. #define y04 f3

  104. #define y05 f4

  105. #define y06 f5

  106. #define y07 f6

  107. #define y08 f7

  108. #define y09 f8

  109. #define y10 f9

  110. #define y11 f10

  111. #define y12 f11

  112. #define y13 f12

  113. #define y14 f13

  114. #define y15 f14

  115. #define y16 f15

  116. 

  117. #define alpha1r f16

  118. #define alpha1i f17

  119. #define alpha2r f18

  120. #define alpha2i f19

  121. #define alpha3r f20

  122. #define alpha3i f21

  123. #define alpha4r f22

  124. #define alpha4i f23

  125. 

  126. #define a1     f24

  127. #define a2     f25

  128. #define a3     f26

  129. #define a4     f27

  130. #define a5     f28

  131. #define a6     f29

  132. #define a7     f30

  133. #define a8     f31

  134. 

  135. #define alpha_r f14

  136. #define alpha_i f15

  137. 

  138. #if defined(PPCG4)

  139. #define PREFETCHSIZE_A  34

  140. #define PREFETCHSIZE_C  16

  141. #endif

  142. 

  143. #if defined(PPC440) || defined(PPC440FP2)

  144. #define PREFETCHSIZE_A  34

  145. #define PREFETCHSIZE_C  16

  146. #endif

  147. 

  148. #ifdef PPC970

  149. #define PREFETCHSIZE_A  56

  150. #define PREFETCHSIZE_C  16

  151. #endif

  152. 

  153. #ifdef CELL

  154. #define PREFETCHSIZE_A  56

  155. #define PREFETCHSIZE_C  16

  156. #endif

  157. 

  158. #ifdef POWER3

  159. #define PREFETCHSIZE_A  34

  160. #define PREFETCHSIZE_C  16

  161. #endif

  162. 

  163. #ifdef POWER4

  164. #define PREFETCHSIZE_A  34

  165. #define PREFETCHSIZE_C  16

  166. #endif

  167. 

  168. #ifdef POWER5

  169. #define PREFETCHSIZE_A  40

  170. #define PREFETCHSIZE_C  24

  171. #endif

  172. 

  173. #ifdef POWER6

  174. #define PREFETCHSIZE_A  24

  175. #define PREFETCHSIZE_C  24

  176. #endif

  177. 

  178. #ifdef POWER8

  179. #define PREFETCHSIZE_A  24

  180. #define PREFETCHSIZE_C  24

  181. #endif

  182. 

  183. #ifndef XCONJ

  184. #define FMADDR FMADD

  185. #define FMSUBR FNMSUB

  186. #else

  187. #define FMADDR FNMSUB

  188. #define FMSUBR FMADD

  189. #endif

  190. 

  191. #ifndef  CONJ

  192. #define FMADDX FMADD

  193. #define FMSUBX FNMSUB

  194. #else

  195. #define FMADDX FNMSUB

  196. #define FMSUBX FMADD

  197. #endif

  198. 

  199. #ifndef NEEDPARAM

  200. 

  201. #ifndef __64BIT__

  202. #define STACKSIZE 224

  203. #define ALPHA_R	  208(SP)

  204. #define ALPHA_I	  216(SP)

  205. #else

  206. #define STACKSIZE 280

  207. #define ALPHA_R   256(SP)

  208. #define	ALPHA_I	  264(SP)

  209. #endif

  210. 

  211. 	PROLOGUE

  212. 	PROFCODE

  213. 

  214. 	addi	SP,   SP, -STACKSIZE

  215. 	li	r0,   0

  216. 

  217. 	stfd	f14,     0(SP)

  218. 	stfd	f15,     8(SP)

  219. 	stfd	f16,    16(SP)

  220. 	stfd	f17,    24(SP)

  221. 	stfd	f18,    32(SP)

  222. 	stfd	f19,    40(SP)

  223. 	stfd	f20,    48(SP)

  224. 	stfd	f21,    56(SP)

  225. 	stfd	f22,    64(SP)

  226. 	stfd	f23,    72(SP)

  227. 	stfd	f24,    80(SP)

  228. 	stfd	f25,    88(SP)

  229. 	stfd	f26,    96(SP)

  230. 	stfd	f27,   104(SP)

  231. 	stfd	f28,   112(SP)

  232. 	stfd	f29,   120(SP)

  233. 	stfd	f30,   128(SP)

  234. 	stfd	f31,   136(SP)

  235. 

  236. #ifdef __64BIT__

  237. 	std	r14,   144(SP)

  238. 	std	r15,   152(SP)

  239. 	std	r16,   160(SP)

  240. 	std	r17,   168(SP)

  241. 	std	r18,   176(SP)

  242. 	std	r19,   184(SP)

  243. 	std	r20,   192(SP)

  244. 	std	r21,   200(SP)

  245. 	std	r22,   208(SP)

  246. #else

  247. 	stw	r14,   144(SP)

  248. 	stw	r15,   148(SP)

  249. 	stw	r16,   152(SP)

  250. 	stw	r17,   156(SP)

  251. 	stw	r18,   160(SP)

  252. 	stw	r19,   164(SP)

  253. 	stw	r20,   168(SP)

  254. 	stw	r21,   172(SP)

  255. 	stw	r22,   176(SP)

  256. #endif

  257. 

  258. #if defined(linux) || defined(__FreeBSD__)

  259. #ifndef __64BIT__

  260. 	lwz	INCY,  FRAMESLOT(0) + STACKSIZE(SP)

  261. #else

  262. 	ld	INCX,  FRAMESLOT(0) + STACKSIZE(SP)

  263. 	ld	Y,     FRAMESLOT(1) + STACKSIZE(SP)

  264. 	ld	INCY,  FRAMESLOT(2) + STACKSIZE(SP)

  265. #endif

  266. #endif

  267. 

  268. #if defined(_AIX) || defined(__APPLE__)

  269. #ifndef __64BIT__

  270. #ifdef DOUBLE

  271. 	lwz	LDA,   FRAMESLOT(0) + STACKSIZE(SP)

  272. 	lwz	X,     FRAMESLOT(1) + STACKSIZE(SP)

  273. 	lwz	INCX,  FRAMESLOT(2) + STACKSIZE(SP)

  274. 	lwz	Y,     FRAMESLOT(3) + STACKSIZE(SP)

  275. 	lwz	INCY,  FRAMESLOT(4) + STACKSIZE(SP)

  276. #else

  277. 	lwz	INCX,  FRAMESLOT(0) + STACKSIZE(SP)

  278. 	lwz	Y,     FRAMESLOT(1) + STACKSIZE(SP)

  279. 	lwz	INCY,  FRAMESLOT(2) + STACKSIZE(SP)

  280. #endif

  281. #else

  282. 	ld	INCX,  FRAMESLOT(0) + STACKSIZE(SP)

  283. 	ld	Y,     FRAMESLOT(1) + STACKSIZE(SP)

  284. 	ld	INCY,  FRAMESLOT(2) + STACKSIZE(SP)

  285. #endif

  286. #endif

  287. 

  288. 	stfd	f1, ALPHA_R

  289. 	stfd	f2, ALPHA_I

  290. 

  291. 	slwi	LDA4, LDA,  ZBASE_SHIFT + 2

  292. 	slwi	LDA,  LDA,  ZBASE_SHIFT

  293. 	slwi	INCX, INCX, ZBASE_SHIFT

  294. 	slwi	INCY, INCY, ZBASE_SHIFT

  295. 

  296. 	li	PREA, PREFETCHSIZE_A * SIZE

  297. 	li	PREC, PREFETCHSIZE_C * SIZE

  298. 

  299. 	cmpwi	cr0, M, 0

  300. 	ble-	LL(999)

  301. 

  302. 	cmpwi	cr0, N, 0

  303. 	ble-	LL(999)

  304. 

  305. 	cmpi	cr0, 0, INCY, 2 * SIZE

  306. 	bne	LL(100)

  307. 

  308. 	srawi.	J, N, 2

  309. 	ble	LL(20)

  310. 	.align 4

  311. 

  312. LL(11):

  313. 	lfd	alpha_r, ALPHA_R

  314. 	lfd	alpha_i, ALPHA_I

  315. 

  316. 	LFD	a1, 0 * SIZE(X)

  317. 	LFD	a2, 1 * SIZE(X)

  318. 	add	X, X, INCX

  319. 	LFD	a3, 0 * SIZE(X)

  320. 	LFD	a4, 1 * SIZE(X)

  321. 	add	X, X, INCX

  322. 	LFD	a5, 0 * SIZE(X)

  323. 	LFD	a6, 1 * SIZE(X)

  324. 	add	X, X, INCX

  325. 	LFD	a7, 0 * SIZE(X)

  326. 	LFD	a8, 1 * SIZE(X)

  327. 	add	X, X, INCX

  328. 

  329. 	FMUL	alpha1r, alpha_r, a1

  330. 	FMUL	alpha1i, alpha_i, a1

  331. 	FMUL	alpha2r, alpha_r, a3

  332. 	FMUL	alpha2i, alpha_i, a3

  333. 	FMUL	alpha3r, alpha_r, a5

  334. 	FMUL	alpha3i, alpha_i, a5

  335. 	FMUL	alpha4r, alpha_r, a7

  336. 	FMUL	alpha4i, alpha_i, a7

  337. 

  338. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  339. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  340. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  341. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  342. 	FMSUBR	alpha3r, alpha_i, a6, alpha3r

  343. 	FMADDR	alpha3i, alpha_r, a6, alpha3i

  344. 	FMSUBR	alpha4r, alpha_i, a8, alpha4r

  345. 	FMADDR	alpha4i, alpha_r, a8, alpha4i

  346. 

  347. 	mr	AO1, A

  348. 	add	AO2, A,   LDA

  349. 	add	AO3, AO2, LDA

  350. 	add	AO4, AO3, LDA

  351. 	add	A,   AO4, LDA

  352. 

  353. 	mr	Y1, Y

  354. 	mr	Y2, Y

  355. 

  356. 	srawi.	r0,  M, 3

  357. 	mtspr	CTR, r0

  358. 	ble	LL(15)

  359. 	.align 4

  360. 

  361. 	LFD	a1,  0 * SIZE(AO1)

  362. 	LFD	a2,  1 * SIZE(AO1)

  363. 	LFD	a3,  2 * SIZE(AO1)

  364. 	LFD	a4,  3 * SIZE(AO1)

  365. 

  366. 	LFD	y01,  0 * SIZE(Y1)

  367. 	LFD	y02,  1 * SIZE(Y1)

  368. 	LFD	y03,  2 * SIZE(Y1)

  369. 	LFD	y04,  3 * SIZE(Y1)

  370. 

  371. 	LFD	a5,  4 * SIZE(AO1)

  372. 	LFD	a6,  5 * SIZE(AO1)

  373. 	LFD	a7,  6 * SIZE(AO1)

  374. 	LFD	a8,  7 * SIZE(AO1)

  375. 

  376. 	LFD	y05,  4 * SIZE(Y1)

  377. 	LFD	y06,  5 * SIZE(Y1)

  378. 	LFD	y07,  6 * SIZE(Y1)

  379. 	LFD	y08,  7 * SIZE(Y1)

  380. 	LFD	y09,  8 * SIZE(Y1)

  381. 	LFD	y10,  9 * SIZE(Y1)

  382. 	LFD	y11, 10 * SIZE(Y1)

  383. 	LFD	y12, 11 * SIZE(Y1)

  384. 	LFD	y13, 12 * SIZE(Y1)

  385. 	LFD	y14, 13 * SIZE(Y1)

  386. 	LFD	y15, 14 * SIZE(Y1)

  387. 	LFD	y16, 15 * SIZE(Y1)

  388. 	addi	Y1, Y1, 16 * SIZE

  389. 

  390. 	bdz	LL(13)

  391. 	.align 4

  392. 

  393. LL(12):

  394. 	FMADD	y01, alpha1r, a1, y01

  395. 	FMADD	y02, alpha1i, a1, y02

  396. 	FMADD	y03, alpha1r, a3, y03

  397. 	FMADD	y04, alpha1i, a3, y04

  398. 

  399. 	FMADD	y05, alpha1r, a5, y05

  400. 	FMADD	y06, alpha1i, a5, y06

  401. 	FMADD	y07, alpha1r, a7, y07

  402. 	FMADD	y08, alpha1i, a7, y08

  403. 

  404. 	LFD	a1,  8 * SIZE(AO1)

  405. 	LFD	a3, 10 * SIZE(AO1)

  406. 	LFD	a5, 12 * SIZE(AO1)

  407. 	LFD	a7, 14 * SIZE(AO1)

  408. 

  409. 	FMSUBX	y01, alpha1i, a2, y01

  410. 	FMADDX	y02, alpha1r, a2, y02

  411. 	FMSUBX	y03, alpha1i, a4, y03

  412. 	FMADDX	y04, alpha1r, a4, y04

  413. 

  414. 	FMSUBX	y05, alpha1i, a6, y05

  415. 	FMADDX	y06, alpha1r, a6, y06

  416. 	FMSUBX	y07, alpha1i, a8, y07

  417. 	FMADDX	y08, alpha1r, a8, y08

  418. 

  419. 	LFD	a2,  9 * SIZE(AO1)

  420. 	LFD	a4, 11 * SIZE(AO1)

  421. 	LFD	a6, 13 * SIZE(AO1)

  422. 	LFD	a8, 15 * SIZE(AO1)

  423. 

  424. 	addi	AO1, AO1, 16 * SIZE

  425. 	nop

  426. 	DCBT(AO1, PREA)

  427. 	nop

  428. 

  429. 	FMADD	y09, alpha1r, a1, y09

  430. 	FMADD	y10, alpha1i, a1, y10

  431. 	FMADD	y11, alpha1r, a3, y11

  432. 	FMADD	y12, alpha1i, a3, y12

  433. 

  434. 	FMADD	y13, alpha1r, a5, y13

  435. 	FMADD	y14, alpha1i, a5, y14

  436. 	FMADD	y15, alpha1r, a7, y15

  437. 	FMADD	y16, alpha1i, a7, y16

  438. 

  439. 	LFD	a1,  0 * SIZE(AO2)

  440. 	LFD	a3,  2 * SIZE(AO2)

  441. 	LFD	a5,  4 * SIZE(AO2)

  442. 	LFD	a7,  6 * SIZE(AO2)

  443. 

  444. 	FMSUBX	y09, alpha1i, a2, y09

  445. 	FMADDX	y10, alpha1r, a2, y10

  446. 	FMSUBX	y11, alpha1i, a4, y11

  447. 	FMADDX	y12, alpha1r, a4, y12

  448. 

  449. 	FMSUBX	y13, alpha1i, a6, y13

  450. 	FMADDX	y14, alpha1r, a6, y14

  451. 	FMSUBX	y15, alpha1i, a8, y15

  452. 	FMADDX	y16, alpha1r, a8, y16

  453. 

  454. 	LFD	a2,  1 * SIZE(AO2)

  455. 	LFD	a4,  3 * SIZE(AO2)

  456. 	LFD	a6,  5 * SIZE(AO2)

  457. 	LFD	a8,  7 * SIZE(AO2)

  458. 

  459. 	FMADD	y01, alpha2r, a1, y01

  460. 	FMADD	y02, alpha2i, a1, y02

  461. 	FMADD	y03, alpha2r, a3, y03

  462. 	FMADD	y04, alpha2i, a3, y04

  463. 

  464. 	FMADD	y05, alpha2r, a5, y05

  465. 	FMADD	y06, alpha2i, a5, y06

  466. 	FMADD	y07, alpha2r, a7, y07

  467. 	FMADD	y08, alpha2i, a7, y08

  468. 

  469. 	LFD	a1,  8 * SIZE(AO2)

  470. 	LFD	a3, 10 * SIZE(AO2)

  471. 	LFD	a5, 12 * SIZE(AO2)

  472. 	LFD	a7, 14 * SIZE(AO2)

  473. 

  474. 	FMSUBX	y01, alpha2i, a2, y01

  475. 	FMADDX	y02, alpha2r, a2, y02

  476. 	FMSUBX	y03, alpha2i, a4, y03

  477. 	FMADDX	y04, alpha2r, a4, y04

  478. 

  479. 	FMSUBX	y05, alpha2i, a6, y05

  480. 	FMADDX	y06, alpha2r, a6, y06

  481. 	FMSUBX	y07, alpha2i, a8, y07

  482. 	FMADDX	y08, alpha2r, a8, y08

  483. 

  484. 	LFD	a2,  9 * SIZE(AO2)

  485. 	LFD	a4, 11 * SIZE(AO2)

  486. 	LFD	a6, 13 * SIZE(AO2)

  487. 	LFD	a8, 15 * SIZE(AO2)

  488. 

  489. 	addi	AO2, AO2, 16 * SIZE

  490. 	nop

  491. 	DCBT(AO2, PREA)

  492. 	nop

  493. 

  494. 	FMADD	y09, alpha2r, a1, y09

  495. 	FMADD	y10, alpha2i, a1, y10

  496. 	FMADD	y11, alpha2r, a3, y11

  497. 	FMADD	y12, alpha2i, a3, y12

  498. 

  499. 	FMADD	y13, alpha2r, a5, y13

  500. 	FMADD	y14, alpha2i, a5, y14

  501. 	FMADD	y15, alpha2r, a7, y15

  502. 	FMADD	y16, alpha2i, a7, y16

  503. 

  504. 	LFD	a1,  0 * SIZE(AO3)

  505. 	LFD	a3,  2 * SIZE(AO3)

  506. 	LFD	a5,  4 * SIZE(AO3)

  507. 	LFD	a7,  6 * SIZE(AO3)

  508. 

  509. 	FMSUBX	y09, alpha2i, a2, y09

  510. 	FMADDX	y10, alpha2r, a2, y10

  511. 	FMSUBX	y11, alpha2i, a4, y11

  512. 	FMADDX	y12, alpha2r, a4, y12

  513. 

  514. 	FMSUBX	y13, alpha2i, a6, y13

  515. 	FMADDX	y14, alpha2r, a6, y14

  516. 	FMSUBX	y15, alpha2i, a8, y15

  517. 	FMADDX	y16, alpha2r, a8, y16

  518. 

  519. 	LFD	a2,  1 * SIZE(AO3)

  520. 	LFD	a4,  3 * SIZE(AO3)

  521. 	LFD	a6,  5 * SIZE(AO3)

  522. 	LFD	a8,  7 * SIZE(AO3)

  523. 

  524. 	FMADD	y01, alpha3r, a1, y01

  525. 	FMADD	y02, alpha3i, a1, y02

  526. 	FMADD	y03, alpha3r, a3, y03

  527. 	FMADD	y04, alpha3i, a3, y04

  528. 

  529. 	FMADD	y05, alpha3r, a5, y05

  530. 	FMADD	y06, alpha3i, a5, y06

  531. 	FMADD	y07, alpha3r, a7, y07

  532. 	FMADD	y08, alpha3i, a7, y08

  533. 

  534. 	LFD	a1,  8 * SIZE(AO3)

  535. 	LFD	a3, 10 * SIZE(AO3)

  536. 	LFD	a5, 12 * SIZE(AO3)

  537. 	LFD	a7, 14 * SIZE(AO3)

  538. 

  539. 	FMSUBX	y01, alpha3i, a2, y01

  540. 	FMADDX	y02, alpha3r, a2, y02

  541. 	FMSUBX	y03, alpha3i, a4, y03

  542. 	FMADDX	y04, alpha3r, a4, y04

  543. 

  544. 	FMSUBX	y05, alpha3i, a6, y05

  545. 	FMADDX	y06, alpha3r, a6, y06

  546. 	FMSUBX	y07, alpha3i, a8, y07

  547. 	FMADDX	y08, alpha3r, a8, y08

  548. 

  549. 	LFD	a2,  9 * SIZE(AO3)

  550. 	LFD	a4, 11 * SIZE(AO3)

  551. 	LFD	a6, 13 * SIZE(AO3)

  552. 	LFD	a8, 15 * SIZE(AO3)

  553. 

  554. 	addi	AO3, AO3, 16 * SIZE

  555. 	nop

  556. 	DCBT(AO3, PREA)

  557. 	nop

  558. 

  559. 	FMADD	y09, alpha3r, a1, y09

  560. 	FMADD	y10, alpha3i, a1, y10

  561. 	FMADD	y11, alpha3r, a3, y11

  562. 	FMADD	y12, alpha3i, a3, y12

  563. 

  564. 	FMADD	y13, alpha3r, a5, y13

  565. 	FMADD	y14, alpha3i, a5, y14

  566. 	FMADD	y15, alpha3r, a7, y15

  567. 	FMADD	y16, alpha3i, a7, y16

  568. 

  569. 	LFD	a1,  0 * SIZE(AO4)

  570. 	LFD	a3,  2 * SIZE(AO4)

  571. 	LFD	a5,  4 * SIZE(AO4)

  572. 	LFD	a7,  6 * SIZE(AO4)

  573. 

  574. 	FMSUBX	y09, alpha3i, a2, y09

  575. 	FMADDX	y10, alpha3r, a2, y10

  576. 	FMSUBX	y11, alpha3i, a4, y11

  577. 	FMADDX	y12, alpha3r, a4, y12

  578. 

  579. 	FMSUBX	y13, alpha3i, a6, y13

  580. 	FMADDX	y14, alpha3r, a6, y14

  581. 	FMSUBX	y15, alpha3i, a8, y15

  582. 	FMADDX	y16, alpha3r, a8, y16

  583. 

  584. 	LFD	a2,  1 * SIZE(AO4)

  585. 	LFD	a4,  3 * SIZE(AO4)

  586. 	LFD	a6,  5 * SIZE(AO4)

  587. 	LFD	a8,  7 * SIZE(AO4)

  588. 

  589. 	FMADD	y01, alpha4r, a1, y01

  590. 	FMADD	y02, alpha4i, a1, y02

  591. 	FMADD	y03, alpha4r, a3, y03

  592. 	FMADD	y04, alpha4i, a3, y04

  593. 

  594. 	FMADD	y05, alpha4r, a5, y05

  595. 	FMADD	y06, alpha4i, a5, y06

  596. 	FMADD	y07, alpha4r, a7, y07

  597. 	FMADD	y08, alpha4i, a7, y08

  598. 

  599. 	LFD	a1,  8 * SIZE(AO4)

  600. 	LFD	a3, 10 * SIZE(AO4)

  601. 	LFD	a5, 12 * SIZE(AO4)

  602. 	LFD	a7, 14 * SIZE(AO4)

  603. 

  604. 	FMSUBX	y01, alpha4i, a2, y01

  605. 	FMADDX	y02, alpha4r, a2, y02

  606. 	FMSUBX	y03, alpha4i, a4, y03

  607. 	FMADDX	y04, alpha4r, a4, y04

  608. 

  609. 	STFD	y01,  0 * SIZE(Y2)

  610. 	STFD	y02,  1 * SIZE(Y2)

  611. 	STFD	y03,  2 * SIZE(Y2)

  612. 	STFD	y04,  3 * SIZE(Y2)

  613. 

  614. 	LFD	y01,  0 * SIZE(Y1)

  615. 	LFD	y02,  1 * SIZE(Y1)

  616. 	LFD	y03,  2 * SIZE(Y1)

  617. 	LFD	y04,  3 * SIZE(Y1)

  618. 

  619. 	FMSUBX	y05, alpha4i, a6, y05

  620. 	FMADDX	y06, alpha4r, a6, y06

  621. 	FMSUBX	y07, alpha4i, a8, y07

  622. 	FMADDX	y08, alpha4r, a8, y08

  623. 

  624. 	LFD	a2,  9 * SIZE(AO4)

  625. 	LFD	a4, 11 * SIZE(AO4)

  626. 	LFD	a6, 13 * SIZE(AO4)

  627. 	LFD	a8, 15 * SIZE(AO4)

  628. 

  629. 	addi	AO4, AO4, 16 * SIZE

  630. 	nop

  631. 	DCBT(AO4, PREA)

  632. 	nop

  633. 

  634. 	STFD	y05,  4 * SIZE(Y2)

  635. 	STFD	y06,  5 * SIZE(Y2)

  636. 	STFD	y07,  6 * SIZE(Y2)

  637. 	STFD	y08,  7 * SIZE(Y2)

  638. 

  639. 	LFD	y05,  4 * SIZE(Y1)

  640. 	LFD	y06,  5 * SIZE(Y1)

  641. 	LFD	y07,  6 * SIZE(Y1)

  642. 	LFD	y08,  7 * SIZE(Y1)

  643. 

  644. 	FMADD	y09, alpha4r, a1, y09

  645. 	FMADD	y10, alpha4i, a1, y10

  646. 	FMADD	y11, alpha4r, a3, y11

  647. 	FMADD	y12, alpha4i, a3, y12

  648. 

  649. 	FMADD	y13, alpha4r, a5, y13

  650. 	FMADD	y14, alpha4i, a5, y14

  651. 	FMADD	y15, alpha4r, a7, y15

  652. 	FMADD	y16, alpha4i, a7, y16

  653. 

  654. 	LFD	a1,  0 * SIZE(AO1)

  655. 	LFD	a3,  2 * SIZE(AO1)

  656. 	LFD	a5,  4 * SIZE(AO1)

  657. 	LFD	a7,  6 * SIZE(AO1)

  658. 

  659. 	FMSUBX	y09, alpha4i, a2, y09

  660. 	FMADDX	y10, alpha4r, a2, y10

  661. 	FMSUBX	y11, alpha4i, a4, y11

  662. 	FMADDX	y12, alpha4r, a4, y12

  663. 

  664. 	STFD	y09,  8 * SIZE(Y2)

  665. 	STFD	y10,  9 * SIZE(Y2)

  666. 	STFD	y11, 10 * SIZE(Y2)

  667. 	STFD	y12, 11 * SIZE(Y2)

  668. 

  669. 	LFD	y09,  8 * SIZE(Y1)

  670. 	LFD	y10,  9 * SIZE(Y1)

  671. 	LFD	y11, 10 * SIZE(Y1)

  672. 	LFD	y12, 11 * SIZE(Y1)

  673. 

  674. 	FMSUBX	y13, alpha4i, a6, y13

  675. 	FMADDX	y14, alpha4r, a6, y14

  676. 	FMSUBX	y15, alpha4i, a8, y15

  677. 	FMADDX	y16, alpha4r, a8, y16

  678. 

  679. 	LFD	a2,  1 * SIZE(AO1)

  680. 	LFD	a4,  3 * SIZE(AO1)

  681. 	LFD	a6,  5 * SIZE(AO1)

  682. 	LFD	a8,  7 * SIZE(AO1)

  683. 

  684. 	STFD	y13, 12 * SIZE(Y2)

  685. 	STFD	y14, 13 * SIZE(Y2)

  686. 	STFD	y15, 14 * SIZE(Y2)

  687. 	STFD	y16, 15 * SIZE(Y2)

  688. 

  689. 	LFD	y13, 12 * SIZE(Y1)

  690. 	LFD	y14, 13 * SIZE(Y1)

  691. 	LFD	y15, 14 * SIZE(Y1)

  692. 	LFD	y16, 15 * SIZE(Y1)

  693. 

  694. 	addi	Y2, Y2, 16 * SIZE

  695. 	addi	Y1, Y1, 16 * SIZE

  696. 	DCBT(Y1, PREC)

  697. 	bdnz	LL(12)

  698. 	.align 4

  699. 

  700. LL(13):

  701. 	FMADD	y01, alpha1r, a1, y01

  702. 	FMADD	y02, alpha1i, a1, y02

  703. 	FMADD	y03, alpha1r, a3, y03

  704. 	FMADD	y04, alpha1i, a3, y04

  705. 

  706. 	FMADD	y05, alpha1r, a5, y05

  707. 	FMADD	y06, alpha1i, a5, y06

  708. 	FMADD	y07, alpha1r, a7, y07

  709. 	FMADD	y08, alpha1i, a7, y08

  710. 

  711. 	LFD	a1,  8 * SIZE(AO1)

  712. 	LFD	a3, 10 * SIZE(AO1)

  713. 	LFD	a5, 12 * SIZE(AO1)

  714. 	LFD	a7, 14 * SIZE(AO1)

  715. 

  716. 	FMSUBX	y01, alpha1i, a2, y01

  717. 	FMADDX	y02, alpha1r, a2, y02

  718. 	FMSUBX	y03, alpha1i, a4, y03

  719. 	FMADDX	y04, alpha1r, a4, y04

  720. 

  721. 	FMSUBX	y05, alpha1i, a6, y05

  722. 	FMADDX	y06, alpha1r, a6, y06

  723. 	FMSUBX	y07, alpha1i, a8, y07

  724. 	FMADDX	y08, alpha1r, a8, y08

  725. 

  726. 	LFD	a2,  9 * SIZE(AO1)

  727. 	LFD	a4, 11 * SIZE(AO1)

  728. 	LFD	a6, 13 * SIZE(AO1)

  729. 	LFD	a8, 15 * SIZE(AO1)

  730. 

  731. 	FMADD	y09, alpha1r, a1, y09

  732. 	FMADD	y10, alpha1i, a1, y10

  733. 	FMADD	y11, alpha1r, a3, y11

  734. 	FMADD	y12, alpha1i, a3, y12

  735. 

  736. 	FMADD	y13, alpha1r, a5, y13

  737. 	FMADD	y14, alpha1i, a5, y14

  738. 	FMADD	y15, alpha1r, a7, y15

  739. 	FMADD	y16, alpha1i, a7, y16

  740. 

  741. 	LFD	a1,  0 * SIZE(AO2)

  742. 	LFD	a3,  2 * SIZE(AO2)

  743. 	LFD	a5,  4 * SIZE(AO2)

  744. 	LFD	a7,  6 * SIZE(AO2)

  745. 

  746. 	FMSUBX	y09, alpha1i, a2, y09

  747. 	FMADDX	y10, alpha1r, a2, y10

  748. 	FMSUBX	y11, alpha1i, a4, y11

  749. 	FMADDX	y12, alpha1r, a4, y12

  750. 

  751. 	FMSUBX	y13, alpha1i, a6, y13

  752. 	FMADDX	y14, alpha1r, a6, y14

  753. 	FMSUBX	y15, alpha1i, a8, y15

  754. 	FMADDX	y16, alpha1r, a8, y16

  755. 

  756. 	LFD	a2,  1 * SIZE(AO2)

  757. 	LFD	a4,  3 * SIZE(AO2)

  758. 	LFD	a6,  5 * SIZE(AO2)

  759. 	LFD	a8,  7 * SIZE(AO2)

  760. 

  761. 	FMADD	y01, alpha2r, a1, y01

  762. 	FMADD	y02, alpha2i, a1, y02

  763. 	FMADD	y03, alpha2r, a3, y03

  764. 	FMADD	y04, alpha2i, a3, y04

  765. 

  766. 	FMADD	y05, alpha2r, a5, y05

  767. 	FMADD	y06, alpha2i, a5, y06

  768. 	FMADD	y07, alpha2r, a7, y07

  769. 	FMADD	y08, alpha2i, a7, y08

  770. 

  771. 	LFD	a1,  8 * SIZE(AO2)

  772. 	LFD	a3, 10 * SIZE(AO2)

  773. 	LFD	a5, 12 * SIZE(AO2)

  774. 	LFD	a7, 14 * SIZE(AO2)

  775. 

  776. 	FMSUBX	y01, alpha2i, a2, y01

  777. 	FMADDX	y02, alpha2r, a2, y02

  778. 	FMSUBX	y03, alpha2i, a4, y03

  779. 	FMADDX	y04, alpha2r, a4, y04

  780. 

  781. 	FMSUBX	y05, alpha2i, a6, y05

  782. 	FMADDX	y06, alpha2r, a6, y06

  783. 	FMSUBX	y07, alpha2i, a8, y07

  784. 	FMADDX	y08, alpha2r, a8, y08

  785. 

  786. 	LFD	a2,  9 * SIZE(AO2)

  787. 	LFD	a4, 11 * SIZE(AO2)

  788. 	LFD	a6, 13 * SIZE(AO2)

  789. 	LFD	a8, 15 * SIZE(AO2)

  790. 

  791. 	FMADD	y09, alpha2r, a1, y09

  792. 	FMADD	y10, alpha2i, a1, y10

  793. 	FMADD	y11, alpha2r, a3, y11

  794. 	FMADD	y12, alpha2i, a3, y12

  795. 

  796. 	FMADD	y13, alpha2r, a5, y13

  797. 	FMADD	y14, alpha2i, a5, y14

  798. 	FMADD	y15, alpha2r, a7, y15

  799. 	FMADD	y16, alpha2i, a7, y16

  800. 

  801. 	LFD	a1,  0 * SIZE(AO3)

  802. 	LFD	a3,  2 * SIZE(AO3)

  803. 	LFD	a5,  4 * SIZE(AO3)

  804. 	LFD	a7,  6 * SIZE(AO3)

  805. 

  806. 	FMSUBX	y09, alpha2i, a2, y09

  807. 	FMADDX	y10, alpha2r, a2, y10

  808. 	FMSUBX	y11, alpha2i, a4, y11

  809. 	FMADDX	y12, alpha2r, a4, y12

  810. 

  811. 	FMSUBX	y13, alpha2i, a6, y13

  812. 	FMADDX	y14, alpha2r, a6, y14

  813. 	FMSUBX	y15, alpha2i, a8, y15

  814. 	FMADDX	y16, alpha2r, a8, y16

  815. 

  816. 	LFD	a2,  1 * SIZE(AO3)

  817. 	LFD	a4,  3 * SIZE(AO3)

  818. 	LFD	a6,  5 * SIZE(AO3)

  819. 	LFD	a8,  7 * SIZE(AO3)

  820. 

  821. 	FMADD	y01, alpha3r, a1, y01

  822. 	FMADD	y02, alpha3i, a1, y02

  823. 	FMADD	y03, alpha3r, a3, y03

  824. 	FMADD	y04, alpha3i, a3, y04

  825. 

  826. 	FMADD	y05, alpha3r, a5, y05

  827. 	FMADD	y06, alpha3i, a5, y06

  828. 	FMADD	y07, alpha3r, a7, y07

  829. 	FMADD	y08, alpha3i, a7, y08

  830. 

  831. 	LFD	a1,  8 * SIZE(AO3)

  832. 	LFD	a3, 10 * SIZE(AO3)

  833. 	LFD	a5, 12 * SIZE(AO3)

  834. 	LFD	a7, 14 * SIZE(AO3)

  835. 

  836. 	FMSUBX	y01, alpha3i, a2, y01

  837. 	FMADDX	y02, alpha3r, a2, y02

  838. 	FMSUBX	y03, alpha3i, a4, y03

  839. 	FMADDX	y04, alpha3r, a4, y04

  840. 

  841. 	FMSUBX	y05, alpha3i, a6, y05

  842. 	FMADDX	y06, alpha3r, a6, y06

  843. 	FMSUBX	y07, alpha3i, a8, y07

  844. 	FMADDX	y08, alpha3r, a8, y08

  845. 

  846. 	LFD	a2,  9 * SIZE(AO3)

  847. 	LFD	a4, 11 * SIZE(AO3)

  848. 	LFD	a6, 13 * SIZE(AO3)

  849. 	LFD	a8, 15 * SIZE(AO3)

  850. 

  851. 	FMADD	y09, alpha3r, a1, y09

  852. 	FMADD	y10, alpha3i, a1, y10

  853. 	FMADD	y11, alpha3r, a3, y11

  854. 	FMADD	y12, alpha3i, a3, y12

  855. 

  856. 	FMADD	y13, alpha3r, a5, y13

  857. 	FMADD	y14, alpha3i, a5, y14

  858. 	FMADD	y15, alpha3r, a7, y15

  859. 	FMADD	y16, alpha3i, a7, y16

  860. 

  861. 	LFD	a1,  0 * SIZE(AO4)

  862. 	LFD	a3,  2 * SIZE(AO4)

  863. 	LFD	a5,  4 * SIZE(AO4)

  864. 	LFD	a7,  6 * SIZE(AO4)

  865. 

  866. 	FMSUBX	y09, alpha3i, a2, y09

  867. 	FMADDX	y10, alpha3r, a2, y10

  868. 	FMSUBX	y11, alpha3i, a4, y11

  869. 	FMADDX	y12, alpha3r, a4, y12

  870. 

  871. 	FMSUBX	y13, alpha3i, a6, y13

  872. 	FMADDX	y14, alpha3r, a6, y14

  873. 	FMSUBX	y15, alpha3i, a8, y15

  874. 	FMADDX	y16, alpha3r, a8, y16

  875. 

  876. 	LFD	a2,  1 * SIZE(AO4)

  877. 	LFD	a4,  3 * SIZE(AO4)

  878. 	LFD	a6,  5 * SIZE(AO4)

  879. 	LFD	a8,  7 * SIZE(AO4)

  880. 

  881. 	FMADD	y01, alpha4r, a1, y01

  882. 	FMADD	y02, alpha4i, a1, y02

  883. 	FMADD	y03, alpha4r, a3, y03

  884. 	FMADD	y04, alpha4i, a3, y04

  885. 

  886. 	FMADD	y05, alpha4r, a5, y05

  887. 	FMADD	y06, alpha4i, a5, y06

  888. 	FMADD	y07, alpha4r, a7, y07

  889. 	FMADD	y08, alpha4i, a7, y08

  890. 

  891. 	LFD	a1,  8 * SIZE(AO4)

  892. 	LFD	a3, 10 * SIZE(AO4)

  893. 	LFD	a5, 12 * SIZE(AO4)

  894. 	LFD	a7, 14 * SIZE(AO4)

  895. 

  896. 	FMSUBX	y01, alpha4i, a2, y01

  897. 	FMADDX	y02, alpha4r, a2, y02

  898. 	FMSUBX	y03, alpha4i, a4, y03

  899. 	FMADDX	y04, alpha4r, a4, y04

  900. 

  901. 	FMSUBX	y05, alpha4i, a6, y05

  902. 	FMADDX	y06, alpha4r, a6, y06

  903. 	FMSUBX	y07, alpha4i, a8, y07

  904. 	FMADDX	y08, alpha4r, a8, y08

  905. 

  906. 	LFD	a2,  9 * SIZE(AO4)

  907. 	LFD	a4, 11 * SIZE(AO4)

  908. 	LFD	a6, 13 * SIZE(AO4)

  909. 	LFD	a8, 15 * SIZE(AO4)

  910. 

  911. 	FMADD	y09, alpha4r, a1, y09

  912. 	FMADD	y10, alpha4i, a1, y10

  913. 	FMADD	y11, alpha4r, a3, y11

  914. 	FMADD	y12, alpha4i, a3, y12

  915. 

  916. 	FMADD	y13, alpha4r, a5, y13

  917. 	FMADD	y14, alpha4i, a5, y14

  918. 	FMADD	y15, alpha4r, a7, y15

  919. 	FMADD	y16, alpha4i, a7, y16

  920. 

  921. 	LFD	a1, 16 * SIZE(AO1)

  922. 	LFD	a3, 18 * SIZE(AO1)

  923. 	LFD	a5, 20 * SIZE(AO1)

  924. 	LFD	a7, 22 * SIZE(AO1)

  925. 

  926. 	FMSUBX	y09, alpha4i, a2, y09

  927. 	FMADDX	y10, alpha4r, a2, y10

  928. 	FMSUBX	y11, alpha4i, a4, y11

  929. 	FMADDX	y12, alpha4r, a4, y12

  930. 

  931. 	FMSUBX	y13, alpha4i, a6, y13

  932. 	FMADDX	y14, alpha4r, a6, y14

  933. 	FMSUBX	y15, alpha4i, a8, y15

  934. 	FMADDX	y16, alpha4r, a8, y16

  935. 

  936. 	LFD	a2, 17 * SIZE(AO1)

  937. 	LFD	a4, 19 * SIZE(AO1)

  938. 	LFD	a6, 21 * SIZE(AO1)

  939. 	LFD	a8, 23 * SIZE(AO1)

  940. 

  941. 	addi	AO1, AO1, 16 * SIZE

  942. 	addi	AO2, AO2, 16 * SIZE

  943. 	addi	AO3, AO3, 16 * SIZE

  944. 	addi	AO4, AO4, 16 * SIZE

  945. 

  946. 	STFD	y01,  0 * SIZE(Y2)

  947. 	STFD	y02,  1 * SIZE(Y2)

  948. 	STFD	y03,  2 * SIZE(Y2)

  949. 	STFD	y04,  3 * SIZE(Y2)

  950. 	STFD	y05,  4 * SIZE(Y2)

  951. 	STFD	y06,  5 * SIZE(Y2)

  952. 	STFD	y07,  6 * SIZE(Y2)

  953. 	STFD	y08,  7 * SIZE(Y2)

  954. 	STFD	y09,  8 * SIZE(Y2)

  955. 	STFD	y10,  9 * SIZE(Y2)

  956. 	STFD	y11, 10 * SIZE(Y2)

  957. 	STFD	y12, 11 * SIZE(Y2)

  958. 	STFD	y13, 12 * SIZE(Y2)

  959. 	STFD	y14, 13 * SIZE(Y2)

  960. 	STFD	y15, 14 * SIZE(Y2)

  961. 	STFD	y16, 15 * SIZE(Y2)

  962. 	addi	Y2, Y2, 16 * SIZE

  963. 	.align 4

  964. 

  965. LL(15):

  966. 	andi.	r0, M, 7

  967. 	ble	LL(19)

  968. 	andi.	r0, M, 4

  969. 	ble	LL(16)

  970. 

  971. 	LFD	y01, 0 * SIZE(Y1)

  972. 	LFD	y02, 1 * SIZE(Y1)

  973. 	LFD	y03, 2 * SIZE(Y1)

  974. 	LFD	y04, 3 * SIZE(Y1)

  975. 

  976. 	LFD	a1,  0 * SIZE(AO1)

  977. 	LFD	a3,  2 * SIZE(AO1)

  978. 	LFD	a5,  4 * SIZE(AO1)

  979. 	LFD	a7,  6 * SIZE(AO1)

  980. 

  981. 	LFD	y05, 4 * SIZE(Y1)

  982. 	LFD	y06, 5 * SIZE(Y1)

  983. 	LFD	y07, 6 * SIZE(Y1)

  984. 	LFD	y08, 7 * SIZE(Y1)

  985. 

  986. 	LFD	a2,  1 * SIZE(AO1)

  987. 	LFD	a4,  3 * SIZE(AO1)

  988. 	LFD	a6,  5 * SIZE(AO1)

  989. 	LFD	a8,  7 * SIZE(AO1)

  990. 

  991. 	FMADD	y01, alpha1r, a1, y01

  992. 	FMADD	y02, alpha1i, a1, y02

  993. 	FMADD	y03, alpha1r, a3, y03

  994. 	FMADD	y04, alpha1i, a3, y04

  995. 

  996. 	FMADD	y05, alpha1r, a5, y05

  997. 	FMADD	y06, alpha1i, a5, y06

  998. 	FMADD	y07, alpha1r, a7, y07

  999. 	FMADD	y08, alpha1i, a7, y08

  1000. 

  1001. 	LFD	a1,  0 * SIZE(AO2)

  1002. 	LFD	a3,  2 * SIZE(AO2)

  1003. 	LFD	a5,  4 * SIZE(AO2)

  1004. 	LFD	a7,  6 * SIZE(AO2)

  1005. 

  1006. 	FMSUBX	y01, alpha1i, a2, y01

  1007. 	FMADDX	y02, alpha1r, a2, y02

  1008. 	FMSUBX	y03, alpha1i, a4, y03

  1009. 	FMADDX	y04, alpha1r, a4, y04

  1010. 

  1011. 	FMSUBX	y05, alpha1i, a6, y05

  1012. 	FMADDX	y06, alpha1r, a6, y06

  1013. 	FMSUBX	y07, alpha1i, a8, y07

  1014. 	FMADDX	y08, alpha1r, a8, y08

  1015. 

  1016. 	LFD	a2,  1 * SIZE(AO2)

  1017. 	LFD	a4,  3 * SIZE(AO2)

  1018. 	LFD	a6,  5 * SIZE(AO2)

  1019. 	LFD	a8,  7 * SIZE(AO2)

  1020. 

  1021. 	FMADD	y01, alpha2r, a1, y01

  1022. 	FMADD	y02, alpha2i, a1, y02

  1023. 	FMADD	y03, alpha2r, a3, y03

  1024. 	FMADD	y04, alpha2i, a3, y04

  1025. 

  1026. 	FMADD	y05, alpha2r, a5, y05

  1027. 	FMADD	y06, alpha2i, a5, y06

  1028. 	FMADD	y07, alpha2r, a7, y07

  1029. 	FMADD	y08, alpha2i, a7, y08

  1030. 

  1031. 	LFD	a1,  0 * SIZE(AO3)

  1032. 	LFD	a3,  2 * SIZE(AO3)

  1033. 	LFD	a5,  4 * SIZE(AO3)

  1034. 	LFD	a7,  6 * SIZE(AO3)

  1035. 

  1036. 	FMSUBX	y01, alpha2i, a2, y01

  1037. 	FMADDX	y02, alpha2r, a2, y02

  1038. 	FMSUBX	y03, alpha2i, a4, y03

  1039. 	FMADDX	y04, alpha2r, a4, y04

  1040. 

  1041. 	FMSUBX	y05, alpha2i, a6, y05

  1042. 	FMADDX	y06, alpha2r, a6, y06

  1043. 	FMSUBX	y07, alpha2i, a8, y07

  1044. 	FMADDX	y08, alpha2r, a8, y08

  1045. 

  1046. 	LFD	a2,  1 * SIZE(AO3)

  1047. 	LFD	a4,  3 * SIZE(AO3)

  1048. 	LFD	a6,  5 * SIZE(AO3)

  1049. 	LFD	a8,  7 * SIZE(AO3)

  1050. 

  1051. 	FMADD	y01, alpha3r, a1, y01

  1052. 	FMADD	y02, alpha3i, a1, y02

  1053. 	FMADD	y03, alpha3r, a3, y03

  1054. 	FMADD	y04, alpha3i, a3, y04

  1055. 

  1056. 	FMADD	y05, alpha3r, a5, y05

  1057. 	FMADD	y06, alpha3i, a5, y06

  1058. 	FMADD	y07, alpha3r, a7, y07

  1059. 	FMADD	y08, alpha3i, a7, y08

  1060. 

  1061. 	LFD	a1,  0 * SIZE(AO4)

  1062. 	LFD	a3,  2 * SIZE(AO4)

  1063. 	LFD	a5,  4 * SIZE(AO4)

  1064. 	LFD	a7,  6 * SIZE(AO4)

  1065. 

  1066. 	FMSUBX	y01, alpha3i, a2, y01

  1067. 	FMADDX	y02, alpha3r, a2, y02

  1068. 	FMSUBX	y03, alpha3i, a4, y03

  1069. 	FMADDX	y04, alpha3r, a4, y04

  1070. 

  1071. 	FMSUBX	y05, alpha3i, a6, y05

  1072. 	FMADDX	y06, alpha3r, a6, y06

  1073. 	FMSUBX	y07, alpha3i, a8, y07

  1074. 	FMADDX	y08, alpha3r, a8, y08

  1075. 

  1076. 	LFD	a2,  1 * SIZE(AO4)

  1077. 	LFD	a4,  3 * SIZE(AO4)

  1078. 	LFD	a6,  5 * SIZE(AO4)

  1079. 	LFD	a8,  7 * SIZE(AO4)

  1080. 

  1081. 	FMADD	y01, alpha4r, a1, y01

  1082. 	FMADD	y02, alpha4i, a1, y02

  1083. 	FMADD	y03, alpha4r, a3, y03

  1084. 	FMADD	y04, alpha4i, a3, y04

  1085. 

  1086. 	FMADD	y05, alpha4r, a5, y05

  1087. 	FMADD	y06, alpha4i, a5, y06

  1088. 	FMADD	y07, alpha4r, a7, y07

  1089. 	FMADD	y08, alpha4i, a7, y08

  1090. 

  1091. 	FMSUBX	y01, alpha4i, a2, y01

  1092. 	FMADDX	y02, alpha4r, a2, y02

  1093. 	FMSUBX	y03, alpha4i, a4, y03

  1094. 	FMADDX	y04, alpha4r, a4, y04

  1095. 

  1096. 	STFD	y01, 0 * SIZE(Y2)

  1097. 	STFD	y02, 1 * SIZE(Y2)

  1098. 	STFD	y03, 2 * SIZE(Y2)

  1099. 	STFD	y04, 3 * SIZE(Y2)

  1100. 

  1101. 	FMSUBX	y05, alpha4i, a6, y05

  1102. 	FMADDX	y06, alpha4r, a6, y06

  1103. 	FMSUBX	y07, alpha4i, a8, y07

  1104. 	FMADDX	y08, alpha4r, a8, y08

  1105. 

  1106. 	STFD	y05, 4 * SIZE(Y2)

  1107. 	STFD	y06, 5 * SIZE(Y2)

  1108. 	STFD	y07, 6 * SIZE(Y2)

  1109. 	STFD	y08, 7 * SIZE(Y2)

  1110. 

  1111. 	addi	AO1, AO1, 8 * SIZE

  1112. 	addi	AO2, AO2, 8 * SIZE

  1113. 	addi	AO3, AO3, 8 * SIZE

  1114. 	addi	AO4, AO4, 8 * SIZE

  1115. 

  1116. 	addi	Y1, Y1, 8 * SIZE

  1117. 	addi	Y2, Y2, 8 * SIZE

  1118. 	.align 4

  1119. 

  1120. LL(16):

  1121. 	andi.	r0, M, 2

  1122. 	nop

  1123. 	nop

  1124. 	ble	LL(17)

  1125. 

  1126. 	LFD	a1,  0 * SIZE(AO1)

  1127. 	LFD	a2,  1 * SIZE(AO1)

  1128. 	LFD	a3,  2 * SIZE(AO1)

  1129. 	LFD	a4,  3 * SIZE(AO1)

  1130. 

  1131. 	LFD	y01, 0 * SIZE(Y1)

  1132. 	LFD	y02, 1 * SIZE(Y1)

  1133. 	LFD	y03, 2 * SIZE(Y1)

  1134. 	LFD	y04, 3 * SIZE(Y1)

  1135. 

  1136. 	LFD	a5,  0 * SIZE(AO2)

  1137. 	LFD	a6,  1 * SIZE(AO2)

  1138. 	LFD	a7,  2 * SIZE(AO2)

  1139. 	LFD	a8,  3 * SIZE(AO2)

  1140. 

  1141. 	FMADD	y01, alpha1r, a1, y01

  1142. 	FMADD	y02, alpha1i, a1, y02

  1143. 	FMADD	y03, alpha1r, a3, y03

  1144. 	FMADD	y04, alpha1i, a3, y04

  1145. 

  1146. 	FMSUBX	y01, alpha1i, a2, y01

  1147. 	FMADDX	y02, alpha1r, a2, y02

  1148. 	FMSUBX	y03, alpha1i, a4, y03

  1149. 	FMADDX	y04, alpha1r, a4, y04

  1150. 

  1151. 	LFD	a1,  0 * SIZE(AO3)

  1152. 	LFD	a2,  1 * SIZE(AO3)

  1153. 	LFD	a3,  2 * SIZE(AO3)

  1154. 	LFD	a4,  3 * SIZE(AO3)

  1155. 

  1156. 	FMADD	y01, alpha2r, a5, y01

  1157. 	FMADD	y02, alpha2i, a5, y02

  1158. 	FMADD	y03, alpha2r, a7, y03

  1159. 	FMADD	y04, alpha2i, a7, y04

  1160. 

  1161. 	FMSUBX	y01, alpha2i, a6, y01

  1162. 	FMADDX	y02, alpha2r, a6, y02

  1163. 	FMSUBX	y03, alpha2i, a8, y03

  1164. 	FMADDX	y04, alpha2r, a8, y04

  1165. 

  1166. 	LFD	a5,  0 * SIZE(AO4)

  1167. 	LFD	a6,  1 * SIZE(AO4)

  1168. 	LFD	a7,  2 * SIZE(AO4)

  1169. 	LFD	a8,  3 * SIZE(AO4)

  1170. 

  1171. 	FMADD	y01, alpha3r, a1, y01

  1172. 	FMADD	y02, alpha3i, a1, y02

  1173. 	FMADD	y03, alpha3r, a3, y03

  1174. 	FMADD	y04, alpha3i, a3, y04

  1175. 

  1176. 	FMSUBX	y01, alpha3i, a2, y01

  1177. 	FMADDX	y02, alpha3r, a2, y02

  1178. 	FMSUBX	y03, alpha3i, a4, y03

  1179. 	FMADDX	y04, alpha3r, a4, y04

  1180. 

  1181. 	FMADD	y01, alpha4r, a5, y01

  1182. 	FMADD	y02, alpha4i, a5, y02

  1183. 	FMADD	y03, alpha4r, a7, y03

  1184. 	FMADD	y04, alpha4i, a7, y04

  1185. 

  1186. 	FMSUBX	y01, alpha4i, a6, y01

  1187. 	FMADDX	y02, alpha4r, a6, y02

  1188. 	FMSUBX	y03, alpha4i, a8, y03

  1189. 	FMADDX	y04, alpha4r, a8, y04

  1190. 

  1191. 	STFD	y01, 0 * SIZE(Y2)

  1192. 	STFD	y02, 1 * SIZE(Y2)

  1193. 	STFD	y03, 2 * SIZE(Y2)

  1194. 	STFD	y04, 3 * SIZE(Y2)

  1195. 

  1196. 	addi	AO1, AO1, 4 * SIZE

  1197. 	addi	AO2, AO2, 4 * SIZE

  1198. 	addi	AO3, AO3, 4 * SIZE

  1199. 	addi	AO4, AO4, 4 * SIZE

  1200. 

  1201. 	addi	Y1, Y1, 4 * SIZE

  1202. 	addi	Y2, Y2, 4 * SIZE

  1203. 	.align 4

  1204. 

  1205. LL(17):

  1206. 	andi.	r0, M, 1

  1207. 	ble	LL(19)

  1208. 

  1209. 	LFD	y01, 0 * SIZE(Y1)

  1210. 	LFD	y02, 1 * SIZE(Y1)

  1211. 

  1212. 	LFD	a1,  0 * SIZE(AO1)

  1213. 	LFD	a2,  1 * SIZE(AO1)

  1214. 	LFD	a3,  0 * SIZE(AO2)

  1215. 	LFD	a4,  1 * SIZE(AO2)

  1216. 

  1217. 	LFD	a5,  0 * SIZE(AO3)

  1218. 	LFD	a6,  1 * SIZE(AO3)

  1219. 	LFD	a7,  0 * SIZE(AO4)

  1220. 	LFD	a8,  1 * SIZE(AO4)

  1221. 

  1222. 	FMADD	y01, alpha1r, a1, y01

  1223. 	FMADD	y02, alpha1i, a1, y02

  1224. 	FMSUBX	y01, alpha1i, a2, y01

  1225. 	FMADDX	y02, alpha1r, a2, y02

  1226. 

  1227. 	FMADD	y01, alpha2r, a3, y01

  1228. 	FMADD	y02, alpha2i, a3, y02

  1229. 	FMSUBX	y01, alpha2i, a4, y01

  1230. 	FMADDX	y02, alpha2r, a4, y02

  1231. 

  1232. 	FMADD	y01, alpha3r, a5, y01

  1233. 	FMADD	y02, alpha3i, a5, y02

  1234. 	FMSUBX	y01, alpha3i, a6, y01

  1235. 	FMADDX	y02, alpha3r, a6, y02

  1236. 

  1237. 	FMADD	y01, alpha4r, a7, y01

  1238. 	FMADD	y02, alpha4i, a7, y02

  1239. 	FMSUBX	y01, alpha4i, a8, y01

  1240. 	FMADDX	y02, alpha4r, a8, y02

  1241. 

  1242. 	STFD	y01, 0 * SIZE(Y2)

  1243. 	STFD	y02, 1 * SIZE(Y2)

  1244. 

  1245. 	add	Y1, Y1, INCY

  1246. 	add	Y2, Y2, INCY

  1247. 	.align 4

  1248. 

  1249. LL(19):

  1250. 	addi	J, J, -1

  1251. 	cmpi	cr0, 0, J, 0

  1252. 	bgt	LL(11)

  1253. 	.align 4

  1254. 

  1255. LL(20):

  1256. 	andi.	J, N, 2

  1257. 	ble	LL(30)

  1258. 	.align 4

  1259. 

  1260. LL(21):

  1261. 	lfd	alpha_r, ALPHA_R

  1262. 	lfd	alpha_i, ALPHA_I

  1263. 

  1264. 	LFD	a1, 0 * SIZE(X)

  1265. 	LFD	a2, 1 * SIZE(X)

  1266. 	add	X, X, INCX

  1267. 	LFD	a3, 0 * SIZE(X)

  1268. 	LFD	a4, 1 * SIZE(X)

  1269. 	add	X, X, INCX

  1270. 

  1271. 	FMUL	alpha1r, alpha_r, a1

  1272. 	FMUL	alpha1i, alpha_i, a1

  1273. 	FMUL	alpha2r, alpha_r, a3

  1274. 	FMUL	alpha2i, alpha_i, a3

  1275. 

  1276. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  1277. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  1278. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  1279. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  1280. 

  1281. 	mr	AO1, A

  1282. 	add	AO2, A,   LDA

  1283. 	add	A,   AO2, LDA

  1284. 

  1285. 	mr	Y1, Y

  1286. 	mr	Y2, Y

  1287. 

  1288. 	srawi.	r0,  M, 3

  1289. 	mtspr	CTR, r0

  1290. 	ble	LL(25)

  1291. 	.align 4

  1292. 

  1293. 	LFD	a1,  0 * SIZE(AO1)

  1294. 	LFD	a2,  1 * SIZE(AO1)

  1295. 	LFD	a3,  2 * SIZE(AO1)

  1296. 	LFD	a4,  3 * SIZE(AO1)

  1297. 

  1298. 	LFD	y01,  0 * SIZE(Y1)

  1299. 	LFD	y02,  1 * SIZE(Y1)

  1300. 	LFD	y03,  2 * SIZE(Y1)

  1301. 	LFD	y04,  3 * SIZE(Y1)

  1302. 

  1303. 	LFD	a5,  4 * SIZE(AO1)

  1304. 	LFD	a6,  5 * SIZE(AO1)

  1305. 	LFD	a7,  6 * SIZE(AO1)

  1306. 	LFD	a8,  7 * SIZE(AO1)

  1307. 

  1308. 	LFD	y05,  4 * SIZE(Y1)

  1309. 	LFD	y06,  5 * SIZE(Y1)

  1310. 	LFD	y07,  6 * SIZE(Y1)

  1311. 	LFD	y08,  7 * SIZE(Y1)

  1312. 	LFD	y09,  8 * SIZE(Y1)

  1313. 	LFD	y10,  9 * SIZE(Y1)

  1314. 	LFD	y11, 10 * SIZE(Y1)

  1315. 	LFD	y12, 11 * SIZE(Y1)

  1316. 	LFD	y13, 12 * SIZE(Y1)

  1317. 	LFD	y14, 13 * SIZE(Y1)

  1318. 	LFD	y15, 14 * SIZE(Y1)

  1319. 	LFD	y16, 15 * SIZE(Y1)

  1320. 	addi	Y1, Y1, 16 * SIZE

  1321. 	bdz	LL(23)

  1322. 	.align 4

  1323. 

  1324. LL(22):

  1325. 	FMADD	y01, alpha1r, a1, y01

  1326. 	FMADD	y02, alpha1i, a1, y02

  1327. 	FMADD	y03, alpha1r, a3, y03

  1328. 	FMADD	y04, alpha1i, a3, y04

  1329. 

  1330. 	FMADD	y05, alpha1r, a5, y05

  1331. 	FMADD	y06, alpha1i, a5, y06

  1332. 	FMADD	y07, alpha1r, a7, y07

  1333. 	FMADD	y08, alpha1i, a7, y08

  1334. 

  1335. 	LFD	a1,  8 * SIZE(AO1)

  1336. 	LFD	a3, 10 * SIZE(AO1)

  1337. 	LFD	a5, 12 * SIZE(AO1)

  1338. 	LFD	a7, 14 * SIZE(AO1)

  1339. 

  1340. 	FMSUBX	y01, alpha1i, a2, y01

  1341. 	FMADDX	y02, alpha1r, a2, y02

  1342. 	FMSUBX	y03, alpha1i, a4, y03

  1343. 	FMADDX	y04, alpha1r, a4, y04

  1344. 

  1345. 	FMSUBX	y05, alpha1i, a6, y05

  1346. 	FMADDX	y06, alpha1r, a6, y06

  1347. 	FMSUBX	y07, alpha1i, a8, y07

  1348. 	FMADDX	y08, alpha1r, a8, y08

  1349. 

  1350. 	LFD	a2,  9 * SIZE(AO1)

  1351. 	LFD	a4, 11 * SIZE(AO1)

  1352. 	LFD	a6, 13 * SIZE(AO1)

  1353. 	LFD	a8, 15 * SIZE(AO1)

  1354. 

  1355. 	addi	AO1, AO1, 16 * SIZE

  1356. 	nop

  1357. 	DCBT(AO1, PREA)

  1358. 	nop

  1359. 

  1360. 	FMADD	y09, alpha1r, a1, y09

  1361. 	FMADD	y10, alpha1i, a1, y10

  1362. 	FMADD	y11, alpha1r, a3, y11

  1363. 	FMADD	y12, alpha1i, a3, y12

  1364. 

  1365. 	FMADD	y13, alpha1r, a5, y13

  1366. 	FMADD	y14, alpha1i, a5, y14

  1367. 	FMADD	y15, alpha1r, a7, y15

  1368. 	FMADD	y16, alpha1i, a7, y16

  1369. 

  1370. 	LFD	a1,  0 * SIZE(AO2)

  1371. 	LFD	a3,  2 * SIZE(AO2)

  1372. 	LFD	a5,  4 * SIZE(AO2)

  1373. 	LFD	a7,  6 * SIZE(AO2)

  1374. 

  1375. 	FMSUBX	y09, alpha1i, a2, y09

  1376. 	FMADDX	y10, alpha1r, a2, y10

  1377. 	FMSUBX	y11, alpha1i, a4, y11

  1378. 	FMADDX	y12, alpha1r, a4, y12

  1379. 

  1380. 	FMSUBX	y13, alpha1i, a6, y13

  1381. 	FMADDX	y14, alpha1r, a6, y14

  1382. 	FMSUBX	y15, alpha1i, a8, y15

  1383. 	FMADDX	y16, alpha1r, a8, y16

  1384. 

  1385. 	LFD	a2,  1 * SIZE(AO2)

  1386. 	LFD	a4,  3 * SIZE(AO2)

  1387. 	LFD	a6,  5 * SIZE(AO2)

  1388. 	LFD	a8,  7 * SIZE(AO2)

  1389. 

  1390. 	FMADD	y01, alpha2r, a1, y01

  1391. 	FMADD	y02, alpha2i, a1, y02

  1392. 	FMADD	y03, alpha2r, a3, y03

  1393. 	FMADD	y04, alpha2i, a3, y04

  1394. 

  1395. 	FMADD	y05, alpha2r, a5, y05

  1396. 	FMADD	y06, alpha2i, a5, y06

  1397. 	FMADD	y07, alpha2r, a7, y07

  1398. 	FMADD	y08, alpha2i, a7, y08

  1399. 

  1400. 	LFD	a1,  8 * SIZE(AO2)

  1401. 	LFD	a3, 10 * SIZE(AO2)

  1402. 	LFD	a5, 12 * SIZE(AO2)

  1403. 	LFD	a7, 14 * SIZE(AO2)

  1404. 

  1405. 	FMSUBX	y01, alpha2i, a2, y01

  1406. 	FMADDX	y02, alpha2r, a2, y02

  1407. 	FMSUBX	y03, alpha2i, a4, y03

  1408. 	FMADDX	y04, alpha2r, a4, y04

  1409. 

  1410. 	STFD	y01,  0 * SIZE(Y2)

  1411. 	STFD	y02,  1 * SIZE(Y2)

  1412. 	STFD	y03,  2 * SIZE(Y2)

  1413. 	STFD	y04,  3 * SIZE(Y2)

  1414. 

  1415. 	LFD	y01,  0 * SIZE(Y1)

  1416. 	LFD	y02,  1 * SIZE(Y1)

  1417. 	LFD	y03,  2 * SIZE(Y1)

  1418. 	LFD	y04,  3 * SIZE(Y1)

  1419. 

  1420. 	FMSUBX	y05, alpha2i, a6, y05

  1421. 	FMADDX	y06, alpha2r, a6, y06

  1422. 	FMSUBX	y07, alpha2i, a8, y07

  1423. 	FMADDX	y08, alpha2r, a8, y08

  1424. 

  1425. 	LFD	a2,  9 * SIZE(AO2)

  1426. 	LFD	a4, 11 * SIZE(AO2)

  1427. 	LFD	a6, 13 * SIZE(AO2)

  1428. 	LFD	a8, 15 * SIZE(AO2)

  1429. 

  1430. 	STFD	y05,  4 * SIZE(Y2)

  1431. 	STFD	y06,  5 * SIZE(Y2)

  1432. 	STFD	y07,  6 * SIZE(Y2)

  1433. 	STFD	y08,  7 * SIZE(Y2)

  1434. 

  1435. 	LFD	y05,  4 * SIZE(Y1)

  1436. 	LFD	y06,  5 * SIZE(Y1)

  1437. 	LFD	y07,  6 * SIZE(Y1)

  1438. 	LFD	y08,  7 * SIZE(Y1)

  1439. 

  1440. 	addi	AO2, AO2, 16 * SIZE

  1441. 	nop

  1442. 	DCBT(AO2, PREA)

  1443. 	nop

  1444. 

  1445. 	FMADD	y09, alpha2r, a1, y09

  1446. 	FMADD	y10, alpha2i, a1, y10

  1447. 	FMADD	y11, alpha2r, a3, y11

  1448. 	FMADD	y12, alpha2i, a3, y12

  1449. 

  1450. 	FMADD	y13, alpha2r, a5, y13

  1451. 	FMADD	y14, alpha2i, a5, y14

  1452. 	FMADD	y15, alpha2r, a7, y15

  1453. 	FMADD	y16, alpha2i, a7, y16

  1454. 

  1455. 	LFD	a1,  0 * SIZE(AO1)

  1456. 	LFD	a3,  2 * SIZE(AO1)

  1457. 	LFD	a5,  4 * SIZE(AO1)

  1458. 	LFD	a7,  6 * SIZE(AO1)

  1459. 

  1460. 	FMSUBX	y09, alpha2i, a2, y09

  1461. 	FMADDX	y10, alpha2r, a2, y10

  1462. 	FMSUBX	y11, alpha2i, a4, y11

  1463. 	FMADDX	y12, alpha2r, a4, y12

  1464. 

  1465. 	STFD	y09,  8 * SIZE(Y2)

  1466. 	STFD	y10,  9 * SIZE(Y2)

  1467. 	STFD	y11, 10 * SIZE(Y2)

  1468. 	STFD	y12, 11 * SIZE(Y2)

  1469. 

  1470. 	LFD	y09,  8 * SIZE(Y1)

  1471. 	LFD	y10,  9 * SIZE(Y1)

  1472. 	LFD	y11, 10 * SIZE(Y1)

  1473. 	LFD	y12, 11 * SIZE(Y1)

  1474. 

  1475. 	FMSUBX	y13, alpha2i, a6, y13

  1476. 	FMADDX	y14, alpha2r, a6, y14

  1477. 	FMSUBX	y15, alpha2i, a8, y15

  1478. 	FMADDX	y16, alpha2r, a8, y16

  1479. 

  1480. 	LFD	a2,  1 * SIZE(AO1)

  1481. 	LFD	a4,  3 * SIZE(AO1)

  1482. 	LFD	a6,  5 * SIZE(AO1)

  1483. 	LFD	a8,  7 * SIZE(AO1)

  1484. 

  1485. 	STFD	y13, 12 * SIZE(Y2)

  1486. 	STFD	y14, 13 * SIZE(Y2)

  1487. 	STFD	y15, 14 * SIZE(Y2)

  1488. 	STFD	y16, 15 * SIZE(Y2)

  1489. 

  1490. 	LFD	y13, 12 * SIZE(Y1)

  1491. 	LFD	y14, 13 * SIZE(Y1)

  1492. 	LFD	y15, 14 * SIZE(Y1)

  1493. 	LFD	y16, 15 * SIZE(Y1)

  1494. 

  1495. 	addi	Y2, Y2, 16 * SIZE

  1496. 	addi	Y1, Y1, 16 * SIZE

  1497. 	DCBT(Y1, PREC)

  1498. 	bdnz	LL(22)

  1499. 	.align 4

  1500. 

  1501. LL(23):

  1502. 	FMADD	y01, alpha1r, a1, y01

  1503. 	FMADD	y02, alpha1i, a1, y02

  1504. 	FMADD	y03, alpha1r, a3, y03

  1505. 	FMADD	y04, alpha1i, a3, y04

  1506. 

  1507. 	FMADD	y05, alpha1r, a5, y05

  1508. 	FMADD	y06, alpha1i, a5, y06

  1509. 	FMADD	y07, alpha1r, a7, y07

  1510. 	FMADD	y08, alpha1i, a7, y08

  1511. 

  1512. 	LFD	a1,  8 * SIZE(AO1)

  1513. 	LFD	a3, 10 * SIZE(AO1)

  1514. 	LFD	a5, 12 * SIZE(AO1)

  1515. 	LFD	a7, 14 * SIZE(AO1)

  1516. 

  1517. 	FMSUBX	y01, alpha1i, a2, y01

  1518. 	FMADDX	y02, alpha1r, a2, y02

  1519. 	FMSUBX	y03, alpha1i, a4, y03

  1520. 	FMADDX	y04, alpha1r, a4, y04

  1521. 

  1522. 	FMSUBX	y05, alpha1i, a6, y05

  1523. 	FMADDX	y06, alpha1r, a6, y06

  1524. 	FMSUBX	y07, alpha1i, a8, y07

  1525. 	FMADDX	y08, alpha1r, a8, y08

  1526. 

  1527. 	LFD	a2,  9 * SIZE(AO1)

  1528. 	LFD	a4, 11 * SIZE(AO1)

  1529. 	LFD	a6, 13 * SIZE(AO1)

  1530. 	LFD	a8, 15 * SIZE(AO1)

  1531. 

  1532. 	FMADD	y09, alpha1r, a1, y09

  1533. 	FMADD	y10, alpha1i, a1, y10

  1534. 	FMADD	y11, alpha1r, a3, y11

  1535. 	FMADD	y12, alpha1i, a3, y12

  1536. 

  1537. 	FMADD	y13, alpha1r, a5, y13

  1538. 	FMADD	y14, alpha1i, a5, y14

  1539. 	FMADD	y15, alpha1r, a7, y15

  1540. 	FMADD	y16, alpha1i, a7, y16

  1541. 

  1542. 	LFD	a1,  0 * SIZE(AO2)

  1543. 	LFD	a3,  2 * SIZE(AO2)

  1544. 	LFD	a5,  4 * SIZE(AO2)

  1545. 	LFD	a7,  6 * SIZE(AO2)

  1546. 

  1547. 	FMSUBX	y09, alpha1i, a2, y09

  1548. 	FMADDX	y10, alpha1r, a2, y10

  1549. 	FMSUBX	y11, alpha1i, a4, y11

  1550. 	FMADDX	y12, alpha1r, a4, y12

  1551. 

  1552. 	FMSUBX	y13, alpha1i, a6, y13

  1553. 	FMADDX	y14, alpha1r, a6, y14

  1554. 	FMSUBX	y15, alpha1i, a8, y15

  1555. 	FMADDX	y16, alpha1r, a8, y16

  1556. 

  1557. 	LFD	a2,  1 * SIZE(AO2)

  1558. 	LFD	a4,  3 * SIZE(AO2)

  1559. 	LFD	a6,  5 * SIZE(AO2)

  1560. 	LFD	a8,  7 * SIZE(AO2)

  1561. 

  1562. 	FMADD	y01, alpha2r, a1, y01

  1563. 	FMADD	y02, alpha2i, a1, y02

  1564. 	FMADD	y03, alpha2r, a3, y03

  1565. 	FMADD	y04, alpha2i, a3, y04

  1566. 

  1567. 	FMADD	y05, alpha2r, a5, y05

  1568. 	FMADD	y06, alpha2i, a5, y06

  1569. 	FMADD	y07, alpha2r, a7, y07

  1570. 	FMADD	y08, alpha2i, a7, y08

  1571. 

  1572. 	LFD	a1,  8 * SIZE(AO2)

  1573. 	LFD	a3, 10 * SIZE(AO2)

  1574. 	LFD	a5, 12 * SIZE(AO2)

  1575. 	LFD	a7, 14 * SIZE(AO2)

  1576. 

  1577. 	FMSUBX	y01, alpha2i, a2, y01

  1578. 	FMADDX	y02, alpha2r, a2, y02

  1579. 	FMSUBX	y03, alpha2i, a4, y03

  1580. 	FMADDX	y04, alpha2r, a4, y04

  1581. 

  1582. 	STFD	y01,  0 * SIZE(Y2)

  1583. 	STFD	y02,  1 * SIZE(Y2)

  1584. 	STFD	y03,  2 * SIZE(Y2)

  1585. 	STFD	y04,  3 * SIZE(Y2)

  1586. 

  1587. 	FMSUBX	y05, alpha2i, a6, y05

  1588. 	FMADDX	y06, alpha2r, a6, y06

  1589. 	FMSUBX	y07, alpha2i, a8, y07

  1590. 	FMADDX	y08, alpha2r, a8, y08

  1591. 

  1592. 	LFD	a2,  9 * SIZE(AO2)

  1593. 	LFD	a4, 11 * SIZE(AO2)

  1594. 	LFD	a6, 13 * SIZE(AO2)

  1595. 	LFD	a8, 15 * SIZE(AO2)

  1596. 

  1597. 	STFD	y05,  4 * SIZE(Y2)

  1598. 	STFD	y06,  5 * SIZE(Y2)

  1599. 	STFD	y07,  6 * SIZE(Y2)

  1600. 	STFD	y08,  7 * SIZE(Y2)

  1601. 

  1602. 	FMADD	y09, alpha2r, a1, y09

  1603. 	FMADD	y10, alpha2i, a1, y10

  1604. 	FMADD	y11, alpha2r, a3, y11

  1605. 	FMADD	y12, alpha2i, a3, y12

  1606. 

  1607. 	FMADD	y13, alpha2r, a5, y13

  1608. 	FMADD	y14, alpha2i, a5, y14

  1609. 	FMADD	y15, alpha2r, a7, y15

  1610. 	FMADD	y16, alpha2i, a7, y16

  1611. 

  1612. 	FMSUBX	y09, alpha2i, a2, y09

  1613. 	FMADDX	y10, alpha2r, a2, y10

  1614. 	FMSUBX	y11, alpha2i, a4, y11

  1615. 	FMADDX	y12, alpha2r, a4, y12

  1616. 

  1617. 	FMSUBX	y13, alpha2i, a6, y13

  1618. 	FMADDX	y14, alpha2r, a6, y14

  1619. 	FMSUBX	y15, alpha2i, a8, y15

  1620. 	FMADDX	y16, alpha2r, a8, y16

  1621. 

  1622. 	STFD	y09,  8 * SIZE(Y2)

  1623. 	STFD	y10,  9 * SIZE(Y2)

  1624. 	STFD	y11, 10 * SIZE(Y2)

  1625. 	STFD	y12, 11 * SIZE(Y2)

  1626. 

  1627. 	STFD	y13, 12 * SIZE(Y2)

  1628. 	STFD	y14, 13 * SIZE(Y2)

  1629. 	STFD	y15, 14 * SIZE(Y2)

  1630. 	STFD	y16, 15 * SIZE(Y2)

  1631. 

  1632. 	addi	AO1, AO1, 16 * SIZE

  1633. 	addi	AO2, AO2, 16 * SIZE

  1634. 	addi	Y2, Y2, 16 * SIZE

  1635. 	.align 4

  1636. 

  1637. LL(25):

  1638. 	andi.	r0, M, 7

  1639. 	ble	LL(30)

  1640. 	andi.	r0, M, 4

  1641. 	ble	LL(26)

  1642. 

  1643. 	LFD	y01, 0 * SIZE(Y1)

  1644. 	LFD	y02, 1 * SIZE(Y1)

  1645. 	LFD	y03, 2 * SIZE(Y1)

  1646. 	LFD	y04, 3 * SIZE(Y1)

  1647. 

  1648. 	LFD	a1,  0 * SIZE(AO1)

  1649. 	LFD	a3,  2 * SIZE(AO1)

  1650. 	LFD	a5,  4 * SIZE(AO1)

  1651. 	LFD	a7,  6 * SIZE(AO1)

  1652. 

  1653. 	LFD	y05, 4 * SIZE(Y1)

  1654. 	LFD	y06, 5 * SIZE(Y1)

  1655. 	LFD	y07, 6 * SIZE(Y1)

  1656. 	LFD	y08, 7 * SIZE(Y1)

  1657. 

  1658. 	LFD	a2,  1 * SIZE(AO1)

  1659. 	LFD	a4,  3 * SIZE(AO1)

  1660. 	LFD	a6,  5 * SIZE(AO1)

  1661. 	LFD	a8,  7 * SIZE(AO1)

  1662. 

  1663. 	FMADD	y01, alpha1r, a1, y01

  1664. 	FMADD	y02, alpha1i, a1, y02

  1665. 	FMADD	y03, alpha1r, a3, y03

  1666. 	FMADD	y04, alpha1i, a3, y04

  1667. 

  1668. 	FMADD	y05, alpha1r, a5, y05

  1669. 	FMADD	y06, alpha1i, a5, y06

  1670. 	FMADD	y07, alpha1r, a7, y07

  1671. 	FMADD	y08, alpha1i, a7, y08

  1672. 

  1673. 	LFD	a1,  0 * SIZE(AO2)

  1674. 	LFD	a3,  2 * SIZE(AO2)

  1675. 	LFD	a5,  4 * SIZE(AO2)

  1676. 	LFD	a7,  6 * SIZE(AO2)

  1677. 

  1678. 	FMSUBX	y01, alpha1i, a2, y01

  1679. 	FMADDX	y02, alpha1r, a2, y02

  1680. 	FMSUBX	y03, alpha1i, a4, y03

  1681. 	FMADDX	y04, alpha1r, a4, y04

  1682. 

  1683. 	FMSUBX	y05, alpha1i, a6, y05

  1684. 	FMADDX	y06, alpha1r, a6, y06

  1685. 	FMSUBX	y07, alpha1i, a8, y07

  1686. 	FMADDX	y08, alpha1r, a8, y08

  1687. 

  1688. 	LFD	a2,  1 * SIZE(AO2)

  1689. 	LFD	a4,  3 * SIZE(AO2)

  1690. 	LFD	a6,  5 * SIZE(AO2)

  1691. 	LFD	a8,  7 * SIZE(AO2)

  1692. 

  1693. 	FMADD	y01, alpha2r, a1, y01

  1694. 	FMADD	y02, alpha2i, a1, y02

  1695. 	FMADD	y03, alpha2r, a3, y03

  1696. 	FMADD	y04, alpha2i, a3, y04

  1697. 

  1698. 	FMADD	y05, alpha2r, a5, y05

  1699. 	FMADD	y06, alpha2i, a5, y06

  1700. 	FMADD	y07, alpha2r, a7, y07

  1701. 	FMADD	y08, alpha2i, a7, y08

  1702. 

  1703. 	FMSUBX	y01, alpha2i, a2, y01

  1704. 	FMADDX	y02, alpha2r, a2, y02

  1705. 	FMSUBX	y03, alpha2i, a4, y03

  1706. 	FMADDX	y04, alpha2r, a4, y04

  1707. 

  1708. 	STFD	y01, 0 * SIZE(Y2)

  1709. 	STFD	y02, 1 * SIZE(Y2)

  1710. 	STFD	y03, 2 * SIZE(Y2)

  1711. 	STFD	y04, 3 * SIZE(Y2)

  1712. 

  1713. 	FMSUBX	y05, alpha2i, a6, y05

  1714. 	FMADDX	y06, alpha2r, a6, y06

  1715. 	FMSUBX	y07, alpha2i, a8, y07

  1716. 	FMADDX	y08, alpha2r, a8, y08

  1717. 

  1718. 	STFD	y05, 4 * SIZE(Y2)

  1719. 	STFD	y06, 5 * SIZE(Y2)

  1720. 	STFD	y07, 6 * SIZE(Y2)

  1721. 	STFD	y08, 7 * SIZE(Y2)

  1722. 

  1723. 	addi	AO1, AO1, 8 * SIZE

  1724. 	addi	AO2, AO2, 8 * SIZE

  1725. 	addi	Y1, Y1, 8 * SIZE

  1726. 	addi	Y2, Y2, 8 * SIZE

  1727. 	.align 4

  1728. 

  1729. LL(26):

  1730. 	andi.	r0, M, 2

  1731. 	ble	LL(27)

  1732. 

  1733. 	LFD	a1,  0 * SIZE(AO1)

  1734. 	LFD	a3,  2 * SIZE(AO1)

  1735. 	LFD	a5,  0 * SIZE(AO2)

  1736. 	LFD	a7,  2 * SIZE(AO2)

  1737. 

  1738. 	LFD	y01, 0 * SIZE(Y1)

  1739. 	LFD	y02, 1 * SIZE(Y1)

  1740. 	LFD	y03, 2 * SIZE(Y1)

  1741. 	LFD	y04, 3 * SIZE(Y1)

  1742. 

  1743. 	LFD	a2,  1 * SIZE(AO1)

  1744. 	LFD	a4,  3 * SIZE(AO1)

  1745. 	LFD	a6,  1 * SIZE(AO2)

  1746. 	LFD	a8,  3 * SIZE(AO2)

  1747. 

  1748. 	FMADD	y01, alpha1r, a1, y01

  1749. 	FMADD	y02, alpha1i, a1, y02

  1750. 	FMADD	y03, alpha1r, a3, y03

  1751. 	FMADD	y04, alpha1i, a3, y04

  1752. 

  1753. 	FMSUBX	y01, alpha1i, a2, y01

  1754. 	FMADDX	y02, alpha1r, a2, y02

  1755. 	FMSUBX	y03, alpha1i, a4, y03

  1756. 	FMADDX	y04, alpha1r, a4, y04

  1757. 

  1758. 	FMADD	y01, alpha2r, a5, y01

  1759. 	FMADD	y02, alpha2i, a5, y02

  1760. 	FMADD	y03, alpha2r, a7, y03

  1761. 	FMADD	y04, alpha2i, a7, y04

  1762. 

  1763. 	FMSUBX	y01, alpha2i, a6, y01

  1764. 	FMADDX	y02, alpha2r, a6, y02

  1765. 	FMSUBX	y03, alpha2i, a8, y03

  1766. 	FMADDX	y04, alpha2r, a8, y04

  1767. 

  1768. 	STFD	y01, 0 * SIZE(Y2)

  1769. 	STFD	y02, 1 * SIZE(Y2)

  1770. 	STFD	y03, 2 * SIZE(Y2)

  1771. 	STFD	y04, 3 * SIZE(Y2)

  1772. 

  1773. 	addi	AO1, AO1, 4 * SIZE

  1774. 	addi	AO2, AO2, 4 * SIZE

  1775. 	addi	Y1, Y1, 4 * SIZE

  1776. 	addi	Y2, Y2, 4 * SIZE

  1777. 	.align 4

  1778. 

  1779. LL(27):

  1780. 	andi.	r0, M, 1

  1781. 	ble	LL(30)

  1782. 

  1783. 	LFD	y01, 0 * SIZE(Y1)

  1784. 	LFD	y02, 1 * SIZE(Y1)

  1785. 

  1786. 	LFD	a1,  0 * SIZE(AO1)

  1787. 	LFD	a2,  1 * SIZE(AO1)

  1788. 	LFD	a3,  0 * SIZE(AO2)

  1789. 	LFD	a4,  1 * SIZE(AO2)

  1790. 

  1791. 	FMADD	y01, alpha1r, a1, y01

  1792. 	FMADD	y02, alpha1i, a1, y02

  1793. 	FMSUBX	y01, alpha1i, a2, y01

  1794. 	FMADDX	y02, alpha1r, a2, y02

  1795. 

  1796. 	FMADD	y01, alpha2r, a3, y01

  1797. 	FMADD	y02, alpha2i, a3, y02

  1798. 	FMSUBX	y01, alpha2i, a4, y01

  1799. 	FMADDX	y02, alpha2r, a4, y02

  1800. 

  1801. 	STFD	y01, 0 * SIZE(Y2)

  1802. 	STFD	y02, 1 * SIZE(Y2)

  1803. 	add	Y1, Y1, INCY

  1804. 	add	Y2, Y2, INCY

  1805. 	.align 4

  1806. 

  1807. LL(30):

  1808. 	andi.	J, N, 1

  1809. 	ble	LL(999)

  1810. 	.align 4

  1811. 

  1812. LL(31):

  1813. 	lfd	alpha_r, ALPHA_R

  1814. 	lfd	alpha_i, ALPHA_I

  1815. 

  1816. 	LFD	a1, 0 * SIZE(X)

  1817. 	LFD	a2, 1 * SIZE(X)

  1818. 	add	X, X, INCX

  1819. 

  1820. 	FMUL	alpha1r, alpha_r, a1

  1821. 	FMUL	alpha1i, alpha_i, a1

  1822. 

  1823. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  1824. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  1825. 

  1826. 	mr	AO1, A

  1827. 	add	A,   AO1, LDA

  1828. 

  1829. 	mr	Y1, Y

  1830. 	mr	Y2, Y

  1831. 

  1832. 	srawi.	r0,  M, 3

  1833. 	mtspr	CTR, r0

  1834. 	ble	LL(35)

  1835. 	.align 4

  1836. 

  1837. 	LFD	y01,  0 * SIZE(Y1)

  1838. 	LFD	y02,  1 * SIZE(Y1)

  1839. 	LFD	y03,  2 * SIZE(Y1)

  1840. 	LFD	y04,  3 * SIZE(Y1)

  1841. 	LFD	y05,  4 * SIZE(Y1)

  1842. 	LFD	y06,  5 * SIZE(Y1)

  1843. 	LFD	y07,  6 * SIZE(Y1)

  1844. 	LFD	y08,  7 * SIZE(Y1)

  1845. 	LFD	y09,  8 * SIZE(Y1)

  1846. 	LFD	y10,  9 * SIZE(Y1)

  1847. 	LFD	y11, 10 * SIZE(Y1)

  1848. 	LFD	y12, 11 * SIZE(Y1)

  1849. 	LFD	y13, 12 * SIZE(Y1)

  1850. 	LFD	y14, 13 * SIZE(Y1)

  1851. 	LFD	y15, 14 * SIZE(Y1)

  1852. 	LFD	y16, 15 * SIZE(Y1)

  1853. 

  1854. 	LFD	a1,  0 * SIZE(AO1)

  1855. 	LFD	a2,  1 * SIZE(AO1)

  1856. 	LFD	a3,  2 * SIZE(AO1)

  1857. 	LFD	a4,  3 * SIZE(AO1)

  1858. 	LFD	a5,  4 * SIZE(AO1)

  1859. 	LFD	a6,  5 * SIZE(AO1)

  1860. 	LFD	a7,  6 * SIZE(AO1)

  1861. 	LFD	a8,  7 * SIZE(AO1)

  1862. 

  1863. 	addi	Y1, Y1, 16 * SIZE

  1864. 	bdz	LL(33)

  1865. 	.align 4

  1866. 

  1867. LL(32):

  1868. 	FMADD	y01, alpha1r, a1, y01

  1869. 	FMADD	y02, alpha1i, a1, y02

  1870. 	FMADD	y03, alpha1r, a3, y03

  1871. 	FMADD	y04, alpha1i, a3, y04

  1872. 

  1873. 	FMADD	y05, alpha1r, a5, y05

  1874. 	FMADD	y06, alpha1i, a5, y06

  1875. 	FMADD	y07, alpha1r, a7, y07

  1876. 	FMADD	y08, alpha1i, a7, y08

  1877. 

  1878. 	LFD	a1,  8 * SIZE(AO1)

  1879. 	LFD	a3, 10 * SIZE(AO1)

  1880. 	LFD	a5, 12 * SIZE(AO1)

  1881. 	LFD	a7, 14 * SIZE(AO1)

  1882. 

  1883. 	FMSUBX	y01, alpha1i, a2, y01

  1884. 	FMADDX	y02, alpha1r, a2, y02

  1885. 	FMSUBX	y03, alpha1i, a4, y03

  1886. 	FMADDX	y04, alpha1r, a4, y04

  1887. 

  1888. 	STFD	y01,  0 * SIZE(Y2)

  1889. 	STFD	y02,  1 * SIZE(Y2)

  1890. 	STFD	y03,  2 * SIZE(Y2)

  1891. 	STFD	y04,  3 * SIZE(Y2)

  1892. 

  1893. 	LFD	y01,  0 * SIZE(Y1)

  1894. 	LFD	y02,  1 * SIZE(Y1)

  1895. 	LFD	y03,  2 * SIZE(Y1)

  1896. 	LFD	y04,  3 * SIZE(Y1)

  1897. 

  1898. 	FMSUBX	y05, alpha1i, a6, y05

  1899. 	FMADDX	y06, alpha1r, a6, y06

  1900. 	FMSUBX	y07, alpha1i, a8, y07

  1901. 	FMADDX	y08, alpha1r, a8, y08

  1902. 

  1903. 	LFD	a2,  9 * SIZE(AO1)

  1904. 	LFD	a4, 11 * SIZE(AO1)

  1905. 	LFD	a6, 13 * SIZE(AO1)

  1906. 	LFD	a8, 15 * SIZE(AO1)

  1907. 

  1908. 	addi	AO1, AO1, 16 * SIZE

  1909. 	nop

  1910. 	DCBT(AO1, PREA)

  1911. 	nop

  1912. 

  1913. 	STFD	y05,  4 * SIZE(Y2)

  1914. 	STFD	y06,  5 * SIZE(Y2)

  1915. 	STFD	y07,  6 * SIZE(Y2)

  1916. 	STFD	y08,  7 * SIZE(Y2)

  1917. 

  1918. 	LFD	y05,  4 * SIZE(Y1)

  1919. 	LFD	y06,  5 * SIZE(Y1)

  1920. 	LFD	y07,  6 * SIZE(Y1)

  1921. 	LFD	y08,  7 * SIZE(Y1)

  1922. 

  1923. 	FMADD	y09, alpha1r, a1, y09

  1924. 	FMADD	y10, alpha1i, a1, y10

  1925. 	FMADD	y11, alpha1r, a3, y11

  1926. 	FMADD	y12, alpha1i, a3, y12

  1927. 

  1928. 	FMADD	y13, alpha1r, a5, y13

  1929. 	FMADD	y14, alpha1i, a5, y14

  1930. 	FMADD	y15, alpha1r, a7, y15

  1931. 	FMADD	y16, alpha1i, a7, y16

  1932. 

  1933. 	LFD	a1,  0 * SIZE(AO1)

  1934. 	LFD	a3,  2 * SIZE(AO1)

  1935. 	LFD	a5,  4 * SIZE(AO1)

  1936. 	LFD	a7,  6 * SIZE(AO1)

  1937. 

  1938. 	FMSUBX	y09, alpha1i, a2, y09

  1939. 	FMADDX	y10, alpha1r, a2, y10

  1940. 	FMSUBX	y11, alpha1i, a4, y11

  1941. 	FMADDX	y12, alpha1r, a4, y12

  1942. 

  1943. 	STFD	y09,  8 * SIZE(Y2)

  1944. 	STFD	y10,  9 * SIZE(Y2)

  1945. 	STFD	y11, 10 * SIZE(Y2)

  1946. 	STFD	y12, 11 * SIZE(Y2)

  1947. 

  1948. 	LFD	y09,  8 * SIZE(Y1)

  1949. 	LFD	y10,  9 * SIZE(Y1)

  1950. 	LFD	y11, 10 * SIZE(Y1)

  1951. 	LFD	y12, 11 * SIZE(Y1)

  1952. 

  1953. 	FMSUBX	y13, alpha1i, a6, y13

  1954. 	FMADDX	y14, alpha1r, a6, y14

  1955. 	FMSUBX	y15, alpha1i, a8, y15

  1956. 	FMADDX	y16, alpha1r, a8, y16

  1957. 

  1958. 	LFD	a2,  1 * SIZE(AO1)

  1959. 	LFD	a4,  3 * SIZE(AO1)

  1960. 	LFD	a6,  5 * SIZE(AO1)

  1961. 	LFD	a8,  7 * SIZE(AO1)

  1962. 

  1963. 	STFD	y13, 12 * SIZE(Y2)

  1964. 	STFD	y14, 13 * SIZE(Y2)

  1965. 	STFD	y15, 14 * SIZE(Y2)

  1966. 	STFD	y16, 15 * SIZE(Y2)

  1967. 

  1968. 	LFD	y13, 12 * SIZE(Y1)

  1969. 	LFD	y14, 13 * SIZE(Y1)

  1970. 	LFD	y15, 14 * SIZE(Y1)

  1971. 	LFD	y16, 15 * SIZE(Y1)

  1972. 

  1973. 	addi	Y1, Y1, 16 * SIZE

  1974. 	addi	Y2, Y2, 16 * SIZE

  1975. 	DCBT(Y1, PREC)

  1976. 	bdnz	LL(32)

  1977. 	.align 4

  1978. 

  1979. LL(33):

  1980. 	FMADD	y01, alpha1r, a1, y01

  1981. 	FMADD	y02, alpha1i, a1, y02

  1982. 	FMADD	y03, alpha1r, a3, y03

  1983. 	FMADD	y04, alpha1i, a3, y04

  1984. 

  1985. 	FMADD	y05, alpha1r, a5, y05

  1986. 	FMADD	y06, alpha1i, a5, y06

  1987. 	FMADD	y07, alpha1r, a7, y07

  1988. 	FMADD	y08, alpha1i, a7, y08

  1989. 

  1990. 	LFD	a1,  8 * SIZE(AO1)

  1991. 	LFD	a3, 10 * SIZE(AO1)

  1992. 	LFD	a5, 12 * SIZE(AO1)

  1993. 	LFD	a7, 14 * SIZE(AO1)

  1994. 

  1995. 	FMSUBX	y01, alpha1i, a2, y01

  1996. 	FMADDX	y02, alpha1r, a2, y02

  1997. 	FMSUBX	y03, alpha1i, a4, y03

  1998. 	FMADDX	y04, alpha1r, a4, y04

  1999. 

  2000. 	STFD	y01,  0 * SIZE(Y2)

  2001. 	STFD	y02,  1 * SIZE(Y2)

  2002. 	STFD	y03,  2 * SIZE(Y2)

  2003. 	STFD	y04,  3 * SIZE(Y2)

  2004. 

  2005. 	FMSUBX	y05, alpha1i, a6, y05

  2006. 	FMADDX	y06, alpha1r, a6, y06

  2007. 	FMSUBX	y07, alpha1i, a8, y07

  2008. 	FMADDX	y08, alpha1r, a8, y08

  2009. 

  2010. 	LFD	a2,  9 * SIZE(AO1)

  2011. 	LFD	a4, 11 * SIZE(AO1)

  2012. 	LFD	a6, 13 * SIZE(AO1)

  2013. 	LFD	a8, 15 * SIZE(AO1)

  2014. 

  2015. 	STFD	y05,  4 * SIZE(Y2)

  2016. 	STFD	y06,  5 * SIZE(Y2)

  2017. 	STFD	y07,  6 * SIZE(Y2)

  2018. 	STFD	y08,  7 * SIZE(Y2)

  2019. 

  2020. 	FMADD	y09, alpha1r, a1, y09

  2021. 	FMADD	y10, alpha1i, a1, y10

  2022. 	FMADD	y11, alpha1r, a3, y11

  2023. 	FMADD	y12, alpha1i, a3, y12

  2024. 

  2025. 	FMADD	y13, alpha1r, a5, y13

  2026. 	FMADD	y14, alpha1i, a5, y14

  2027. 	FMADD	y15, alpha1r, a7, y15

  2028. 	FMADD	y16, alpha1i, a7, y16

  2029. 

  2030. 	FMSUBX	y09, alpha1i, a2, y09

  2031. 	FMADDX	y10, alpha1r, a2, y10

  2032. 	FMSUBX	y11, alpha1i, a4, y11

  2033. 	FMADDX	y12, alpha1r, a4, y12

  2034. 

  2035. 	STFD	y09,  8 * SIZE(Y2)

  2036. 	STFD	y10,  9 * SIZE(Y2)

  2037. 	STFD	y11, 10 * SIZE(Y2)

  2038. 	STFD	y12, 11 * SIZE(Y2)

  2039. 

  2040. 	FMSUBX	y13, alpha1i, a6, y13

  2041. 	FMADDX	y14, alpha1r, a6, y14

  2042. 	FMSUBX	y15, alpha1i, a8, y15

  2043. 	FMADDX	y16, alpha1r, a8, y16

  2044. 

  2045. 	STFD	y13, 12 * SIZE(Y2)

  2046. 	STFD	y14, 13 * SIZE(Y2)

  2047. 	STFD	y15, 14 * SIZE(Y2)

  2048. 	STFD	y16, 15 * SIZE(Y2)

  2049. 

  2050. 	addi	AO1, AO1, 16 * SIZE

  2051. 	addi	Y2, Y2, 16 * SIZE

  2052. 	.align 4

  2053. 

  2054. LL(35):

  2055. 	andi.	r0, M, 7

  2056. 	ble	LL(999)

  2057. 	andi.	r0, M, 4

  2058. 	ble	LL(36)

  2059. 

  2060. 	LFD	y01, 0 * SIZE(Y1)

  2061. 	LFD	y02, 1 * SIZE(Y1)

  2062. 	LFD	y03, 2 * SIZE(Y1)

  2063. 	LFD	y04, 3 * SIZE(Y1)

  2064. 

  2065. 	LFD	a1,  0 * SIZE(AO1)

  2066. 	LFD	a3,  2 * SIZE(AO1)

  2067. 	LFD	a5,  4 * SIZE(AO1)

  2068. 	LFD	a7,  6 * SIZE(AO1)

  2069. 

  2070. 	LFD	y05, 4 * SIZE(Y1)

  2071. 	LFD	y06, 5 * SIZE(Y1)

  2072. 	LFD	y07, 6 * SIZE(Y1)

  2073. 	LFD	y08, 7 * SIZE(Y1)

  2074. 

  2075. 	LFD	a2,  1 * SIZE(AO1)

  2076. 	LFD	a4,  3 * SIZE(AO1)

  2077. 	LFD	a6,  5 * SIZE(AO1)

  2078. 	LFD	a8,  7 * SIZE(AO1)

  2079. 

  2080. 	FMADD	y01, alpha1r, a1, y01

  2081. 	FMADD	y02, alpha1i, a1, y02

  2082. 	FMADD	y03, alpha1r, a3, y03

  2083. 	FMADD	y04, alpha1i, a3, y04

  2084. 

  2085. 	FMADD	y05, alpha1r, a5, y05

  2086. 	FMADD	y06, alpha1i, a5, y06

  2087. 	FMADD	y07, alpha1r, a7, y07

  2088. 	FMADD	y08, alpha1i, a7, y08

  2089. 

  2090. 	FMSUBX	y01, alpha1i, a2, y01

  2091. 	FMADDX	y02, alpha1r, a2, y02

  2092. 	FMSUBX	y03, alpha1i, a4, y03

  2093. 	FMADDX	y04, alpha1r, a4, y04

  2094. 

  2095. 	FMSUBX	y05, alpha1i, a6, y05

  2096. 	FMADDX	y06, alpha1r, a6, y06

  2097. 	FMSUBX	y07, alpha1i, a8, y07

  2098. 	FMADDX	y08, alpha1r, a8, y08

  2099. 

  2100. 	STFD	y01, 0 * SIZE(Y2)

  2101. 	STFD	y02, 1 * SIZE(Y2)

  2102. 	STFD	y03, 2 * SIZE(Y2)

  2103. 	STFD	y04, 3 * SIZE(Y2)

  2104. 	STFD	y05, 4 * SIZE(Y2)

  2105. 	STFD	y06, 5 * SIZE(Y2)

  2106. 	STFD	y07, 6 * SIZE(Y2)

  2107. 	STFD	y08, 7 * SIZE(Y2)

  2108. 

  2109. 	addi	AO1, AO1, 8 * SIZE

  2110. 	addi	Y1, Y1, 8 * SIZE

  2111. 	addi	Y2, Y2, 8 * SIZE

  2112. 	.align 4

  2113. 

  2114. LL(36):

  2115. 	andi.	r0, M, 2

  2116. 	ble	LL(37)

  2117. 

  2118. 	LFD	a1,  0 * SIZE(AO1)

  2119. 	LFD	a2,  1 * SIZE(AO1)

  2120. 	LFD	a3,  2 * SIZE(AO1)

  2121. 	LFD	a4,  3 * SIZE(AO1)

  2122. 

  2123. 	LFD	y01, 0 * SIZE(Y1)

  2124. 	LFD	y02, 1 * SIZE(Y1)

  2125. 	LFD	y03, 2 * SIZE(Y1)

  2126. 	LFD	y04, 3 * SIZE(Y1)

  2127. 

  2128. 	FMADD	y01, alpha1r, a1, y01

  2129. 	FMADD	y02, alpha1i, a1, y02

  2130. 	FMADD	y03, alpha1r, a3, y03

  2131. 	FMADD	y04, alpha1i, a3, y04

  2132. 

  2133. 	FMSUBX	y01, alpha1i, a2, y01

  2134. 	FMADDX	y02, alpha1r, a2, y02

  2135. 	FMSUBX	y03, alpha1i, a4, y03

  2136. 	FMADDX	y04, alpha1r, a4, y04

  2137. 

  2138. 	STFD	y01, 0 * SIZE(Y2)

  2139. 	STFD	y02, 1 * SIZE(Y2)

  2140. 	STFD	y03, 2 * SIZE(Y2)

  2141. 	STFD	y04, 3 * SIZE(Y2)

  2142. 

  2143. 	addi	AO1, AO1, 4 * SIZE

  2144. 	addi	Y1, Y1, 4 * SIZE

  2145. 	addi	Y2, Y2, 4 * SIZE

  2146. 	.align 4

  2147. 

  2148. LL(37):

  2149. 	andi.	r0, M, 1

  2150. 	ble	LL(999)

  2151. 

  2152. 	LFD	y01, 0 * SIZE(Y1)

  2153. 	LFD	y02, 1 * SIZE(Y1)

  2154. 	LFD	a1,  0 * SIZE(AO1)

  2155. 	LFD	a2,  1 * SIZE(AO1)

  2156. 

  2157. 	FMADD	y01, alpha1r, a1, y01

  2158. 	FMADD	y02, alpha1i, a1, y02

  2159. 	FMSUBX	y01, alpha1i, a2, y01

  2160. 	FMADDX	y02, alpha1r, a2, y02

  2161. 

  2162. 	STFD	y01, 0 * SIZE(Y2)

  2163. 	STFD	y02, 1 * SIZE(Y2)

  2164. 	add	Y1, Y1, INCY

  2165. 	add	Y2, Y2, INCY

  2166. 	b	LL(999)

  2167. 	.align 4

  2168. 

  2169. LL(100):

  2170. 	srawi.	J, N, 2

  2171. 	ble	LL(120)

  2172. 	.align 4

  2173. 

  2174. LL(111):

  2175. 	lfd	alpha_r, ALPHA_R

  2176. 	lfd	alpha_i, ALPHA_I

  2177. 

  2178. 	LFD	a1, 0 * SIZE(X)

  2179. 	LFD	a2, 1 * SIZE(X)

  2180. 	add	X, X, INCX

  2181. 	LFD	a3, 0 * SIZE(X)

  2182. 	LFD	a4, 1 * SIZE(X)

  2183. 	add	X, X, INCX

  2184. 	LFD	a5, 0 * SIZE(X)

  2185. 	LFD	a6, 1 * SIZE(X)

  2186. 	add	X, X, INCX

  2187. 	LFD	a7, 0 * SIZE(X)

  2188. 	LFD	a8, 1 * SIZE(X)

  2189. 	add	X, X, INCX

  2190. 

  2191. 	FMUL	alpha1r, alpha_r, a1

  2192. 	FMUL	alpha1i, alpha_i, a1

  2193. 	FMUL	alpha2r, alpha_r, a3

  2194. 	FMUL	alpha2i, alpha_i, a3

  2195. 	FMUL	alpha3r, alpha_r, a5

  2196. 	FMUL	alpha3i, alpha_i, a5

  2197. 	FMUL	alpha4r, alpha_r, a7

  2198. 	FMUL	alpha4i, alpha_i, a7

  2199. 

  2200. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  2201. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  2202. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  2203. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  2204. 	FMSUBR	alpha3r, alpha_i, a6, alpha3r

  2205. 	FMADDR	alpha3i, alpha_r, a6, alpha3i

  2206. 	FMSUBR	alpha4r, alpha_i, a8, alpha4r

  2207. 	FMADDR	alpha4i, alpha_r, a8, alpha4i

  2208. 

  2209. 	mr	AO1, A

  2210. 	add	AO2, A,   LDA

  2211. 	add	AO3, AO2, LDA

  2212. 	add	AO4, AO3, LDA

  2213. 	add	A,   AO4, LDA

  2214. 

  2215. 	mr	Y1, Y

  2216. 	mr	Y2, Y

  2217. 

  2218. 	srawi.	r0,  M, 3

  2219. 	mtspr	CTR, r0

  2220. 	ble	LL(115)

  2221. 	.align 4

  2222. 

  2223. 	LFD	y01, 0 * SIZE(Y1)

  2224. 	LFD	y02, 1 * SIZE(Y1)

  2225. 	add	Y1, Y1, INCY

  2226. 	LFD	y03, 0 * SIZE(Y1)

  2227. 	LFD	y04, 1 * SIZE(Y1)

  2228. 	add	Y1, Y1, INCY

  2229. 	LFD	y05, 0 * SIZE(Y1)

  2230. 	LFD	y06, 1 * SIZE(Y1)

  2231. 	add	Y1, Y1, INCY

  2232. 	LFD	y07, 0 * SIZE(Y1)

  2233. 	LFD	y08, 1 * SIZE(Y1)

  2234. 	add	Y1, Y1, INCY

  2235. 	LFD	y09, 0 * SIZE(Y1)

  2236. 	LFD	y10, 1 * SIZE(Y1)

  2237. 	add	Y1, Y1, INCY

  2238. 	LFD	y11, 0 * SIZE(Y1)

  2239. 	LFD	y12, 1 * SIZE(Y1)

  2240. 	add	Y1, Y1, INCY

  2241. 	LFD	y13, 0 * SIZE(Y1)

  2242. 	LFD	y14, 1 * SIZE(Y1)

  2243. 	add	Y1, Y1, INCY

  2244. 	LFD	y15, 0 * SIZE(Y1)

  2245. 	LFD	y16, 1 * SIZE(Y1)

  2246. 	add	Y1, Y1, INCY

  2247. 

  2248. 	LFD	a1,  0 * SIZE(AO1)

  2249. 	LFD	a2,  1 * SIZE(AO1)

  2250. 	LFD	a3,  2 * SIZE(AO1)

  2251. 	LFD	a4,  3 * SIZE(AO1)

  2252. 	LFD	a5,  4 * SIZE(AO1)

  2253. 	LFD	a6,  5 * SIZE(AO1)

  2254. 	LFD	a7,  6 * SIZE(AO1)

  2255. 	LFD	a8,  7 * SIZE(AO1)

  2256. 	bdz	LL(113)

  2257. 	.align 4

  2258. 

  2259. LL(112):

  2260. 	FMADD	y01, alpha1r, a1, y01

  2261. 	FMADD	y02, alpha1i, a1, y02

  2262. 	FMADD	y03, alpha1r, a3, y03

  2263. 	FMADD	y04, alpha1i, a3, y04

  2264. 

  2265. 	FMADD	y05, alpha1r, a5, y05

  2266. 	FMADD	y06, alpha1i, a5, y06

  2267. 	FMADD	y07, alpha1r, a7, y07

  2268. 	FMADD	y08, alpha1i, a7, y08

  2269. 

  2270. 	LFD	a1,  8 * SIZE(AO1)

  2271. 	LFD	a3, 10 * SIZE(AO1)

  2272. 	LFD	a5, 12 * SIZE(AO1)

  2273. 	LFD	a7, 14 * SIZE(AO1)

  2274. 

  2275. 	FMSUBX	y01, alpha1i, a2, y01

  2276. 	FMADDX	y02, alpha1r, a2, y02

  2277. 	FMSUBX	y03, alpha1i, a4, y03

  2278. 	FMADDX	y04, alpha1r, a4, y04

  2279. 

  2280. 	FMSUBX	y05, alpha1i, a6, y05

  2281. 	FMADDX	y06, alpha1r, a6, y06

  2282. 	FMSUBX	y07, alpha1i, a8, y07

  2283. 	FMADDX	y08, alpha1r, a8, y08

  2284. 

  2285. 	LFD	a2,  9 * SIZE(AO1)

  2286. 	LFD	a4, 11 * SIZE(AO1)

  2287. 	LFD	a6, 13 * SIZE(AO1)

  2288. 	LFD	a8, 15 * SIZE(AO1)

  2289. 

  2290. 	addi	AO1, AO1, 16 * SIZE

  2291. 	nop

  2292. 	DCBT(AO1, PREA)

  2293. 	nop

  2294. 

  2295. 	FMADD	y09, alpha1r, a1, y09

  2296. 	FMADD	y10, alpha1i, a1, y10

  2297. 	FMADD	y11, alpha1r, a3, y11

  2298. 	FMADD	y12, alpha1i, a3, y12

  2299. 

  2300. 	FMADD	y13, alpha1r, a5, y13

  2301. 	FMADD	y14, alpha1i, a5, y14

  2302. 	FMADD	y15, alpha1r, a7, y15

  2303. 	FMADD	y16, alpha1i, a7, y16

  2304. 

  2305. 	LFD	a1,  0 * SIZE(AO2)

  2306. 	LFD	a3,  2 * SIZE(AO2)

  2307. 	LFD	a5,  4 * SIZE(AO2)

  2308. 	LFD	a7,  6 * SIZE(AO2)

  2309. 

  2310. 	FMSUBX	y09, alpha1i, a2, y09

  2311. 	FMADDX	y10, alpha1r, a2, y10

  2312. 	FMSUBX	y11, alpha1i, a4, y11

  2313. 	FMADDX	y12, alpha1r, a4, y12

  2314. 

  2315. 	FMSUBX	y13, alpha1i, a6, y13

  2316. 	FMADDX	y14, alpha1r, a6, y14

  2317. 	FMSUBX	y15, alpha1i, a8, y15

  2318. 	FMADDX	y16, alpha1r, a8, y16

  2319. 

  2320. 	LFD	a2,  1 * SIZE(AO2)

  2321. 	LFD	a4,  3 * SIZE(AO2)

  2322. 	LFD	a6,  5 * SIZE(AO2)

  2323. 	LFD	a8,  7 * SIZE(AO2)

  2324. 

  2325. 	FMADD	y01, alpha2r, a1, y01

  2326. 	FMADD	y02, alpha2i, a1, y02

  2327. 	FMADD	y03, alpha2r, a3, y03

  2328. 	FMADD	y04, alpha2i, a3, y04

  2329. 

  2330. 	FMADD	y05, alpha2r, a5, y05

  2331. 	FMADD	y06, alpha2i, a5, y06

  2332. 	FMADD	y07, alpha2r, a7, y07

  2333. 	FMADD	y08, alpha2i, a7, y08

  2334. 

  2335. 	LFD	a1,  8 * SIZE(AO2)

  2336. 	LFD	a3, 10 * SIZE(AO2)

  2337. 	LFD	a5, 12 * SIZE(AO2)

  2338. 	LFD	a7, 14 * SIZE(AO2)

  2339. 

  2340. 	FMSUBX	y01, alpha2i, a2, y01

  2341. 	FMADDX	y02, alpha2r, a2, y02

  2342. 	FMSUBX	y03, alpha2i, a4, y03

  2343. 	FMADDX	y04, alpha2r, a4, y04

  2344. 

  2345. 	FMSUBX	y05, alpha2i, a6, y05

  2346. 	FMADDX	y06, alpha2r, a6, y06

  2347. 	FMSUBX	y07, alpha2i, a8, y07

  2348. 	FMADDX	y08, alpha2r, a8, y08

  2349. 

  2350. 	LFD	a2,  9 * SIZE(AO2)

  2351. 	LFD	a4, 11 * SIZE(AO2)

  2352. 	LFD	a6, 13 * SIZE(AO2)

  2353. 	LFD	a8, 15 * SIZE(AO2)

  2354. 

  2355. 	addi	AO2, AO2, 16 * SIZE

  2356. 	nop

  2357. 	DCBT(AO2, PREA)

  2358. 	nop

  2359. 

  2360. 	FMADD	y09, alpha2r, a1, y09

  2361. 	FMADD	y10, alpha2i, a1, y10

  2362. 	FMADD	y11, alpha2r, a3, y11

  2363. 	FMADD	y12, alpha2i, a3, y12

  2364. 

  2365. 	FMADD	y13, alpha2r, a5, y13

  2366. 	FMADD	y14, alpha2i, a5, y14

  2367. 	FMADD	y15, alpha2r, a7, y15

  2368. 	FMADD	y16, alpha2i, a7, y16

  2369. 

  2370. 	LFD	a1,  0 * SIZE(AO3)

  2371. 	LFD	a3,  2 * SIZE(AO3)

  2372. 	LFD	a5,  4 * SIZE(AO3)

  2373. 	LFD	a7,  6 * SIZE(AO3)

  2374. 

  2375. 	FMSUBX	y09, alpha2i, a2, y09

  2376. 	FMADDX	y10, alpha2r, a2, y10

  2377. 	FMSUBX	y11, alpha2i, a4, y11

  2378. 	FMADDX	y12, alpha2r, a4, y12

  2379. 

  2380. 	FMSUBX	y13, alpha2i, a6, y13

  2381. 	FMADDX	y14, alpha2r, a6, y14

  2382. 	FMSUBX	y15, alpha2i, a8, y15

  2383. 	FMADDX	y16, alpha2r, a8, y16

  2384. 

  2385. 	LFD	a2,  1 * SIZE(AO3)

  2386. 	LFD	a4,  3 * SIZE(AO3)

  2387. 	LFD	a6,  5 * SIZE(AO3)

  2388. 	LFD	a8,  7 * SIZE(AO3)

  2389. 

  2390. 	FMADD	y01, alpha3r, a1, y01

  2391. 	FMADD	y02, alpha3i, a1, y02

  2392. 	FMADD	y03, alpha3r, a3, y03

  2393. 	FMADD	y04, alpha3i, a3, y04

  2394. 

  2395. 	FMADD	y05, alpha3r, a5, y05

  2396. 	FMADD	y06, alpha3i, a5, y06

  2397. 	FMADD	y07, alpha3r, a7, y07

  2398. 	FMADD	y08, alpha3i, a7, y08

  2399. 

  2400. 	LFD	a1,  8 * SIZE(AO3)

  2401. 	LFD	a3, 10 * SIZE(AO3)

  2402. 	LFD	a5, 12 * SIZE(AO3)

  2403. 	LFD	a7, 14 * SIZE(AO3)

  2404. 

  2405. 	FMSUBX	y01, alpha3i, a2, y01

  2406. 	FMADDX	y02, alpha3r, a2, y02

  2407. 	FMSUBX	y03, alpha3i, a4, y03

  2408. 	FMADDX	y04, alpha3r, a4, y04

  2409. 

  2410. 	FMSUBX	y05, alpha3i, a6, y05

  2411. 	FMADDX	y06, alpha3r, a6, y06

  2412. 	FMSUBX	y07, alpha3i, a8, y07

  2413. 	FMADDX	y08, alpha3r, a8, y08

  2414. 

  2415. 	LFD	a2,  9 * SIZE(AO3)

  2416. 	LFD	a4, 11 * SIZE(AO3)

  2417. 	LFD	a6, 13 * SIZE(AO3)

  2418. 	LFD	a8, 15 * SIZE(AO3)

  2419. 

  2420. 	addi	AO3, AO3, 16 * SIZE

  2421. 	nop

  2422. 	DCBT(AO3, PREA)

  2423. 	nop

  2424. 

  2425. 	FMADD	y09, alpha3r, a1, y09

  2426. 	FMADD	y10, alpha3i, a1, y10

  2427. 	FMADD	y11, alpha3r, a3, y11

  2428. 	FMADD	y12, alpha3i, a3, y12

  2429. 

  2430. 	FMADD	y13, alpha3r, a5, y13

  2431. 	FMADD	y14, alpha3i, a5, y14

  2432. 	FMADD	y15, alpha3r, a7, y15

  2433. 	FMADD	y16, alpha3i, a7, y16

  2434. 

  2435. 	LFD	a1,  0 * SIZE(AO4)

  2436. 	LFD	a3,  2 * SIZE(AO4)

  2437. 	LFD	a5,  4 * SIZE(AO4)

  2438. 	LFD	a7,  6 * SIZE(AO4)

  2439. 

  2440. 	FMSUBX	y09, alpha3i, a2, y09

  2441. 	FMADDX	y10, alpha3r, a2, y10

  2442. 	FMSUBX	y11, alpha3i, a4, y11

  2443. 	FMADDX	y12, alpha3r, a4, y12

  2444. 

  2445. 	FMSUBX	y13, alpha3i, a6, y13

  2446. 	FMADDX	y14, alpha3r, a6, y14

  2447. 	FMSUBX	y15, alpha3i, a8, y15

  2448. 	FMADDX	y16, alpha3r, a8, y16

  2449. 

  2450. 	LFD	a2,  1 * SIZE(AO4)

  2451. 	LFD	a4,  3 * SIZE(AO4)

  2452. 	LFD	a6,  5 * SIZE(AO4)

  2453. 	LFD	a8,  7 * SIZE(AO4)

  2454. 

  2455. 	FMADD	y01, alpha4r, a1, y01

  2456. 	FMADD	y02, alpha4i, a1, y02

  2457. 	FMADD	y03, alpha4r, a3, y03

  2458. 	FMADD	y04, alpha4i, a3, y04

  2459. 

  2460. 	FMADD	y05, alpha4r, a5, y05

  2461. 	FMADD	y06, alpha4i, a5, y06

  2462. 	FMADD	y07, alpha4r, a7, y07

  2463. 	FMADD	y08, alpha4i, a7, y08

  2464. 

  2465. 	LFD	a1,  8 * SIZE(AO4)

  2466. 	LFD	a3, 10 * SIZE(AO4)

  2467. 	LFD	a5, 12 * SIZE(AO4)

  2468.  	LFD	a7, 14 * SIZE(AO4)

  2469. 

  2470. 	FMSUBX	y01, alpha4i, a2, y01

  2471. 	FMADDX	y02, alpha4r, a2, y02

  2472. 	FMSUBX	y03, alpha4i, a4, y03

  2473. 	FMADDX	y04, alpha4r, a4, y04

  2474. 

  2475. 	STFD	y01, 0 * SIZE(Y2)

  2476. 	nop

  2477. 	STFD	y02, 1 * SIZE(Y2)

  2478. 	add	Y2, Y2, INCY

  2479. 

  2480. 	LFD	y01, 0 * SIZE(Y1)

  2481. 	nop

  2482. 	LFD	y02, 1 * SIZE(Y1)

  2483. 	add	Y1, Y1, INCY

  2484. 

  2485. 	STFD	y03, 0 * SIZE(Y2)

  2486. 	nop

  2487. 	STFD	y04, 1 * SIZE(Y2)

  2488. 	add	Y2, Y2, INCY

  2489. 

  2490. 	LFD	y03, 0 * SIZE(Y1)

  2491. 	nop

  2492. 	LFD	y04, 1 * SIZE(Y1)

  2493. 	add	Y1, Y1, INCY

  2494. 

  2495. 	FMSUBX	y05, alpha4i, a6, y05

  2496. 	FMADDX	y06, alpha4r, a6, y06

  2497. 	FMSUBX	y07, alpha4i, a8, y07

  2498. 	FMADDX	y08, alpha4r, a8, y08

  2499. 

  2500. 	LFD	a2,  9 * SIZE(AO4)

  2501. 	LFD	a4, 11 * SIZE(AO4)

  2502. 	LFD	a6, 13 * SIZE(AO4)

  2503. 	LFD	a8, 15 * SIZE(AO4)

  2504. 

  2505. 	addi	AO4, AO4, 16 * SIZE

  2506. 	nop

  2507. 	DCBT(AO4, PREA)

  2508. 	nop

  2509. 

  2510. 	STFD	y05, 0 * SIZE(Y2)

  2511. 	nop

  2512. 	STFD	y06, 1 * SIZE(Y2)

  2513. 	add	Y2, Y2, INCY

  2514. 

  2515. 	LFD	y05, 0 * SIZE(Y1)

  2516. 	nop

  2517. 	LFD	y06, 1 * SIZE(Y1)

  2518. 	add	Y1, Y1, INCY

  2519. 

  2520. 	STFD	y07, 0 * SIZE(Y2)

  2521. 	nop

  2522. 	STFD	y08, 1 * SIZE(Y2)

  2523. 	add	Y2, Y2, INCY

  2524. 

  2525. 	LFD	y07, 0 * SIZE(Y1)

  2526. 	nop

  2527. 	LFD	y08, 1 * SIZE(Y1)

  2528. 	add	Y1, Y1, INCY

  2529. 

  2530. 	FMADD	y09, alpha4r, a1, y09

  2531. 	FMADD	y10, alpha4i, a1, y10

  2532. 	FMADD	y11, alpha4r, a3, y11

  2533. 	FMADD	y12, alpha4i, a3, y12

  2534. 

  2535. 	FMADD	y13, alpha4r, a5, y13

  2536. 	FMADD	y14, alpha4i, a5, y14

  2537. 	FMADD	y15, alpha4r, a7, y15

  2538. 	FMADD	y16, alpha4i, a7, y16

  2539. 

  2540. 	LFD	a1,  0 * SIZE(AO1)

  2541. 	LFD	a3,  2 * SIZE(AO1)

  2542. 	LFD	a5,  4 * SIZE(AO1)

  2543. 	LFD	a7,  6 * SIZE(AO1)

  2544. 

  2545. 	FMSUBX	y09, alpha4i, a2, y09

  2546. 	FMADDX	y10, alpha4r, a2, y10

  2547. 	FMSUBX	y11, alpha4i, a4, y11

  2548. 	FMADDX	y12, alpha4r, a4, y12

  2549. 

  2550. 	STFD	y09, 0 * SIZE(Y2)

  2551. 	nop

  2552. 	STFD	y10, 1 * SIZE(Y2)

  2553. 	add	Y2, Y2, INCY

  2554. 

  2555. 	LFD	y09, 0 * SIZE(Y1)

  2556. 	nop

  2557. 	LFD	y10, 1 * SIZE(Y1)

  2558. 	add	Y1, Y1, INCY

  2559. 

  2560. 	STFD	y11, 0 * SIZE(Y2)

  2561. 	nop

  2562. 	STFD	y12, 1 * SIZE(Y2)

  2563. 	add	Y2, Y2, INCY

  2564. 

  2565. 	LFD	y11, 0 * SIZE(Y1)

  2566. 	nop

  2567. 	LFD	y12, 1 * SIZE(Y1)

  2568. 	add	Y1, Y1, INCY

  2569. 

  2570. 	FMSUBX	y13, alpha4i, a6, y13

  2571. 	FMADDX	y14, alpha4r, a6, y14

  2572. 	FMSUBX	y15, alpha4i, a8, y15

  2573. 	FMADDX	y16, alpha4r, a8, y16

  2574. 

  2575. 	LFD	a2,  1 * SIZE(AO1)

  2576. 	LFD	a4,  3 * SIZE(AO1)

  2577. 	LFD	a6,  5 * SIZE(AO1)

  2578. 	LFD	a8,  7 * SIZE(AO1)

  2579. 

  2580. 	STFD	y13, 0 * SIZE(Y2)

  2581. 	nop

  2582. 	STFD	y14, 1 * SIZE(Y2)

  2583. 	add	Y2, Y2, INCY

  2584. 

  2585. 	LFD	y13, 0 * SIZE(Y1)

  2586. 	nop

  2587. 	LFD	y14, 1 * SIZE(Y1)

  2588. 	add	Y1, Y1, INCY

  2589. 

  2590. 	STFD	y15, 0 * SIZE(Y2)

  2591. 	nop

  2592. 	STFD	y16, 1 * SIZE(Y2)

  2593. 	add	Y2, Y2, INCY

  2594. 

  2595. 	LFD	y15, 0 * SIZE(Y1)

  2596. 	nop

  2597. 	LFD	y16, 1 * SIZE(Y1)

  2598. 	add	Y1, Y1, INCY

  2599. 

  2600. 	DCBT(Y1, PREC)

  2601. 	bdnz	LL(112)

  2602. 	.align 4

  2603. 

  2604. LL(113):

  2605. 	FMADD	y01, alpha1r, a1, y01

  2606. 	FMADD	y02, alpha1i, a1, y02

  2607. 	FMADD	y03, alpha1r, a3, y03

  2608. 	FMADD	y04, alpha1i, a3, y04

  2609. 

  2610. 	FMADD	y05, alpha1r, a5, y05

  2611. 	FMADD	y06, alpha1i, a5, y06

  2612. 	FMADD	y07, alpha1r, a7, y07

  2613. 	FMADD	y08, alpha1i, a7, y08

  2614. 

  2615. 	LFD	a1,  8 * SIZE(AO1)

  2616. 	LFD	a3, 10 * SIZE(AO1)

  2617. 	LFD	a5, 12 * SIZE(AO1)

  2618. 	LFD	a7, 14 * SIZE(AO1)

  2619. 

  2620. 	FMSUBX	y01, alpha1i, a2, y01

  2621. 	FMADDX	y02, alpha1r, a2, y02

  2622. 	FMSUBX	y03, alpha1i, a4, y03

  2623. 	FMADDX	y04, alpha1r, a4, y04

  2624. 

  2625. 	FMSUBX	y05, alpha1i, a6, y05

  2626. 	FMADDX	y06, alpha1r, a6, y06

  2627. 	FMSUBX	y07, alpha1i, a8, y07

  2628. 	FMADDX	y08, alpha1r, a8, y08

  2629. 

  2630. 	LFD	a2,  9 * SIZE(AO1)

  2631. 	LFD	a4, 11 * SIZE(AO1)

  2632. 	LFD	a6, 13 * SIZE(AO1)

  2633. 	LFD	a8, 15 * SIZE(AO1)

  2634. 

  2635. 	FMADD	y09, alpha1r, a1, y09

  2636. 	FMADD	y10, alpha1i, a1, y10

  2637. 	FMADD	y11, alpha1r, a3, y11

  2638. 	FMADD	y12, alpha1i, a3, y12

  2639. 

  2640. 	FMADD	y13, alpha1r, a5, y13

  2641. 	FMADD	y14, alpha1i, a5, y14

  2642. 	FMADD	y15, alpha1r, a7, y15

  2643. 	FMADD	y16, alpha1i, a7, y16

  2644. 

  2645. 	LFD	a1,  0 * SIZE(AO2)

  2646. 	LFD	a3,  2 * SIZE(AO2)

  2647. 	LFD	a5,  4 * SIZE(AO2)

  2648. 	LFD	a7,  6 * SIZE(AO2)

  2649. 

  2650. 	FMSUBX	y09, alpha1i, a2, y09

  2651. 	FMADDX	y10, alpha1r, a2, y10

  2652. 	FMSUBX	y11, alpha1i, a4, y11

  2653. 	FMADDX	y12, alpha1r, a4, y12

  2654. 

  2655. 	FMSUBX	y13, alpha1i, a6, y13

  2656. 	FMADDX	y14, alpha1r, a6, y14

  2657. 	FMSUBX	y15, alpha1i, a8, y15

  2658. 	FMADDX	y16, alpha1r, a8, y16

  2659. 

  2660. 	LFD	a2,  1 * SIZE(AO2)

  2661. 	LFD	a4,  3 * SIZE(AO2)

  2662. 	LFD	a6,  5 * SIZE(AO2)

  2663. 	LFD	a8,  7 * SIZE(AO2)

  2664. 

  2665. 	FMADD	y01, alpha2r, a1, y01

  2666. 	FMADD	y02, alpha2i, a1, y02

  2667. 	FMADD	y03, alpha2r, a3, y03

  2668. 	FMADD	y04, alpha2i, a3, y04

  2669. 

  2670. 	FMADD	y05, alpha2r, a5, y05

  2671. 	FMADD	y06, alpha2i, a5, y06

  2672. 	FMADD	y07, alpha2r, a7, y07

  2673. 	FMADD	y08, alpha2i, a7, y08

  2674. 

  2675. 	LFD	a1,  8 * SIZE(AO2)

  2676. 	LFD	a3, 10 * SIZE(AO2)

  2677. 	LFD	a5, 12 * SIZE(AO2)

  2678. 	LFD	a7, 14 * SIZE(AO2)

  2679. 

  2680. 	FMSUBX	y01, alpha2i, a2, y01

  2681. 	FMADDX	y02, alpha2r, a2, y02

  2682. 	FMSUBX	y03, alpha2i, a4, y03

  2683. 	FMADDX	y04, alpha2r, a4, y04

  2684. 

  2685. 	FMSUBX	y05, alpha2i, a6, y05

  2686. 	FMADDX	y06, alpha2r, a6, y06

  2687. 	FMSUBX	y07, alpha2i, a8, y07

  2688. 	FMADDX	y08, alpha2r, a8, y08

  2689. 

  2690. 	LFD	a2,  9 * SIZE(AO2)

  2691. 	LFD	a4, 11 * SIZE(AO2)

  2692. 	LFD	a6, 13 * SIZE(AO2)

  2693. 	LFD	a8, 15 * SIZE(AO2)

  2694. 

  2695. 	FMADD	y09, alpha2r, a1, y09

  2696. 	FMADD	y10, alpha2i, a1, y10

  2697. 	FMADD	y11, alpha2r, a3, y11

  2698. 	FMADD	y12, alpha2i, a3, y12

  2699. 

  2700. 	FMADD	y13, alpha2r, a5, y13

  2701. 	FMADD	y14, alpha2i, a5, y14

  2702. 	FMADD	y15, alpha2r, a7, y15

  2703. 	FMADD	y16, alpha2i, a7, y16

  2704. 

  2705. 	LFD	a1,  0 * SIZE(AO3)

  2706. 	LFD	a3,  2 * SIZE(AO3)

  2707. 	LFD	a5,  4 * SIZE(AO3)

  2708. 	LFD	a7,  6 * SIZE(AO3)

  2709. 

  2710. 	FMSUBX	y09, alpha2i, a2, y09

  2711. 	FMADDX	y10, alpha2r, a2, y10

  2712. 	FMSUBX	y11, alpha2i, a4, y11

  2713. 	FMADDX	y12, alpha2r, a4, y12

  2714. 

  2715. 	FMSUBX	y13, alpha2i, a6, y13

  2716. 	FMADDX	y14, alpha2r, a6, y14

  2717. 	FMSUBX	y15, alpha2i, a8, y15

  2718. 	FMADDX	y16, alpha2r, a8, y16

  2719. 

  2720. 	LFD	a2,  1 * SIZE(AO3)

  2721. 	LFD	a4,  3 * SIZE(AO3)

  2722. 	LFD	a6,  5 * SIZE(AO3)

  2723. 	LFD	a8,  7 * SIZE(AO3)

  2724. 

  2725. 	FMADD	y01, alpha3r, a1, y01

  2726. 	FMADD	y02, alpha3i, a1, y02

  2727. 	FMADD	y03, alpha3r, a3, y03

  2728. 	FMADD	y04, alpha3i, a3, y04

  2729. 

  2730. 	FMADD	y05, alpha3r, a5, y05

  2731. 	FMADD	y06, alpha3i, a5, y06

  2732. 	FMADD	y07, alpha3r, a7, y07

  2733. 	FMADD	y08, alpha3i, a7, y08

  2734. 

  2735. 	LFD	a1,  8 * SIZE(AO3)

  2736. 	LFD	a3, 10 * SIZE(AO3)

  2737. 	LFD	a5, 12 * SIZE(AO3)

  2738. 	LFD	a7, 14 * SIZE(AO3)

  2739. 

  2740. 	FMSUBX	y01, alpha3i, a2, y01

  2741. 	FMADDX	y02, alpha3r, a2, y02

  2742. 	FMSUBX	y03, alpha3i, a4, y03

  2743. 	FMADDX	y04, alpha3r, a4, y04

  2744. 

  2745. 	FMSUBX	y05, alpha3i, a6, y05

  2746. 	FMADDX	y06, alpha3r, a6, y06

  2747. 	FMSUBX	y07, alpha3i, a8, y07

  2748. 	FMADDX	y08, alpha3r, a8, y08

  2749. 

  2750. 	LFD	a2,  9 * SIZE(AO3)

  2751. 	LFD	a4, 11 * SIZE(AO3)

  2752. 	LFD	a6, 13 * SIZE(AO3)

  2753. 	LFD	a8, 15 * SIZE(AO3)

  2754. 

  2755. 	FMADD	y09, alpha3r, a1, y09

  2756. 	FMADD	y10, alpha3i, a1, y10

  2757. 	FMADD	y11, alpha3r, a3, y11

  2758. 	FMADD	y12, alpha3i, a3, y12

  2759. 

  2760. 	FMADD	y13, alpha3r, a5, y13

  2761. 	FMADD	y14, alpha3i, a5, y14

  2762. 	FMADD	y15, alpha3r, a7, y15

  2763. 	FMADD	y16, alpha3i, a7, y16

  2764. 

  2765. 	LFD	a1,  0 * SIZE(AO4)

  2766. 	LFD	a3,  2 * SIZE(AO4)

  2767. 	LFD	a5,  4 * SIZE(AO4)

  2768. 	LFD	a7,  6 * SIZE(AO4)

  2769. 

  2770. 	FMSUBX	y09, alpha3i, a2, y09

  2771. 	FMADDX	y10, alpha3r, a2, y10

  2772. 	FMSUBX	y11, alpha3i, a4, y11

  2773. 	FMADDX	y12, alpha3r, a4, y12

  2774. 

  2775. 	FMSUBX	y13, alpha3i, a6, y13

  2776. 	FMADDX	y14, alpha3r, a6, y14

  2777. 	FMSUBX	y15, alpha3i, a8, y15

  2778. 	FMADDX	y16, alpha3r, a8, y16

  2779. 

  2780. 	LFD	a2,  1 * SIZE(AO4)

  2781. 	LFD	a4,  3 * SIZE(AO4)

  2782. 	LFD	a6,  5 * SIZE(AO4)

  2783. 	LFD	a8,  7 * SIZE(AO4)

  2784. 

  2785. 	FMADD	y01, alpha4r, a1, y01

  2786. 	FMADD	y02, alpha4i, a1, y02

  2787. 	FMADD	y03, alpha4r, a3, y03

  2788. 	FMADD	y04, alpha4i, a3, y04

  2789. 

  2790. 	FMADD	y05, alpha4r, a5, y05

  2791. 	FMADD	y06, alpha4i, a5, y06

  2792. 	FMADD	y07, alpha4r, a7, y07

  2793. 	FMADD	y08, alpha4i, a7, y08

  2794. 

  2795. 	LFD	a1,  8 * SIZE(AO4)

  2796. 	LFD	a3, 10 * SIZE(AO4)

  2797. 	LFD	a5, 12 * SIZE(AO4)

  2798. 	LFD	a7, 14 * SIZE(AO4)

  2799. 

  2800. 	FMSUBX	y01, alpha4i, a2, y01

  2801. 	FMADDX	y02, alpha4r, a2, y02

  2802. 	FMSUBX	y03, alpha4i, a4, y03

  2803. 	FMADDX	y04, alpha4r, a4, y04

  2804. 

  2805. 	STFD	y01, 0 * SIZE(Y2)

  2806. 	nop

  2807. 	STFD	y02, 1 * SIZE(Y2)

  2808. 	add	Y2, Y2, INCY

  2809. 

  2810. 	STFD	y03, 0 * SIZE(Y2)

  2811. 	nop

  2812. 	STFD	y04, 1 * SIZE(Y2)

  2813. 	add	Y2, Y2, INCY

  2814. 

  2815. 	FMSUBX	y05, alpha4i, a6, y05

  2816. 	FMADDX	y06, alpha4r, a6, y06

  2817. 	FMSUBX	y07, alpha4i, a8, y07

  2818. 	FMADDX	y08, alpha4r, a8, y08

  2819. 

  2820. 	LFD	a2,  9 * SIZE(AO4)

  2821. 	LFD	a4, 11 * SIZE(AO4)

  2822. 	LFD	a6, 13 * SIZE(AO4)

  2823. 	LFD	a8, 15 * SIZE(AO4)

  2824. 

  2825. 	STFD	y05, 0 * SIZE(Y2)

  2826. 	nop

  2827. 	STFD	y06, 1 * SIZE(Y2)

  2828. 	add	Y2, Y2, INCY

  2829. 

  2830. 	STFD	y07, 0 * SIZE(Y2)

  2831. 	nop

  2832. 	STFD	y08, 1 * SIZE(Y2)

  2833. 	add	Y2, Y2, INCY

  2834. 

  2835. 	FMADD	y09, alpha4r, a1, y09

  2836. 	FMADD	y10, alpha4i, a1, y10

  2837. 	FMADD	y11, alpha4r, a3, y11

  2838. 	FMADD	y12, alpha4i, a3, y12

  2839. 

  2840. 	FMADD	y13, alpha4r, a5, y13

  2841. 	FMADD	y14, alpha4i, a5, y14

  2842. 	FMADD	y15, alpha4r, a7, y15

  2843. 	FMADD	y16, alpha4i, a7, y16

  2844. 

  2845. 	FMSUBX	y09, alpha4i, a2, y09

  2846. 	FMADDX	y10, alpha4r, a2, y10

  2847. 	FMSUBX	y11, alpha4i, a4, y11

  2848. 	FMADDX	y12, alpha4r, a4, y12

  2849. 

  2850. 	STFD	y09, 0 * SIZE(Y2)

  2851. 	nop

  2852. 	STFD	y10, 1 * SIZE(Y2)

  2853. 	add	Y2, Y2, INCY

  2854. 

  2855. 	STFD	y11, 0 * SIZE(Y2)

  2856. 	nop

  2857. 	STFD	y12, 1 * SIZE(Y2)

  2858. 	add	Y2, Y2, INCY

  2859. 

  2860. 	FMSUBX	y13, alpha4i, a6, y13

  2861. 	FMADDX	y14, alpha4r, a6, y14

  2862. 	FMSUBX	y15, alpha4i, a8, y15

  2863. 	FMADDX	y16, alpha4r, a8, y16

  2864. 

  2865. 	STFD	y13, 0 * SIZE(Y2)

  2866. 	nop

  2867. 	STFD	y14, 1 * SIZE(Y2)

  2868. 	add	Y2, Y2, INCY

  2869. 

  2870. 	STFD	y15, 0 * SIZE(Y2)

  2871. 	nop

  2872. 	STFD	y16, 1 * SIZE(Y2)

  2873. 	add	Y2, Y2, INCY

  2874. 

  2875. 	addi	AO1, AO1, 16 * SIZE

  2876. 	addi	AO2, AO2, 16 * SIZE

  2877. 	addi	AO3, AO3, 16 * SIZE

  2878. 	addi	AO4, AO4, 16 * SIZE

  2879. 	.align 4

  2880. 

  2881. LL(115):

  2882. 	andi.	r0, M, 7

  2883. 	ble	LL(119)

  2884. 	andi.	r0, M, 4

  2885. 	ble	LL(116)

  2886. 

  2887. 	LFD	y01, 0 * SIZE(Y1)

  2888. 	LFD	y02, 1 * SIZE(Y1)

  2889. 	add	Y1, Y1, INCY

  2890. 	LFD	y03, 0 * SIZE(Y1)

  2891. 	LFD	y04, 1 * SIZE(Y1)

  2892. 	add	Y1, Y1, INCY

  2893. 

  2894. 	LFD	a1,  0 * SIZE(AO1)

  2895. 	LFD	a3,  2 * SIZE(AO1)

  2896. 	LFD	a5,  4 * SIZE(AO1)

  2897. 	LFD	a7,  6 * SIZE(AO1)

  2898. 

  2899. 	LFD	y05, 0 * SIZE(Y1)

  2900. 	LFD	y06, 1 * SIZE(Y1)

  2901. 	add	Y1, Y1, INCY

  2902. 	LFD	y07, 0 * SIZE(Y1)

  2903. 	LFD	y08, 1 * SIZE(Y1)

  2904. 	add	Y1, Y1, INCY

  2905. 

  2906. 	LFD	a2,  1 * SIZE(AO1)

  2907. 	LFD	a4,  3 * SIZE(AO1)

  2908. 	LFD	a6,  5 * SIZE(AO1)

  2909. 	LFD	a8,  7 * SIZE(AO1)

  2910. 

  2911. 	FMADD	y01, alpha1r, a1, y01

  2912. 	FMADD	y02, alpha1i, a1, y02

  2913. 	FMADD	y03, alpha1r, a3, y03

  2914. 	FMADD	y04, alpha1i, a3, y04

  2915. 

  2916. 	FMADD	y05, alpha1r, a5, y05

  2917. 	FMADD	y06, alpha1i, a5, y06

  2918. 	FMADD	y07, alpha1r, a7, y07

  2919. 	FMADD	y08, alpha1i, a7, y08

  2920. 

  2921. 	LFD	a1,  0 * SIZE(AO2)

  2922. 	LFD	a3,  2 * SIZE(AO2)

  2923. 	LFD	a5,  4 * SIZE(AO2)

  2924. 	LFD	a7,  6 * SIZE(AO2)

  2925. 

  2926. 	FMSUBX	y01, alpha1i, a2, y01

  2927. 	FMADDX	y02, alpha1r, a2, y02

  2928. 	FMSUBX	y03, alpha1i, a4, y03

  2929. 	FMADDX	y04, alpha1r, a4, y04

  2930. 

  2931. 	FMSUBX	y05, alpha1i, a6, y05

  2932. 	FMADDX	y06, alpha1r, a6, y06

  2933. 	FMSUBX	y07, alpha1i, a8, y07

  2934. 	FMADDX	y08, alpha1r, a8, y08

  2935. 

  2936. 	LFD	a2,  1 * SIZE(AO2)

  2937. 	LFD	a4,  3 * SIZE(AO2)

  2938. 	LFD	a6,  5 * SIZE(AO2)

  2939. 	LFD	a8,  7 * SIZE(AO2)

  2940. 

  2941. 	FMADD	y01, alpha2r, a1, y01

  2942. 	FMADD	y02, alpha2i, a1, y02

  2943. 	FMADD	y03, alpha2r, a3, y03

  2944. 	FMADD	y04, alpha2i, a3, y04

  2945. 

  2946. 	FMADD	y05, alpha2r, a5, y05

  2947. 	FMADD	y06, alpha2i, a5, y06

  2948. 	FMADD	y07, alpha2r, a7, y07

  2949. 	FMADD	y08, alpha2i, a7, y08

  2950. 

  2951. 	LFD	a1,  0 * SIZE(AO3)

  2952. 	LFD	a3,  2 * SIZE(AO3)

  2953. 	LFD	a5,  4 * SIZE(AO3)

  2954. 	LFD	a7,  6 * SIZE(AO3)

  2955. 

  2956. 	FMSUBX	y01, alpha2i, a2, y01

  2957. 	FMADDX	y02, alpha2r, a2, y02

  2958. 	FMSUBX	y03, alpha2i, a4, y03

  2959. 	FMADDX	y04, alpha2r, a4, y04

  2960. 

  2961. 	FMSUBX	y05, alpha2i, a6, y05

  2962. 	FMADDX	y06, alpha2r, a6, y06

  2963. 	FMSUBX	y07, alpha2i, a8, y07

  2964. 	FMADDX	y08, alpha2r, a8, y08

  2965. 

  2966. 	LFD	a2,  1 * SIZE(AO3)

  2967. 	LFD	a4,  3 * SIZE(AO3)

  2968. 	LFD	a6,  5 * SIZE(AO3)

  2969. 	LFD	a8,  7 * SIZE(AO3)

  2970. 

  2971. 	FMADD	y01, alpha3r, a1, y01

  2972. 	FMADD	y02, alpha3i, a1, y02

  2973. 	FMADD	y03, alpha3r, a3, y03

  2974. 	FMADD	y04, alpha3i, a3, y04

  2975. 

  2976. 	FMADD	y05, alpha3r, a5, y05

  2977. 	FMADD	y06, alpha3i, a5, y06

  2978. 	FMADD	y07, alpha3r, a7, y07

  2979. 	FMADD	y08, alpha3i, a7, y08

  2980. 

  2981. 	LFD	a1,  0 * SIZE(AO4)

  2982. 	LFD	a3,  2 * SIZE(AO4)

  2983. 	LFD	a5,  4 * SIZE(AO4)

  2984. 	LFD	a7,  6 * SIZE(AO4)

  2985. 

  2986. 	FMSUBX	y01, alpha3i, a2, y01

  2987. 	FMADDX	y02, alpha3r, a2, y02

  2988. 	FMSUBX	y03, alpha3i, a4, y03

  2989. 	FMADDX	y04, alpha3r, a4, y04

  2990. 

  2991. 	FMSUBX	y05, alpha3i, a6, y05

  2992. 	FMADDX	y06, alpha3r, a6, y06

  2993. 	FMSUBX	y07, alpha3i, a8, y07

  2994. 	FMADDX	y08, alpha3r, a8, y08

  2995. 

  2996. 	LFD	a2,  1 * SIZE(AO4)

  2997. 	LFD	a4,  3 * SIZE(AO4)

  2998. 	LFD	a6,  5 * SIZE(AO4)

  2999. 	LFD	a8,  7 * SIZE(AO4)

  3000. 

  3001. 	FMADD	y01, alpha4r, a1, y01

  3002. 	FMADD	y02, alpha4i, a1, y02

  3003. 	FMADD	y03, alpha4r, a3, y03

  3004. 	FMADD	y04, alpha4i, a3, y04

  3005. 

  3006. 	FMADD	y05, alpha4r, a5, y05

  3007. 	FMADD	y06, alpha4i, a5, y06

  3008. 	FMADD	y07, alpha4r, a7, y07

  3009. 	FMADD	y08, alpha4i, a7, y08

  3010. 

  3011. 	FMSUBX	y01, alpha4i, a2, y01

  3012. 	FMADDX	y02, alpha4r, a2, y02

  3013. 	FMSUBX	y03, alpha4i, a4, y03

  3014. 	FMADDX	y04, alpha4r, a4, y04

  3015. 

  3016. 	FMSUBX	y05, alpha4i, a6, y05

  3017. 	FMADDX	y06, alpha4r, a6, y06

  3018. 	FMSUBX	y07, alpha4i, a8, y07

  3019. 	FMADDX	y08, alpha4r, a8, y08

  3020. 

  3021. 	STFD	y01, 0 * SIZE(Y2)

  3022. 	addi	AO1, AO1, 8 * SIZE

  3023. 	STFD	y02, 1 * SIZE(Y2)

  3024. 	add	Y2, Y2, INCY

  3025. 

  3026. 	STFD	y03, 0 * SIZE(Y2)

  3027. 	addi	AO2, AO2, 8 * SIZE

  3028. 	STFD	y04, 1 * SIZE(Y2)

  3029. 	add	Y2, Y2, INCY

  3030. 

  3031. 	STFD	y05, 0 * SIZE(Y2)

  3032. 	addi	AO3, AO3, 8 * SIZE

  3033. 	STFD	y06, 1 * SIZE(Y2)

  3034. 	add	Y2, Y2, INCY

  3035. 

  3036. 	STFD	y07, 0 * SIZE(Y2)

  3037. 	addi	AO4, AO4, 8 * SIZE

  3038. 	STFD	y08, 1 * SIZE(Y2)

  3039. 	add	Y2, Y2, INCY

  3040. 	.align 4

  3041. 

  3042. LL(116):

  3043. 	andi.	r0, M, 2

  3044. 	ble	LL(117)

  3045. 

  3046. 	LFD	a1,  0 * SIZE(AO1)

  3047. 	LFD	a2,  1 * SIZE(AO1)

  3048. 	LFD	a3,  2 * SIZE(AO1)

  3049. 	LFD	a4,  3 * SIZE(AO1)

  3050. 

  3051. 	LFD	y01, 0 * SIZE(Y1)

  3052. 	LFD	y02, 1 * SIZE(Y1)

  3053. 	add	Y1, Y1, INCY

  3054. 	LFD	y03, 0 * SIZE(Y1)

  3055. 	LFD	y04, 1 * SIZE(Y1)

  3056. 	add	Y1, Y1, INCY

  3057. 

  3058. 	LFD	a5,  0 * SIZE(AO2)

  3059. 	LFD	a6,  1 * SIZE(AO2)

  3060. 	LFD	a7,  2 * SIZE(AO2)

  3061. 	LFD	a8,  3 * SIZE(AO2)

  3062. 

  3063. 	FMADD	y01, alpha1r, a1, y01

  3064. 	FMADD	y02, alpha1i, a1, y02

  3065. 	FMADD	y03, alpha1r, a3, y03

  3066. 	FMADD	y04, alpha1i, a3, y04

  3067. 

  3068. 	FMSUBX	y01, alpha1i, a2, y01

  3069. 	FMADDX	y02, alpha1r, a2, y02

  3070. 	FMSUBX	y03, alpha1i, a4, y03

  3071. 	FMADDX	y04, alpha1r, a4, y04

  3072. 

  3073. 	LFD	a1,  0 * SIZE(AO3)

  3074. 	LFD	a2,  1 * SIZE(AO3)

  3075. 	LFD	a3,  2 * SIZE(AO3)

  3076. 	LFD	a4,  3 * SIZE(AO3)

  3077. 

  3078. 	FMADD	y01, alpha2r, a5, y01

  3079. 	FMADD	y02, alpha2i, a5, y02

  3080. 	FMADD	y03, alpha2r, a7, y03

  3081. 	FMADD	y04, alpha2i, a7, y04

  3082. 

  3083. 	FMSUBX	y01, alpha2i, a6, y01

  3084. 	FMADDX	y02, alpha2r, a6, y02

  3085. 	FMSUBX	y03, alpha2i, a8, y03

  3086. 	FMADDX	y04, alpha2r, a8, y04

  3087. 

  3088. 	LFD	a5,  0 * SIZE(AO4)

  3089. 	LFD	a6,  1 * SIZE(AO4)

  3090. 	LFD	a7,  2 * SIZE(AO4)

  3091. 	LFD	a8,  3 * SIZE(AO4)

  3092. 

  3093. 	FMADD	y01, alpha3r, a1, y01

  3094. 	FMADD	y02, alpha3i, a1, y02

  3095. 	FMADD	y03, alpha3r, a3, y03

  3096. 	FMADD	y04, alpha3i, a3, y04

  3097. 

  3098. 	FMSUBX	y01, alpha3i, a2, y01

  3099. 	FMADDX	y02, alpha3r, a2, y02

  3100. 	FMSUBX	y03, alpha3i, a4, y03

  3101. 	FMADDX	y04, alpha3r, a4, y04

  3102. 

  3103. 	FMADD	y01, alpha4r, a5, y01

  3104. 	FMADD	y02, alpha4i, a5, y02

  3105. 	FMADD	y03, alpha4r, a7, y03

  3106. 	FMADD	y04, alpha4i, a7, y04

  3107. 

  3108. 	FMSUBX	y01, alpha4i, a6, y01

  3109. 	FMADDX	y02, alpha4r, a6, y02

  3110. 	FMSUBX	y03, alpha4i, a8, y03

  3111. 	FMADDX	y04, alpha4r, a8, y04

  3112. 

  3113. 	STFD	y01, 0 * SIZE(Y2)

  3114. 	addi	AO1, AO1, 4 * SIZE

  3115. 	STFD	y02, 1 * SIZE(Y2)

  3116. 	add	Y2, Y2, INCY

  3117. 

  3118. 	STFD	y03, 0 * SIZE(Y2)

  3119. 	addi	AO2, AO2, 4 * SIZE

  3120. 	STFD	y04, 1 * SIZE(Y2)

  3121. 	add	Y2, Y2, INCY

  3122. 

  3123. 	addi	AO3, AO3, 4 * SIZE

  3124. 	addi	AO4, AO4, 4 * SIZE

  3125. 	.align 4

  3126. 

  3127. LL(117):

  3128. 	andi.	r0, M, 1

  3129. 	ble	LL(119)

  3130. 

  3131. 	LFD	y01, 0 * SIZE(Y1)

  3132. 	LFD	y02, 1 * SIZE(Y1)

  3133. 	add	Y1, Y1, INCY

  3134. 

  3135. 	LFD	a1,  0 * SIZE(AO1)

  3136. 	LFD	a2,  1 * SIZE(AO1)

  3137. 	LFD	a3,  0 * SIZE(AO2)

  3138. 	LFD	a4,  1 * SIZE(AO2)

  3139. 	LFD	a5,  0 * SIZE(AO3)

  3140. 	LFD	a6,  1 * SIZE(AO3)

  3141. 	LFD	a7,  0 * SIZE(AO4)

  3142. 	LFD	a8,  1 * SIZE(AO4)

  3143. 

  3144. 	FMADD	y01, alpha1r, a1, y01

  3145. 	FMADD	y02, alpha1i, a1, y02

  3146. 	FMSUBX	y01, alpha1i, a2, y01

  3147. 	FMADDX	y02, alpha1r, a2, y02

  3148. 

  3149. 	FMADD	y01, alpha2r, a3, y01

  3150. 	FMADD	y02, alpha2i, a3, y02

  3151. 	FMSUBX	y01, alpha2i, a4, y01

  3152. 	FMADDX	y02, alpha2r, a4, y02

  3153. 

  3154. 	FMADD	y01, alpha3r, a5, y01

  3155. 	FMADD	y02, alpha3i, a5, y02

  3156. 	FMSUBX	y01, alpha3i, a6, y01

  3157. 	FMADDX	y02, alpha3r, a6, y02

  3158. 

  3159. 	FMADD	y01, alpha4r, a7, y01

  3160. 	FMADD	y02, alpha4i, a7, y02

  3161. 	FMSUBX	y01, alpha4i, a8, y01

  3162. 	FMADDX	y02, alpha4r, a8, y02

  3163. 

  3164. 	STFD	y01, 0 * SIZE(Y2)

  3165. 	STFD	y02, 1 * SIZE(Y2)

  3166. 	add	Y2, Y2, INCY

  3167. 	.align 4

  3168. 

  3169. LL(119):

  3170. 	addi	J, J, -1

  3171. 	cmpi	cr0, 0, J, 0

  3172. 	bgt	LL(111)

  3173. 	.align 4

  3174. 

  3175. LL(120):

  3176. 	andi.	J, N, 2

  3177. 	ble	LL(130)

  3178. 	.align 4

  3179. 

  3180. LL(121):

  3181. 	lfd	alpha_r, ALPHA_R

  3182. 	lfd	alpha_i, ALPHA_I

  3183. 

  3184. 	LFD	a1, 0 * SIZE(X)

  3185. 	LFD	a2, 1 * SIZE(X)

  3186. 	add	X, X, INCX

  3187. 	LFD	a3, 0 * SIZE(X)

  3188. 	LFD	a4, 1 * SIZE(X)

  3189. 	add	X, X, INCX

  3190. 

  3191. 	FMUL	alpha1r, alpha_r, a1

  3192. 	FMUL	alpha1i, alpha_i, a1

  3193. 	FMUL	alpha2r, alpha_r, a3

  3194. 	FMUL	alpha2i, alpha_i, a3

  3195. 

  3196. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  3197. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  3198. 	FMSUBR	alpha2r, alpha_i, a4, alpha2r

  3199. 	FMADDR	alpha2i, alpha_r, a4, alpha2i

  3200. 

  3201. 	mr	AO1, A

  3202. 	add	AO2, A,   LDA

  3203. 	add	A,   AO2, LDA

  3204. 

  3205. 	mr	Y1, Y

  3206. 	mr	Y2, Y

  3207. 

  3208. 	srawi.	r0,  M, 3

  3209. 	mtspr	CTR, r0

  3210. 	ble	LL(125)

  3211. 	.align 4

  3212. 

  3213. 	LFD	y01, 0 * SIZE(Y1)

  3214. 	LFD	y02, 1 * SIZE(Y1)

  3215. 	add	Y1, Y1, INCY

  3216. 	LFD	y03, 0 * SIZE(Y1)

  3217. 	LFD	y04, 1 * SIZE(Y1)

  3218. 	add	Y1, Y1, INCY

  3219. 

  3220. 	LFD	a1,  0 * SIZE(AO1)

  3221. 	LFD	a3,  2 * SIZE(AO1)

  3222. 	LFD	a5,  4 * SIZE(AO1)

  3223. 	LFD	a7,  6 * SIZE(AO1)

  3224. 

  3225. 	LFD	y05, 0 * SIZE(Y1)

  3226. 	LFD	y06, 1 * SIZE(Y1)

  3227. 	add	Y1, Y1, INCY

  3228. 	LFD	y07, 0 * SIZE(Y1)

  3229. 	LFD	y08, 1 * SIZE(Y1)

  3230. 	add	Y1, Y1, INCY

  3231. 

  3232. 	LFD	a2,  1 * SIZE(AO1)

  3233. 	LFD	a4,  3 * SIZE(AO1)

  3234. 	LFD	a6,  5 * SIZE(AO1)

  3235. 	LFD	a8,  7 * SIZE(AO1)

  3236. 

  3237. 	LFD	y09, 0 * SIZE(Y1)

  3238. 	LFD	y10, 1 * SIZE(Y1)

  3239. 	add	Y1, Y1, INCY

  3240. 	LFD	y11, 0 * SIZE(Y1)

  3241. 	LFD	y12, 1 * SIZE(Y1)

  3242. 	add	Y1, Y1, INCY

  3243. 	LFD	y13, 0 * SIZE(Y1)

  3244. 	LFD	y14, 1 * SIZE(Y1)

  3245. 	add	Y1, Y1, INCY

  3246. 	LFD	y15, 0 * SIZE(Y1)

  3247. 	LFD	y16, 1 * SIZE(Y1)

  3248. 	add	Y1, Y1, INCY

  3249. 

  3250. 	bdz	LL(123)

  3251. 	.align 4

  3252. 

  3253. LL(122):

  3254. 	FMADD	y01, alpha1r, a1, y01

  3255. 	FMADD	y02, alpha1i, a1, y02

  3256. 	FMADD	y03, alpha1r, a3, y03

  3257. 	FMADD	y04, alpha1i, a3, y04

  3258. 

  3259. 	FMADD	y05, alpha1r, a5, y05

  3260. 	FMADD	y06, alpha1i, a5, y06

  3261. 	FMADD	y07, alpha1r, a7, y07

  3262. 	FMADD	y08, alpha1i, a7, y08

  3263. 

  3264. 	LFD	a1,  8 * SIZE(AO1)

  3265. 	LFD	a3, 10 * SIZE(AO1)

  3266. 	LFD	a5, 12 * SIZE(AO1)

  3267. 	LFD	a7, 14 * SIZE(AO1)

  3268. 

  3269. 	FMSUBX	y01, alpha1i, a2, y01

  3270. 	FMADDX	y02, alpha1r, a2, y02

  3271. 	FMSUBX	y03, alpha1i, a4, y03

  3272. 	FMADDX	y04, alpha1r, a4, y04

  3273. 

  3274. 	FMSUBX	y05, alpha1i, a6, y05

  3275. 	FMADDX	y06, alpha1r, a6, y06

  3276. 	FMSUBX	y07, alpha1i, a8, y07

  3277. 	FMADDX	y08, alpha1r, a8, y08

  3278. 

  3279. 	LFD	a2,  9 * SIZE(AO1)

  3280. 	LFD	a4, 11 * SIZE(AO1)

  3281. 	LFD	a6, 13 * SIZE(AO1)

  3282. 	LFD	a8, 15 * SIZE(AO1)

  3283. 

  3284. 	addi	AO1, AO1, 16 * SIZE

  3285. 	nop

  3286. 	DCBT(AO1, PREA)

  3287. 	nop

  3288. 

  3289. 	FMADD	y09, alpha1r, a1, y09

  3290. 	FMADD	y10, alpha1i, a1, y10

  3291. 	FMADD	y11, alpha1r, a3, y11

  3292. 	FMADD	y12, alpha1i, a3, y12

  3293. 

  3294. 	FMADD	y13, alpha1r, a5, y13

  3295. 	FMADD	y14, alpha1i, a5, y14

  3296. 	FMADD	y15, alpha1r, a7, y15

  3297. 	FMADD	y16, alpha1i, a7, y16

  3298. 

  3299. 	LFD	a1,  0 * SIZE(AO2)

  3300. 	LFD	a3,  2 * SIZE(AO2)

  3301. 	LFD	a5,  4 * SIZE(AO2)

  3302. 	LFD	a7,  6 * SIZE(AO2)

  3303. 

  3304. 	FMSUBX	y09, alpha1i, a2, y09

  3305. 	FMADDX	y10, alpha1r, a2, y10

  3306. 	FMSUBX	y11, alpha1i, a4, y11

  3307. 	FMADDX	y12, alpha1r, a4, y12

  3308. 

  3309. 	FMSUBX	y13, alpha1i, a6, y13

  3310. 	FMADDX	y14, alpha1r, a6, y14

  3311. 	FMSUBX	y15, alpha1i, a8, y15

  3312. 	FMADDX	y16, alpha1r, a8, y16

  3313. 

  3314. 	LFD	a2,  1 * SIZE(AO2)

  3315. 	LFD	a4,  3 * SIZE(AO2)

  3316. 	LFD	a6,  5 * SIZE(AO2)

  3317. 	LFD	a8,  7 * SIZE(AO2)

  3318. 

  3319. 	FMADD	y01, alpha2r, a1, y01

  3320. 	FMADD	y02, alpha2i, a1, y02

  3321. 	FMADD	y03, alpha2r, a3, y03

  3322. 	FMADD	y04, alpha2i, a3, y04

  3323. 

  3324. 	FMADD	y05, alpha2r, a5, y05

  3325. 	FMADD	y06, alpha2i, a5, y06

  3326. 	FMADD	y07, alpha2r, a7, y07

  3327. 	FMADD	y08, alpha2i, a7, y08

  3328. 

  3329. 	LFD	a1,  8 * SIZE(AO2)

  3330. 	LFD	a3, 10 * SIZE(AO2)

  3331. 	LFD	a5, 12 * SIZE(AO2)

  3332. 	LFD	a7, 14 * SIZE(AO2)

  3333. 

  3334. 	FMSUBX	y01, alpha2i, a2, y01

  3335. 	FMADDX	y02, alpha2r, a2, y02

  3336. 	FMSUBX	y03, alpha2i, a4, y03

  3337. 	FMADDX	y04, alpha2r, a4, y04

  3338. 

  3339. 	STFD	y01, 0 * SIZE(Y2)

  3340. 	nop

  3341. 	STFD	y02, 1 * SIZE(Y2)

  3342. 	add	Y2, Y2, INCY

  3343. 

  3344. 	LFD	y01, 0 * SIZE(Y1)

  3345. 	nop

  3346. 	LFD	y02, 1 * SIZE(Y1)

  3347. 	add	Y1, Y1, INCY

  3348. 

  3349. 	STFD	y03, 0 * SIZE(Y2)

  3350. 	nop

  3351. 	STFD	y04, 1 * SIZE(Y2)

  3352. 	add	Y2, Y2, INCY

  3353. 

  3354. 	LFD	y03, 0 * SIZE(Y1)

  3355. 	nop

  3356. 	LFD	y04, 1 * SIZE(Y1)

  3357. 	add	Y1, Y1, INCY

  3358. 

  3359. 	FMSUBX	y05, alpha2i, a6, y05

  3360. 	FMADDX	y06, alpha2r, a6, y06

  3361. 	FMSUBX	y07, alpha2i, a8, y07

  3362. 	FMADDX	y08, alpha2r, a8, y08

  3363. 

  3364. 	LFD	a2,  9 * SIZE(AO2)

  3365. 	LFD	a4, 11 * SIZE(AO2)

  3366. 	LFD	a6, 13 * SIZE(AO2)

  3367. 	LFD	a8, 15 * SIZE(AO2)

  3368. 

  3369. 	addi	AO2, AO2, 16 * SIZE

  3370. 	nop

  3371. 	DCBT(AO2, PREA)

  3372. 	nop

  3373. 

  3374. 	STFD	y05, 0 * SIZE(Y2)

  3375. 	nop

  3376. 	STFD	y06, 1 * SIZE(Y2)

  3377. 	add	Y2, Y2, INCY

  3378. 

  3379. 	LFD	y05, 0 * SIZE(Y1)

  3380. 	nop

  3381. 	LFD	y06, 1 * SIZE(Y1)

  3382. 	add	Y1, Y1, INCY

  3383. 

  3384. 	STFD	y07, 0 * SIZE(Y2)

  3385. 	nop

  3386. 	STFD	y08, 1 * SIZE(Y2)

  3387. 	add	Y2, Y2, INCY

  3388. 

  3389. 	LFD	y07, 0 * SIZE(Y1)

  3390. 	nop

  3391. 	LFD	y08, 1 * SIZE(Y1)

  3392. 	add	Y1, Y1, INCY

  3393. 

  3394. 	FMADD	y09, alpha2r, a1, y09

  3395. 	FMADD	y10, alpha2i, a1, y10

  3396. 	FMADD	y11, alpha2r, a3, y11

  3397. 	FMADD	y12, alpha2i, a3, y12

  3398. 

  3399. 	FMADD	y13, alpha2r, a5, y13

  3400. 	FMADD	y14, alpha2i, a5, y14

  3401. 	FMADD	y15, alpha2r, a7, y15

  3402. 	FMADD	y16, alpha2i, a7, y16

  3403. 

  3404. 	LFD	a1,  0 * SIZE(AO1)

  3405. 	LFD	a3,  2 * SIZE(AO1)

  3406. 	LFD	a5,  4 * SIZE(AO1)

  3407. 	LFD	a7,  6 * SIZE(AO1)

  3408. 

  3409. 	FMSUBX	y09, alpha2i, a2, y09

  3410. 	FMADDX	y10, alpha2r, a2, y10

  3411. 	FMSUBX	y11, alpha2i, a4, y11

  3412. 	FMADDX	y12, alpha2r, a4, y12

  3413. 

  3414. 	STFD	y09, 0 * SIZE(Y2)

  3415. 	nop

  3416. 	STFD	y10, 1 * SIZE(Y2)

  3417. 	add	Y2, Y2, INCY

  3418. 

  3419. 	LFD	y09, 0 * SIZE(Y1)

  3420. 	nop

  3421. 	LFD	y10, 1 * SIZE(Y1)

  3422. 	add	Y1, Y1, INCY

  3423. 

  3424. 	STFD	y11, 0 * SIZE(Y2)

  3425. 	nop

  3426. 	STFD	y12, 1 * SIZE(Y2)

  3427. 	add	Y2, Y2, INCY

  3428. 

  3429. 	LFD	y11, 0 * SIZE(Y1)

  3430. 	nop

  3431. 	LFD	y12, 1 * SIZE(Y1)

  3432. 	add	Y1, Y1, INCY

  3433. 

  3434. 	FMSUBX	y13, alpha2i, a6, y13

  3435. 	FMADDX	y14, alpha2r, a6, y14

  3436. 	FMSUBX	y15, alpha2i, a8, y15

  3437. 	FMADDX	y16, alpha2r, a8, y16

  3438. 

  3439. 	LFD	a2,  1 * SIZE(AO1)

  3440. 	LFD	a4,  3 * SIZE(AO1)

  3441. 	LFD	a6,  5 * SIZE(AO1)

  3442. 	LFD	a8,  7 * SIZE(AO1)

  3443. 

  3444. 	STFD	y13, 0 * SIZE(Y2)

  3445. 	nop

  3446. 	STFD	y14, 1 * SIZE(Y2)

  3447. 	add	Y2, Y2, INCY

  3448. 

  3449. 	STFD	y15, 0 * SIZE(Y2)

  3450. 	nop

  3451. 	STFD	y16, 1 * SIZE(Y2)

  3452. 	add	Y2, Y2, INCY

  3453. 

  3454. 	LFD	y13, 0 * SIZE(Y1)

  3455. 	nop

  3456. 	LFD	y14, 1 * SIZE(Y1)

  3457. 	add	Y1, Y1, INCY

  3458. 

  3459. 	LFD	y15, 0 * SIZE(Y1)

  3460. 	nop

  3461. 	LFD	y16, 1 * SIZE(Y1)

  3462. 	add	Y1, Y1, INCY

  3463. 

  3464. 	DCBT(Y1, PREC)

  3465. 	bdnz	LL(122)

  3466. 	.align 4

  3467. 

  3468. LL(123):

  3469. 	FMADD	y01, alpha1r, a1, y01

  3470. 	FMADD	y02, alpha1i, a1, y02

  3471. 	FMADD	y03, alpha1r, a3, y03

  3472. 	FMADD	y04, alpha1i, a3, y04

  3473. 

  3474. 	FMADD	y05, alpha1r, a5, y05

  3475. 	FMADD	y06, alpha1i, a5, y06

  3476. 	FMADD	y07, alpha1r, a7, y07

  3477. 	FMADD	y08, alpha1i, a7, y08

  3478. 

  3479. 	LFD	a1,  8 * SIZE(AO1)

  3480. 	LFD	a3, 10 * SIZE(AO1)

  3481. 	LFD	a5, 12 * SIZE(AO1)

  3482. 	LFD	a7, 14 * SIZE(AO1)

  3483. 

  3484. 	FMSUBX	y01, alpha1i, a2, y01

  3485. 	FMADDX	y02, alpha1r, a2, y02

  3486. 	FMSUBX	y03, alpha1i, a4, y03

  3487. 	FMADDX	y04, alpha1r, a4, y04

  3488. 

  3489. 	FMSUBX	y05, alpha1i, a6, y05

  3490. 	FMADDX	y06, alpha1r, a6, y06

  3491. 	FMSUBX	y07, alpha1i, a8, y07

  3492. 	FMADDX	y08, alpha1r, a8, y08

  3493. 

  3494. 	LFD	a2,  9 * SIZE(AO1)

  3495. 	LFD	a4, 11 * SIZE(AO1)

  3496. 	LFD	a6, 13 * SIZE(AO1)

  3497. 	LFD	a8, 15 * SIZE(AO1)

  3498. 

  3499. 	FMADD	y09, alpha1r, a1, y09

  3500. 	FMADD	y10, alpha1i, a1, y10

  3501. 	FMADD	y11, alpha1r, a3, y11

  3502. 	FMADD	y12, alpha1i, a3, y12

  3503. 

  3504. 	FMADD	y13, alpha1r, a5, y13

  3505. 	FMADD	y14, alpha1i, a5, y14

  3506. 	FMADD	y15, alpha1r, a7, y15

  3507. 	FMADD	y16, alpha1i, a7, y16

  3508. 

  3509. 	LFD	a1,  0 * SIZE(AO2)

  3510. 	LFD	a3,  2 * SIZE(AO2)

  3511. 	LFD	a5,  4 * SIZE(AO2)

  3512. 	LFD	a7,  6 * SIZE(AO2)

  3513. 

  3514. 	FMSUBX	y09, alpha1i, a2, y09

  3515. 	FMADDX	y10, alpha1r, a2, y10

  3516. 	FMSUBX	y11, alpha1i, a4, y11

  3517. 	FMADDX	y12, alpha1r, a4, y12

  3518. 

  3519. 	FMSUBX	y13, alpha1i, a6, y13

  3520. 	FMADDX	y14, alpha1r, a6, y14

  3521. 	FMSUBX	y15, alpha1i, a8, y15

  3522. 	FMADDX	y16, alpha1r, a8, y16

  3523. 

  3524. 	LFD	a2,  1 * SIZE(AO2)

  3525. 	LFD	a4,  3 * SIZE(AO2)

  3526. 	LFD	a6,  5 * SIZE(AO2)

  3527. 	LFD	a8,  7 * SIZE(AO2)

  3528. 

  3529. 	FMADD	y01, alpha2r, a1, y01

  3530. 	FMADD	y02, alpha2i, a1, y02

  3531. 	FMADD	y03, alpha2r, a3, y03

  3532. 	FMADD	y04, alpha2i, a3, y04

  3533. 

  3534. 	FMADD	y05, alpha2r, a5, y05

  3535. 	FMADD	y06, alpha2i, a5, y06

  3536. 	FMADD	y07, alpha2r, a7, y07

  3537. 	FMADD	y08, alpha2i, a7, y08

  3538. 

  3539. 	LFD	a1,  8 * SIZE(AO2)

  3540. 	LFD	a3, 10 * SIZE(AO2)

  3541. 	LFD	a5, 12 * SIZE(AO2)

  3542. 	LFD	a7, 14 * SIZE(AO2)

  3543. 

  3544. 	FMSUBX	y01, alpha2i, a2, y01

  3545. 	FMADDX	y02, alpha2r, a2, y02

  3546. 	FMSUBX	y03, alpha2i, a4, y03

  3547. 	FMADDX	y04, alpha2r, a4, y04

  3548. 

  3549. 	STFD	y01, 0 * SIZE(Y2)

  3550. 	addi	AO1, AO1, 16 * SIZE

  3551. 	STFD	y02, 1 * SIZE(Y2)

  3552. 	add	Y2, Y2, INCY

  3553. 

  3554. 	STFD	y03, 0 * SIZE(Y2)

  3555. 	nop

  3556. 	STFD	y04, 1 * SIZE(Y2)

  3557. 	add	Y2, Y2, INCY

  3558. 

  3559. 	FMSUBX	y05, alpha2i, a6, y05

  3560. 	FMADDX	y06, alpha2r, a6, y06

  3561. 	FMSUBX	y07, alpha2i, a8, y07

  3562. 	FMADDX	y08, alpha2r, a8, y08

  3563. 

  3564. 	LFD	a2,  9 * SIZE(AO2)

  3565. 	LFD	a4, 11 * SIZE(AO2)

  3566. 	LFD	a6, 13 * SIZE(AO2)

  3567. 	LFD	a8, 15 * SIZE(AO2)

  3568. 

  3569. 	STFD	y05, 0 * SIZE(Y2)

  3570. 	addi	AO2, AO2, 16 * SIZE

  3571. 	STFD	y06, 1 * SIZE(Y2)

  3572. 	add	Y2, Y2, INCY

  3573. 

  3574. 	STFD	y07, 0 * SIZE(Y2)

  3575. 	nop

  3576. 	STFD	y08, 1 * SIZE(Y2)

  3577. 	add	Y2, Y2, INCY

  3578. 

  3579. 	FMADD	y09, alpha2r, a1, y09

  3580. 	FMADD	y10, alpha2i, a1, y10

  3581. 	FMADD	y11, alpha2r, a3, y11

  3582. 	FMADD	y12, alpha2i, a3, y12

  3583. 

  3584. 	FMADD	y13, alpha2r, a5, y13

  3585. 	FMADD	y14, alpha2i, a5, y14

  3586. 	FMADD	y15, alpha2r, a7, y15

  3587. 	FMADD	y16, alpha2i, a7, y16

  3588. 

  3589. 	FMSUBX	y09, alpha2i, a2, y09

  3590. 	FMADDX	y10, alpha2r, a2, y10

  3591. 	FMSUBX	y11, alpha2i, a4, y11

  3592. 	FMADDX	y12, alpha2r, a4, y12

  3593. 

  3594. 	STFD	y09, 0 * SIZE(Y2)

  3595. 	nop

  3596. 	STFD	y10, 1 * SIZE(Y2)

  3597. 	add	Y2, Y2, INCY

  3598. 

  3599. 	STFD	y11, 0 * SIZE(Y2)

  3600. 	nop

  3601. 	STFD	y12, 1 * SIZE(Y2)

  3602. 	add	Y2, Y2, INCY

  3603. 

  3604. 	FMSUBX	y13, alpha2i, a6, y13

  3605. 	FMADDX	y14, alpha2r, a6, y14

  3606. 	FMSUBX	y15, alpha2i, a8, y15

  3607. 	FMADDX	y16, alpha2r, a8, y16

  3608. 

  3609. 	STFD	y13, 0 * SIZE(Y2)

  3610. 	nop

  3611. 	STFD	y14, 1 * SIZE(Y2)

  3612. 	add	Y2, Y2, INCY

  3613. 

  3614. 	STFD	y15, 0 * SIZE(Y2)

  3615. 	nop

  3616. 	STFD	y16, 1 * SIZE(Y2)

  3617. 	add	Y2, Y2, INCY

  3618. 	.align 4

  3619. 

  3620. LL(125):

  3621. 	andi.	r0, M, 7

  3622. 	ble	LL(130)

  3623. 	andi.	r0, M, 4

  3624. 	ble	LL(126)

  3625. 

  3626. 	LFD	y01, 0 * SIZE(Y1)

  3627. 	LFD	y02, 1 * SIZE(Y1)

  3628. 	add	Y1, Y1, INCY

  3629. 	LFD	y03, 0 * SIZE(Y1)

  3630. 	LFD	y04, 1 * SIZE(Y1)

  3631. 	add	Y1, Y1, INCY

  3632. 

  3633. 	LFD	a1,  0 * SIZE(AO1)

  3634. 	LFD	a3,  2 * SIZE(AO1)

  3635. 	LFD	a5,  4 * SIZE(AO1)

  3636. 	LFD	a7,  6 * SIZE(AO1)

  3637. 

  3638. 	LFD	y05, 0 * SIZE(Y1)

  3639. 	LFD	y06, 1 * SIZE(Y1)

  3640. 	add	Y1, Y1, INCY

  3641. 	LFD	y07, 0 * SIZE(Y1)

  3642. 	LFD	y08, 1 * SIZE(Y1)

  3643. 	add	Y1, Y1, INCY

  3644. 

  3645. 	LFD	a2,  1 * SIZE(AO1)

  3646. 	LFD	a4,  3 * SIZE(AO1)

  3647. 	LFD	a6,  5 * SIZE(AO1)

  3648. 	LFD	a8,  7 * SIZE(AO1)

  3649. 

  3650. 	FMADD	y01, alpha1r, a1, y01

  3651. 	FMADD	y02, alpha1i, a1, y02

  3652. 	FMADD	y03, alpha1r, a3, y03

  3653. 	FMADD	y04, alpha1i, a3, y04

  3654. 

  3655. 	FMADD	y05, alpha1r, a5, y05

  3656. 	FMADD	y06, alpha1i, a5, y06

  3657. 	FMADD	y07, alpha1r, a7, y07

  3658. 	FMADD	y08, alpha1i, a7, y08

  3659. 

  3660. 	LFD	a1,  0 * SIZE(AO2)

  3661. 	LFD	a3,  2 * SIZE(AO2)

  3662. 	LFD	a5,  4 * SIZE(AO2)

  3663. 	LFD	a7,  6 * SIZE(AO2)

  3664. 

  3665. 	FMSUBX	y01, alpha1i, a2, y01

  3666. 	FMADDX	y02, alpha1r, a2, y02

  3667. 	FMSUBX	y03, alpha1i, a4, y03

  3668. 	FMADDX	y04, alpha1r, a4, y04

  3669. 

  3670. 	FMSUBX	y05, alpha1i, a6, y05

  3671. 	FMADDX	y06, alpha1r, a6, y06

  3672. 	FMSUBX	y07, alpha1i, a8, y07

  3673. 	FMADDX	y08, alpha1r, a8, y08

  3674. 

  3675. 	LFD	a2,  1 * SIZE(AO2)

  3676. 	LFD	a4,  3 * SIZE(AO2)

  3677. 	LFD	a6,  5 * SIZE(AO2)

  3678. 	LFD	a8,  7 * SIZE(AO2)

  3679. 

  3680. 	FMADD	y01, alpha2r, a1, y01

  3681. 	FMADD	y02, alpha2i, a1, y02

  3682. 	FMADD	y03, alpha2r, a3, y03

  3683. 	FMADD	y04, alpha2i, a3, y04

  3684. 

  3685. 	FMADD	y05, alpha2r, a5, y05

  3686. 	FMADD	y06, alpha2i, a5, y06

  3687. 	FMADD	y07, alpha2r, a7, y07

  3688. 	FMADD	y08, alpha2i, a7, y08

  3689. 

  3690. 	FMSUBX	y01, alpha2i, a2, y01

  3691. 	FMADDX	y02, alpha2r, a2, y02

  3692. 	FMSUBX	y03, alpha2i, a4, y03

  3693. 	FMADDX	y04, alpha2r, a4, y04

  3694. 

  3695. 	STFD	y01, 0 * SIZE(Y2)

  3696. 	addi	AO1, AO1, 8 * SIZE

  3697. 	STFD	y02, 1 * SIZE(Y2)

  3698. 	add	Y2, Y2, INCY

  3699. 

  3700. 	STFD	y03, 0 * SIZE(Y2)

  3701. 	addi	AO2, AO2, 8 * SIZE

  3702. 	STFD	y04, 1 * SIZE(Y2)

  3703. 	add	Y2, Y2, INCY

  3704. 

  3705. 	FMSUBX	y05, alpha2i, a6, y05

  3706. 	FMADDX	y06, alpha2r, a6, y06

  3707. 	FMSUBX	y07, alpha2i, a8, y07

  3708. 	FMADDX	y08, alpha2r, a8, y08

  3709. 

  3710. 	STFD	y05, 0 * SIZE(Y2)

  3711. 	nop

  3712. 	STFD	y06, 1 * SIZE(Y2)

  3713. 	add	Y2, Y2, INCY

  3714. 

  3715. 	STFD	y07, 0 * SIZE(Y2)

  3716. 	nop

  3717. 	STFD	y08, 1 * SIZE(Y2)

  3718. 	add	Y2, Y2, INCY

  3719. 	.align 4

  3720. 

  3721. LL(126):

  3722. 	andi.	r0, M, 2

  3723. 	ble	LL(127)

  3724. 

  3725. 	LFD	a1,  0 * SIZE(AO1)

  3726. 	LFD	a2,  1 * SIZE(AO1)

  3727. 	LFD	a3,  2 * SIZE(AO1)

  3728. 	LFD	a4,  3 * SIZE(AO1)

  3729. 

  3730. 	LFD	y01, 0 * SIZE(Y1)

  3731. 	LFD	y02, 1 * SIZE(Y1)

  3732. 	add	Y1, Y1, INCY

  3733. 	LFD	y03, 0 * SIZE(Y1)

  3734. 	LFD	y04, 1 * SIZE(Y1)

  3735. 	add	Y1, Y1, INCY

  3736. 

  3737. 	LFD	a5,  0 * SIZE(AO2)

  3738. 	LFD	a6,  1 * SIZE(AO2)

  3739. 	LFD	a7,  2 * SIZE(AO2)

  3740. 	LFD	a8,  3 * SIZE(AO2)

  3741. 

  3742. 	FMADD	y01, alpha1r, a1, y01

  3743. 	FMADD	y02, alpha1i, a1, y02

  3744. 	FMADD	y03, alpha1r, a3, y03

  3745. 	FMADD	y04, alpha1i, a3, y04

  3746. 

  3747. 	FMSUBX	y01, alpha1i, a2, y01

  3748. 	FMADDX	y02, alpha1r, a2, y02

  3749. 	FMSUBX	y03, alpha1i, a4, y03

  3750. 	FMADDX	y04, alpha1r, a4, y04

  3751. 

  3752. 	FMADD	y01, alpha2r, a5, y01

  3753. 	FMADD	y02, alpha2i, a5, y02

  3754. 	FMADD	y03, alpha2r, a7, y03

  3755. 	FMADD	y04, alpha2i, a7, y04

  3756. 

  3757. 	FMSUBX	y01, alpha2i, a6, y01

  3758. 	FMADDX	y02, alpha2r, a6, y02

  3759. 	FMSUBX	y03, alpha2i, a8, y03

  3760. 	FMADDX	y04, alpha2r, a8, y04

  3761. 

  3762. 	STFD	y01, 0 * SIZE(Y2)

  3763. 	addi	AO1, AO1, 4 * SIZE

  3764. 	STFD	y02, 1 * SIZE(Y2)

  3765. 	add	Y2, Y2, INCY

  3766. 

  3767. 	STFD	y03, 0 * SIZE(Y2)

  3768. 	addi	AO2, AO2, 4 * SIZE

  3769. 	STFD	y04, 1 * SIZE(Y2)

  3770. 	add	Y2, Y2, INCY

  3771. 	.align 4

  3772. 

  3773. LL(127):

  3774. 	andi.	r0, M, 1

  3775. 	ble	LL(130)

  3776. 

  3777. 	LFD	y01, 0 * SIZE(Y1)

  3778. 	LFD	y02, 1 * SIZE(Y1)

  3779. 	add	Y1, Y1, INCY

  3780. 

  3781. 	LFD	a1,  0 * SIZE(AO1)

  3782. 	LFD	a2,  1 * SIZE(AO1)

  3783. 	LFD	a3,  0 * SIZE(AO2)

  3784. 	LFD	a4,  1 * SIZE(AO2)

  3785. 

  3786. 	FMADD	y01, alpha1r, a1, y01

  3787. 	FMADD	y02, alpha1i, a1, y02

  3788. 	FMSUBX	y01, alpha1i, a2, y01

  3789. 	FMADDX	y02, alpha1r, a2, y02

  3790. 

  3791. 	FMADD	y01, alpha2r, a3, y01

  3792. 	FMADD	y02, alpha2i, a3, y02

  3793. 	FMSUBX	y01, alpha2i, a4, y01

  3794. 	FMADDX	y02, alpha2r, a4, y02

  3795. 

  3796. 	STFD	y01, 0 * SIZE(Y2)

  3797. 	STFD	y02, 1 * SIZE(Y2)

  3798. 	add	Y2, Y2, INCY

  3799. 	.align 4

  3800. 

  3801. LL(130):

  3802. 	andi.	J, N, 1

  3803. 	ble	LL(999)

  3804. 	.align 4

  3805. 

  3806. LL(131):

  3807. 	lfd	alpha_r, ALPHA_R

  3808. 	lfd	alpha_i, ALPHA_I

  3809. 

  3810. 	LFD	a1, 0 * SIZE(X)

  3811. 	LFD	a2, 1 * SIZE(X)

  3812. 	add	X, X, INCX

  3813. 

  3814. 	FMUL	alpha1r, alpha_r, a1

  3815. 	FMUL	alpha1i, alpha_i, a1

  3816. 

  3817. 	FMSUBR	alpha1r, alpha_i, a2, alpha1r

  3818. 	FMADDR	alpha1i, alpha_r, a2, alpha1i

  3819. 

  3820. 	mr	AO1, A

  3821. 	add	A,   AO1, LDA

  3822. 

  3823. 	mr	Y1, Y

  3824. 	mr	Y2, Y

  3825. 

  3826. 	srawi.	r0,  M, 3

  3827. 	mtspr	CTR, r0

  3828. 	ble	LL(135)

  3829. 	.align 4

  3830. 

  3831. 	LFD	y01, 0 * SIZE(Y1)

  3832. 	LFD	y02, 1 * SIZE(Y1)

  3833. 	add	Y1, Y1, INCY

  3834. 	LFD	y03, 0 * SIZE(Y1)

  3835. 	LFD	y04, 1 * SIZE(Y1)

  3836. 	add	Y1, Y1, INCY

  3837. 

  3838. 	LFD	a1,  0 * SIZE(AO1)

  3839. 	LFD	a3,  2 * SIZE(AO1)

  3840. 	LFD	a5,  4 * SIZE(AO1)

  3841. 	LFD	a7,  6 * SIZE(AO1)

  3842. 

  3843. 	LFD	y05, 0 * SIZE(Y1)

  3844. 	LFD	y06, 1 * SIZE(Y1)

  3845. 	add	Y1, Y1, INCY

  3846. 	LFD	y07, 0 * SIZE(Y1)

  3847. 	LFD	y08, 1 * SIZE(Y1)

  3848. 	add	Y1, Y1, INCY

  3849. 

  3850. 	LFD	a2,  1 * SIZE(AO1)

  3851. 	LFD	a4,  3 * SIZE(AO1)

  3852. 	LFD	a6,  5 * SIZE(AO1)

  3853. 	LFD	a8,  7 * SIZE(AO1)

  3854. 

  3855. 	LFD	y09, 0 * SIZE(Y1)

  3856. 	LFD	y10, 1 * SIZE(Y1)

  3857. 	add	Y1, Y1, INCY

  3858. 	LFD	y11, 0 * SIZE(Y1)

  3859. 	LFD	y12, 1 * SIZE(Y1)

  3860. 	add	Y1, Y1, INCY

  3861. 	LFD	y13, 0 * SIZE(Y1)

  3862. 	LFD	y14, 1 * SIZE(Y1)

  3863. 	add	Y1, Y1, INCY

  3864. 	LFD	y15, 0 * SIZE(Y1)

  3865. 	LFD	y16, 1 * SIZE(Y1)

  3866. 	add	Y1, Y1, INCY

  3867. 

  3868. 	bdz	LL(133)

  3869. 	.align 4

  3870. 

  3871. LL(132):

  3872. 	FMADD	y01, alpha1r, a1, y01

  3873. 	FMADD	y02, alpha1i, a1, y02

  3874. 	FMADD	y03, alpha1r, a3, y03

  3875. 	FMADD	y04, alpha1i, a3, y04

  3876. 

  3877. 	FMADD	y05, alpha1r, a5, y05

  3878. 	FMADD	y06, alpha1i, a5, y06

  3879. 	FMADD	y07, alpha1r, a7, y07

  3880. 	FMADD	y08, alpha1i, a7, y08

  3881. 

  3882. 	LFD	a1,  8 * SIZE(AO1)

  3883. 	LFD	a3, 10 * SIZE(AO1)

  3884. 	LFD	a5, 12 * SIZE(AO1)

  3885. 	LFD	a7, 14 * SIZE(AO1)

  3886. 

  3887. 	FMSUBX	y01, alpha1i, a2, y01

  3888. 	FMADDX	y02, alpha1r, a2, y02

  3889. 	FMSUBX	y03, alpha1i, a4, y03

  3890. 	FMADDX	y04, alpha1r, a4, y04

  3891. 

  3892. 	STFD	y01, 0 * SIZE(Y2)

  3893. 	nop

  3894. 	STFD	y02, 1 * SIZE(Y2)

  3895. 	add	Y2, Y2, INCY

  3896. 

  3897. 	LFD	y01, 0 * SIZE(Y1)

  3898. 	nop

  3899. 	LFD	y02, 1 * SIZE(Y1)

  3900. 	add	Y1, Y1, INCY

  3901. 

  3902. 	STFD	y03, 0 * SIZE(Y2)

  3903. 	nop

  3904. 	STFD	y04, 1 * SIZE(Y2)

  3905. 	add	Y2, Y2, INCY

  3906. 

  3907. 	LFD	y03, 0 * SIZE(Y1)

  3908. 	nop

  3909. 	LFD	y04, 1 * SIZE(Y1)

  3910. 	add	Y1, Y1, INCY

  3911. 

  3912. 	FMSUBX	y05, alpha1i, a6, y05

  3913. 	FMADDX	y06, alpha1r, a6, y06

  3914. 	FMSUBX	y07, alpha1i, a8, y07

  3915. 	FMADDX	y08, alpha1r, a8, y08

  3916. 

  3917. 	LFD	a2,  9 * SIZE(AO1)

  3918. 	LFD	a4, 11 * SIZE(AO1)

  3919. 	LFD	a6, 13 * SIZE(AO1)

  3920. 	LFD	a8, 15 * SIZE(AO1)

  3921. 

  3922. 	addi	AO1, AO1, 16 * SIZE

  3923. 	nop

  3924. 	DCBT(AO1, PREA)

  3925. 	nop

  3926. 

  3927. 	STFD	y05, 0 * SIZE(Y2)

  3928. 	nop

  3929. 	STFD	y06, 1 * SIZE(Y2)

  3930. 	add	Y2, Y2, INCY

  3931. 

  3932. 	LFD	y05, 0 * SIZE(Y1)

  3933. 	nop

  3934. 	LFD	y06, 1 * SIZE(Y1)

  3935. 	add	Y1, Y1, INCY

  3936. 

  3937. 	STFD	y07, 0 * SIZE(Y2)

  3938. 	nop

  3939. 	STFD	y08, 1 * SIZE(Y2)

  3940. 	add	Y2, Y2, INCY

  3941. 

  3942. 	LFD	y07, 0 * SIZE(Y1)

  3943. 	nop

  3944. 	LFD	y08, 1 * SIZE(Y1)

  3945. 	add	Y1, Y1, INCY

  3946. 

  3947. 	FMADD	y09, alpha1r, a1, y09

  3948. 	FMADD	y10, alpha1i, a1, y10

  3949. 	FMADD	y11, alpha1r, a3, y11

  3950. 	FMADD	y12, alpha1i, a3, y12

  3951. 

  3952. 	FMADD	y13, alpha1r, a5, y13

  3953. 	FMADD	y14, alpha1i, a5, y14

  3954. 	FMADD	y15, alpha1r, a7, y15

  3955. 	FMADD	y16, alpha1i, a7, y16

  3956. 

  3957. 	LFD	a1,  0 * SIZE(AO1)

  3958. 	LFD	a3,  2 * SIZE(AO1)

  3959. 	LFD	a5,  4 * SIZE(AO1)

  3960. 	LFD	a7,  6 * SIZE(AO1)

  3961. 

  3962. 	FMSUBX	y09, alpha1i, a2, y09

  3963. 	FMADDX	y10, alpha1r, a2, y10

  3964. 	FMSUBX	y11, alpha1i, a4, y11

  3965. 	FMADDX	y12, alpha1r, a4, y12

  3966. 

  3967. 	STFD	y09, 0 * SIZE(Y2)

  3968. 	nop

  3969. 	STFD	y10, 1 * SIZE(Y2)

  3970. 	add	Y2, Y2, INCY

  3971. 

  3972. 	LFD	y09, 0 * SIZE(Y1)

  3973. 	nop

  3974. 	LFD	y10, 1 * SIZE(Y1)

  3975. 	add	Y1, Y1, INCY

  3976. 

  3977. 	STFD	y11, 0 * SIZE(Y2)

  3978. 	nop

  3979. 	STFD	y12, 1 * SIZE(Y2)

  3980. 	add	Y2, Y2, INCY

  3981. 

  3982. 	LFD	y11, 0 * SIZE(Y1)

  3983. 	nop

  3984. 	LFD	y12, 1 * SIZE(Y1)

  3985. 	add	Y1, Y1, INCY

  3986. 

  3987. 	FMSUBX	y13, alpha1i, a6, y13

  3988. 	FMADDX	y14, alpha1r, a6, y14

  3989. 	FMSUBX	y15, alpha1i, a8, y15

  3990. 	FMADDX	y16, alpha1r, a8, y16

  3991. 

  3992. 	LFD	a2,  1 * SIZE(AO1)

  3993. 	LFD	a4,  3 * SIZE(AO1)

  3994. 	LFD	a6,  5 * SIZE(AO1)

  3995. 	LFD	a8,  7 * SIZE(AO1)

  3996. 

  3997. 	STFD	y13, 0 * SIZE(Y2)

  3998. 	nop

  3999. 	STFD	y14, 1 * SIZE(Y2)

  4000. 	add	Y2, Y2, INCY

  4001. 

  4002. 	STFD	y15, 0 * SIZE(Y2)

  4003. 	nop

  4004. 	STFD	y16, 1 * SIZE(Y2)

  4005. 	add	Y2, Y2, INCY

  4006. 

  4007. 	LFD	y13, 0 * SIZE(Y1)

  4008. 	nop

  4009. 	LFD	y14, 1 * SIZE(Y1)

  4010. 	add	Y1, Y1, INCY

  4011. 

  4012. 	LFD	y15, 0 * SIZE(Y1)

  4013. 	nop

  4014. 	LFD	y16, 1 * SIZE(Y1)

  4015. 	add	Y1, Y1, INCY

  4016. 

  4017. 	DCBT(Y1, PREC)

  4018. 	bdnz	LL(132)

  4019. 	.align 4

  4020. 

  4021. LL(133):

  4022. 	FMADD	y01, alpha1r, a1, y01

  4023. 	FMADD	y02, alpha1i, a1, y02

  4024. 	FMADD	y03, alpha1r, a3, y03

  4025. 	FMADD	y04, alpha1i, a3, y04

  4026. 

  4027. 	FMADD	y05, alpha1r, a5, y05

  4028. 	FMADD	y06, alpha1i, a5, y06

  4029. 	FMADD	y07, alpha1r, a7, y07

  4030. 	FMADD	y08, alpha1i, a7, y08

  4031. 

  4032. 	LFD	a1,  8 * SIZE(AO1)

  4033.  	LFD	a3, 10 * SIZE(AO1)

  4034. 	LFD	a5, 12 * SIZE(AO1)

  4035. 	LFD	a7, 14 * SIZE(AO1)

  4036. 

  4037. 	FMSUBX	y01, alpha1i, a2, y01

  4038. 	FMADDX	y02, alpha1r, a2, y02

  4039. 	FMSUBX	y03, alpha1i, a4, y03

  4040. 	FMADDX	y04, alpha1r, a4, y04

  4041. 

  4042. 	FMSUBX	y05, alpha1i, a6, y05

  4043. 	FMADDX	y06, alpha1r, a6, y06

  4044. 	FMSUBX	y07, alpha1i, a8, y07

  4045. 	FMADDX	y08, alpha1r, a8, y08

  4046. 

  4047. 	LFD	a2,  9 * SIZE(AO1)

  4048. 	LFD	a4, 11 * SIZE(AO1)

  4049. 	LFD	a6, 13 * SIZE(AO1)

  4050. 	LFD	a8, 15 * SIZE(AO1)

  4051. 

  4052. 	FMADD	y09, alpha1r, a1, y09

  4053. 	FMADD	y10, alpha1i, a1, y10

  4054. 	FMADD	y11, alpha1r, a3, y11

  4055. 	FMADD	y12, alpha1i, a3, y12

  4056. 

  4057. 	FMADD	y13, alpha1r, a5, y13

  4058. 	FMADD	y14, alpha1i, a5, y14

  4059. 	FMADD	y15, alpha1r, a7, y15

  4060. 	FMADD	y16, alpha1i, a7, y16

  4061. 

  4062. 	FMSUBX	y09, alpha1i, a2, y09

  4063. 	FMADDX	y10, alpha1r, a2, y10

  4064. 	FMSUBX	y11, alpha1i, a4, y11

  4065. 	FMADDX	y12, alpha1r, a4, y12

  4066. 

  4067. 	FMSUBX	y13, alpha1i, a6, y13

  4068. 	FMADDX	y14, alpha1r, a6, y14

  4069. 	FMSUBX	y15, alpha1i, a8, y15

  4070. 	FMADDX	y16, alpha1r, a8, y16

  4071. 

  4072. 	STFD	y01, 0 * SIZE(Y2)

  4073. 	addi	AO1, AO1, 16 * SIZE

  4074. 	STFD	y02, 1 * SIZE(Y2)

  4075. 	add	Y2, Y2, INCY

  4076. 

  4077. 	STFD	y03, 0 * SIZE(Y2)

  4078. 	nop

  4079. 	STFD	y04, 1 * SIZE(Y2)

  4080. 	add	Y2, Y2, INCY

  4081. 

  4082. 	STFD	y05, 0 * SIZE(Y2)

  4083. 	nop

  4084. 	STFD	y06, 1 * SIZE(Y2)

  4085. 	add	Y2, Y2, INCY

  4086. 

  4087. 	STFD	y07, 0 * SIZE(Y2)

  4088. 	nop

  4089. 	STFD	y08, 1 * SIZE(Y2)

  4090. 	add	Y2, Y2, INCY

  4091. 

  4092. 	STFD	y09, 0 * SIZE(Y2)

  4093. 	nop

  4094. 	STFD	y10, 1 * SIZE(Y2)

  4095. 	add	Y2, Y2, INCY

  4096. 

  4097. 	STFD	y11, 0 * SIZE(Y2)

  4098. 	nop

  4099. 	STFD	y12, 1 * SIZE(Y2)

  4100. 	add	Y2, Y2, INCY

  4101. 

  4102. 	STFD	y13, 0 * SIZE(Y2)

  4103. 	nop

  4104. 	STFD	y14, 1 * SIZE(Y2)

  4105. 	add	Y2, Y2, INCY

  4106. 

  4107. 	STFD	y15, 0 * SIZE(Y2)

  4108. 	nop

  4109. 	STFD	y16, 1 * SIZE(Y2)

  4110. 	add	Y2, Y2, INCY

  4111. 	.align 4

  4112. 

  4113. LL(135):

  4114. 	andi.	r0, M, 7

  4115. 	ble	LL(999)

  4116. 	andi.	r0, M, 4

  4117. 	ble	LL(136)

  4118. 

  4119. 	LFD	y01, 0 * SIZE(Y1)

  4120. 	nop

  4121. 	LFD	y02, 1 * SIZE(Y1)

  4122. 	add	Y1, Y1, INCY

  4123. 

  4124. 	LFD	y03, 0 * SIZE(Y1)

  4125. 	nop

  4126. 	LFD	y04, 1 * SIZE(Y1)

  4127. 	add	Y1, Y1, INCY

  4128. 

  4129. 	LFD	y05, 0 * SIZE(Y1)

  4130. 	nop

  4131. 	LFD	y06, 1 * SIZE(Y1)

  4132. 	add	Y1, Y1, INCY

  4133. 

  4134. 	LFD	y07, 0 * SIZE(Y1)

  4135. 	nop

  4136. 	LFD	y08, 1 * SIZE(Y1)

  4137. 	add	Y1, Y1, INCY

  4138. 

  4139. 	LFD	a1,  0 * SIZE(AO1)

  4140. 	LFD	a3,  2 * SIZE(AO1)

  4141. 	LFD	a5,  4 * SIZE(AO1)

  4142.  	LFD	a7,  6 * SIZE(AO1)

  4143. 

  4144. 	LFD	a2,  1 * SIZE(AO1)

  4145. 	LFD	a4,  3 * SIZE(AO1)

  4146. 	LFD	a6,  5 * SIZE(AO1)

  4147. 	LFD	a8,  7 * SIZE(AO1)

  4148. 

  4149. 	FMADD	y01, alpha1r, a1, y01

  4150. 	FMADD	y02, alpha1i, a1, y02

  4151. 	FMADD	y03, alpha1r, a3, y03

  4152. 	FMADD	y04, alpha1i, a3, y04

  4153. 

  4154. 	FMADD	y05, alpha1r, a5, y05

  4155. 	FMADD	y06, alpha1i, a5, y06

  4156. 	FMADD	y07, alpha1r, a7, y07

  4157. 	FMADD	y08, alpha1i, a7, y08

  4158. 

  4159. 	FMSUBX	y01, alpha1i, a2, y01

  4160. 	FMADDX	y02, alpha1r, a2, y02

  4161. 	FMSUBX	y03, alpha1i, a4, y03

  4162. 	FMADDX	y04, alpha1r, a4, y04

  4163. 

  4164. 	FMSUBX	y05, alpha1i, a6, y05

  4165. 	FMADDX	y06, alpha1r, a6, y06

  4166. 	FMSUBX	y07, alpha1i, a8, y07

  4167. 	FMADDX	y08, alpha1r, a8, y08

  4168. 

  4169. 	STFD	y01, 0 * SIZE(Y2)

  4170. 	addi	AO1, AO1, 8 * SIZE

  4171. 	STFD	y02, 1 * SIZE(Y2)

  4172. 	add	Y2, Y2, INCY

  4173. 

  4174. 	STFD	y03, 0 * SIZE(Y2)

  4175. 	nop

  4176. 	STFD	y04, 1 * SIZE(Y2)

  4177. 	add	Y2, Y2, INCY

  4178. 

  4179. 	STFD	y05, 0 * SIZE(Y2)

  4180. 	nop

  4181. 	STFD	y06, 1 * SIZE(Y2)

  4182. 	add	Y2, Y2, INCY

  4183. 

  4184. 	STFD	y07, 0 * SIZE(Y2)

  4185. 	nop

  4186. 	STFD	y08, 1 * SIZE(Y2)

  4187. 	add	Y2, Y2, INCY

  4188. 	.align 4

  4189. 

  4190. LL(136):

  4191. 	andi.	r0, M, 2

  4192. 	ble	LL(137)

  4193. 

  4194. 	LFD	a1,  0 * SIZE(AO1)

  4195. 	LFD	a2,  1 * SIZE(AO1)

  4196. 	LFD	a3,  2 * SIZE(AO1)

  4197. 	LFD	a4,  3 * SIZE(AO1)

  4198. 

  4199. 	LFD	y01, 0 * SIZE(Y1)

  4200. 	nop

  4201. 	LFD	y02, 1 * SIZE(Y1)

  4202. 	add	Y1, Y1, INCY

  4203. 	LFD	y03, 0 * SIZE(Y1)

  4204. 	nop

  4205. 	LFD	y04, 1 * SIZE(Y1)

  4206. 	add	Y1, Y1, INCY

  4207. 

  4208. 	FMADD	y01, alpha1r, a1, y01

  4209. 	FMADD	y02, alpha1i, a1, y02

  4210. 	FMADD	y03, alpha1r, a3, y03

  4211. 	FMADD	y04, alpha1i, a3, y04

  4212. 

  4213. 	FMSUBX	y01, alpha1i, a2, y01

  4214. 	FMADDX	y02, alpha1r, a2, y02

  4215. 	FMSUBX	y03, alpha1i, a4, y03

  4216. 	FMADDX	y04, alpha1r, a4, y04

  4217. 

  4218. 	STFD	y01, 0 * SIZE(Y2)

  4219. 	addi	AO1, AO1, 4 * SIZE

  4220. 	STFD	y02, 1 * SIZE(Y2)

  4221. 	add	Y2, Y2, INCY

  4222. 

  4223. 	STFD	y03, 0 * SIZE(Y2)

  4224. 	nop

  4225. 	STFD	y04, 1 * SIZE(Y2)

  4226. 	add	Y2, Y2, INCY

  4227. 	.align 4

  4228. 

  4229. LL(137):

  4230. 	andi.	r0, M, 1

  4231. 	ble	LL(999)

  4232. 

  4233. 	LFD	y01, 0 * SIZE(Y1)

  4234. 	nop

  4235. 	LFD	y02, 1 * SIZE(Y1)

  4236. 	add	Y1, Y1, INCY

  4237. 

  4238. 	LFD	a1,  0 * SIZE(AO1)

  4239. 	LFD	a2,  1 * SIZE(AO1)

  4240. 

  4241. 	FMADD	y01, alpha1r, a1, y01

  4242. 	FMADD	y02, alpha1i, a1, y02

  4243. 	FMSUBX	y01, alpha1i, a2, y01

  4244. 	FMADDX	y02, alpha1r, a2, y02

  4245. 

  4246. 	STFD	y01, 0 * SIZE(Y2)

  4247. 	nop

  4248. 	STFD	y02, 1 * SIZE(Y2)

  4249. 	add	Y2, Y2, INCY

  4250. 	.align 4

  4251. 

  4252. LL(999):

  4253. 	li	r3, 0

  4254. 

  4255. 	lfd	f14,     0(SP)

  4256. 	lfd	f15,     8(SP)

  4257. 	lfd	f16,    16(SP)

  4258. 	lfd	f17,    24(SP)

  4259. 	lfd	f18,    32(SP)

  4260. 	lfd	f19,    40(SP)

  4261. 	lfd	f20,    48(SP)

  4262. 	lfd	f21,    56(SP)

  4263. 	lfd	f22,    64(SP)

  4264. 	lfd	f23,    72(SP)

  4265. 	lfd	f24,    80(SP)

  4266. 	lfd	f25,    88(SP)

  4267. 	lfd	f26,    96(SP)

  4268. 	lfd	f27,   104(SP)

  4269. 	lfd	f28,   112(SP)

  4270. 	lfd	f29,   120(SP)

  4271. 	lfd	f30,   128(SP)

  4272. 	lfd	f31,   136(SP)

  4273. 

  4274. #ifdef __64BIT__

  4275. 	ld	r14,   144(SP)

  4276. 	ld	r15,   152(SP)

  4277. 	ld	r16,   160(SP)

  4278. 	ld	r17,   168(SP)

  4279. 	ld	r18,   176(SP)

  4280. 	ld	r19,   184(SP)

  4281. 	ld	r20,   192(SP)

  4282. 	ld	r21,   200(SP)

  4283. 	ld	r22,   208(SP)

  4284. #else

  4285. 	lwz	r14,   144(SP)

  4286. 	lwz	r15,   148(SP)

  4287. 	lwz	r16,   152(SP)

  4288. 	lwz	r17,   156(SP)

  4289. 	lwz	r18,   160(SP)

  4290. 	lwz	r19,   164(SP)

  4291. 	lwz	r20,   168(SP)

  4292. 	lwz	r21,   172(SP)

  4293. 	lwz	r22,   176(SP)

  4294. #endif

  4295. 

  4296. 	addi	SP, SP, STACKSIZE

  4297. 	blr

  4298. 

  4299. 	EPILOGUE

  4300. #endif