OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/zgemv_t_ppc440.S

1295 lines
26 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define P 1024

  43. 

  44. #ifndef __64BIT__

  45. #define STACKSIZE 224

  46. #else

  47. #define STACKSIZE 304

  48. #endif

  49. 

  50. #if defined(linux) || defined(__FreeBSD__)

  51. #ifndef __64BIT__

  52. #define M	r3

  53. #define	N	r4

  54. #define A	r6

  55. #define LDA	r7

  56. #define X	r8

  57. #define	INCX	r9

  58. #define	Y	r10

  59. #define	INCY	r5

  60. #else

  61. #define M	r3

  62. #define	N	r4

  63. #define A	r8

  64. #define LDA	r9

  65. #define X	r10

  66. #define	INCX	r5

  67. #define	Y	r6

  68. #define	INCY	r7

  69. #endif

  70. #endif

  71. 

  72. #if defined(_AIX) || defined(__APPLE__)

  73. #if !defined(__64BIT__) && defined(DOUBLE)

  74. #define M	r3

  75. #define	N	r4

  76. #define A	r10

  77. #define LDA	r5

  78. #define X	r6

  79. #define	INCX	r7

  80. #define	Y	r8

  81. #define	INCY	r9

  82. #else

  83. #define M	r3

  84. #define	N	r4

  85. #define A	r8

  86. #define LDA	r9

  87. #define X	r10

  88. #define	INCX	r5

  89. #define	Y	r6

  90. #define	INCY	r7

  91. #endif

  92. #endif

  93. 

  94. #define	BUFFER	r11

  95. #define	XP	r12

  96. #define	X1	r14

  97. #define	J	r15

  98. #define	AO1	r16

  99. #define	AO2	r17

  100. #define	AO3	r18

  101. #define	AO4	r19

  102. #define	PREA	r20

  103. #define	PREC	r21

  104. #define	YY	r22

  105. 

  106. #if defined(PPCG4)

  107. #define PREFETCHSIZE_A  (3 * 8)

  108. #define PREFETCHSIZE_C   7

  109. #endif

  110. 

  111. #if defined(POWER6)

  112. #define PREFETCHSIZE_A  (3 * 8)

  113. #define PREFETCHSIZE_C   7

  114. #endif

  115. 

  116. #if !(defined(CONJ) && defined(XCONJ))

  117. #define FMADDR FMADD

  118. #define FMSUBR FNMSUB

  119. #else

  120. #define FMADDR FNMSUB

  121. #define FMSUBR FMADD

  122. #endif

  123. 

  124. #ifndef NEEDPARAM

  125. 

  126. #ifndef __64BIT__

  127. #define FZERO	200(SP)

  128. #else

  129. #define FZERO	256(SP)

  130. #endif

  131. 

  132. 	PROLOGUE

  133. 	PROFCODE

  134. 

  135. 	addi	SP, SP,  -STACKSIZE

  136. 	li	r0,   0

  137. 

  138. 	stfd	f14,     0(SP)

  139. 	stfd	f15,     8(SP)

  140. 	stfd	f16,    16(SP)

  141. 	stfd	f17,    24(SP)

  142. 	stfd	f18,    32(SP)

  143. 	stfd	f19,    40(SP)

  144. 	stfd	f20,    48(SP)

  145. 	stfd	f21,    56(SP)

  146. 	stfd	f22,    64(SP)

  147. 	stfd	f23,    72(SP)

  148. 	stfd	f24,    80(SP)

  149. 	stfd	f25,    88(SP)

  150. 	stfd	f26,    96(SP)

  151. 	stfd	f27,   104(SP)

  152. 	stfd	f28,   112(SP)

  153. 	stfd	f29,   120(SP)

  154. 	stfd	f30,   128(SP)

  155. 	stfd	f31,   136(SP)

  156. 

  157. #ifdef __64BIT__

  158. 	std	r14,   144(SP)

  159. 	std	r15,   152(SP)

  160. 	std	r16,   160(SP)

  161. 	std	r17,   168(SP)

  162. 	std	r18,   176(SP)

  163. 	std	r19,   184(SP)

  164. 	std	r20,   192(SP)

  165. 	std	r21,   200(SP)

  166. 	std	r22,   208(SP)

  167. 	std	r0,    FZERO

  168. #else

  169. 	stw	r14,   144(SP)

  170. 	stw	r15,   148(SP)

  171. 	stw	r16,   152(SP)

  172. 	stw	r17,   156(SP)

  173. 	stw	r18,   160(SP)

  174. 	stw	r19,   164(SP)

  175. 	stw	r20,   168(SP)

  176. 	stw	r21,   172(SP)

  177. 	stw	r22,   176(SP)

  178. 	stw	r0,    FZERO

  179. 	stw	r0,    4 + FZERO

  180. #endif

  181. 

  182. #if defined(linux) || defined(__FreeBSD__)

  183. #ifndef __64BIT__

  184. 	lwz	INCY,	 FRAMESLOT(0) + STACKSIZE(SP)

  185. 	lwz	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  186. #else

  187. 	ld	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  188. 	ld	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  189. 	ld	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  190. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  191. #endif

  192. #endif

  193. 

  194. #if defined(_AIX) || defined(__APPLE__)

  195. #ifndef __64BIT__

  196. #ifdef DOUBLE

  197. 	lwz	LDA,     FRAMESLOT(0) + STACKSIZE(SP)

  198. 	lwz	X,       FRAMESLOT(1) + STACKSIZE(SP)

  199. 	lwz	INCX,    FRAMESLOT(2) + STACKSIZE(SP)

  200. 	lwz	Y,       FRAMESLOT(3) + STACKSIZE(SP)

  201. 	lwz	INCY,    FRAMESLOT(4) + STACKSIZE(SP)

  202. 	lwz	BUFFER,  FRAMESLOT(5) + STACKSIZE(SP)

  203. #else

  204. 	lwz	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  205. 	lwz	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  206. 	lwz	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  207. 	lwz	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  208. #endif

  209. #else

  210. 	ld	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  211. 	ld	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  212. 	ld	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  213. 	ld	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  214. #endif

  215. #endif

  216. 

  217. #ifndef XCONJ

  218. #ifndef CONJ

  219. #define FMADD1	FMADD

  220. #define FMADD2	FMADD

  221. #define FMADD3	FNMSUB

  222. #define FMADD4	FMADD

  223. #else

  224. #define FMADD1	FMADD

  225. #define FMADD2	FMADD

  226. #define FMADD3	FMADD

  227. #define FMADD4	FNMSUB

  228. #endif

  229. #else

  230. #ifndef CONJ

  231. #define FMADD1	FMADD

  232. #define FMADD2	FNMSUB

  233. #define FMADD3	FMADD

  234. #define FMADD4	FMADD

  235. #else

  236. #define FMADD1	FMADD

  237. #define FMADD2	FMADD

  238. #define FMADD3	FNMSUB

  239. #define FMADD4	FMADD

  240. #endif

  241. #endif

  242. 

  243. #define y1 f0

  244. #define y2 f1

  245. #define y3 f2

  246. #define y4 f3

  247. #define y5 f4

  248. #define y6 f5

  249. #define y7 f6

  250. #define y8 f7

  251. 

  252. #define a1	f8

  253. #define a2	f9

  254. #define a3	f10

  255. #define a4	f11

  256. #define a5	f12

  257. #define a6	f13

  258. #define a7	f14

  259. #define a8	f15

  260. 

  261. #define b1	f16

  262. #define b2	f17

  263. #define b3	f18

  264. #define b4	f19

  265. #define b5	f20

  266. #define b6	f21

  267. #define b7	f22

  268. #define b8	f23

  269. 

  270. #define alpha_r	f24

  271. #define alpha_i	f25

  272. 

  273. 	fmr	alpha_r, f1

  274. 	fmr	alpha_i, f2

  275. 

  276. 	slwi	LDA,  LDA,  ZBASE_SHIFT

  277. 	slwi	INCX, INCX, ZBASE_SHIFT

  278. 	slwi	INCY, INCY, ZBASE_SHIFT

  279. 

  280. 	li	PREA, PREFETCHSIZE_A * SIZE

  281. 	li	PREC, PREFETCHSIZE_C * SIZE

  282. 

  283. 	addi	A, A, -SIZE

  284. 	addi	INCX, INCX, -SIZE

  285. 	addi	INCY, INCY, -SIZE

  286. 

  287. 	sub	X, X, INCX

  288. 	sub	Y, Y, INCY

  289. 

  290. 	mr	YY, Y

  291. 

  292. 	cmpwi	cr0, M, 0

  293. 	ble	LL(999)

  294. 	cmpwi	cr0, N, 0

  295. 	ble	LL(999)

  296. 

  297. 	mr	XP, X

  298. 	cmpwi	cr0, INCX, SIZE

  299. 	beq	LL(10)

  300. 

  301. 	addi	XP, BUFFER, -SIZE

  302. 	addi	X1, BUFFER, -SIZE

  303. 

  304. 	srawi.	r0, M, 2

  305. 	mtspr	CTR, r0

  306. 	ble	LL(05)

  307. 	.align 4

  308. 

  309. LL(02):

  310. 	LFDUX	f0, X, INCX

  311. 	LFDU	f1, 1 * SIZE(X)

  312. 	LFDUX	f2, X, INCX

  313. 	LFDU	f3, 1 * SIZE(X)

  314. 	LFDUX	f4, X, INCX

  315. 	LFDU	f5, 1 * SIZE(X)

  316. 	LFDUX	f6, X, INCX

  317. 	LFDU	f7, 1 * SIZE(X)

  318. 

  319. 	STFDU	f0,  1 * SIZE(X1)

  320. 	STFDU	f1,  1 * SIZE(X1)

  321. 	STFDU	f2,  1 * SIZE(X1)

  322. 	STFDU	f3,  1 * SIZE(X1)

  323. 	STFDU	f4,  1 * SIZE(X1)

  324. 	STFDU	f5,  1 * SIZE(X1)

  325. 	STFDU	f6,  1 * SIZE(X1)

  326. 	STFDU	f7,  1 * SIZE(X1)

  327. 	bdnz	LL(02)

  328. 	.align 4

  329. 

  330. LL(05):

  331. 	andi.	r0, M, 3

  332. 	mtspr	CTR, r0

  333. 	ble	LL(10)

  334. 	.align 4

  335. 

  336. LL(06):

  337. 	LFDUX	f0, X, INCX

  338. 	LFDU	f1,  1 * SIZE(X)

  339. 	STFDU	f0,  1 * SIZE(X1)

  340. 	STFDU	f1,  1 * SIZE(X1)

  341. 	bdnz	LL(06)

  342. 	.align 4

  343. 

  344. LL(10):

  345. 	srawi.	J, N, 2

  346. 	ble	LL(20)

  347. 	.align 4

  348. 

  349. LL(11):

  350. 	lfd	 y1,  FZERO

  351. 	mr     AO1, A

  352. 	fmr	 y2,  y1

  353. 	mr     X1, XP

  354. 	fmr	 y3,  y1

  355. 	add    AO2, A,   LDA

  356. 	fmr	 y4,  y1

  357. 	add    AO3, AO2, LDA

  358. 	fmr	 y5,  y1

  359. 	add    AO4, AO3, LDA

  360. 	fmr	 y6,  y1

  361. 	add    A,   AO4, LDA

  362. 	fmr	 y7,  y1

  363. 

  364. 	dcbtst	 PREC, Y

  365. 	fmr	 y8,  y1

  366. 

  367. 	srawi.	r0,  M, 2

  368. 	mtspr	CTR, r0

  369. 	ble	LL(15)

  370. 

  371. 	LFDU	a1, 1 * SIZE(AO1)

  372. 	LFDU	b1, 1 * SIZE(X1)

  373. 	LFDU	a2, 1 * SIZE(AO1)

  374. 	LFDU	b2, 1 * SIZE(X1)

  375. 	LFDU	a3, 1 * SIZE(AO2)

  376. 	LFDU	a4, 1 * SIZE(AO2)

  377. 	LFDU	a5, 1 * SIZE(AO3)

  378. 	LFDU	a6, 1 * SIZE(AO3)

  379. 	LFDU	a7, 1 * SIZE(AO4)

  380. 	bdz	LL(13)

  381. 	.align 5

  382. 

  383. LL(12):

  384. 	FMADD1	y1,  a1,  b1, y1

  385. 	LFDU	a8, 1 * SIZE(AO4)

  386. 	FMADD2	y2,  a1,  b2, y2

  387. 	LFDU	b3, 1 * SIZE(X1)

  388. 	FMADD1	y3,  a3,  b1, y3

  389. 	LFDU	b4, 1 * SIZE(X1)

  390. 	FMADD2	y4,  a3,  b2, y4

  391. 

  392. #ifdef PPCG4

  393. 	dcbt	AO1, PREA

  394. #endif

  395. 

  396. 	FMADD3	y1,  a2,  b2, y1

  397. 	LFDU	a1, 1 * SIZE(AO1)

  398. 	FMADD4	y2,  a2,  b1, y2

  399. 	LFDU	a2, 1 * SIZE(AO1)

  400. 	FMADD3	y3,  a4,  b2, y3

  401. 	LFDU	a3, 1 * SIZE(AO2)

  402. 	FMADD4	y4,  a4,  b1, y4

  403. 	LFDU	a4, 1 * SIZE(AO2)

  404. 

  405. #ifdef PPCG4

  406. 	dcbt	X1, PREA

  407. #endif

  408. 

  409. 	FMADD1	y5,  a5,  b1, y5

  410. 	FMADD2	y6,  a5,  b2, y6

  411. 	FMADD1	y7,  a7,  b1, y7

  412. 	FMADD2	y8,  a7,  b2, y8

  413. 

  414. #ifdef PPCG4

  415. 	dcbt	AO2, PREA

  416. #endif

  417. 

  418. 	FMADD3	y5,  a6,  b2, y5

  419. 	LFDU	a5, 1 * SIZE(AO3)

  420. 	FMADD4	y6,  a6,  b1, y6

  421. 	LFDU	a6, 1 * SIZE(AO3)

  422. 	FMADD3	y7,  a8,  b2, y7

  423. 	LFDU	a7, 1 * SIZE(AO4)

  424. 	FMADD4	y8,  a8,  b1, y8

  425. 	LFDU	a8, 1 * SIZE(AO4)

  426. 

  427. 

  428. 	FMADD1	y1,  a1,  b3, y1

  429. 	LFDU	b1, 1 * SIZE(X1)

  430. 	FMADD2	y2,  a1,  b4, y2

  431. 	LFDU	b2, 1 * SIZE(X1)

  432. 	FMADD1	y3,  a3,  b3, y3

  433. 	FMADD2	y4,  a3,  b4, y4

  434. 

  435. #ifdef PPCG4

  436. 	dcbt	AO3, PREA

  437. #endif

  438. 

  439. 	FMADD3	y1,  a2,  b4, y1

  440. 	LFDU	a1, 1 * SIZE(AO1)

  441. 	FMADD4	y2,  a2,  b3, y2

  442. 	LFDU	a2, 1 * SIZE(AO1)

  443. 	FMADD3	y3,  a4,  b4, y3

  444. 	LFDU	a3, 1 * SIZE(AO2)

  445. 	FMADD4	y4,  a4,  b3, y4

  446. 	LFDU	a4, 1 * SIZE(AO2)

  447. 

  448. 	FMADD1	y5,  a5,  b3, y5

  449. 	FMADD2	y6,  a5,  b4, y6

  450. 	FMADD1	y7,  a7,  b3, y7

  451. 	FMADD2	y8,  a7,  b4, y8

  452. 

  453. #ifdef PPCG4

  454. 	dcbt	AO4, PREA

  455. #endif

  456. 

  457. 	FMADD3	y5,  a6,  b4, y5

  458. 	LFDU	a5, 1 * SIZE(AO3)

  459. 	FMADD4	y6,  a6,  b3, y6

  460. 	LFDU	a6, 1 * SIZE(AO3)

  461. 	FMADD3	y7,  a8,  b4, y7

  462. 	LFDU	a7, 1 * SIZE(AO4)

  463. 	FMADD4	y8,  a8,  b3, y8

  464. 	LFDU	a8, 1 * SIZE(AO4)

  465. 

  466. 	FMADD1	y1,  a1,  b1, y1

  467. 	LFDU	b3, 1 * SIZE(X1)

  468. 	FMADD2	y2,  a1,  b2, y2

  469. 	LFDU	b4, 1 * SIZE(X1)

  470. 	FMADD1	y3,  a3,  b1, y3

  471. 	FMADD2	y4,  a3,  b2, y4

  472. 

  473. #if defined(PPCG4) && defined(DOUBLE)

  474. 	dcbt	AO1, PREA

  475. #endif

  476. 

  477. 	FMADD3	y1,  a2,  b2, y1

  478. 	LFDU	a1, 1 * SIZE(AO1)

  479. 	FMADD4	y2,  a2,  b1, y2

  480. 	LFDU	a2, 1 * SIZE(AO1)

  481. 	FMADD3	y3,  a4,  b2, y3

  482. 	LFDU	a3, 1 * SIZE(AO2)

  483. 	FMADD4	y4,  a4,  b1, y4

  484. 	LFDU	a4, 1 * SIZE(AO2)

  485. 

  486. #if defined(PPCG4) && defined(DOUBLE)

  487. 	dcbt	X1, PREA

  488. #endif

  489. 

  490. 	FMADD1	y5,  a5,  b1, y5

  491. 	FMADD2	y6,  a5,  b2, y6

  492. 	FMADD1	y7,  a7,  b1, y7

  493. 	FMADD2	y8,  a7,  b2, y8

  494. 

  495. #if defined(PPCG4) && defined(DOUBLE)

  496. 	dcbt	AO2, PREA

  497. #endif

  498. 

  499. 	FMADD3	y5,  a6,  b2, y5

  500. 	LFDU	a5, 1 * SIZE(AO3)

  501. 	FMADD4	y6,  a6,  b1, y6

  502. 	LFDU	a6, 1 * SIZE(AO3)

  503. 	FMADD3	y7,  a8,  b2, y7

  504. 	LFDU	a7, 1 * SIZE(AO4)

  505. 	FMADD4	y8,  a8,  b1, y8

  506. 	LFDU	a8, 1 * SIZE(AO4)

  507. 

  508. 	FMADD1	y1,  a1,  b3, y1

  509. 	FMADD2	y2,  a1,  b4, y2

  510. 	FMADD1	y3,  a3,  b3, y3

  511. 	FMADD2	y4,  a3,  b4, y4

  512. 

  513. #if defined(PPCG4) && defined(DOUBLE)

  514. 	dcbt	AO3, PREA

  515. #endif

  516. 

  517. 	FMADD3	y1,  a2,  b4, y1

  518. 	LFDU	a1, 1 * SIZE(AO1)

  519. 	FMADD4	y2,  a2,  b3, y2

  520. 	LFDU	a2, 1 * SIZE(AO1)

  521. 	FMADD3	y3,  a4,  b4, y3

  522. 	LFDU	a3, 1 * SIZE(AO2)

  523. 	FMADD4	y4,  a4,  b3, y4

  524. 	LFDU	a4, 1 * SIZE(AO2)

  525. 

  526. 	FMADD1	y5,  a5,  b3, y5

  527. 	LFDU	b1, 1 * SIZE(X1)

  528. 	FMADD2	y6,  a5,  b4, y6

  529. 	LFDU	b2, 1 * SIZE(X1)

  530. 	FMADD1	y7,  a7,  b3, y7

  531. 	FMADD2	y8,  a7,  b4, y8

  532. 

  533. #if defined(PPCG4) && defined(DOUBLE)

  534. 	dcbt	AO4, PREA

  535. #endif

  536. 

  537. 	FMADD3	y5,  a6,  b4, y5

  538. 	LFDU	a5, 1 * SIZE(AO3)

  539. 	FMADD4	y6,  a6,  b3, y6

  540. 	LFDU	a6, 1 * SIZE(AO3)

  541. 	FMADD3	y7,  a8,  b4, y7

  542. 	LFDU	a7, 1 * SIZE(AO4)

  543. 	FMADD4	y8,  a8,  b3, y8

  544.  	bdnz	LL(12)

  545. 	.align 4

  546. 

  547. LL(13):

  548. 	FMADD1	y1,  a1,  b1, y1

  549. 	LFDU	a8, 1 * SIZE(AO4)

  550. 	FMADD2	y2,  a1,  b2, y2

  551. 	LFDU	b3, 1 * SIZE(X1)

  552. 	FMADD1	y3,  a3,  b1, y3

  553. 	LFDU	b4, 1 * SIZE(X1)

  554. 	FMADD2	y4,  a3,  b2, y4

  555. 

  556. 	FMADD3	y1,  a2,  b2, y1

  557. 	LFDU	a1, 1 * SIZE(AO1)

  558. 	FMADD4	y2,  a2,  b1, y2

  559. 	LFDU	a2, 1 * SIZE(AO1)

  560. 	FMADD3	y3,  a4,  b2, y3

  561. 	LFDU	a3, 1 * SIZE(AO2)

  562. 	FMADD4	y4,  a4,  b1, y4

  563. 	LFDU	a4, 1 * SIZE(AO2)

  564. 

  565. 	FMADD1	y5,  a5,  b1, y5

  566. 	FMADD2	y6,  a5,  b2, y6

  567. 	FMADD1	y7,  a7,  b1, y7

  568. 	FMADD2	y8,  a7,  b2, y8

  569. 

  570. 	FMADD3	y5,  a6,  b2, y5

  571. 	LFDU	a5, 1 * SIZE(AO3)

  572. 	FMADD4	y6,  a6,  b1, y6

  573. 	LFDU	a6, 1 * SIZE(AO3)

  574. 	FMADD3	y7,  a8,  b2, y7

  575. 	LFDU	a7, 1 * SIZE(AO4)

  576. 	FMADD4	y8,  a8,  b1, y8

  577. 	LFDU	a8, 1 * SIZE(AO4)

  578. 

  579. 	FMADD1	y1,  a1,  b3, y1

  580. 	LFDU	b1, 1 * SIZE(X1)

  581. 	FMADD2	y2,  a1,  b4, y2

  582. 	LFDU	b2, 1 * SIZE(X1)

  583. 	FMADD1	y3,  a3,  b3, y3

  584. 	FMADD2	y4,  a3,  b4, y4

  585. 

  586. 	FMADD3	y1,  a2,  b4, y1

  587. 	LFDU	a1, 1 * SIZE(AO1)

  588. 	FMADD4	y2,  a2,  b3, y2

  589. 	LFDU	a2, 1 * SIZE(AO1)

  590. 	FMADD3	y3,  a4,  b4, y3

  591. 	LFDU	a3, 1 * SIZE(AO2)

  592. 	FMADD4	y4,  a4,  b3, y4

  593. 	LFDU	a4, 1 * SIZE(AO2)

  594. 

  595. 	FMADD1	y5,  a5,  b3, y5

  596. 	FMADD2	y6,  a5,  b4, y6

  597. 	FMADD1	y7,  a7,  b3, y7

  598. 	FMADD2	y8,  a7,  b4, y8

  599. 

  600. 	FMADD3	y5,  a6,  b4, y5

  601. 	LFDU	a5, 1 * SIZE(AO3)

  602. 	FMADD4	y6,  a6,  b3, y6

  603. 	LFDU	a6, 1 * SIZE(AO3)

  604. 	FMADD3	y7,  a8,  b4, y7

  605. 	LFDU	a7, 1 * SIZE(AO4)

  606. 	FMADD4	y8,  a8,  b3, y8

  607. 	LFDU	a8, 1 * SIZE(AO4)

  608. 

  609. 	FMADD1	y1,  a1,  b1, y1

  610. 	LFDU	b3, 1 * SIZE(X1)

  611. 	FMADD2	y2,  a1,  b2, y2

  612. 	LFDU	b4, 1 * SIZE(X1)

  613. 	FMADD1	y3,  a3,  b1, y3

  614. 	FMADD2	y4,  a3,  b2, y4

  615. 

  616. 	FMADD3	y1,  a2,  b2, y1

  617. 	LFDU	a1, 1 * SIZE(AO1)

  618. 	FMADD4	y2,  a2,  b1, y2

  619. 	LFDU	a2, 1 * SIZE(AO1)

  620. 	FMADD3	y3,  a4,  b2, y3

  621. 	LFDU	a3, 1 * SIZE(AO2)

  622. 	FMADD4	y4,  a4,  b1, y4

  623. 	LFDU	a4, 1 * SIZE(AO2)

  624. 

  625. 	FMADD1	y5,  a5,  b1, y5

  626. 	FMADD2	y6,  a5,  b2, y6

  627. 	FMADD1	y7,  a7,  b1, y7

  628. 	FMADD2	y8,  a7,  b2, y8

  629. 

  630. 	FMADD3	y5,  a6,  b2, y5

  631. 	LFDU	a5, 1 * SIZE(AO3)

  632. 	FMADD4	y6,  a6,  b1, y6

  633. 	LFDU	a6, 1 * SIZE(AO3)

  634. 	FMADD3	y7,  a8,  b2, y7

  635. 	LFDU	a7, 1 * SIZE(AO4)

  636. 	FMADD4	y8,  a8,  b1, y8

  637. 	LFDU	a8, 1 * SIZE(AO4)

  638. 

  639. 	FMADD1	y1,  a1,  b3, y1

  640. 	FMADD2	y2,  a1,  b4, y2

  641. 	FMADD1	y3,  a3,  b3, y3

  642. 	FMADD2	y4,  a3,  b4, y4

  643. 

  644. 	FMADD3	y1,  a2,  b4, y1

  645. 	FMADD4	y2,  a2,  b3, y2

  646. 	FMADD3	y3,  a4,  b4, y3

  647. 	FMADD4	y4,  a4,  b3, y4

  648. 

  649. 	FMADD1	y5,  a5,  b3, y5

  650. 	FMADD2	y6,  a5,  b4, y6

  651. 	FMADD1	y7,  a7,  b3, y7

  652. 	FMADD2	y8,  a7,  b4, y8

  653. 

  654. 	FMADD3	y5,  a6,  b4, y5

  655. 	FMADD4	y6,  a6,  b3, y6

  656. 	FMADD3	y7,  a8,  b4, y7

  657. 	FMADD4	y8,  a8,  b3, y8

  658. 	.align 4

  659. 

  660. LL(15):

  661. 	andi.	r0, M, 2

  662. 	ble	LL(17)

  663. 

  664. 	LFDU	a1, 1 * SIZE(AO1)

  665. 	LFDU	b1, 1 * SIZE(X1)

  666. 	LFDU	a2, 1 * SIZE(AO1)

  667. 	LFDU	b2, 1 * SIZE(X1)

  668. 	LFDU	a3, 1 * SIZE(AO2)

  669. 	LFDU	b3, 1 * SIZE(X1)

  670. 	LFDU	a4, 1 * SIZE(AO2)

  671. 	LFDU	b4, 1 * SIZE(X1)

  672. 

  673. 	FMADD1	y1,  a1,  b1, y1

  674. 	LFDU	a5, 1 * SIZE(AO3)

  675. 	FMADD2	y2,  a1,  b2, y2

  676. 	LFDU	a6, 1 * SIZE(AO3)

  677. 	FMADD1	y3,  a3,  b1, y3

  678. 	LFDU	a7, 1 * SIZE(AO4)

  679. 	FMADD2	y4,  a3,  b2, y4

  680. 	LFDU	a8, 1 * SIZE(AO4)

  681. 

  682. 	FMADD3	y1,  a2,  b2, y1

  683. 	LFDU	a1, 1 * SIZE(AO1)

  684. 	FMADD4	y2,  a2,  b1, y2

  685. 	LFDU	a2, 1 * SIZE(AO1)

  686. 	FMADD3	y3,  a4,  b2, y3

  687. 	LFDU	a3, 1 * SIZE(AO2)

  688. 	FMADD4	y4,  a4,  b1, y4

  689. 	LFDU	a4, 1 * SIZE(AO2)

  690. 

  691. 	FMADD1	y5,  a5,  b1, y5

  692. 	FMADD2	y6,  a5,  b2, y6

  693. 	FMADD1	y7,  a7,  b1, y7

  694. 	FMADD2	y8,  a7,  b2, y8

  695. 

  696. 	FMADD3	y5,  a6,  b2, y5

  697. 	LFDU	a5, 1 * SIZE(AO3)

  698. 	FMADD4	y6,  a6,  b1, y6

  699. 	LFDU	a6, 1 * SIZE(AO3)

  700. 	FMADD3	y7,  a8,  b2, y7

  701. 	LFDU	a7, 1 * SIZE(AO4)

  702. 	FMADD4	y8,  a8,  b1, y8

  703. 	LFDU	a8, 1 * SIZE(AO4)

  704. 

  705. 	FMADD1	y1,  a1,  b3, y1

  706. 	FMADD2	y2,  a1,  b4, y2

  707. 	FMADD1	y3,  a3,  b3, y3

  708. 	FMADD2	y4,  a3,  b4, y4

  709. 

  710. 	FMADD3	y1,  a2,  b4, y1

  711. 	FMADD4	y2,  a2,  b3, y2

  712. 	FMADD3	y3,  a4,  b4, y3

  713. 	FMADD4	y4,  a4,  b3, y4

  714. 

  715. 	FMADD1	y5,  a5,  b3, y5

  716. 	FMADD2	y6,  a5,  b4, y6

  717. 	FMADD1	y7,  a7,  b3, y7

  718. 	FMADD2	y8,  a7,  b4, y8

  719. 

  720. 	FMADD3	y5,  a6,  b4, y5

  721. 	FMADD4	y6,  a6,  b3, y6

  722. 	FMADD3	y7,  a8,  b4, y7

  723. 	FMADD4	y8,  a8,  b3, y8

  724. 	.align 4

  725. 

  726. LL(17):

  727. 	andi.	r0, M, 1

  728. 	ble	LL(19)

  729. 

  730. 	LFDU	a1, 1 * SIZE(AO1)

  731. 	LFDU	a2, 1 * SIZE(AO1)

  732. 	LFDU	a3, 1 * SIZE(AO2)

  733. 	LFDU	a4, 1 * SIZE(AO2)

  734. 	LFDU	a5, 1 * SIZE(AO3)

  735. 	LFDU	a6, 1 * SIZE(AO3)

  736. 	LFDU	a7, 1 * SIZE(AO4)

  737. 	LFDU	a8, 1 * SIZE(AO4)

  738. 

  739. 	LFDU	b1, 1 * SIZE(X1)

  740. 	LFDU	b2, 1 * SIZE(X1)

  741. 

  742. 	FMADD1	y1,  a1,  b1, y1

  743. 	FMADD2	y2,  a1,  b2, y2

  744. 	FMADD1	y3,  a3,  b1, y3

  745. 	FMADD2	y4,  a3,  b2, y4

  746. 

  747. 	FMADD3	y1,  a2,  b2, y1

  748. 	FMADD4	y2,  a2,  b1, y2

  749. 	FMADD3	y3,  a4,  b2, y3

  750. 	FMADD4	y4,  a4,  b1, y4

  751. 

  752. 	FMADD1	y5,  a5,  b1, y5

  753. 	FMADD2	y6,  a5,  b2, y6

  754. 	FMADD1	y7,  a7,  b1, y7

  755. 	FMADD2	y8,  a7,  b2, y8

  756. 

  757. 	FMADD3	y5,  a6,  b2, y5

  758. 	FMADD4	y6,  a6,  b1, y6

  759. 	FMADD3	y7,  a8,  b2, y7

  760. 	FMADD4	y8,  a8,  b1, y8

  761. 	.align 4

  762. 

  763. LL(19):

  764. 	LFDUX	b1,  Y, INCY

  765. 	LFDU	b2,  1 * SIZE(Y)

  766. 	LFDUX	b3,  Y, INCY

  767. 	LFDU	b4,  1 * SIZE(Y)

  768. 	LFDUX	b5,  Y, INCY

  769. 	LFDU	b6,  1 * SIZE(Y)

  770. 	LFDUX	b7,  Y, INCY

  771. 	LFDU	b8,  1 * SIZE(Y)

  772. 

  773. 	FMADD	b1, alpha_r, y1, b1

  774. 	FMADDR	b2, alpha_r, y2, b2

  775. 	FMADD	b3, alpha_r, y3, b3

  776. 	FMADDR	b4, alpha_r, y4, b4

  777. 

  778. 	FMADD	b5, alpha_r, y5, b5

  779. 	FMADDR	b6, alpha_r, y6, b6

  780. 	FMADD	b7, alpha_r, y7, b7

  781. 	FMADDR	b8, alpha_r, y8, b8

  782. 

  783. 	FMSUBR	b1, alpha_i, y2, b1

  784. 	FMADD	b2, alpha_i, y1, b2

  785. 	FMSUBR	b3, alpha_i, y4, b3

  786. 	FMADD	b4, alpha_i, y3, b4

  787. 

  788. 	FMSUBR	b5, alpha_i, y6, b5

  789. 	FMADD	b6, alpha_i, y5, b6

  790. 	FMSUBR	b7, alpha_i, y8, b7

  791. 	FMADD	b8, alpha_i, y7, b8

  792. 

  793. 	STFDUX	b1,  YY, INCY

  794. 	STFDU	b2,  1 * SIZE(YY)

  795. 	STFDUX	b3,  YY, INCY

  796. 	STFDU	b4,  1 * SIZE(YY)

  797. 

  798. 	STFDUX	b5,  YY, INCY

  799. 	STFDU	b6,  1 * SIZE(YY)

  800. 	STFDUX	b7,  YY, INCY

  801. 	STFDU	b8,  1 * SIZE(YY)

  802. 

  803. 	addi	J, J, -1

  804. 	cmpwi	cr0, J, 0

  805. 	bgt	LL(11)

  806. 	.align 4

  807. 

  808. LL(20):

  809. 	andi.	J, N, 2

  810. 	ble	LL(30)

  811. 

  812. 	lfd	 y1,  FZERO

  813. 	mr     AO1, A

  814. 	fmr	 y2,  y1

  815. 	mr     X1, XP

  816. 	fmr	 y3,  y1

  817. 	add    AO2, A,   LDA

  818. 	fmr	 y4,  y1

  819. 	add    A,   AO2, LDA

  820. 

  821. 	srawi.	r0,  M, 2

  822. 	mtspr	CTR, r0

  823. 	ble	LL(25)

  824. 

  825. 	LFDU	a1, 1 * SIZE(AO1)

  826. 	LFDU	b1, 1 * SIZE(X1)

  827. 	LFDU	a2, 1 * SIZE(AO1)

  828. 	LFDU	b2, 1 * SIZE(X1)

  829. 	LFDU	a3, 1 * SIZE(AO2)

  830. 	bdz	LL(23)

  831. 	.align 5

  832. 

  833. LL(22):

  834. 	FMADD1	y1,  a1,  b1, y1

  835. 	LFDU	a4, 1 * SIZE(AO2)

  836. 	FMADD2	y2,  a1,  b2, y2

  837. 	LFDU	b3, 1 * SIZE(X1)

  838. 	FMADD1	y3,  a3,  b1, y3

  839. 	LFDU	b4, 1 * SIZE(X1)

  840. 	FMADD2	y4,  a3,  b2, y4

  841. 

  842. #ifdef PPCG4

  843. 	dcbt	AO1, PREA

  844. #endif

  845. 

  846. 	FMADD3	y1,  a2,  b2, y1

  847. 	LFDU	a1, 1 * SIZE(AO1)

  848. 	FMADD4	y2,  a2,  b1, y2

  849. 	LFDU	a2, 1 * SIZE(AO1)

  850. 	FMADD3	y3,  a4,  b2, y3

  851. 	LFDU	a3, 1 * SIZE(AO2)

  852. 	FMADD4	y4,  a4,  b1, y4

  853. 	LFDU	a4, 1 * SIZE(AO2)

  854. 

  855. #ifdef PPCG4

  856. 	dcbt	AO2, PREA

  857. #endif

  858. 

  859. 	FMADD1	y1,  a1,  b3, y1

  860. 	LFDU	b1, 1 * SIZE(X1)

  861. 	FMADD2	y2,  a1,  b4, y2

  862. 	LFDU	b2, 1 * SIZE(X1)

  863. 	FMADD1	y3,  a3,  b3, y3

  864. 	FMADD2	y4,  a3,  b4, y4

  865. 

  866. #ifdef PPCG4

  867. 	dcbt	X1, PREA

  868. #endif

  869. 

  870. 	FMADD3	y1,  a2,  b4, y1

  871. 	LFDU	a1, 1 * SIZE(AO1)

  872. 	FMADD4	y2,  a2,  b3, y2

  873. 	LFDU	a2, 1 * SIZE(AO1)

  874. 	FMADD3	y3,  a4,  b4, y3

  875. 	LFDU	a3, 1 * SIZE(AO2)

  876. 	FMADD4	y4,  a4,  b3, y4

  877. 	LFDU	a4, 1 * SIZE(AO2)

  878. 

  879. 	FMADD1	y1,  a1,  b1, y1

  880. 	LFDU	b3, 1 * SIZE(X1)

  881. 	FMADD2	y2,  a1,  b2, y2

  882. 	LFDU	b4, 1 * SIZE(X1)

  883. 	FMADD1	y3,  a3,  b1, y3

  884. 	FMADD2	y4,  a3,  b2, y4

  885. 

  886. #if defined(PPCG4) && defined(DOUBLE)

  887. 	dcbt	AO1, PREA

  888. #endif

  889. 

  890. 	FMADD3	y1,  a2,  b2, y1

  891. 	LFDU	a1, 1 * SIZE(AO1)

  892. 	FMADD4	y2,  a2,  b1, y2

  893. 	LFDU	a2, 1 * SIZE(AO1)

  894. 	FMADD3	y3,  a4,  b2, y3

  895. 	LFDU	a3, 1 * SIZE(AO2)

  896. 	FMADD4	y4,  a4,  b1, y4

  897. 	LFDU	a4, 1 * SIZE(AO2)

  898. 

  899. #if defined(PPCG4) && defined(DOUBLE)

  900. 	dcbt	AO2, PREA

  901. #endif

  902. 

  903. 	FMADD1	y1,  a1,  b3, y1

  904. 	LFDU	b1, 1 * SIZE(X1)

  905. 	FMADD2	y2,  a1,  b4, y2

  906. 	LFDU	b2, 1 * SIZE(X1)

  907. 	FMADD1	y3,  a3,  b3, y3

  908. 	FMADD2	y4,  a3,  b4, y4

  909. 

  910. #if defined(PPCG4) && defined(DOUBLE)

  911. 	dcbt	X1, PREA

  912. #endif

  913. 

  914. 	FMADD3	y1,  a2,  b4, y1

  915. 	LFDU	a1, 1 * SIZE(AO1)

  916. 	FMADD4	y2,  a2,  b3, y2

  917. 	LFDU	a2, 1 * SIZE(AO1)

  918. 	FMADD3	y3,  a4,  b4, y3

  919. 	LFDU	a3, 1 * SIZE(AO2)

  920. 	FMADD4	y4,  a4,  b3, y4

  921. 

  922.  	bdnz	LL(22)

  923. 	.align 4

  924. 

  925. LL(23):

  926. 	FMADD1	y1,  a1,  b1, y1

  927. 	LFDU	a4, 1 * SIZE(AO2)

  928. 	FMADD2	y2,  a1,  b2, y2

  929. 	LFDU	b3, 1 * SIZE(X1)

  930. 	FMADD1	y3,  a3,  b1, y3

  931. 	LFDU	b4, 1 * SIZE(X1)

  932. 	FMADD2	y4,  a3,  b2, y4

  933. 

  934. 	FMADD3	y1,  a2,  b2, y1

  935. 	LFDU	a1, 1 * SIZE(AO1)

  936. 	FMADD4	y2,  a2,  b1, y2

  937. 	LFDU	a2, 1 * SIZE(AO1)

  938. 	FMADD3	y3,  a4,  b2, y3

  939. 	LFDU	a3, 1 * SIZE(AO2)

  940. 	FMADD4	y4,  a4,  b1, y4

  941. 	LFDU	a4, 1 * SIZE(AO2)

  942. 

  943. 	FMADD1	y1,  a1,  b3, y1

  944. 	LFDU	b1, 1 * SIZE(X1)

  945. 	FMADD2	y2,  a1,  b4, y2

  946. 	LFDU	b2, 1 * SIZE(X1)

  947. 	FMADD1	y3,  a3,  b3, y3

  948. 	FMADD2	y4,  a3,  b4, y4

  949. 

  950. 	FMADD3	y1,  a2,  b4, y1

  951. 	LFDU	a1, 1 * SIZE(AO1)

  952. 	FMADD4	y2,  a2,  b3, y2

  953. 	LFDU	a2, 1 * SIZE(AO1)

  954. 	FMADD3	y3,  a4,  b4, y3

  955. 	LFDU	a3, 1 * SIZE(AO2)

  956. 	FMADD4	y4,  a4,  b3, y4

  957. 	LFDU	a4, 1 * SIZE(AO2)

  958. 

  959. 	FMADD1	y1,  a1,  b1, y1

  960. 	LFDU	b3, 1 * SIZE(X1)

  961. 	FMADD2	y2,  a1,  b2, y2

  962. 	LFDU	b4, 1 * SIZE(X1)

  963. 	FMADD1	y3,  a3,  b1, y3

  964. 	FMADD2	y4,  a3,  b2, y4

  965. 

  966. 	FMADD3	y1,  a2,  b2, y1

  967. 	LFDU	a1, 1 * SIZE(AO1)

  968. 	FMADD4	y2,  a2,  b1, y2

  969. 	LFDU	a2, 1 * SIZE(AO1)

  970. 	FMADD3	y3,  a4,  b2, y3

  971. 	LFDU	a3, 1 * SIZE(AO2)

  972. 	FMADD4	y4,  a4,  b1, y4

  973. 	LFDU	a4, 1 * SIZE(AO2)

  974. 

  975. 	FMADD1	y1,  a1,  b3, y1

  976. 	FMADD2	y2,  a1,  b4, y2

  977. 	FMADD1	y3,  a3,  b3, y3

  978. 	FMADD2	y4,  a3,  b4, y4

  979. 

  980. 	FMADD3	y1,  a2,  b4, y1

  981. 	FMADD4	y2,  a2,  b3, y2

  982. 	FMADD3	y3,  a4,  b4, y3

  983. 	FMADD4	y4,  a4,  b3, y4

  984. 	.align 4

  985. 

  986. LL(25):

  987. 	andi.	r0, M, 2

  988. 	ble	LL(27)

  989. 

  990. 	LFDU	a1, 1 * SIZE(AO1)

  991. 	LFDU	b1, 1 * SIZE(X1)

  992. 	LFDU	a2, 1 * SIZE(AO1)

  993. 	LFDU	b2, 1 * SIZE(X1)

  994. 	LFDU	a3, 1 * SIZE(AO2)

  995. 	LFDU	b3, 1 * SIZE(X1)

  996. 	LFDU	a4, 1 * SIZE(AO2)

  997. 	LFDU	b4, 1 * SIZE(X1)

  998. 

  999. 	FMADD1	y1,  a1,  b1, y1

  1000. 	FMADD2	y2,  a1,  b2, y2

  1001. 	FMADD1	y3,  a3,  b1, y3

  1002. 	FMADD2	y4,  a3,  b2, y4

  1003. 

  1004. 	FMADD3	y1,  a2,  b2, y1

  1005. 	LFDU	a1, 1 * SIZE(AO1)

  1006. 	FMADD4	y2,  a2,  b1, y2

  1007. 	LFDU	a2, 1 * SIZE(AO1)

  1008. 	FMADD3	y3,  a4,  b2, y3

  1009. 	LFDU	a3, 1 * SIZE(AO2)

  1010. 	FMADD4	y4,  a4,  b1, y4

  1011. 	LFDU	a4, 1 * SIZE(AO2)

  1012. 

  1013. 	FMADD1	y1,  a1,  b3, y1

  1014. 	FMADD2	y2,  a1,  b4, y2

  1015. 	FMADD1	y3,  a3,  b3, y3

  1016. 	FMADD2	y4,  a3,  b4, y4

  1017. 

  1018. 	FMADD3	y1,  a2,  b4, y1

  1019. 	FMADD4	y2,  a2,  b3, y2

  1020. 	FMADD3	y3,  a4,  b4, y3

  1021. 	FMADD4	y4,  a4,  b3, y4

  1022. 	.align 4

  1023. 

  1024. LL(27):

  1025. 	andi.	r0, M, 1

  1026. 	ble	LL(29)

  1027. 

  1028. 	LFDU	a1, 1 * SIZE(AO1)

  1029. 	LFDU	a2, 1 * SIZE(AO1)

  1030. 	LFDU	a3, 1 * SIZE(AO2)

  1031. 	LFDU	a4, 1 * SIZE(AO2)

  1032. 

  1033. 	LFDU	b1, 1 * SIZE(X1)

  1034. 	LFDU	b2, 1 * SIZE(X1)

  1035. 

  1036. 	FMADD1	y1,  a1,  b1, y1

  1037. 	FMADD2	y2,  a1,  b2, y2

  1038. 	FMADD1	y3,  a3,  b1, y3

  1039. 	FMADD2	y4,  a3,  b2, y4

  1040. 

  1041. 	FMADD3	y1,  a2,  b2, y1

  1042. 	FMADD4	y2,  a2,  b1, y2

  1043. 	FMADD3	y3,  a4,  b2, y3

  1044. 	FMADD4	y4,  a4,  b1, y4

  1045. 	.align 4

  1046. 

  1047. LL(29):

  1048. 	LFDUX	b1,  Y, INCY

  1049. 	LFDU	b2,  1 * SIZE(Y)

  1050. 	LFDUX	b3,  Y, INCY

  1051. 	LFDU	b4,  1 * SIZE(Y)

  1052. 

  1053. 	FMADD	b1, alpha_r, y1, b1

  1054. 	FMADDR	b2, alpha_r, y2, b2

  1055. 	FMADD	b3, alpha_r, y3, b3

  1056. 	FMADDR	b4, alpha_r, y4, b4

  1057. 

  1058. 	FMSUBR	b1, alpha_i, y2, b1

  1059. 	FMADD	b2, alpha_i, y1, b2

  1060. 	FMSUBR	b3, alpha_i, y4, b3

  1061. 	FMADD	b4, alpha_i, y3, b4

  1062. 

  1063. 	STFDUX	b1,  YY, INCY

  1064. 	STFDU	b2,  1 * SIZE(YY)

  1065. 	STFDUX	b3,  YY, INCY

  1066. 	STFDU	b4,  1 * SIZE(YY)

  1067. 	.align 4

  1068. 

  1069. LL(30):

  1070. 	andi.	J, N, 1

  1071. 	ble	LL(999)

  1072. 

  1073. 	lfd	 y1,  FZERO

  1074. 	mr     AO1, A

  1075. 	fmr	 y2,  y1

  1076. 	mr     X1, XP

  1077. 	fmr	 y3,  y1

  1078. 	fmr	 y4,  y1

  1079. 	add    A,   A, LDA

  1080. 

  1081. 	srawi.	r0,  M, 2

  1082. 	mtspr	CTR, r0

  1083. 	ble	LL(35)

  1084. 

  1085. 	LFDU	a1, 1 * SIZE(AO1)

  1086. 	LFDU	b1, 1 * SIZE(X1)

  1087. 	LFDU	a2, 1 * SIZE(AO1)

  1088. 	LFDU	b2, 1 * SIZE(X1)

  1089. 	bdz	LL(33)

  1090. 	.align 5

  1091. 

  1092. LL(32):

  1093. 	FMADD1	y1,  a1,  b1, y1

  1094. 	LFDU	b3, 1 * SIZE(X1)

  1095. 	FMADD2	y2,  a1,  b2, y2

  1096. 	LFDU	b4, 1 * SIZE(X1)

  1097. 

  1098. #ifdef PPCG4

  1099. 	dcbt	AO1, PREA

  1100. #endif

  1101. 

  1102. 	FMADD3	y3,  a2,  b2, y3

  1103. 	LFDU	a1, 1 * SIZE(AO1)

  1104. 	FMADD4	y4,  a2,  b1, y4

  1105. 	LFDU	a2, 1 * SIZE(AO1)

  1106. 

  1107. 	FMADD1	y1,  a1,  b3, y1

  1108. 	LFDU	b1, 1 * SIZE(X1)

  1109. 	FMADD2	y2,  a1,  b4, y2

  1110. 	LFDU	b2, 1 * SIZE(X1)

  1111. 

  1112. #ifdef PPCG4

  1113. 	dcbt	X1, PREA

  1114. #endif

  1115. 

  1116. 	FMADD3	y3,  a2,  b4, y3

  1117. 	LFDU	a1, 1 * SIZE(AO1)

  1118. 	FMADD4	y4,  a2,  b3, y4

  1119. 	LFDU	a2, 1 * SIZE(AO1)

  1120. 

  1121. 	FMADD1	y1,  a1,  b1, y1

  1122. 	LFDU	b3, 1 * SIZE(X1)

  1123. 	FMADD2	y2,  a1,  b2, y2

  1124. 	LFDU	b4, 1 * SIZE(X1)

  1125. 

  1126. #if defined(PPCG4) && defined(DOUBLE)

  1127. 	dcbt	AO1, PREA

  1128. #endif

  1129. 

  1130. 	FMADD3	y3,  a2,  b2, y3

  1131. 	LFDU	a1, 1 * SIZE(AO1)

  1132. 	FMADD4	y4,  a2,  b1, y4

  1133. 	LFDU	a2, 1 * SIZE(AO1)

  1134. 

  1135. 	FMADD1	y1,  a1,  b3, y1

  1136. 	LFDU	b1, 1 * SIZE(X1)

  1137. 	FMADD2	y2,  a1,  b4, y2

  1138. 	LFDU	b2, 1 * SIZE(X1)

  1139. 

  1140. #if defined(PPCG4) && defined(DOUBLE)

  1141. 	dcbt	X1, PREA

  1142. #endif

  1143. 

  1144. 	FMADD3	y3,  a2,  b4, y3

  1145. 	LFDU	a1, 1 * SIZE(AO1)

  1146. 	FMADD4	y4,  a2,  b3, y4

  1147. 	LFDU	a2, 1 * SIZE(AO1)

  1148. 

  1149.  	bdnz	LL(32)

  1150. 	.align 4

  1151. 

  1152. LL(33):

  1153. 	FMADD1	y1,  a1,  b1, y1

  1154. 	LFDU	b3, 1 * SIZE(X1)

  1155. 	FMADD2	y2,  a1,  b2, y2

  1156. 	LFDU	b4, 1 * SIZE(X1)

  1157. 

  1158. 	FMADD3	y3,  a2,  b2, y3

  1159. 	LFDU	a1, 1 * SIZE(AO1)

  1160. 	FMADD4	y4,  a2,  b1, y4

  1161. 	LFDU	a2, 1 * SIZE(AO1)

  1162. 

  1163. 	FMADD1	y1,  a1,  b3, y1

  1164. 	LFDU	b1, 1 * SIZE(X1)

  1165. 	FMADD2	y2,  a1,  b4, y2

  1166. 	LFDU	b2, 1 * SIZE(X1)

  1167. 

  1168. 	FMADD3	y3,  a2,  b4, y3

  1169. 	LFDU	a1, 1 * SIZE(AO1)

  1170. 	FMADD4	y4,  a2,  b3, y4

  1171. 	LFDU	a2, 1 * SIZE(AO1)

  1172. 

  1173. 	FMADD1	y1,  a1,  b1, y1

  1174. 	LFDU	b3, 1 * SIZE(X1)

  1175. 	FMADD2	y2,  a1,  b2, y2

  1176. 	LFDU	b4, 1 * SIZE(X1)

  1177. 

  1178. 	FMADD3	y3,  a2,  b2, y3

  1179. 	LFDU	a1, 1 * SIZE(AO1)

  1180. 	FMADD4	y4,  a2,  b1, y4

  1181. 	LFDU	a2, 1 * SIZE(AO1)

  1182. 

  1183. 	FMADD1	y1,  a1,  b3, y1

  1184. 	FMADD2	y2,  a1,  b4, y2

  1185. 	FMADD3	y3,  a2,  b4, y3

  1186. 	FMADD4	y4,  a2,  b3, y4

  1187. 	.align 4

  1188. 

  1189. LL(35):

  1190. 	andi.	r0, M, 2

  1191. 	ble	LL(37)

  1192. 

  1193. 	LFDU	a1, 1 * SIZE(AO1)

  1194. 	LFDU	b1, 1 * SIZE(X1)

  1195. 	LFDU	a2, 1 * SIZE(AO1)

  1196. 	LFDU	b2, 1 * SIZE(X1)

  1197. 

  1198. 	FMADD1	y1,  a1,  b1, y1

  1199. 	LFDU	b3, 1 * SIZE(X1)

  1200. 	FMADD2	y2,  a1,  b2, y2

  1201. 	LFDU	a3, 1 * SIZE(AO1)

  1202. 	FMADD3	y3,  a2,  b2, y3

  1203. 	LFDU	b4, 1 * SIZE(X1)

  1204. 	FMADD4	y4,  a2,  b1, y4

  1205. 	LFDU	a4, 1 * SIZE(AO1)

  1206. 

  1207. 	FMADD1	y1,  a3,  b3, y1

  1208. 	FMADD2	y2,  a3,  b4, y2

  1209. 	FMADD3	y3,  a4,  b4, y3

  1210. 	FMADD4	y4,  a4,  b3, y4

  1211. 	.align 4

  1212. 

  1213. LL(37):

  1214. 	andi.	r0, M, 1

  1215. 	ble	LL(39)

  1216. 

  1217. 	LFDU	a1, 1 * SIZE(AO1)

  1218. 	LFDU	b1, 1 * SIZE(X1)

  1219. 	LFDU	a2, 1 * SIZE(AO1)

  1220. 	LFDU	b2, 1 * SIZE(X1)

  1221. 

  1222. 	FMADD1	y1,  a1,  b1, y1

  1223. 	FMADD2	y2,  a1,  b2, y2

  1224. 	FMADD3	y3,  a2,  b2, y3

  1225. 	FMADD4	y4,  a2,  b1, y4

  1226. 	.align 4

  1227. 

  1228. LL(39):

  1229. 	LFDUX	b1,  Y, INCY

  1230. 	LFDU	b2,  1 * SIZE(Y)

  1231. 

  1232. 	FADD	y1, y1, y3

  1233. 	FADD	y2, y2, y4

  1234. 

  1235. 	FMADD	b1, alpha_r, y1, b1

  1236. 	FMADDR	b2, alpha_r, y2, b2

  1237. 	FMSUBR	b1, alpha_i, y2, b1

  1238. 	FMADD	b2, alpha_i, y1, b2

  1239. 

  1240. 	STFDUX	b1,  YY, INCY

  1241. 	STFDU	b2,  1 * SIZE(YY)

  1242. 	.align 4

  1243. 

  1244. LL(999):

  1245. 	li	r3, 0

  1246. 

  1247. 	lfd	f14,     0(SP)

  1248. 	lfd	f15,     8(SP)

  1249. 	lfd	f16,    16(SP)

  1250. 	lfd	f17,    24(SP)

  1251. 	lfd	f18,    32(SP)

  1252. 	lfd	f19,    40(SP)

  1253. 	lfd	f20,    48(SP)

  1254. 	lfd	f21,    56(SP)

  1255. 	lfd	f22,    64(SP)

  1256. 	lfd	f23,    72(SP)

  1257. 	lfd	f24,    80(SP)

  1258. 	lfd	f25,    88(SP)

  1259. 	lfd	f26,    96(SP)

  1260. 	lfd	f27,   104(SP)

  1261. 	lfd	f28,   112(SP)

  1262. 	lfd	f29,   120(SP)

  1263. 	lfd	f30,   128(SP)

  1264. 	lfd	f31,   136(SP)

  1265. 

  1266. #ifdef __64BIT__

  1267. 	ld	r14,   144(SP)

  1268. 	ld	r15,   152(SP)

  1269. 	ld	r16,   160(SP)

  1270. 	ld	r17,   168(SP)

  1271. 	ld	r18,   176(SP)

  1272. 	ld	r19,   184(SP)

  1273. 	ld	r20,   192(SP)

  1274. 	ld	r21,   200(SP)

  1275. 	ld	r22,   208(SP)

  1276. #else

  1277. 	lwz	r14,   144(SP)

  1278. 	lwz	r15,   148(SP)

  1279. 	lwz	r16,   152(SP)

  1280. 	lwz	r17,   156(SP)

  1281. 	lwz	r18,   160(SP)

  1282. 	lwz	r19,   164(SP)

  1283. 	lwz	r20,   168(SP)

  1284. 	lwz	r21,   172(SP)

  1285. 	lwz	r22,   176(SP)

  1286. #endif

  1287. 

  1288. 	addi	SP, SP, STACKSIZE

  1289. 

  1290. 	blr

  1291. 

  1292. 	EPILOGUE

  1293. 

  1294. #endif