OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e9a912591
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e9a912591'
${ noResults }
OpenBLAS/kernel/power/gemv_t_ppc440.S

1095 lines
20 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(linux) || defined(__FreeBSD__)

  43. #ifndef __64BIT__

  44. #define M	r3

  45. #define	N	r4

  46. #define A	r6

  47. #define LDA	r7

  48. #define X	r8

  49. #define	INCX	r9

  50. #define	Y	r10

  51. #define	INCY	r5

  52. #else

  53. #define M	r3

  54. #define	N	r4

  55. #define A	r7

  56. #define LDA	r8

  57. #define X	r9

  58. #define	INCX	r10

  59. #define	Y	r5

  60. #define	INCY	r6

  61. #endif

  62. #endif

  63. 

  64. #if defined(_AIX) || defined(__APPLE__)

  65. #if !defined(__64BIT__) && defined(DOUBLE)

  66. #define M	r3

  67. #define	N	r4

  68. #define A	r8

  69. #define LDA	r9

  70. #define X	r10

  71. #define	INCX	r5

  72. #define	Y	r6

  73. #define	INCY	r7

  74. #else

  75. #define M	r3

  76. #define	N	r4

  77. #define A	r7

  78. #define LDA	r8

  79. #define X	r9

  80. #define	INCX	r10

  81. #define	Y	r5

  82. #define	INCY	r6

  83. #endif

  84. #endif

  85. 

  86. #define	BUFFER	r11

  87. #define	XP	r12

  88. #define	AO1	r14

  89. #define	AO2	r15

  90. #define	AO3	r16

  91. #define	AO4	r17

  92. #define	J	r18

  93. #define YY	r19

  94. #define	PREA	r20

  95. #define	PREC	r21

  96. #define	X1	r22

  97. 

  98. 

  99. #if defined(PPC440)

  100. #define PREFETCHSIZE_A  42

  101. #define PREFETCHSIZE_C   7

  102. #endif

  103. 

  104. #if defined(PPCG4)

  105. #define PREFETCHSIZE_A  42

  106. #define PREFETCHSIZE_C   7

  107. #endif

  108. 

  109. #if defined(POWER6)

  110. #define PREFETCHSIZE_A  42

  111. #define PREFETCHSIZE_C   7

  112. #endif

  113. 

  114. #define y01 f0

  115. #define y02 f1

  116. #define y03 f2

  117. #define y04 f3

  118. #define y05 f4

  119. #define y06 f5

  120. #define y07 f6

  121. #define y08 f7

  122. 

  123. #define a1	f8

  124. #define a2	f9

  125. #define a3	f10

  126. #define a4	f11

  127. #define a5	f12

  128. #define a6	f13

  129. #define a7	f14

  130. #define a8	f15

  131. 

  132. #define b1	f16

  133. #define b2	f17

  134. #define b3	f18

  135. #define b4	f19

  136. #define b5	f20

  137. #define b6	f21

  138. #define b7	f22

  139. #define b8	f23

  140. 

  141. #define alpha	f23

  142. 

  143. #ifndef NEEDPARAM

  144. 

  145. #ifndef __64BIT__

  146. #define STACKSIZE 224

  147. #else

  148. #define STACKSIZE 288

  149. #endif

  150. 

  151. #define FZERO     144(SP)

  152. #define ALPHA     152(SP)

  153. 

  154. 	PROLOGUE

  155. 	PROFCODE

  156. 

  157. 	addi	SP, SP,  -STACKSIZE

  158. 	li	r0,   0

  159. 

  160. 	stfd	f14,     0(SP)

  161. 	stfd	f15,     8(SP)

  162. 	stfd	f16,    16(SP)

  163. 	stfd	f17,    24(SP)

  164. 	stfd	f18,    32(SP)

  165. 	stfd	f19,    40(SP)

  166. 	stfd	f20,    48(SP)

  167. 	stfd	f21,    56(SP)

  168. 	stfd	f22,    64(SP)

  169. 	stfd	f23,    72(SP)

  170. 

  171. #ifdef __64BIT__

  172. 	std	r0,    FZERO

  173. 	stfd	f1,    ALPHA

  174. 	std	r14,   160(SP)

  175. 	std	r15,   168(SP)

  176. 	std	r16,   176(SP)

  177. 	std	r17,   184(SP)

  178. 	std	r18,   192(SP)

  179. 	std	r19,   200(SP)

  180. 	std	r20,   208(SP)

  181. 	std	r21,   216(SP)

  182. 	std	r22,   224(SP)

  183. #else

  184. 	stw	r0,    0 + FZERO

  185. 	stw	r0,    4 + FZERO

  186. 	stfd	f1,    ALPHA

  187. 	stw	r14,   160(SP)

  188. 	stw	r15,   164(SP)

  189. 	stw	r16,   168(SP)

  190. 	stw	r17,   172(SP)

  191. 	stw	r18,   176(SP)

  192. 	stw	r19,   180(SP)

  193. 	stw	r20,   184(SP)

  194. 	stw	r21,   188(SP)

  195. 	stw	r22,   192(SP)

  196. #endif

  197. 

  198. #if defined(linux) || defined(__FreeBSD__)

  199. #ifndef __64BIT__

  200. 	lwz	INCY,	 FRAMESLOT(0) + STACKSIZE(SP)

  201. 	lwz	BUFFER,  FRAMESLOT(1) + STACKSIZE(SP)

  202. #else

  203. 	ld	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  204. 	ld	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  205. 	ld	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  206. #endif

  207. #endif

  208. 

  209. #if defined(_AIX) || defined(__APPLE__)

  210. #ifndef __64BIT__

  211. #ifdef DOUBLE

  212. 	lwz	INCX,    FRAMESLOT(0) + STACKSIZE(SP)

  213. 	lwz	Y,       FRAMESLOT(1) + STACKSIZE(SP)

  214. 	lwz	INCY,    FRAMESLOT(2) + STACKSIZE(SP)

  215. 	lwz	BUFFER,  FRAMESLOT(3) + STACKSIZE(SP)

  216. #else

  217. 	lwz	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  218. 	lwz	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  219. 	lwz	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  220. #endif

  221. #else

  222. 	ld	Y,       FRAMESLOT(0) + STACKSIZE(SP)

  223. 	ld	INCY,    FRAMESLOT(1) + STACKSIZE(SP)

  224. 	ld	BUFFER,  FRAMESLOT(2) + STACKSIZE(SP)

  225. #endif

  226. #endif

  227. 

  228. 	slwi	LDA,  LDA,  BASE_SHIFT

  229. 	slwi	INCX, INCX, BASE_SHIFT

  230. 	slwi	INCY, INCY, BASE_SHIFT

  231. 

  232. 	addi	A, A, -SIZE

  233. 	sub	X, X, INCX

  234. 	sub	Y, Y, INCY

  235. 

  236. 	li	PREA, PREFETCHSIZE_A * SIZE

  237. 	li	PREC, PREFETCHSIZE_C * SIZE

  238. 

  239. 	cmpi	cr0, 0, M, 0

  240. 	ble	LL(999)

  241. 	cmpi	cr0, 0, N, 0

  242. 	ble	LL(999)

  243. 

  244. 	mr	XP, X

  245. 

  246. 	cmpi	cr0, 0, INCX, SIZE

  247. 	beq	LL(10)

  248. 

  249. 	addi	XP, BUFFER, -SIZE

  250. 	addi	X1, BUFFER, -SIZE

  251. 

  252. 	srawi.	r0, M, 3

  253. 	mtspr	CTR, r0

  254. 	ble	LL(CopyRemain)

  255. 	.align 4

  256. 

  257. LL(CopyKernel):

  258. 	LFDUX	f0, X, INCX

  259. 	LFDUX	f1, X, INCX

  260. 	LFDUX	f2, X, INCX

  261. 	LFDUX	f3, X, INCX

  262. 	LFDUX	f4, X, INCX

  263. 	LFDUX	f5, X, INCX

  264. 	LFDUX	f6, X, INCX

  265. 	LFDUX	f7, X, INCX

  266. 

  267. 	STFDU	f0,  1 * SIZE(X1)

  268. 	STFDU	f1,  1 * SIZE(X1)

  269. 	STFDU	f2,  1 * SIZE(X1)

  270. 	STFDU	f3,  1 * SIZE(X1)

  271. 	STFDU	f4,  1 * SIZE(X1)

  272. 	STFDU	f5,  1 * SIZE(X1)

  273. 	STFDU	f6,  1 * SIZE(X1)

  274. 	STFDU	f7,  1 * SIZE(X1)

  275. 	bdnz	LL(CopyKernel)

  276. 	.align 4

  277. 

  278. LL(CopyRemain):

  279. 	andi.	r0, M, 7

  280. 	mtspr	CTR, r0

  281. 	ble	LL(10)

  282. 	.align 4

  283. 

  284. LL(CopySub):

  285. 	LFDUX	f0, X,  INCX

  286. 	STFDU	f0,  1 * SIZE(X1)

  287. 	bdnz	LL(CopySub)

  288. 	.align 4

  289. 

  290. LL(10):

  291. 	mr	YY, Y

  292. 

  293. 	srawi.	J, N, 2

  294. 	ble	LL(30)

  295. 	.align 4

  296. 

  297. LL(21):

  298. 	mr     AO1, A

  299. 	add    AO2, A,   LDA

  300. 	add    AO3, AO2, LDA

  301. 	add    AO4, AO3, LDA

  302. 	add    A,   AO4, LDA

  303. 

  304. 	mr     X1, XP

  305. 

  306. 	lfd	 y01, FZERO

  307. 	fmr	 y02, y01

  308. 	fmr	 y03, y01

  309. 	fmr	 y04, y01

  310. 	fmr	 y05, y01

  311. 	fmr	 y06, y01

  312. 	fmr	 y07, y01

  313. 	fmr	 y08, y01

  314. 

  315. 	dcbtst	 Y, PREC

  316. 

  317. 	srawi.	r0,  M, 3

  318. 	mtspr	CTR, r0

  319. 	ble	LL(24)

  320. 

  321. 	LFDU	a1,  1 * SIZE(AO1)

  322. 	LFDU	a2,  1 * SIZE(AO2)

  323. 	LFDU	a3,  1 * SIZE(AO3)

  324. 	LFDU	a4,  1 * SIZE(AO4)

  325. 

  326. 	LFDU	b1,  1 * SIZE(X1)

  327. 	LFDU	b2,  1 * SIZE(X1)

  328. 

  329. 	LFDU	a5,  1 * SIZE(AO1)

  330. 	LFDU	a6,  1 * SIZE(AO2)

  331. 	LFDU	a7,  1 * SIZE(AO3)

  332. 	LFDU	a8,  1 * SIZE(AO4)

  333. 

  334. 	LFDU	b3,  1 * SIZE(X1)

  335. 	LFDU	b4,  1 * SIZE(X1)

  336. 	bdz	LL(23)

  337. 	.align 4

  338. 

  339. LL(22):

  340. #ifdef PPCG4

  341. 	dcbt	X1, PREA

  342. #endif

  343. 

  344. 	FMADD	y01, a1, b1, y01

  345. 	LFDU	a1,  1 * SIZE(AO1)

  346. 	FMADD	y02, a2, b1, y02

  347. 	LFDU	a2,  1 * SIZE(AO2)

  348. 	FMADD	y03, a3, b1, y03

  349. 	LFDU	a3,  1 * SIZE(AO3)

  350. 	FMADD	y04, a4, b1, y04

  351. 	LFDU	a4,  1 * SIZE(AO4)

  352. 

  353. 	LFDU	b1,  1 * SIZE(X1)

  354. #ifdef PPCG4

  355. 	dcbt	AO1, PREA

  356. #endif

  357. 

  358. 	FMADD	y05, a5, b2, y05

  359. 	LFDU	a5,  1 * SIZE(AO1)

  360. 	FMADD	y06, a6, b2, y06

  361. 	LFDU	a6,  1 * SIZE(AO2)

  362. 	FMADD	y07, a7, b2, y07

  363. 	LFDU	a7,  1 * SIZE(AO3)

  364. 	FMADD	y08, a8, b2, y08

  365. 	LFDU	a8,  1 * SIZE(AO4)

  366. 

  367. 	LFDU	b2,  1 * SIZE(X1)

  368. #ifdef PPCG4

  369. 	dcbt	AO2, PREA

  370. #endif

  371. 

  372. 	FMADD	y01, a1, b3, y01

  373. 	LFDU	a1,  1 * SIZE(AO1)

  374. 	FMADD	y02, a2, b3, y02

  375. 	LFDU	a2,  1 * SIZE(AO2)

  376. 	FMADD	y03, a3, b3, y03

  377. 	LFDU	a3,  1 * SIZE(AO3)

  378. 	FMADD	y04, a4, b3, y04

  379. 	LFDU	a4,  1 * SIZE(AO4)

  380. 

  381. 	LFDU	b3,  1 * SIZE(X1)

  382. #ifdef PPCG4

  383. 	dcbt	AO3, PREA

  384. #endif

  385. 

  386. 	FMADD	y05, a5, b4, y05

  387. 	LFDU	a5,  1 * SIZE(AO1)

  388. 	FMADD	y06, a6, b4, y06

  389. 	LFDU	a6,  1 * SIZE(AO2)

  390. 	FMADD	y07, a7, b4, y07

  391. 	LFDU	a7,  1 * SIZE(AO3)

  392. 	FMADD	y08, a8, b4, y08

  393. 	LFDU	a8,  1 * SIZE(AO4)

  394. 

  395. #ifdef PPCG4

  396. 	dcbt	AO4, PREA

  397. #endif

  398. 	LFDU	b4,  1 * SIZE(X1)

  399. #if defined(PPCG4) && defined(DOUBLE)

  400. 	dcbt	X1, PREA

  401. #endif

  402. 

  403. 	FMADD	y01, a1, b1, y01

  404. 	LFDU	a1,  1 * SIZE(AO1)

  405. 	FMADD	y02, a2, b1, y02

  406. 	LFDU	a2,  1 * SIZE(AO2)

  407. 	FMADD	y03, a3, b1, y03

  408. 	LFDU	a3,  1 * SIZE(AO3)

  409. 	FMADD	y04, a4, b1, y04

  410. 	LFDU	a4,  1 * SIZE(AO4)

  411. 

  412. 	LFDU	b1,  1 * SIZE(X1)

  413. #if defined(PPCG4) && defined(DOUBLE)

  414. 	dcbt	AO1, PREA

  415. #endif

  416. 

  417. 	FMADD	y05, a5, b2, y05

  418. 	LFDU	a5,  1 * SIZE(AO1)

  419. 	FMADD	y06, a6, b2, y06

  420. 	LFDU	a6,  1 * SIZE(AO2)

  421. 	FMADD	y07, a7, b2, y07

  422. 	LFDU	a7,  1 * SIZE(AO3)

  423. 	FMADD	y08, a8, b2, y08

  424. 	LFDU	a8,  1 * SIZE(AO4)

  425. 

  426. 	LFDU	b2,  1 * SIZE(X1)

  427. #if defined(PPCG4) && defined(DOUBLE)

  428. 	dcbt	AO2, PREA

  429. #endif

  430. 

  431. 	FMADD	y01, a1, b3, y01

  432. 	LFDU	a1,  1 * SIZE(AO1)

  433. 	FMADD	y02, a2, b3, y02

  434. 	LFDU	a2,  1 * SIZE(AO2)

  435. 	FMADD	y03, a3, b3, y03

  436. 	LFDU	a3,  1 * SIZE(AO3)

  437. 	FMADD	y04, a4, b3, y04

  438. 	LFDU	a4,  1 * SIZE(AO4)

  439. 

  440. 	LFDU	b3,  1 * SIZE(X1)

  441. #if defined(PPCG4) && defined(DOUBLE)

  442. 	dcbt	AO3, PREA

  443. #endif

  444. 

  445. 	FMADD	y05, a5, b4, y05

  446. 	LFDU	a5,  1 * SIZE(AO1)

  447. 	FMADD	y06, a6, b4, y06

  448. 	LFDU	a6,  1 * SIZE(AO2)

  449. 	FMADD	y07, a7, b4, y07

  450. 	LFDU	a7,  1 * SIZE(AO3)

  451. 	FMADD	y08, a8, b4, y08

  452. 	LFDU	a8,  1 * SIZE(AO4)

  453. 

  454. 	LFDU	b4,  1 * SIZE(X1)

  455. #if defined(PPCG4) && defined(DOUBLE)

  456. 	dcbt	AO4, PREA

  457. #endif

  458. 

  459.  	bdnz	LL(22)

  460. 	.align 4

  461. 

  462. LL(23):

  463. 	FMADD	y01, a1, b1, y01

  464. 	LFDU	a1,  1 * SIZE(AO1)

  465. 	FMADD	y02, a2, b1, y02

  466. 	LFDU	a2,  1 * SIZE(AO2)

  467. 	FMADD	y03, a3, b1, y03

  468. 	LFDU	a3,  1 * SIZE(AO3)

  469. 	FMADD	y04, a4, b1, y04

  470. 	LFDU	a4,  1 * SIZE(AO4)

  471. 

  472. 	LFDU	b1,  1 * SIZE(X1)

  473. 

  474. 	FMADD	y05, a5, b2, y05

  475. 	LFDU	a5,  1 * SIZE(AO1)

  476. 	FMADD	y06, a6, b2, y06

  477. 	LFDU	a6,  1 * SIZE(AO2)

  478. 	FMADD	y07, a7, b2, y07

  479. 	LFDU	a7,  1 * SIZE(AO3)

  480. 	FMADD	y08, a8, b2, y08

  481. 	LFDU	a8,  1 * SIZE(AO4)

  482. 

  483. 	LFDU	b2,  1 * SIZE(X1)

  484. 

  485. 	FMADD	y01, a1, b3, y01

  486. 	LFDU	a1,  1 * SIZE(AO1)

  487. 	FMADD	y02, a2, b3, y02

  488. 	LFDU	a2,  1 * SIZE(AO2)

  489. 	FMADD	y03, a3, b3, y03

  490. 	LFDU	a3,  1 * SIZE(AO3)

  491. 	FMADD	y04, a4, b3, y04

  492. 	LFDU	a4,  1 * SIZE(AO4)

  493. 

  494. 	LFDU	b3,  1 * SIZE(X1)

  495. 

  496. 	FMADD	y05, a5, b4, y05

  497. 	LFDU	a5,  1 * SIZE(AO1)

  498. 	FMADD	y06, a6, b4, y06

  499. 	LFDU	a6,  1 * SIZE(AO2)

  500. 	FMADD	y07, a7, b4, y07

  501. 	LFDU	a7,  1 * SIZE(AO3)

  502. 	FMADD	y08, a8, b4, y08

  503. 	LFDU	a8,  1 * SIZE(AO4)

  504. 

  505. 	LFDU	b4,  1 * SIZE(X1)

  506. 

  507. 	FMADD	y01, a1, b1, y01

  508. 	LFDU	a1,  1 * SIZE(AO1)

  509. 	FMADD	y02, a2, b1, y02

  510. 	LFDU	a2,  1 * SIZE(AO2)

  511. 	FMADD	y03, a3, b1, y03

  512. 	LFDU	a3,  1 * SIZE(AO3)

  513. 	FMADD	y04, a4, b1, y04

  514. 	LFDU	a4,  1 * SIZE(AO4)

  515. 

  516. 	FMADD	y05, a5, b2, y05

  517. 	LFDU	a5,  1 * SIZE(AO1)

  518. 	FMADD	y06, a6, b2, y06

  519. 	LFDU	a6,  1 * SIZE(AO2)

  520. 	FMADD	y07, a7, b2, y07

  521. 	LFDU	a7,  1 * SIZE(AO3)

  522. 	FMADD	y08, a8, b2, y08

  523. 	LFDU	a8,  1 * SIZE(AO4)

  524. 

  525. 	FMADD	y01, a1, b3, y01

  526. 	FMADD	y02, a2, b3, y02

  527. 	FMADD	y03, a3, b3, y03

  528. 	FMADD	y04, a4, b3, y04

  529. 

  530. 	FMADD	y05, a5, b4, y05

  531. 	FMADD	y06, a6, b4, y06

  532. 	FMADD	y07, a7, b4, y07

  533. 	FMADD	y08, a8, b4, y08

  534. 	.align 4

  535. 

  536. LL(24):

  537. 	andi.	r0, M,  7

  538. 	ble	LL(28)

  539. 

  540. 	andi.	r0, M,  4

  541. 	ble	LL(26)

  542. 

  543. 	LFDU	a1,  1 * SIZE(AO1)

  544. 	LFDU	a2,  1 * SIZE(AO2)

  545. 	LFDU	b1,  1 * SIZE(X1)

  546. 	LFDU	a3,  1 * SIZE(AO3)

  547. 	LFDU	a4,  1 * SIZE(AO4)

  548. 	LFDU	b2,  1 * SIZE(X1)

  549. 

  550. 	FMADD	y01, a1, b1, y01

  551. 	LFDU	a5,  1 * SIZE(AO1)

  552. 	FMADD	y02, a2, b1, y02

  553. 	LFDU	a6,  1 * SIZE(AO2)

  554. 	FMADD	y03, a3, b1, y03

  555. 	LFDU	a7,  1 * SIZE(AO3)

  556. 	FMADD	y04, a4, b1, y04

  557. 	LFDU	a8,  1 * SIZE(AO4)

  558. 

  559. 	LFDU	b3,  1 * SIZE(X1)

  560. 

  561. 	FMADD	y05, a5, b2, y05

  562. 	LFDU	a1,  1 * SIZE(AO1)

  563. 	FMADD	y06, a6, b2, y06

  564. 	LFDU	a2,  1 * SIZE(AO2)

  565. 	FMADD	y07, a7, b2, y07

  566. 	LFDU	a3,  1 * SIZE(AO3)

  567. 	FMADD	y08, a8, b2, y08

  568. 	LFDU	a4,  1 * SIZE(AO4)

  569. 

  570. 	LFDU	b4,  1 * SIZE(X1)

  571. 

  572. 	FMADD	y01, a1, b3, y01

  573. 	LFDU	a5,  1 * SIZE(AO1)

  574. 	FMADD	y02, a2, b3, y02

  575. 	LFDU	a6,  1 * SIZE(AO2)

  576. 	FMADD	y03, a3, b3, y03

  577. 	LFDU	a7,  1 * SIZE(AO3)

  578. 	FMADD	y04, a4, b3, y04

  579. 	LFDU	a8,  1 * SIZE(AO4)

  580. 

  581. 	FMADD	y05, a5, b4, y05

  582. 	FMADD	y06, a6, b4, y06

  583. 	FMADD	y07, a7, b4, y07

  584. 	FMADD	y08, a8, b4, y08

  585. 	.align 4

  586. 

  587. LL(26):

  588. 	andi.	r0, M,  2

  589. 	ble	LL(27)

  590. 

  591. 	LFDU	b1,  1 * SIZE(X1)

  592. 	LFDU	a1,  1 * SIZE(AO1)

  593. 	LFDU	a2,  1 * SIZE(AO2)

  594. 	LFDU	a3,  1 * SIZE(AO3)

  595. 	LFDU	a4,  1 * SIZE(AO4)

  596. 	LFDU	b2,  1 * SIZE(X1)

  597. 

  598. 	FMADD	y01, a1, b1, y01

  599. 	LFDU	a5,  1 * SIZE(AO1)

  600. 	FMADD	y02, a2, b1, y02

  601. 	LFDU	a6,  1 * SIZE(AO2)

  602. 	FMADD	y03, a3, b1, y03

  603. 	LFDU	a7,  1 * SIZE(AO3)

  604. 	FMADD	y04, a4, b1, y04

  605. 	LFDU	a8,  1 * SIZE(AO4)

  606. 

  607. 	FMADD	y05, a5, b2, y05

  608. 	FMADD	y06, a6, b2, y06

  609. 	FMADD	y07, a7, b2, y07

  610. 	FMADD	y08, a8, b2, y08

  611. 	.align 4

  612. 

  613. LL(27):

  614. 	andi.	r0, M,  1

  615. 	ble	LL(28)

  616. 

  617. 	LFDU	a1,  1 * SIZE(AO1)

  618. 	LFDU	b1,  1 * SIZE(X1)

  619. 

  620. 	LFDU	a2,  1 * SIZE(AO2)

  621. 	LFDU	a3,  1 * SIZE(AO3)

  622. 	LFDU	a4,  1 * SIZE(AO4)

  623. 

  624. 	FMADD	y01, a1, b1, y01

  625. 	FMADD	y02, a2, b1, y02

  626. 	FMADD	y03, a3, b1, y03

  627. 	FMADD	y04, a4, b1, y04

  628. 	.align 4

  629. 

  630. LL(28):

  631. 	lfd	alpha, ALPHA

  632. 

  633. 	LFDUX	a1, Y, INCY

  634. 	LFDUX	a2, Y, INCY

  635. 	LFDUX	a3, Y, INCY

  636. 	LFDUX	a4, Y, INCY

  637. 

  638. 	FADD	y01, y05, y01

  639. 	FADD	y02, y06, y02

  640. 	FADD	y03, y07, y03

  641. 	FADD	y04, y08, y04

  642. 

  643. 	FMADD	a1, alpha, f0, a1

  644. 	FMADD	a2, alpha, f1, a2

  645. 	FMADD	a3, alpha, f2, a3

  646. 	FMADD	a4, alpha, f3, a4

  647. 

  648. 	STFDUX	a1, YY, INCY

  649. 	addi	J, J, -1

  650. 	STFDUX	a2, YY, INCY

  651. 	cmpi	cr0, 0, J, 0

  652. 	STFDUX	a3, YY, INCY

  653. 	STFDUX	a4, YY, INCY

  654. 	bgt	LL(21)

  655. 	.align 4

  656. 

  657. LL(30):

  658. 	andi.	J, N, 2

  659. 	ble	LL(40)

  660. 

  661. 	mr     AO1, A

  662. 	add    AO2, A,   LDA

  663. 	add    A,   AO2, LDA

  664. 

  665. 	mr     X1, XP

  666. 

  667. 	lfd	 y01, FZERO

  668. 	fmr	 y02, y01

  669. 	fmr	 y03, y01

  670. 	fmr	 y04, y01

  671. 

  672. 	srawi.	r0,  M, 3

  673. 	mtspr	CTR, r0

  674. 	ble	LL(34)

  675. 

  676. 	LFDU	a1,  1 * SIZE(AO1)

  677. 	LFDU	a2,  1 * SIZE(AO2)

  678. 	LFDU	b1,  1 * SIZE(X1)

  679. 	LFDU	b2,  1 * SIZE(X1)

  680. 

  681. 	LFDU	a5,  1 * SIZE(AO1)

  682. 	LFDU	a6,  1 * SIZE(AO2)

  683. 	LFDU	b3,  1 * SIZE(X1)

  684. 	LFDU	b4,  1 * SIZE(X1)

  685. 	bdz	LL(33)

  686. 	.align 4

  687. 

  688. LL(32):

  689. #ifdef PPCG4

  690. 	dcbt	X1, PREA

  691. #endif

  692. 

  693. 	FMADD	y01, a1, b1, y01

  694. 	LFDU	a1,  1 * SIZE(AO1)

  695. 	FMADD	y02, a2, b1, y02

  696. 	LFDU	a2,  1 * SIZE(AO2)

  697. 

  698. 	LFDU	b1,  1 * SIZE(X1)

  699. #ifdef PPCG4

  700. 	dcbt	AO1, PREA

  701. #endif

  702. 

  703. 	FMADD	y03, a5, b2, y03

  704. 	LFDU	a5,  1 * SIZE(AO1)

  705. 	FMADD	y04, a6, b2, y04

  706. 	LFDU	a6,  1 * SIZE(AO2)

  707. 

  708. 	LFDU	b2,  1 * SIZE(X1)

  709. 

  710. 	FMADD	y01, a1, b3, y01

  711. 	LFDU	a1,  1 * SIZE(AO1)

  712. 	FMADD	y02, a2, b3, y02

  713. 	LFDU	a2,  1 * SIZE(AO2)

  714. 

  715. 	LFDU	b3,  1 * SIZE(X1)

  716. #ifdef PPCG4

  717. 	dcbt	AO2, PREA

  718. #endif

  719. 

  720. 	FMADD	y03, a5, b4, y03

  721. 	LFDU	a5,  1 * SIZE(AO1)

  722. 	FMADD	y04, a6, b4, y04

  723. 	LFDU	a6,  1 * SIZE(AO2)

  724. 

  725. 	LFDU	b4,  1 * SIZE(X1)

  726. 

  727. 	FMADD	y01, a1, b1, y01

  728. 	LFDU	a1,  1 * SIZE(AO1)

  729. 	FMADD	y02, a2, b1, y02

  730. 	LFDU	a2,  1 * SIZE(AO2)

  731. 

  732. #if defined(PPCG4) && defined(DOUBLE)

  733. 	dcbt	X1, PREA

  734. #endif

  735. 	LFDU	b1,  1 * SIZE(X1)

  736. #if defined(PPCG4) && defined(DOUBLE)

  737. 	dcbt	AO1, PREA

  738. #endif

  739. 

  740. 	FMADD	y03, a5, b2, y03

  741. 	LFDU	a5,  1 * SIZE(AO1)

  742. 	FMADD	y04, a6, b2, y04

  743. 	LFDU	a6,  1 * SIZE(AO2)

  744. 

  745. 	LFDU	b2,  1 * SIZE(X1)

  746. 

  747. 	FMADD	y01, a1, b3, y01

  748. 	LFDU	a1,  1 * SIZE(AO1)

  749. 	FMADD	y02, a2, b3, y02

  750. 	LFDU	a2,  1 * SIZE(AO2)

  751. 

  752. 	LFDU	b3,  1 * SIZE(X1)

  753. #if defined(PPCG4) && defined(DOUBLE)

  754. 	dcbt	AO2, PREA

  755. #endif

  756. 

  757. 	FMADD	y03, a5, b4, y03

  758. 	LFDU	a5,  1 * SIZE(AO1)

  759. 	FMADD	y04, a6, b4, y04

  760. 	LFDU	a6,  1 * SIZE(AO2)

  761. 

  762. 	LFDU	b4,  1 * SIZE(X1)

  763.  	bdnz	LL(32)

  764. 	.align 4

  765. 

  766. LL(33):

  767. 	FMADD	y01, a1, b1, y01

  768. 	LFDU	a1,  1 * SIZE(AO1)

  769. 	FMADD	y02, a2, b1, y02

  770. 	LFDU	a2,  1 * SIZE(AO2)

  771. 

  772. 	LFDU	b1,  1 * SIZE(X1)

  773. 

  774. 	FMADD	y03, a5, b2, y03

  775. 	LFDU	a5,  1 * SIZE(AO1)

  776. 	FMADD	y04, a6, b2, y04

  777. 	LFDU	a6,  1 * SIZE(AO2)

  778. 

  779. 	LFDU	b2,  1 * SIZE(X1)

  780. 

  781. 	FMADD	y01, a1, b3, y01

  782. 	LFDU	a1,  1 * SIZE(AO1)

  783. 	FMADD	y02, a2, b3, y02

  784. 	LFDU	a2,  1 * SIZE(AO2)

  785. 

  786. 	LFDU	b3,  1 * SIZE(X1)

  787. 

  788. 	FMADD	y03, a5, b4, y03

  789. 	LFDU	a5,  1 * SIZE(AO1)

  790. 	FMADD	y04, a6, b4, y04

  791. 	LFDU	a6,  1 * SIZE(AO2)

  792. 

  793. 	LFDU	b4,  1 * SIZE(X1)

  794. 

  795. 	FMADD	y01, a1, b1, y01

  796. 	LFDU	a1,  1 * SIZE(AO1)

  797. 	FMADD	y02, a2, b1, y02

  798. 	LFDU	a2,  1 * SIZE(AO2)

  799. 

  800. 	FMADD	y03, a5, b2, y03

  801. 	LFDU	a5,  1 * SIZE(AO1)

  802. 	FMADD	y04, a6, b2, y04

  803. 	LFDU	a6,  1 * SIZE(AO2)

  804. 

  805. 	FMADD	y01, a1, b3, y01

  806. 	FMADD	y02, a2, b3, y02

  807. 

  808. 	FMADD	y03, a5, b4, y03

  809. 	FMADD	y04, a6, b4, y04

  810. 	.align 4

  811. 

  812. LL(34):

  813. 	andi.	r0, M,  7

  814. 	ble	LL(38)

  815. 

  816. 	andi.	r0, M,  4

  817. 	ble	LL(36)

  818. 

  819. 	LFDU	a1,  1 * SIZE(AO1)

  820. 	LFDU	a2,  1 * SIZE(AO2)

  821. 	LFDU	b1,  1 * SIZE(X1)

  822. 

  823. 	LFDU	b2,  1 * SIZE(X1)

  824. 	FMADD	y01, a1, b1, y01

  825. 	LFDU	a5,  1 * SIZE(AO1)

  826. 	FMADD	y02, a2, b1, y02

  827. 	LFDU	a6,  1 * SIZE(AO2)

  828. 

  829. 	LFDU	b3,  1 * SIZE(X1)

  830. 	FMADD	y03, a5, b2, y03

  831. 	LFDU	a1,  1 * SIZE(AO1)

  832. 	FMADD	y04, a6, b2, y04

  833. 	LFDU	a2,  1 * SIZE(AO2)

  834. 

  835. 	LFDU	b4,  1 * SIZE(X1)

  836. 	FMADD	y01, a1, b3, y01

  837. 	LFDU	a5,  1 * SIZE(AO1)

  838. 	FMADD	y02, a2, b3, y02

  839. 	LFDU	a6,  1 * SIZE(AO2)

  840. 

  841. 	FMADD	y03, a5, b4, y03

  842. 	FMADD	y04, a6, b4, y04

  843. 	.align 4

  844. 

  845. LL(36):

  846. 	andi.	r0, M,  2

  847. 	ble	LL(37)

  848. 

  849. 	LFDU	b1,  1 * SIZE(X1)

  850. 	LFDU	a1,  1 * SIZE(AO1)

  851. 	LFDU	a2,  1 * SIZE(AO2)

  852. 	LFDU	b2,  1 * SIZE(X1)

  853. 	LFDU	a3,  1 * SIZE(AO1)

  854. 	LFDU	a4,  1 * SIZE(AO2)

  855. 

  856. 	FMADD	y01, a1, b1, y01

  857. 	FMADD	y02, a2, b1, y02

  858. 	FMADD	y03, a3, b2, y03

  859. 	FMADD	y04, a4, b2, y04

  860. 	.align 4

  861. 

  862. LL(37):

  863. 	andi.	r0, M,  1

  864. 	ble	LL(38)

  865. 

  866. 	LFDU	a1,  1 * SIZE(AO1)

  867. 	LFDU	b1,  1 * SIZE(X1)

  868. 	LFDU	a2,  1 * SIZE(AO2)

  869. 

  870. 	FMADD	y01, a1, b1, y01

  871. 	FMADD	y02, a2, b1, y02

  872. 	.align 4

  873. 

  874. LL(38):

  875. 	lfd	alpha, ALPHA

  876. 

  877. 	LFDUX	a1, Y, INCY

  878. 	LFDUX	a2, Y, INCY

  879. 

  880. 	FADD	y01, y03, y01

  881. 	FADD	y02, y04, y02

  882. 

  883. 	FMADD	a1, alpha, f0, a1

  884. 	FMADD	a2, alpha, f1, a2

  885. 

  886. 	STFDUX	a1, YY, INCY

  887. 	STFDUX	a2, YY, INCY

  888. 	.align 4

  889. 

  890. LL(40):

  891. 	andi.	J, N, 1

  892. 	ble	LL(999)

  893. 

  894. 	mr     AO1, A

  895. 	add    A,   A, LDA

  896. 

  897. 	mr     X1, XP

  898. 

  899. 	lfd	 y01, FZERO

  900. 	fmr	 y02, y01

  901. 

  902. 	srawi.	r0,  M, 3

  903. 	mtspr	CTR, r0

  904. 	ble	LL(44)

  905. 

  906. 	LFDU	a1,  1 * SIZE(AO1)

  907. 	LFDU	a2,  1 * SIZE(AO1)

  908. 	LFDU	a3,  1 * SIZE(AO1)

  909. 	LFDU	a4,  1 * SIZE(AO1)

  910. 

  911. 	LFDU	b1,  1 * SIZE(X1)

  912. 	LFDU	b2,  1 * SIZE(X1)

  913. 	LFDU	b3,  1 * SIZE(X1)

  914. 	LFDU	b4,  1 * SIZE(X1)

  915. 	bdz	LL(43)

  916. 	.align 4

  917. 

  918. LL(42):

  919. 	FMADD	y01, a1, b1, y01

  920. 	LFDU	a1,  1 * SIZE(AO1)

  921. 	LFDU	b1,  1 * SIZE(X1)

  922. 

  923. #ifdef PPCG4

  924. 	dcbt	X1, PREA

  925. #endif

  926. 

  927. 	FMADD	y02, a2, b2, y02

  928. 	LFDU	a2,  1 * SIZE(AO1)

  929. 	LFDU	b2,  1 * SIZE(X1)

  930. 

  931. #ifdef PPCG4

  932. 	dcbt	AO1, PREA

  933. #endif

  934. 

  935. 	FMADD	y01, a3, b3, y01

  936. 	LFDU	a3,  1 * SIZE(AO1)

  937. 	LFDU	b3,  1 * SIZE(X1)

  938. 

  939. 	FMADD	y02, a4, b4, y02

  940.  	LFDU	a4,  1 * SIZE(AO1)

  941. 	LFDU	b4,  1 * SIZE(X1)

  942. 

  943. 	FMADD	y01, a1, b1, y01

  944. 	LFDU	a1,  1 * SIZE(AO1)

  945. 	LFDU	b1,  1 * SIZE(X1)

  946. 

  947. 	FMADD	y02, a2, b2, y02

  948. 	LFDU	a2,  1 * SIZE(AO1)

  949. 	LFDU	b2,  1 * SIZE(X1)

  950. 

  951. #if defined(PPCG4) && defined(DOUBLE)

  952. 	dcbt	AO1, PREA

  953. #endif

  954. 

  955. 	FMADD	y01, a3, b3, y01

  956. 	LFDU	a3,  1 * SIZE(AO1)

  957. 	LFDU	b3,  1 * SIZE(X1)

  958. 

  959. #if defined(PPCG4) && defined(DOUBLE)

  960. 	dcbt	X1, PREA

  961. #endif

  962. 

  963. 	FMADD	y02, a4, b4, y02

  964.  	LFDU	a4,  1 * SIZE(AO1)

  965. 	LFDU	b4,  1 * SIZE(X1)

  966. 

  967.  	bdnz	LL(42)

  968. 	.align 4

  969. 

  970. LL(43):

  971. 	FMADD	y01, a1, b1, y01

  972. 	LFDU	a1,  1 * SIZE(AO1)

  973. 	LFDU	b1,  1 * SIZE(X1)

  974. 

  975. 	FMADD	y02, a2, b2, y02

  976. 	LFDU	a2,  1 * SIZE(AO1)

  977. 	LFDU	b2,  1 * SIZE(X1)

  978. 

  979. 	FMADD	y01, a3, b3, y01

  980. 	LFDU	a3,  1 * SIZE(AO1)

  981. 	LFDU	b3,  1 * SIZE(X1)

  982. 

  983. 	FMADD	y02, a4, b4, y02

  984.  	LFDU	a4,  1 * SIZE(AO1)

  985. 	LFDU	b4,  1 * SIZE(X1)

  986. 

  987. 	FMADD	y01, a1, b1, y01

  988. 	FMADD	y02, a2, b2, y02

  989. 	FMADD	y01, a3, b3, y01

  990. 	FMADD	y02, a4, b4, y02

  991. 	.align 4

  992. 

  993. LL(44):

  994. 	andi.	r0, M,  7

  995. 	ble	LL(48)

  996. 

  997. 	andi.	r0, M,  4

  998. 	ble	LL(46)

  999. 

  1000. 	LFDU	a1,  1 * SIZE(AO1)

  1001. 	LFDU	b1,  1 * SIZE(X1)

  1002. 	LFDU	a2,  1 * SIZE(AO1)

  1003. 	LFDU	b2,  1 * SIZE(X1)

  1004. 

  1005. 	FMADD	y01, a1, b1, y01

  1006. 	LFDU	a3,  1 * SIZE(AO1)

  1007. 	LFDU	b3,  1 * SIZE(X1)

  1008. 

  1009. 	FMADD	y02, a2, b2, y02

  1010. 	LFDU	a4,  1 * SIZE(AO1)

  1011. 	LFDU	b4,  1 * SIZE(X1)

  1012. 

  1013. 	FMADD	y01, a3, b3, y01

  1014. 	FMADD	y02, a4, b4, y02

  1015. 	.align 4

  1016. 

  1017. LL(46):

  1018. 	andi.	r0, M,  2

  1019. 	ble	LL(47)

  1020. 

  1021. 	LFDU	b1,  1 * SIZE(X1)

  1022. 	LFDU	a1,  1 * SIZE(AO1)

  1023. 	LFDU	b2,  1 * SIZE(X1)

  1024. 	LFDU	a2,  1 * SIZE(AO1)

  1025. 

  1026. 	FMADD	y01, a1, b1, y01

  1027. 	FMADD	y02, a2, b2, y02

  1028. 	.align 4

  1029. 

  1030. LL(47):

  1031. 	andi.	r0, M,  1

  1032. 	ble	LL(48)

  1033. 

  1034. 	LFDU	a1,  1 * SIZE(AO1)

  1035. 	LFDU	b1,  1 * SIZE(X1)

  1036. 

  1037. 	FMADD	y01, a1, b1, y01

  1038. 	.align 4

  1039. 

  1040. LL(48):

  1041. 	lfd	alpha, ALPHA

  1042. 

  1043. 	LFDUX	a1, Y, INCY

  1044. 

  1045. 	FADD	y01, y02, y01

  1046. 

  1047. 	FMADD	a1, alpha, f0, a1

  1048. 

  1049. 	STFDUX	a1, YY, INCY

  1050. 	.align 4

  1051. 

  1052. LL(999):

  1053. 	li	r3, 0

  1054. 

  1055. 	lfd	f14,     0(SP)

  1056. 	lfd	f15,     8(SP)

  1057. 	lfd	f16,    16(SP)

  1058. 	lfd	f17,    24(SP)

  1059. 	lfd	f18,    32(SP)

  1060. 	lfd	f19,    40(SP)

  1061. 	lfd	f20,    48(SP)

  1062. 	lfd	f21,    56(SP)

  1063. 	lfd	f22,    64(SP)

  1064. 	lfd	f23,    72(SP)

  1065. 

  1066. #ifdef __64BIT__

  1067. 	ld	r14,   160(SP)

  1068. 	ld	r15,   168(SP)

  1069. 	ld	r16,   176(SP)

  1070. 	ld	r17,   184(SP)

  1071. 	ld	r18,   192(SP)

  1072. 	ld	r19,   200(SP)

  1073. 	ld	r20,   208(SP)

  1074. 	ld	r21,   216(SP)

  1075. 	ld	r22,   224(SP)

  1076. #else

  1077. 	lwz	r14,   160(SP)

  1078. 	lwz	r15,   164(SP)

  1079. 	lwz	r16,   168(SP)

  1080. 	lwz	r17,   172(SP)

  1081. 	lwz	r18,   176(SP)

  1082. 	lwz	r19,   180(SP)

  1083. 	lwz	r20,   184(SP)

  1084. 	lwz	r21,   188(SP)

  1085. 	lwz	r22,   192(SP)

  1086. #endif

  1087. 

  1088. 	addi	SP, SP, STACKSIZE

  1089. 

  1090. 	blr

  1091. 

  1092. 	EPILOGUE

  1093. 

  1094. #endif