OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5a8447e97e
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5a8447e97e'
${ noResults }
OpenBLAS/kernel/power/ztrsm_kernel_LT.S

2289 lines
40 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef __64BIT__

  43. #define LOAD	lwz

  44. #else

  45. #define LOAD	ld

  46. #endif

  47. 

  48. #ifdef __64BIT__

  49. #define STACKSIZE 320

  50. #define ALPHA_R 296(SP)

  51. #define ALPHA_I 304(SP)

  52. #define FZERO	312(SP)

  53. #else

  54. #define STACKSIZE 256

  55. #define ALPHA_R 224(SP)

  56. #define ALPHA_I 232(SP)

  57. #define FZERO	240(SP)

  58. #endif

  59. 

  60. #define	M	r3

  61. #define	N	r4

  62. #define	K	r5

  63. 

  64. #ifdef linux

  65. #ifndef __64BIT__

  66. #define A	r6

  67. #define	B	r7

  68. #define	C	r8

  69. #define	LDC	r9

  70. #define OFFSET	r10

  71. #else

  72. #define A	r8

  73. #define	B	r9

  74. #define	C	r10

  75. #define	LDC	r6

  76. #define OFFSET	r7

  77. #endif

  78. #endif

  79. 

  80. #if defined(_AIX) || defined(__APPLE__)

  81. #if !defined(__64BIT__) && defined(DOUBLE)

  82. #define A	r10

  83. #define	B	r6

  84. #define	C	r7

  85. #define	LDC	r8

  86. #define OFFSET	r9

  87. #else

  88. #define A	r8

  89. #define	B	r9

  90. #define	C	r10

  91. #define	LDC	r6

  92. #define OFFSET	r7

  93. #endif

  94. #endif

  95. 

  96. #define AORIG	r21

  97. #define TEMP	r22

  98. #define KK	r23

  99. #define	I	r24

  100. #define J	r25

  101. #define AO	r26

  102. #define	BO	r27

  103. #define	CO1	r28

  104. #define CO2	r29

  105. 

  106. #define PREA	r30

  107. #define PREC	r31

  108. #define PREB	PREA

  109. 

  110. #ifndef NEEDPARAM

  111. 

  112. 	PROLOGUE

  113. 	PROFCODE

  114. 

  115. 	addi	SP, SP, -STACKSIZE

  116. 	li	r0, 0

  117. 

  118. 	stfd	f14,    0(SP)

  119. 	stfd	f15,    8(SP)

  120. 	stfd	f16,   16(SP)

  121. 	stfd	f17,   24(SP)

  122. 

  123. 	stfd	f18,   32(SP)

  124. 	stfd	f19,   40(SP)

  125. 	stfd	f20,   48(SP)

  126. 	stfd	f21,   56(SP)

  127. 

  128. 	stfd	f22,   64(SP)

  129. 	stfd	f23,   72(SP)

  130. 	stfd	f24,   80(SP)

  131. 	stfd	f25,   88(SP)

  132. 

  133. 	stfd	f26,   96(SP)

  134. 	stfd	f27,  104(SP)

  135. 	stfd	f28,  112(SP)

  136. 	stfd	f29,  120(SP)

  137. 

  138. 	stfd	f30,  128(SP)

  139. 	stfd	f31,  136(SP)

  140. 

  141. #ifdef __64BIT__

  142. 	std	r31,  144(SP)

  143. 	std	r30,  152(SP)

  144. 	std	r29,  160(SP)

  145. 	std	r28,  168(SP)

  146. 	std	r27,  176(SP)

  147. 	std	r26,  184(SP)

  148. 	std	r25,  192(SP)

  149. 	std	r24,  200(SP)

  150. 	std	r23,  208(SP)

  151. 	std	r22,  216(SP)

  152. 	std	r21,  224(SP)

  153. #else

  154. 	stw	r31,  144(SP)

  155. 	stw	r30,  148(SP)

  156. 	stw	r29,  152(SP)

  157. 	stw	r28,  156(SP)

  158. 	stw	r27,  160(SP)

  159. 	stw	r26,  164(SP)

  160. 	stw	r25,  168(SP)

  161. 	stw	r24,  172(SP)

  162. 	stw	r23,  176(SP)

  163. 	stw	r22,  180(SP)

  164. 	stw	r21,  184(SP)

  165. #endif

  166. 

  167. 	stw	r0,  FZERO

  168. 

  169. #ifdef linux

  170. #ifdef __64BIT__

  171. 	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  172. #endif

  173. #endif

  174. 

  175. #if defined(_AIX) || defined(__APPLE__)

  176. #ifdef __64BIT__

  177. 	ld	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  178. #else

  179. #ifdef DOUBLE

  180. 	lwz	B,   FRAMESLOT(0) + STACKSIZE(SP)

  181. 	lwz	C,   FRAMESLOT(1) + STACKSIZE(SP)

  182. 	lwz	LDC, FRAMESLOT(2) + STACKSIZE(SP)

  183. #else

  184. 	lwz	LDC, FRAMESLOT(0) + STACKSIZE(SP)

  185. #endif

  186. #endif

  187. #endif

  188. 

  189. #if defined(linux) && defined(__64BIT__)

  190. 	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  191. #endif

  192. 

  193. #if defined(_AIX) || defined(__APPLE__)

  194. #ifdef __64BIT__

  195. 	ld	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  196. #else

  197. #ifdef DOUBLE

  198. 	lwz	OFFSET,  FRAMESLOT(3) + STACKSIZE(SP)

  199. #else

  200. 	lwz	OFFSET,  FRAMESLOT(1) + STACKSIZE(SP)

  201. #endif

  202. #endif

  203. #endif

  204. 

  205. 	slwi	LDC, LDC, ZBASE_SHIFT

  206. 

  207. #ifdef LN

  208. 	mullw	r0, M, K

  209. 	slwi	r0, r0, ZBASE_SHIFT

  210. 	add	A, A, r0

  211. 

  212. 	slwi	r0, M, ZBASE_SHIFT

  213. 	add	C, C, r0

  214. #endif

  215. 

  216. #ifdef RN

  217. 	neg	KK, OFFSET

  218. #endif

  219. 

  220. #ifdef RT

  221. 	mullw	r0, N, K

  222. 	slwi	r0, r0, ZBASE_SHIFT

  223. 	add	B, B, r0

  224. 

  225. 	mullw	r0, N, LDC

  226. 	add	C, C, r0

  227. 

  228. 	sub	KK, N, OFFSET

  229. #endif

  230. 

  231. 	cmpwi	cr0, M, 0

  232. 	ble	LL(999)

  233. 	cmpwi	cr0, N, 0

  234. 	ble	LL(999)

  235. 	cmpwi	cr0, K, 0

  236. 	ble	LL(999)

  237. 

  238. #ifndef PREFETCHTEST

  239. #ifdef PPC970

  240. 	li	PREC,   4 * SIZE

  241. #endif

  242. #ifdef POWER4

  243. 	li	PREC,   4 * SIZE   /* is 12 best? */

  244. #endif

  245. #ifdef POWER5

  246. 	li	PREC,   4 * SIZE   /* is 12 best? */

  247. #endif

  248. #else

  249. 

  250. #ifdef linux

  251. #ifndef __64BIT__

  252. 	lwz	PREA,  FRAMESLOT(2) + STACKSIZE(SP)

  253. 	lwz	PREC,  FRAMESLOT(3) + STACKSIZE(SP)

  254. #else

  255. 	ld	PREA,  FRAMESLOT(3) + STACKSIZE(SP)

  256. 	ld	PREC,  FRAMESLOT(4) + STACKSIZE(SP)

  257. #endif

  258. #endif

  259. 

  260. #if defined(_AIX) || defined(__APPLE__)

  261. #ifdef __64BIT__

  262. 	ld	PREA,  FRAMESLOT(3) + STACKSIZE(SP)

  263. 	ld	PREC,  FRAMESLOT(4) + STACKSIZE(SP)

  264. #else

  265. #ifdef DOUBLE

  266. 	lwz	PREA,  FRAMESLOT(4) + STACKSIZE(SP)

  267. 	lwz	PREC,  FRAMESLOT(5) + STACKSIZE(SP)

  268. #else

  269. 	lwz	PREA,  FRAMESLOT(3) + STACKSIZE(SP)

  270. 	lwz	PREC,  FRAMESLOT(4) + STACKSIZE(SP)

  271. #endif

  272. #endif

  273. #endif

  274. 

  275. #endif

  276. 

  277. #ifndef PREFETCHTEST

  278. #ifdef PPC970

  279. #ifdef ALLOC_HUGETLB

  280. 	li	PREA,   (16 *  5 * SIZE + 16)

  281. #else

  282. 	li	PREA,   (16 *  9 * SIZE + 16)

  283. #endif

  284. #endif

  285. #ifdef POWER4

  286. #ifdef ALLOC_HUGETLB

  287. 	li	PREA,   (16 *  1 * SIZE + 16)

  288. #else

  289. 	li	PREA,   (16 *  2 * SIZE + 16)

  290. #endif

  291. #endif

  292. #ifdef POWER5

  293. #ifdef ALLOC_HUGETLB

  294. 	li	PREA,   (16 *  7 * SIZE | 1)

  295. 	li	PREB,   (16 *  7 * SIZE | 3)

  296. #else

  297. 	li	PREA,   (16 * 12 * SIZE | 1)

  298. 	li	PREB,   (16 *  6 * SIZE | 3)

  299. #endif

  300. #endif

  301. #endif

  302. 

  303. 	srawi.	J, N,  1

  304. 	ble	LL(30)

  305. 	.align 4

  306. 

  307. LL(10):

  308. #ifdef RT

  309. 	slwi	r0, K, 1 + ZBASE_SHIFT

  310. 	sub	B, B, r0

  311. 

  312. 	slwi	r0, LDC, 1

  313. 	sub	C, C, r0

  314. #endif

  315. 

  316. 	mr	CO1, C

  317. 	add	CO2, C,  LDC

  318. 

  319. #ifdef LN

  320. 	add	KK, M, OFFSET

  321. #endif

  322. 

  323. #ifdef LT

  324. 	mr	KK, OFFSET

  325. #endif

  326. 

  327. 	srawi.	I, M,  1

  328. #if defined(LN) || defined(RT)

  329. 	mr	AORIG, A

  330. #else

  331. 	mr	AO, A

  332. #endif

  333. #ifndef RT

  334. 	add	C,  CO2, LDC

  335. #endif

  336. 	ble	LL(20)

  337. 	.align 4

  338. 

  339. LL(11):

  340. #if defined(LT) || defined(RN)

  341. 	LFD	f16,  0 * SIZE(AO)

  342. 	LFD	f17,  1 * SIZE(AO)

  343. 	LFD	f18,  2 * SIZE(AO)

  344. 	LFD	f19,  3 * SIZE(AO)

  345. 

  346. 	LFD	f20,  0 * SIZE(B)

  347. 	LFD	f21,  1 * SIZE(B)

  348. 	LFD	f22,  2 * SIZE(B)

  349. 	LFD	f23,  3 * SIZE(B)

  350. 

  351. 	lfs	f0,  FZERO

  352.  	fmr	f1,  f0

  353. 	fmr	f2,  f0

  354. 	fmr	f3,  f0

  355. 

  356. 	fmr	f4,  f0

  357. 	fmr	f5,  f0

  358. 	fmr	f6,  f0

  359. 	fmr	f7,  f0

  360. 

  361. 	fmr	f8,  f0

  362. 	fmr	f9,  f0

  363. 	fmr	f10, f0

  364. 	fmr	f11, f0

  365. 

  366. 	fmr	f12, f0

  367. 	fmr	f13, f0

  368. 	fmr	f14, f0

  369. 	fmr	f15, f0

  370. 

  371. 	dcbt	CO1, PREC

  372. 	dcbt	CO2, PREC

  373. 

  374. 	srawi.	r0, KK,  2

  375. 	mtspr	CTR, r0

  376. 	mr	BO,  B

  377. #else

  378. 

  379. #ifdef LN

  380. 	slwi	r0,   K,  1 + ZBASE_SHIFT

  381. 	sub	AORIG, AORIG, r0

  382. #endif

  383. 

  384. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  385. 	add	AO, AORIG, TEMP

  386. 	add	BO, B,     TEMP

  387. 

  388. 	sub	TEMP, K, KK

  389. 

  390. 	LFD	f16,  0 * SIZE(AO)

  391. 	LFD	f17,  1 * SIZE(AO)

  392. 	LFD	f18,  2 * SIZE(AO)

  393. 	LFD	f19,  3 * SIZE(AO)

  394. 

  395. 	LFD	f20,  0 * SIZE(BO)

  396. 	LFD	f21,  1 * SIZE(BO)

  397. 	LFD	f22,  2 * SIZE(BO)

  398. 	LFD	f23,  3 * SIZE(BO)

  399. 

  400. 	lfs	f0,  FZERO

  401.  	fmr	f1,  f0

  402. 	fmr	f2,  f0

  403. 	fmr	f3,  f0

  404. 

  405. 	fmr	f4,  f0

  406. 	fmr	f5,  f0

  407. 	fmr	f6,  f0

  408. 	fmr	f7,  f0

  409. 

  410. 	fmr	f8,  f0

  411. 	fmr	f9,  f0

  412. 	fmr	f10, f0

  413. 	fmr	f11, f0

  414. 

  415. 	fmr	f12, f0

  416. 	fmr	f13, f0

  417. 	fmr	f14, f0

  418. 	fmr	f15, f0

  419. 

  420. 	dcbt	CO1, PREC

  421. 	dcbt	CO2, PREC

  422. 

  423. 	srawi.	r0, TEMP,  2

  424. 	mtspr	CTR, r0

  425. #endif

  426. 	ble	LL(15)

  427. 	.align 4

  428. 

  429. LL(12):

  430. 	fmadd	f0,  f16, f20, f0

  431. 	fmadd	f5,  f17, f21, f5

  432. 	fmadd	f10, f18, f22, f10

  433. 	fmadd	f15, f19, f23, f15

  434. 

  435. 	LFD	f28,  4 * SIZE(BO)

  436. 	LFD	f29,  5 * SIZE(BO)

  437. 	LFD	f30,  6 * SIZE(BO)

  438. 	LFD	f31,  7 * SIZE(BO)

  439. 

  440. 	fmadd	f1,  f17, f20, f1

  441. 	fmadd	f2,  f18, f20, f2

  442. 	fmadd	f3,  f19, f20, f3

  443. 	fmadd	f4,  f16, f21, f4

  444. 

  445. 	LFD	f24,  4 * SIZE(AO)

  446. 	LFD	f25,  5 * SIZE(AO)

  447. 	LFD	f26,  6 * SIZE(AO)

  448. 	LFD	f27,  7 * SIZE(AO)

  449. 

  450. 	fmadd	f6,  f18, f21, f6

  451. 	fmadd	f7,  f19, f21, f7

  452. 	fmadd	f8,  f16, f22, f8

  453. 	fmadd	f9,  f17, f22, f9

  454. 

  455. 	fmadd	f11, f19, f22, f11

  456. 	fmadd	f12, f16, f23, f12

  457. 	fmadd	f13, f17, f23, f13

  458. 	fmadd	f14, f18, f23, f14

  459. 

  460. 	LFD	f20,  8 * SIZE(BO)

  461. 	LFD	f21,  9 * SIZE(BO)

  462. 	LFD	f22, 10 * SIZE(BO)

  463. 	LFD	f23, 11 * SIZE(BO)

  464. 

  465. 	fmadd	f0,  f24, f28, f0

  466. 	fmadd	f5,  f25, f29, f5

  467. 	fmadd	f10, f26, f30, f10

  468. 	fmadd	f15, f27, f31, f15

  469. 

  470. 	LFD	f16,  8 * SIZE(AO)

  471. 	LFD	f17,  9 * SIZE(AO)

  472. 	LFD	f18, 10 * SIZE(AO)

  473. 	LFD	f19, 11 * SIZE(AO)

  474. 

  475. 	fmadd	f1,  f25, f28, f1

  476. 	fmadd	f2,  f26, f28, f2

  477. 	fmadd	f3,  f27, f28, f3

  478. 	fmadd	f4,  f24, f29, f4

  479. 

  480. 	fmadd	f6,  f26, f29, f6

  481. 	fmadd	f7,  f27, f29, f7

  482. 	fmadd	f8,  f24, f30, f8

  483. 	fmadd	f9,  f25, f30, f9

  484. 

  485. 	fmadd	f11, f27, f30, f11

  486. 	fmadd	f12, f24, f31, f12

  487. 	fmadd	f13, f25, f31, f13

  488. 	fmadd	f14, f26, f31, f14

  489. 

  490. 	LFD	f28, 12 * SIZE(BO)

  491. 	LFD	f29, 13 * SIZE(BO)

  492. 	LFD	f30, 14 * SIZE(BO)

  493. 	LFD	f31, 15 * SIZE(BO)

  494. 

  495. 	fmadd	f0,  f16, f20, f0

  496. 	fmadd	f5,  f17, f21, f5

  497. 	fmadd	f10, f18, f22, f10

  498. 	fmadd	f15, f19, f23, f15

  499. 

  500. 	LFD	f24, 12 * SIZE(AO)

  501. 	LFD	f25, 13 * SIZE(AO)

  502. 	LFD	f26, 14 * SIZE(AO)

  503. 	LFD	f27, 15 * SIZE(AO)

  504. 

  505. 	fmadd	f1,  f17, f20, f1

  506. 	fmadd	f2,  f18, f20, f2

  507. 	fmadd	f3,  f19, f20, f3

  508. 	fmadd	f4,  f16, f21, f4

  509. 

  510. 	fmadd	f6,  f18, f21, f6

  511. 	fmadd	f7,  f19, f21, f7

  512. 	fmadd	f8,  f16, f22, f8

  513. 	fmadd	f9,  f17, f22, f9

  514. 

  515. 	fmadd	f11, f19, f22, f11

  516. 	fmadd	f12, f16, f23, f12

  517. 	fmadd	f13, f17, f23, f13

  518. 	fmadd	f14, f18, f23, f14

  519. 

  520. 	LFD	f16, 16 * SIZE(AO)

  521. 	LFD	f17, 17 * SIZE(AO)

  522. 	LFD	f18, 18 * SIZE(AO)

  523. 	LFD	f19, 19 * SIZE(AO)

  524. 

  525. 	fmadd	f0,  f24, f28, f0

  526. 	fmadd	f5,  f25, f29, f5

  527. 	fmadd	f10, f26, f30, f10

  528. 	fmadd	f15, f27, f31, f15

  529. 

  530. 	LFD	f20, 16 * SIZE(BO)

  531. 	LFD	f21, 17 * SIZE(BO)

  532. 	LFD	f22, 18 * SIZE(BO)

  533. 	LFD	f23, 19 * SIZE(BO)

  534. 

  535. 	fmadd	f1,  f25, f28, f1

  536. 	fmadd	f2,  f26, f28, f2

  537. 	fmadd	f3,  f27, f28, f3

  538. 	fmadd	f4,  f24, f29, f4

  539. 

  540. 	fmadd	f6,  f26, f29, f6

  541. 	fmadd	f7,  f27, f29, f7

  542. 	fmadd	f8,  f24, f30, f8

  543. 	fmadd	f9,  f25, f30, f9

  544. 

  545. 	fmadd	f11, f27, f30, f11

  546. 	fmadd	f12, f24, f31, f12

  547. 	fmadd	f13, f25, f31, f13

  548. 	fmadd	f14, f26, f31, f14

  549. 

  550. 	addi	AO, AO, 16 * SIZE

  551. 	addi	BO, BO, 16 * SIZE

  552. #ifdef PPC970

  553. #ifndef ALLOC_HUGETLB

  554. 	DCBT(AO, PREA)

  555. #endif

  556. 	DCBT(BO, PREB)

  557. #endif

  558. 

  559. #ifdef POWER4

  560. #ifndef ALLOC_HUGETLB

  561. 	DCBT(AO, PREA)

  562. #endif

  563. 	DCBT(BO, PREB)

  564. #endif

  565. 

  566. #ifdef POWER5

  567. 	DCBT(AO, PREA)

  568. 	DCBT(BO, PREB)

  569. #endif

  570. 	bdnz	LL(12)

  571. 	.align 4

  572. 

  573. LL(15):

  574. #if defined(LT) || defined(RN)

  575. 	andi.	r0, KK,  3

  576. #else

  577. 	andi.	r0, TEMP, 3

  578. #endif

  579. 	mtspr	CTR, r0

  580. 	ble	LL(KERNEL_MainFinish)

  581. 	.align 4

  582. 

  583. LL(16):

  584. 	fmadd	f0,  f16, f20, f0

  585. 	fmadd	f5,  f17, f21, f5

  586. 	fmadd	f10, f18, f22, f10

  587. 	fmadd	f15, f19, f23, f15

  588. 

  589. 	fmadd	f1,  f17, f20, f1

  590. 	fmadd	f2,  f18, f20, f2

  591. 	fmadd	f3,  f19, f20, f3

  592. 	fmadd	f4,  f16, f21, f4

  593. 

  594. 	fmadd	f6,  f18, f21, f6

  595. 	fmadd	f7,  f19, f21, f7

  596. 	fmadd	f8,  f16, f22, f8

  597. 	fmadd	f9,  f17, f22, f9

  598. 

  599. 	fmadd	f11, f19, f22, f11

  600. 	fmadd	f12, f16, f23, f12

  601. 	fmadd	f13, f17, f23, f13

  602. 	fmadd	f14, f18, f23, f14

  603. 

  604. 	LFD	f16,  4 * SIZE(AO)

  605. 	LFD	f17,  5 * SIZE(AO)

  606. 	LFD	f18,  6 * SIZE(AO)

  607. 	LFD	f19,  7 * SIZE(AO)

  608. 

  609. 	LFD	f20,  4 * SIZE(BO)

  610. 	LFD	f21,  5 * SIZE(BO)

  611. 	LFD	f22,  6 * SIZE(BO)

  612. 	LFD	f23,  7 * SIZE(BO)

  613. 

  614. 	addi	BO, BO,  4 * SIZE

  615. 	addi	AO, AO,  4 * SIZE

  616. 	bdnz	LL(16)

  617. 	.align 4

  618. 

  619. LL(KERNEL_MainFinish):

  620. #ifndef CONJ

  621. 	FSUB	  f0,  f0,  f5

  622. 	FADD	  f1,  f1,  f4

  623. 	FSUB	  f2,  f2,  f7

  624. 	FADD	  f3,  f3,  f6

  625. 

  626. 	FSUB	  f8,  f8,  f13

  627. 	FADD	  f9,  f9,  f12

  628. 	FSUB	  f10, f10, f15

  629. 	FADD	  f11, f11, f14

  630. 

  631. #else

  632. 	FADD	  f0,  f0,  f5

  633. 	FSUB	  f1,  f4,  f1

  634. 	FADD	  f2,  f2,  f7

  635. 	FSUB	  f3,  f6,  f3

  636. 

  637. 	FADD	  f8,  f8,  f13

  638. 	FSUB	  f9,  f12, f9

  639. 	FADD	  f10, f10, f15

  640. 	FSUB	  f11, f14, f11

  641. #endif

  642. 

  643. #if defined(LN) || defined(RT)

  644. 	subi	r0, KK, 2

  645. 	slwi	r0, r0, 1 + ZBASE_SHIFT

  646. 	add	AO, AORIG, r0

  647. 	add	BO, B,     r0

  648. #endif

  649. 

  650. #if defined(LN) || defined(LT)

  651. 	LFD	f16,  0 * SIZE(BO)

  652. 	LFD	f17,  1 * SIZE(BO)

  653. 	LFD	f18,  2 * SIZE(BO)

  654. 	LFD	f19,  3 * SIZE(BO)

  655. 

  656. 	LFD	f20,  4 * SIZE(BO)

  657.  	LFD	f21,  5 * SIZE(BO)

  658. 	LFD	f22,  6 * SIZE(BO)

  659. 	LFD	f23,  7 * SIZE(BO)

  660. 

  661. 	FSUB	f0,  f16, f0

  662. 	FSUB	f1,  f17, f1

  663. 	FSUB	f8,  f18, f8

  664. 	FSUB	f9,  f19, f9

  665. 

  666. 	FSUB	f2,  f20, f2

  667. 	FSUB	f3,  f21, f3

  668. 	FSUB	f10, f22, f10

  669. 	FSUB	f11, f23, f11

  670. #else

  671. 	LFD	f16,  0 * SIZE(AO)

  672. 	LFD	f17,  1 * SIZE(AO)

  673. 	LFD	f18,  2 * SIZE(AO)

  674. 	LFD	f19,  3 * SIZE(AO)

  675. 

  676. 	LFD	f20,  4 * SIZE(AO)

  677.  	LFD	f21,  5 * SIZE(AO)

  678. 	LFD	f22,  6 * SIZE(AO)

  679. 	LFD	f23,  7 * SIZE(AO)

  680. 

  681. #ifndef CONJ

  682. 	FSUB	f0,  f16, f0

  683. 	FSUB	f1,  f17, f1

  684. 	FSUB	f2,  f18, f2

  685. 	FSUB	f3,  f19, f3

  686. 

  687. 	FSUB	f8,  f20, f8

  688. 	FSUB	f9,  f21, f9

  689. 	FSUB	f10, f22, f10

  690. 	FSUB	f11, f23, f11

  691. #else

  692. 	FSUB	f0,  f16, f0

  693. 	FADD	f1,  f17, f1

  694. 	FSUB	f2,  f18, f2

  695. 	FADD	f3,  f19, f3

  696. 

  697. 	FSUB	f8,  f20, f8

  698. 	FADD	f9,  f21, f9

  699. 	FSUB	f10, f22, f10

  700. 	FADD	f11, f23, f11

  701. #endif

  702. #endif

  703. 

  704. #ifdef LN

  705. 	LFD	f16,  6 * SIZE(AO)

  706. 	LFD	f17,  7 * SIZE(AO)

  707. 	LFD	f18,  4 * SIZE(AO)

  708. 	LFD	f19,  5 * SIZE(AO)

  709. 	LFD	f20,  0 * SIZE(AO)

  710. 	LFD	f21,  1 * SIZE(AO)

  711. 

  712. 	FMUL	f6,  f17, f3

  713. 	FMUL	f7,  f17, f2

  714. 	FMUL	f14, f17, f11

  715. 	FMUL	f15, f17, f10

  716. 

  717. #ifndef CONJ

  718. 	FMSUB	f2,  f16, f2,  f6

  719. 	FMADD	f3,  f16, f3,  f7

  720. 	FMSUB	f10, f16, f10, f14

  721. 	FMADD	f11, f16, f11, f15

  722. 

  723. 	FMADD	f0,  f19, f3,  f0

  724. 	FNMSUB	f1,  f19, f2,  f1

  725. 	FMADD	f8,  f19, f11, f8

  726. 	FNMSUB	f9,  f19, f10, f9

  727. 

  728. 	FNMSUB	f0,  f18, f2,  f0

  729. 	FNMSUB	f1,  f18, f3,  f1

  730. 	FNMSUB	f8,  f18, f10, f8

  731. 	FNMSUB	f9,  f18, f11, f9

  732. 

  733. 	FMUL	f4,  f21, f1

  734. 	FMUL	f5,  f21, f0

  735. 	FMUL	f12, f21, f9

  736. 	FMUL	f13, f21, f8

  737. 

  738. 	FMSUB	f0,  f20, f0,  f4

  739. 	FMADD	f1,  f20, f1,  f5

  740. 	FMSUB	f8,  f20, f8,  f12

  741. 	FMADD	f9,  f20, f9,  f13

  742. 

  743. #else

  744. 	FMADD	f2,  f16, f2,  f6

  745. 	FMSUB	f3,  f16, f3,  f7

  746. 	FMADD	f10, f16, f10, f14

  747. 	FMSUB	f11, f16, f11, f15

  748. 

  749. 	FMSUB	f0,  f19, f3,  f0

  750. 	FNMADD	f1,  f19, f2,  f1

  751. 	FMSUB	f8,  f19, f11, f8

  752. 	FNMADD	f9,  f19, f10, f9

  753. 

  754. 	FNMADD	f0,  f18, f2,  f0

  755. 	FNMADD	f1,  f18, f3,  f1

  756. 	FNMADD	f8,  f18, f10, f8

  757. 	FNMADD	f9,  f18, f11, f9

  758. 

  759. 	FMUL	f4,  f21, f1

  760. 	FMUL	f5,  f21, f0

  761. 	FMUL	f12, f21, f9

  762. 	FMUL	f13, f21, f8

  763. 

  764. 	FMADD	f0,  f20, f0,  f4

  765. 	FMSUB	f1,  f20, f1,  f5

  766. 	FMADD	f8,  f20, f8,  f12

  767. 	FMSUB	f9,  f20, f9,  f13

  768. #endif

  769. #endif

  770. 

  771. #ifdef LT

  772. 	LFD	f16,  0 * SIZE(AO)

  773. 	LFD	f17,  1 * SIZE(AO)

  774. 	LFD	f18,  2 * SIZE(AO)

  775. 	LFD	f19,  3 * SIZE(AO)

  776. 	LFD	f20,  6 * SIZE(AO)

  777. 	LFD	f21,  7 * SIZE(AO)

  778. 

  779. 	FMUL	f4,  f17, f1

  780. 	FMUL	f5,  f17, f0

  781. 	FMUL	f12, f17, f9

  782. 	FMUL	f13, f17, f8

  783. 

  784. #ifndef CONJ

  785. 	FMSUB	f0,  f16, f0,  f4

  786. 	FMADD	f1,  f16, f1,  f5

  787. 	FMSUB	f8,  f16, f8,  f12

  788. 	FMADD	f9,  f16, f9,  f13

  789. 

  790. 	FMADD	f2,  f19, f1, f2

  791. 	FNMSUB	f3,  f19, f0, f3

  792. 	FMADD	f10, f19, f9, f10

  793. 	FNMSUB	f11, f19, f8, f11

  794. 

  795. 	FNMSUB	f2,  f18, f0,  f2

  796. 	FNMSUB	f3,  f18, f1,  f3

  797. 	FNMSUB	f10, f18, f8,  f10

  798. 	FNMSUB	f11, f18, f9,  f11

  799. 

  800. 	FMUL	f4,  f21, f3

  801. 	FMUL	f5,  f21, f2

  802. 	FMUL	f12, f21, f11

  803. 	FMUL	f13, f21, f10

  804. 

  805. 	FMSUB	f2,  f20, f2,  f4

  806. 	FMADD	f3,  f20, f3,  f5

  807. 	FMSUB	f10, f20, f10, f12

  808. 	FMADD	f11, f20, f11, f13

  809. 

  810. #else

  811. 	FMADD	f0,  f16, f0,  f4

  812. 	FMSUB	f1,  f16, f1,  f5

  813. 	FMADD	f8,  f16, f8,  f12

  814. 	FMSUB	f9,  f16, f9,  f13

  815. 

  816. 	FMSUB	f2,  f19, f1, f2

  817. 	FNMADD	f3,  f19, f0, f3

  818. 	FMSUB	f10, f19, f9, f10

  819. 	FNMADD	f11, f19, f8, f11

  820. 

  821. 	FNMADD	f2,  f18, f0,  f2

  822. 	FNMADD	f3,  f18, f1,  f3

  823. 	FNMADD	f10, f18, f8,  f10

  824. 	FNMADD	f11, f18, f9,  f11

  825. 

  826. 	FMUL	f4,  f21, f3

  827. 	FMUL	f5,  f21, f2

  828. 	FMUL	f12, f21, f11

  829. 	FMUL	f13, f21, f10

  830. 

  831. 	FMADD	f2,  f20, f2,  f4

  832. 	FMSUB	f3,  f20, f3,  f5

  833. 	FMADD	f10, f20, f10, f12

  834. 	FMSUB	f11, f20, f11, f13

  835. #endif

  836. #endif

  837. 

  838. #ifdef RN

  839. 	LFD	f16,  0 * SIZE(BO)

  840. 	LFD	f17,  1 * SIZE(BO)

  841. 	LFD	f18,  2 * SIZE(BO)

  842. 	LFD	f19,  3 * SIZE(BO)

  843. 	LFD	f20,  6 * SIZE(BO)

  844. 	LFD	f21,  7 * SIZE(BO)

  845. 

  846. 	FMUL	f4,  f17, f1

  847. 	FMUL	f5,  f17, f0

  848. 	FMUL	f6,  f17, f3

  849. 	FMUL	f7,  f17, f2

  850. 

  851. #ifndef CONJ

  852. 	FMSUB	f0,  f16, f0,  f4

  853. 	FMADD	f1,  f16, f1,  f5

  854. 	FMSUB	f2,  f16, f2,  f6

  855. 	FMADD	f3,  f16, f3,  f7

  856. 

  857. 	FMADD	f8,  f19, f1, f8

  858. 	FNMSUB	f9,  f19, f0, f9

  859. 	FMADD	f10, f19, f3, f10

  860. 	FNMSUB	f11, f19, f2, f11

  861. 

  862. 	FNMSUB	f8,  f18, f0,  f8

  863. 	FNMSUB	f9,  f18, f1,  f9

  864. 	FNMSUB	f10, f18, f2,  f10

  865. 	FNMSUB	f11, f18, f3,  f11

  866. 

  867. 	FMUL	f4,  f21, f9

  868. 	FMUL	f5,  f21, f8

  869. 	FMUL	f6,  f21, f11

  870. 	FMUL	f7,  f21, f10

  871. 

  872. 	FMSUB	f8,  f20, f8,  f4

  873. 	FMADD	f9,  f20, f9,  f5

  874. 	FMSUB	f10, f20, f10, f6

  875. 	FMADD	f11, f20, f11, f7

  876. #else

  877. 	FMADD	f0,  f16, f0,  f4

  878. 	FMSUB	f1,  f16, f1,  f5

  879. 	FMADD	f2,  f16, f2,  f6

  880. 	FMSUB	f3,  f16, f3,  f7

  881. 

  882. 	FMSUB	f8,  f19, f1, f8

  883. 	FNMADD	f9,  f19, f0, f9

  884. 	FMSUB	f10, f19, f3, f10

  885. 	FNMADD	f11, f19, f2, f11

  886. 

  887. 	FNMADD	f8,  f18, f0,  f8

  888. 	FNMADD	f9,  f18, f1,  f9

  889. 	FNMADD	f10, f18, f2,  f10

  890. 	FNMADD	f11, f18, f3,  f11

  891. 

  892. 	FMUL	f4,  f21, f9

  893. 	FMUL	f5,  f21, f8

  894. 	FMUL	f6,  f21, f11

  895. 	FMUL	f7,  f21, f10

  896. 

  897. 	FMADD	f8,  f20, f8,  f4

  898. 	FMSUB	f9,  f20, f9,  f5

  899. 	FMADD	f10, f20, f10, f6

  900. 	FMSUB	f11, f20, f11, f7

  901. #endif

  902. #endif

  903. 

  904. #ifdef RT

  905. 	LFD	f16,  6 * SIZE(BO)

  906. 	LFD	f17,  7 * SIZE(BO)

  907. 	LFD	f18,  4 * SIZE(BO)

  908. 	LFD	f19,  5 * SIZE(BO)

  909. 	LFD	f20,  0 * SIZE(BO)

  910. 	LFD	f21,  1 * SIZE(BO)

  911. 

  912. 	FMUL	f12, f17, f9

  913. 	FMUL	f13, f17, f8

  914. 	FMUL	f14, f17, f11

  915. 	FMUL	f15, f17, f10

  916. 

  917. #ifndef CONJ

  918. 	FMSUB	f8,  f16, f8,  f12

  919. 	FMADD	f9,  f16, f9,  f13

  920. 	FMSUB	f10, f16, f10, f14

  921. 	FMADD	f11, f16, f11, f15

  922. 

  923. 	FMADD	f0,  f19, f9,  f0

  924. 	FNMSUB	f1,  f19, f8,  f1

  925. 	FMADD	f2,  f19, f11, f2

  926. 	FNMSUB	f3,  f19, f10, f3

  927. 

  928. 	FNMSUB	f0,  f18, f8,  f0

  929. 	FNMSUB	f1,  f18, f9,  f1

  930. 	FNMSUB	f2,  f18, f10, f2

  931. 	FNMSUB	f3,  f18, f11, f3

  932. 

  933. 	FMUL	f4,  f21, f1

  934. 	FMUL	f5,  f21, f0

  935. 	FMUL	f6,  f21, f3

  936. 	FMUL	f7,  f21, f2

  937. 

  938. 	FMSUB	f0,  f20, f0,  f4

  939. 	FMADD	f1,  f20, f1,  f5

  940. 	FMSUB	f2,  f20, f2,  f6

  941. 	FMADD	f3,  f20, f3,  f7

  942. 

  943. #else

  944. 	FMADD	f8,  f16, f8,  f12

  945. 	FMSUB	f9,  f16, f9,  f13

  946. 	FMADD	f10, f16, f10, f14

  947. 	FMSUB	f11, f16, f11, f15

  948. 

  949. 	FMSUB	f0,  f19, f9,  f0

  950. 	FNMADD	f1,  f19, f8,  f1

  951. 	FMSUB	f2,  f19, f11, f2

  952. 	FNMADD	f3,  f19, f10, f3

  953. 

  954. 	FNMADD	f0,  f18, f8,  f0

  955. 	FNMADD	f1,  f18, f9,  f1

  956. 	FNMADD	f2,  f18, f10, f2

  957. 	FNMADD	f3,  f18, f11, f3

  958. 

  959. 	FMUL	f4,  f21, f1

  960. 	FMUL	f5,  f21, f0

  961. 	FMUL	f6,  f21, f3

  962. 	FMUL	f7,  f21, f2

  963. 

  964. 	FMADD	f0,  f20, f0,  f4

  965. 	FMSUB	f1,  f20, f1,  f5

  966. 	FMADD	f2,  f20, f2,  f6

  967. 	FMSUB	f3,  f20, f3,  f7

  968. #endif

  969. #endif

  970. 

  971. #ifdef LN

  972. 	subi	CO1, CO1, 4 * SIZE

  973. 	subi	CO2, CO2, 4 * SIZE

  974. #endif

  975. 

  976. #if defined(LN) || defined(LT)

  977. 	STFD	f0,   0 * SIZE(BO)

  978. 	STFD	f1,   1 * SIZE(BO)

  979. 	STFD	f8,   2 * SIZE(BO)

  980. 	STFD	f9,   3 * SIZE(BO)

  981. 

  982. 	STFD	f2,   4 * SIZE(BO)

  983. 	STFD	f3,   5 * SIZE(BO)

  984. 	STFD	f10,  6 * SIZE(BO)

  985. 	STFD	f11,  7 * SIZE(BO)

  986. #else

  987. 	STFD	f0,   0 * SIZE(AO)

  988. 	STFD	f1,   1 * SIZE(AO)

  989. 	STFD	f2,   2 * SIZE(AO)

  990. 	STFD	f3,   3 * SIZE(AO)

  991. 

  992. 	STFD	f8,   4 * SIZE(AO)

  993. 	STFD	f9,   5 * SIZE(AO)

  994. 	STFD	f10,  6 * SIZE(AO)

  995. 	STFD	f11,  7 * SIZE(AO)

  996. #endif

  997. 

  998. 	STFD	f0,   0 * SIZE(CO1)

  999. 	STFD	f1,   1 * SIZE(CO1)

  1000. 	STFD	f2,   2 * SIZE(CO1)

  1001. 	STFD	f3,   3 * SIZE(CO1)

  1002. 

  1003. 	STFD	f8,   0 * SIZE(CO2)

  1004. 	STFD	f9,   1 * SIZE(CO2)

  1005. 	STFD	f10,  2 * SIZE(CO2)

  1006. 	STFD	f11,  3 * SIZE(CO2)

  1007. 

  1008. #ifndef LN

  1009. 	addi	CO1, CO1, 4 * SIZE

  1010. 	addi	CO2, CO2, 4 * SIZE

  1011. #endif

  1012. 

  1013. #ifdef RT

  1014. 	slwi	r0, K, 1 + ZBASE_SHIFT

  1015. 	add	AORIG, AORIG, r0

  1016. #endif

  1017. 

  1018. #if defined(LT) || defined(RN)

  1019. 	sub	TEMP, K, KK

  1020. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  1021. 	add	AO, AO, TEMP

  1022. 	add	BO, BO, TEMP

  1023. #endif

  1024. 

  1025. #ifdef LT

  1026. 	addi	KK, KK, 2

  1027. #endif

  1028. 

  1029. #ifdef LN

  1030. 	subi	KK, KK, 2

  1031. #endif

  1032. 

  1033. 	addic.	I, I, -1

  1034. 	bgt	LL(11)

  1035. 	.align 4

  1036. 

  1037. LL(20):

  1038. 	andi.	I,  M,  1

  1039. 	ble	LL(29)

  1040. 

  1041. #if defined(LT) || defined(RN)

  1042. 	LFD	f16,  0 * SIZE(AO)

  1043. 	LFD	f17,  1 * SIZE(AO)

  1044. 	LFD	f18,  2 * SIZE(AO)

  1045. 	LFD	f19,  3 * SIZE(AO)

  1046. 

  1047. 	LFD	f20,  0 * SIZE(B)

  1048. 	LFD	f21,  1 * SIZE(B)

  1049. 	LFD	f22,  2 * SIZE(B)

  1050. 	LFD	f23,  3 * SIZE(B)

  1051. 

  1052. 	LFD	f24,  4 * SIZE(B)

  1053. 	LFD	f25,  5 * SIZE(B)

  1054. 	LFD	f26,  6 * SIZE(B)

  1055. 	LFD	f27,  7 * SIZE(B)

  1056. 

  1057. 	lfs	f0,  FZERO

  1058.  	fmr	f1,  f0

  1059. 	fmr	f2,  f0

  1060. 	fmr	f3,  f0

  1061. 

  1062. 	fmr	f4,  f0

  1063. 	fmr	f5,  f0

  1064. 	fmr	f6,  f0

  1065. 	fmr	f7,  f0

  1066. 

  1067. 	srawi.	r0, KK,  2

  1068. 	mr	BO,  B

  1069. 	mtspr	CTR, r0

  1070. #else

  1071. 

  1072. #ifdef LN

  1073. 	slwi	r0,   K,  0 + ZBASE_SHIFT

  1074. 	sub	AORIG, AORIG, r0

  1075. #endif

  1076. 

  1077. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  1078. 	slwi	TEMP, KK, 1 + ZBASE_SHIFT

  1079. 	add	AO, AORIG, r0

  1080. 	add	BO, B,     TEMP

  1081. 

  1082. 	sub	TEMP, K, KK

  1083. 

  1084. 	LFD	f16,  0 * SIZE(AO)

  1085. 	LFD	f17,  1 * SIZE(AO)

  1086. 	LFD	f18,  2 * SIZE(AO)

  1087. 	LFD	f19,  3 * SIZE(AO)

  1088. 

  1089. 	LFD	f20,  0 * SIZE(BO)

  1090. 	LFD	f21,  1 * SIZE(BO)

  1091. 	LFD	f22,  2 * SIZE(BO)

  1092. 	LFD	f23,  3 * SIZE(BO)

  1093. 

  1094. 	LFD	f24,  4 * SIZE(BO)

  1095. 	LFD	f25,  5 * SIZE(BO)

  1096. 	LFD	f26,  6 * SIZE(BO)

  1097. 	LFD	f27,  7 * SIZE(BO)

  1098. 

  1099. 	lfs	f0,  FZERO

  1100.  	fmr	f1,  f0

  1101. 	fmr	f2,  f0

  1102. 	fmr	f3,  f0

  1103. 

  1104. 	fmr	f4,  f0

  1105. 	fmr	f5,  f0

  1106. 	fmr	f6,  f0

  1107. 	fmr	f7,  f0

  1108. 

  1109. 	srawi.	r0, TEMP,  2

  1110. 	mtspr	CTR, r0

  1111. #endif

  1112. 	ble	LL(25)

  1113. 	.align 4

  1114. 

  1115. LL(22):

  1116. 	fmadd	f0,  f16, f20, f0

  1117. 	fmadd	f1,  f16, f21, f1

  1118. 	fmadd	f2,  f16, f22, f2

  1119. 	fmadd	f3,  f16, f23, f3

  1120. 

  1121. 	fmadd	f4,  f17, f20, f4

  1122. 	fmadd	f5,  f17, f21, f5

  1123. 	fmadd	f6,  f17, f22, f6

  1124. 	fmadd	f7,  f17, f23, f7

  1125. 

  1126.  	LFD	f20,  8 * SIZE(BO)

  1127. 	LFD	f21,  9 * SIZE(BO)

  1128. 	LFD	f22, 10 * SIZE(BO)

  1129. 	LFD	f23, 11 * SIZE(BO)

  1130. 

  1131. 	fmadd	f0,  f18, f24, f0

  1132. 	fmadd	f1,  f18, f25, f1

  1133. 	fmadd	f2,  f18, f26, f2

  1134. 	fmadd	f3,  f18, f27, f3

  1135. 

  1136. 	fmadd	f4,  f19, f24, f4

  1137. 	fmadd	f5,  f19, f25, f5

  1138. 	fmadd	f6,  f19, f26, f6

  1139. 	fmadd	f7,  f19, f27, f7

  1140. 

  1141.  	LFD	f24, 12 * SIZE(BO)

  1142. 	LFD	f25, 13 * SIZE(BO)

  1143. 	LFD	f26, 14 * SIZE(BO)

  1144. 	LFD	f27, 15 * SIZE(BO)

  1145. 

  1146. 	LFD	f16,  4 * SIZE(AO)

  1147. 	LFD	f17,  5 * SIZE(AO)

  1148. 	LFD	f18,  6 * SIZE(AO)

  1149. 	LFD	f19,  7 * SIZE(AO)

  1150. 

  1151. 	fmadd	f0,  f16, f20, f0

  1152. 	fmadd	f1,  f16, f21, f1

  1153. 	fmadd	f2,  f16, f22, f2

  1154. 	fmadd	f3,  f16, f23, f3

  1155. 

  1156. 	fmadd	f4,  f17, f20, f4

  1157. 	fmadd	f5,  f17, f21, f5

  1158. 	fmadd	f6,  f17, f22, f6

  1159. 	fmadd	f7,  f17, f23, f7

  1160. 

  1161.  	LFD	f20, 16 * SIZE(BO)

  1162. 	LFD	f21, 17 * SIZE(BO)

  1163. 	LFD	f22, 18 * SIZE(BO)

  1164. 	LFD	f23, 19 * SIZE(BO)

  1165. 

  1166. 	fmadd	f0,  f18, f24, f0

  1167. 	fmadd	f1,  f18, f25, f1

  1168. 	fmadd	f2,  f18, f26, f2

  1169. 	fmadd	f3,  f18, f27, f3

  1170. 

  1171. 	fmadd	f4,  f19, f24, f4

  1172. 	fmadd	f5,  f19, f25, f5

  1173. 	fmadd	f6,  f19, f26, f6

  1174. 	fmadd	f7,  f19, f27, f7

  1175. 

  1176. 	LFD	f16,  8 * SIZE(AO)

  1177. 	LFD	f17,  9 * SIZE(AO)

  1178. 	LFD	f18, 10 * SIZE(AO)

  1179. 	LFD	f19, 11 * SIZE(AO)

  1180. 

  1181.  	LFD	f24, 20 * SIZE(BO)

  1182. 	LFD	f25, 21 * SIZE(BO)

  1183. 	LFD	f26, 22 * SIZE(BO)

  1184. 	LFD	f27, 23 * SIZE(BO)

  1185. 

  1186. 	addi	BO,  BO, 16 * SIZE

  1187. 	addi	AO,  AO,  8 * SIZE

  1188. 	bdnz	LL(22)

  1189. 	.align 4

  1190. 

  1191. LL(25):

  1192. #if defined(LT) || defined(RN)

  1193. 	andi.	r0, KK,  3

  1194. #else

  1195. 	andi.	r0, TEMP, 3

  1196. #endif

  1197. 	mtspr	CTR, r0

  1198. 	ble	LL(27)

  1199. 	.align 4

  1200. 

  1201. LL(26):

  1202. 	fmadd	f0,  f16, f20, f0

  1203. 	fmadd	f1,  f16, f21, f1

  1204. 	fmadd	f2,  f16, f22, f2

  1205. 	fmadd	f3,  f16, f23, f3

  1206. 

  1207. 	fmadd	f4,  f17, f20, f4

  1208. 	fmadd	f5,  f17, f21, f5

  1209. 	fmadd	f6,  f17, f22, f6

  1210. 	fmadd	f7,  f17, f23, f7

  1211. 

  1212.  	LFD	f20,  4 * SIZE(BO)

  1213. 	LFD	f21,  5 * SIZE(BO)

  1214. 	LFD	f22,  6 * SIZE(BO)

  1215. 	LFD	f23,  7 * SIZE(BO)

  1216. 

  1217. 	LFD	f16,  2 * SIZE(AO)

  1218. 	LFD	f17,  3 * SIZE(AO)

  1219. 	addi	AO, AO,  2 * SIZE

  1220. 	addi	BO, BO,  4 * SIZE

  1221. 	bdnz	LL(26)

  1222. 	.align 4

  1223. 

  1224. LL(27):

  1225. #ifndef CONJ

  1226. 	FSUB	  f0,  f0,  f5

  1227. 	FADD	  f1,  f1,  f4

  1228. 	FSUB	  f2,  f2,  f7

  1229. 	FADD	  f3,  f3,  f6

  1230. #else

  1231. #if defined(LN) || defined(LT)

  1232. 	FADD	  f0,  f0,  f5

  1233. 	FSUB	  f1,  f1,  f4

  1234. 	FADD	  f2,  f2,  f7

  1235. 	FSUB	  f3,  f3,  f6

  1236. #else

  1237. 	FADD	  f0,  f0,  f5

  1238. 	FSUB	  f1,  f4,  f1

  1239. 	FADD	  f2,  f2,  f7

  1240. 	FSUB	  f3,  f6,  f3

  1241. #endif

  1242. #endif

  1243. 

  1244. #if defined(LN) || defined(RT)

  1245. #ifdef LN

  1246. 	subi	r0, KK, 1

  1247. #else

  1248. 	subi	r0, KK, 2

  1249. #endif

  1250. 	slwi	TEMP, r0, 0 + ZBASE_SHIFT

  1251. 	slwi	r0,   r0, 1 + ZBASE_SHIFT

  1252. 	add	AO, AORIG, TEMP

  1253. 	add	BO, B,     r0

  1254. #endif

  1255. 

  1256. 

  1257. #if defined(LN) || defined(LT)

  1258. 	LFD	f16,  0 * SIZE(BO)

  1259. 	LFD	f17,  1 * SIZE(BO)

  1260. 	LFD	f18,  2 * SIZE(BO)

  1261. 	LFD	f19,  3 * SIZE(BO)

  1262. 

  1263. 	FSUB	f0,  f16, f0

  1264. 	FSUB	f1,  f17, f1

  1265. 	FSUB	f2,  f18, f2

  1266. 	FSUB	f3,  f19, f3

  1267. #else

  1268. 	LFD	f16,  0 * SIZE(AO)

  1269. 	LFD	f17,  1 * SIZE(AO)

  1270. 	LFD	f20,  2 * SIZE(AO)

  1271.  	LFD	f21,  3 * SIZE(AO)

  1272. 

  1273. 	FSUB	f0,  f16, f0

  1274. 	FSUB	f1,  f17, f1

  1275. 	FSUB	f2,  f20, f2

  1276. 	FSUB	f3,  f21, f3

  1277. #endif

  1278. 

  1279. #ifdef LN

  1280. 	LFD	f20,  0 * SIZE(AO)

  1281. 	LFD	f21,  1 * SIZE(AO)

  1282. 

  1283. 	FMUL	f4,  f21, f1

  1284. 	FMUL	f5,  f21, f0

  1285. 	FMUL	f12, f21, f3

  1286. 	FMUL	f13, f21, f2

  1287. 

  1288. #ifndef CONJ

  1289. 	FMSUB	f0,  f20, f0,  f4

  1290. 	FMADD	f1,  f20, f1,  f5

  1291. 	FMSUB	f2,  f20, f2,  f12

  1292. 	FMADD	f3,  f20, f3,  f13

  1293. #else

  1294. 	FMADD	f0,  f20, f0,  f4

  1295. 	FMSUB	f1,  f20, f1,  f5

  1296. 	FMADD	f2,  f20, f2,  f12

  1297. 	FMSUB	f3,  f20, f3,  f13

  1298. #endif

  1299. #endif

  1300. 

  1301. #ifdef LT

  1302. 	LFD	f16,  0 * SIZE(AO)

  1303. 	LFD	f17,  1 * SIZE(AO)

  1304. 

  1305. 	FMUL	f4,  f17, f1

  1306. 	FMUL	f5,  f17, f0

  1307. 	FMUL	f12, f17, f3

  1308. 	FMUL	f13, f17, f2

  1309. 

  1310. #ifndef CONJ

  1311. 	FMSUB	f0,  f16, f0,  f4

  1312. 	FMADD	f1,  f16, f1,  f5

  1313. 	FMSUB	f2,  f16, f2,  f12

  1314. 	FMADD	f3,  f16, f3,  f13

  1315. #else

  1316. 	FMADD	f0,  f16, f0,  f4

  1317. 	FMSUB	f1,  f16, f1,  f5

  1318. 	FMADD	f2,  f16, f2,  f12

  1319. 	FMSUB	f3,  f16, f3,  f13

  1320. #endif

  1321. #endif

  1322. 

  1323. #ifdef RN

  1324. 	LFD	f16,  0 * SIZE(BO)

  1325. 	LFD	f17,  1 * SIZE(BO)

  1326. 	LFD	f18,  2 * SIZE(BO)

  1327. 	LFD	f19,  3 * SIZE(BO)

  1328. 	LFD	f20,  6 * SIZE(BO)

  1329. 	LFD	f21,  7 * SIZE(BO)

  1330. 

  1331. 	FMUL	f4,  f17, f1

  1332. 	FMUL	f5,  f17, f0

  1333. 

  1334. #ifndef CONJ

  1335. 	FMSUB	f0,  f16, f0,  f4

  1336. 	FMADD	f1,  f16, f1,  f5

  1337. 

  1338. 	FMADD	f2,  f19, f1, f2

  1339. 	FNMSUB	f3,  f19, f0, f3

  1340. 

  1341. 	FNMSUB	f2,  f18, f0,  f2

  1342. 	FNMSUB	f3,  f18, f1,  f3

  1343. 

  1344. 	FMUL	f4,  f21, f3

  1345. 	FMUL	f5,  f21, f2

  1346. 

  1347. 	FMSUB	f2,  f20, f2,  f4

  1348. 	FMADD	f3,  f20, f3,  f5

  1349. #else

  1350. 	FMADD	f0,  f16, f0,  f4

  1351. 	FMSUB	f1,  f16, f1,  f5

  1352. 

  1353. 	FMSUB	f2,  f19, f1, f2

  1354. 	FNMADD	f3,  f19, f0, f3

  1355. 

  1356. 	FNMADD	f2,  f18, f0,  f2

  1357. 	FNMADD	f3,  f18, f1,  f3

  1358. 

  1359. 	FMUL	f4,  f21, f3

  1360. 	FMUL	f5,  f21, f2

  1361. 

  1362. 	FMADD	f2,  f20, f2,  f4

  1363. 	FMSUB	f3,  f20, f3,  f5

  1364. #endif

  1365. #endif

  1366. 

  1367. #ifdef RT

  1368. 	LFD	f16,  6 * SIZE(BO)

  1369. 	LFD	f17,  7 * SIZE(BO)

  1370. 	LFD	f18,  4 * SIZE(BO)

  1371. 	LFD	f19,  5 * SIZE(BO)

  1372. 	LFD	f20,  0 * SIZE(BO)

  1373. 	LFD	f21,  1 * SIZE(BO)

  1374. 

  1375. 	FMUL	f12, f17, f3

  1376. 	FMUL	f13, f17, f2

  1377. 

  1378. #ifndef CONJ

  1379. 	FMSUB	f2,  f16, f2,  f12

  1380. 	FMADD	f3,  f16, f3,  f13

  1381. 

  1382. 	FMADD	f0,  f19, f3,  f0

  1383. 	FNMSUB	f1,  f19, f2,  f1

  1384. 

  1385. 	FNMSUB	f0,  f18, f2,  f0

  1386. 	FNMSUB	f1,  f18, f3,  f1

  1387. 

  1388. 	FMUL	f4,  f21, f1

  1389. 	FMUL	f5,  f21, f0

  1390. 

  1391. 	FMSUB	f0,  f20, f0,  f4

  1392. 	FMADD	f1,  f20, f1,  f5

  1393. #else

  1394. 	FMADD	f2,  f16, f2,  f12

  1395. 	FMSUB	f3,  f16, f3,  f13

  1396. 

  1397. 	FMSUB	f0,  f19, f3,  f0

  1398. 	FNMADD	f1,  f19, f2,  f1

  1399. 

  1400. 	FNMADD	f0,  f18, f2,  f0

  1401. 	FNMADD	f1,  f18, f3,  f1

  1402. 

  1403. 	FMUL	f4,  f21, f1

  1404. 	FMUL	f5,  f21, f0

  1405. 

  1406. 	FMADD	f0,  f20, f0,  f4

  1407. 	FMSUB	f1,  f20, f1,  f5

  1408. #endif

  1409. #endif

  1410. 

  1411. #ifdef LN

  1412. 	subi	CO1, CO1, 2 * SIZE

  1413. 	subi	CO2, CO2, 2 * SIZE

  1414. #endif

  1415. 

  1416. #if defined(LN) || defined(LT)

  1417. 	STFD	f0,   0 * SIZE(BO)

  1418. 	STFD	f1,   1 * SIZE(BO)

  1419. 	STFD	f2,   2 * SIZE(BO)

  1420. 	STFD	f3,   3 * SIZE(BO)

  1421. #else

  1422. 	STFD	f0,   0 * SIZE(AO)

  1423. 	STFD	f1,   1 * SIZE(AO)

  1424. 	STFD	f2,   2 * SIZE(AO)

  1425. 	STFD	f3,   3 * SIZE(AO)

  1426. #endif

  1427. 

  1428. 	STFD	f0,   0 * SIZE(CO1)

  1429. 	STFD	f1,   1 * SIZE(CO1)

  1430. 	STFD	f2,   0 * SIZE(CO2)

  1431. 	STFD	f3,   1 * SIZE(CO2)

  1432. 

  1433. #ifndef LN

  1434. 	addi	CO1, CO1, 2 * SIZE

  1435. 	addi	CO2, CO2, 2 * SIZE

  1436. #endif

  1437. 

  1438. #ifdef RT

  1439. 	slwi	r0, K, 0 + ZBASE_SHIFT

  1440. 	add	AORIG, AORIG, r0

  1441. #endif

  1442. 

  1443. #if defined(LT) || defined(RN)

  1444. 	sub	TEMP, K, KK

  1445. 	slwi	r0,   TEMP, 0 + ZBASE_SHIFT

  1446. 	slwi	TEMP, TEMP, 1 + ZBASE_SHIFT

  1447. 	add	AO, AO, r0

  1448. 	add	BO, BO, TEMP

  1449. #endif

  1450. 

  1451. #ifdef LT

  1452. 	addi	KK, KK, 1

  1453. #endif

  1454. 

  1455. #ifdef LN

  1456. 	subi	KK, KK, 1

  1457. #endif

  1458. 	.align 4

  1459. 

  1460. LL(29):

  1461. #ifdef LN

  1462. 	slwi	r0, K, 1 + ZBASE_SHIFT

  1463. 	add	B, B, r0

  1464. #endif

  1465. 

  1466. #if defined(LT) || defined(RN)

  1467. 	mr	B,  BO

  1468. #endif

  1469. 

  1470. #ifdef RN

  1471. 	addi	KK, KK, 2

  1472. #endif

  1473. 

  1474. #ifdef RT

  1475. 	subi	KK, KK, 2

  1476. #endif

  1477. 

  1478. 	addic.	J, J, -1

  1479. 	bgt	LL(10)

  1480. 	.align 4

  1481. 

  1482. LL(30):

  1483. 	andi.	J, N,  1

  1484. 	ble	LL(999)

  1485. 

  1486. #ifdef RT

  1487. 	slwi	r0, K, 0 + ZBASE_SHIFT

  1488. 	sub	B, B, r0

  1489. 

  1490. 	sub	C, C, LDC

  1491. #endif

  1492. 

  1493. 	mr	CO1, C

  1494. 

  1495. #ifdef LN

  1496. 	add	KK, M, OFFSET

  1497. #endif

  1498. 

  1499. #ifdef LT

  1500. 	mr	KK, OFFSET

  1501. #endif

  1502. 

  1503. 	srawi.	I,  M,  1

  1504. #if defined(LN) || defined(RT)

  1505. 	mr	AORIG, A

  1506. #else

  1507. 	mr	AO, A

  1508. #endif

  1509. #ifndef RT

  1510. 	add	C, C, LDC

  1511. #endif

  1512. 	ble	LL(40)

  1513. 	.align 4

  1514. 

  1515. LL(31):

  1516. #if defined(LT) || defined(RN)

  1517. 	LFD	f20,  0 * SIZE(AO)

  1518. 	LFD	f21,  1 * SIZE(AO)

  1519. 	LFD	f22,  2 * SIZE(AO)

  1520. 	LFD	f23,  3 * SIZE(AO)

  1521. 

  1522. 	LFD	f24,  4 * SIZE(AO)

  1523. 	LFD	f25,  5 * SIZE(AO)

  1524. 	LFD	f26,  6 * SIZE(AO)

  1525. 	LFD	f27,  7 * SIZE(AO)

  1526. 

  1527. 	LFD	f16, 0 * SIZE(B)

  1528. 	LFD	f17, 1 * SIZE(B)

  1529. 	LFD	f18, 2 * SIZE(B)

  1530. 	LFD	f19, 3 * SIZE(B)

  1531. 

  1532. 	lfs	f0, FZERO

  1533. 	fmr	f1, f0

  1534. 	fmr	f2, f0

  1535. 	fmr	f3, f0

  1536. 	fmr	f4, f0

  1537. 	fmr	f5, f0

  1538. 	fmr	f6, f0

  1539. 	fmr	f7, f0

  1540. 

  1541. 	dcbt	CO1, PREC

  1542. 

  1543. 	srawi.	r0, KK,  2

  1544. 	mr	BO,  B

  1545. 	mtspr	CTR, r0

  1546. #else

  1547. 

  1548. #ifdef LN

  1549. 	slwi	r0,   K,  1 + ZBASE_SHIFT

  1550. 	sub	AORIG, AORIG, r0

  1551. #endif

  1552. 

  1553. 	slwi	r0,   KK, 1 + ZBASE_SHIFT

  1554. 	slwi	TEMP, KK, 0 + ZBASE_SHIFT

  1555. 

  1556. 	add	AO, AORIG, r0

  1557. 	add	BO, B,     TEMP

  1558. 

  1559. 	sub	TEMP, K, KK

  1560. 

  1561. 	LFD	f20,  0 * SIZE(AO)

  1562. 	LFD	f21,  1 * SIZE(AO)

  1563. 	LFD	f22,  2 * SIZE(AO)

  1564. 	LFD	f23,  3 * SIZE(AO)

  1565. 

  1566. 	LFD	f24,  4 * SIZE(AO)

  1567. 	LFD	f25,  5 * SIZE(AO)

  1568. 	LFD	f26,  6 * SIZE(AO)

  1569. 	LFD	f27,  7 * SIZE(AO)

  1570. 

  1571. 	LFD	f16, 0 * SIZE(BO)

  1572. 	LFD	f17, 1 * SIZE(BO)

  1573. 	LFD	f18, 2 * SIZE(BO)

  1574. 	LFD	f19, 3 * SIZE(BO)

  1575. 

  1576. 	lfs	f0, FZERO

  1577. 	fmr	f1, f0

  1578. 	fmr	f2, f0

  1579. 	fmr	f3, f0

  1580. 	fmr	f4, f0

  1581. 	fmr	f5, f0

  1582. 	fmr	f6, f0

  1583. 	fmr	f7, f0

  1584. 

  1585. 	srawi.	r0, TEMP,  2

  1586. 	mtspr	CTR, r0

  1587. #endif

  1588. 	ble	LL(35)

  1589. 	.align 4

  1590. 

  1591. LL(32):

  1592. 	fmadd	f0,  f16, f20, f0

  1593. 	fmadd	f1,  f16, f21, f1

  1594. 	fmadd	f2,  f16, f22, f2

  1595. 	fmadd	f3,  f16, f23, f3

  1596. 

  1597. 	fmadd	f4,  f17, f20, f4

  1598. 	fmadd	f5,  f17, f21, f5

  1599. 	fmadd	f6,  f17, f22, f6

  1600. 	fmadd	f7,  f17, f23, f7

  1601. 

  1602. 	LFD	f20,  8 * SIZE(AO)

  1603. 	LFD	f21,  9 * SIZE(AO)

  1604. 	LFD	f22, 10 * SIZE(AO)

  1605. 	LFD	f23, 11 * SIZE(AO)

  1606. 

  1607. 	fmadd	f0,  f18, f24, f0

  1608. 	fmadd	f1,  f18, f25, f1

  1609. 	fmadd	f2,  f18, f26, f2

  1610. 	fmadd	f3,  f18, f27, f3

  1611. 

  1612. 	fmadd	f4,  f19, f24, f4

  1613. 	fmadd	f5,  f19, f25, f5

  1614. 	fmadd	f6,  f19, f26, f6

  1615. 	fmadd	f7,  f19, f27, f7

  1616. 

  1617. 	LFD	f24, 12 * SIZE(AO)

  1618. 	LFD	f25, 13 * SIZE(AO)

  1619. 	LFD	f26, 14 * SIZE(AO)

  1620. 	LFD	f27, 15 * SIZE(AO)

  1621. 

  1622. 	LFD	f16,  4 * SIZE(BO)

  1623. 	LFD	f17,  5 * SIZE(BO)

  1624. 	LFD	f18,  6 * SIZE(BO)

  1625. 	LFD	f19,  7 * SIZE(BO)

  1626. 

  1627. 	fmadd	f0,  f16, f20, f0

  1628. 	fmadd	f1,  f16, f21, f1

  1629. 	fmadd	f2,  f16, f22, f2

  1630. 	fmadd	f3,  f16, f23, f3

  1631. 

  1632. 	fmadd	f4,  f17, f20, f4

  1633. 	fmadd	f5,  f17, f21, f5

  1634. 	fmadd	f6,  f17, f22, f6

  1635. 	fmadd	f7,  f17, f23, f7

  1636. 

  1637. 	LFD	f20, 16 * SIZE(AO)

  1638. 	LFD	f21, 17 * SIZE(AO)

  1639. 	LFD	f22, 18 * SIZE(AO)

  1640. 	LFD	f23, 19 * SIZE(AO)

  1641. 

  1642. 	fmadd	f0,  f18, f24, f0

  1643. 	fmadd	f1,  f18, f25, f1

  1644. 	fmadd	f2,  f18, f26, f2

  1645. 	fmadd	f3,  f18, f27, f3

  1646. 

  1647. 	fmadd	f4,  f19, f24, f4

  1648. 	fmadd	f5,  f19, f25, f5

  1649. 	fmadd	f6,  f19, f26, f6

  1650. 	fmadd	f7,  f19, f27, f7

  1651. 

  1652. 	LFD	f24, 20 * SIZE(AO)

  1653. 	LFD	f25, 21 * SIZE(AO)

  1654. 	LFD	f26, 22 * SIZE(AO)

  1655. 	LFD	f27, 23 * SIZE(AO)

  1656. 

  1657. 	LFD	f16,  8 * SIZE(BO)

  1658. 	LFD	f17,  9 * SIZE(BO)

  1659. 	LFD	f18, 10 * SIZE(BO)

  1660. 	LFD	f19, 11 * SIZE(BO)

  1661. 

  1662. 	addi	AO, AO, 16 * SIZE

  1663. 	addi	BO, BO,  8 * SIZE

  1664.  	dcbt	PREA, AO

  1665.  	dcbt	PREA, BO

  1666. 	bdnz	LL(32)

  1667. 	.align 4

  1668. 

  1669. LL(35):

  1670. #if defined(LT) || defined(RN)

  1671. 	andi.	r0, KK,  3

  1672. #else

  1673. 	andi.	r0, TEMP, 3

  1674. #endif

  1675. 	mtspr	CTR, r0

  1676. 	ble	LL(37)

  1677. 	.align 4

  1678. 

  1679. LL(36):

  1680. 	fmadd	f0,  f16, f20, f0

  1681. 	fmadd	f1,  f16, f21, f1

  1682. 	fmadd	f2,  f16, f22, f2

  1683. 	fmadd	f3,  f16, f23, f3

  1684. 

  1685. 	fmadd	f4,  f17, f20, f4

  1686. 	fmadd	f5,  f17, f21, f5

  1687. 	fmadd	f6,  f17, f22, f6

  1688. 	fmadd	f7,  f17, f23, f7

  1689. 

  1690. 	LFD	f20, 4 * SIZE(AO)

  1691. 	LFD	f21, 5 * SIZE(AO)

  1692. 	LFD	f22, 6 * SIZE(AO)

  1693. 	LFD	f23, 7 * SIZE(AO)

  1694. 

  1695. 	LFD	f16, 2 * SIZE(BO)

  1696. 	LFD	f17, 3 * SIZE(BO)

  1697. 

  1698. 	addi	BO,  BO, 2 * SIZE

  1699. 	addi	AO,  AO, 4 * SIZE

  1700. 	bdnz	LL(36)

  1701. 	.align 4

  1702. 

  1703. LL(37):

  1704. #ifndef CONJ

  1705. 	FSUB	  f0,  f0,  f5

  1706. 	FADD	  f1,  f1,  f4

  1707. 	FSUB	  f2,  f2,  f7

  1708. 	FADD	  f3,  f3,  f6

  1709. #else

  1710. 	FADD	  f0,  f0,  f5

  1711. 	FSUB	  f1,  f4,  f1

  1712. 	FADD	  f2,  f2,  f7

  1713. 	FSUB	  f3,  f6,  f3

  1714. #endif

  1715. 

  1716. #if defined(LN) || defined(RT)

  1717. #ifdef LN

  1718. 	subi	r0, KK, 2

  1719. #else

  1720. 	subi	r0, KK, 1

  1721. #endif

  1722. 	slwi	TEMP, r0, 1 + ZBASE_SHIFT

  1723. 	slwi	r0,   r0, 0 + ZBASE_SHIFT

  1724. 

  1725. 	add	AO, AORIG, TEMP

  1726. 	add	BO, B,     r0

  1727. #endif

  1728. 

  1729. #if defined(LN) || defined(LT)

  1730. 	LFD	f16,  0 * SIZE(BO)

  1731. 	LFD	f17,  1 * SIZE(BO)

  1732. 	LFD	f18,  2 * SIZE(BO)

  1733. 	LFD	f19,  3 * SIZE(BO)

  1734. 

  1735. 	FSUB	f0,  f16, f0

  1736. 	FSUB	f1,  f17, f1

  1737. 	FSUB	f2,  f18, f2

  1738. 	FSUB	f3,  f19, f3

  1739. #else

  1740. 	LFD	f16,  0 * SIZE(AO)

  1741. 	LFD	f17,  1 * SIZE(AO)

  1742. 	LFD	f18,  2 * SIZE(AO)

  1743. 	LFD	f19,  3 * SIZE(AO)

  1744. 

  1745. #ifndef CONJ

  1746. 	FSUB	f0,  f16, f0

  1747. 	FSUB	f1,  f17, f1

  1748. 	FSUB	f2,  f18, f2

  1749. 	FSUB	f3,  f19, f3

  1750. #else

  1751. 	FSUB	f0,  f16, f0

  1752. 	FADD	f1,  f17, f1

  1753. 	FSUB	f2,  f18, f2

  1754. 	FADD	f3,  f19, f3

  1755. #endif

  1756. #endif

  1757. 

  1758. #ifdef LN

  1759. 	LFD	f16,  6 * SIZE(AO)

  1760. 	LFD	f17,  7 * SIZE(AO)

  1761. 	LFD	f18,  4 * SIZE(AO)

  1762. 	LFD	f19,  5 * SIZE(AO)

  1763. 	LFD	f20,  0 * SIZE(AO)

  1764. 	LFD	f21,  1 * SIZE(AO)

  1765. 

  1766. 	FMUL	f6,  f17, f3

  1767. 	FMUL	f7,  f17, f2

  1768. 

  1769. #ifndef CONJ

  1770. 	FMSUB	f2,  f16, f2,  f6

  1771. 	FMADD	f3,  f16, f3,  f7

  1772. 

  1773. 	FMADD	f0,  f19, f3,  f0

  1774. 	FNMSUB	f1,  f19, f2,  f1

  1775. 	FNMSUB	f0,  f18, f2,  f0

  1776. 	FNMSUB	f1,  f18, f3,  f1

  1777. 

  1778. 	FMUL	f4,  f21, f1

  1779. 	FMUL	f5,  f21, f0

  1780. 	FMSUB	f0,  f20, f0,  f4

  1781. 	FMADD	f1,  f20, f1,  f5

  1782. #else

  1783. 	FMADD	f2,  f16, f2,  f6

  1784. 	FMSUB	f3,  f16, f3,  f7

  1785. 

  1786. 	FMSUB	f0,  f19, f3,  f0

  1787. 	FNMADD	f1,  f19, f2,  f1

  1788. 	FNMADD	f0,  f18, f2,  f0

  1789. 	FNMADD	f1,  f18, f3,  f1

  1790. 

  1791. 	FMUL	f4,  f21, f1

  1792. 	FMUL	f5,  f21, f0

  1793. 	FMADD	f0,  f20, f0,  f4

  1794. 	FMSUB	f1,  f20, f1,  f5

  1795. #endif

  1796. #endif

  1797. 

  1798. #ifdef LT

  1799. 	LFD	f16,  0 * SIZE(AO)

  1800. 	LFD	f17,  1 * SIZE(AO)

  1801. 	LFD	f18,  2 * SIZE(AO)

  1802. 	LFD	f19,  3 * SIZE(AO)

  1803. 	LFD	f20,  6 * SIZE(AO)

  1804. 	LFD	f21,  7 * SIZE(AO)

  1805. 

  1806. 	FMUL	f4,  f17, f1

  1807. 	FMUL	f5,  f17, f0

  1808. 

  1809. #ifndef CONJ

  1810. 	FMSUB	f0,  f16, f0,  f4

  1811. 	FMADD	f1,  f16, f1,  f5

  1812. 

  1813. 	FMADD	f2,  f19, f1, f2

  1814. 	FNMSUB	f3,  f19, f0, f3

  1815. 	FNMSUB	f2,  f18, f0,  f2

  1816. 	FNMSUB	f3,  f18, f1,  f3

  1817. 

  1818. 	FMUL	f4,  f21, f3

  1819. 	FMUL	f5,  f21, f2

  1820. 	FMSUB	f2,  f20, f2,  f4

  1821. 	FMADD	f3,  f20, f3,  f5

  1822. #else

  1823. 	FMADD	f0,  f16, f0,  f4

  1824. 	FMSUB	f1,  f16, f1,  f5

  1825. 

  1826. 	FMSUB	f2,  f19, f1, f2

  1827. 	FNMADD	f3,  f19, f0, f3

  1828. 	FNMADD	f2,  f18, f0,  f2

  1829. 	FNMADD	f3,  f18, f1,  f3

  1830. 

  1831. 	FMUL	f4,  f21, f3

  1832. 	FMUL	f5,  f21, f2

  1833. 	FMADD	f2,  f20, f2,  f4

  1834. 	FMSUB	f3,  f20, f3,  f5

  1835. #endif

  1836. #endif

  1837. 

  1838. #ifdef RN

  1839. 	LFD	f16,  0 * SIZE(BO)

  1840. 	LFD	f17,  1 * SIZE(BO)

  1841. 

  1842. 	FMUL	f4,  f17, f1

  1843. 	FMUL	f5,  f17, f0

  1844. 	FMUL	f6,  f17, f3

  1845. 	FMUL	f7,  f17, f2

  1846. 

  1847. #ifndef CONJ

  1848. 	FMSUB	f0,  f16, f0,  f4

  1849. 	FMADD	f1,  f16, f1,  f5

  1850. 	FMSUB	f2,  f16, f2,  f6

  1851. 	FMADD	f3,  f16, f3,  f7

  1852. #else

  1853. 	FMADD	f0,  f16, f0,  f4

  1854. 	FMSUB	f1,  f16, f1,  f5

  1855. 	FMADD	f2,  f16, f2,  f6

  1856. 	FMSUB	f3,  f16, f3,  f7

  1857. #endif

  1858. #endif

  1859. 

  1860. #ifdef RT

  1861. 	LFD	f20,  0 * SIZE(BO)

  1862. 	LFD	f21,  1 * SIZE(BO)

  1863. 

  1864. 	FMUL	f4,  f21, f1

  1865. 	FMUL	f5,  f21, f0

  1866. 	FMUL	f6,  f21, f3

  1867. 	FMUL	f7,  f21, f2

  1868. 

  1869. #ifndef CONJ

  1870. 	FMSUB	f0,  f20, f0,  f4

  1871. 	FMADD	f1,  f20, f1,  f5

  1872. 	FMSUB	f2,  f20, f2,  f6

  1873. 	FMADD	f3,  f20, f3,  f7

  1874. 

  1875. #else

  1876. 	FMADD	f0,  f20, f0,  f4

  1877. 	FMSUB	f1,  f20, f1,  f5

  1878. 	FMADD	f2,  f20, f2,  f6

  1879. 	FMSUB	f3,  f20, f3,  f7

  1880. #endif

  1881. #endif

  1882. 

  1883. #ifdef LN

  1884. 	subi	CO1, CO1, 4 * SIZE

  1885. #endif

  1886. 

  1887. #if defined(LN) || defined(LT)

  1888. 	STFD	f0,   0 * SIZE(BO)

  1889. 	STFD	f1,   1 * SIZE(BO)

  1890. 	STFD	f2,   2 * SIZE(BO)

  1891. 	STFD	f3,   3 * SIZE(BO)

  1892. #else

  1893. 	STFD	f0,   0 * SIZE(AO)

  1894. 	STFD	f1,   1 * SIZE(AO)

  1895. 	STFD	f2,   2 * SIZE(AO)

  1896. 	STFD	f3,   3 * SIZE(AO)

  1897. #endif

  1898. 

  1899. 	STFD	f0,   0 * SIZE(CO1)

  1900. 	STFD	f1,   1 * SIZE(CO1)

  1901. 	STFD	f2,   2 * SIZE(CO1)

  1902. 	STFD	f3,   3 * SIZE(CO1)

  1903. 

  1904. #ifndef LN

  1905. 	addi	CO1, CO1, 4 * SIZE

  1906. #endif

  1907. 

  1908. #ifdef RT

  1909. 	slwi	r0, K, 1 + ZBASE_SHIFT

  1910. 	add	AORIG, AORIG, r0

  1911. #endif

  1912. 

  1913. #if defined(LT) || defined(RN)

  1914. 	sub	TEMP, K, KK

  1915. 	slwi	r0,   TEMP, 1 + ZBASE_SHIFT

  1916. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  1917. 	add	AO, AO, r0

  1918. 	add	BO, BO, TEMP

  1919. #endif

  1920. 

  1921. #ifdef LT

  1922. 	addi	KK, KK, 2

  1923. #endif

  1924. 

  1925. #ifdef LN

  1926. 	subi	KK, KK, 2

  1927. #endif

  1928. 

  1929. 	addic.	I, I, -1

  1930. 	bgt	LL(31)

  1931. 	.align 4

  1932. 

  1933. LL(40):

  1934. 	andi.	I,  M,  1

  1935. 	ble	LL(49)

  1936. 

  1937. #if defined(LT) || defined(RN)

  1938. 	LFD	f16,  0 * SIZE(AO)

  1939. 	LFD	f17,  1 * SIZE(AO)

  1940. 	LFD	f18,  2 * SIZE(AO)

  1941. 	LFD	f19,  3 * SIZE(AO)

  1942. 

  1943. 	LFD	f20,  0 * SIZE(B)

  1944. 	LFD	f21,  1 * SIZE(B)

  1945. 	LFD	f22,  2 * SIZE(B)

  1946. 	LFD	f23,  3 * SIZE(B)

  1947. 

  1948. 	lfs	f0, FZERO

  1949. 	fmr	f1, f0

  1950. 	fmr	f2, f0

  1951. 	fmr	f3, f0

  1952. 	fmr	f4, f0

  1953. 	fmr	f5, f0

  1954. 	fmr	f6, f0

  1955. 	fmr	f7, f0

  1956. 

  1957. 	srawi.	r0, KK,  2

  1958. 	mr	BO,  B

  1959. 	mtspr	CTR, r0

  1960. #else

  1961. 

  1962. #ifdef LN

  1963. 	slwi	r0,   K,  0 + ZBASE_SHIFT

  1964. 	sub	AORIG, AORIG, r0

  1965. #endif

  1966. 

  1967. 	slwi	r0,   KK, 0 + ZBASE_SHIFT

  1968. 

  1969. 	add	AO, AORIG, r0

  1970. 	add	BO, B,     r0

  1971. 

  1972. 	sub	TEMP, K, KK

  1973. 

  1974. 	LFD	f16,  0 * SIZE(AO)

  1975. 	LFD	f17,  1 * SIZE(AO)

  1976. 	LFD	f18,  2 * SIZE(AO)

  1977. 	LFD	f19,  3 * SIZE(AO)

  1978. 

  1979. 	LFD	f20,  0 * SIZE(BO)

  1980. 	LFD	f21,  1 * SIZE(BO)

  1981. 	LFD	f22,  2 * SIZE(BO)

  1982. 	LFD	f23,  3 * SIZE(BO)

  1983. 

  1984. 	lfs	f0, FZERO

  1985. 	fmr	f1, f0

  1986. 	fmr	f2, f0

  1987. 	fmr	f3, f0

  1988. 	fmr	f4, f0

  1989. 	fmr	f5, f0

  1990. 	fmr	f6, f0

  1991. 	fmr	f7, f0

  1992. 

  1993. 	srawi.	r0, TEMP,  2

  1994. 	mtspr	CTR, r0

  1995. #endif

  1996. 	ble	LL(45)

  1997. 	.align 4

  1998. 

  1999. LL(42):

  2000. 	fmadd	f0,  f16,  f20,  f0

  2001. 	fmadd	f1,  f17,  f21,  f1

  2002. 	fmadd	f2,  f17,  f20,  f2

  2003. 	fmadd	f3,  f16,  f21,  f3

  2004. 

  2005. 	LFD	f16,  4 * SIZE(AO)

  2006. 	LFD	f17,  5 * SIZE(AO)

  2007. 	LFD	f20,  4 * SIZE(BO)

  2008. 	LFD	f21,  5 * SIZE(BO)

  2009. 

  2010. 	fmadd	f4,  f18,  f22,  f4

  2011. 	fmadd	f5,  f19,  f23,  f5

  2012. 	fmadd	f6,  f19,  f22,  f6

  2013. 	fmadd	f7,  f18,  f23,  f7

  2014. 

  2015. 	LFD	f18,  6 * SIZE(AO)

  2016. 	LFD	f19,  7 * SIZE(AO)

  2017. 	LFD	f22,  6 * SIZE(BO)

  2018. 	LFD	f23,  7 * SIZE(BO)

  2019. 

  2020. 	fmadd	f0,  f16,  f20,  f0

  2021. 	fmadd	f1,  f17,  f21,  f1

  2022. 	fmadd	f2,  f17,  f20,  f2

  2023. 	fmadd	f3,  f16,  f21,  f3

  2024. 

  2025. 	LFD	f16,  8 * SIZE(AO)

  2026. 	LFD	f17,  9 * SIZE(AO)

  2027. 	LFD	f20,  8 * SIZE(BO)

  2028. 	LFD	f21,  9 * SIZE(BO)

  2029. 

  2030. 	fmadd	f4,  f18,  f22,  f4

  2031. 	fmadd	f5,  f19,  f23,  f5

  2032. 	fmadd	f6,  f19,  f22,  f6

  2033. 	fmadd	f7,  f18,  f23,  f7

  2034. 

  2035. 	LFD	f18, 10 * SIZE(AO)

  2036. 	LFD	f19, 11 * SIZE(AO)

  2037. 	LFD	f22, 10 * SIZE(BO)

  2038. 	LFD	f23, 11 * SIZE(BO)

  2039. 

  2040. 	addi	AO, AO,  8 * SIZE

  2041. 	addi	BO, BO,  8 * SIZE

  2042. 	bdnz	LL(42)

  2043. 	.align 4

  2044. 

  2045. LL(45):

  2046. 	fadd	f0, f0, f4

  2047. 	fadd	f1, f1, f5

  2048. 	fadd	f2, f2, f6

  2049. 	fadd	f3, f3, f7

  2050. 

  2051. #if defined(LT) || defined(RN)

  2052. 	andi.	r0, KK,  3

  2053. #else

  2054. 	andi.	r0, TEMP, 3

  2055. #endif

  2056. 	mtspr	CTR,r0

  2057. 	ble	LL(47)

  2058. 	.align 4

  2059. 

  2060. LL(46):

  2061. 	fmadd	f0,  f16,  f20,  f0

  2062. 	fmadd	f1,  f17,  f21,  f1

  2063. 	fmadd	f2,  f17,  f20,  f2

  2064. 	fmadd	f3,  f16,  f21,  f3

  2065. 

  2066. 	LFD	f16, 2 * SIZE(AO)

  2067. 	LFD	f17, 3 * SIZE(AO)

  2068. 	LFD	f20, 2 * SIZE(BO)

  2069. 	LFD	f21, 3 * SIZE(BO)

  2070. 

  2071. 	addi	AO, AO, 2 * SIZE

  2072. 	addi	BO, BO, 2 * SIZE

  2073. 

  2074. 	bdnz	LL(46)

  2075. 	.align 4

  2076. 

  2077. LL(47):

  2078. #ifndef CONJ

  2079. 	FSUB	  f0,  f0,  f1

  2080. 	FADD	  f1,  f2,  f3

  2081. #else

  2082. 	FADD	  f0,  f0,  f1

  2083. 	FSUB	  f1,  f3,  f2

  2084. #endif

  2085. 

  2086. #if defined(LN) || defined(RT)

  2087. 	subi	r0, KK, 1

  2088. 	slwi	r0,   r0, 0 + ZBASE_SHIFT

  2089. 

  2090. 	add	AO, AORIG, r0

  2091. 	add	BO, B,     r0

  2092. #endif

  2093. 

  2094. #if defined(LN) || defined(LT)

  2095. 	LFD	f16,  0 * SIZE(BO)

  2096. 	LFD	f17,  1 * SIZE(BO)

  2097. 	FSUB	f0,  f16, f0

  2098. 	FSUB	f1,  f17, f1

  2099. #else

  2100. 	LFD	f16,  0 * SIZE(AO)

  2101. 	LFD	f17,  1 * SIZE(AO)

  2102. #ifndef CONJ

  2103. 	FSUB	f0,  f16, f0

  2104. 	FSUB	f1,  f17, f1

  2105. #else

  2106. 	FSUB	f0,  f16, f0

  2107. 	FADD	f1,  f17, f1

  2108. #endif

  2109. #endif

  2110. 

  2111. #ifdef LN

  2112. 	LFD	f20,  0 * SIZE(AO)

  2113. 	LFD	f21,  1 * SIZE(AO)

  2114. 

  2115. 	FMUL	f4,  f21, f1

  2116. 	FMUL	f5,  f21, f0

  2117. #ifndef CONJ

  2118. 	FMSUB	f0,  f20, f0,  f4

  2119. 	FMADD	f1,  f20, f1,  f5

  2120. #else

  2121. 	FMADD	f0,  f20, f0,  f4

  2122. 	FMSUB	f1,  f20, f1,  f5

  2123. #endif

  2124. #endif

  2125. 

  2126. #ifdef LT

  2127. 	LFD	f16,  0 * SIZE(AO)

  2128. 	LFD	f17,  1 * SIZE(AO)

  2129. 

  2130. 	FMUL	f4,  f17, f1

  2131. 	FMUL	f5,  f17, f0

  2132. 

  2133. #ifndef CONJ

  2134. 	FMSUB	f0,  f16, f0,  f4

  2135. 	FMADD	f1,  f16, f1,  f5

  2136. #else

  2137. 	FMADD	f0,  f16, f0,  f4

  2138. 	FMSUB	f1,  f16, f1,  f5

  2139. #endif

  2140. #endif

  2141. 

  2142. #ifdef RN

  2143. 	LFD	f16,  0 * SIZE(BO)

  2144. 	LFD	f17,  1 * SIZE(BO)

  2145. 

  2146. 	FMUL	f4,  f17, f1

  2147. 	FMUL	f5,  f17, f0

  2148. #ifndef CONJ

  2149. 	FMSUB	f0,  f16, f0,  f4

  2150. 	FMADD	f1,  f16, f1,  f5

  2151. #else

  2152. 	FMADD	f0,  f16, f0,  f4

  2153. 	FMSUB	f1,  f16, f1,  f5

  2154. #endif

  2155. #endif

  2156. 

  2157. #ifdef RT

  2158. 	LFD	f20,  0 * SIZE(BO)

  2159. 	LFD	f21,  1 * SIZE(BO)

  2160. 

  2161. 	FMUL	f4,  f21, f1

  2162. 	FMUL	f5,  f21, f0

  2163. #ifndef CONJ

  2164. 	FMSUB	f0,  f20, f0,  f4

  2165. 	FMADD	f1,  f20, f1,  f5

  2166. #else

  2167. 	FMADD	f0,  f20, f0,  f4

  2168. 	FMSUB	f1,  f20, f1,  f5

  2169. #endif

  2170. #endif

  2171. 

  2172. #ifdef LN

  2173. 	subi	CO1, CO1, 2 * SIZE

  2174. #endif

  2175. 

  2176. #if defined(LN) || defined(LT)

  2177. 	STFD	f0,   0 * SIZE(BO)

  2178. 	STFD	f1,   1 * SIZE(BO)

  2179. #else

  2180. 	STFD	f0,   0 * SIZE(AO)

  2181. 	STFD	f1,   1 * SIZE(AO)

  2182. #endif

  2183. 

  2184. 	STFD	f0,   0 * SIZE(CO1)

  2185. 	STFD	f1,   1 * SIZE(CO1)

  2186. 

  2187. #ifndef LN

  2188. 	addi	CO1, CO1, 2 * SIZE

  2189. #endif

  2190. 

  2191. #ifdef RT

  2192. 	slwi	r0, K, 0 + ZBASE_SHIFT

  2193. 	add	AORIG, AORIG, r0

  2194. #endif

  2195. 

  2196. #if defined(LT) || defined(RN)

  2197. 	sub	TEMP, K, KK

  2198. 	slwi	TEMP, TEMP, 0 + ZBASE_SHIFT

  2199. 	add	AO, AO, TEMP

  2200. 	add	BO, BO, TEMP

  2201. #endif

  2202. 

  2203. #ifdef LT

  2204. 	addi	KK, KK, 1

  2205. #endif

  2206. 

  2207. #ifdef LN

  2208. 	subi	KK, KK, 1

  2209. #endif

  2210. 	.align 4

  2211. 

  2212. LL(49):

  2213. #ifdef LN

  2214. 	slwi	r0, K, 0 + ZBASE_SHIFT

  2215. 	add	B, B, r0

  2216. #endif

  2217. 

  2218. #if defined(LT) || defined(RN)

  2219. 	mr	B,  BO

  2220. #endif

  2221. 

  2222. #ifdef RN

  2223. 	addi	KK, KK, 1

  2224. #endif

  2225. 

  2226. #ifdef RT

  2227. 	subi	KK, KK, 1

  2228. #endif

  2229. 	.align 4

  2230. 

  2231. LL(999):

  2232. 	addi	r3, 0, 0

  2233. 

  2234. 	lfd	f14,    0(SP)

  2235. 	lfd	f15,    8(SP)

  2236. 	lfd	f16,   16(SP)

  2237. 	lfd	f17,   24(SP)

  2238. 

  2239. 	lfd	f18,   32(SP)

  2240. 	lfd	f19,   40(SP)

  2241. 	lfd	f20,   48(SP)

  2242. 	lfd	f21,   56(SP)

  2243. 

  2244. 	lfd	f22,   64(SP)

  2245. 	lfd	f23,   72(SP)

  2246. 	lfd	f24,   80(SP)

  2247. 	lfd	f25,   88(SP)

  2248. 

  2249. 	lfd	f26,   96(SP)

  2250. 	lfd	f27,  104(SP)

  2251. 	lfd	f28,  112(SP)

  2252. 	lfd	f29,  120(SP)

  2253. 

  2254. 	lfd	f30,  128(SP)

  2255. 	lfd	f31,  136(SP)

  2256. 

  2257. #ifdef __64BIT__

  2258. 	ld	r31,  144(SP)

  2259. 	ld	r30,  152(SP)

  2260. 	ld	r29,  160(SP)

  2261. 	ld	r28,  168(SP)

  2262. 	ld	r27,  176(SP)

  2263. 	ld	r26,  184(SP)

  2264. 	ld	r25,  192(SP)

  2265. 	ld	r24,  200(SP)

  2266. 	ld	r23,  208(SP)

  2267. 	ld	r22,  216(SP)

  2268. 	ld	r21,  224(SP)

  2269. #else

  2270. 	lwz	r31,  144(SP)

  2271. 	lwz	r30,  148(SP)

  2272. 	lwz	r29,  152(SP)

  2273. 	lwz	r28,  156(SP)

  2274. 	lwz	r27,  160(SP)

  2275. 	lwz	r26,  164(SP)

  2276. 	lwz	r25,  168(SP)

  2277. 	lwz	r24,  172(SP)

  2278. 	lwz	r23,  176(SP)

  2279. 	lwz	r22,  180(SP)

  2280. 	lwz	r21,  184(SP)

  2281. #endif

  2282. 

  2283. 	addi	SP, SP, STACKSIZE

  2284. 

  2285. 	blr

  2286. 

  2287. 	EPILOGUE

  2288. #endif