OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/gemm_ncopy_hummer_4.S

799 lines
14 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define	M	r3

  43. #define	N	r4

  44. #define	A	r5

  45. #define	LDA	r6

  46. #define B	r7

  47. 

  48. #define AO1	r8

  49. #define AO2	r9

  50. #define AO3	r10

  51. #define AO4	r11

  52. 

  53. #define J	r12

  54. 

  55. #define INC	r30

  56. #define INC2	r31

  57. 

  58. #define c01	f0

  59. #define c02	f1

  60. #define c03	f2

  61. #define c04	f3

  62. #define c05	f4

  63. #define c06	f5

  64. #define c07	f6

  65. #define c08	f7

  66. #define c09	f8

  67. #define c10	f9

  68. #define c11	f10

  69. #define c12	f11

  70. #define c13	f12

  71. #define c14	f13

  72. #define c15	f14

  73. #define c16	f15

  74. 

  75. #define	sel_p	f16

  76. #define	sel_s	f17

  77. 

  78. #define c17	f18

  79. #define c18	f19

  80. 

  81. 

  82. 	PROLOGUE

  83. 	PROFCODE

  84. 

  85. 	li	r0, -16

  86. 

  87. 	stfpdux	f14, SP, r0

  88. 	stfpdux	f15, SP, r0

  89. 

  90. 	stfpdux	f16, SP, r0

  91. 	stfpdux	f17, SP, r0

  92. 	stfpdux	f18, SP, r0

  93. 	stfpdux	f19, SP, r0

  94. 

  95. 	stwu	r31,  -4(SP)

  96. 	stwu	r30,  -4(SP)

  97. 

  98. 	lis	r9,   0x3f80

  99. 	lis	r10,  0xbf80

  100. 

  101. 	stwu	r9,    -4(SP)

  102. 	stwu	r10,   -4(SP)

  103. 	stwu	r10,   -4(SP)

  104. 	stwu	r9,    -4(SP)

  105. 

  106. 	slwi	LDA, LDA, BASE_SHIFT

  107. 

  108. 	cmpwi	cr0, M, 0

  109. 	ble-	.L99

  110. 	cmpwi	cr0, N, 0

  111. 	ble-	.L99

  112. 

  113. 	andi.	r0, A,   2 * SIZE - 1

  114. 	bne	.L100

  115. 	andi.	r0, LDA, 2 * SIZE - 1

  116. 	bne	.L100

  117. 

  118. 	li	r0, 8

  119. 	addi	SP, SP, -8

  120. 

  121. 	lfpsux	sel_p, SP, r0

  122. 	lfpsux	sel_s, SP, r0

  123. 

  124. 	li	INC,  1 * SIZE

  125. 	li	INC2, 2 * SIZE

  126. 

  127. 	subi	A, A, 2 * SIZE

  128. 	subi	B, B, 2 * SIZE

  129. 

  130. 	srawi.	J,  N,  2

  131. 	ble	.L20

  132. 	.align 4

  133. .L11:

  134. 	mr	AO1, A

  135. 	add	AO2, A,   LDA

  136. 	add	AO3, AO2, LDA

  137. 	add	AO4, AO3, LDA

  138. 	add	A,   AO4, LDA

  139. 

  140. 	srawi.	r0,  M,  3

  141. 	mtspr	CTR, r0

  142. 	ble	.L15

  143. 	.align 4

  144. 

  145. .L12:

  146. 	LFPDUX	c01,   AO1, INC2

  147. 	LFXDUX	c05,   AO2, INC2

  148. 	LFPDUX	c09,   AO3, INC2

  149. 	LFXDUX	c13,   AO4, INC2

  150. 

  151. 	LFPDUX	c02,   AO1, INC2

  152. 	LFXDUX	c06,   AO2, INC2

  153. 	LFPDUX	c10,   AO3, INC2

  154. 	LFXDUX	c14,   AO4, INC2

  155. 

  156. 	LFPDUX	c03,   AO1, INC2

  157. 	LFXDUX	c07,   AO2, INC2

  158. 	LFPDUX	c11,   AO3, INC2

  159. 	LFXDUX	c15,   AO4, INC2

  160. 

  161. 	LFPDUX	c04,   AO1, INC2

  162. 	LFXDUX	c08,   AO2, INC2

  163. 	LFPDUX	c12,   AO3, INC2

  164. 	LFXDUX	c16,   AO4, INC2

  165. 

  166. 	fpsel	c17, sel_p, c01, c05

  167. 	fpsel	c18, sel_p, c09, c13

  168. 	fpsel	c01, sel_s, c01, c05

  169. 	fpsel	c05, sel_s, c09, c13

  170. 

  171. 	fpsel	c09, sel_p, c02, c06

  172. 	fpsel	c13, sel_p, c10, c14

  173. 	STFPDUX	c17,   B, INC2

  174. 	fpsel	c02, sel_s, c02, c06

  175.  	STFPDUX	c18,   B, INC2

  176. 	fpsel	c06, sel_s, c10, c14

  177. 	STFXDUX	c01,   B, INC2

  178. 

  179. 	fpsel	c10, sel_p, c03, c07

  180. 	STFXDUX	c05,   B, INC2

  181. 	fpsel	c14, sel_p, c11, c15

  182. 	STFPDUX	c09,   B, INC2

  183. 	fpsel	c03, sel_s, c03, c07

  184. 	STFPDUX	c13,   B, INC2

  185. 	fpsel	c07, sel_s, c11, c15

  186. 	STFXDUX	c02,   B, INC2

  187. 

  188. 	fpsel	c11, sel_p, c04, c08

  189. 	STFXDUX	c06,   B, INC2

  190. 	fpsel	c15, sel_p, c12, c16

  191. 	STFPDUX	c10,   B, INC2

  192. 	fpsel	c04, sel_s, c04, c08

  193. 	STFPDUX	c14,   B, INC2

  194. 	fpsel	c08, sel_s, c12, c16

  195. 	STFXDUX	c03,   B, INC2

  196. 

  197. 	STFXDUX	c07,   B, INC2

  198. 	STFPDUX	c11,   B, INC2

  199. 	STFPDUX	c15,   B, INC2

  200. 	STFXDUX	c04,   B, INC2

  201. 	STFXDUX	c08,   B, INC2

  202. 	bdnz	.L12

  203. 	.align 4

  204. 

  205. .L15:

  206. 	andi.	r0,  M,  7

  207. 	ble	.L19

  208. 

  209. 	andi.	r0,  M,  4

  210. 	beq	.L16

  211. 

  212. 	LFPDUX	c01,   AO1, INC2

  213. 	LFXDUX	c05,   AO2, INC2

  214. 	LFPDUX	c09,   AO3, INC2

  215. 	LFXDUX	c13,   AO4, INC2

  216. 

  217. 	LFPDUX	c02,   AO1, INC2

  218. 	LFXDUX	c06,   AO2, INC2

  219. 	LFPDUX	c10,   AO3, INC2

  220. 	LFXDUX	c14,   AO4, INC2

  221. 

  222. 	fpsel	c17, sel_p, c01, c05

  223. 	fpsel	c18, sel_p, c09, c13

  224. 	fpsel	c01, sel_s, c01, c05

  225. 	fpsel	c05, sel_s, c09, c13

  226. 

  227. 	fpsel	c09, sel_p, c02, c06

  228. 	fpsel	c13, sel_p, c10, c14

  229. 	STFPDUX	c17,   B, INC2

  230. 	fpsel	c02, sel_s, c02, c06

  231. 	STFPDUX	c18,   B, INC2

  232. 	fpsel	c06, sel_s, c10, c14

  233. 	STFXDUX	c01,   B, INC2

  234. 	STFXDUX	c05,   B, INC2

  235. 	STFPDUX	c09,   B, INC2

  236. 	STFPDUX	c13,   B, INC2

  237. 	STFXDUX	c02,   B, INC2

  238. 	STFXDUX	c06,   B, INC2

  239. 	.align 4

  240. 

  241. .L16:

  242. 	andi.	r0,  M,  2

  243. 	beq	.L17

  244. 

  245. 	LFPDUX	c01,   AO1, INC2

  246. 	LFXDUX	c05,   AO2, INC2

  247. 	LFPDUX	c09,   AO3, INC2

  248. 	LFXDUX	c13,   AO4, INC2

  249. 

  250. 	fpsel	c17, sel_p, c01, c05

  251. 	fpsel	c18, sel_p, c09, c13

  252. 	fpsel	c01, sel_s, c01, c05

  253. 	fpsel	c05, sel_s, c09, c13

  254. 

  255. 	STFPDUX	c17,   B, INC2

  256. 	STFPDUX	c18,   B, INC2

  257. 	STFXDUX	c01,   B, INC2

  258. 	STFXDUX	c05,   B, INC2

  259. 	.align 4

  260. 

  261. .L17:

  262. 	andi.	r0,  M,  1

  263. 	beq	.L19

  264. 

  265. 	LFDUX	c01,   AO1, INC2

  266. 	LFDUX	c02,   AO2, INC2

  267. 	LFDUX	c03,   AO3, INC2

  268. 	LFDUX	c04,   AO4, INC2

  269. 

  270. 	fsmfp	c01, c02

  271. 	fsmfp	c03, c04

  272. 

  273. 	STFPDUX	c01,   B, INC2

  274. 	STFPDUX	c03,   B, INC2

  275. 	.align 4

  276. 

  277. .L19:

  278. 	addic.	J, J, -1

  279. 	bgt	.L11

  280. 	.align 4

  281. 

  282. .L20:

  283. 	andi.	J,  N,  2

  284. 	ble	.L30

  285. 

  286. 	mr	AO1, A

  287. 	add	AO2, A,   LDA

  288. 	add	A,   AO2, LDA

  289. 

  290. 	srawi.	r0,  M,  3

  291. 	mtspr	CTR, r0

  292. 	ble	.L25

  293. 	.align 4

  294. 

  295. .L22:

  296. 	LFPDUX	c01,   AO1, INC2

  297. 	LFXDUX	c05,   AO2, INC2

  298. 	LFPDUX	c02,   AO1, INC2

  299. 	LFXDUX	c06,   AO2, INC2

  300. 

  301. 	LFPDUX	c03,   AO1, INC2

  302. 	LFXDUX	c07,   AO2, INC2

  303. 	LFPDUX	c04,   AO1, INC2

  304. 	LFXDUX	c08,   AO2, INC2

  305. 

  306. 	fpsel	c17, sel_p, c01, c05

  307. 	fpsel	c01, sel_s, c01, c05

  308. 	fpsel	c09, sel_p, c02, c06

  309. 	fpsel	c02, sel_s, c02, c06

  310. 

  311. 	fpsel	c10, sel_p, c03, c07

  312. 	fpsel	c03, sel_s, c03, c07

  313. 	STFPDUX	c17,   B, INC2

  314. 	fpsel	c11, sel_p, c04, c08

  315. 	STFXDUX	c01,   B, INC2

  316. 	fpsel	c04, sel_s, c04, c08

  317. 	STFPDUX	c09,   B, INC2

  318. 

  319. 	STFXDUX	c02,   B, INC2

  320. 	STFPDUX	c10,   B, INC2

  321. 	STFXDUX	c03,   B, INC2

  322. 	STFPDUX	c11,   B, INC2

  323. 	STFXDUX	c04,   B, INC2

  324. 	bdnz	.L22

  325. 	.align 4

  326. 

  327. .L25:

  328. 	andi.	r0,  M,  7

  329. 	ble	.L30

  330. 

  331. 	andi.	r0,  M,  4

  332. 	beq	.L26

  333. 

  334. 	LFPDUX	c01,   AO1, INC2

  335. 	LFXDUX	c05,   AO2, INC2

  336. 	LFPDUX	c02,   AO1, INC2

  337. 	LFXDUX	c06,   AO2, INC2

  338. 

  339. 	fpsel	c17, sel_p, c01, c05

  340. 	fpsel	c01, sel_s, c01, c05

  341. 	fpsel	c09, sel_p, c02, c06

  342. 	fpsel	c02, sel_s, c02, c06

  343. 

  344. 	STFPDUX	c17,   B, INC2

  345. 	STFXDUX	c01,   B, INC2

  346. 	STFPDUX	c09,   B, INC2

  347. 	STFXDUX	c02,   B, INC2

  348. 	.align 4

  349. 

  350. .L26:

  351. 	andi.	r0,  M,  2

  352. 	beq	.L27

  353. 

  354. 	LFPDUX	c01,   AO1, INC2

  355. 	LFXDUX	c05,   AO2, INC2

  356. 

  357. 	fpsel	c17, sel_p, c01, c05

  358. 	fpsel	c01, sel_s, c01, c05

  359. 

  360. 	STFPDUX	c17,   B, INC2

  361. 	STFXDUX	c01,   B, INC2

  362. 	.align 4

  363. 

  364. .L27:

  365. 	andi.	r0,  M,  1

  366. 	beq	.L30

  367. 

  368. 	LFDUX	c01,   AO1, INC2

  369. 	LFDUX	c02,   AO2, INC2

  370. 

  371. 	fsmfp	c01, c02

  372. 	STFPDUX	c01,   B, INC2

  373. 	.align 4

  374. 

  375. .L30:

  376. 	andi.	J,  N,  1

  377. 	ble	.L99

  378. 

  379. 	mr	AO1, A

  380. 

  381. 	srawi.	r0,  M,  3

  382. 	mtspr	CTR, r0

  383. 	ble	.L35

  384. 	.align 4

  385. 

  386. .L32:

  387. 	LFPDUX	c01,   AO1, INC2

  388. 	LFPDUX	c02,   AO1, INC2

  389. 	LFPDUX	c03,   AO1, INC2

  390. 	LFPDUX	c04,   AO1, INC2

  391. 

  392. 	STFPDUX	c01,   B, INC2

  393. 	STFPDUX	c02,   B, INC2

  394. 	STFPDUX	c03,   B, INC2

  395. 	STFPDUX	c04,   B, INC2

  396. 	bdnz	.L32

  397. 	.align 4

  398. 

  399. .L35:

  400. 	andi.	r0,  M,  7

  401. 	ble	.L99

  402. 

  403. 	andi.	r0,  M,  4

  404. 	beq	.L36

  405. 

  406. 	LFPDUX	c01,   AO1, INC2

  407. 	LFPDUX	c02,   AO1, INC2

  408. 

  409. 	STFPDUX	c01,   B, INC2

  410. 	STFPDUX	c02,   B, INC2

  411. 	.align 4

  412. 

  413. .L36:

  414. 	andi.	r0,  M,  2

  415. 	beq	.L37

  416. 

  417. 	LFPDUX	c01,   AO1, INC2

  418. 

  419. 	STFPDUX	c01,   B, INC2

  420. 	.align 4

  421. 

  422. .L37:

  423. 	andi.	r0,  M,  1

  424. 	beq	.L99

  425. 

  426. 	LFDX	c01,   AO1, INC2

  427. 	STFDX	c01,   B,  INC2

  428. 	.align 4

  429. 

  430. .L99:

  431. 	addi	SP, SP, 4

  432. 

  433. 	lwzu	r30,   4(SP)

  434. 	lwzu	r31,   4(SP)

  435. 

  436. 	subi	SP, SP, 12

  437. 	li	r0, 16

  438. 

  439. 	lfpdux	f19, SP, r0

  440. 	lfpdux	f18, SP, r0

  441. 	lfpdux	f17, SP, r0

  442. 	lfpdux	f16, SP, r0

  443. 

  444. 	lfpdux	f15, SP, r0

  445. 	lfpdux	f14, SP, r0

  446. 	addi	SP, SP, 16

  447. 	blr

  448. 	.align 4

  449. 

  450. .L100:

  451. 	li	INC,  1 * SIZE

  452. 	li	INC2, 2 * SIZE

  453. 

  454. 	subi	A, A, 1 * SIZE

  455. 	subi	B, B, 2 * SIZE

  456. 

  457. 	srawi.	J,  N,  2

  458. 	ble	.L120

  459. 	.align 4

  460. .L111:

  461. 	mr	AO1, A

  462. 	add	AO2, A,   LDA

  463. 	add	AO3, AO2, LDA

  464. 	add	AO4, AO3, LDA

  465. 	add	A,   AO4, LDA

  466. 

  467. 	srawi.	r0,  M,  3

  468. 	mtspr	CTR, r0

  469. 	ble	.L115

  470. 	.align 4

  471. 

  472. .L112:

  473. 	LFDUX	c01,   AO1, INC

  474. 	LFDUX	c02,   AO1, INC

  475. 	LFDUX	c03,   AO1, INC

  476. 	LFDUX	c04,   AO1, INC

  477. 

  478. 	LFDUX	c09,   AO1, INC

  479. 	LFDUX	c10,   AO1, INC

  480. 	LFDUX	c11,   AO1, INC

  481. 	LFDUX	c12,   AO1, INC

  482. 

  483. 	LFSDUX	c01,   AO2, INC

  484. 	LFSDUX	c02,   AO2, INC

  485. 	LFSDUX	c03,   AO2, INC

  486. 	LFSDUX	c04,   AO2, INC

  487. 

  488. 	LFSDUX	c09,   AO2, INC

  489. 	LFSDUX	c10,   AO2, INC

  490. 	LFSDUX	c11,   AO2, INC

  491. 	LFSDUX	c12,   AO2, INC

  492. 

  493. 	LFDUX	c05,   AO3, INC

  494. 	LFDUX	c06,   AO3, INC

  495. 	LFDUX	c07,   AO3, INC

  496. 	LFDUX	c08,   AO3, INC

  497. 

  498. 	LFDUX	c13,   AO3, INC

  499. 	LFDUX	c14,   AO3, INC

  500. 	LFDUX	c15,   AO3, INC

  501. 	LFDUX	c16,   AO3, INC

  502. 

  503. 	LFSDUX	c05,   AO4, INC

  504. 	LFSDUX	c06,   AO4, INC

  505. 	LFSDUX	c07,   AO4, INC

  506. 	LFSDUX	c08,   AO4, INC

  507. 

  508. 	LFSDUX	c13,   AO4, INC

  509. 	LFSDUX	c14,   AO4, INC

  510. 	LFSDUX	c15,   AO4, INC

  511. 	LFSDUX	c16,   AO4, INC

  512. 

  513. 	STFPDUX	c01,   B, INC2

  514. 	STFPDUX	c05,   B, INC2

  515. 	STFPDUX	c02,   B, INC2

  516. 	STFPDUX	c06,   B, INC2

  517. 	STFPDUX	c03,   B, INC2

  518. 	STFPDUX	c07,   B, INC2

  519. 	STFPDUX	c04,   B, INC2

  520. 	STFPDUX	c08,   B, INC2

  521. 

  522. 	STFPDUX	c09,   B, INC2

  523. 	STFPDUX	c13,   B, INC2

  524. 	STFPDUX	c10,   B, INC2

  525. 	STFPDUX	c14,   B, INC2

  526. 	STFPDUX	c11,   B, INC2

  527. 	STFPDUX	c15,   B, INC2

  528. 	STFPDUX	c12,   B, INC2

  529. 	STFPDUX	c16,   B, INC2

  530. 	bdnz	.L112

  531. 	.align 4

  532. 

  533. .L115:

  534. 	andi.	r0,  M,  7

  535. 	ble	.L119

  536. 

  537. 	andi.	r0,  M,  4

  538. 	beq	.L116

  539. 

  540. 	LFDUX	c01,   AO1, INC

  541. 	LFDUX	c02,   AO1, INC

  542. 	LFDUX	c03,   AO1, INC

  543. 	LFDUX	c04,   AO1, INC

  544. 

  545. 	LFSDUX	c01,   AO2, INC

  546. 	LFSDUX	c02,   AO2, INC

  547. 	LFSDUX	c03,   AO2, INC

  548. 	LFSDUX	c04,   AO2, INC

  549. 

  550. 	LFDUX	c05,   AO3, INC

  551. 	LFDUX	c06,   AO3, INC

  552. 	LFDUX	c07,   AO3, INC

  553. 	LFDUX	c08,   AO3, INC

  554. 

  555. 	LFSDUX	c05,   AO4, INC

  556. 	LFSDUX	c06,   AO4, INC

  557. 	LFSDUX	c07,   AO4, INC

  558. 	LFSDUX	c08,   AO4, INC

  559. 

  560. 	STFPDUX	c01,   B, INC2

  561. 	STFPDUX	c05,   B, INC2

  562. 	STFPDUX	c02,   B, INC2

  563. 	STFPDUX	c06,   B, INC2

  564. 	STFPDUX	c03,   B, INC2

  565. 	STFPDUX	c07,   B, INC2

  566. 	STFPDUX	c04,   B, INC2

  567. 	STFPDUX	c08,   B, INC2

  568. 	.align 4

  569. 

  570. .L116:

  571. 	andi.	r0,  M,  2

  572. 	beq	.L117

  573. 

  574. 	LFDUX	c01,   AO1, INC

  575. 	LFDUX	c02,   AO1, INC

  576. 

  577. 	LFSDUX	c01,   AO2, INC

  578. 	LFSDUX	c02,   AO2, INC

  579. 

  580. 	LFDUX	c05,   AO3, INC

  581. 	LFDUX	c06,   AO3, INC

  582. 

  583. 	LFSDUX	c05,   AO4, INC

  584. 	LFSDUX	c06,   AO4, INC

  585. 

  586. 	STFPDUX	c01,   B, INC2

  587. 	STFPDUX	c05,   B, INC2

  588. 	STFPDUX	c02,   B, INC2

  589. 	STFPDUX	c06,   B, INC2

  590. 	.align 4

  591. 

  592. .L117:

  593. 	andi.	r0,  M,  1

  594. 	beq	.L119

  595. 

  596. 	LFDUX	c01,   AO1, INC

  597. 	LFDUX	c05,   AO3, INC

  598. 

  599. 	nop

  600. 	nop

  601. 

  602. 	LFSDUX	c01,   AO2, INC

  603. 	LFSDUX	c05,   AO4, INC

  604. 

  605. 	STFPDUX	c01,   B, INC2

  606. 	STFPDUX	c05,   B, INC2

  607. 	.align 4

  608. 

  609. .L119:

  610. 	addic.	J, J, -1

  611. 	bgt	.L111

  612. 	.align 4

  613. 

  614. .L120:

  615. 	andi.	J,  N,  2

  616. 	ble	.L130

  617. 

  618. 	mr	AO1, A

  619. 	add	AO2, A,   LDA

  620. 	add	A,   AO2, LDA

  621. 

  622. 	srawi.	r0,  M,  3

  623. 	mtspr	CTR, r0

  624. 	ble	.L125

  625. 	.align 4

  626. 

  627. .L122:

  628. 	LFDUX	c01,   AO1, INC

  629. 	LFDUX	c02,   AO1, INC

  630. 	LFDUX	c03,   AO1, INC

  631. 	LFDUX	c04,   AO1, INC

  632. 

  633. 	LFDUX	c09,   AO1, INC

  634. 	LFDUX	c10,   AO1, INC

  635. 	LFDUX	c11,   AO1, INC

  636. 	LFDUX	c12,   AO1, INC

  637. 

  638. 	LFSDUX	c01,   AO2, INC

  639. 	LFSDUX	c02,   AO2, INC

  640. 	LFSDUX	c03,   AO2, INC

  641. 	LFSDUX	c04,   AO2, INC

  642. 

  643. 	LFSDUX	c09,   AO2, INC

  644. 	LFSDUX	c10,   AO2, INC

  645. 	LFSDUX	c11,   AO2, INC

  646. 	LFSDUX	c12,   AO2, INC

  647. 

  648. 	STFPDUX	c01,   B, INC2

  649. 	STFPDUX	c02,   B, INC2

  650. 	STFPDUX	c03,   B, INC2

  651. 	STFPDUX	c04,   B, INC2

  652. 

  653. 	STFPDUX	c09,   B, INC2

  654. 	STFPDUX	c10,   B, INC2

  655. 	STFPDUX	c11,   B, INC2

  656. 	STFPDUX	c12,   B, INC2

  657. 	bdnz	.L122

  658. 	.align 4

  659. 

  660. .L125:

  661. 	andi.	r0,  M,  7

  662. 	ble	.L130

  663. 

  664. 	andi.	r0,  M,  4

  665. 	beq	.L126

  666. 

  667. 	LFDUX	c01,   AO1, INC

  668. 	LFDUX	c02,   AO1, INC

  669. 	LFDUX	c03,   AO1, INC

  670. 	LFDUX	c04,   AO1, INC

  671. 

  672. 	LFSDUX	c01,   AO2, INC

  673. 	LFSDUX	c02,   AO2, INC

  674. 	LFSDUX	c03,   AO2, INC

  675. 	LFSDUX	c04,   AO2, INC

  676. 

  677. 	STFPDUX	c01,   B, INC2

  678. 	STFPDUX	c02,   B, INC2

  679. 	STFPDUX	c03,   B, INC2

  680. 	STFPDUX	c04,   B, INC2

  681. 	.align 4

  682. 

  683. .L126:

  684. 	andi.	r0,  M,  2

  685. 	beq	.L127

  686. 

  687. 	LFDUX	c01,   AO1, INC

  688. 	LFDUX	c02,   AO1, INC

  689. 

  690. 	LFSDUX	c01,   AO2, INC

  691. 	LFSDUX	c02,   AO2, INC

  692. 

  693. 	STFPDUX	c01,   B, INC2

  694. 	STFPDUX	c02,   B, INC2

  695. 	.align 4

  696. 

  697. .L127:

  698. 	andi.	r0,  M,  1

  699. 	beq	.L130

  700. 

  701. 	LFDUX	c01,   AO1, INC

  702. 	LFDUX	c02,   AO2, INC

  703. 

  704. 	fsmfp	c01, c02

  705. 	STFPDUX	c01,   B, INC2

  706. 	.align 4

  707. 

  708. .L130:

  709. 	andi.	J,  N,  1

  710. 	ble	.L999

  711. 

  712. 	mr	AO1, A

  713. 

  714. 	srawi.	r0,  M,  3

  715. 	mtspr	CTR, r0

  716. 	ble	.L135

  717. 	.align 4

  718. 

  719. .L132:

  720. 	LFDUX	c01,   AO1, INC

  721. 	LFDUX	c02,   AO1, INC

  722. 	LFDUX	c03,   AO1, INC

  723. 	LFDUX	c04,   AO1, INC

  724. 

  725. 	LFDUX	c05,   AO1, INC

  726. 	LFDUX	c06,   AO1, INC

  727. 	LFDUX	c07,   AO1, INC

  728. 	LFDUX	c08,   AO1, INC

  729. 

  730. 	fsmfp	c01, c02

  731. 	fsmfp	c03, c04

  732. 	fsmfp	c05, c06

  733. 	fsmfp	c07, c08

  734. 

  735. 	STFPDUX	c01,   B, INC2

  736. 	STFPDUX	c03,   B, INC2

  737. 	STFPDUX	c05,   B, INC2

  738. 	STFPDUX	c07,   B, INC2

  739. 	bdnz	.L132

  740. 	.align 4

  741. 

  742. .L135:

  743. 	andi.	r0,  M,  7

  744. 	ble	.L999

  745. 

  746. 	andi.	r0,  M,  4

  747. 	beq	.L136

  748. 

  749. 	LFDUX	c01,   AO1, INC

  750. 	LFDUX	c02,   AO1, INC

  751. 	LFDUX	c03,   AO1, INC

  752. 	LFDUX	c04,   AO1, INC

  753. 

  754. 	fsmfp	c01, c02

  755. 	fsmfp	c03, c04

  756. 

  757. 	STFPDUX	c01,   B, INC2

  758. 	STFPDUX	c03,   B, INC2

  759. 	.align 4

  760. 

  761. .L136:

  762. 	andi.	r0,  M,  2

  763. 	beq	.L137

  764. 

  765. 	LFDUX	c01,   AO1, INC

  766. 	LFDUX	c02,   AO1, INC

  767. 

  768. 	fsmfp	c01, c02

  769. 	STFPDUX	c01,   B, INC2

  770. 	.align 4

  771. 

  772. .L137:

  773. 	andi.	r0,  M,  1

  774. 	beq	.L999

  775. 

  776. 	LFDX	c01,   AO1, INC

  777. 	STFDX	c01,   B,  INC2

  778. 	.align 4

  779. 

  780. .L999:

  781. 	addi	SP, SP, 12

  782. 

  783. 	lwzu	r30,   4(SP)

  784. 	lwzu	r31,   4(SP)

  785. 

  786. 	subi	SP, SP, 12

  787. 	li	r0, 16

  788. 

  789. 	lfpdux	f19, SP, r0

  790. 	lfpdux	f18, SP, r0

  791. 	lfpdux	f17, SP, r0

  792. 	lfpdux	f16, SP, r0

  793. 

  794. 	lfpdux	f15, SP, r0

  795. 	lfpdux	f14, SP, r0

  796. 	addi	SP, SP, 16

  797. 	blr

  798. 	EPILOGUE