OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 57ed58cefe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '57ed58cefe'
${ noResults }
OpenBLAS/kernel/x86/qtrsm_kernel_RT_2x2.S

1232 lines
19 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef OPTERON

  43. #define PREFETCH	prefetch

  44. #define PREFETCHW	prefetchw

  45. #else

  46. #define PREFETCH	prefetcht0

  47. #define PREFETCHW	prefetcht0

  48. #endif

  49. 

  50. #define PREFETCHSIZE (5 + 4 * 10)

  51. #define STACK	16

  52. #define ARGS	16

  53. 

  54. #define J	 0 + STACK(%esp)

  55. #define KK	 4 + STACK(%esp)

  56. #define KKK	 8 + STACK(%esp)

  57. #define AORIG	12 + STACK(%esp)

  58. 

  59. #define M	 4 + STACK + ARGS(%esp)

  60. #define N	 8 + STACK + ARGS(%esp)

  61. #define K	12 + STACK + ARGS(%esp)

  62. #define ALPHA	16 + STACK + ARGS(%esp)

  63. #define A	32 + STACK + ARGS(%esp)

  64. #define ARG_B	36 + STACK + ARGS(%esp)

  65. #define C	40 + STACK + ARGS(%esp)

  66. #define ARG_LDC	44 + STACK + ARGS(%esp)

  67. #define OFFSET	48 + STACK + ARGS(%esp)

  68. 

  69. #define I	%esi

  70. #define B	%ebx

  71. #define CO	%edi

  72. #define AO	%edx

  73. #define BO	%ecx

  74. #define LDC	%ebp

  75. 

  76. #define PREFETCH_OFFSET 48

  77. 

  78. 	PROLOGUE

  79. 

  80. 	subl	$ARGS, %esp	# Generate Stack Frame

  81. 

  82. 	pushl	%ebp

  83. 	pushl	%edi

  84. 	pushl	%esi

  85. 	pushl	%ebx

  86. 

  87. 	PROFCODE

  88. 

  89. 	movl	ARG_LDC, LDC

  90. 	movl	ARG_B,   B

  91. 	sall	$BASE_SHIFT, LDC

  92. 

  93. 	addl	$8 * SIZE, A

  94. 	addl	$8 * SIZE, B

  95. 

  96. 

  97. #ifdef LN

  98.        movl	M, %eax

  99.        sall	$BASE_SHIFT, %eax

  100.        addl	%eax, C

  101.        imull	K, %eax

  102.        addl	%eax, A

  103. #endif

  104. 

  105. #ifdef RT

  106.        movl	N, %eax

  107.        sall	$BASE_SHIFT, %eax

  108.        imull	K, %eax

  109.        addl	%eax, B

  110. 

  111.        movl	N,    %eax

  112.        imull	%ebp, %eax

  113.        addl	%eax, C

  114. #endif

  115. 

  116. #ifdef RN

  117.        movl	OFFSET, %eax

  118.        negl	%eax

  119.        movl	%eax, KK

  120. #endif

  121. 

  122. #ifdef RT

  123.        movl	N, %eax

  124.        subl	OFFSET, %eax

  125.        movl	%eax, KK

  126. #endif

  127. 

  128. 	movl	N,  %eax

  129. 	testl	$1, %eax

  130. 	je	.L30

  131. 

  132. #if defined(LT) || defined(RN)

  133. 	movl	A, AO

  134. #else

  135. 	movl	A, %eax

  136. 	movl	%eax, AORIG

  137. #endif

  138. 

  139. #ifdef RT

  140. 	movl	K, %eax

  141. 	sall	$0 + BASE_SHIFT, %eax

  142. 	subl	%eax, B

  143. #endif

  144. 

  145. #ifdef RT

  146. 	subl	LDC, C

  147. #endif

  148. 	movl	C, CO

  149. #ifndef RT

  150. 	addl	LDC, C

  151. #endif

  152. 

  153. #ifdef LN

  154. 	movl	OFFSET, %eax

  155. 	addl	M, %eax

  156. 	movl	%eax, KK

  157. #endif

  158. 

  159. #ifdef LT

  160. 	movl	OFFSET, %eax

  161. 	movl	%eax, KK

  162. #endif

  163. 

  164. 	movl	M,  I

  165. 	sarl	$1, I

  166. 	je	.L40

  167. 	ALIGN_4

  168. 

  169. .L31:

  170. #ifdef LN

  171.        movl	K, %eax

  172.        sall	$1 + BASE_SHIFT, %eax

  173.        subl	%eax, AORIG

  174. #endif

  175. 

  176. #if defined(LN) || defined(RT)

  177. 	movl	KK, %eax

  178. 	sall	$BASE_SHIFT, %eax

  179. 	movl	AORIG, AO

  180. 	leal	(AO, %eax, 2), AO

  181. 	leal	(B,  %eax, 1), BO

  182. #else

  183. 	movl	B, BO

  184. #endif

  185. 

  186. 	fldz

  187. 	fldz

  188. 

  189. #if   defined(HAVE_3DNOW)

  190. 	prefetchw	2 * SIZE(CO)

  191. #elif defined(HAVE_SSE)

  192. 	prefetchnta	2 * SIZE(CO)

  193. #endif

  194. 

  195. #if defined(LT) || defined(RN)

  196. 	movl	KK, %eax

  197. #else

  198. 	movl	K,  %eax

  199. 	subl	KK, %eax

  200. #endif

  201. 	sarl	$2, %eax

  202.  	je	.L35

  203. 	ALIGN_4

  204. 

  205. .L32:

  206. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  207. 

  208. 	FLD	 -8 * SIZE(BO)

  209. 	FLD	 -8 * SIZE(AO)

  210. 	fmul	 %st(1), %st

  211. 	faddp	 %st, %st(2)

  212. 

  213. 	FLD	 -7 * SIZE(AO)

  214. 	fmulp	 %st, %st(1)

  215. 	faddp	 %st, %st(2)

  216. 

  217. 	FLD	 -7 * SIZE(BO)

  218. 	FLD	 -6 * SIZE(AO)

  219. 	fmul	 %st(1), %st

  220. 	faddp	 %st, %st(2)

  221. 

  222. 	FLD	 -5 * SIZE(AO)

  223. 	fmulp	 %st, %st(1)

  224. 	faddp	 %st, %st(2)

  225. 

  226. 	FLD	 -6 * SIZE(BO)

  227. 	FLD	 -4 * SIZE(AO)

  228. 	fmul	 %st(1), %st

  229. 	faddp	 %st, %st(2)

  230. 

  231. 	FLD	 -3 * SIZE(AO)

  232. 	fmulp	 %st, %st(1)

  233. 	faddp	 %st, %st(2)

  234. 

  235. 	FLD	 -5 * SIZE(BO)

  236. 	FLD	 -2 * SIZE(AO)

  237. 	fmul	 %st(1), %st

  238. 	faddp	 %st, %st(2)

  239. 

  240. 	FLD	 -1 * SIZE(AO)

  241. 	fmulp	 %st, %st(1)

  242. 	faddp	 %st, %st(2)

  243. 

  244. 	addl	$8 * SIZE,AO

  245. 	addl	$4 * SIZE,BO

  246. 

  247. 	decl	%eax

  248. 	jne	.L32

  249. 	ALIGN_4

  250. 

  251. .L35:

  252. #if defined(LT) || defined(RN)

  253. 	movl	KK, %eax

  254. #else

  255. 	movl	K,  %eax

  256. 	subl	KK, %eax

  257. #endif

  258. 	and	$3,  %eax

  259. 	je	.L38

  260. 	ALIGN_4

  261. 

  262. .L36:

  263. 	FLD	 -8 * SIZE(BO)

  264. 

  265. 	FLD	 -8 * SIZE(AO)

  266. 	fmul	 %st(1), %st

  267. 	faddp	 %st, %st(2)

  268. 

  269. 	FLD	 -7 * SIZE(AO)

  270. 	fmulp	 %st, %st(1)

  271. 	faddp	 %st, %st(2)

  272. 

  273. 	addl	$2 * SIZE,AO

  274. 	addl	$1 * SIZE,BO

  275. 

  276. 	decl	%eax

  277. 	jne	 .L36

  278. 	ALIGN_4

  279. 

  280. .L38:

  281. #if defined(LN) || defined(RT)

  282. 	movl	KK, %eax

  283. #ifdef LN

  284. 	subl	$2, %eax

  285. #else

  286. 	subl	$1, %eax

  287. #endif

  288. 

  289. 	sall	$BASE_SHIFT, %eax

  290. 

  291. 	movl	AORIG, AO

  292. 	leal	(AO, %eax, 2), AO

  293. 	leal	(B,  %eax, 1), BO

  294. #endif

  295. 

  296. #if defined(LN) || defined(LT)

  297. 	FLD	-8 * SIZE(BO)

  298. 	fsubp	%st, %st(1)

  299. 	FLD	-7 * SIZE(BO)

  300. 	fsubp	%st, %st(2)

  301. #else

  302. 	FLD	-8 * SIZE(AO)

  303. 	fsubp	%st, %st(1)

  304. 	FLD	-7 * SIZE(AO)

  305. 	fsubp	%st, %st(2)

  306. #endif

  307. 

  308. #ifdef LN

  309.        FLD	-5 * SIZE(AO)

  310.        fmulp	%st, %st(2)

  311. 

  312.        FLD	-6 * SIZE(AO)

  313.        fmul	%st(2), %st

  314. 

  315.        fsubrp	%st, %st(1)

  316.        FLD	-8 * SIZE(AO)

  317.        fmulp	%st, %st(1)

  318. #endif

  319. 

  320. #ifdef LT

  321.        FLD	-8 * SIZE(AO)

  322.        fmulp	%st, %st(1)

  323. 

  324.        FLD	-7 * SIZE(AO)

  325.        fmul	%st(1), %st

  326. 

  327.        fsubrp	%st, %st(2)

  328. 

  329.        FLD	-5 * SIZE(AO)

  330.        fmulp	%st, %st(2)

  331. #endif

  332. 

  333. #ifdef RN

  334.        FLD	-8 * SIZE(BO)

  335.        fmul	%st, %st(1)

  336.        fmulp	%st, %st(2)

  337. #endif

  338. 

  339. #ifdef RT

  340.        FLD	-8 * SIZE(BO)

  341.        fmul	%st, %st(1)

  342.        fmulp	%st, %st(2)

  343. #endif

  344. 

  345. #ifdef LN

  346. 	subl	$2 * SIZE, CO

  347. #endif

  348. 

  349. #if defined(LN) || defined(LT)

  350. 	fld	%st

  351. 	FST	-8 * SIZE(BO)

  352. 	fxch	%st(1)

  353. 	fld	%st

  354. 	FST	-7 * SIZE(BO)

  355. #else

  356. 	fld	%st

  357. 	FST	-8 * SIZE(AO)

  358. 	fxch	%st(1)

  359. 	fld	%st

  360. 	FST	-7 * SIZE(AO)

  361. #endif

  362. 

  363. 	FST	1 * SIZE(CO)

  364. 	FST	0 * SIZE(CO)

  365. 

  366. #ifndef LN

  367. 	addl	$2 * SIZE, CO

  368. #endif

  369. 

  370. #if defined(LT) || defined(RN)

  371. 	movl	K,  %eax

  372. 	subl	KK, %eax

  373. 	sall	$BASE_SHIFT, %eax

  374. 	leal	(AO, %eax, 2), AO

  375. 	leal	(BO, %eax, 1), BO

  376. #endif

  377. 

  378. #ifdef LN

  379. 	subl	$2, KK

  380. #endif

  381. 

  382. #ifdef LT

  383. 	addl	$2, KK

  384. #endif

  385. 

  386. #ifdef RT

  387.        movl	K, %eax

  388.        sall	$1 + BASE_SHIFT, %eax

  389.        addl	%eax, AORIG

  390. #endif

  391. 

  392. 	decl	I

  393. 	jne	.L31

  394. 	ALIGN_4

  395. 

  396. .L40:

  397. 	movl	 M, %eax

  398. 	andl	$1, %eax

  399. 	je	.L49

  400. 	ALIGN_4

  401. 

  402. .L41:

  403. #ifdef LN

  404.        movl	K, %eax

  405.        sall	$0 + BASE_SHIFT, %eax

  406.        subl	%eax, AORIG

  407. #endif

  408. 

  409. #if defined(LN) || defined(RT)

  410. 	movl	KK, %eax

  411. 	sall	$BASE_SHIFT, %eax

  412. 	movl	AORIG, AO

  413. 	leal	(AO, %eax, 1), AO

  414. 	leal	(B,  %eax, 1), BO

  415. #else

  416. 	movl	B, BO

  417. #endif

  418. 

  419. 	fldz

  420. 

  421. #if defined(LT) || defined(RN)

  422. 	movl	KK, %eax

  423. #else

  424. 	movl	K,  %eax

  425. 	subl	KK, %eax

  426. #endif

  427. 	sarl	$2, %eax

  428.  	je	.L45

  429. 	ALIGN_4

  430. 

  431. .L42:

  432. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  433. 

  434. 	FLD	 -8 * SIZE(AO)

  435. 	FLD	 -8 * SIZE(BO)

  436. 	fmulp	 %st, %st(1)

  437. 	faddp	 %st, %st(1)

  438. 

  439. 	FLD	 -7 * SIZE(AO)

  440. 	FLD	 -7 * SIZE(BO)

  441. 	fmulp	 %st, %st(1)

  442. 	faddp	 %st, %st(1)

  443. 

  444. 	FLD	 -6 * SIZE(AO)

  445. 	FLD	 -6 * SIZE(BO)

  446. 	fmulp	 %st, %st(1)

  447. 	faddp	 %st, %st(1)

  448. 

  449. 	FLD	 -5 * SIZE(AO)

  450. 	FLD	 -5 * SIZE(BO)

  451. 	fmulp	 %st, %st(1)

  452. 	faddp	 %st, %st(1)

  453. 

  454. 	addl	$4 * SIZE,AO

  455. 	addl	$4 * SIZE,BO

  456. 

  457. 	decl	%eax

  458. 	jne	.L42

  459. 	ALIGN_4

  460. 

  461. .L45:

  462. #if defined(LT) || defined(RN)

  463. 	movl	KK, %eax

  464. #else

  465. 	movl	K,  %eax

  466. 	subl	KK, %eax

  467. #endif

  468. 	and	$3,  %eax

  469. 	je	.L48

  470. 	ALIGN_4

  471. 

  472. .L46:

  473. 	FLD	 -8 * SIZE(AO)

  474. 

  475. 	FLD	 -8 * SIZE(BO)

  476. 	fmulp	 %st, %st(1)

  477. 	faddp	 %st, %st(1)

  478. 

  479. 	addl	$1 * SIZE,AO

  480. 	addl	$1 * SIZE,BO

  481. 

  482. 	decl	%eax

  483. 	jne	 .L46

  484. 	ALIGN_4

  485. 

  486. .L48:

  487. #if defined(LN) || defined(RT)

  488. 	movl	KK, %eax

  489. #ifdef LN

  490. 	subl	$1, %eax

  491. #else

  492. 	subl	$1, %eax

  493. #endif

  494. 

  495. 	sall	$BASE_SHIFT, %eax

  496. 

  497. 	movl	AORIG, AO

  498. 	leal	(AO, %eax, 1), AO

  499. 	leal	(B,  %eax, 1), BO

  500. #endif

  501. 

  502. #if defined(LN) || defined(LT)

  503. 	FLD	-8 * SIZE(BO)

  504. 	fsubp	%st, %st(1)

  505. #else

  506. 	FLD	-8 * SIZE(AO)

  507. 	fsubp	%st, %st(1)

  508. #endif

  509. 

  510. #ifdef LN

  511.        FLD	-8 * SIZE(AO)

  512.        fmulp	%st, %st(1)

  513. #endif

  514. 

  515. #ifdef LT

  516.        FLD	-8 * SIZE(AO)

  517.        fmulp	%st, %st(1)

  518. #endif

  519. 

  520. #ifdef RN

  521.        FLD	-8 * SIZE(BO)

  522.        fmulp	%st, %st(1)

  523. #endif

  524. 

  525. #ifdef RT

  526.        FLD	-8 * SIZE(BO)

  527.        fmulp	%st, %st(1)

  528. #endif

  529. 

  530. #ifdef LN

  531. 	subl	$1 * SIZE, CO

  532. #endif

  533. 

  534. #if defined(LN) || defined(LT)

  535. 	fld	%st

  536. 	FST	-8 * SIZE(BO)

  537. #else

  538. 	fld	%st

  539. 	FST	-8 * SIZE(AO)

  540. #endif

  541. 

  542. 	FST	0 * SIZE(CO)

  543. 

  544. #ifndef LN

  545. 	addl	$1 * SIZE, CO

  546. #endif

  547. 

  548. #if defined(LT) || defined(RN)

  549. 	movl	K,  %eax

  550. 	subl	KK, %eax

  551. 	sall	$BASE_SHIFT, %eax

  552. 	leal	(AO, %eax, 1), AO

  553. 	leal	(BO, %eax, 1), BO

  554. #endif

  555. 

  556. #ifdef LN

  557. 	subl	$1, KK

  558. #endif

  559. 

  560. #ifdef LT

  561. 	addl	$1, KK

  562. #endif

  563. 

  564. #ifdef RT

  565.        movl	K, %eax

  566.        sall	$0 + BASE_SHIFT, %eax

  567.        addl	%eax, AORIG

  568. #endif

  569. 	ALIGN_4

  570. 

  571. .L49:

  572. #ifdef LN

  573.        movl	K, %eax

  574.        sall	$BASE_SHIFT, %eax

  575.        leal	(B, %eax, 1), B

  576. #endif

  577. 

  578. #if defined(LT) || defined(RN)

  579. 	movl	BO, B

  580. #endif

  581. 

  582. #ifdef RN

  583. 	addl	$1, KK

  584. #endif

  585. 

  586. #ifdef RT

  587. 	subl	$1, KK

  588. #endif

  589. 	ALIGN_4

  590. 

  591. .L30:

  592. 	movl	N,   %eax

  593. 	sarl	$1,  %eax

  594. 	movl	%eax, J

  595. 	je	.L999

  596. 	ALIGN_4

  597. 

  598. .L01:

  599. #if defined(LT) || defined(RN)

  600. 	movl	A, AO

  601. #else

  602. 	movl	A, %eax

  603. 	movl	%eax, AORIG

  604. #endif

  605. 

  606. #ifdef RT

  607. 	movl	K, %eax

  608. 	sall	$1 + BASE_SHIFT, %eax

  609. 	subl	%eax, B

  610. #endif

  611. 

  612. 	lea	(, LDC, 2), %eax

  613. 

  614. #ifdef RT

  615. 	subl	%eax, C

  616. #endif

  617. 	movl	C, CO

  618. #ifndef RT

  619. 	addl	%eax, C

  620. #endif

  621. 

  622. #ifdef LN

  623. 	movl	OFFSET, %eax

  624. 	addl	M, %eax

  625. 	movl	%eax, KK

  626. #endif

  627. 

  628. #ifdef LT

  629. 	movl	OFFSET, %eax

  630. 	movl	%eax, KK

  631. #endif

  632. 

  633. 	movl	M,  I

  634. 	sarl	$1, I

  635. 	je	.L20

  636. 	ALIGN_4

  637. 

  638. .L11:

  639. #ifdef LN

  640.        movl	K, %eax

  641.        sall	$1 + BASE_SHIFT, %eax

  642.        subl	%eax, AORIG

  643. #endif

  644. 

  645. #if defined(LN) || defined(RT)

  646. 	movl	KK, %eax

  647. 	sall	$BASE_SHIFT, %eax

  648. 	movl	AORIG, AO

  649. 	leal	(AO, %eax, 2), AO

  650. 	leal	(B,  %eax, 2), BO

  651. #else

  652. 	movl	B, BO

  653. #endif

  654. 

  655. 	fldz

  656. 	fldz

  657. 	fldz

  658. 	fldz

  659. 

  660. #if   defined(HAVE_3DNOW)

  661. 	prefetchw	2 * SIZE(CO)

  662.  	prefetchw	2 * SIZE(CO, LDC, 1)

  663. #elif defined(HAVE_SSE)

  664. 	prefetchnta	2 * SIZE(CO)

  665.  	prefetchnta	2 * SIZE(CO, LDC, 1)

  666. #endif

  667. 

  668. #if defined(LT) || defined(RN)

  669. 	movl	KK, %eax

  670. #else

  671. 	movl	K,  %eax

  672. 	subl	KK, %eax

  673. #endif

  674. 	sarl	$2, %eax

  675.  	je	.L15

  676. 	ALIGN_4

  677. 

  678. .L12:

  679. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  680. 

  681. 	FLD	 -8 * SIZE(AO)

  682. 

  683. 	FLD	 -8 * SIZE(BO)

  684. 	fld	 %st(1)

  685. 	fmul	 %st(1), %st

  686. 	faddp	 %st, %st(3)

  687. 

  688. 	FLD	 -7 * SIZE(BO)

  689. 	fmul	 %st, %st(2)

  690. 

  691. 	FLD	 -7 * SIZE(AO)

  692. 	fmul	 %st, %st(2)

  693. 	fmulp	 %st, %st(1)

  694. 

  695. 	faddp	 %st, %st(6)

  696. 	faddp	 %st, %st(4)

  697. 	faddp	 %st, %st(2)

  698. 

  699. 	FLD	 -6 * SIZE(AO)

  700. 

  701. 	FLD	 -6 * SIZE(BO)

  702. 	fld	 %st(1)

  703. 	fmul	 %st(1), %st

  704. 	faddp	 %st, %st(3)

  705. 

  706. 	FLD	 -5 * SIZE(BO)

  707. 	fmul	 %st, %st(2)

  708. 

  709. 	FLD	 -5 * SIZE(AO)

  710. 	fmul	 %st, %st(2)

  711. 	fmulp	 %st, %st(1)

  712. 

  713. 	faddp	 %st, %st(6)

  714. 	faddp	 %st, %st(4)

  715. 	faddp	 %st, %st(2)

  716. 

  717. 	PREFETCH	(PREFETCHSIZE + 4) * SIZE(AO)

  718. 

  719. 	FLD	 -4 * SIZE(AO)

  720. 

  721. 	FLD	 -4 * SIZE(BO)

  722. 	fld	 %st(1)

  723. 	fmul	 %st(1), %st

  724. 	faddp	 %st, %st(3)

  725. 

  726. 	FLD	 -3 * SIZE(BO)

  727. 	fmul	 %st, %st(2)

  728. 

  729. 	FLD	 -3 * SIZE(AO)

  730. 	fmul	 %st, %st(2)

  731. 	fmulp	 %st, %st(1)

  732. 

  733. 	faddp	 %st, %st(6)

  734. 	faddp	 %st, %st(4)

  735. 	faddp	 %st, %st(2)

  736. 

  737. 	FLD	 -2 * SIZE(AO)

  738. 

  739. 	FLD	 -2 * SIZE(BO)

  740. 	fld	 %st(1)

  741. 	fmul	 %st(1), %st

  742. 	faddp	 %st, %st(3)

  743. 

  744. 	FLD	 -1 * SIZE(BO)

  745. 	fmul	 %st, %st(2)

  746. 

  747. 	FLD	 -1 * SIZE(AO)

  748. 	fmul	 %st, %st(2)

  749. 	fmulp	 %st, %st(1)

  750. 

  751. 	faddp	 %st, %st(6)

  752. 	faddp	 %st, %st(4)

  753. 	faddp	 %st, %st(2)

  754. 

  755. 	addl	$8 * SIZE,AO

  756. 	addl	$8 * SIZE,BO

  757. 

  758. 	decl	%eax

  759. 	jne	.L12

  760. 	ALIGN_4

  761. 

  762. .L15:

  763. #if defined(LT) || defined(RN)

  764. 	movl	KK, %eax

  765. #else

  766. 	movl	K,  %eax

  767. 	subl	KK, %eax

  768. #endif

  769. 	and	$3,  %eax

  770. 	je	.L18

  771. 	ALIGN_4

  772. 

  773. .L16:

  774. 	FLD	 -8 * SIZE(AO)

  775. 

  776. 	FLD	 -8 * SIZE(BO)

  777. 	fld	 %st(1)

  778. 	fmul	 %st(1), %st

  779. 	faddp	 %st, %st(3)

  780. 

  781. 	FLD	 -7 * SIZE(BO)

  782. 	fmul	 %st, %st(2)

  783. 

  784. 	FLD	 -7 * SIZE(AO)

  785. 	fmul	 %st, %st(2)

  786. 	fmulp	 %st, %st(1)

  787. 

  788. 	faddp	 %st, %st(6)

  789. 	faddp	 %st, %st(4)

  790. 	faddp	 %st, %st(2)

  791. 

  792. 	addl	$2 * SIZE,AO

  793. 	addl	$2 * SIZE,BO

  794. 

  795. 	decl	%eax

  796. 	jne	 .L16

  797. 	ALIGN_4

  798. 

  799. .L18:

  800. #if defined(LN) || defined(RT)

  801. 	movl	KK, %eax

  802. #ifdef LN

  803. 	subl	$2, %eax

  804. #else

  805. 	subl	$2, %eax

  806. #endif

  807. 

  808. 	sall	$BASE_SHIFT, %eax

  809. 

  810. 	movl	AORIG, AO

  811. 	leal	(AO, %eax, 2), AO

  812. 	leal	(B,  %eax, 2), BO

  813. #endif

  814. 

  815. #if defined(LN) || defined(LT)

  816. 	FLD	-8 * SIZE(BO)

  817. 	fsubp	%st, %st(1)

  818. 	FLD	-7 * SIZE(BO)

  819. 	fsubp	%st, %st(2)

  820. 	FLD	-6 * SIZE(BO)

  821. 	fsubp	%st, %st(3)

  822. 	FLD	-5 * SIZE(BO)

  823. 	fsubp	%st, %st(4)

  824. #else

  825. 	FLD	-8 * SIZE(AO)

  826. 	fsubp	%st, %st(1)

  827. 	FLD	-7 * SIZE(AO)

  828. 	fsubp	%st, %st(3)

  829. 	FLD	-6 * SIZE(AO)

  830. 	fsubp	%st, %st(2)

  831. 	FLD	-5 * SIZE(AO)

  832. 	fsubp	%st, %st(4)

  833. #endif

  834. 

  835. #ifdef LN

  836.        FLD	-5 * SIZE(AO)

  837.        fmul	%st, %st(3)

  838.        fmulp	%st, %st(4)

  839. 

  840.        FLD	-6 * SIZE(AO)

  841.        fmul	%st(3), %st

  842.        FLD	-6 * SIZE(AO)

  843.        fmul	%st(5), %st

  844. 

  845.        fsubrp	%st, %st(3)

  846.        fsubrp	%st, %st(1)

  847. 

  848.        FLD	-8 * SIZE(AO)

  849.        fmul	%st, %st(1)

  850.        fmulp	%st, %st(2)

  851. #endif

  852. 

  853. #ifdef LT

  854.        FLD	-8 * SIZE(AO)

  855.        fmul	%st, %st(1)

  856.        fmulp	%st, %st(2)

  857. 

  858.        FLD	-7 * SIZE(AO)

  859.        fmul	%st(1), %st

  860.        FLD	-7 * SIZE(AO)

  861.        fmul	%st(3), %st

  862. 

  863.        fsubrp	%st, %st(5)

  864.        fsubrp	%st, %st(3)

  865. 

  866.        FLD	-5 * SIZE(AO)

  867.        fmul	%st, %st(3)

  868.        fmulp	%st, %st(4)

  869. #endif

  870. 

  871. #ifdef RN

  872.        FLD	-8 * SIZE(BO)

  873.        fmul	%st, %st(1)

  874.        fmulp	%st, %st(3)

  875. 

  876.        FLD	-7 * SIZE(BO)

  877.        fmul	%st(1), %st

  878.        FLD	-7 * SIZE(BO)

  879.        fmul	%st(4), %st

  880. 

  881.        fsubrp	%st, %st(5)

  882.        fsubrp	%st, %st(2)

  883. 

  884.        FLD	-5 * SIZE(BO)

  885.        fmul	%st, %st(2)

  886.        fmulp	%st, %st(4)

  887. #endif

  888. 

  889. #ifdef RT

  890.        FLD	-5 * SIZE(BO)

  891.        fmul	%st, %st(2)

  892.        fmulp	%st, %st(4)

  893. 

  894.        FLD	-6 * SIZE(BO)

  895.        fmul	%st(2), %st

  896.        FLD	-6 * SIZE(BO)

  897.        fmul	%st(5), %st

  898. 

  899.        fsubrp	%st, %st(4)

  900.        fsubrp	%st, %st(1)

  901. 

  902.        FLD	-8 * SIZE(BO)

  903.        fmul	%st, %st(1)

  904.        fmulp	%st, %st(3)

  905. #endif

  906. 

  907. #ifdef LN

  908. 	subl	$2 * SIZE, CO

  909. #endif

  910. 

  911. #if defined(LN) || defined(LT)

  912. 	fld	%st

  913. 	FST	-8 * SIZE(BO)

  914. 	fxch	%st(1)

  915. 	fld	%st

  916. 	FST	-7 * SIZE(BO)

  917. 	fxch	%st(2)

  918. 	fld	%st

  919. 	FST	-6 * SIZE(BO)

  920. 	fxch	%st(3)

  921. 	fld	%st

  922. 	FST	-5 * SIZE(BO)

  923. 

  924. 	FST	1 * SIZE(CO, LDC)

  925. 	FST	0 * SIZE(CO)

  926. 	FST	0 * SIZE(CO, LDC)

  927. 	FST	1 * SIZE(CO)

  928. #else

  929. 	fld	%st

  930. 	FST	-8 * SIZE(AO)

  931. 	fxch	%st(2)

  932. 	fld	%st

  933. 	FST	-7 * SIZE(AO)

  934. 	fxch	%st(1)

  935. 	fld	%st

  936. 	FST	-6 * SIZE(AO)

  937. 	fxch	%st(3)

  938. 	fld	%st

  939. 	FST	-5 * SIZE(AO)

  940. 

  941. 	FST	1 * SIZE(CO, LDC)

  942. 	FST	1 * SIZE(CO)

  943. 	FST	0 * SIZE(CO)

  944. 	FST	0 * SIZE(CO, LDC)

  945. #endif

  946. 

  947. #ifndef LN

  948. 	addl	$2 * SIZE, CO

  949. #endif

  950. 

  951. #if defined(LT) || defined(RN)

  952. 	movl	K,  %eax

  953. 	subl	KK, %eax

  954. 	sall	$BASE_SHIFT, %eax

  955. 	leal	(AO, %eax, 2), AO

  956. 	leal	(BO, %eax, 2), BO

  957. #endif

  958. 

  959. #ifdef LN

  960. 	subl	$2, KK

  961. #endif

  962. 

  963. #ifdef LT

  964. 	addl	$2, KK

  965. #endif

  966. 

  967. #ifdef RT

  968.        movl	K, %eax

  969.        sall	$1 + BASE_SHIFT, %eax

  970.        addl	%eax, AORIG

  971. #endif

  972. 

  973. 	decl	I

  974. 	jne	.L11

  975. 	ALIGN_4

  976. 

  977. .L20:

  978. 	movl	 M, %eax

  979. 	andl	$1, %eax

  980. 	je	.L29

  981. 	ALIGN_4

  982. 

  983. .L21:

  984. #ifdef LN

  985.        movl	K, %eax

  986.        sall	$0 + BASE_SHIFT, %eax

  987.        subl	%eax, AORIG

  988. #endif

  989. 

  990. #if defined(LN) || defined(RT)

  991. 	movl	KK, %eax

  992. 	sall	$BASE_SHIFT, %eax

  993. 	movl	AORIG, AO

  994. 	leal	(AO, %eax, 1), AO

  995. 	leal	(B,  %eax, 2), BO

  996. #else

  997. 	movl	B, BO

  998. #endif

  999. 

  1000. 	fldz

  1001. 	fldz

  1002. 

  1003. #if defined(LT) || defined(RN)

  1004. 	movl	KK, %eax

  1005. #else

  1006. 	movl	K,  %eax

  1007. 	subl	KK, %eax

  1008. #endif

  1009. 	sarl	$2, %eax

  1010.  	je	.L25

  1011. 	ALIGN_4

  1012. 

  1013. .L22:

  1014. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  1015. 

  1016. 	FLD	 -8 * SIZE(AO)

  1017. 

  1018. 	FLD	 -8 * SIZE(BO)

  1019. 	fmul	 %st(1), %st

  1020. 	faddp	 %st, %st(2)

  1021. 

  1022. 	FLD	 -7 * SIZE(BO)

  1023. 	fmulp	 %st, %st(1)

  1024. 	faddp	 %st, %st(2)

  1025. 

  1026. 	FLD	 -7 * SIZE(AO)

  1027. 

  1028. 	FLD	 -6 * SIZE(BO)

  1029. 	fmul	 %st(1), %st

  1030. 	faddp	 %st, %st(2)

  1031. 

  1032. 	FLD	 -5 * SIZE(BO)

  1033. 	fmulp	 %st, %st(1)

  1034. 	faddp	 %st, %st(2)

  1035. 

  1036. 	FLD	 -6 * SIZE(AO)

  1037. 

  1038. 	FLD	 -4 * SIZE(BO)

  1039. 	fmul	 %st(1), %st

  1040. 	faddp	 %st, %st(2)

  1041. 

  1042. 	FLD	 -3 * SIZE(BO)

  1043. 	fmulp	 %st, %st(1)

  1044. 	faddp	 %st, %st(2)

  1045. 

  1046. 	FLD	 -5 * SIZE(AO)

  1047. 

  1048. 	FLD	 -2 * SIZE(BO)

  1049. 	fmul	 %st(1), %st

  1050. 	faddp	 %st, %st(2)

  1051. 

  1052. 	FLD	 -1 * SIZE(BO)

  1053. 	fmulp	 %st, %st(1)

  1054. 	faddp	 %st, %st(2)

  1055. 

  1056. 	addl	$4 * SIZE,AO

  1057. 	addl	$8 * SIZE,BO

  1058. 

  1059. 	decl	%eax

  1060. 	jne	.L22

  1061. 	ALIGN_4

  1062. 

  1063. .L25:

  1064. #if defined(LT) || defined(RN)

  1065. 	movl	KK, %eax

  1066. #else

  1067. 	movl	K,  %eax

  1068. 	subl	KK, %eax

  1069. #endif

  1070. 	and	$3,  %eax

  1071. 	je	.L28

  1072. 	ALIGN_4

  1073. 

  1074. .L26:

  1075. 	FLD	 -8 * SIZE(AO)

  1076. 

  1077. 	FLD	 -8 * SIZE(BO)

  1078. 	fmul	 %st(1), %st

  1079. 	faddp	 %st, %st(2)

  1080. 

  1081. 	FLD	 -7 * SIZE(BO)

  1082. 	fmulp	 %st, %st(1)

  1083. 	faddp	 %st, %st(2)

  1084. 

  1085. 	addl	$1 * SIZE,AO

  1086. 	addl	$2 * SIZE,BO

  1087. 

  1088. 	decl	%eax

  1089. 	jne	 .L26

  1090. 	ALIGN_4

  1091. 

  1092. .L28:

  1093. #if defined(LN) || defined(RT)

  1094. 	movl	KK, %eax

  1095. #ifdef LN

  1096. 	subl	$1, %eax

  1097. #else

  1098. 	subl	$2, %eax

  1099. #endif

  1100. 

  1101. 	sall	$BASE_SHIFT, %eax

  1102. 

  1103. 	movl	AORIG, AO

  1104. 	leal	(AO, %eax, 1), AO

  1105. 	leal	(B,  %eax, 2), BO

  1106. #endif

  1107. 

  1108. #if defined(LN) || defined(LT)

  1109. 	FLD	-8 * SIZE(BO)

  1110. 	fsubp	%st, %st(1)

  1111. 	FLD	-7 * SIZE(BO)

  1112. 	fsubp	%st, %st(2)

  1113. #else

  1114. 	FLD	-8 * SIZE(AO)

  1115. 	fsubp	%st, %st(1)

  1116. 	FLD	-7 * SIZE(AO)

  1117. 	fsubp	%st, %st(2)

  1118. #endif

  1119. 

  1120. #if defined(LN) || defined(LT)

  1121.        FLD	-8 * SIZE(AO)

  1122.        fmul	%st, %st(1)

  1123.        fmulp	%st, %st(2)

  1124. #endif

  1125. 

  1126. #ifdef RN

  1127.        FLD	-8 * SIZE(BO)

  1128.        fmulp	%st, %st(1)

  1129. 

  1130.        FLD	-7 * SIZE(BO)

  1131.        fmul	%st(1), %st

  1132. 

  1133.        fsubrp	%st, %st(2)

  1134. 

  1135.        FLD	-5 * SIZE(BO)

  1136.        fmulp	%st, %st(2)

  1137. #endif

  1138. 

  1139. #ifdef RT

  1140.        FLD	-5 * SIZE(BO)

  1141.        fmulp	%st, %st(2)

  1142. 

  1143.        FLD	-6 * SIZE(BO)

  1144.        fmul	%st(2), %st

  1145. 

  1146.        fsubrp	%st, %st(1)

  1147. 

  1148.        FLD	-8 * SIZE(BO)

  1149.        fmulp	%st, %st(1)

  1150. #endif

  1151. 

  1152. #ifdef LN

  1153. 	subl	$1 * SIZE, CO

  1154. #endif

  1155. 

  1156. #if defined(LN) || defined(LT)

  1157. 	fld	%st

  1158. 	FST	-8 * SIZE(BO)

  1159. 	fxch	%st(1)

  1160. 	fld	%st

  1161. 	FST	-7 * SIZE(BO)

  1162. #else

  1163. 	fld	%st

  1164. 	FST	-8 * SIZE(AO)

  1165. 	fxch	%st(1)

  1166. 	fld	%st

  1167. 	FST	-7 * SIZE(AO)

  1168. #endif

  1169. 

  1170. 	FST	0 * SIZE(CO, LDC)

  1171. 	FST	0 * SIZE(CO)

  1172. 

  1173. #ifndef LN

  1174. 	addl	$1 * SIZE, CO

  1175. #endif

  1176. 

  1177. #if defined(LT) || defined(RN)

  1178. 	movl	K,  %eax

  1179. 	subl	KK, %eax

  1180. 	sall	$BASE_SHIFT, %eax

  1181. 	leal	(AO, %eax, 1), AO

  1182. 	leal	(BO, %eax, 2), BO

  1183. #endif

  1184. 

  1185. #ifdef LN

  1186. 	subl	$1, KK

  1187. #endif

  1188. 

  1189. #ifdef LT

  1190. 	addl	$1, KK

  1191. #endif

  1192. 

  1193. #ifdef RT

  1194.        movl	K, %eax

  1195.        sall	$0 + BASE_SHIFT, %eax

  1196.        addl	%eax, AORIG

  1197. #endif

  1198. 	ALIGN_4

  1199. 

  1200. .L29:

  1201. #ifdef LN

  1202.        movl	K, %eax

  1203.        sall	$BASE_SHIFT, %eax

  1204.        leal	(B, %eax, 2), B

  1205. #endif

  1206. 

  1207. #if defined(LT) || defined(RN)

  1208. 	movl	BO, B

  1209. #endif

  1210. 

  1211. #ifdef RN

  1212. 	addl	$2, KK

  1213. #endif

  1214. 

  1215. #ifdef RT

  1216. 	subl	$2, KK

  1217. #endif

  1218. 

  1219. 	decl	J

  1220. 	jne	.L01

  1221. 	ALIGN_4

  1222. 

  1223. .L999:

  1224. 	popl	%ebx

  1225. 	popl	%esi

  1226. 	popl	%edi

  1227. 	popl	%ebp

  1228. 	addl	$ARGS, %esp

  1229. 	ret

  1230. 

  1231. 	EPILOGUE