OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfaaa975e6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfaaa975e6'
${ noResults }
OpenBLAS/kernel/x86/qgemm_kernel_2x2.S

811 lines
14 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(OPTERON) || defined(BARCELONA)

  43. #define PREFETCH	prefetch

  44. #define PREFETCHW	prefetchw

  45. #else

  46. #define PREFETCH	prefetcht0

  47. #define PREFETCHW	prefetcht0

  48. #endif

  49. 

  50. #define PREFETCHSIZE (5 + 4 * 10)

  51. #define STACK	16

  52. #define ARGS	16

  53. 	

  54. #define J	 0 + STACK(%esp)

  55. #define KK	 4 + STACK(%esp)

  56. #define KKK	 8 + STACK(%esp)

  57. 

  58. #define M	 4 + STACK + ARGS(%esp)

  59. #define N	 8 + STACK + ARGS(%esp)

  60. #define K	12 + STACK + ARGS(%esp)

  61. #define ALPHA	16 + STACK + ARGS(%esp)

  62. #define A	32 + STACK + ARGS(%esp)

  63. #define ARG_B	36 + STACK + ARGS(%esp)

  64. #define C	40 + STACK + ARGS(%esp)

  65. #define ARG_LDC	44 + STACK + ARGS(%esp)

  66. #define OFFSET	48 + STACK + ARGS(%esp)

  67. 

  68. #define I	%esi

  69. #define B	%ebx

  70. #define CO	%edi

  71. #define AO	%edx

  72. #define BO	%ecx

  73. #define LDC	%ebp

  74. 

  75. #define PREFETCH_OFFSET 48

  76. 

  77. 	PROLOGUE

  78. 

  79. 	subl	$ARGS, %esp	# Generate Stack Frame

  80. 

  81. 	pushl	%ebp

  82. 	pushl	%edi

  83. 	pushl	%esi

  84. 	pushl	%ebx

  85. 

  86. 	PROFCODE

  87. 

  88. #if defined(TRMMKERNEL) && !defined(LEFT)

  89. 	movl	OFFSET, %eax

  90. 	negl	%eax

  91. 	movl	%eax, KK

  92. #endif

  93. 	

  94. 	movl	ARG_LDC, LDC

  95. 	movl	ARG_B,   B

  96. 

  97. 	addl	$8 * SIZE, A

  98. 	addl	$8 * SIZE, B

  99. 	

  100. 	sall	$BASE_SHIFT, LDC

  101. 

  102. 	movl	N,   %eax

  103. 	sarl	$1,  %eax

  104. 	movl	%eax, J

  105. 	je	.L30

  106. 	ALIGN_4

  107. 

  108. .L01:

  109. #if defined(TRMMKERNEL) && defined(LEFT)

  110. 	movl	OFFSET, %eax

  111. 	movl	%eax, KK

  112. #endif	

  113. 

  114. 	movl	A, AO

  115. 

  116. 	movl	C, CO

  117. 	lea	(, LDC, 2), %eax

  118. 	addl	%eax, C

  119. 

  120. 	movl	M,  I

  121. 	sarl	$1, I

  122. 	je	.L20

  123. 	ALIGN_4

  124. 

  125. .L11:

  126. #if !defined(TRMMKERNEL) || \

  127. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  128. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  129. 	movl	B, BO

  130. #else

  131. 	movl	KK,   %eax

  132. 	sall	$BASE_SHIFT, %eax

  133. 	leal	(AO, %eax, 2), AO

  134. 	leal	(B,  %eax, 2), BO

  135. #endif	

  136. 

  137. 	fldz

  138. 	fldz

  139. 	fldz

  140. 	fldz

  141. 

  142. #if   defined(HAVE_3DNOW)

  143. 	prefetchw	2 * SIZE(CO)

  144.  	prefetchw	2 * SIZE(CO, LDC, 1)

  145. #elif defined(HAVE_SSE)

  146. 	prefetchnta	2 * SIZE(CO)

  147.  	prefetchnta	2 * SIZE(CO, LDC, 1)

  148. #endif

  149. 

  150. #ifndef TRMMKERNEL

  151. 	movl	K, %eax

  152. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  153. 	movl	K, %eax

  154. 	subl	KK, %eax

  155. 	movl	%eax, KKK	

  156. #else

  157. 	movl	KK, %eax

  158. #ifdef LEFT

  159. 	addl	$2, %eax

  160. #else

  161. 	addl	$2, %eax

  162. #endif

  163. 	movl	%eax, KKK

  164. #endif

  165. 	sarl	$2, %eax

  166.  	je	.L15

  167. 	ALIGN_4

  168. 

  169. .L12:

  170. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  171. 

  172. 	FLD	 -8 * SIZE(AO)

  173. 

  174. 	FLD	 -8 * SIZE(BO)

  175. 	fld	 %st(1)

  176. 	fmul	 %st(1), %st

  177. 	faddp	 %st, %st(3)

  178. 

  179. 	FLD	 -7 * SIZE(BO)

  180. 	fmul	 %st, %st(2)

  181. 	 

  182. 	FLD	 -7 * SIZE(AO)

  183. 	fmul	 %st, %st(2)

  184. 	fmulp	 %st, %st(1)

  185. 

  186. 	faddp	 %st, %st(6)

  187. 	faddp	 %st, %st(3)

  188. 	faddp	 %st, %st(3)

  189. 

  190. 	FLD	 -6 * SIZE(AO)

  191. 

  192. 	FLD	 -6 * SIZE(BO)

  193. 	fld	 %st(1)

  194. 	fmul	 %st(1), %st

  195. 	faddp	 %st, %st(3)

  196. 

  197. 	FLD	 -5 * SIZE(BO)

  198. 	fmul	 %st, %st(2)

  199. 	 

  200. 	FLD	 -5 * SIZE(AO)

  201. 	fmul	 %st, %st(2)

  202. 	fmulp	 %st, %st(1)

  203. 

  204. 	faddp	 %st, %st(6)

  205. 	faddp	 %st, %st(3)

  206. 	faddp	 %st, %st(3)

  207. 

  208. 	PREFETCH	(PREFETCHSIZE + 4) * SIZE(AO)

  209. 

  210. 	FLD	 -4 * SIZE(AO)

  211. 

  212. 	FLD	 -4 * SIZE(BO)

  213. 	fld	 %st(1)

  214. 	fmul	 %st(1), %st

  215. 	faddp	 %st, %st(3)

  216. 

  217. 	FLD	 -3 * SIZE(BO)

  218. 	fmul	 %st, %st(2)

  219. 	 

  220. 	FLD	 -3 * SIZE(AO)

  221. 	fmul	 %st, %st(2)

  222. 	fmulp	 %st, %st(1)

  223. 

  224. 	faddp	 %st, %st(6)

  225. 	faddp	 %st, %st(3)

  226. 	faddp	 %st, %st(3)

  227. 

  228. 	FLD	 -2 * SIZE(AO)

  229. 

  230. 	FLD	 -2 * SIZE(BO)

  231. 	fld	 %st(1)

  232. 	fmul	 %st(1), %st

  233. 	faddp	 %st, %st(3)

  234. 

  235. 	FLD	 -1 * SIZE(BO)

  236. 	fmul	 %st, %st(2)

  237. 	 

  238. 	FLD	 -1 * SIZE(AO)

  239. 	fmul	 %st, %st(2)

  240. 	fmulp	 %st, %st(1)

  241. 

  242. 	faddp	 %st, %st(6)

  243. 	faddp	 %st, %st(3)

  244. 	faddp	 %st, %st(3)

  245. 

  246. 	addl	$8 * SIZE,AO

  247. 	addl	$8 * SIZE,BO

  248. 

  249. 	decl	%eax

  250. 	jne	.L12

  251. 	ALIGN_4

  252. 

  253. .L15:

  254. #ifndef TRMMKERNEL

  255. 	movl	K, %eax

  256. #else

  257. 	movl	KKK, %eax

  258. #endif

  259. 	and	$3,  %eax

  260. 	je	.L18

  261. 	ALIGN_4

  262. 

  263. .L16:

  264. 	FLD	 -8 * SIZE(AO)

  265. 

  266. 	FLD	 -8 * SIZE(BO)

  267. 	fld	 %st(1)

  268. 	fmul	 %st(1), %st

  269. 	faddp	 %st, %st(3)

  270. 

  271. 	FLD	 -7 * SIZE(BO)

  272. 	fmul	 %st, %st(2)

  273. 	 

  274. 	FLD	 -7 * SIZE(AO)

  275. 	fmul	 %st, %st(2)

  276. 	fmulp	 %st, %st(1)

  277. 

  278. 	faddp	 %st, %st(6)

  279. 	faddp	 %st, %st(3)

  280. 	faddp	 %st, %st(3)

  281. 

  282. 	addl	$2 * SIZE,AO

  283. 	addl	$2 * SIZE,BO

  284. 

  285. 	decl	%eax

  286. 	jne	 .L16

  287. 	ALIGN_4

  288. 

  289. .L18:

  290. #ifndef TRMMKERNEL

  291. 	FLD	ALPHA

  292. 

  293. 	fmul	%st, %st(1)

  294. 	fmul	%st, %st(2)

  295. 	fmul	%st, %st(3)

  296. 	fmulp	%st, %st(4)

  297. 

  298. 	FLD	0 * SIZE(CO)

  299. 	faddp	%st, %st(1)

  300. 	FST	0 * SIZE(CO)

  301. 

  302. 	FLD	1 * SIZE(CO)

  303. 	faddp	%st, %st(1)

  304. 	FST	1 * SIZE(CO)

  305. 

  306. 	FLD	0 * SIZE(CO, LDC)

  307. 	faddp	%st, %st(1)

  308. 	FST	0 * SIZE(CO, LDC)

  309. 

  310. 	FLD	1 * SIZE(CO, LDC)

  311. 	faddp	%st, %st(1)

  312. 	FST	1 * SIZE(CO, LDC)

  313. #else

  314. 	FST	0 * SIZE(CO)

  315. 	FST	1 * SIZE(CO)

  316. 	FST	0 * SIZE(CO, LDC)

  317. 	FST	1 * SIZE(CO, LDC)

  318. #endif

  319. 

  320. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  321.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  322. 	movl	K, %eax

  323. 	subl	KKK, %eax

  324. 	sall	$BASE_SHIFT, %eax

  325. 	leal	(AO, %eax, 2), AO

  326. 	leal	(BO, %eax, 2), BO

  327. #endif

  328. 

  329. #if defined(TRMMKERNEL) && defined(LEFT)

  330. 	addl	$2, KK

  331. #endif

  332. 

  333. 	addl	$2 * SIZE, CO

  334. 	decl	I

  335. 	jne	.L11

  336. 	ALIGN_4

  337. 

  338. .L20:

  339. 	movl	 M, %eax

  340. 	andl	$1, %eax

  341. 	je	.L29

  342. 	ALIGN_4

  343. 

  344. .L21:

  345. #if !defined(TRMMKERNEL) || \

  346. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  347. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  348. 	movl	B, BO

  349. #else

  350. 	movl	KK,   %eax

  351. 	sall	$BASE_SHIFT, %eax

  352. 	leal	(AO, %eax, 1), AO

  353. 	leal	( B, %eax, 2), BO

  354. #endif	

  355. 

  356. 	fldz

  357. 	fldz

  358. 

  359. #ifndef TRMMKERNEL

  360. 	movl	K, %eax

  361. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  362. 	movl	K, %eax

  363. 	subl	KK, %eax

  364. 	movl	%eax, KKK	

  365. #else

  366. 	movl	KK, %eax

  367. #ifdef LEFT

  368. 	addl	$1, %eax

  369. #else

  370. 	addl	$2, %eax

  371. #endif

  372. 	movl	%eax, KKK

  373. #endif

  374. 	sarl	$2, %eax

  375.  	je	.L25

  376. 	ALIGN_4

  377. 

  378. .L22:

  379. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  380. 

  381. 	FLD	 -8 * SIZE(AO)

  382. 

  383. 	FLD	 -8 * SIZE(BO)

  384. 	fmul	 %st(1), %st

  385. 	faddp	 %st, %st(2)

  386. 

  387. 	FLD	 -7 * SIZE(BO)

  388. 	fmulp	 %st, %st(1)

  389. 	faddp	 %st, %st(2)

  390. 

  391. 	FLD	 -7 * SIZE(AO)

  392. 

  393. 	FLD	 -6 * SIZE(BO)

  394. 	fmul	 %st(1), %st

  395. 	faddp	 %st, %st(2)

  396. 

  397. 	FLD	 -5 * SIZE(BO)

  398. 	fmulp	 %st, %st(1)

  399. 	faddp	 %st, %st(2)

  400. 

  401. 	FLD	 -6 * SIZE(AO)

  402. 

  403. 	FLD	 -4 * SIZE(BO)

  404. 	fmul	 %st(1), %st

  405. 	faddp	 %st, %st(2)

  406. 

  407. 	FLD	 -3 * SIZE(BO)

  408. 	fmulp	 %st, %st(1)

  409. 	faddp	 %st, %st(2)

  410. 

  411. 	FLD	 -5 * SIZE(AO)

  412. 

  413. 	FLD	 -2 * SIZE(BO)

  414. 	fmul	 %st(1), %st

  415. 	faddp	 %st, %st(2)

  416. 

  417. 	FLD	 -1 * SIZE(BO)

  418. 	fmulp	 %st, %st(1)

  419. 	faddp	 %st, %st(2)

  420. 

  421. 	addl	$4 * SIZE,AO

  422. 	addl	$8 * SIZE,BO

  423. 

  424. 	decl	%eax

  425. 	jne	.L22

  426. 	ALIGN_4

  427. 

  428. .L25:

  429. #ifndef TRMMKERNEL

  430. 	movl	K, %eax

  431. #else

  432. 	movl	KKK, %eax

  433. #endif

  434. 	and	$3,  %eax

  435. 	je	.L28

  436. 	ALIGN_4

  437. 

  438. .L26:

  439. 	FLD	 -8 * SIZE(AO)

  440. 

  441. 	FLD	 -8 * SIZE(BO)

  442. 	fmul	 %st(1), %st

  443. 	faddp	 %st, %st(2)

  444. 

  445. 	FLD	 -7 * SIZE(BO)

  446. 	fmulp	 %st, %st(1)

  447. 	faddp	 %st, %st(2)

  448. 

  449. 	addl	$1 * SIZE,AO

  450. 	addl	$2 * SIZE,BO

  451. 

  452. 	decl	%eax

  453. 	jne	 .L26

  454. 	ALIGN_4

  455. 

  456. .L28:

  457. #ifndef TRMMKERNEL

  458. 	FLD	ALPHA

  459. 

  460. 	fmul	%st, %st(1)

  461. 	fmulp	%st, %st(2)

  462. 

  463. 	FLD	0 * SIZE(CO)

  464. 	faddp	%st, %st(1)

  465. 	FST	0 * SIZE(CO)

  466. 

  467. 	FLD	0 * SIZE(CO, LDC)

  468. 	faddp	%st, %st(1)

  469. 	FST	0 * SIZE(CO, LDC)

  470. #else

  471. 	FST	0 * SIZE(CO)

  472. 	FST	0 * SIZE(CO, LDC)

  473. #endif

  474. 

  475. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  476.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  477. 	movl	K, %eax

  478. 	subl	KKK, %eax

  479. 	sall	$BASE_SHIFT, %eax

  480. 	leal	(AO, %eax, 1), AO

  481. 	leal	(BO, %eax, 2), BO

  482. #endif

  483. 

  484. #if defined(TRMMKERNEL) && defined(LEFT)

  485. 	addl	$1, KK

  486. #endif

  487. 

  488. 	addl	$1 * SIZE, CO

  489. 	ALIGN_4

  490. 

  491. .L29:

  492. #if defined(TRMMKERNEL) && !defined(LEFT)

  493. 	addl	$2, KK

  494. #endif

  495. 

  496. 	movl	BO, B

  497. 	decl	J

  498. 	jne	.L01

  499. 	ALIGN_4

  500. 

  501. .L30:

  502. 	movl	N,  %eax

  503. 	testl	$1, %eax	

  504. 	je	.L999

  505. 

  506. #if defined(TRMMKERNEL) && defined(LEFT)

  507. 	movl	OFFSET, %eax

  508. 	movl	%eax, KK

  509. #endif	

  510. 

  511. 	movl	A, AO

  512. 

  513. 	movl	C, CO

  514. 	addl	LDC, C

  515. 

  516. 	movl	M,  I

  517. 	sarl	$1, I

  518. 	je	.L40

  519. 	ALIGN_4

  520. 

  521. .L31:

  522. #if !defined(TRMMKERNEL) || \

  523. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  524. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  525. 	movl	B, BO

  526. #else

  527. 	movl	KK,   %eax

  528. 	sall	$BASE_SHIFT, %eax

  529. 	leal	(AO, %eax, 2), AO

  530. 	leal	( B, %eax, 1), BO

  531. #endif	

  532. 

  533. 	fldz

  534. 	fldz

  535. 

  536. #if   defined(HAVE_3DNOW)

  537. 	prefetchw	2 * SIZE(CO)

  538. #elif defined(HAVE_SSE)

  539. 	prefetchnta	2 * SIZE(CO)

  540. #endif

  541. 

  542. #ifndef TRMMKERNEL

  543. 	movl	K, %eax

  544. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  545. 	movl	K, %eax

  546. 	subl	KK, %eax

  547. 	movl	%eax, KKK	

  548. #else

  549. 	movl	KK, %eax

  550. #ifdef LEFT

  551. 	addl	$2, %eax

  552. #else

  553. 	addl	$1, %eax

  554. #endif

  555. 	movl	%eax, KKK

  556. #endif

  557. 	sarl	$2, %eax

  558.  	je	.L35

  559. 	ALIGN_4

  560. 

  561. .L32:

  562. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  563. 

  564. 	FLD	 -8 * SIZE(BO)

  565. 	FLD	 -8 * SIZE(AO)

  566. 	fmul	 %st(1), %st

  567. 	faddp	 %st, %st(2)

  568. 

  569. 	FLD	 -7 * SIZE(AO)

  570. 	fmulp	 %st, %st(1)

  571. 	faddp	 %st, %st(2)

  572. 

  573. 	FLD	 -7 * SIZE(BO)

  574. 	FLD	 -6 * SIZE(AO)

  575. 	fmul	 %st(1), %st

  576. 	faddp	 %st, %st(2)

  577. 

  578. 	FLD	 -5 * SIZE(AO)

  579. 	fmulp	 %st, %st(1)

  580. 	faddp	 %st, %st(2)

  581. 

  582. 	FLD	 -6 * SIZE(BO)

  583. 	FLD	 -4 * SIZE(AO)

  584. 	fmul	 %st(1), %st

  585. 	faddp	 %st, %st(2)

  586. 

  587. 	FLD	 -3 * SIZE(AO)

  588. 	fmulp	 %st, %st(1)

  589. 	faddp	 %st, %st(2)

  590. 

  591. 	FLD	 -5 * SIZE(BO)

  592. 	FLD	 -2 * SIZE(AO)

  593. 	fmul	 %st(1), %st

  594. 	faddp	 %st, %st(2)

  595. 

  596. 	FLD	 -1 * SIZE(AO)

  597. 	fmulp	 %st, %st(1)

  598. 	faddp	 %st, %st(2)

  599. 

  600. 	addl	$8 * SIZE,AO

  601. 	addl	$4 * SIZE,BO

  602. 

  603. 	decl	%eax

  604. 	jne	.L32

  605. 	ALIGN_4

  606. 

  607. .L35:

  608. #ifndef TRMMKERNEL

  609. 	movl	K, %eax

  610. #else

  611. 	movl	KKK, %eax

  612. #endif

  613. 	and	$3,  %eax

  614. 	je	.L38

  615. 	ALIGN_4

  616. 

  617. .L36:

  618. 	FLD	 -8 * SIZE(BO)

  619. 

  620. 	FLD	 -8 * SIZE(AO)

  621. 	fmul	 %st(1), %st

  622. 	faddp	 %st, %st(2)

  623. 

  624. 	FLD	 -7 * SIZE(AO)

  625. 	fmulp	 %st, %st(1)

  626. 	faddp	 %st, %st(2)

  627. 

  628. 	addl	$2 * SIZE,AO

  629. 	addl	$1 * SIZE,BO

  630. 

  631. 	decl	%eax

  632. 	jne	 .L36

  633. 	ALIGN_4

  634. 

  635. .L38:

  636. #ifndef TRMMKERNEL

  637. 	FLD	ALPHA

  638. 

  639. 	fmul	%st, %st(1)

  640. 	fmulp	%st, %st(2)

  641. 

  642. 	FLD	0 * SIZE(CO)

  643. 	faddp	%st, %st(1)

  644. 	FST	0 * SIZE(CO)

  645. 

  646. 	FLD	1 * SIZE(CO)

  647. 	faddp	%st, %st(1)

  648. 	FST	1 * SIZE(CO)

  649. #else

  650. 	FST	0 * SIZE(CO)

  651. 	FST	1 * SIZE(CO)

  652. #endif

  653. 

  654. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  655.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  656. 	movl	K, %eax

  657. 	subl	KKK, %eax

  658. 	sall	$BASE_SHIFT, %eax

  659. 	leal	(AO, %eax, 2), AO

  660. 	leal	(BO, %eax, 1), BO

  661. #endif

  662. 

  663. #if defined(TRMMKERNEL) && defined(LEFT)

  664. 	addl	$2, KK

  665. #endif

  666. 

  667. 	addl	$2 * SIZE, CO

  668. 	decl	I

  669. 	jne	.L31

  670. 	ALIGN_4

  671. 

  672. .L40:

  673. 	movl	 M, %eax

  674. 	andl	$1, %eax

  675. 	je	.L49

  676. 	ALIGN_4

  677. 

  678. .L41:

  679. #if !defined(TRMMKERNEL) || \

  680. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  681. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  682. 	movl	B, BO

  683. #else

  684. 	movl	KK,   %eax

  685. 	sall	$BASE_SHIFT, %eax

  686. 	leal	(AO, %eax, 1), AO

  687. 	leal	( B, %eax, 1), BO

  688. #endif	

  689. 

  690. 	fldz

  691. 

  692. #ifndef TRMMKERNEL

  693. 	movl	K, %eax

  694. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  695. 	movl	K, %eax

  696. 	subl	KK, %eax

  697. 	movl	%eax, KKK	

  698. #else

  699. 	movl	KK, %eax

  700. #ifdef LEFT

  701. 	addl	$1, %eax

  702. #else

  703. 	addl	$1, %eax

  704. #endif

  705. 	movl	%eax, KKK

  706. #endif

  707. 	sarl	$2, %eax

  708.  	je	.L45

  709. 	ALIGN_4

  710. 

  711. .L42:

  712. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  713. 

  714. 	FLD	 -8 * SIZE(AO)

  715. 	FLD	 -8 * SIZE(BO)

  716. 	fmulp	 %st, %st(1)

  717. 	faddp	 %st, %st(1)

  718. 

  719. 	FLD	 -7 * SIZE(AO)

  720. 	FLD	 -7 * SIZE(BO)

  721. 	fmulp	 %st, %st(1)

  722. 	faddp	 %st, %st(1)

  723. 

  724. 	FLD	 -6 * SIZE(AO)

  725. 	FLD	 -6 * SIZE(BO)

  726. 	fmulp	 %st, %st(1)

  727. 	faddp	 %st, %st(1)

  728. 

  729. 	FLD	 -5 * SIZE(AO)

  730. 	FLD	 -5 * SIZE(BO)

  731. 	fmulp	 %st, %st(1)

  732. 	faddp	 %st, %st(1)

  733. 

  734. 	addl	$4 * SIZE,AO

  735. 	addl	$4 * SIZE,BO

  736. 

  737. 	decl	%eax

  738. 	jne	.L42

  739. 	ALIGN_4

  740. 

  741. .L45:

  742. #ifndef TRMMKERNEL

  743. 	movl	K, %eax

  744. #else

  745. 	movl	KKK, %eax

  746. #endif

  747. 	and	$3,  %eax

  748. 	je	.L48

  749. 	ALIGN_4

  750. 

  751. .L46:

  752. 	FLD	 -8 * SIZE(AO)

  753. 

  754. 	FLD	 -8 * SIZE(BO)

  755. 	fmulp	 %st, %st(1)

  756. 	faddp	 %st, %st(1)

  757. 

  758. 	addl	$1 * SIZE,AO

  759. 	addl	$1 * SIZE,BO

  760. 

  761. 	decl	%eax

  762. 	jne	 .L46

  763. 	ALIGN_4

  764. 

  765. .L48:

  766. #ifndef TRMMKERNEL

  767. 	FLD	ALPHA

  768. 

  769. 	fmulp	%st, %st(1)

  770. 

  771. 	FLD	0 * SIZE(CO)

  772. 	faddp	%st, %st(1)

  773. 	FST	0 * SIZE(CO)

  774. #else

  775. 	FST	0 * SIZE(CO)

  776. #endif

  777. 

  778. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  779.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  780. 	movl	K, %eax

  781. 	subl	KKK, %eax

  782. 	sall	$BASE_SHIFT, %eax

  783. 	leal	(AO, %eax, 1), AO

  784. 	leal	(BO, %eax, 1), BO

  785. #endif

  786. 

  787. #if defined(TRMMKERNEL) && defined(LEFT)

  788. 	addl	$1, KK

  789. #endif

  790. 

  791. 	addl	$1 * SIZE, CO

  792. 	ALIGN_4

  793. 

  794. .L49:

  795. #if defined(TRMMKERNEL) && !defined(LEFT)

  796. 	addl	$1, KK

  797. #endif

  798. 

  799. 	movl	BO, B

  800. 	ALIGN_4

  801. 

  802. .L999:

  803. 	popl	%ebx

  804. 	popl	%esi

  805. 	popl	%edi

  806. 	popl	%ebp

  807. 	addl	$ARGS, %esp

  808. 	ret

  809. 

  810. 	EPILOGUE