OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/qgemm_kernel_2x2.S

811 lines
14 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define M	ARG1

  43. #define N	ARG2

  44. #define K	ARG3

  45. #define A	ARG4

  46. #define B	ARG5

  47. #define C	ARG6

  48. #define LDC	%r10

  49. 

  50. #define I	%r12

  51. #define J	%r13

  52. #define AO	%r14

  53. #define BO	%r15

  54. #define	CO	%rbp

  55. 

  56. #define KK	%r11

  57. #define KKK	 48(%rsp)

  58. 

  59. #define STACKSIZE 64

  60. 

  61. #define ALPHA	 8 + STACKSIZE(%rsp)

  62. #define OFFSET	32 + STACKSIZE(%rsp)

  63. 

  64. #ifdef OPTERON

  65. #define PREFETCH	prefetch

  66. #define PREFETCHW	prefetchw

  67. #else

  68. #define PREFETCH	prefetcht0

  69. #define PREFETCHW	prefetcht0

  70. #endif

  71. 

  72. #define PREFETCHSIZE (5 + 4 * 10)

  73. 

  74. 	PROLOGUE

  75. 	PROFCODE

  76. 

  77. 	subq	$STACKSIZE, %rsp

  78. 	movq	%rbx,  0(%rsp)

  79. 	movq	%rbp,  8(%rsp)

  80. 	movq	%r12, 16(%rsp)

  81. 	movq	%r13, 24(%rsp)

  82. 	movq	%r14, 32(%rsp)

  83. 	movq	%r15, 40(%rsp)

  84. 

  85. 	movq	24 + STACKSIZE(%rsp), LDC

  86. 

  87. #if defined(TRMMKERNEL) && !defined(LEFT)

  88. 	movq	OFFSET, %rax

  89. 	negq	%rax

  90. 	movq	%rax, KK

  91. #endif

  92. 

  93. 	addq	$8 * SIZE, A

  94. 	addq	$8 * SIZE, B

  95. 

  96. 	salq	$BASE_SHIFT, LDC

  97. 

  98. 	movq	N,   %rax

  99. 	sarq	$1,  %rax

  100. 	movq	%rax, J

  101. 	je	.L30

  102. 	ALIGN_4

  103. 

  104. .L01:

  105. #if defined(TRMMKERNEL) && defined(LEFT)

  106. 	movq	OFFSET, %rax

  107. 	movq	%rax, KK

  108. #endif

  109. 

  110. 	movq	A, AO

  111. 

  112. 	movq	C, CO

  113. 	leaq	(, LDC, 2), %rax

  114. 	addq	%rax, C

  115. 

  116. 	movq	M,  I

  117. 	sarq	$1, I

  118. 	je	.L20

  119. 	ALIGN_4

  120. 

  121. .L11:

  122. #if !defined(TRMMKERNEL) || \

  123. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  124. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  125. 	movq	B, BO

  126. #else

  127. 	movq	KK,   %rax

  128. 	salq	$BASE_SHIFT, %rax

  129. 	leaq	(AO, %rax, 2), AO

  130. 	leaq	(B,  %rax, 2), BO

  131. #endif

  132. 

  133. 	fldz

  134. 	fldz

  135. 	fldz

  136. 	fldz

  137. 

  138. #if   defined(HAVE_3DNOW)

  139. 	prefetchw	2 * SIZE(CO)

  140.  	prefetchw	2 * SIZE(CO, LDC, 1)

  141. #elif defined(HAVE_SSE)

  142. 	prefetchnta	2 * SIZE(CO)

  143.  	prefetchnta	2 * SIZE(CO, LDC, 1)

  144. #endif

  145. 

  146. #ifndef TRMMKERNEL

  147. 	movq	K, %rax

  148. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  149. 	movq	K, %rax

  150. 	subq	KK, %rax

  151. 	movq	%rax, KKK

  152. #else

  153. 	movq	KK, %rax

  154. #ifdef LEFT

  155. 	addq	$2, %rax

  156. #else

  157. 	addq	$2, %rax

  158. #endif

  159. 	movq	%rax, KKK

  160. #endif

  161. 	sarq	$2, %rax

  162.  	je	.L15

  163. 	ALIGN_4

  164. 

  165. .L12:

  166. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  167. 

  168. 	FLD	 -8 * SIZE(AO)

  169. 

  170. 	FLD	 -8 * SIZE(BO)

  171. 	fld	 %st(1)

  172. 	fmul	 %st(1), %st

  173. 	faddp	 %st, %st(3)

  174. 

  175. 	FLD	 -7 * SIZE(BO)

  176. 	fmul	 %st, %st(2)

  177. 

  178. 	FLD	 -7 * SIZE(AO)

  179. 	fmul	 %st, %st(2)

  180. 	fmulp	 %st, %st(1)

  181. 

  182. 	faddp	 %st, %st(6)

  183. 	faddp	 %st, %st(3)

  184. 	faddp	 %st, %st(3)

  185. 

  186. 	FLD	 -6 * SIZE(AO)

  187. 

  188. 	FLD	 -6 * SIZE(BO)

  189. 	fld	 %st(1)

  190. 	fmul	 %st(1), %st

  191. 	faddp	 %st, %st(3)

  192. 

  193. 	FLD	 -5 * SIZE(BO)

  194. 	fmul	 %st, %st(2)

  195. 

  196. 	FLD	 -5 * SIZE(AO)

  197. 	fmul	 %st, %st(2)

  198. 	fmulp	 %st, %st(1)

  199. 

  200. 	faddp	 %st, %st(6)

  201. 	faddp	 %st, %st(3)

  202. 	faddp	 %st, %st(3)

  203. 

  204. 	PREFETCH	(PREFETCHSIZE + 4) * SIZE(AO)

  205. 

  206. 	FLD	 -4 * SIZE(AO)

  207. 

  208. 	FLD	 -4 * SIZE(BO)

  209. 	fld	 %st(1)

  210. 	fmul	 %st(1), %st

  211. 	faddp	 %st, %st(3)

  212. 

  213. 	FLD	 -3 * SIZE(BO)

  214. 	fmul	 %st, %st(2)

  215. 

  216. 	FLD	 -3 * SIZE(AO)

  217. 	fmul	 %st, %st(2)

  218. 	fmulp	 %st, %st(1)

  219. 

  220. 	faddp	 %st, %st(6)

  221. 	faddp	 %st, %st(3)

  222. 	faddp	 %st, %st(3)

  223. 

  224. 	FLD	 -2 * SIZE(AO)

  225. 

  226. 	FLD	 -2 * SIZE(BO)

  227. 	fld	 %st(1)

  228. 	fmul	 %st(1), %st

  229. 	faddp	 %st, %st(3)

  230. 

  231. 	FLD	 -1 * SIZE(BO)

  232. 	fmul	 %st, %st(2)

  233. 

  234. 	FLD	 -1 * SIZE(AO)

  235. 	fmul	 %st, %st(2)

  236. 	fmulp	 %st, %st(1)

  237. 

  238. 	faddp	 %st, %st(6)

  239. 	faddp	 %st, %st(3)

  240. 	faddp	 %st, %st(3)

  241. 

  242. 	addq	$8 * SIZE,AO

  243. 	addq	$8 * SIZE,BO

  244. 

  245. 	decq	%rax

  246. 	jne	.L12

  247. 	ALIGN_4

  248. 

  249. .L15:

  250. #ifndef TRMMKERNEL

  251. 	movq	K, %rax

  252. #else

  253. 	movq	KKK, %rax

  254. #endif

  255. 	and	$3,  %rax

  256. 	je	.L18

  257. 	ALIGN_4

  258. 

  259. .L16:

  260. 	FLD	 -8 * SIZE(AO)

  261. 

  262. 	FLD	 -8 * SIZE(BO)

  263. 	fld	 %st(1)

  264. 	fmul	 %st(1), %st

  265. 	faddp	 %st, %st(3)

  266. 

  267. 	FLD	 -7 * SIZE(BO)

  268. 	fmul	 %st, %st(2)

  269. 

  270. 	FLD	 -7 * SIZE(AO)

  271. 	fmul	 %st, %st(2)

  272. 	fmulp	 %st, %st(1)

  273. 

  274. 	faddp	 %st, %st(6)

  275. 	faddp	 %st, %st(3)

  276. 	faddp	 %st, %st(3)

  277. 

  278. 	addq	$2 * SIZE,AO

  279. 	addq	$2 * SIZE,BO

  280. 

  281. 	decq	%rax

  282. 	jne	 .L16

  283. 	ALIGN_4

  284. 

  285. .L18:

  286. #ifndef TRMMKERNEL

  287. 	FLD	ALPHA

  288. 

  289. 	fmul	%st, %st(1)

  290. 	fmul	%st, %st(2)

  291. 	fmul	%st, %st(3)

  292. 	fmulp	%st, %st(4)

  293. 

  294. 	FLD	0 * SIZE(CO)

  295. 	faddp	%st, %st(1)

  296. 	FST	0 * SIZE(CO)

  297. 

  298. 	FLD	1 * SIZE(CO)

  299. 	faddp	%st, %st(1)

  300. 	FST	1 * SIZE(CO)

  301. 

  302. 	FLD	0 * SIZE(CO, LDC)

  303. 	faddp	%st, %st(1)

  304. 	FST	0 * SIZE(CO, LDC)

  305. 

  306. 	FLD	1 * SIZE(CO, LDC)

  307. 	faddp	%st, %st(1)

  308. 	FST	1 * SIZE(CO, LDC)

  309. #else

  310. 	FST	0 * SIZE(CO)

  311. 	FST	1 * SIZE(CO)

  312. 	FST	0 * SIZE(CO, LDC)

  313. 	FST	1 * SIZE(CO, LDC)

  314. #endif

  315. 

  316. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  317.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  318. 	movq	K, %rax

  319. 	subq	KKK, %rax

  320. 	salq	$BASE_SHIFT, %rax

  321. 	leaq	(AO, %rax, 2), AO

  322. 	leaq	(BO, %rax, 2), BO

  323. #endif

  324. 

  325. #if defined(TRMMKERNEL) && defined(LEFT)

  326. 	addq	$2, KK

  327. #endif

  328. 

  329. 	addq	$2 * SIZE, CO

  330. 	decq	I

  331. 	jne	.L11

  332. 	ALIGN_4

  333. 

  334. .L20:

  335. 	movq	 M, %rax

  336. 	andq	$1, %rax

  337. 	je	.L29

  338. 	ALIGN_4

  339. 

  340. .L21:

  341. #if !defined(TRMMKERNEL) || \

  342. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  343. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  344. 	movq	B, BO

  345. #else

  346. 	movq	KK,   %rax

  347. 	salq	$BASE_SHIFT, %rax

  348. 	leaq	(AO, %rax, 1), AO

  349. 	leaq	( B, %rax, 2), BO

  350. #endif

  351. 

  352. 	fldz

  353. 	fldz

  354. 

  355. #ifndef TRMMKERNEL

  356. 	movq	K, %rax

  357. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  358. 	movq	K, %rax

  359. 	subq	KK, %rax

  360. 	movq	%rax, KKK

  361. #else

  362. 	movq	KK, %rax

  363. #ifdef LEFT

  364. 	addq	$1, %rax

  365. #else

  366. 	addq	$2, %rax

  367. #endif

  368. 	movq	%rax, KKK

  369. #endif

  370. 	sarq	$2, %rax

  371.  	je	.L25

  372. 	ALIGN_4

  373. 

  374. .L22:

  375. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  376. 

  377. 	FLD	 -8 * SIZE(AO)

  378. 

  379. 	FLD	 -8 * SIZE(BO)

  380. 	fmul	 %st(1), %st

  381. 	faddp	 %st, %st(2)

  382. 

  383. 	FLD	 -7 * SIZE(BO)

  384. 	fmulp	 %st, %st(1)

  385. 	faddp	 %st, %st(2)

  386. 

  387. 	FLD	 -7 * SIZE(AO)

  388. 

  389. 	FLD	 -6 * SIZE(BO)

  390. 	fmul	 %st(1), %st

  391. 	faddp	 %st, %st(2)

  392. 

  393. 	FLD	 -5 * SIZE(BO)

  394. 	fmulp	 %st, %st(1)

  395. 	faddp	 %st, %st(2)

  396. 

  397. 	FLD	 -6 * SIZE(AO)

  398. 

  399. 	FLD	 -4 * SIZE(BO)

  400. 	fmul	 %st(1), %st

  401. 	faddp	 %st, %st(2)

  402. 

  403. 	FLD	 -3 * SIZE(BO)

  404. 	fmulp	 %st, %st(1)

  405. 	faddp	 %st, %st(2)

  406. 

  407. 	FLD	 -5 * SIZE(AO)

  408. 

  409. 	FLD	 -2 * SIZE(BO)

  410. 	fmul	 %st(1), %st

  411. 	faddp	 %st, %st(2)

  412. 

  413. 	FLD	 -1 * SIZE(BO)

  414. 	fmulp	 %st, %st(1)

  415. 	faddp	 %st, %st(2)

  416. 

  417. 	addq	$4 * SIZE,AO

  418. 	addq	$8 * SIZE,BO

  419. 

  420. 	decq	%rax

  421. 	jne	.L22

  422. 	ALIGN_4

  423. 

  424. .L25:

  425. #ifndef TRMMKERNEL

  426. 	movq	K, %rax

  427. #else

  428. 	movq	KKK, %rax

  429. #endif

  430. 	and	$3,  %rax

  431. 	je	.L28

  432. 	ALIGN_4

  433. 

  434. .L26:

  435. 	FLD	 -8 * SIZE(AO)

  436. 

  437. 	FLD	 -8 * SIZE(BO)

  438. 	fmul	 %st(1), %st

  439. 	faddp	 %st, %st(2)

  440. 

  441. 	FLD	 -7 * SIZE(BO)

  442. 	fmulp	 %st, %st(1)

  443. 	faddp	 %st, %st(2)

  444. 

  445. 	addq	$1 * SIZE,AO

  446. 	addq	$2 * SIZE,BO

  447. 

  448. 	decq	%rax

  449. 	jne	 .L26

  450. 	ALIGN_4

  451. 

  452. .L28:

  453. #ifndef TRMMKERNEL

  454. 	FLD	ALPHA

  455. 

  456. 	fmul	%st, %st(1)

  457. 	fmulp	%st, %st(2)

  458. 

  459. 	FLD	0 * SIZE(CO)

  460. 	faddp	%st, %st(1)

  461. 	FST	0 * SIZE(CO)

  462. 

  463. 	FLD	0 * SIZE(CO, LDC)

  464. 	faddp	%st, %st(1)

  465. 	FST	0 * SIZE(CO, LDC)

  466. #else

  467. 	FST	0 * SIZE(CO)

  468. 	FST	0 * SIZE(CO, LDC)

  469. #endif

  470. 

  471. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  472.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  473. 	movq	K, %rax

  474. 	subq	KKK, %rax

  475. 	salq	$BASE_SHIFT, %rax

  476. 	leaq	(AO, %rax, 1), AO

  477. 	leaq	(BO, %rax, 2), BO

  478. #endif

  479. 

  480. #if defined(TRMMKERNEL) && defined(LEFT)

  481. 	addq	$1, KK

  482. #endif

  483. 

  484. 	addq	$1 * SIZE, CO

  485. 	ALIGN_4

  486. 

  487. .L29:

  488. #if defined(TRMMKERNEL) && !defined(LEFT)

  489. 	addq	$2, KK

  490. #endif

  491. 

  492. 	movq	BO, B

  493. 	decq	J

  494. 	jne	.L01

  495. 	ALIGN_4

  496. 

  497. .L30:

  498. 	movq	N,  %rax

  499. 	testq	$1, %rax

  500. 	je	.L999

  501. 

  502. #if defined(TRMMKERNEL) && defined(LEFT)

  503. 	movq	OFFSET, %rax

  504. 	movq	%rax, KK

  505. #endif

  506. 

  507. 	movq	A, AO

  508. 

  509. 	movq	C, CO

  510. 	addq	LDC, C

  511. 

  512. 	movq	M,  I

  513. 	sarq	$1, I

  514. 	je	.L40

  515. 	ALIGN_4

  516. 

  517. .L31:

  518. #if !defined(TRMMKERNEL) || \

  519. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  520. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  521. 	movq	B, BO

  522. #else

  523. 	movq	KK,   %rax

  524. 	salq	$BASE_SHIFT, %rax

  525. 	leaq	(AO, %rax, 2), AO

  526. 	leaq	( B, %rax, 1), BO

  527. #endif

  528. 

  529. 	fldz

  530. 	fldz

  531. 

  532. #if   defined(HAVE_3DNOW)

  533. 	prefetchw	2 * SIZE(CO)

  534. #elif defined(HAVE_SSE)

  535. 	prefetchnta	2 * SIZE(CO)

  536. #endif

  537. 

  538. #ifndef TRMMKERNEL

  539. 	movq	K, %rax

  540. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  541. 	movq	K, %rax

  542. 	subq	KK, %rax

  543. 	movq	%rax, KKK

  544. #else

  545. 	movq	KK, %rax

  546. #ifdef LEFT

  547. 	addq	$2, %rax

  548. #else

  549. 	addq	$1, %rax

  550. #endif

  551. 	movq	%rax, KKK

  552. #endif

  553. 	sarq	$2, %rax

  554.  	je	.L35

  555. 	ALIGN_4

  556. 

  557. .L32:

  558. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  559. 

  560. 	FLD	 -8 * SIZE(BO)

  561. 	FLD	 -8 * SIZE(AO)

  562. 	fmul	 %st(1), %st

  563. 	faddp	 %st, %st(2)

  564. 

  565. 	FLD	 -7 * SIZE(AO)

  566. 	fmulp	 %st, %st(1)

  567. 	faddp	 %st, %st(2)

  568. 

  569. 	FLD	 -7 * SIZE(BO)

  570. 	FLD	 -6 * SIZE(AO)

  571. 	fmul	 %st(1), %st

  572. 	faddp	 %st, %st(2)

  573. 

  574. 	FLD	 -5 * SIZE(AO)

  575. 	fmulp	 %st, %st(1)

  576. 	faddp	 %st, %st(2)

  577. 

  578. 	FLD	 -6 * SIZE(BO)

  579. 	FLD	 -4 * SIZE(AO)

  580. 	fmul	 %st(1), %st

  581. 	faddp	 %st, %st(2)

  582. 

  583. 	FLD	 -3 * SIZE(AO)

  584. 	fmulp	 %st, %st(1)

  585. 	faddp	 %st, %st(2)

  586. 

  587. 	FLD	 -5 * SIZE(BO)

  588. 	FLD	 -2 * SIZE(AO)

  589. 	fmul	 %st(1), %st

  590. 	faddp	 %st, %st(2)

  591. 

  592. 	FLD	 -1 * SIZE(AO)

  593. 	fmulp	 %st, %st(1)

  594. 	faddp	 %st, %st(2)

  595. 

  596. 	addq	$8 * SIZE,AO

  597. 	addq	$4 * SIZE,BO

  598. 

  599. 	decq	%rax

  600. 	jne	.L32

  601. 	ALIGN_4

  602. 

  603. .L35:

  604. #ifndef TRMMKERNEL

  605. 	movq	K, %rax

  606. #else

  607. 	movq	KKK, %rax

  608. #endif

  609. 	and	$3,  %rax

  610. 	je	.L38

  611. 	ALIGN_4

  612. 

  613. .L36:

  614. 	FLD	 -8 * SIZE(BO)

  615. 

  616. 	FLD	 -8 * SIZE(AO)

  617. 	fmul	 %st(1), %st

  618. 	faddp	 %st, %st(2)

  619. 

  620. 	FLD	 -7 * SIZE(AO)

  621. 	fmulp	 %st, %st(1)

  622. 	faddp	 %st, %st(2)

  623. 

  624. 	addq	$2 * SIZE,AO

  625. 	addq	$1 * SIZE,BO

  626. 

  627. 	decq	%rax

  628. 	jne	 .L36

  629. 	ALIGN_4

  630. 

  631. .L38:

  632. #ifndef TRMMKERNEL

  633. 	FLD	ALPHA

  634. 

  635. 	fmul	%st, %st(1)

  636. 	fmulp	%st, %st(2)

  637. 

  638. 	FLD	0 * SIZE(CO)

  639. 	faddp	%st, %st(1)

  640. 	FST	0 * SIZE(CO)

  641. 

  642. 	FLD	1 * SIZE(CO)

  643. 	faddp	%st, %st(1)

  644. 	FST	1 * SIZE(CO)

  645. #else

  646. 	FST	0 * SIZE(CO)

  647. 	FST	1 * SIZE(CO)

  648. #endif

  649. 

  650. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  651.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  652. 	movq	K, %rax

  653. 	subq	KKK, %rax

  654. 	salq	$BASE_SHIFT, %rax

  655. 	leaq	(AO, %rax, 2), AO

  656. 	leaq	(BO, %rax, 1), BO

  657. #endif

  658. 

  659. #if defined(TRMMKERNEL) && defined(LEFT)

  660. 	addq	$2, KK

  661. #endif

  662. 

  663. 	addq	$2 * SIZE, CO

  664. 	decq	I

  665. 	jne	.L31

  666. 	ALIGN_4

  667. 

  668. .L40:

  669. 	movq	 M, %rax

  670. 	andq	$1, %rax

  671. 	je	.L49

  672. 	ALIGN_4

  673. 

  674. .L41:

  675. #if !defined(TRMMKERNEL) || \

  676. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  677. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  678. 	movq	B, BO

  679. #else

  680. 	movq	KK,   %rax

  681. 	salq	$BASE_SHIFT, %rax

  682. 	leaq	(AO, %rax, 1), AO

  683. 	leaq	( B, %rax, 1), BO

  684. #endif

  685. 

  686. 	fldz

  687. 

  688. #ifndef TRMMKERNEL

  689. 	movq	K, %rax

  690. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  691. 	movq	K, %rax

  692. 	subq	KK, %rax

  693. 	movq	%rax, KKK

  694. #else

  695. 	movq	KK, %rax

  696. #ifdef LEFT

  697. 	addq	$1, %rax

  698. #else

  699. 	addq	$1, %rax

  700. #endif

  701. 	movq	%rax, KKK

  702. #endif

  703. 	sarq	$2, %rax

  704.  	je	.L45

  705. 	ALIGN_4

  706. 

  707. .L42:

  708. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(AO)

  709. 

  710. 	FLD	 -8 * SIZE(AO)

  711. 	FLD	 -8 * SIZE(BO)

  712. 	fmulp	 %st, %st(1)

  713. 	faddp	 %st, %st(1)

  714. 

  715. 	FLD	 -7 * SIZE(AO)

  716. 	FLD	 -7 * SIZE(BO)

  717. 	fmulp	 %st, %st(1)

  718. 	faddp	 %st, %st(1)

  719. 

  720. 	FLD	 -6 * SIZE(AO)

  721. 	FLD	 -6 * SIZE(BO)

  722. 	fmulp	 %st, %st(1)

  723. 	faddp	 %st, %st(1)

  724. 

  725. 	FLD	 -5 * SIZE(AO)

  726. 	FLD	 -5 * SIZE(BO)

  727. 	fmulp	 %st, %st(1)

  728. 	faddp	 %st, %st(1)

  729. 

  730. 	addq	$4 * SIZE,AO

  731. 	addq	$4 * SIZE,BO

  732. 

  733. 	decq	%rax

  734. 	jne	.L42

  735. 	ALIGN_4

  736. 

  737. .L45:

  738. #ifndef TRMMKERNEL

  739. 	movq	K, %rax

  740. #else

  741. 	movq	KKK, %rax

  742. #endif

  743. 	and	$3,  %rax

  744. 	je	.L48

  745. 	ALIGN_4

  746. 

  747. .L46:

  748. 	FLD	 -8 * SIZE(AO)

  749. 

  750. 	FLD	 -8 * SIZE(BO)

  751. 	fmulp	 %st, %st(1)

  752. 	faddp	 %st, %st(1)

  753. 

  754. 	addq	$1 * SIZE,AO

  755. 	addq	$1 * SIZE,BO

  756. 

  757. 	decq	%rax

  758. 	jne	 .L46

  759. 	ALIGN_4

  760. 

  761. .L48:

  762. #ifndef TRMMKERNEL

  763. 	FLD	ALPHA

  764. 

  765. 	fmulp	%st, %st(1)

  766. 

  767. 	FLD	0 * SIZE(CO)

  768. 	faddp	%st, %st(1)

  769. 	FST	0 * SIZE(CO)

  770. #else

  771. 	FST	0 * SIZE(CO)

  772. #endif

  773. 

  774. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  775.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  776. 	movq	K, %rax

  777. 	subq	KKK, %rax

  778. 	salq	$BASE_SHIFT, %rax

  779. 	leaq	(AO, %rax, 1), AO

  780. 	leaq	(BO, %rax, 1), BO

  781. #endif

  782. 

  783. #if defined(TRMMKERNEL) && defined(LEFT)

  784. 	addq	$1, KK

  785. #endif

  786. 

  787. 	addq	$1 * SIZE, CO

  788. 	ALIGN_4

  789. 

  790. .L49:

  791. #if defined(TRMMKERNEL) && !defined(LEFT)

  792. 	addq	$1, KK

  793. #endif

  794. 

  795. 	movq	BO, B

  796. 	ALIGN_4

  797. 

  798. .L999:

  799. 	EMMS

  800. 

  801. 	movq	  0(%rsp), %rbx

  802. 	movq	  8(%rsp), %rbp

  803. 	movq	 16(%rsp), %r12

  804. 	movq	 24(%rsp), %r13

  805. 	movq	 32(%rsp), %r14

  806. 	movq	 40(%rsp), %r15

  807. 	addq	$STACKSIZE, %rsp

  808. 	ret

  809. 

  810. 	EPILOGUE