OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 869a169c57
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '869a169c57'
${ noResults }
OpenBLAS/kernel/x86/gemm_kernel_2x2_atom.S

737 lines
14 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS	16

  44. 

  45. #define M	 4 + STACK + ARGS(%esp)

  46. #define N	 8 + STACK + ARGS(%esp)

  47. #define K	12 + STACK + ARGS(%esp)

  48. #define ALPHA	16 + STACK + ARGS(%esp)

  49. #define A	24 + STACK + ARGS(%esp)

  50. #define ARG_B	28 + STACK + ARGS(%esp)

  51. #define C	32 + STACK + ARGS(%esp)

  52. #define ARG_LDC	36 + STACK + ARGS(%esp)

  53. #define OFFSET	40 + STACK + ARGS(%esp)

  54. 

  55. #define J	 0 + STACK(%esp)

  56. #define BX	 4 + STACK(%esp)

  57. #define KK	 8 + STACK(%esp)

  58. #define KKK	12 + STACK(%esp)

  59. 

  60. #define PREFETCH     prefetcht0

  61. #define PREFETCHSIZE 84

  62. 

  63. #define AA	%edx

  64. #define BB	%ecx

  65. #define CO1	%esi

  66. #define LDC	%ebp

  67. #define B	%edi

  68. 

  69. 	PROLOGUE

  70. 

  71. 	subl	$ARGS, %esp

  72. 

  73. 	pushl	%ebp

  74. 	pushl	%edi

  75. 	pushl	%esi

  76. 	pushl	%ebx

  77. 

  78. 	PROFCODE

  79. 

  80. 	movl	ARG_B,   B

  81. 	movl	ARG_LDC, LDC

  82. 

  83. #ifdef TRMMKERNEL

  84. 	movl	OFFSET, %eax

  85. #ifndef LEFT

  86. 	negl	%eax

  87. #endif

  88. 	movl	%eax, KK

  89. #endif

  90. 

  91. 	leal	(, LDC, SIZE), LDC

  92. 

  93. 	movl	N,  %eax

  94. 	sarl	$1, %eax

  95. 	movl	%eax, J

  96. 	jle	.L30

  97. 	ALIGN_2

  98. 

  99. .L10:

  100. #if defined(TRMMKERNEL) && defined(LEFT)

  101. 	movl	OFFSET, %eax

  102. 	movl	%eax, KK

  103. #endif

  104. 

  105. 	movl	K, %eax

  106. 	sall	$BASE_SHIFT + 1, %eax

  107. 	leal	(B, %eax), %eax

  108. 	movl	%eax, BX

  109. 

  110. 	movl	C, CO1		# coffset = c

  111. 	leal	(, LDC, 2), %eax

  112. 	addl	%eax, C

  113. 

  114. 	movl	A, AA		# aoffset = a

  115. 

  116. 	movl	M,  %ebx

  117. 	sarl	$1, %ebx	# i = (m >> 2)

  118. 	jle	.L20

  119. 	ALIGN_4

  120. 

  121. .L11:

  122. #if !defined(TRMMKERNEL) || \

  123. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  124. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  125. 

  126. 	movl	B, BB

  127. #else

  128. 	movl	KK, %eax

  129. 	leal	(, %eax, SIZE), %eax

  130. 	leal	(AA, %eax, 2), AA

  131. 	leal	(B,  %eax, 2), BB

  132. #endif

  133. 

  134. 	movl	BX, %eax

  135. 	prefetcht0  0 * SIZE(%eax)

  136. 	subl	$-8 * SIZE, BX

  137. 

  138. 	movsd	 0 * SIZE(AA), %xmm0

  139. 	xorps	%xmm2, %xmm2

  140. 	xorps	%xmm3, %xmm3

  141. 

  142. 	xorps	%xmm4, %xmm4

  143. 	prefetcht0	3 * SIZE(CO1)

  144. 	xorps	%xmm5, %xmm5

  145. 	prefetcht0	3 * SIZE(CO1, LDC)

  146. 	xorps	%xmm6, %xmm6

  147. 	xorps	%xmm7, %xmm7

  148. 

  149. #ifndef TRMMKERNEL

  150. 	movl	K, %eax

  151. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  152. 	movl	K, %eax

  153. 	subl	KK, %eax

  154. 	movl	%eax, KKK

  155. #else

  156. 	movl	KK, %eax

  157. #ifdef LEFT

  158. 	addl	$2, %eax

  159. #else

  160. 	addl	$2, %eax

  161. #endif

  162. 	movl	%eax, KKK

  163. #endif

  164. 	sarl	$2, %eax

  165. 	je	.L15

  166. 	ALIGN_4

  167. 

  168. .L12:

  169. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE(AA)

  170. 

  171. 	addsd	%xmm2, %xmm6

  172. 	movsd	 1 * SIZE(AA), %xmm2

  173. 	movaps	%xmm0, %xmm1

  174. 	mulsd	 0 * SIZE(BB), %xmm0

  175. 	addsd	%xmm3, %xmm7

  176. 	mulsd	 1 * SIZE(BB), %xmm1

  177. 

  178. 	addsd	%xmm0, %xmm4

  179. 	movsd	 2 * SIZE(AA), %xmm0

  180. 	movaps	%xmm2, %xmm3

  181. 	mulsd	 0 * SIZE(BB), %xmm2

  182. 	addsd	%xmm1, %xmm5

  183. 	mulsd	 1 * SIZE(BB), %xmm3

  184. 

  185. 	addsd	%xmm2, %xmm6

  186. 	movsd	 3 * SIZE(AA), %xmm2

  187. 	movaps	%xmm0, %xmm1

  188. 	mulsd	 2 * SIZE(BB), %xmm0

  189. 	addsd	%xmm3, %xmm7

  190. 	mulsd	 3 * SIZE(BB), %xmm1

  191. 

  192. 	addsd	%xmm0, %xmm4

  193. 	movsd	 4 * SIZE(AA), %xmm0

  194. 	movaps	%xmm2, %xmm3

  195. 	mulsd	 2 * SIZE(BB), %xmm2

  196. 	addsd	%xmm1, %xmm5

  197. 	mulsd	 3 * SIZE(BB), %xmm3

  198. 

  199. 	addsd	%xmm2, %xmm6

  200. 	movsd	 5 * SIZE(AA), %xmm2

  201. 	movaps	%xmm0, %xmm1

  202. 	mulsd	 4 * SIZE(BB), %xmm0

  203. 	addsd	%xmm3, %xmm7

  204. 	mulsd	 5 * SIZE(BB), %xmm1

  205. 

  206. 	addsd	%xmm0, %xmm4

  207. 	movsd	 6 * SIZE(AA), %xmm0

  208. 	movaps	%xmm2, %xmm3

  209. 	mulsd	 4 * SIZE(BB), %xmm2

  210. 	addsd	%xmm1, %xmm5

  211. 	mulsd	 5 * SIZE(BB), %xmm3

  212. 

  213. 	addsd	%xmm2, %xmm6

  214. 	movsd	 7 * SIZE(AA), %xmm2

  215. 	movaps	%xmm0, %xmm1

  216. 	mulsd	 6 * SIZE(BB), %xmm0

  217. 	addsd	%xmm3, %xmm7

  218. 	mulsd	 7 * SIZE(BB), %xmm1

  219. 

  220. 	addsd	%xmm0, %xmm4

  221. 	movsd	 8 * SIZE(AA), %xmm0

  222. 	movaps	%xmm2, %xmm3

  223. 	mulsd	 6 * SIZE(BB), %xmm2

  224. 	addsd	%xmm1, %xmm5

  225. 	mulsd	 7 * SIZE(BB), %xmm3

  226. 

  227. 	addl   $8 * SIZE, BB

  228. 	addl   $8 * SIZE, AA

  229. 	decl   %eax

  230. 	jne    .L12

  231. 	ALIGN_4

  232. 

  233. .L15:

  234. #ifndef TRMMKERNEL

  235. 	movl	K, %eax

  236. #else

  237. 	movl	KKK, %eax

  238. #endif

  239. 	andl	$3, %eax		# if (k & 1)

  240. 	BRANCH

  241. 	je .L18

  242. 	ALIGN_3

  243. 

  244. .L16:

  245. 	addsd	%xmm2, %xmm6

  246. 	movsd	 1 * SIZE(AA), %xmm2

  247. 	movaps	%xmm0, %xmm1

  248. 	mulsd	 0 * SIZE(BB), %xmm0

  249. 	addsd	%xmm3, %xmm7

  250. 	mulsd	 1 * SIZE(BB), %xmm1

  251. 

  252. 	addsd	%xmm0, %xmm4

  253. 	movsd	 2 * SIZE(AA), %xmm0

  254. 	movaps	%xmm2, %xmm3

  255. 	mulsd	 0 * SIZE(BB), %xmm2

  256. 	addsd	%xmm1, %xmm5

  257. 	mulsd	 1 * SIZE(BB), %xmm3

  258. 

  259. 	addl	$2 * SIZE, AA

  260. 	addl	$2 * SIZE, BB

  261. 	decl	%eax

  262. 	jg	.L16

  263. 	ALIGN_4

  264. 

  265. .L18:

  266. 	movsd	ALPHA,  %xmm0

  267. 

  268. 	addsd	%xmm2, %xmm6

  269. 	addsd	%xmm3, %xmm7

  270. 

  271. 	mulsd	%xmm0, %xmm4

  272. 	mulsd	%xmm0, %xmm5

  273. 	mulsd	%xmm0, %xmm6

  274. 	mulsd	%xmm0, %xmm7

  275. 

  276. #ifndef TRMMKERNEL

  277. 	addsd	0 * SIZE(CO1), %xmm4

  278. 	addsd	1 * SIZE(CO1), %xmm6

  279. 

  280. 	addsd	0 * SIZE(CO1, LDC), %xmm5

  281. 	addsd	1 * SIZE(CO1, LDC), %xmm7

  282. #endif

  283. 

  284. 	movsd	%xmm4, 0 * SIZE(CO1)

  285. 	movsd	%xmm6, 1 * SIZE(CO1)

  286. 	movsd	%xmm5, 0 * SIZE(CO1, LDC)

  287. 	movsd	%xmm7, 1 * SIZE(CO1, LDC)

  288. 

  289. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  290.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  291. 	movl	K, %eax

  292. 	subl	KKK, %eax

  293. 	leal	(,%eax, SIZE), %eax

  294. 	leal	(AA, %eax, 2), AA

  295. 	leal	(BB, %eax, 2), BB

  296. #endif

  297. 

  298. #if defined(TRMMKERNEL) && defined(LEFT)

  299. 	addl	$2, KK

  300. #endif

  301. 

  302. 	addl	$2 * SIZE, CO1

  303. 	decl	%ebx

  304. 	jg	.L11

  305. 	ALIGN_4

  306. 

  307. .L20:

  308. 	movl	M,  %ebx

  309. 	testl	$1, %ebx

  310. 	jle	.L29

  311. 

  312. #if !defined(TRMMKERNEL) || \

  313. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  314. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  315. 

  316. 	movl	B, BB

  317. #else

  318. 	movl	KK, %eax

  319. 	leal	(, %eax, SIZE), %eax

  320. 	leal	(AA, %eax, 1), AA

  321. 	leal	(B,  %eax, 2), BB

  322. #endif

  323. 

  324. 	movsd	 0 * SIZE(AA), %xmm0

  325. 	xorps	%xmm2, %xmm2

  326. 	xorps	%xmm3, %xmm3

  327. 	xorps	%xmm4, %xmm4

  328. 	xorps	%xmm5, %xmm5

  329. 

  330. #ifndef TRMMKERNEL

  331. 	movl	K, %eax

  332. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  333. 	movl	K, %eax

  334. 	subl	KK, %eax

  335. 	movl	%eax, KKK

  336. #else

  337. 	movl	KK, %eax

  338. #ifdef LEFT

  339. 	addl	$1, %eax

  340. #else

  341. 	addl	$2, %eax

  342. #endif

  343. 	movl	%eax, KKK

  344. #endif

  345. 	sarl	$2, %eax

  346. 	je	.L25

  347. 	ALIGN_4

  348. 

  349. .L22:

  350. 	addsd	%xmm2, %xmm4

  351. 	movsd	 0 * SIZE(BB), %xmm2

  352. 	addsd	%xmm3, %xmm5

  353. 	movsd	 1 * SIZE(BB), %xmm3

  354. 

  355. 	mulsd	%xmm0, %xmm2

  356. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE(AA)

  357. 	mulsd	%xmm0, %xmm3

  358. 	movsd	 1 * SIZE(AA), %xmm0

  359. 

  360. 	addsd	%xmm2, %xmm4

  361. 	movsd	 2 * SIZE(BB), %xmm2

  362. 	addsd	%xmm3, %xmm5

  363. 	movsd	 3 * SIZE(BB), %xmm3

  364. 

  365. 	mulsd	%xmm0, %xmm2

  366. 	mulsd	%xmm0, %xmm3

  367. 	movsd	 2 * SIZE(AA), %xmm0

  368. 

  369. 	addsd	%xmm2, %xmm4

  370. 	movsd	 4 * SIZE(BB), %xmm2

  371. 	addsd	%xmm3, %xmm5

  372. 	movsd	 5 * SIZE(BB), %xmm3

  373. 

  374. 	mulsd	%xmm0, %xmm2

  375. 	mulsd	%xmm0, %xmm3

  376. 	movsd	 3 * SIZE(AA), %xmm0

  377. 

  378. 	addsd	%xmm2, %xmm4

  379. 	movsd	 6 * SIZE(BB), %xmm2

  380. 	addsd	%xmm3, %xmm5

  381. 	movsd	 7 * SIZE(BB), %xmm3

  382. 

  383. 	mulsd	%xmm0, %xmm2

  384. 	mulsd	%xmm0, %xmm3

  385. 	movsd	 4 * SIZE(AA), %xmm0

  386. 

  387. 	addl   $4 * SIZE, AA

  388. 	addl   $8 * SIZE, BB

  389. 	decl   %eax

  390. 	jne    .L22

  391. 	ALIGN_4

  392. 

  393. .L25:

  394. #ifndef TRMMKERNEL

  395. 	movl	K, %eax

  396. #else

  397. 	movl	KKK, %eax

  398. #endif

  399. 	andl	$3, %eax		# if (k & 1)

  400. 	BRANCH

  401. 	je .L28

  402. 	ALIGN_3

  403. 

  404. .L26:

  405. 	addsd	%xmm2, %xmm4

  406. 	movsd	 0 * SIZE(BB), %xmm2

  407. 	addsd	%xmm3, %xmm5

  408. 	movsd	 1 * SIZE(BB), %xmm3

  409. 

  410. 	mulsd	%xmm0, %xmm2

  411. 	mulsd	%xmm0, %xmm3

  412. 	movsd	 1 * SIZE(AA), %xmm0

  413. 

  414. 	addl	$1 * SIZE, AA

  415. 	addl	$2 * SIZE, BB

  416. 	decl	%eax

  417. 	jg	.L26

  418. 	ALIGN_4

  419. 

  420. .L28:

  421. 	movsd	ALPHA,  %xmm0

  422. 

  423. 	addsd	%xmm2, %xmm4

  424. 	addsd	%xmm3, %xmm5

  425. 

  426. 	mulsd	%xmm0, %xmm4

  427. 	mulsd	%xmm0, %xmm5

  428. 

  429. #ifndef TRMMKERNEL

  430. 	addsd	0 * SIZE(CO1), %xmm4

  431. 	addsd	0 * SIZE(CO1, LDC), %xmm5

  432. #endif

  433. 

  434. 	movsd	%xmm4, 0 * SIZE(CO1)

  435. 	movsd	%xmm5, 0 * SIZE(CO1, LDC)

  436. 

  437. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  438.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  439. 	movl	K, %eax

  440. 	subl	KKK, %eax

  441. 	leal	(,%eax, SIZE), %eax

  442. 	leal	(AA, %eax, 1), AA

  443. 	leal	(BB, %eax, 2), BB

  444. #endif

  445. 

  446. #if defined(TRMMKERNEL) && defined(LEFT)

  447. 	addl	$1, KK

  448. #endif

  449. 

  450. 	addl	$1 * SIZE, CO1

  451. 	ALIGN_4

  452. 

  453. .L29:

  454. #if defined(TRMMKERNEL) && !defined(LEFT)

  455. 	addl	$2, KK

  456. #endif

  457. 

  458. 	movl	BB, B

  459. 	decl	J

  460. 	jg	.L10

  461. 	ALIGN_4

  462. 

  463. .L30:

  464. 	testl	$1, N

  465. 	je	.L999

  466. 

  467. #if defined(TRMMKERNEL) && defined(LEFT)

  468. 	movl	OFFSET, %eax

  469. 	movl	%eax, KK

  470. #endif

  471. 

  472. 	movl	C, CO1

  473. 	addl	LDC, C

  474. 

  475. 	movl	A, AA

  476. 

  477. 	movl	M,  %ebx

  478. 	sarl	$1, %ebx

  479. 	jle	.L40

  480. 	ALIGN_4

  481. 

  482. .L31:

  483. #if !defined(TRMMKERNEL) || \

  484. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  485. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  486. 

  487. 	movl	B, BB

  488. #else

  489. 	movl	KK, %eax

  490. 	leal	(, %eax, SIZE), %eax

  491. 	leal	(AA, %eax, 2), AA

  492. 	leal	(B,  %eax, 1), BB

  493. #endif

  494. 

  495. 	movsd	 0 * SIZE(BB), %xmm1

  496. 	xorps	%xmm0, %xmm0

  497. 	prefetcht0	3 * SIZE(CO1)

  498. 	xorps	%xmm2, %xmm2

  499. 	xorps	%xmm4, %xmm4

  500. 	xorps	%xmm6, %xmm6

  501. 

  502. #ifndef TRMMKERNEL

  503. 	movl	K, %eax

  504. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  505. 	movl	K, %eax

  506. 	subl	KK, %eax

  507. 	movl	%eax, KKK

  508. #else

  509. 	movl	KK, %eax

  510. #ifdef LEFT

  511. 	addl	$2, %eax

  512. #else

  513. 	addl	$1, %eax

  514. #endif

  515. 	movl	%eax, KKK

  516. #endif

  517. 	sarl	$2, %eax

  518. 	je	.L35

  519. 	ALIGN_4

  520. 

  521. .L32:

  522. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE(AA)

  523. 

  524. 	addsd	%xmm0, %xmm4

  525. 	movsd	 0 * SIZE(AA), %xmm0

  526. 	addsd	%xmm2, %xmm6

  527. 	movsd	 1 * SIZE(AA), %xmm2

  528. 	mulsd	 %xmm1, %xmm0

  529. 	mulsd	 %xmm1, %xmm2

  530. 	movsd	 1 * SIZE(BB), %xmm1

  531. 

  532. 	addsd	%xmm0, %xmm4

  533. 	movsd	 2 * SIZE(AA), %xmm0

  534. 	addsd	%xmm2, %xmm6

  535. 	movsd	 3 * SIZE(AA), %xmm2

  536. 	mulsd	 %xmm1, %xmm0

  537. 	mulsd	 %xmm1, %xmm2

  538. 	movsd	 2 * SIZE(BB), %xmm1

  539. 

  540. 	addsd	%xmm0, %xmm4

  541. 	movsd	 4 * SIZE(AA), %xmm0

  542. 	addsd	%xmm2, %xmm6

  543. 	movsd	 5 * SIZE(AA), %xmm2

  544. 	mulsd	 %xmm1, %xmm0

  545. 	mulsd	 %xmm1, %xmm2

  546. 	movsd	 3 * SIZE(BB), %xmm1

  547. 

  548. 	addsd	%xmm0, %xmm4

  549. 	movsd	 6 * SIZE(AA), %xmm0

  550. 	addsd	%xmm2, %xmm6

  551. 	movsd	 7 * SIZE(AA), %xmm2

  552. 	mulsd	 %xmm1, %xmm0

  553. 	mulsd	 %xmm1, %xmm2

  554. 	movsd	 4 * SIZE(BB), %xmm1

  555. 

  556. 	addl   $8 * SIZE, AA

  557. 	addl   $4 * SIZE, BB

  558. 	decl   %eax

  559. 	jne    .L32

  560. 	ALIGN_4

  561. 

  562. .L35:

  563. #ifndef TRMMKERNEL

  564. 	movl	K, %eax

  565. #else

  566. 	movl	KKK, %eax

  567. #endif

  568. 	andl	$3, %eax		# if (k & 1)

  569. 	BRANCH

  570. 	je .L38

  571. 	ALIGN_3

  572. 

  573. .L36:

  574. 	addsd	%xmm0, %xmm4

  575. 	movsd	 0 * SIZE(AA), %xmm0

  576. 	addsd	%xmm2, %xmm6

  577. 	movsd	 1 * SIZE(AA), %xmm2

  578. 	mulsd	 %xmm1, %xmm0

  579. 	mulsd	 %xmm1, %xmm2

  580. 	movsd	 1 * SIZE(BB), %xmm1

  581. 

  582. 	addl	$2 * SIZE, AA

  583. 	addl	$1 * SIZE, BB

  584. 	decl	%eax

  585. 	jg	.L36

  586. 	ALIGN_4

  587. 

  588. .L38:

  589. 	movsd	ALPHA,  %xmm3

  590. 

  591. 	addsd	%xmm0, %xmm4

  592. 	addsd	%xmm2, %xmm6

  593. 

  594. 	mulsd	%xmm3, %xmm4

  595. 	mulsd	%xmm3, %xmm6

  596. 

  597. #ifndef TRMMKERNEL

  598. 	addsd	0 * SIZE(CO1), %xmm4

  599. 	addsd	1 * SIZE(CO1), %xmm6

  600. #endif

  601. 

  602. 	movsd	%xmm4, 0 * SIZE(CO1)

  603. 	movsd	%xmm6, 1 * SIZE(CO1)

  604. 

  605. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  606.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  607. 	movl	K, %eax

  608. 	subl	KKK, %eax

  609. 	leal	(,%eax, SIZE), %eax

  610. 	leal	(AA, %eax, 2), AA

  611. 	leal	(BB, %eax, 1), BB

  612. #endif

  613. 

  614. #if defined(TRMMKERNEL) && defined(LEFT)

  615. 	addl	$2, KK

  616. #endif

  617. 

  618. 	addl	$2 * SIZE, CO1

  619. 	decl	%ebx

  620. 	jg	.L31

  621. 	ALIGN_4

  622. 

  623. .L40:

  624. 	movl	M,  %ebx

  625. 	testl	$1, %ebx

  626. 	jle	.L999

  627. 

  628. #if !defined(TRMMKERNEL) || \

  629. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  630. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  631. 

  632. 	movl	B, BB

  633. #else

  634. 	movl	KK, %eax

  635. 	leal	(, %eax, SIZE), %eax

  636. 	leal	(AA, %eax, 1), AA

  637. 	leal	(B,  %eax, 1), BB

  638. #endif

  639. 

  640. 	movsd	 0 * SIZE(AA), %xmm0

  641. 	xorps	%xmm4, %xmm4

  642. 	movsd	 0 * SIZE(BB), %xmm2

  643. 	xorps	%xmm5, %xmm5

  644. 

  645. #ifndef TRMMKERNEL

  646. 	movl	K, %eax

  647. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  648. 	movl	K, %eax

  649. 	subl	KK, %eax

  650. 	movl	%eax, KKK

  651. #else

  652. 	movl	KK, %eax

  653. #ifdef LEFT

  654. 	addl	$1, %eax

  655. #else

  656. 	addl	$1, %eax

  657. #endif

  658. 	movl	%eax, KKK

  659. #endif

  660. 	sarl	$2, %eax

  661. 	je	.L45

  662. 	ALIGN_4

  663. 

  664. .L42:

  665. 	mulsd	%xmm0, %xmm2

  666. 	movsd	 1 * SIZE(AA), %xmm0

  667. 	addsd	%xmm2, %xmm4

  668. 	movsd	 1 * SIZE(BB), %xmm2

  669. 

  670. 	mulsd	%xmm0, %xmm2

  671. 	movsd	 2 * SIZE(AA), %xmm0

  672. 	addsd	%xmm2, %xmm5

  673. 	movsd	 2 * SIZE(BB), %xmm2

  674. 

  675. 	mulsd	%xmm0, %xmm2

  676. 	movsd	 3 * SIZE(AA), %xmm0

  677. 	addsd	%xmm2, %xmm4

  678. 	movsd	 3 * SIZE(BB), %xmm2

  679. 

  680. 	mulsd	%xmm0, %xmm2

  681. 	movsd	 4 * SIZE(AA), %xmm0

  682. 	addsd	%xmm2, %xmm5

  683. 	movsd	 4 * SIZE(BB), %xmm2

  684. 

  685. 	addl   $4 * SIZE, AA

  686. 	addl   $4 * SIZE, BB

  687. 	decl   %eax

  688. 	jne    .L42

  689. 	ALIGN_4

  690. 

  691. .L45:

  692. #ifndef TRMMKERNEL

  693. 	movl	K, %eax

  694. #else

  695. 	movl	KKK, %eax

  696. #endif

  697. 	andl	$3, %eax		# if (k & 1)

  698. 	BRANCH

  699. 	je .L48

  700. 	ALIGN_3

  701. 

  702. .L46:

  703. 	mulsd	%xmm0, %xmm2

  704. 	movsd	 1 * SIZE(AA), %xmm0

  705. 	addsd	%xmm2, %xmm4

  706. 	movsd	 1 * SIZE(BB), %xmm2

  707. 

  708. 	addl	$1 * SIZE, AA

  709. 	addl	$1 * SIZE, BB

  710. 	decl	%eax

  711. 	jg	.L46

  712. 	ALIGN_4

  713. 

  714. .L48:

  715. 	movsd	ALPHA,  %xmm0

  716. 

  717. 	addsd	%xmm5, %xmm4

  718. 	mulsd	%xmm0, %xmm4

  719. 

  720. #ifndef TRMMKERNEL

  721. 	addsd	0 * SIZE(CO1), %xmm4

  722. #endif

  723. 

  724. 	movsd	%xmm4, 0 * SIZE(CO1)

  725. 	ALIGN_4

  726. 

  727. .L999:

  728. 	popl	%ebx

  729. 	popl	%esi

  730. 	popl	%edi

  731. 	popl	%ebp

  732. 

  733. 	addl	$ARGS, %esp

  734. 	ret

  735. 

  736. 	EPILOGUE