OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 886cbaf4e4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '886cbaf4e4'
${ noResults }
OpenBLAS/kernel/x86_64/zgemm_kernel_2x1_atom.S

770 lines
16 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41.  

  42. #define M	%rdi

  43. #define N	%rsi

  44. #define K	%rdx

  45. 

  46. #define A	%rcx

  47. #define B	%r8

  48. #define C	%r9

  49. #define LDC	%r10

  50. 

  51. #define I	%r11

  52. #define J	%r12

  53. #define AO	%r13

  54. #define BO	%r14

  55. #define	CO1	%r15

  56. #define BB	%rbp

  57. 	

  58. #ifndef WINDOWS_ABI

  59. 

  60. #define STACKSIZE 128

  61. 

  62. #define OLD_LDC		 8 + STACKSIZE(%rsp)

  63. #define OLD_OFFSET	16 + STACKSIZE(%rsp)

  64. 

  65. #define ALPHA_R	 48(%rsp)

  66. #define ALPHA_I	 56(%rsp)

  67. #define OFFSET	 64(%rsp)

  68. #define KKK	 72(%rsp)

  69. #define KK	 80(%rsp)

  70. 

  71. #else

  72. 

  73. #define STACKSIZE 512

  74. 

  75. #define OLD_ALPHA_I	40 + STACKSIZE(%rsp)

  76. #define OLD_A		48 + STACKSIZE(%rsp)

  77. #define OLD_B		56 + STACKSIZE(%rsp)

  78. #define OLD_C		64 + STACKSIZE(%rsp)

  79. #define OLD_LDC		72 + STACKSIZE(%rsp)

  80. #define OLD_OFFSET	80 + STACKSIZE(%rsp)

  81. 

  82. #define ALPHA_R	 224(%rsp)

  83. #define ALPHA_I	 232(%rsp)

  84. #define OFFSET	 240(%rsp)

  85. #define KKK	 248(%rsp)

  86. #define KK	 256(%rsp)

  87. #endif

  88. 

  89. #define PREFETCH     prefetcht0

  90. #define PREFETCHSIZE (8 * 8 + 3)

  91. 

  92. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  93. #define ADDSD1	addsd

  94. #define ADDSD2	addsd

  95. #define ADDSD3	addsd

  96. #define ADDSD4	subsd

  97. #elif defined(NR) || defined(NC) || defined(TR) || defined(TC)

  98. #define ADDSD1	addsd

  99. #define ADDSD2	subsd

  100. #define ADDSD3	addsd

  101. #define ADDSD4	addsd

  102. #elif defined(RN) || defined(RT) || defined(CN) || defined(CT)

  103. #define ADDSD1	addsd

  104. #define ADDSD2	addsd

  105. #define ADDSD3	subsd

  106. #define ADDSD4	addsd

  107. #else

  108. #define ADDSD1	addsd

  109. #define ADDSD2	subsd

  110. #define ADDSD3	subsd

  111. #define ADDSD4	subsd

  112. #endif

  113. 

  114. 	PROLOGUE

  115. 	PROFCODE

  116. 	

  117. 	subq	$STACKSIZE, %rsp

  118. 	

  119. 	movq	%rbx,  0(%rsp)

  120. 	movq	%rbp,  8(%rsp)

  121. 	movq	%r12, 16(%rsp)

  122. 	movq	%r13, 24(%rsp)

  123. 	movq	%r14, 32(%rsp)

  124. 	movq	%r15, 40(%rsp)

  125. 

  126. #ifdef WINDOWS_ABI

  127. 	movq	%rdi,    48(%rsp)

  128. 	movq	%rsi,    56(%rsp)

  129. 	movups	%xmm6,   64(%rsp)

  130. 	movups	%xmm7,   80(%rsp)

  131. 	movups	%xmm8,   96(%rsp)

  132. 	movups	%xmm9,  112(%rsp)

  133. 	movups	%xmm10, 128(%rsp)

  134. 	movups	%xmm11, 144(%rsp)

  135. 	movups	%xmm12, 160(%rsp)

  136. 	movups	%xmm13, 176(%rsp)

  137. 	movups	%xmm14, 192(%rsp)

  138. 	movups	%xmm15, 208(%rsp)

  139. 

  140. 	movq	ARG1,      M

  141. 	movq	ARG2,      N

  142. 	movq	ARG3,      K

  143. 	movq	OLD_A,     A

  144. 	movq	OLD_B,     B

  145. 	movq	OLD_C,     C

  146. 	movq	OLD_LDC,   LDC

  147. #ifdef TRMMKERNEL

  148. 	movsd	OLD_OFFSET,  %xmm4

  149. #endif

  150. 	movaps	%xmm3,       %xmm0

  151. 	movsd	OLD_ALPHA_I, %xmm1

  152. #else

  153. 	movq	OLD_LDC,   LDC

  154. #ifdef TRMMKERNEL

  155. 	movsd	OLD_OFFSET,  %xmm4

  156. #endif

  157. 

  158. #endif

  159. 

  160. 	movsd	 %xmm0, ALPHA_R

  161. 	movsd	 %xmm1, ALPHA_I

  162. 

  163. #ifdef TRMMKERNEL

  164. 	movsd	%xmm4, OFFSET

  165. 	movsd	%xmm4, KK

  166. #ifndef LEFT

  167. 	negq	KK

  168. #endif	

  169. #endif

  170. 

  171. 	salq	$ZBASE_SHIFT, LDC

  172. 

  173. 	movq	N,  J

  174. 	testq	N, N

  175. 	jle	.L999

  176. 	ALIGN_4

  177. 

  178. .L01:

  179. #if defined(TRMMKERNEL) && defined(LEFT)

  180. 	movq	OFFSET, %rax

  181. 	movq	%rax, KK

  182. #endif	

  183. 

  184. 	movq	C, CO1

  185. 	addq	LDC, C

  186. 

  187. 	movq	A, AO

  188. 

  189. 	movq	K, %rax

  190. 	salq	$ZBASE_SHIFT, %rax

  191. 	leaq	(B, %rax), BB

  192. 

  193. 	movq	M,  I

  194. 	sarq	$1, I

  195. 	jle	.L20

  196. 	ALIGN_4

  197. 

  198. .L10:

  199. #if !defined(TRMMKERNEL) || \

  200. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  201. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  202. 

  203. 	movq	B, BO

  204. #else

  205. 	movq	KK, %rax

  206. 	leaq	(, %rax, SIZE), %rax

  207. 	leaq	(AO, %rax, 4), AO

  208. 	leaq	(B,  %rax, 2), BO

  209. #endif	

  210. 

  211. 	prefetcht0	 0 * SIZE(BB)

  212. 	subq	   $-8 * SIZE, BB

  213. 

  214. 	movsd	 0 * SIZE(AO), %xmm0

  215. 	xorps	%xmm2,  %xmm2

  216. 	movsd	 1 * SIZE(AO), %xmm4

  217. 	xorps	%xmm5,  %xmm5

  218. 	movsd	 2 * SIZE(AO), %xmm5

  219. 	xorps	%xmm6,  %xmm6

  220. 	xorps	%xmm7,  %xmm7

  221. 

  222. 	movsd	 0 * SIZE(BO), %xmm1

  223. 	xorps	%xmm8,  %xmm8

  224. 	xorps	%xmm9,  %xmm9

  225. 	movsd	 1 * SIZE(BO), %xmm3

  226. 	xorps	%xmm10, %xmm10

  227. 	xorps	%xmm11, %xmm11

  228. 

  229. 	prefetcht0     3 * SIZE(CO1)

  230. 	xorps	%xmm12, %xmm12

  231. 	xorps	%xmm13, %xmm13

  232. 	xorps	%xmm14, %xmm14

  233. 	xorps	%xmm15, %xmm15

  234. 

  235. #ifndef TRMMKERNEL

  236. 	movq	K, %rax

  237. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  238. 	movq	K, %rax

  239. 	subq	KK, %rax

  240. 	movq	%rax, KKK	

  241. #else

  242. 	movq	KK, %rax

  243. #ifdef LEFT

  244. 	addq	$2, %rax

  245. #else

  246. 	addq	$1, %rax

  247. #endif

  248. 	movq	%rax, KKK

  249. #endif

  250. 

  251. 	sarq	$2, %rax

  252. 	je	.L15

  253. 	ALIGN_4

  254. 

  255. .L12:

  256. 	ADDSD2	 %xmm2, %xmm13

  257. 	PREFETCH (PREFETCHSIZE + 0) * SIZE(AO)

  258. 	movaps	 %xmm0, %xmm2

  259. 	mulsd	 %xmm1, %xmm0

  260. 

  261. 	ADDSD3	 %xmm7, %xmm14

  262. 	movsd	 3 * SIZE(AO), %xmm7

  263. 	mulsd	 %xmm3, %xmm2

  264.  

  265. 	ADDSD4	 %xmm6, %xmm15

  266. 	PREFETCH ((PREFETCHSIZE) >> 1 + 0) * SIZE(BO)

  267. 	movaps	 %xmm4, %xmm6

  268. 	mulsd	 %xmm1, %xmm4

  269. 

  270. 	ADDSD1	 %xmm0, %xmm8

  271. 	movsd	 4 * SIZE(AO), %xmm0

  272. 	mulsd	 %xmm3, %xmm6

  273. 

  274. 	ADDSD2	 %xmm2, %xmm9

  275. 	movaps	 %xmm5, %xmm2

  276. 	mulsd	 %xmm1, %xmm5

  277. 

  278. 	ADDSD3	 %xmm4, %xmm10

  279. 	movsd	 5 * SIZE(AO), %xmm4

  280. 	mulsd	 %xmm3, %xmm2

  281. 

  282. 	ADDSD4	 %xmm6, %xmm11

  283. 	movaps	 %xmm7, %xmm6

  284. 	mulsd	 %xmm1, %xmm7

  285. 	movsd	 2 * SIZE(BO), %xmm1

  286. 

  287. 	ADDSD1	 %xmm5, %xmm12

  288. 	movsd	 6 * SIZE(AO), %xmm5

  289. 	mulsd	 %xmm3, %xmm6

  290. 	movsd	 3 * SIZE(BO), %xmm3

  291. 

  292. 	ADDSD2	 %xmm2, %xmm13

  293. 	movaps	 %xmm0, %xmm2

  294. 	mulsd	 %xmm1, %xmm0

  295. 

  296. 	ADDSD3	 %xmm7, %xmm14

  297. 	movsd	 7 * SIZE(AO), %xmm7

  298. 	mulsd	 %xmm3, %xmm2

  299. 

  300. 	ADDSD4	 %xmm6, %xmm15

  301. 	movaps	 %xmm4, %xmm6

  302. 	mulsd	 %xmm1, %xmm4

  303. 

  304. 	ADDSD1	 %xmm0, %xmm8

  305. 	movsd	 8 * SIZE(AO), %xmm0

  306. 	mulsd	 %xmm3, %xmm6

  307. 

  308. 	ADDSD2	 %xmm2, %xmm9

  309. 	movaps	 %xmm5, %xmm2

  310. 	mulsd	 %xmm1, %xmm5

  311. 

  312. 	ADDSD3	 %xmm4, %xmm10

  313. 	movsd	 9 * SIZE(AO), %xmm4

  314. 	mulsd	 %xmm3, %xmm2

  315. 

  316. 	ADDSD4	 %xmm6, %xmm11

  317. 	movaps	 %xmm7, %xmm6

  318. 	mulsd	 %xmm1, %xmm7

  319. 	movsd	 4 * SIZE(BO), %xmm1

  320. 

  321. 	ADDSD1	 %xmm5, %xmm12

  322. 	movsd	10 * SIZE(AO), %xmm5

  323. 	mulsd	 %xmm3, %xmm6

  324. 	movsd	 5 * SIZE(BO), %xmm3

  325. 

  326. 	ADDSD2	 %xmm2, %xmm13

  327. 	PREFETCH (PREFETCHSIZE + 8) * SIZE(AO)

  328. 	movaps	 %xmm0, %xmm2

  329. 	mulsd	 %xmm1, %xmm0

  330. 

  331. 	ADDSD3	 %xmm7, %xmm14

  332. 	movsd	11 * SIZE(AO), %xmm7

  333. 	mulsd	 %xmm3, %xmm2

  334. 

  335. 	ADDSD4	 %xmm6, %xmm15

  336. 	movaps	 %xmm4, %xmm6

  337. 	mulsd	 %xmm1, %xmm4

  338. 

  339. 	ADDSD1	 %xmm0, %xmm8

  340. 	movsd	12 * SIZE(AO), %xmm0

  341. 	mulsd	 %xmm3, %xmm6

  342. 

  343. 	ADDSD2	 %xmm2, %xmm9

  344. 	movaps	 %xmm5, %xmm2

  345. 	mulsd	 %xmm1, %xmm5

  346. 

  347. 	ADDSD3	 %xmm4, %xmm10

  348. 	movsd	13 * SIZE(AO), %xmm4

  349. 	mulsd	 %xmm3, %xmm2

  350. 

  351. 	ADDSD4	 %xmm6, %xmm11

  352. 	movaps	 %xmm7, %xmm6

  353. 	mulsd	 %xmm1, %xmm7

  354. 	movsd	 6 * SIZE(BO), %xmm1

  355. 

  356. 	ADDSD1	 %xmm5, %xmm12

  357. 	movsd	14 * SIZE(AO), %xmm5

  358. 	mulsd	 %xmm3, %xmm6

  359. 	movsd	 7 * SIZE(BO), %xmm3

  360. 

  361. 	ADDSD2	 %xmm2, %xmm13

  362. 	movaps	 %xmm0, %xmm2

  363. 	mulsd	 %xmm1, %xmm0

  364. 

  365. 	ADDSD3	 %xmm7, %xmm14

  366. 	movsd	15 * SIZE(AO), %xmm7

  367. 	mulsd	 %xmm3, %xmm2

  368. 	subq   $-16 * SIZE, AO

  369. 

  370. 	ADDSD4	 %xmm6, %xmm15

  371. 	movaps	 %xmm4, %xmm6

  372. 	mulsd	 %xmm1, %xmm4

  373. 

  374. 	ADDSD1	 %xmm0, %xmm8

  375. 	movsd	 0 * SIZE(AO), %xmm0

  376. 	mulsd	 %xmm3, %xmm6

  377. 

  378. 	ADDSD2	 %xmm2, %xmm9

  379. 	movaps	 %xmm5, %xmm2

  380. 	mulsd	 %xmm1, %xmm5

  381. 	addq   $  8 * SIZE, BO

  382. 

  383. 	ADDSD3	 %xmm4, %xmm10

  384. 	movsd	 1 * SIZE(AO), %xmm4

  385. 	mulsd	 %xmm3, %xmm2

  386. 	decq   %rax

  387. 

  388. 	ADDSD4	 %xmm6, %xmm11

  389. 	movaps	 %xmm7, %xmm6

  390. 	mulsd	 %xmm1, %xmm7

  391. 	movsd	 0 * SIZE(BO), %xmm1

  392. 

  393. 	ADDSD1	 %xmm5, %xmm12

  394. 	movsd	 2 * SIZE(AO), %xmm5

  395. 	mulsd	 %xmm3, %xmm6

  396. 	movsd	 1 * SIZE(BO), %xmm3

  397. 

  398. 	jne    .L12

  399. 	ALIGN_4

  400. 

  401. .L15:

  402. #ifndef TRMMKERNEL

  403. 	movq	K, %rax

  404. #else

  405. 	movq	KKK, %rax

  406. #endif

  407. 	andq	$3, %rax

  408. 	BRANCH

  409. 	BRANCH

  410. 	je	.L18

  411. 	ALIGN_4

  412. 

  413. .L16:

  414. 	ADDSD2	 %xmm2, %xmm13

  415. 	movaps	 %xmm0, %xmm2

  416. 	mulsd	 %xmm1, %xmm0

  417. 

  418. 	ADDSD3	 %xmm7, %xmm14

  419. 	movsd	 3 * SIZE(AO), %xmm7

  420. 	mulsd	 %xmm3, %xmm2

  421. 

  422. 	ADDSD4	 %xmm6, %xmm15

  423. 	movaps	 %xmm4, %xmm6

  424. 	mulsd	 %xmm1, %xmm4

  425. 

  426. 	ADDSD1	 %xmm0, %xmm8

  427. 	movsd	 4 * SIZE(AO), %xmm0

  428. 	mulsd	 %xmm3, %xmm6

  429. 

  430. 	ADDSD2	 %xmm2, %xmm9

  431. 	movaps	 %xmm5, %xmm2

  432. 	mulsd	 %xmm1, %xmm5

  433. 

  434. 	ADDSD3	 %xmm4, %xmm10

  435. 	movsd	 5 * SIZE(AO), %xmm4

  436. 	mulsd	 %xmm3, %xmm2

  437. 

  438. 	ADDSD4	 %xmm6, %xmm11

  439. 	movaps	 %xmm7, %xmm6

  440. 	mulsd	 %xmm1, %xmm7

  441. 	movsd	 2 * SIZE(BO), %xmm1

  442. 

  443. 	ADDSD1	 %xmm5, %xmm12

  444. 	movsd	 6 * SIZE(AO), %xmm5

  445. 	mulsd	 %xmm3, %xmm6

  446. 	movsd	 3 * SIZE(BO), %xmm3

  447. 

  448. 	addq	$4 * SIZE, AO

  449. 	addq	$2 * SIZE, BO

  450. 	decq	%rax

  451. 	BRANCH

  452. 	jg	.L16

  453. 	ALIGN_4

  454. 

  455. .L18:

  456. 	movsd	ALPHA_R, %xmm0

  457. 	movsd	ALPHA_I, %xmm1

  458. 

  459. 	ADDSD2	 %xmm2, %xmm13

  460. 	ADDSD3	 %xmm7, %xmm14

  461. 	ADDSD4	 %xmm6, %xmm15

  462. 

  463. 	addsd	%xmm11, %xmm8

  464. 	addsd	%xmm9,  %xmm10

  465. 	addsd	%xmm15, %xmm12

  466. 	addsd	%xmm13, %xmm14

  467. 

  468. 	movaps	%xmm8,  %xmm9

  469. 	movaps	%xmm10, %xmm11

  470. 	movaps	%xmm12, %xmm13

  471. 	movaps	%xmm14, %xmm15

  472. 

  473. 	mulsd	%xmm0, %xmm8

  474. 	mulsd	%xmm1, %xmm9

  475. 	mulsd	%xmm1, %xmm10

  476. 	mulsd	%xmm0, %xmm11

  477. 

  478. 	subsd	%xmm10, %xmm8

  479. 	addsd	%xmm11, %xmm9

  480. 

  481. 	mulsd	%xmm0, %xmm12

  482. 	mulsd	%xmm1, %xmm13

  483. 	mulsd	%xmm1, %xmm14

  484. 	mulsd	%xmm0, %xmm15

  485. 

  486. 	subsd	%xmm14, %xmm12

  487. 	addsd	%xmm15, %xmm13

  488. 

  489. #if !defined(TRMMKERNEL) && !defined(BETAZERO)

  490. 	addsd	0 * SIZE(CO1), %xmm8

  491. 	addsd	1 * SIZE(CO1), %xmm9

  492. 	addsd	2 * SIZE(CO1), %xmm12

  493. 	addsd	3 * SIZE(CO1), %xmm13

  494. #endif

  495. 

  496. 	movsd	%xmm8,  0 * SIZE(CO1)

  497. 	movsd	%xmm9,  1 * SIZE(CO1)

  498. 	movsd	%xmm12, 2 * SIZE(CO1)

  499. 	movsd	%xmm13, 3 * SIZE(CO1)

  500. 

  501. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  502.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  503. 	movq	K, %rax

  504. 	subq	KKK, %rax

  505. 	leaq	(,%rax, SIZE), %rax

  506. 	leaq	(AO, %rax, 4), AO

  507. 	leaq	(BO, %rax, 2), BO

  508. #endif

  509. 

  510. #if defined(TRMMKERNEL) && defined(LEFT)

  511. 	addq	$2, KK

  512. #endif

  513. 

  514. 	addq	$4 * SIZE, CO1

  515. 	decq	I

  516. 	jg	.L10

  517. 	ALIGN_4	

  518. 

  519. .L20:

  520. 	testq	$1, M

  521. 	jle	.L99

  522. 

  523. #if !defined(TRMMKERNEL) || \

  524. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  525. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  526. 

  527. 	movq	B, BO

  528. #else

  529. 	movq	KK, %rax

  530. 	leaq	(, %rax, SIZE), %rax

  531. 	leaq	(AO, %rax, 2), AO

  532. 	leaq	(B,  %rax, 2), BO

  533. #endif	

  534. 

  535. 

  536. 	movsd	 0 * SIZE(AO), %xmm0

  537. 	xorps	%xmm2,  %xmm2

  538. 	movsd	 1 * SIZE(AO), %xmm4

  539. 	xorps	%xmm5,  %xmm5

  540. 	movsd	 2 * SIZE(AO), %xmm5

  541. 	xorps	%xmm6,  %xmm6

  542. 	movsd	 3 * SIZE(AO), %xmm7

  543. 

  544. 	movsd	 0 * SIZE(BO), %xmm1

  545. 	xorps	%xmm8,  %xmm8

  546. 	xorps	%xmm9,  %xmm9

  547. 	movsd	 1 * SIZE(BO), %xmm3

  548. 	xorps	%xmm10, %xmm10

  549. 	xorps	%xmm11, %xmm11

  550. 

  551. #ifndef TRMMKERNEL

  552. 	movq	K, %rax

  553. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  554. 	movq	K, %rax

  555. 	subq	KK, %rax

  556. 	movq	%rax, KKK	

  557. #else

  558. 	movq	KK, %rax

  559. #ifdef LEFT

  560. 	addq	$1, %rax

  561. #else

  562. 	addq	$1, %rax

  563. #endif

  564. 	movq	%rax, KKK

  565. #endif

  566. 

  567. 	sarq	$2, %rax

  568. 	je	.L25

  569. 	ALIGN_4

  570. 

  571. .L22:

  572. 	PREFETCH (PREFETCHSIZE + 0) * SIZE(AO)

  573. 	ADDSD2	 %xmm2, %xmm9

  574. 	movaps	 %xmm0, %xmm2

  575. 	mulsd	 %xmm1, %xmm0

  576. 

  577. 	ADDSD4	 %xmm6, %xmm11

  578. 	movaps	 %xmm4, %xmm6

  579. 	mulsd	 %xmm1, %xmm4

  580. 	movsd	 2 * SIZE(BO), %xmm1

  581. 

  582. 	ADDSD1	 %xmm0, %xmm8

  583. 	movsd	 4 * SIZE(AO), %xmm0

  584. 	mulsd	 %xmm3, %xmm2

  585. 

  586. 	ADDSD3	 %xmm4, %xmm10

  587. 	movsd	 5 * SIZE(AO), %xmm4

  588. 	mulsd	 %xmm3, %xmm6

  589. 	movsd	 3 * SIZE(BO), %xmm3

  590. 

  591. 	ADDSD2	 %xmm2, %xmm9

  592. 	movaps	 %xmm5, %xmm2

  593. 	mulsd	 %xmm1, %xmm5

  594. 

  595. 	ADDSD4	 %xmm6, %xmm11

  596. 	movaps	 %xmm7, %xmm6

  597. 	mulsd	 %xmm1, %xmm7

  598. 	movsd	 4 * SIZE(BO), %xmm1

  599. 

  600. 	ADDSD1	 %xmm5, %xmm8

  601. 	movsd	 6 * SIZE(AO), %xmm5

  602. 	mulsd	 %xmm3, %xmm2

  603. 

  604. 	ADDSD3	 %xmm7, %xmm10

  605. 	movsd	 7 * SIZE(AO), %xmm7

  606. 	mulsd	 %xmm3, %xmm6

  607. 	movsd	 5 * SIZE(BO), %xmm3

  608. 

  609. 	ADDSD2	 %xmm2, %xmm9

  610. 	movaps	 %xmm0, %xmm2

  611. 	mulsd	 %xmm1, %xmm0

  612. 

  613. 	ADDSD4	 %xmm6, %xmm11

  614. 	movaps	 %xmm4, %xmm6

  615. 	mulsd	 %xmm1, %xmm4

  616. 	movsd	 6 * SIZE(BO), %xmm1

  617. 

  618. 	ADDSD1	 %xmm0, %xmm8

  619. 	movsd	 8 * SIZE(AO), %xmm0

  620. 	mulsd	 %xmm3, %xmm2

  621. 

  622. 	ADDSD3	 %xmm4, %xmm10

  623. 	movsd	 9 * SIZE(AO), %xmm4

  624. 	mulsd	 %xmm3, %xmm6

  625. 	movsd	 7 * SIZE(BO), %xmm3

  626. 

  627. 	ADDSD2	 %xmm2, %xmm9

  628. 	movaps	 %xmm5, %xmm2

  629. 	mulsd	 %xmm1, %xmm5

  630. 

  631. 	ADDSD4	 %xmm6, %xmm11

  632. 	movaps	 %xmm7, %xmm6

  633. 	mulsd	 %xmm1, %xmm7

  634. 	movsd	 8 * SIZE(BO), %xmm1

  635. 

  636. 	ADDSD1	 %xmm5, %xmm8

  637. 	movsd	10 * SIZE(AO), %xmm5

  638. 	mulsd	 %xmm3, %xmm2

  639. 

  640. 	ADDSD3	 %xmm7, %xmm10

  641. 	movsd	11 * SIZE(AO), %xmm7

  642. 	mulsd	 %xmm3, %xmm6

  643. 	movsd	 9 * SIZE(BO), %xmm3

  644. 

  645. 	addq	$8 * SIZE, AO

  646. 	addq	$8 * SIZE, BO

  647. 

  648. 	decq	%rax

  649. 	jne    .L22

  650. 	ALIGN_4

  651. 

  652. .L25:

  653. #ifndef TRMMKERNEL

  654. 	movq	K, %rax

  655. #else

  656. 	movq	KKK, %rax

  657. #endif

  658. 	andq	$3, %rax

  659. 	BRANCH

  660. 	BRANCH

  661. 	je	.L29

  662. 	ALIGN_4

  663. 

  664. .L26:

  665. 	ADDSD2	 %xmm2, %xmm9

  666. 	movaps	 %xmm0, %xmm2

  667. 	mulsd	 %xmm1, %xmm0

  668. 

  669. 	ADDSD4	 %xmm6, %xmm11

  670. 	movaps	 %xmm4, %xmm6

  671. 	mulsd	 %xmm1, %xmm4

  672. 	movsd	 2 * SIZE(BO), %xmm1

  673. 

  674. 	mulsd	 %xmm3, %xmm2

  675. 	ADDSD1	 %xmm0, %xmm8

  676. 	movsd	 2 * SIZE(AO), %xmm0

  677. 

  678. 	mulsd	 %xmm3, %xmm6

  679. 	movsd	 3 * SIZE(BO), %xmm3

  680. 	ADDSD3	 %xmm4, %xmm10

  681. 	movsd	 3 * SIZE(AO), %xmm4

  682. 

  683. 	addq	$2 * SIZE, AO

  684. 	addq	$2 * SIZE, BO

  685. 	decq	%rax

  686. 	BRANCH

  687. 	jg	.L26

  688. 	ALIGN_4

  689. 

  690. .L29:

  691. 	movsd	ALPHA_R, %xmm0

  692. 	movsd	ALPHA_I, %xmm1

  693. 

  694. 	ADDSD2	 %xmm2, %xmm9

  695. 	ADDSD4	 %xmm6, %xmm11

  696. 

  697. 	addsd	%xmm11,  %xmm8

  698. 	addsd	%xmm9,  %xmm10

  699. 

  700. 	movaps	%xmm8,  %xmm9

  701. 	movaps	%xmm10, %xmm11

  702. 

  703. 	mulsd	%xmm0, %xmm8

  704. 	mulsd	%xmm1, %xmm9

  705. 	mulsd	%xmm1, %xmm10

  706. 	mulsd	%xmm0, %xmm11

  707. 

  708. 	subsd	%xmm10, %xmm8

  709. 	addsd	%xmm11, %xmm9

  710. 

  711. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  712. 	addsd	0 * SIZE(CO1), %xmm8

  713. 	addsd	1 * SIZE(CO1), %xmm9

  714. #endif

  715. 

  716. 	movsd	%xmm8,  0 * SIZE(CO1)

  717. 	movsd	%xmm9,  1 * SIZE(CO1)

  718. 

  719. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  720.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  721. 	movq	K, %rax

  722. 	subq	KKK, %rax

  723. 	leaq	(,%rax, SIZE), %rax

  724. 	leaq	(AO, %rax, 2), AO

  725. 	leaq	(BO, %rax, 2), BO

  726. #endif

  727. 

  728. #if defined(TRMMKERNEL) && defined(LEFT)

  729. 	addq	$1, KK

  730. #endif

  731. 	ALIGN_4

  732. 	

  733. .L99:

  734. #if defined(TRMMKERNEL) && !defined(LEFT)

  735. 	addq	$1, KK

  736. #endif

  737. 

  738. 	movq	BO, B

  739. 	decq	J			# j --

  740. 	jg	.L01

  741. 	ALIGN_4

  742. 	

  743. .L999:

  744. 	movq	  0(%rsp), %rbx

  745. 	movq	  8(%rsp), %rbp

  746. 	movq	 16(%rsp), %r12

  747. 	movq	 24(%rsp), %r13

  748. 	movq	 32(%rsp), %r14

  749. 	movq	 40(%rsp), %r15

  750. 

  751. #ifdef WINDOWS_ABI

  752. 	movq	 48(%rsp), %rdi

  753. 	movq	 56(%rsp), %rsi

  754. 	movups	 64(%rsp), %xmm6

  755. 	movups	 80(%rsp), %xmm7

  756. 	movups	 96(%rsp), %xmm8

  757. 	movups	112(%rsp), %xmm9

  758. 	movups	128(%rsp), %xmm10

  759. 	movups	144(%rsp), %xmm11

  760. 	movups	160(%rsp), %xmm12

  761. 	movups	176(%rsp), %xmm13

  762. 	movups	192(%rsp), %xmm14

  763. 	movups	208(%rsp), %xmm15

  764. #endif

  765. 

  766. 	addq	$STACKSIZE, %rsp

  767. 	ret

  768. 

  769. 	EPILOGUE