OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/dgemv_n_atom.S

789 lines
16 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. #include "l2param.h"

  42. 

  43. #define PREFETCH	prefetchnta

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCH_SIZE	(8 * 6)

  46. 

  47. #ifndef WINDOWS_ABI

  48. 

  49. #define STACKSIZE	64

  50. 

  51. #define OLD_INCX	 8 + STACKSIZE(%rsp)

  52. #define OLD_Y		16 + STACKSIZE(%rsp)

  53. #define OLD_INCY	24 + STACKSIZE(%rsp)

  54. #define OLD_BUFFER	32 + STACKSIZE(%rsp)

  55. #define STACK_ALPHA	48	      (%rsp)

  56. 

  57. #define M	  %rdi

  58. #define N	  %rsi

  59. #define A	  %rcx

  60. #define LDA	  %r8

  61. #define X	  %r9

  62. #define INCX	  %rdx

  63. #define Y	  %rbp

  64. #define INCY	  %r10

  65. 

  66. #else

  67. 

  68. #define STACKSIZE	256

  69. 

  70. #define OLD_A		 40 + STACKSIZE(%rsp)

  71. #define OLD_LDA		 48 + STACKSIZE(%rsp)

  72. #define OLD_X		 56 + STACKSIZE(%rsp)

  73. #define OLD_INCX	 64 + STACKSIZE(%rsp)

  74. #define OLD_Y		 72 + STACKSIZE(%rsp)

  75. #define OLD_INCY	 80 + STACKSIZE(%rsp)

  76. #define OLD_BUFFER	 88 + STACKSIZE(%rsp)

  77. 

  78. #define STACK_ALPHA	224	       (%rsp)

  79. 

  80. #define M	  %rcx

  81. #define N	  %rdx

  82. #define A	  %r8

  83. #define LDA	  %r9

  84. #define X	  %rdi

  85. #define INCX	  %rsi

  86. #define Y	  %rbp

  87. #define INCY	  %r10

  88. 

  89. #endif

  90. 

  91. #define I	%rax

  92. #define J	%r11

  93. #define A1	%r12

  94. #define A2	%r13

  95. #define Y1	%r14

  96. #define BUFFER	%r15

  97. #define MM	%rbx

  98. 

  99. #define ALPHA %xmm15

  100. 

  101. 	PROLOGUE

  102. 	PROFCODE

  103. 

  104. 	subq	$STACKSIZE, %rsp

  105. 	movq	%rbx,  0(%rsp)

  106. 	movq	%rbp,  8(%rsp)

  107. 	movq	%r12, 16(%rsp)

  108. 	movq	%r13, 24(%rsp)

  109. 	movq	%r14, 32(%rsp)

  110. 	movq	%r15, 40(%rsp)

  111. 

  112. #ifdef WINDOWS_ABI

  113. 	movq	%rdi,    48(%rsp)

  114. 	movq	%rsi,    56(%rsp)

  115. 	movups	%xmm6,   64(%rsp)

  116. 	movups	%xmm7,   80(%rsp)

  117. 	movups	%xmm8,   96(%rsp)

  118. 	movups	%xmm9,  112(%rsp)

  119. 	movups	%xmm10, 128(%rsp)

  120. 	movups	%xmm11, 144(%rsp)

  121. 	movups	%xmm12, 160(%rsp)

  122. 	movups	%xmm13, 176(%rsp)

  123. 	movups	%xmm14, 192(%rsp)

  124. 	movups	%xmm15, 208(%rsp)

  125. 

  126. 	movq	OLD_A,     A

  127. 	movq	OLD_LDA,   LDA

  128. 	movq	OLD_X,     X

  129. #endif

  130. 	movq	OLD_INCX,   INCX

  131. 	movq	OLD_Y,      Y

  132. 	movq	OLD_INCY,   INCY

  133. 	movq	OLD_BUFFER, BUFFER

  134. 

  135. #ifndef WINDOWS_ABI

  136. 	movsd	 %xmm0, STACK_ALPHA

  137. #else

  138. 	movsd	 %xmm3, STACK_ALPHA

  139. #endif

  140. 

  141. 	leaq	(,INCX, SIZE), INCX

  142. 	leaq	(,INCY, SIZE), INCY

  143. 	leaq	(,LDA,  SIZE), LDA

  144. 

  145. 	testq	N, N

  146. 	jle	.L999

  147. 	testq	M, M

  148. 	jle	.L999

  149. 

  150. 	cmpq	$SIZE, INCY

  151. 	cmoveq	Y, BUFFER

  152. 	je	.L10

  153. 

  154. 	movq	BUFFER, Y1

  155. 	xorps	%xmm4, %xmm4

  156. 

  157. 	movq	M,  %rax

  158. 	addq	$7, %rax

  159. 	sarq	$3, %rax

  160. 	ALIGN_3

  161. 

  162. .L01:

  163. 	movsd	%xmm4, 0 * SIZE(Y1)

  164. 	movsd	%xmm4, 1 * SIZE(Y1)

  165. 	movsd	%xmm4, 2 * SIZE(Y1)

  166. 	movsd	%xmm4, 3 * SIZE(Y1)

  167. 	movsd	%xmm4, 4 * SIZE(Y1)

  168. 	movsd	%xmm4, 5 * SIZE(Y1)

  169. 	movsd	%xmm4, 6 * SIZE(Y1)

  170. 	movsd	%xmm4, 7 * SIZE(Y1)

  171. 

  172. 	addq	$8 * SIZE, Y1

  173. 	decq	%rax

  174. 	jg	.L01

  175. 	ALIGN_3

  176. 

  177. .L10:

  178. 	movq	N,  J

  179. 	sarq	$1, J

  180. 	jle	.L20

  181. 	ALIGN_3

  182. 

  183. .L11:

  184. 	movq	BUFFER, Y1

  185. 

  186. 	movq	A,  A1

  187. 	leaq	(A, LDA, 1), A2

  188. 	leaq	(A, LDA, 2), A

  189. 

  190. 	movsd	STACK_ALPHA, %xmm0

  191. 

  192. 	movsd	(X), %xmm14

  193. 	addq	INCX, X

  194. 	movsd	(X), %xmm15

  195. 	addq	INCX, X

  196. 

  197. 	mulsd	%xmm0, %xmm14

  198. 	mulsd	%xmm0, %xmm15

  199. 

  200. 	movq	M,  I

  201. 	sarq	$3, I

  202. 	jle	.L15

  203. 

  204. 	movsd	 0 * SIZE(A1), %xmm0

  205. 	movsd	 1 * SIZE(A1), %xmm1

  206. 	movsd	 2 * SIZE(A1), %xmm2

  207. 	movsd	 3 * SIZE(A1), %xmm3

  208. 

  209. 	movsd	 0 * SIZE(A2), %xmm4

  210. 	movsd	 1 * SIZE(A2), %xmm5

  211. 	movsd	 2 * SIZE(A2), %xmm6

  212. 	movsd	 3 * SIZE(A2), %xmm7

  213. 

  214. 	movsd	 0 * SIZE(Y1), %xmm8

  215. 	mulsd	 %xmm14, %xmm0

  216. 	movsd	 1 * SIZE(Y1), %xmm9

  217. 	mulsd	 %xmm14, %xmm1

  218. 	movsd	 2 * SIZE(Y1), %xmm10

  219. 	mulsd	 %xmm14, %xmm2

  220. 	movsd	 3 * SIZE(Y1), %xmm11

  221. 	mulsd	 %xmm14, %xmm3

  222. 

  223. 	decq	 I

  224. 	jle	 .L14

  225. 	ALIGN_3

  226. 

  227. .L13:

  228. 	PREFETCH	PREFETCH_SIZE * SIZE(A1)

  229. 	mulsd	 %xmm15, %xmm4

  230. 	PREFETCH	PREFETCH_SIZE * SIZE(A2)

  231. 	addsd	 %xmm0,  %xmm8

  232. 	movsd	 4 * SIZE(A1), %xmm0

  233. 

  234. 	mulsd	 %xmm15, %xmm5

  235. 	addsd	 %xmm1,  %xmm9

  236. 	movsd	 5 * SIZE(A1), %xmm1

  237. 

  238. 	mulsd	 %xmm15, %xmm6

  239. 	addsd	 %xmm2,  %xmm10

  240. 	movsd	 6 * SIZE(A1), %xmm2

  241. 

  242. 	mulsd	 %xmm15, %xmm7

  243. 	addsd	 %xmm3,  %xmm11

  244. 	movsd	 7 * SIZE(A1), %xmm3

  245. 

  246. 	addsd	 %xmm4,  %xmm8

  247. 	mulsd	 %xmm14, %xmm0

  248. 	movsd	 4 * SIZE(A2), %xmm4

  249. 

  250. 	addsd	 %xmm5,  %xmm9

  251. 	mulsd	 %xmm14, %xmm1

  252. 	movsd	 5 * SIZE(A2), %xmm5

  253. 

  254. 	addsd	 %xmm6,  %xmm10

  255. 	mulsd	 %xmm14, %xmm2

  256. 	movsd	 6 * SIZE(A2), %xmm6

  257. 

  258. 	addsd	 %xmm7,  %xmm11

  259. 	mulsd	 %xmm14, %xmm3

  260. 	movsd	 7 * SIZE(A2), %xmm7

  261. 

  262. 	movsd	 %xmm8,  0 * SIZE(Y1)

  263. 	movsd	 4 * SIZE(Y1), %xmm8

  264. 	movsd	 %xmm9,  1 * SIZE(Y1)

  265. 	movsd	 5 * SIZE(Y1), %xmm9

  266. 

  267. 	movsd	 %xmm10, 2 * SIZE(Y1)

  268. 	movsd	 6 * SIZE(Y1), %xmm10

  269. 	movsd	 %xmm11, 3 * SIZE(Y1)

  270. 	movsd	 7 * SIZE(Y1), %xmm11

  271. 

  272. 	mulsd	 %xmm15, %xmm4

  273. 	addsd	 %xmm0,  %xmm8

  274. 	movsd	 8 * SIZE(A1), %xmm0

  275. 

  276. 	mulsd	 %xmm15, %xmm5

  277. 	addsd	 %xmm1,  %xmm9

  278. 	movsd	 9 * SIZE(A1), %xmm1

  279. 

  280. 	mulsd	 %xmm15, %xmm6

  281. 	addsd	 %xmm2,  %xmm10

  282. 	movsd	10 * SIZE(A1), %xmm2

  283. 

  284. 	mulsd	 %xmm15, %xmm7

  285. 	addq	 $8 * SIZE, A2

  286. 	addsd	 %xmm3,  %xmm11

  287. 	movsd	11 * SIZE(A1), %xmm3

  288. 

  289. 	mulsd	 %xmm14, %xmm0

  290. 	addsd	 %xmm4,  %xmm8

  291. 	movsd	 0 * SIZE(A2), %xmm4

  292. 

  293. 	mulsd	 %xmm14, %xmm1

  294. 	addq	 $8 * SIZE, Y1

  295. 	addsd	 %xmm5,  %xmm9

  296. 	movsd	 1 * SIZE(A2), %xmm5

  297. 

  298. 	mulsd	 %xmm14, %xmm2

  299. 	addq	 $8 * SIZE, A1

  300. 	addsd	 %xmm6,  %xmm10

  301. 	movsd	 2 * SIZE(A2), %xmm6

  302. 

  303. 	mulsd	 %xmm14, %xmm3

  304. 	decq	 I

  305. 	addsd	 %xmm7,  %xmm11

  306. 	movsd	 3 * SIZE(A2), %xmm7

  307. 

  308. 	movsd	 %xmm8, -4 * SIZE(Y1)

  309. 	movsd	 0 * SIZE(Y1), %xmm8

  310. 	movsd	 %xmm9, -3 * SIZE(Y1)

  311. 	movsd	 1 * SIZE(Y1), %xmm9

  312. 

  313. 	movsd	 %xmm10,-2 * SIZE(Y1)

  314. 	movsd	 2 * SIZE(Y1), %xmm10

  315. 	movsd	 %xmm11,-1 * SIZE(Y1)

  316. 	movsd	 3 * SIZE(Y1), %xmm11

  317. 	jg	.L13

  318. 	ALIGN_3

  319. 

  320. .L14:

  321. 	mulsd	 %xmm15, %xmm4

  322. 	addsd	 %xmm0,  %xmm8

  323. 	movsd	 4 * SIZE(A1), %xmm0

  324. 

  325. 	mulsd	 %xmm15, %xmm5

  326. 	addsd	 %xmm1,  %xmm9

  327. 	movsd	 5 * SIZE(A1), %xmm1

  328. 

  329. 	mulsd	 %xmm15, %xmm6

  330. 	addsd	 %xmm2,  %xmm10

  331. 	movsd	 6 * SIZE(A1), %xmm2

  332. 

  333. 	mulsd	 %xmm15, %xmm7

  334. 	addsd	 %xmm3,  %xmm11

  335. 	movsd	 7 * SIZE(A1), %xmm3

  336. 

  337. 	addsd	 %xmm4,  %xmm8

  338. 	mulsd	 %xmm14, %xmm0

  339. 	movsd	 4 * SIZE(A2), %xmm4

  340. 

  341. 	addsd	 %xmm5,  %xmm9

  342. 	mulsd	 %xmm14, %xmm1

  343. 	movsd	 5 * SIZE(A2), %xmm5

  344. 

  345. 	addsd	 %xmm6,  %xmm10

  346. 	mulsd	 %xmm14, %xmm2

  347. 	movsd	 6 * SIZE(A2), %xmm6

  348. 

  349. 	addsd	 %xmm7,  %xmm11

  350. 	mulsd	 %xmm14, %xmm3

  351. 	movsd	 7 * SIZE(A2), %xmm7

  352. 

  353. 	movsd	 %xmm8,  0 * SIZE(Y1)

  354. 	movsd	 4 * SIZE(Y1), %xmm8

  355. 	movsd	 %xmm9,  1 * SIZE(Y1)

  356. 	movsd	 5 * SIZE(Y1), %xmm9

  357. 

  358. 	movsd	 %xmm10, 2 * SIZE(Y1)

  359. 	movsd	 6 * SIZE(Y1), %xmm10

  360. 	movsd	 %xmm11, 3 * SIZE(Y1)

  361. 	movsd	 7 * SIZE(Y1), %xmm11

  362. 

  363. 	mulsd	 %xmm15, %xmm4

  364. 	addsd	 %xmm0,  %xmm8

  365. 

  366. 	mulsd	 %xmm15, %xmm5

  367. 	addsd	 %xmm1,  %xmm9

  368. 

  369. 	mulsd	 %xmm15, %xmm6

  370. 	addsd	 %xmm2,  %xmm10

  371. 

  372. 	mulsd	 %xmm15, %xmm7

  373. 	addq	 $8 * SIZE, A2

  374. 	addsd	 %xmm3,  %xmm11

  375. 

  376. 	mulsd	 %xmm14, %xmm0

  377. 	addsd	 %xmm4,  %xmm8

  378. 

  379. 	mulsd	 %xmm14, %xmm1

  380. 	addq	 $8 * SIZE, Y1

  381. 	addsd	 %xmm5,  %xmm9

  382. 

  383. 	mulsd	 %xmm14, %xmm2

  384. 	addq	 $8 * SIZE, A1

  385. 	addsd	 %xmm6,  %xmm10

  386. 

  387. 	mulsd	 %xmm14, %xmm3

  388. 	addsd	 %xmm7,  %xmm11

  389. 

  390. 	movsd	 %xmm8, -4 * SIZE(Y1)

  391. 	movsd	 %xmm9, -3 * SIZE(Y1)

  392. 	movsd	 %xmm10,-2 * SIZE(Y1)

  393. 	movsd	 %xmm11,-1 * SIZE(Y1)

  394. 	ALIGN_3

  395. 

  396. .L15:

  397. 	testq	$4, M

  398. 	je	.L17

  399. 

  400. 	movsd	 0 * SIZE(A1), %xmm0

  401. 	movsd	 1 * SIZE(A1), %xmm1

  402. 	movsd	 2 * SIZE(A1), %xmm2

  403. 	movsd	 3 * SIZE(A1), %xmm3

  404. 

  405. 	movsd	 0 * SIZE(A2), %xmm4

  406. 	movsd	 1 * SIZE(A2), %xmm5

  407. 	movsd	 2 * SIZE(A2), %xmm6

  408. 	movsd	 3 * SIZE(A2), %xmm7

  409. 

  410. 	movsd	 0 * SIZE(Y1), %xmm8

  411. 	mulsd	 %xmm14, %xmm0

  412. 	movsd	 1 * SIZE(Y1), %xmm9

  413. 	mulsd	 %xmm14, %xmm1

  414. 	movsd	 2 * SIZE(Y1), %xmm10

  415. 	mulsd	 %xmm14, %xmm2

  416. 	movsd	 3 * SIZE(Y1), %xmm11

  417. 	mulsd	 %xmm14, %xmm3

  418. 

  419. 	mulsd	 %xmm15, %xmm4

  420. 	addsd	 %xmm0,  %xmm8

  421. 	mulsd	 %xmm15, %xmm5

  422. 	addsd	 %xmm1,  %xmm9

  423. 	mulsd	 %xmm15, %xmm6

  424. 	addsd	 %xmm2,  %xmm10

  425. 	mulsd	 %xmm15, %xmm7

  426. 	addsd	 %xmm3,  %xmm11

  427. 

  428. 	addsd	 %xmm4,  %xmm8

  429. 	addsd	 %xmm5,  %xmm9

  430. 	addsd	 %xmm6,  %xmm10

  431. 	addsd	 %xmm7,  %xmm11

  432. 

  433. 	movsd	 %xmm8,  0 * SIZE(Y1)

  434. 	movsd	 %xmm9,  1 * SIZE(Y1)

  435. 	movsd	 %xmm10, 2 * SIZE(Y1)

  436. 	movsd	 %xmm11, 3 * SIZE(Y1)

  437. 

  438. 	addq	 $4 * SIZE, A1

  439. 	addq	 $4 * SIZE, A2

  440. 	addq	 $4 * SIZE, Y1

  441. 	ALIGN_3

  442. 

  443. .L17:

  444. 	testq	$2, M

  445. 	je	.L18

  446. 

  447. 	movsd	 0 * SIZE(A1), %xmm0

  448. 	movsd	 1 * SIZE(A1), %xmm1

  449. 

  450. 	movsd	 0 * SIZE(A2), %xmm4

  451. 	movsd	 1 * SIZE(A2), %xmm5

  452. 

  453. 	mulsd	 %xmm14, %xmm0

  454. 	movsd	 0 * SIZE(Y1), %xmm8

  455. 	mulsd	 %xmm14, %xmm1

  456. 	movsd	 1 * SIZE(Y1), %xmm9

  457. 	mulsd	 %xmm15, %xmm4

  458. 	mulsd	 %xmm15, %xmm5

  459. 

  460. 	addsd	 %xmm0,  %xmm8

  461. 	addsd	 %xmm1,  %xmm9

  462. 	addsd	 %xmm4,  %xmm8

  463. 	addsd	 %xmm5,  %xmm9

  464. 

  465. 	movsd	 %xmm8, 0 * SIZE(Y1)

  466. 	movsd	 %xmm9, 1 * SIZE(Y1)

  467. 

  468. 	addq	 $2 * SIZE, A1

  469. 	addq	 $2 * SIZE, A2

  470. 	addq	 $2 * SIZE, Y1

  471. 	ALIGN_3

  472. 

  473. .L18:

  474. 	testq	$1, M

  475. 	je	.L19

  476. 

  477. 	movsd	 0 * SIZE(Y1), %xmm8

  478. 

  479. 	movsd	 0 * SIZE(A1), %xmm0

  480. 	movsd	 0 * SIZE(A2), %xmm4

  481. 

  482. 	mulsd	 %xmm14, %xmm0

  483. 	mulsd	 %xmm15, %xmm4

  484. 

  485. 	addsd	 %xmm0,  %xmm8

  486. 	addsd	 %xmm4,  %xmm8

  487. 

  488. 	movsd	 %xmm8, 0 * SIZE(Y1)

  489. 	ALIGN_3

  490. 

  491. .L19:

  492. 	decq	J

  493. 	jg	.L11

  494. 	ALIGN_3

  495. 

  496. .L20:

  497. 	testq	$1, N

  498. 	je	.L990

  499. 

  500. 	movq	BUFFER, Y1

  501. 	movq	A,  A1

  502. 

  503. 	movsd	(X), %xmm14

  504. 	mulsd	STACK_ALPHA, %xmm14

  505. 

  506. 	movq	M,  I

  507. 	sarq	$3, I

  508. 	jle	.L25

  509. 

  510. 	movsd	 0 * SIZE(A1), %xmm0

  511. 	movsd	 1 * SIZE(A1), %xmm1

  512. 	movsd	 2 * SIZE(A1), %xmm2

  513. 	movsd	 3 * SIZE(A1), %xmm3

  514. 

  515.  	movsd	 4 * SIZE(A1), %xmm4

  516.  	movsd	 5 * SIZE(A1), %xmm5

  517. 	movsd	 6 * SIZE(A1), %xmm6

  518. 	movsd	 7 * SIZE(A1), %xmm7

  519. 

  520. 	movsd	 0 * SIZE(Y1), %xmm8

  521. 	mulsd	 %xmm14, %xmm0

  522. 	movsd	 1 * SIZE(Y1), %xmm9

  523. 	mulsd	 %xmm14, %xmm1

  524. 	movsd	 2 * SIZE(Y1), %xmm10

  525. 	mulsd	 %xmm14, %xmm2

  526. 	movsd	 3 * SIZE(Y1), %xmm11

  527. 	mulsd	 %xmm14, %xmm3

  528. 

  529. 	decq	 I

  530. 	jle	 .L24

  531. 	ALIGN_3

  532. 

  533. .L23:

  534. 	PREFETCH	PREFETCH_SIZE * SIZE(A1)

  535. 

  536. 	addsd	 %xmm0,  %xmm8

  537. 	movsd	  8 * SIZE(A1), %xmm0

  538. 	addsd	 %xmm1,  %xmm9

  539. 	movsd	  9 * SIZE(A1), %xmm1

  540. 	addsd	 %xmm2,  %xmm10

  541. 	movsd	 10 * SIZE(A1), %xmm2

  542. 	addsd	 %xmm3,  %xmm11

  543. 	movsd	 11 * SIZE(A1), %xmm3

  544. 

  545. 	movsd	 %xmm8,  0 * SIZE(Y1)

  546. 	movsd	 4 * SIZE(Y1), %xmm8

  547. 	mulsd	 %xmm14, %xmm4

  548. 	movsd	 %xmm9,  1 * SIZE(Y1)

  549. 	movsd	 5 * SIZE(Y1), %xmm9

  550. 	mulsd	 %xmm14, %xmm5

  551. 

  552. 	movsd	 %xmm10, 2 * SIZE(Y1)

  553. 	movsd	 6 * SIZE(Y1), %xmm10

  554. 	mulsd	 %xmm14, %xmm6

  555. 

  556. 	movsd	 %xmm11, 3 * SIZE(Y1)

  557. 	movsd	 7 * SIZE(Y1), %xmm11

  558. 	mulsd	 %xmm14, %xmm7

  559. 

  560. 	addsd	 %xmm4,  %xmm8

  561.  	movsd	 12 * SIZE(A1), %xmm4

  562. 	addsd	 %xmm5,  %xmm9

  563.  	movsd	 13 * SIZE(A1), %xmm5

  564. 	addsd	 %xmm6,  %xmm10

  565. 	movsd	 14 * SIZE(A1), %xmm6

  566. 	addsd	 %xmm7,  %xmm11

  567. 	movsd	 15 * SIZE(A1), %xmm7

  568. 

  569. 	movsd	 %xmm8,  4 * SIZE(Y1)

  570. 	movsd	  8 * SIZE(Y1), %xmm8

  571. 	mulsd	 %xmm14, %xmm0

  572. 	movsd	 %xmm9,  5 * SIZE(Y1)

  573. 	movsd	  9 * SIZE(Y1), %xmm9

  574. 	mulsd	 %xmm14, %xmm1

  575. 	movsd	 %xmm10, 6 * SIZE(Y1)

  576. 	movsd	 10 * SIZE(Y1), %xmm10

  577. 	mulsd	 %xmm14, %xmm2

  578. 	movsd	 %xmm11, 7 * SIZE(Y1)

  579. 	movsd	 11 * SIZE(Y1), %xmm11

  580. 	mulsd	 %xmm14, %xmm3

  581. 

  582. 	addq	 $8 * SIZE, Y1

  583. 	addq	 $8 * SIZE, A1

  584. 

  585. 	decq	 I

  586. 	jg	.L23

  587. 	ALIGN_3

  588. 

  589. .L24:

  590. 	addsd	 %xmm0,  %xmm8

  591. 	addsd	 %xmm1,  %xmm9

  592. 	addsd	 %xmm2,  %xmm10

  593. 	addsd	 %xmm3,  %xmm11

  594. 

  595. 	mulsd	 %xmm14, %xmm4

  596. 	movsd	 %xmm8,  0 * SIZE(Y1)

  597. 	movsd	 4 * SIZE(Y1), %xmm8

  598. 

  599. 	mulsd	 %xmm14, %xmm5

  600. 	movsd	 %xmm9,  1 * SIZE(Y1)

  601. 	movsd	 5 * SIZE(Y1), %xmm9

  602. 

  603. 	mulsd	 %xmm14, %xmm6

  604. 	movsd	 %xmm10, 2 * SIZE(Y1)

  605. 	movsd	 6 * SIZE(Y1), %xmm10

  606. 

  607. 	mulsd	 %xmm14, %xmm7

  608. 	movsd	 %xmm11, 3 * SIZE(Y1)

  609. 	movsd	 7 * SIZE(Y1), %xmm11

  610. 

  611. 	addsd	 %xmm4,  %xmm8

  612. 	addsd	 %xmm5,  %xmm9

  613. 	addsd	 %xmm6,  %xmm10

  614. 	addsd	 %xmm7,  %xmm11

  615. 

  616. 	movsd	 %xmm8,  4 * SIZE(Y1)

  617. 	movsd	 %xmm9,  5 * SIZE(Y1)

  618. 	movsd	 %xmm10, 6 * SIZE(Y1)

  619. 	movsd	 %xmm11, 7 * SIZE(Y1)

  620. 

  621. 	addq	 $8 * SIZE, Y1

  622. 	addq	 $8 * SIZE, A1

  623. 	ALIGN_3

  624. 

  625. .L25:

  626. 	testq	$4, M

  627. 	je	.L27

  628. 

  629. 	movsd	 0 * SIZE(A1), %xmm0

  630. 	movsd	 1 * SIZE(A1), %xmm1

  631. 	movsd	 2 * SIZE(A1), %xmm2

  632. 	movsd	 3 * SIZE(A1), %xmm3

  633. 

  634. 	movsd	 0 * SIZE(Y1), %xmm8

  635. 	mulsd	 %xmm14, %xmm0

  636. 	movsd	 1 * SIZE(Y1), %xmm9

  637. 	mulsd	 %xmm14, %xmm1

  638. 	movsd	 2 * SIZE(Y1), %xmm10

  639. 	mulsd	 %xmm14, %xmm2

  640. 	movsd	 3 * SIZE(Y1), %xmm11

  641. 	mulsd	 %xmm14, %xmm3

  642. 

  643. 	addsd	 %xmm0,  %xmm8

  644. 	addsd	 %xmm1,  %xmm9

  645. 	addsd	 %xmm2,  %xmm10

  646. 	addsd	 %xmm3,  %xmm11

  647. 

  648. 	movsd	 %xmm8,  0 * SIZE(Y1)

  649. 	movsd	 %xmm9,  1 * SIZE(Y1)

  650. 	movsd	 %xmm10, 2 * SIZE(Y1)

  651. 	movsd	 %xmm11, 3 * SIZE(Y1)

  652. 

  653. 	addq	 $4 * SIZE, A1

  654. 	addq	 $4 * SIZE, A2

  655. 	addq	 $4 * SIZE, Y1

  656. 	ALIGN_3

  657. 

  658. .L27:

  659. 	testq	$2, M

  660. 	je	.L28

  661. 

  662. 	movsd	 0 * SIZE(A1), %xmm0

  663. 	movsd	 1 * SIZE(A1), %xmm1

  664. 

  665. 	mulsd	 %xmm14, %xmm0

  666. 	movsd	 0 * SIZE(Y1), %xmm8

  667. 	mulsd	 %xmm14, %xmm1

  668. 	movsd	 1 * SIZE(Y1), %xmm9

  669. 

  670. 	addsd	 %xmm0,  %xmm8

  671. 	addsd	 %xmm1,  %xmm9

  672. 

  673. 	movsd	 %xmm8, 0 * SIZE(Y1)

  674. 	movsd	 %xmm9, 1 * SIZE(Y1)

  675. 

  676. 	addq	 $2 * SIZE, A1

  677. 	addq	 $2 * SIZE, Y1

  678. 	ALIGN_3

  679. 

  680. .L28:

  681. 	testq	$1, M

  682. 	je	.L990

  683. 

  684. 	movsd	 0 * SIZE(Y1), %xmm8

  685. 	movsd	 0 * SIZE(A1), %xmm0

  686. 

  687. 	mulsd	 %xmm14, %xmm0

  688. 	addsd	 %xmm0,  %xmm8

  689. 

  690. 	movsd	 %xmm8, 0 * SIZE(Y1)

  691. 	ALIGN_3

  692. 

  693. .L990:

  694. 	cmpq	$SIZE, INCY

  695. 	je	.L999

  696. 

  697. 	movq	Y,  Y1

  698. 

  699. 	movq	M,  %rax

  700. 	sarq	$2, %rax

  701. 	jle	.L994

  702. 	ALIGN_3

  703. 

  704. .L992:

  705. 	movsd	(Y), %xmm0

  706. 	addq	INCY, Y

  707. 	movsd	(Y), %xmm1

  708. 	addq	INCY, Y

  709. 	movsd	(Y), %xmm2

  710. 	addq	INCY, Y

  711. 	movsd	(Y), %xmm3

  712. 	addq	INCY, Y

  713. 

  714. 	addsd	0 * SIZE(BUFFER), %xmm0

  715. 	addsd	1 * SIZE(BUFFER), %xmm1

  716. 	addsd	2 * SIZE(BUFFER), %xmm2

  717. 	addsd	3 * SIZE(BUFFER), %xmm3

  718. 	addq	$4 * SIZE, BUFFER

  719. 

  720. 	movsd	%xmm0, (Y1)

  721. 	addq	INCY, Y1

  722. 	movsd	%xmm1, (Y1)

  723. 	addq	INCY, Y1

  724. 	movsd	%xmm2, (Y1)

  725. 	addq	INCY, Y1

  726. 	movsd	%xmm3, (Y1)

  727. 	addq	INCY, Y1

  728. 

  729. 	decq	%rax

  730. 	jg	.L992

  731. 	ALIGN_3

  732. 

  733. .L994:

  734. 	testq	$2, M

  735. 	jle	.L996

  736. 

  737. 	movsd	(Y), %xmm0

  738. 	addq	INCY, Y

  739. 	movsd	(Y), %xmm1

  740. 	addq	INCY, Y

  741. 

  742. 	addsd	0 * SIZE(BUFFER), %xmm0

  743. 	addsd	1 * SIZE(BUFFER), %xmm1

  744. 	addq	$2 * SIZE, BUFFER

  745. 

  746. 	movsd	%xmm0, (Y1)

  747. 	addq	INCY, Y1

  748. 	movsd	%xmm1, (Y1)

  749. 	addq	INCY, Y1

  750. 	ALIGN_3

  751. 

  752. .L996:

  753. 	testq	$1, M

  754. 	jle	.L999

  755. 

  756. 	movsd	(Y), %xmm0

  757. 

  758. 	addsd	(BUFFER), %xmm0

  759. 

  760. 	movsd	%xmm0, (Y1)

  761. 	ALIGN_3

  762. 

  763. .L999:

  764. 	movq	  0(%rsp), %rbx

  765. 	movq	  8(%rsp), %rbp

  766. 	movq	 16(%rsp), %r12

  767. 	movq	 24(%rsp), %r13

  768. 	movq	 32(%rsp), %r14

  769. 	movq	 40(%rsp), %r15

  770. 

  771. #ifdef WINDOWS_ABI

  772. 	movq	 48(%rsp), %rdi

  773. 	movq	 56(%rsp), %rsi

  774. 	movups	 64(%rsp), %xmm6

  775. 	movups	 80(%rsp), %xmm7

  776. 	movups	 96(%rsp), %xmm8

  777. 	movups	112(%rsp), %xmm9

  778. 	movups	128(%rsp), %xmm10

  779. 	movups	144(%rsp), %xmm11

  780. 	movups	160(%rsp), %xmm12

  781. 	movups	176(%rsp), %xmm13

  782. 	movups	192(%rsp), %xmm14

  783. 	movups	208(%rsp), %xmm15

  784. #endif

  785. 

  786. 	addq	$STACKSIZE, %rsp

  787. 	ret

  788. 	EPILOGUE