OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 66d9185ebe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '66d9185ebe'
${ noResults }
OpenBLAS/kernel/x86_64/axpy_sse2.S

916 lines
17 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #define Y	ARG6

  47. #define INCY	ARG2

  48. #else

  49. #define M	ARG1

  50. #define X	ARG2

  51. #define INCX	ARG3

  52. #define Y	ARG4

  53. #define INCY	%r10

  54. #endif

  55. 

  56. #define	YY	%r11

  57. #define ALPHA	%xmm15

  58. 

  59. #include "l1param.h"

  60. 

  61. 	PROLOGUE

  62. 	PROFCODE

  63. 

  64. #ifndef WINDOWS_ABI

  65. #ifndef XDOUBLE

  66. 	movq	 8(%rsp), INCY

  67. #else

  68. 	movq	24(%rsp), INCY

  69. #endif

  70. 	movaps	%xmm0,  ALPHA

  71. #else

  72. 

  73. 	movq	40(%rsp), X

  74. 	movq	48(%rsp), INCX

  75. 	movq	56(%rsp), Y

  76. 	movq	64(%rsp), INCY

  77. #endif

  78. 

  79. 	SAVEREGISTERS

  80. 

  81. #ifdef WINDOWS_ABI

  82. 	movaps	%xmm3,  ALPHA

  83. #endif

  84. 

  85. 	unpcklpd ALPHA, ALPHA

  86. 

  87. 	leaq	(, INCX, SIZE), INCX

  88. 	leaq	(, INCY, SIZE), INCY

  89. 

  90. 	testq	M, M

  91. 	jle	.L47

  92. 

  93. 	cmpq	$SIZE, INCX

  94. 	jne	.L40

  95. 	cmpq	$SIZE, INCY

  96. 	jne	.L40

  97. 

  98. 	testq	$SIZE, Y

  99. 	je	.L10

  100. 

  101. 	movsd	(X), %xmm0

  102. 	mulsd	ALPHA, %xmm0

  103. 	addsd	(Y), %xmm0

  104. 	movsd	%xmm0, (Y)

  105. 	addq	$1 * SIZE, X

  106. 	addq	$1 * SIZE, Y

  107. 	decq	M

  108. 	jle	.L19

  109. 	ALIGN_4

  110. 

  111. .L10:

  112. 	subq	$-16 * SIZE, X

  113. 	subq	$-16 * SIZE, Y

  114. 

  115. 	testq	$SIZE, X

  116. 	jne	.L20

  117. 

  118. 	movq	M,  %rax

  119. 	sarq	$4, %rax

  120. 	jle	.L13

  121. 

  122. 	movaps	-16 * SIZE(X), %xmm0

  123. 	movaps	-14 * SIZE(X), %xmm1

  124. 	movaps	-12 * SIZE(X), %xmm2

  125. 	movaps	-10 * SIZE(X), %xmm3

  126. 

  127. 	decq	%rax

  128. 	jle .L12

  129. 	ALIGN_3

  130. 

  131. .L11:

  132. 	movaps	 -8 * SIZE(X), %xmm4

  133. 	movaps	 -6 * SIZE(X), %xmm5

  134. 

  135. #ifdef PREFETCHW

  136. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  137. #endif

  138. 

  139. 	mulpd	ALPHA, %xmm0

  140. 	addpd	-16 * SIZE(Y), %xmm0

  141. 	movaps	%xmm0, -16 * SIZE(Y)

  142. 

  143. 	mulpd	ALPHA, %xmm1

  144. 	addpd	-14 * SIZE(Y), %xmm1

  145. 	movaps	%xmm1, -14 * SIZE(Y)

  146. 

  147. 	movaps	 -4 * SIZE(X), %xmm6

  148. 	movaps	 -2 * SIZE(X), %xmm7

  149. 

  150. #ifdef PREFETCH

  151. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  152. #endif

  153. 

  154. 	mulpd	ALPHA, %xmm2

  155. 	addpd	-12 * SIZE(Y), %xmm2

  156. 	movaps	%xmm2, -12 * SIZE(Y)

  157. 

  158. 	mulpd	ALPHA, %xmm3

  159. 	addpd	-10 * SIZE(Y), %xmm3

  160. 	movaps	%xmm3, -10 * SIZE(Y)

  161. 

  162. 	movaps	 0 * SIZE(X), %xmm0

  163. 	movaps	 2 * SIZE(X), %xmm1

  164. 

  165. #if defined(PREFETCHW) && !defined(FETCH128)

  166. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  167. #endif

  168. 

  169. 	mulpd	ALPHA, %xmm4

  170. 	addpd	 -8 * SIZE(Y), %xmm4

  171. 	movaps	%xmm4, -8 * SIZE(Y)

  172. 

  173. 	mulpd	ALPHA, %xmm5

  174. 	addpd	 -6 * SIZE(Y), %xmm5

  175. 	movaps	%xmm5, -6 * SIZE(Y)

  176. 

  177. 	movaps	 4 * SIZE(X), %xmm2

  178. 	movaps	 6 * SIZE(X), %xmm3

  179. 

  180. #if defined(PREFETCH) && !defined(FETCH128)

  181. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  182. #endif

  183. 

  184. 	mulpd	ALPHA, %xmm6

  185. 	addpd	 -4 * SIZE(Y), %xmm6

  186. 	movaps	%xmm6, -4 * SIZE(Y)

  187. 

  188. 	mulpd	ALPHA, %xmm7

  189. 	addpd	 -2 * SIZE(Y), %xmm7

  190. 	movaps	%xmm7, -2 * SIZE(Y)

  191. 

  192. 	subq	$-16 * SIZE, Y

  193. 	subq	$-16 * SIZE, X

  194. 	decq	%rax

  195. 	jg	.L11

  196. 	ALIGN_3

  197. 

  198. .L12:

  199. 	movaps	 -8 * SIZE(X), %xmm4

  200. 	movaps	 -6 * SIZE(X), %xmm5

  201. 

  202. 	mulpd	ALPHA, %xmm0

  203. 	addpd	-16 * SIZE(Y), %xmm0

  204. 	movaps	%xmm0, -16 * SIZE(Y)

  205. 

  206. 	mulpd	ALPHA, %xmm1

  207. 	addpd	-14 * SIZE(Y), %xmm1

  208. 	movaps	%xmm1, -14 * SIZE(Y)

  209. 

  210. 	movaps	 -4 * SIZE(X), %xmm6

  211. 	movaps	 -2 * SIZE(X), %xmm7

  212. 

  213. 	mulpd	ALPHA, %xmm2

  214. 	addpd	-12 * SIZE(Y), %xmm2

  215. 	movaps	%xmm2, -12 * SIZE(Y)

  216. 

  217. 	mulpd	ALPHA, %xmm3

  218. 	addpd	-10 * SIZE(Y), %xmm3

  219. 	movaps	%xmm3, -10 * SIZE(Y)

  220. 

  221. 	mulpd	ALPHA, %xmm4

  222. 	addpd	 -8 * SIZE(Y), %xmm4

  223. 	movaps	%xmm4,  -8 * SIZE(Y)

  224. 

  225. 	mulpd	ALPHA, %xmm5

  226. 	addpd	 -6 * SIZE(Y), %xmm5

  227. 	movaps	%xmm5,  -6 * SIZE(Y)

  228. 

  229. 	mulpd	ALPHA, %xmm6

  230. 	addpd	 -4 * SIZE(Y), %xmm6

  231. 	movaps	%xmm6,  -4 * SIZE(Y)

  232. 

  233. 	mulpd	ALPHA, %xmm7

  234. 	addpd	 -2 * SIZE(Y), %xmm7

  235. 	movaps	%xmm7,  -2 * SIZE(Y)

  236. 

  237. 	subq	$-16 * SIZE, Y

  238. 	subq	$-16 * SIZE, X

  239. 	ALIGN_3

  240. 

  241. .L13:

  242. 	movq	M,  %rax

  243. 	andq	$8, %rax

  244. 	jle	.L14

  245. 	ALIGN_3

  246. 

  247. 	movaps	-16 * SIZE(X), %xmm0

  248. 	movaps	-14 * SIZE(X), %xmm1

  249. 	movaps	-12 * SIZE(X), %xmm2

  250. 	movaps	-10 * SIZE(X), %xmm3

  251. 

  252. 	mulpd	ALPHA, %xmm0

  253. 	addpd	-16 * SIZE(Y), %xmm0

  254. 	mulpd	ALPHA, %xmm1

  255. 	addpd	-14 * SIZE(Y), %xmm1

  256. 	mulpd	ALPHA, %xmm2

  257. 	addpd	-12 * SIZE(Y), %xmm2

  258. 	mulpd	ALPHA, %xmm3

  259. 	addpd	-10 * SIZE(Y), %xmm3

  260. 

  261. 	movaps	%xmm0, -16 * SIZE(Y)

  262. 	movaps	%xmm1, -14 * SIZE(Y)

  263. 	movaps	%xmm2, -12 * SIZE(Y)

  264. 	movaps	%xmm3, -10 * SIZE(Y)

  265. 

  266. 	addq	$8 * SIZE, X

  267. 	addq	$8 * SIZE, Y

  268. 	ALIGN_3

  269. 

  270. .L14:

  271. 	movq	M,  %rax

  272. 	andq	$4, %rax

  273. 	jle	.L15

  274. 	ALIGN_3

  275. 

  276. 	movaps	-16 * SIZE(X), %xmm0

  277. 	movaps	-14 * SIZE(X), %xmm1

  278. 

  279. 	mulpd	ALPHA, %xmm0

  280. 	mulpd	ALPHA, %xmm1

  281. 

  282. 	addpd	-16 * SIZE(Y), %xmm0

  283. 	addpd	-14 * SIZE(Y), %xmm1

  284. 

  285. 	movaps	%xmm0, -16 * SIZE(Y)

  286. 	movaps	%xmm1, -14 * SIZE(Y)

  287. 

  288. 	addq	$4 * SIZE, X

  289. 	addq	$4 * SIZE, Y

  290. 	ALIGN_3

  291. 

  292. .L15:

  293. 	movq	M,  %rax

  294. 	andq	$2, %rax

  295. 	jle	.L16

  296. 	ALIGN_3

  297. 

  298. 	movaps	-16 * SIZE(X), %xmm0

  299. 	mulpd	ALPHA, %xmm0

  300. 	addpd	-16 * SIZE(Y), %xmm0

  301. 	movaps	%xmm0, -16 * SIZE(Y)

  302. 

  303. 	addq	$2 * SIZE, X

  304. 	addq	$2 * SIZE, Y

  305. 	ALIGN_3

  306. 

  307. .L16:

  308. 	movq	M,  %rax

  309. 	andq	$1, %rax

  310. 	jle	.L19

  311. 	ALIGN_3

  312. 

  313. 	movsd	-16 * SIZE(X), %xmm0

  314. 	mulsd	ALPHA, %xmm0

  315. 	addsd	-16 * SIZE(Y), %xmm0

  316. 

  317. 	movsd	%xmm0, 	-16 * SIZE(Y)

  318. 	ALIGN_3

  319. 

  320. .L19:

  321. 	xorq	%rax,%rax

  322. 

  323. 	RESTOREREGISTERS

  324. 

  325. 	ret

  326. 	ALIGN_3

  327. 

  328. .L20:

  329. #ifdef ALIGNED_ACCESS

  330. 

  331. 	movhps	-16 * SIZE(X), %xmm0

  332. 

  333. 	movq	M,  %rax

  334. 	sarq	$4, %rax

  335. 	jle	.L23

  336. 

  337. 	movaps	-15 * SIZE(X), %xmm1

  338. 	movaps	-13 * SIZE(X), %xmm2

  339. 	movaps	-11 * SIZE(X), %xmm3

  340. 

  341. 	decq	%rax

  342. 	jle .L22

  343. 	ALIGN_4

  344. 

  345. .L21:

  346. 	movaps	 -9 * SIZE(X), %xmm4

  347. 	movaps	 -7 * SIZE(X), %xmm5

  348. 

  349. #ifdef PREFETCHW

  350. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  351. #endif

  352. 

  353. 	SHUFPD_1 %xmm1, %xmm0

  354. 	mulpd	ALPHA, %xmm0

  355. 	addpd	-16 * SIZE(Y), %xmm0

  356. 	movaps	%xmm0, -16 * SIZE(Y)

  357. 

  358. 	SHUFPD_1 %xmm2, %xmm1

  359. 	mulpd	ALPHA, %xmm1

  360. 	addpd	-14 * SIZE(Y), %xmm1

  361. 	movaps	%xmm1, -14 * SIZE(Y)

  362. 

  363. 	movaps	 -5 * SIZE(X), %xmm6

  364. 	movaps	 -3 * SIZE(X), %xmm7

  365. 

  366. #ifdef PREFETCH

  367. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  368. #endif

  369. 

  370. 	SHUFPD_1 %xmm3, %xmm2

  371. 	mulpd	ALPHA, %xmm2

  372. 	addpd	-12 * SIZE(Y), %xmm2

  373. 	movaps	%xmm2, -12 * SIZE(Y)

  374. 

  375. 	SHUFPD_1 %xmm4, %xmm3

  376. 	mulpd	ALPHA, %xmm3

  377. 	addpd	-10 * SIZE(Y), %xmm3

  378. 	movaps	%xmm3, -10 * SIZE(Y)

  379. 

  380. 	movaps	-1 * SIZE(X), %xmm0

  381. 	movaps	 1 * SIZE(X), %xmm1

  382. 

  383. #if defined(PREFETCHW) && !defined(FETCH128)

  384. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  385. #endif

  386. 

  387. 	SHUFPD_1 %xmm5, %xmm4

  388. 	mulpd	ALPHA, %xmm4

  389. 	addpd	 -8 * SIZE(Y), %xmm4

  390. 	movaps	%xmm4,  -8 * SIZE(Y)

  391. 

  392. 	SHUFPD_1 %xmm6, %xmm5

  393. 	mulpd	ALPHA, %xmm5

  394. 	addpd	 -6 * SIZE(Y), %xmm5

  395. 	movaps	%xmm5, -6 * SIZE(Y)

  396. 

  397. 	movaps	 3 * SIZE(X), %xmm2

  398. 	movaps	 5 * SIZE(X), %xmm3

  399. 

  400. #if defined(PREFETCH) && !defined(FETCH128)

  401. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  402. #endif

  403. 

  404. 	SHUFPD_1 %xmm7, %xmm6

  405. 	mulpd	ALPHA, %xmm6

  406. 	addpd	 -4 * SIZE(Y), %xmm6

  407. 	movaps	%xmm6, -4 * SIZE(Y)

  408. 

  409. 	SHUFPD_1 %xmm0, %xmm7

  410. 	mulpd	ALPHA, %xmm7

  411. 	addpd	-2 * SIZE(Y), %xmm7

  412. 	movaps	%xmm7, -2 * SIZE(Y)

  413. 

  414. 	subq	$-16 * SIZE, X

  415. 	subq	$-16 * SIZE, Y

  416. 	decq	%rax

  417. 	jg	.L21

  418. 	ALIGN_3

  419. 

  420. .L22:

  421. 	movaps	 -9 * SIZE(X), %xmm4

  422. 	movaps	 -7 * SIZE(X), %xmm5

  423. 

  424. 	SHUFPD_1 %xmm1, %xmm0

  425. 	mulpd	ALPHA, %xmm0

  426. 	addpd	-16 * SIZE(Y), %xmm0

  427. 	movaps	%xmm0, -16 * SIZE(Y)

  428. 	movaps	-1 * SIZE(X), %xmm0

  429. 

  430. 	SHUFPD_1 %xmm2, %xmm1

  431. 	mulpd	ALPHA, %xmm1

  432. 	addpd	-14 * SIZE(Y), %xmm1

  433. 	movaps	%xmm1, -14 * SIZE(Y)

  434. 

  435. 	movaps	 -5 * SIZE(X), %xmm6

  436. 	movaps	 -3 * SIZE(X), %xmm7

  437. 

  438. 	SHUFPD_1 %xmm3, %xmm2

  439. 	mulpd	ALPHA, %xmm2

  440. 	addpd	-12 * SIZE(Y), %xmm2

  441. 	movaps	%xmm2, -12 * SIZE(Y)

  442. 

  443. 	SHUFPD_1 %xmm4, %xmm3

  444. 	mulpd	ALPHA, %xmm3

  445. 	addpd	-10 * SIZE(Y), %xmm3

  446. 	movaps	%xmm3, -10 * SIZE(Y)

  447. 

  448. 	SHUFPD_1 %xmm5, %xmm4

  449. 	mulpd	ALPHA, %xmm4

  450. 	addpd	 -8 * SIZE(Y), %xmm4

  451. 	movaps	%xmm4,  -8 * SIZE(Y)

  452. 

  453. 	SHUFPD_1 %xmm6, %xmm5

  454. 	mulpd	ALPHA, %xmm5

  455. 	addpd	 -6 * SIZE(Y), %xmm5

  456. 	movaps	%xmm5,  -6 * SIZE(Y)

  457. 

  458. 	SHUFPD_1 %xmm7, %xmm6

  459. 	mulpd	ALPHA, %xmm6

  460. 	addpd	 -4 * SIZE(Y), %xmm6

  461. 	movaps	%xmm6,  -4 * SIZE(Y)

  462. 

  463. 	SHUFPD_1 %xmm0, %xmm7

  464. 	mulpd	ALPHA, %xmm7

  465. 	addpd	 -2 * SIZE(Y), %xmm7

  466. 	movaps	%xmm7,  -2 * SIZE(Y)

  467. 

  468. 	subq	$-16 * SIZE, X

  469. 	subq	$-16 * SIZE, Y

  470. 	ALIGN_3

  471. 

  472. .L23:

  473. 	movq	M,  %rax

  474. 	andq	$8, %rax

  475. 	jle	.L24

  476. 	ALIGN_3

  477. 

  478. 	movaps	-15 * SIZE(X), %xmm1

  479. 	movaps	-13 * SIZE(X), %xmm2

  480. 	movaps	-11 * SIZE(X), %xmm3

  481. 	movaps	 -9 * SIZE(X), %xmm8

  482. 

  483. 	SHUFPD_1 %xmm1, %xmm0

  484. 	mulpd	ALPHA, %xmm0

  485. 	addpd	-16 * SIZE(Y), %xmm0

  486. 	movaps	%xmm0, -16 * SIZE(Y)

  487. 

  488. 	SHUFPD_1 %xmm2, %xmm1

  489. 	mulpd	ALPHA, %xmm1

  490. 	addpd	-14 * SIZE(Y), %xmm1

  491. 	movaps	%xmm1, -14 * SIZE(Y)

  492. 

  493. 	SHUFPD_1 %xmm3, %xmm2

  494. 	mulpd	ALPHA, %xmm2

  495. 	addpd	-12 * SIZE(Y), %xmm2

  496. 	movaps	%xmm2, -12 * SIZE(Y)

  497. 

  498. 	SHUFPD_1 %xmm8, %xmm3

  499. 	mulpd	ALPHA, %xmm3

  500. 	addpd	-10 * SIZE(Y), %xmm3

  501. 	movaps	%xmm3, -10 * SIZE(Y)

  502. 

  503. 	movaps	%xmm8, %xmm0

  504. 

  505. 	addq	$8 * SIZE, X

  506. 	addq	$8 * SIZE, Y

  507. 	ALIGN_3

  508. 

  509. .L24:

  510. 	movq	M,  %rax

  511. 	andq	$4, %rax

  512. 	jle	.L25

  513. 	ALIGN_3

  514. 

  515. 	movaps	-15 * SIZE(X), %xmm1

  516. 	movaps	-13 * SIZE(X), %xmm2

  517. 

  518. 	SHUFPD_1 %xmm1, %xmm0

  519. 	SHUFPD_1 %xmm2, %xmm1

  520. 

  521. 	mulpd	ALPHA, %xmm0

  522. 	mulpd	ALPHA, %xmm1

  523. 

  524. 	addpd	-16 * SIZE(Y), %xmm0

  525. 	addpd	-14 * SIZE(Y), %xmm1

  526. 

  527. 	movaps	%xmm0, -16 * SIZE(Y)

  528. 	movaps	%xmm1, -14 * SIZE(Y)

  529. 	movaps	%xmm2, %xmm0

  530. 

  531. 	addq	$4 * SIZE, X

  532. 	addq	$4 * SIZE, Y

  533. 	ALIGN_3

  534. 

  535. .L25:

  536. 	movq	M,  %rax

  537. 	andq	$2, %rax

  538. 	jle	.L26

  539. 	ALIGN_3

  540. 

  541. 	movaps	-15 * SIZE(X), %xmm1

  542. 	SHUFPD_1 %xmm1, %xmm0

  543. 	mulpd	ALPHA,  %xmm0

  544. 	addpd	-16 * SIZE(Y), %xmm0

  545. 

  546. 	movaps	%xmm0, -16 * SIZE(Y)

  547. 

  548. 	addq	$2 * SIZE, X

  549. 	addq	$2 * SIZE, Y

  550. 	ALIGN_3

  551. 

  552. .L26:

  553. 	movq	M,  %rax

  554. 	andq	$1, %rax

  555. 	jle	.L29

  556. 	ALIGN_3

  557. 

  558. 	movsd	-16 * SIZE(X), %xmm0

  559. 	mulsd	ALPHA, %xmm0

  560. 	addsd	-16 * SIZE(Y), %xmm0

  561. 

  562. 	movsd	%xmm0, 	-16 * SIZE(Y)

  563. 	ALIGN_3

  564. 

  565. .L29:

  566. 	xorq	%rax,%rax

  567. 

  568. 	RESTOREREGISTERS

  569. 

  570. 	ret

  571. 	ALIGN_3

  572. 

  573. #else

  574. 	movq	M,  %rax

  575. 	sarq	$4, %rax

  576. 	jle	.L23

  577. 

  578. 	movsd	-16 * SIZE(X), %xmm0

  579. 	movhps	-15 * SIZE(X), %xmm0

  580. 	movsd	-14 * SIZE(X), %xmm1

  581. 	movhps	-13 * SIZE(X), %xmm1

  582. 	movsd	-12 * SIZE(X), %xmm2

  583. 	movhps	-11 * SIZE(X), %xmm2

  584. 	movsd	-10 * SIZE(X), %xmm3

  585. 	movhps	 -9 * SIZE(X), %xmm3

  586. 

  587. 	decq	%rax

  588. 	jle .L22

  589. 	ALIGN_3

  590. 

  591. .L21:

  592. 	movsd	 -8 * SIZE(X), %xmm4

  593. 	movhps	 -7 * SIZE(X), %xmm4

  594. 	movsd	 -6 * SIZE(X), %xmm5

  595. 	movhps	 -5 * SIZE(X), %xmm5

  596. 

  597. #ifdef PREFETCHW

  598. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  599. #endif

  600. 

  601. 	mulpd	ALPHA, %xmm0

  602. 	addpd	-16 * SIZE(Y), %xmm0

  603. 	movaps	%xmm0, -16 * SIZE(Y)

  604. 

  605. 	mulpd	ALPHA, %xmm1

  606. 	addpd	-14 * SIZE(Y), %xmm1

  607. 	movaps	%xmm1, -14 * SIZE(Y)

  608. 

  609. 	movsd	 -4 * SIZE(X), %xmm6

  610. 	movhps	 -3 * SIZE(X), %xmm6

  611. 	movsd	 -2 * SIZE(X), %xmm7

  612. 	movhps	 -1 * SIZE(X), %xmm7

  613. 

  614. #ifdef PREFETCH

  615. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  616. #endif

  617. 

  618. 	mulpd	ALPHA, %xmm2

  619. 	addpd	-12 * SIZE(Y), %xmm2

  620. 	movaps	%xmm2, -12 * SIZE(Y)

  621. 

  622. 	mulpd	ALPHA, %xmm3

  623. 	addpd	-10 * SIZE(Y), %xmm3

  624. 	movaps	%xmm3, -10 * SIZE(Y)

  625. 

  626. 	movsd	 0 * SIZE(X), %xmm0

  627. 	movhps	 1 * SIZE(X), %xmm0

  628. 	movsd	 2 * SIZE(X), %xmm1

  629. 	movhps	 3 * SIZE(X), %xmm1

  630. 

  631. #if defined(PREFETCHW) && !defined(FETCH128)

  632. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  633. #endif

  634. 

  635. 	mulpd	ALPHA, %xmm4

  636. 	addpd	 -8 * SIZE(Y), %xmm4

  637. 	movaps	%xmm4, -8 * SIZE(Y)

  638. 

  639. 	mulpd	ALPHA, %xmm5

  640. 	addpd	 -6 * SIZE(Y), %xmm5

  641. 	movaps	%xmm5, -6 * SIZE(Y)

  642. 

  643. 	movsd	 4 * SIZE(X), %xmm2

  644. 	movhps	 5 * SIZE(X), %xmm2

  645. 	movsd	 6 * SIZE(X), %xmm3

  646. 	movhps	 7 * SIZE(X), %xmm3

  647. 

  648. #if defined(PREFETCH) && !defined(FETCH128)

  649. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  650. #endif

  651. 

  652. 	mulpd	ALPHA, %xmm6

  653. 	addpd	 -4 * SIZE(Y), %xmm6

  654. 	movaps	%xmm6, -4 * SIZE(Y)

  655. 

  656. 	mulpd	ALPHA, %xmm7

  657. 	addpd	 -2 * SIZE(Y), %xmm7

  658. 	movaps	%xmm7, -2 * SIZE(Y)

  659. 

  660. 	subq	$-16 * SIZE, Y

  661. 	subq	$-16 * SIZE, X

  662. 	decq	%rax

  663. 	jg	.L21

  664. 	ALIGN_3

  665. 

  666. .L22:

  667. 	movsd	 -8 * SIZE(X), %xmm4

  668. 	movhps	 -7 * SIZE(X), %xmm4

  669. 	movsd	 -6 * SIZE(X), %xmm5

  670. 	movhps	 -5 * SIZE(X), %xmm5

  671. 

  672. 	mulpd	ALPHA, %xmm0

  673. 	addpd	-16 * SIZE(Y), %xmm0

  674. 	movaps	%xmm0, -16 * SIZE(Y)

  675. 

  676. 	mulpd	ALPHA, %xmm1

  677. 	addpd	-14 * SIZE(Y), %xmm1

  678. 	movaps	%xmm1, -14 * SIZE(Y)

  679. 

  680. 	movsd	 -4 * SIZE(X), %xmm6

  681. 	movhps	 -3 * SIZE(X), %xmm6

  682. 	movsd	 -2 * SIZE(X), %xmm7

  683. 	movhps	 -1 * SIZE(X), %xmm7

  684. 

  685. 	mulpd	ALPHA, %xmm2

  686. 	addpd	-12 * SIZE(Y), %xmm2

  687. 	movaps	%xmm2, -12 * SIZE(Y)

  688. 

  689. 	mulpd	ALPHA, %xmm3

  690. 	addpd	-10 * SIZE(Y), %xmm3

  691. 	movaps	%xmm3, -10 * SIZE(Y)

  692. 

  693. 	mulpd	ALPHA, %xmm4

  694. 	addpd	 -8 * SIZE(Y), %xmm4

  695. 	movaps	%xmm4,  -8 * SIZE(Y)

  696. 

  697. 	mulpd	ALPHA, %xmm5

  698. 	addpd	 -6 * SIZE(Y), %xmm5

  699. 	movaps	%xmm5,  -6 * SIZE(Y)

  700. 

  701. 	mulpd	ALPHA, %xmm6

  702. 	addpd	 -4 * SIZE(Y), %xmm6

  703. 	movaps	%xmm6,  -4 * SIZE(Y)

  704. 

  705. 	mulpd	ALPHA, %xmm7

  706. 	addpd	 -2 * SIZE(Y), %xmm7

  707. 	movaps	%xmm7,  -2 * SIZE(Y)

  708. 

  709. 	subq	$-16 * SIZE, Y

  710. 	subq	$-16 * SIZE, X

  711. 	ALIGN_3

  712. 

  713. .L23:

  714. 	movq	M,  %rax

  715. 	andq	$8, %rax

  716. 	jle	.L24

  717. 	ALIGN_3

  718. 

  719. 	movsd	-16 * SIZE(X), %xmm0

  720. 	movhps	-15 * SIZE(X), %xmm0

  721. 	movsd	-14 * SIZE(X), %xmm1

  722. 	movhps	-13 * SIZE(X), %xmm1

  723. 	movsd	-12 * SIZE(X), %xmm2

  724. 	movhps	-11 * SIZE(X), %xmm2

  725. 	movsd	-10 * SIZE(X), %xmm3

  726. 	movhps	 -9 * SIZE(X), %xmm3

  727. 

  728. 	mulpd	ALPHA, %xmm0

  729. 	addpd	-16 * SIZE(Y), %xmm0

  730. 	mulpd	ALPHA, %xmm1

  731. 	addpd	-14 * SIZE(Y), %xmm1

  732. 	mulpd	ALPHA, %xmm2

  733. 	addpd	-12 * SIZE(Y), %xmm2

  734. 	mulpd	ALPHA, %xmm3

  735. 	addpd	-10 * SIZE(Y), %xmm3

  736. 

  737. 	movaps	%xmm0, -16 * SIZE(Y)

  738. 	movaps	%xmm1, -14 * SIZE(Y)

  739. 	movaps	%xmm2, -12 * SIZE(Y)

  740. 	movaps	%xmm3, -10 * SIZE(Y)

  741. 

  742. 	addq	$8 * SIZE, X

  743. 	addq	$8 * SIZE, Y

  744. 	ALIGN_3

  745. 

  746. .L24:

  747. 	movq	M,  %rax

  748. 	andq	$4, %rax

  749. 	jle	.L25

  750. 	ALIGN_3

  751. 

  752. 	movsd	-16 * SIZE(X), %xmm0

  753. 	movhps	-15 * SIZE(X), %xmm0

  754. 	movsd	-14 * SIZE(X), %xmm1

  755. 	movhps	-13 * SIZE(X), %xmm1

  756. 

  757. 	mulpd	ALPHA, %xmm0

  758. 	mulpd	ALPHA, %xmm1

  759. 

  760. 	addpd	-16 * SIZE(Y), %xmm0

  761. 	addpd	-14 * SIZE(Y), %xmm1

  762. 

  763. 	movaps	%xmm0, -16 * SIZE(Y)

  764. 	movaps	%xmm1, -14 * SIZE(Y)

  765. 

  766. 	addq	$4 * SIZE, X

  767. 	addq	$4 * SIZE, Y

  768. 	ALIGN_3

  769. 

  770. .L25:

  771. 	movq	M,  %rax

  772. 	andq	$2, %rax

  773. 	jle	.L26

  774. 	ALIGN_3

  775. 

  776. 	movsd	-16 * SIZE(X), %xmm0

  777. 	movhps	-15 * SIZE(X), %xmm0

  778. 	mulpd	ALPHA, %xmm0

  779. 	addpd	-16 * SIZE(Y), %xmm0

  780. 	movaps	%xmm0, -16 * SIZE(Y)

  781. 

  782. 	addq	$2 * SIZE, X

  783. 	addq	$2 * SIZE, Y

  784. 	ALIGN_3

  785. 

  786. .L26:

  787. 	movq	M,  %rax

  788. 	andq	$1, %rax

  789. 	jle	.L29

  790. 	ALIGN_3

  791. 

  792. 	movsd	-16 * SIZE(X), %xmm0

  793. 	mulsd	ALPHA, %xmm0

  794. 	addsd	-16 * SIZE(Y), %xmm0

  795. 

  796. 	movsd	%xmm0, 	-16 * SIZE(Y)

  797. 	ALIGN_3

  798. 

  799. .L29:

  800. 	xorq	%rax,%rax

  801. 

  802. 	RESTOREREGISTERS

  803. 

  804. 	ret

  805. 	ALIGN_3

  806. #endif

  807. 

  808. .L40:

  809. 	movq	Y, YY

  810. 	movq	M,  %rax

  811. //If incx==0 || incy==0, avoid unloop.

  812. 	cmpq	$0, INCX

  813. 	je  .L46

  814. 	cmpq	$0, INCY

  815. 	je  .L46

  816. 

  817. 	sarq	$3, %rax

  818. 	jle	.L45

  819. 	ALIGN_3

  820. 

  821. .L41:

  822. 	movsd	0 * SIZE(X), %xmm0

  823. 	addq	INCX, X

  824. 	movhpd	0 * SIZE(X), %xmm0

  825. 	addq	INCX, X

  826. 	mulpd	ALPHA, %xmm0

  827. 

  828. 	movsd	0 * SIZE(YY), %xmm6

  829. 	addq	INCY, YY

  830. 	movhpd	0 * SIZE(YY), %xmm6

  831. 	addq	INCY, YY

  832. 	addpd	%xmm6, %xmm0

  833. 

  834. 	movsd	0 * SIZE(X), %xmm1

  835. 	addq	INCX, X

  836. 	movhpd	0 * SIZE(X), %xmm1

  837. 	addq	INCX, X

  838. 	mulpd	ALPHA, %xmm1

  839. 

  840. 	movsd	0 * SIZE(YY), %xmm6

  841. 	addq	INCY, YY

  842. 	movhpd	0 * SIZE(YY), %xmm6

  843. 	addq	INCY, YY

  844. 	addpd	%xmm6, %xmm1

  845. 

  846. 	movsd	0 * SIZE(X), %xmm2

  847. 	addq	INCX, X

  848. 	movhpd	0 * SIZE(X), %xmm2

  849. 	addq	INCX, X

  850. 	mulpd	ALPHA, %xmm2

  851. 

  852. 	movsd	0 * SIZE(YY), %xmm6

  853. 	addq	INCY, YY

  854. 	movhpd	0 * SIZE(YY), %xmm6

  855. 	addq	INCY, YY

  856. 	addpd	%xmm6, %xmm2

  857. 

  858. 	movsd	0 * SIZE(X), %xmm3

  859. 	addq	INCX, X

  860. 	movhpd	0 * SIZE(X), %xmm3

  861. 	addq	INCX, X

  862. 	mulpd	ALPHA, %xmm3

  863. 

  864. 	movsd	0 * SIZE(YY), %xmm6

  865. 	addq	INCY, YY

  866. 	movhpd	0 * SIZE(YY), %xmm6

  867. 	addq	INCY, YY

  868. 	addpd	%xmm6, %xmm3

  869. 

  870. 	movsd	%xmm0, 0 * SIZE(Y)

  871. 	addq	INCY, Y

  872. 	movhpd	%xmm0, 0 * SIZE(Y)

  873. 	addq	INCY, Y

  874. 	movsd	%xmm1, 0 * SIZE(Y)

  875. 	addq	INCY, Y

  876. 	movhpd	%xmm1, 0 * SIZE(Y)

  877. 	addq	INCY, Y

  878. 	movsd	%xmm2, 0 * SIZE(Y)

  879. 	addq	INCY, Y

  880. 	movhpd	%xmm2, 0 * SIZE(Y)

  881. 	addq	INCY, Y

  882. 	movsd	%xmm3, 0 * SIZE(Y)

  883. 	addq	INCY, Y

  884. 	movhpd	%xmm3, 0 * SIZE(Y)

  885. 	addq	INCY, Y

  886. 

  887. 	decq	%rax

  888. 	jg	.L41

  889. 	ALIGN_3

  890. 

  891. .L45:

  892. 	movq	M,  %rax

  893. 	andq	$7, %rax

  894. 	jle	.L47

  895. 	ALIGN_3

  896. 

  897. .L46:

  898. 	movsd	(X), %xmm0

  899. 	addq	INCX, X

  900. 	mulsd	%xmm15, %xmm0

  901. 	addsd	(Y), %xmm0

  902. 	movsd	%xmm0, (Y)

  903. 	addq	INCY, Y

  904. 	decq	%rax

  905. 	jg	.L46

  906. 	ALIGN_3

  907. 

  908. .L47:

  909. 	xorq	%rax, %rax

  910. 

  911. 	RESTOREREGISTERS

  912. 

  913. 	ret

  914. 

  915. 	EPILOGUE