OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 549a9f1dbb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '549a9f1dbb'
${ noResults }
OpenBLAS/kernel/x86_64/zaxpy_sse2.S

1804 lines
36 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #define Y	ARG6

  47. #define INCY	ARG2

  48. #else

  49. #define M	ARG1

  50. #define X	ARG2

  51. #define INCX	ARG3

  52. #define Y	ARG4

  53. #define INCY	%r10

  54. #endif

  55. 

  56. #define	YY	%r11

  57. #define ALPHA_R	%xmm14

  58. #define ALPHA_I	%xmm15

  59. 

  60. #define  USE_PSHUFD

  61. 

  62. #if defined(HAVE_SSE3) && !defined(CORE_OPTERON)

  63. #define MOVDDUP(a, b, c)	movddup	a(b), c

  64. #define MOVDDUP2(a, b, c)	movddup	a##b, c

  65. #else

  66. #define MOVDDUP(a, b, c)	movlpd	a(b), c;movhpd	a(b), c

  67. #define MOVDDUP2(a, b, c)	movlpd	a##b, c;movhpd	a##b, c

  68. #endif

  69. 

  70. #include "l1param.h"

  71. 

  72. 	PROLOGUE

  73. 	PROFCODE

  74. 

  75. #ifndef WINDOWS_ABI

  76. 	movq	 8(%rsp), INCY

  77. #else

  78. 	movaps	%xmm3, %xmm0

  79. 	movsd	40(%rsp), %xmm1

  80. 

  81. 	movq	48(%rsp), X

  82. 	movq	56(%rsp), INCX

  83. 	movq	64(%rsp), Y

  84. 	movq	72(%rsp), INCY

  85. #endif

  86. 

  87. 	SAVEREGISTERS

  88. 

  89. 	salq	$ZBASE_SHIFT, INCX

  90. 	salq	$ZBASE_SHIFT, INCY

  91. 

  92. 	testq	M, M

  93. 	jle	.L999

  94. 

  95. 	cmpq	$2 * SIZE, INCX

  96. 	jne	.L50

  97. 	cmpq	$2 * SIZE, INCY

  98. 	jne	.L50

  99. 

  100. 	subq	$-16 * SIZE, X

  101. 	subq	$-16 * SIZE, Y

  102. 

  103. 	pcmpeqb	%xmm7, %xmm7

  104. 	psllq	$63,   %xmm7

  105. 

  106. #ifdef HAVE_SSE3

  107. 	movddup	%xmm0, ALPHA_R

  108. 	movddup	%xmm1, ALPHA_I

  109. #else

  110. 	pshufd	$0x44, %xmm0, ALPHA_R

  111. 	pshufd	$0x44, %xmm1, ALPHA_I

  112. #endif

  113. 

  114. #ifndef CONJ

  115. 	shufps	$0x0c, %xmm7, %xmm7

  116. 	xorpd	%xmm7, ALPHA_I

  117. #else

  118. 	shufps	$0xc0, %xmm7, %xmm7

  119. 	xorpd	%xmm7, ALPHA_R

  120. #endif

  121. 

  122. 	testq	$SIZE, Y

  123. 	jne	.L30

  124. 

  125. 	testq	$SIZE, X

  126. 	jne	.L20

  127. 

  128. 	movq	M,  %rax

  129. 	sarq	$3, %rax

  130. 	jle	.L15

  131. 

  132. 	movaps	-16 * SIZE(X), %xmm0

  133. 	movaps	-14 * SIZE(X), %xmm1

  134. 	movaps	-12 * SIZE(X), %xmm2

  135. 	movaps	-10 * SIZE(X), %xmm3

  136. 

  137. 	decq	%rax

  138. 	jle	.L12

  139. 	ALIGN_3

  140. 

  141. .L11:

  142. 	movaps	 -8 * SIZE(X), %xmm4

  143. 	movaps	 -6 * SIZE(X), %xmm5

  144. 

  145. #ifdef PREFETCHW

  146. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  147. #endif

  148. 

  149. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  150. 	pshufd	$0x4e,   %xmm0, %xmm8

  151. #else

  152. 	movsd	-15 * SIZE(X), %xmm8

  153. 	movhps	-16 * SIZE(X), %xmm8

  154. #endif

  155. 	mulpd	ALPHA_R, %xmm0

  156. 	mulpd	ALPHA_I, %xmm8

  157. 	addpd	-16 * SIZE(Y), %xmm0

  158. 	addpd	%xmm8,   %xmm0

  159. 	movaps	%xmm0, -16 * SIZE(Y)

  160. 

  161. #ifdef USE_PSHUFD

  162. 	pshufd	$0x4e,   %xmm1, %xmm8

  163. #else

  164. 	movsd	-13 * SIZE(X), %xmm8

  165. 	movhps	-14 * SIZE(X), %xmm8

  166. #endif

  167. 	mulpd	ALPHA_R, %xmm1

  168. 	mulpd	ALPHA_I, %xmm8

  169. 	addpd	-14 * SIZE(Y), %xmm1

  170. 	addpd	%xmm8,   %xmm1

  171. 	movaps	%xmm1, -14 * SIZE(Y)

  172. 

  173. 	movaps	 -4 * SIZE(X), %xmm6

  174. 	movaps	 -2 * SIZE(X), %xmm7

  175. 

  176. #ifdef PREFETCH

  177. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  178. #endif

  179. 

  180. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  181. 	pshufd	$0x4e,   %xmm2, %xmm8

  182. #else

  183. 	movsd	-11 * SIZE(X), %xmm8

  184. 	movhps	-12 * SIZE(X), %xmm8

  185. #endif

  186. 	mulpd	ALPHA_R, %xmm2

  187. 	mulpd	ALPHA_I, %xmm8

  188. 	addpd	-12 * SIZE(Y), %xmm2

  189. 	addpd	%xmm8,   %xmm2

  190. 	movaps	%xmm2, -12 * SIZE(Y)

  191. 

  192. #ifdef USE_PSHUFD

  193. 	pshufd	$0x4e,   %xmm3, %xmm8

  194. #else

  195. 	movsd	 -9 * SIZE(X), %xmm8

  196. 	movhps	-10 * SIZE(X), %xmm8

  197. #endif

  198. 	mulpd	ALPHA_R, %xmm3

  199. 	mulpd	ALPHA_I, %xmm8

  200. 	addpd	-10 * SIZE(Y), %xmm3

  201. 	addpd	%xmm8,   %xmm3

  202. 	movaps	%xmm3, -10 * SIZE(Y)

  203. 

  204. 	movaps	  0 * SIZE(X), %xmm0

  205. 	movaps	  2 * SIZE(X), %xmm1

  206. 

  207. #if defined(PREFETCHW) && !defined(FETCH128)

  208. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  209. #endif

  210. 

  211. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  212. 	pshufd	$0x4e,   %xmm4, %xmm8

  213. #else

  214. 	movsd	 -7 * SIZE(X), %xmm8

  215. 	movhps	 -8 * SIZE(X), %xmm8

  216. #endif

  217. 	mulpd	ALPHA_R, %xmm4

  218. 	mulpd	ALPHA_I, %xmm8

  219. 	addpd	 -8 * SIZE(Y), %xmm4

  220. 	addpd	%xmm8,   %xmm4

  221. 	movaps	%xmm4,  -8 * SIZE(Y)

  222. 

  223. #ifdef USE_PSHUFD

  224. 	pshufd	$0x4e,   %xmm5, %xmm8

  225. #else

  226. 	movsd	 -5 * SIZE(X), %xmm8

  227. 	movhps	 -6 * SIZE(X), %xmm8

  228. #endif

  229. 	mulpd	ALPHA_R, %xmm5

  230. 	mulpd	ALPHA_I, %xmm8

  231. 	addpd	 -6 * SIZE(Y), %xmm5

  232. 	addpd	%xmm8,   %xmm5

  233. 	movaps	%xmm5,  -6 * SIZE(Y)

  234. 

  235. 	movaps	  4 * SIZE(X), %xmm2

  236. 	movaps	  6 * SIZE(X), %xmm3

  237. 

  238. #if defined(PREFETCH) && !defined(FETCH128)

  239. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  240. #endif

  241. 

  242. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  243. 	pshufd	$0x4e,   %xmm6, %xmm8

  244. #else

  245. 	movsd	 -3 * SIZE(X), %xmm8

  246. 	movhps	 -4 * SIZE(X), %xmm8

  247. #endif

  248. 	mulpd	ALPHA_R, %xmm6

  249. 	mulpd	ALPHA_I, %xmm8

  250. 	addpd	 -4 * SIZE(Y), %xmm6

  251. 	addpd	%xmm8,   %xmm6

  252. 	movaps	%xmm6,  -4 * SIZE(Y)

  253. 

  254. #ifdef USE_PSHUFD

  255. 	pshufd	$0x4e,   %xmm7, %xmm8

  256. #else

  257. 	movsd	 -1 * SIZE(X), %xmm8

  258. 	movhps	 -2 * SIZE(X), %xmm8

  259. #endif

  260. 	mulpd	ALPHA_R, %xmm7

  261. 	mulpd	ALPHA_I, %xmm8

  262. 	addpd	 -2 * SIZE(Y), %xmm7

  263. 	addpd	%xmm8,   %xmm7

  264. 	movaps	%xmm7,  -2 * SIZE(Y)

  265. 

  266. 	subq	$-16 * SIZE, X

  267. 	subq	$-16 * SIZE, Y

  268. 	decq	%rax

  269. 	jg	.L11

  270. 	ALIGN_3

  271. 

  272. .L12:

  273. 	movaps	 -8 * SIZE(X), %xmm4

  274. 	movaps	 -6 * SIZE(X), %xmm5

  275. 

  276. 	pshufd	$0x4e,   %xmm0, %xmm8

  277. 	mulpd	ALPHA_R, %xmm0

  278. 	mulpd	ALPHA_I, %xmm8

  279. 	addpd	-16 * SIZE(Y), %xmm0

  280. 	addpd	%xmm8,   %xmm0

  281. 	movaps	%xmm0, -16 * SIZE(Y)

  282. 

  283. 	pshufd	$0x4e,   %xmm1, %xmm8

  284. 	mulpd	ALPHA_R, %xmm1

  285. 	mulpd	ALPHA_I, %xmm8

  286. 	addpd	-14 * SIZE(Y), %xmm1

  287. 	addpd	%xmm8,   %xmm1

  288. 	movaps	%xmm1, -14 * SIZE(Y)

  289. 

  290. 	movaps	 -4 * SIZE(X), %xmm6

  291. 	movaps	 -2 * SIZE(X), %xmm7

  292. 

  293. 	pshufd	$0x4e,   %xmm2, %xmm8

  294. 	mulpd	ALPHA_R, %xmm2

  295. 	mulpd	ALPHA_I, %xmm8

  296. 	addpd	-12 * SIZE(Y), %xmm2

  297. 	addpd	%xmm8,   %xmm2

  298. 	movaps	%xmm2, -12 * SIZE(Y)

  299. 

  300. 	pshufd	$0x4e,   %xmm3, %xmm8

  301. 	mulpd	ALPHA_R, %xmm3

  302. 	mulpd	ALPHA_I, %xmm8

  303. 	addpd	-10 * SIZE(Y), %xmm3

  304. 	addpd	%xmm8,   %xmm3

  305. 	movaps	%xmm3, -10 * SIZE(Y)

  306. 

  307. 	pshufd	$0x4e,   %xmm4, %xmm8

  308. 	mulpd	ALPHA_R, %xmm4

  309. 	mulpd	ALPHA_I, %xmm8

  310. 	addpd	 -8 * SIZE(Y), %xmm4

  311. 	addpd	%xmm8,   %xmm4

  312. 	movaps	%xmm4,  -8 * SIZE(Y)

  313. 

  314. 	pshufd	$0x4e,   %xmm5, %xmm8

  315. 	mulpd	ALPHA_R, %xmm5

  316. 	mulpd	ALPHA_I, %xmm8

  317. 	addpd	 -6 * SIZE(Y), %xmm5

  318. 	addpd	%xmm8,   %xmm5

  319. 	movaps	%xmm5,  -6 * SIZE(Y)

  320. 

  321. 	pshufd	$0x4e,   %xmm6, %xmm8

  322. 	mulpd	ALPHA_R, %xmm6

  323. 	mulpd	ALPHA_I, %xmm8

  324. 	addpd	 -4 * SIZE(Y), %xmm6

  325. 	addpd	%xmm8,   %xmm6

  326. 	movaps	%xmm6,  -4 * SIZE(Y)

  327. 

  328. 	pshufd	$0x4e,   %xmm7, %xmm8

  329. 	mulpd	ALPHA_R, %xmm7

  330. 	mulpd	ALPHA_I, %xmm8

  331. 	addpd	 -2 * SIZE(Y), %xmm7

  332. 	addpd	%xmm8,   %xmm7

  333. 	movaps	%xmm7,  -2 * SIZE(Y)

  334. 

  335. 	subq	$-16 * SIZE, X

  336. 	subq	$-16 * SIZE, Y

  337. 	ALIGN_3

  338. 

  339. .L15:

  340. 	movq	M,  %rax

  341. 	andq	$4, %rax

  342. 	jle	.L16

  343. 

  344. 	movaps	-16 * SIZE(X), %xmm0

  345. 	movaps	-14 * SIZE(X), %xmm1

  346. 	movaps	-12 * SIZE(X), %xmm2

  347. 	movaps	-10 * SIZE(X), %xmm3

  348. 

  349. 	pshufd	$0x4e,   %xmm0, %xmm8

  350. 	mulpd	ALPHA_R, %xmm0

  351. 	mulpd	ALPHA_I, %xmm8

  352. 	addpd	-16 * SIZE(Y), %xmm0

  353. 	addpd	%xmm8,   %xmm0

  354. 	movaps	%xmm0, -16 * SIZE(Y)

  355. 

  356. 	pshufd	$0x4e,   %xmm1, %xmm8

  357. 	mulpd	ALPHA_R, %xmm1

  358. 	mulpd	ALPHA_I, %xmm8

  359. 	addpd	-14 * SIZE(Y), %xmm1

  360. 	addpd	%xmm8,   %xmm1

  361. 	movaps	%xmm1, -14 * SIZE(Y)

  362. 

  363. 	pshufd	$0x4e,   %xmm2, %xmm8

  364. 	mulpd	ALPHA_R, %xmm2

  365. 	mulpd	ALPHA_I, %xmm8

  366. 	addpd	-12 * SIZE(Y), %xmm2

  367. 	addpd	%xmm8,   %xmm2

  368. 	movaps	%xmm2, -12 * SIZE(Y)

  369. 

  370. 	pshufd	$0x4e,   %xmm3, %xmm8

  371. 	mulpd	ALPHA_R, %xmm3

  372. 	mulpd	ALPHA_I, %xmm8

  373. 	addpd	-10 * SIZE(Y), %xmm3

  374. 	addpd	%xmm8,   %xmm3

  375. 	movaps	%xmm3, -10 * SIZE(Y)

  376. 

  377. 	addq	$8 * SIZE, X

  378. 	addq	$8 * SIZE, Y

  379. 	ALIGN_3

  380. 

  381. .L16:

  382. 	movq	M,  %rax

  383. 	andq	$2, %rax

  384. 	jle	.L17

  385. 

  386. 	movaps	-16 * SIZE(X), %xmm0

  387. 	movaps	-14 * SIZE(X), %xmm1

  388. 

  389. 	pshufd	$0x4e,   %xmm0, %xmm8

  390. 	mulpd	ALPHA_R, %xmm0

  391. 	mulpd	ALPHA_I, %xmm8

  392. 	addpd	-16 * SIZE(Y), %xmm0

  393. 	addpd	%xmm8,   %xmm0

  394. 	movaps	%xmm0, -16 * SIZE(Y)

  395. 

  396. 	pshufd	$0x4e,   %xmm1, %xmm8

  397. 	mulpd	ALPHA_R, %xmm1

  398. 	mulpd	ALPHA_I, %xmm8

  399. 	addpd	-14 * SIZE(Y), %xmm1

  400. 	addpd	%xmm8,   %xmm1

  401. 	movaps	%xmm1, -14 * SIZE(Y)

  402. 

  403. 	addq	$4 * SIZE, X

  404. 	addq	$4 * SIZE, Y

  405. 	ALIGN_3

  406. 

  407. .L17:

  408. 	movq	M,  %rax

  409. 	andq	$1, %rax

  410. 	jle	.L999

  411. 

  412. 	movaps	-16 * SIZE(X), %xmm0

  413. 

  414. 	pshufd	$0x4e,   %xmm0, %xmm8

  415. 	mulpd	ALPHA_R, %xmm0

  416. 	mulpd	ALPHA_I, %xmm8

  417. 	addpd	-16 * SIZE(Y), %xmm0

  418. 	addpd	%xmm8,   %xmm0

  419. 	movaps	%xmm0, -16 * SIZE(Y)

  420. 	jmp	.L999

  421. 	ALIGN_3

  422. 

  423. .L20:

  424. 	movq	M,  %rax

  425. 	sarq	$3, %rax

  426. 	jle	.L25

  427. 

  428. 	movsd	-16 * SIZE(X), %xmm0

  429. 	movhps	-15 * SIZE(X), %xmm0

  430. 	movsd	-14 * SIZE(X), %xmm1

  431. 	movhps	-13 * SIZE(X), %xmm1

  432. 	movsd	-12 * SIZE(X), %xmm2

  433. 	movhps	-11 * SIZE(X), %xmm2

  434. 	movsd	-10 * SIZE(X), %xmm3

  435. 	movhps	 -9 * SIZE(X), %xmm3

  436. 

  437. 	decq	%rax

  438. 	jle	.L22

  439. 	ALIGN_3

  440. 

  441. .L21:

  442. 	movsd	 -8 * SIZE(X), %xmm4

  443. 	movhps	 -7 * SIZE(X), %xmm4

  444. 

  445. #ifdef PREFETCHW

  446. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  447. #endif

  448. 

  449. 	pshufd	$0x4e,   %xmm0, %xmm8

  450. 	mulpd	ALPHA_R, %xmm0

  451. 	mulpd	ALPHA_I, %xmm8

  452. 	addpd	-16 * SIZE(Y), %xmm0

  453. 	addpd	%xmm8,   %xmm0

  454. 	movaps	%xmm0, -16 * SIZE(Y)

  455. 

  456. 	movsd	 -6 * SIZE(X), %xmm5

  457. 	movhps	 -5 * SIZE(X), %xmm5

  458. 

  459. 	pshufd	$0x4e,   %xmm1, %xmm8

  460. 	mulpd	ALPHA_R, %xmm1

  461. 	mulpd	ALPHA_I, %xmm8

  462. 	addpd	-14 * SIZE(Y), %xmm1

  463. 	addpd	%xmm8,   %xmm1

  464. 	movaps	%xmm1, -14 * SIZE(Y)

  465. 

  466. 	movsd	 -4 * SIZE(X), %xmm6

  467. 	movhps	 -3 * SIZE(X), %xmm6

  468. 

  469. #ifdef PREFETCH

  470. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  471. #endif

  472. 

  473. 	pshufd	$0x4e,   %xmm2, %xmm8

  474. 	mulpd	ALPHA_R, %xmm2

  475. 	mulpd	ALPHA_I, %xmm8

  476. 	addpd	-12 * SIZE(Y), %xmm2

  477. 	addpd	%xmm8,   %xmm2

  478. 	movaps	%xmm2, -12 * SIZE(Y)

  479. 

  480. 	movsd	 -2 * SIZE(X), %xmm7

  481. 	movhps	 -1 * SIZE(X), %xmm7

  482. 

  483. 	pshufd	$0x4e,   %xmm3, %xmm8

  484. 	mulpd	ALPHA_R, %xmm3

  485. 	mulpd	ALPHA_I, %xmm8

  486. 	addpd	-10 * SIZE(Y), %xmm3

  487. 	addpd	%xmm8,   %xmm3

  488. 	movaps	%xmm3, -10 * SIZE(Y)

  489. 

  490. 	movsd	  0 * SIZE(X), %xmm0

  491. 	movhps	  1 * SIZE(X), %xmm0

  492. 

  493. #if defined(PREFETCHW) && !defined(FETCH128)

  494. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  495. #endif

  496. 

  497. 	pshufd	$0x4e,   %xmm4, %xmm8

  498. 	mulpd	ALPHA_R, %xmm4

  499. 	mulpd	ALPHA_I, %xmm8

  500. 	addpd	 -8 * SIZE(Y), %xmm4

  501. 	addpd	%xmm8,   %xmm4

  502. 	movaps	%xmm4,  -8 * SIZE(Y)

  503. 

  504. 	movsd	  2 * SIZE(X), %xmm1

  505. 	movhps	  3 * SIZE(X), %xmm1

  506. 

  507. 	pshufd	$0x4e,   %xmm5, %xmm8

  508. 	mulpd	ALPHA_R, %xmm5

  509. 	mulpd	ALPHA_I, %xmm8

  510. 	addpd	 -6 * SIZE(Y), %xmm5

  511. 	addpd	%xmm8,   %xmm5

  512. 	movaps	%xmm5,  -6 * SIZE(Y)

  513. 

  514. 	movsd	  4 * SIZE(X), %xmm2

  515. 	movhps	  5 * SIZE(X), %xmm2

  516. 

  517. #if defined(PREFETCH) && !defined(FETCH128)

  518. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  519. #endif

  520. 

  521. 	pshufd	$0x4e,   %xmm6, %xmm8

  522. 	mulpd	ALPHA_R, %xmm6

  523. 	mulpd	ALPHA_I, %xmm8

  524. 	addpd	 -4 * SIZE(Y), %xmm6

  525. 	addpd	%xmm8,   %xmm6

  526. 	movaps	%xmm6,  -4 * SIZE(Y)

  527. 

  528. 	movsd	  6 * SIZE(X), %xmm3

  529. 	movhps	  7 * SIZE(X), %xmm3

  530. 

  531. 	pshufd	$0x4e,   %xmm7, %xmm8

  532. 	mulpd	ALPHA_R, %xmm7

  533. 	mulpd	ALPHA_I, %xmm8

  534. 	addpd	 -2 * SIZE(Y), %xmm7

  535. 	addpd	%xmm8,   %xmm7

  536. 	movaps	%xmm7,  -2 * SIZE(Y)

  537. 

  538. 	subq	$-16 * SIZE, X

  539. 	subq	$-16 * SIZE, Y

  540. 	decq	%rax

  541. 	jg	.L21

  542. 	ALIGN_3

  543. 

  544. .L22:

  545. 	movsd	 -8 * SIZE(X), %xmm4

  546. 	movhps	 -7 * SIZE(X), %xmm4

  547. 

  548. 	pshufd	$0x4e,   %xmm0, %xmm8

  549. 	mulpd	ALPHA_R, %xmm0

  550. 	mulpd	ALPHA_I, %xmm8

  551. 	addpd	-16 * SIZE(Y), %xmm0

  552. 	addpd	%xmm8,   %xmm0

  553. 	movaps	%xmm0, -16 * SIZE(Y)

  554. 

  555. 	movsd	 -6 * SIZE(X), %xmm5

  556. 	movhps	 -5 * SIZE(X), %xmm5

  557. 

  558. 	pshufd	$0x4e,   %xmm1, %xmm8

  559. 	mulpd	ALPHA_R, %xmm1

  560. 	mulpd	ALPHA_I, %xmm8

  561. 	addpd	-14 * SIZE(Y), %xmm1

  562. 	addpd	%xmm8,   %xmm1

  563. 	movaps	%xmm1, -14 * SIZE(Y)

  564. 

  565. 	movsd	 -4 * SIZE(X), %xmm6

  566. 	movhps	 -3 * SIZE(X), %xmm6

  567. 

  568. 	pshufd	$0x4e,   %xmm2, %xmm8

  569. 	mulpd	ALPHA_R, %xmm2

  570. 	mulpd	ALPHA_I, %xmm8

  571. 	addpd	-12 * SIZE(Y), %xmm2

  572. 	addpd	%xmm8,   %xmm2

  573. 	movaps	%xmm2, -12 * SIZE(Y)

  574. 

  575. 	movsd	 -2 * SIZE(X), %xmm7

  576. 	movhps	 -1 * SIZE(X), %xmm7

  577. 

  578. 	pshufd	$0x4e,   %xmm3, %xmm8

  579. 	mulpd	ALPHA_R, %xmm3

  580. 	mulpd	ALPHA_I, %xmm8

  581. 	addpd	-10 * SIZE(Y), %xmm3

  582. 	addpd	%xmm8,   %xmm3

  583. 	movaps	%xmm3, -10 * SIZE(Y)

  584. 

  585. 	pshufd	$0x4e,   %xmm4, %xmm8

  586. 	mulpd	ALPHA_R, %xmm4

  587. 	mulpd	ALPHA_I, %xmm8

  588. 	addpd	 -8 * SIZE(Y), %xmm4

  589. 	addpd	%xmm8,   %xmm4

  590. 	movaps	%xmm4,  -8 * SIZE(Y)

  591. 

  592. 	pshufd	$0x4e,   %xmm5, %xmm8

  593. 	mulpd	ALPHA_R, %xmm5

  594. 	mulpd	ALPHA_I, %xmm8

  595. 	addpd	 -6 * SIZE(Y), %xmm5

  596. 	addpd	%xmm8,   %xmm5

  597. 	movaps	%xmm5,  -6 * SIZE(Y)

  598. 

  599. 	pshufd	$0x4e,   %xmm6, %xmm8

  600. 	mulpd	ALPHA_R, %xmm6

  601. 	mulpd	ALPHA_I, %xmm8

  602. 	addpd	 -4 * SIZE(Y), %xmm6

  603. 	addpd	%xmm8,   %xmm6

  604. 	movaps	%xmm6,  -4 * SIZE(Y)

  605. 

  606. 	pshufd	$0x4e,   %xmm7, %xmm8

  607. 	mulpd	ALPHA_R, %xmm7

  608. 	mulpd	ALPHA_I, %xmm8

  609. 	addpd	 -2 * SIZE(Y), %xmm7

  610. 	addpd	%xmm8,   %xmm7

  611. 	movaps	%xmm7,  -2 * SIZE(Y)

  612. 

  613. 	subq	$-16 * SIZE, X

  614. 	subq	$-16 * SIZE, Y

  615. 	ALIGN_3

  616. 

  617. .L25:

  618. 	movq	M,  %rax

  619. 	andq	$4, %rax

  620. 	jle	.L26

  621. 

  622. 	movsd	-16 * SIZE(X), %xmm0

  623. 	movhps	-15 * SIZE(X), %xmm0

  624. 	movsd	-14 * SIZE(X), %xmm1

  625. 	movhps	-13 * SIZE(X), %xmm1

  626. 

  627. 	pshufd	$0x4e,   %xmm0, %xmm8

  628. 	mulpd	ALPHA_R, %xmm0

  629. 	mulpd	ALPHA_I, %xmm8

  630. 	addpd	-16 * SIZE(Y), %xmm0

  631. 	addpd	%xmm8,   %xmm0

  632. 	movaps	%xmm0, -16 * SIZE(Y)

  633. 

  634. 	pshufd	$0x4e,   %xmm1, %xmm8

  635. 	mulpd	ALPHA_R, %xmm1

  636. 	mulpd	ALPHA_I, %xmm8

  637. 	addpd	-14 * SIZE(Y), %xmm1

  638. 	addpd	%xmm8,   %xmm1

  639. 	movaps	%xmm1, -14 * SIZE(Y)

  640. 

  641. 	movsd	-12 * SIZE(X), %xmm2

  642. 	movhps	-11 * SIZE(X), %xmm2

  643. 	movsd	-10 * SIZE(X), %xmm3

  644. 	movhps	 -9 * SIZE(X), %xmm3

  645. 

  646. 	pshufd	$0x4e,   %xmm2, %xmm8

  647. 	mulpd	ALPHA_R, %xmm2

  648. 	mulpd	ALPHA_I, %xmm8

  649. 	addpd	-12 * SIZE(Y), %xmm2

  650. 	addpd	%xmm8,   %xmm2

  651. 	movaps	%xmm2, -12 * SIZE(Y)

  652. 

  653. 	pshufd	$0x4e,   %xmm3, %xmm8

  654. 	mulpd	ALPHA_R, %xmm3

  655. 	mulpd	ALPHA_I, %xmm8

  656. 	addpd	-10 * SIZE(Y), %xmm3

  657. 	addpd	%xmm8,   %xmm3

  658. 	movaps	%xmm3, -10 * SIZE(Y)

  659. 

  660. 	addq	$8 * SIZE, X

  661. 	addq	$8 * SIZE, Y

  662. 	ALIGN_3

  663. 

  664. .L26:

  665. 	movq	M,  %rax

  666. 	andq	$2, %rax

  667. 	jle	.L27

  668. 

  669. 	movsd	-16 * SIZE(X), %xmm0

  670. 	movhps	-15 * SIZE(X), %xmm0

  671. 

  672. 	pshufd	$0x4e,   %xmm0, %xmm8

  673. 	mulpd	ALPHA_R, %xmm0

  674. 	mulpd	ALPHA_I, %xmm8

  675. 	addpd	-16 * SIZE(Y), %xmm0

  676. 	addpd	%xmm8,   %xmm0

  677. 	movaps	%xmm0, -16 * SIZE(Y)

  678. 

  679. 	movsd	-14 * SIZE(X), %xmm1

  680. 	movhps	-13 * SIZE(X), %xmm1

  681. 

  682. 	pshufd	$0x4e,   %xmm1, %xmm8

  683. 	mulpd	ALPHA_R, %xmm1

  684. 	mulpd	ALPHA_I, %xmm8

  685. 	addpd	-14 * SIZE(Y), %xmm1

  686. 	addpd	%xmm8,   %xmm1

  687. 	movaps	%xmm1, -14 * SIZE(Y)

  688. 

  689. 	addq	$4 * SIZE, X

  690. 	addq	$4 * SIZE, Y

  691. 	ALIGN_3

  692. 

  693. .L27:

  694. 	movq	M,  %rax

  695. 	andq	$1, %rax

  696. 	jle	.L999

  697. 

  698. 	movsd	-16 * SIZE(X), %xmm0

  699. 	movhps	-15 * SIZE(X), %xmm0

  700. 

  701. 	pshufd	$0x4e,   %xmm0, %xmm8

  702. 	mulpd	ALPHA_R, %xmm0

  703. 	mulpd	ALPHA_I, %xmm8

  704. 	addpd	-16 * SIZE(Y), %xmm0

  705. 	addpd	%xmm8,   %xmm0

  706. 	movaps	%xmm0, -16 * SIZE(Y)

  707. 	jmp	.L999

  708. 	ALIGN_3

  709. 

  710. .L30:

  711. 	testq	$SIZE, X

  712. 	jne	.L40

  713. 

  714. 	movaps	-16 * SIZE(X), %xmm1

  715. 

  716. 	pshufd	$0x4e,   %xmm1, %xmm8

  717. 	mulpd	ALPHA_R, %xmm1

  718. 	mulpd	ALPHA_I, %xmm8

  719. 	addpd	%xmm8,   %xmm1

  720. 	xorps	%xmm0,   %xmm0

  721. 	SHUFPD_1 %xmm1,  %xmm0

  722. 

  723. 	xorps	%xmm4,   %xmm4

  724. 	movhps	-16 * SIZE(Y), %xmm4

  725. 

  726. 	addpd	%xmm0,   %xmm4

  727. 	movhps	%xmm4, -16 * SIZE(Y)

  728. 	movaps	%xmm1,   %xmm0

  729. 

  730. 	addq	$2 * SIZE, X

  731. 	addq	$1 * SIZE, Y

  732. 	decq	M

  733. 	jle	.L39

  734. 

  735. 	movq	M,  %rax

  736. 	sarq	$3, %rax

  737. 	jle	.L35

  738. 

  739. 	movaps	-16 * SIZE(X), %xmm1

  740. 	movaps	-14 * SIZE(X), %xmm2

  741. 	movaps	-12 * SIZE(X), %xmm3

  742. 

  743. 	decq	%rax

  744. 	jle	.L32

  745. 	ALIGN_3

  746. 

  747. .L31:

  748. #ifdef PREFETCHW

  749. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  750. #endif

  751. 

  752. 	pshufd	$0x4e,   %xmm1, %xmm8

  753. 	mulpd	ALPHA_R, %xmm1

  754. 	mulpd	ALPHA_I, %xmm8

  755. 	addpd	%xmm8,   %xmm1

  756. 	SHUFPD_1 %xmm1,  %xmm0

  757. 

  758. 	addpd	-16 * SIZE(Y), %xmm0

  759. 	movaps	%xmm0, -16 * SIZE(Y)

  760. 	movaps	-10 * SIZE(X), %xmm0

  761. 

  762. 	pshufd	$0x4e,   %xmm2, %xmm8

  763. 	mulpd	ALPHA_R, %xmm2

  764. 	mulpd	ALPHA_I, %xmm8

  765. 	addpd	%xmm8,   %xmm2

  766. 	SHUFPD_1 %xmm2,  %xmm1

  767. 

  768. 	addpd	-14 * SIZE(Y), %xmm1

  769. 	movaps	%xmm1, -14 * SIZE(Y)

  770. 	movaps	 -8 * SIZE(X), %xmm1

  771. 

  772. #ifdef PREFETCH

  773. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  774. #endif

  775. 

  776. 	pshufd	$0x4e,   %xmm3, %xmm8

  777. 	mulpd	ALPHA_R, %xmm3

  778. 	mulpd	ALPHA_I, %xmm8

  779. 	addpd	%xmm8,   %xmm3

  780. 	SHUFPD_1 %xmm3,  %xmm2

  781. 

  782. 	addpd	-12 * SIZE(Y), %xmm2

  783. 	movaps	%xmm2, -12 * SIZE(Y)

  784. 	movaps	 -6 * SIZE(X), %xmm2

  785. 

  786. 	pshufd	$0x4e,   %xmm0, %xmm8

  787. 	mulpd	ALPHA_R, %xmm0

  788. 	mulpd	ALPHA_I, %xmm8

  789. 	addpd	%xmm8,   %xmm0

  790. 	SHUFPD_1 %xmm0,  %xmm3

  791. 

  792. 	addpd	-10 * SIZE(Y), %xmm3

  793. 	movaps	%xmm3, -10 * SIZE(Y)

  794. 	movaps	 -4 * SIZE(X), %xmm3

  795. 

  796. #if defined(PREFETCHW) && !defined(FETCH128)

  797. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  798. #endif

  799. 

  800. 	pshufd	$0x4e,   %xmm1, %xmm8

  801. 	mulpd	ALPHA_R, %xmm1

  802. 	mulpd	ALPHA_I, %xmm8

  803. 	addpd	%xmm8,   %xmm1

  804. 	SHUFPD_1 %xmm1,  %xmm0

  805. 

  806. 	addpd	 -8 * SIZE(Y), %xmm0

  807. 	movaps	%xmm0,  -8 * SIZE(Y)

  808. 	movaps	 -2 * SIZE(X), %xmm0

  809. 

  810. 	pshufd	$0x4e,   %xmm2, %xmm8

  811. 	mulpd	ALPHA_R, %xmm2

  812. 	mulpd	ALPHA_I, %xmm8

  813. 	addpd	%xmm8,   %xmm2

  814. 	SHUFPD_1 %xmm2,  %xmm1

  815. 

  816. 	addpd	 -6 * SIZE(Y), %xmm1

  817. 	movaps	%xmm1,  -6 * SIZE(Y)

  818. 	movaps	  0 * SIZE(X), %xmm1

  819. 

  820. #if defined(PREFETCH) && !defined(FETCH128)

  821. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  822. #endif

  823. 

  824. 	pshufd	$0x4e,   %xmm3, %xmm8

  825. 	mulpd	ALPHA_R, %xmm3

  826. 	mulpd	ALPHA_I, %xmm8

  827. 	addpd	%xmm8,   %xmm3

  828. 	SHUFPD_1 %xmm3,  %xmm2

  829. 

  830. 	addpd	 -4 * SIZE(Y), %xmm2

  831. 	movaps	%xmm2,  -4 * SIZE(Y)

  832. 	movaps	  2 * SIZE(X), %xmm2

  833. 

  834. 	pshufd	$0x4e,   %xmm0, %xmm8

  835. 	mulpd	ALPHA_R, %xmm0

  836. 	mulpd	ALPHA_I, %xmm8

  837. 	addpd	%xmm8,   %xmm0

  838. 	SHUFPD_1 %xmm0,  %xmm3

  839. 

  840. 	addpd	 -2 * SIZE(Y), %xmm3

  841. 	movaps	%xmm3,  -2 * SIZE(Y)

  842. 	movaps	  4 * SIZE(X), %xmm3

  843. 

  844. 	subq	$-16 * SIZE, X

  845. 	subq	$-16 * SIZE, Y

  846. 	decq	%rax

  847. 	jg	.L31

  848. 	ALIGN_3

  849. 

  850. .L32:

  851. 	pshufd	$0x4e,   %xmm1, %xmm8

  852. 	mulpd	ALPHA_R, %xmm1

  853. 	mulpd	ALPHA_I, %xmm8

  854. 	addpd	%xmm8,   %xmm1

  855. 	SHUFPD_1 %xmm1,  %xmm0

  856. 

  857. 	addpd	-16 * SIZE(Y), %xmm0

  858. 	movaps	%xmm0, -16 * SIZE(Y)

  859. 	movaps	-10 * SIZE(X), %xmm0

  860. 

  861. 	pshufd	$0x4e,   %xmm2, %xmm8

  862. 	mulpd	ALPHA_R, %xmm2

  863. 	mulpd	ALPHA_I, %xmm8

  864. 	addpd	%xmm8,   %xmm2

  865. 	SHUFPD_1 %xmm2,  %xmm1

  866. 

  867. 	addpd	-14 * SIZE(Y), %xmm1

  868. 	movaps	%xmm1, -14 * SIZE(Y)

  869. 	movaps	 -8 * SIZE(X), %xmm1

  870. 

  871. 	pshufd	$0x4e,   %xmm3, %xmm8

  872. 	mulpd	ALPHA_R, %xmm3

  873. 	mulpd	ALPHA_I, %xmm8

  874. 	addpd	%xmm8,   %xmm3

  875. 	SHUFPD_1 %xmm3,  %xmm2

  876. 

  877. 	addpd	-12 * SIZE(Y), %xmm2

  878. 	movaps	%xmm2, -12 * SIZE(Y)

  879. 	movaps	 -6 * SIZE(X), %xmm2

  880. 

  881. 	pshufd	$0x4e,   %xmm0, %xmm8

  882. 	mulpd	ALPHA_R, %xmm0

  883. 	mulpd	ALPHA_I, %xmm8

  884. 	addpd	%xmm8,   %xmm0

  885. 	SHUFPD_1 %xmm0,  %xmm3

  886. 

  887. 	addpd	-10 * SIZE(Y), %xmm3

  888. 	movaps	%xmm3, -10 * SIZE(Y)

  889. 	movaps	 -4 * SIZE(X), %xmm3

  890. 

  891. 	pshufd	$0x4e,   %xmm1, %xmm8

  892. 	mulpd	ALPHA_R, %xmm1

  893. 	mulpd	ALPHA_I, %xmm8

  894. 	addpd	%xmm8,   %xmm1

  895. 	SHUFPD_1 %xmm1,  %xmm0

  896. 

  897. 	addpd	 -8 * SIZE(Y), %xmm0

  898. 	movaps	%xmm0,  -8 * SIZE(Y)

  899. 	movaps	 -2 * SIZE(X), %xmm0

  900. 

  901. 	pshufd	$0x4e,   %xmm2, %xmm8

  902. 	mulpd	ALPHA_R, %xmm2

  903. 	mulpd	ALPHA_I, %xmm8

  904. 	addpd	%xmm8,   %xmm2

  905. 	SHUFPD_1 %xmm2,  %xmm1

  906. 

  907. 	addpd	 -6 * SIZE(Y), %xmm1

  908. 	movaps	%xmm1,  -6 * SIZE(Y)

  909. 

  910. 	pshufd	$0x4e,   %xmm3, %xmm8

  911. 	mulpd	ALPHA_R, %xmm3

  912. 	mulpd	ALPHA_I, %xmm8

  913. 	addpd	%xmm8,   %xmm3

  914. 	SHUFPD_1 %xmm3,  %xmm2

  915. 

  916. 	addpd	 -4 * SIZE(Y), %xmm2

  917. 	movaps	%xmm2,  -4 * SIZE(Y)

  918. 

  919. 	pshufd	$0x4e,   %xmm0, %xmm8

  920. 	mulpd	ALPHA_R, %xmm0

  921. 	mulpd	ALPHA_I, %xmm8

  922. 	addpd	%xmm8,   %xmm0

  923. 	SHUFPD_1 %xmm0,  %xmm3

  924. 

  925. 	addpd	 -2 * SIZE(Y), %xmm3

  926. 	movaps	%xmm3,  -2 * SIZE(Y)

  927. 

  928. 	subq	$-16 * SIZE, X

  929. 	subq	$-16 * SIZE, Y

  930. 	ALIGN_3

  931. 

  932. .L35:

  933. 	movq	M,  %rax

  934. 	andq	$4, %rax

  935. 	jle	.L36

  936. 

  937. 	movaps	-16 * SIZE(X), %xmm1

  938. 	movaps	-14 * SIZE(X), %xmm2

  939. 	movaps	-12 * SIZE(X), %xmm3

  940. 	movaps	-10 * SIZE(X), %xmm4

  941. 

  942. 	pshufd	$0x4e,   %xmm1, %xmm8

  943. 	mulpd	ALPHA_R, %xmm1

  944. 	mulpd	ALPHA_I, %xmm8

  945. 	addpd	%xmm8,   %xmm1

  946. 	SHUFPD_1 %xmm1,  %xmm0

  947. 

  948. 	addpd	-16 * SIZE(Y), %xmm0

  949. 	movaps	%xmm0, -16 * SIZE(Y)

  950. 

  951. 	pshufd	$0x4e,   %xmm2, %xmm8

  952. 	mulpd	ALPHA_R, %xmm2

  953. 	mulpd	ALPHA_I, %xmm8

  954. 	addpd	%xmm8,   %xmm2

  955. 	SHUFPD_1 %xmm2,  %xmm1

  956. 

  957. 	addpd	-14 * SIZE(Y), %xmm1

  958. 	movaps	%xmm1, -14 * SIZE(Y)

  959. 

  960. 	pshufd	$0x4e,   %xmm3, %xmm8

  961. 	mulpd	ALPHA_R, %xmm3

  962. 	mulpd	ALPHA_I, %xmm8

  963. 	addpd	%xmm8,   %xmm3

  964. 	SHUFPD_1 %xmm3,  %xmm2

  965. 

  966. 	addpd	-12 * SIZE(Y), %xmm2

  967. 	movaps	%xmm2, -12 * SIZE(Y)

  968. 

  969. 	pshufd	$0x4e,   %xmm4, %xmm8

  970. 	mulpd	ALPHA_R, %xmm4

  971. 	mulpd	ALPHA_I, %xmm8

  972. 	addpd	%xmm8,   %xmm4

  973. 	SHUFPD_1 %xmm4,  %xmm3

  974. 

  975. 	addpd	-10 * SIZE(Y), %xmm3

  976. 	movaps	%xmm3, -10 * SIZE(Y)

  977. 	movaps	%xmm4,   %xmm0

  978. 

  979. 	addq	$8 * SIZE, X

  980. 	addq	$8 * SIZE, Y

  981. 	ALIGN_3

  982. 

  983. .L36:

  984. 	movq	M,  %rax

  985. 	andq	$2, %rax

  986. 	jle	.L37

  987. 

  988. 	movaps	-16 * SIZE(X), %xmm1

  989. 	movaps	-14 * SIZE(X), %xmm2

  990. 

  991. 	pshufd	$0x4e,   %xmm1, %xmm8

  992. 	mulpd	ALPHA_R, %xmm1

  993. 	mulpd	ALPHA_I, %xmm8

  994. 	addpd	%xmm8,   %xmm1

  995. 	SHUFPD_1 %xmm1,  %xmm0

  996. 

  997. 	addpd	-16 * SIZE(Y), %xmm0

  998. 	movaps	%xmm0, -16 * SIZE(Y)

  999. 

  1000. 	pshufd	$0x4e,   %xmm2, %xmm8

  1001. 	mulpd	ALPHA_R, %xmm2

  1002. 	mulpd	ALPHA_I, %xmm8

  1003. 	addpd	%xmm8,   %xmm2

  1004. 	SHUFPD_1 %xmm2,  %xmm1

  1005. 

  1006. 	addpd	-14 * SIZE(Y), %xmm1

  1007. 	movaps	%xmm1, -14 * SIZE(Y)

  1008. 	movaps	%xmm2,   %xmm0

  1009. 

  1010. 	addq	$4 * SIZE, X

  1011. 	addq	$4 * SIZE, Y

  1012. 	ALIGN_3

  1013. 

  1014. .L37:

  1015. 	movq	M,  %rax

  1016. 	andq	$1, %rax

  1017. 	jle	.L39

  1018. 

  1019. 	movaps	-16 * SIZE(X), %xmm1

  1020. 

  1021. 	pshufd	$0x4e,   %xmm1, %xmm8

  1022. 	mulpd	ALPHA_R, %xmm1

  1023. 	mulpd	ALPHA_I, %xmm8

  1024. 	addpd	%xmm8,   %xmm1

  1025. 	SHUFPD_1 %xmm1,  %xmm0

  1026. 

  1027. 	addpd	-16 * SIZE(Y), %xmm0

  1028. 	movaps	%xmm0, -16 * SIZE(Y)

  1029. 	movaps	%xmm1,   %xmm0

  1030. 

  1031. 	addq	$2 * SIZE, X

  1032. 	addq	$2 * SIZE, Y

  1033. 	ALIGN_3

  1034. 

  1035. .L39:

  1036. 	SHUFPD_1 %xmm0,  %xmm0

  1037. 

  1038. 	addsd	-16 * SIZE(Y), %xmm0

  1039. 	movlps	%xmm0, -16 * SIZE(Y)

  1040. 	jmp	.L999

  1041. 	ALIGN_3

  1042. 

  1043. .L40:

  1044. 	movsd	-16 * SIZE(X), %xmm1

  1045. 	movhps	-15 * SIZE(X), %xmm1

  1046. 

  1047. 	pshufd	$0x4e,   %xmm1, %xmm8

  1048. 	mulpd	ALPHA_R, %xmm1

  1049. 	mulpd	ALPHA_I, %xmm8

  1050. 	addpd	%xmm8,   %xmm1

  1051. 	xorps	%xmm0,   %xmm0

  1052. 	SHUFPD_1 %xmm1,  %xmm0

  1053. 

  1054. 	xorps	%xmm4,   %xmm4

  1055. 	movhps	-16 * SIZE(Y), %xmm4

  1056. 

  1057. 	addpd	%xmm0,   %xmm4

  1058. 	movhps	%xmm4, -16 * SIZE(Y)

  1059. 	movaps	%xmm1,   %xmm0

  1060. 

  1061. 	addq	$2 * SIZE, X

  1062. 	addq	$1 * SIZE, Y

  1063. 	decq	M

  1064. 	jle	.L49

  1065. 

  1066. 	movq	M,  %rax

  1067. 	sarq	$3, %rax

  1068. 	jle	.L45

  1069. 

  1070. 	movsd	-16 * SIZE(X), %xmm1

  1071. 	movhps	-15 * SIZE(X), %xmm1

  1072. 	movsd	-14 * SIZE(X), %xmm2

  1073. 	movhps	-13 * SIZE(X), %xmm2

  1074. 	movsd	-12 * SIZE(X), %xmm3

  1075. 	movhps	-11 * SIZE(X), %xmm3

  1076. 

  1077. 	decq	%rax

  1078. 	jle	.L42

  1079. 	ALIGN_3

  1080. 

  1081. .L41:

  1082. #ifdef PREFETCHW

  1083. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  1084. #endif

  1085. 

  1086. 	pshufd	$0x4e,   %xmm1, %xmm8

  1087. 	mulpd	ALPHA_R, %xmm1

  1088. 	mulpd	ALPHA_I, %xmm8

  1089. 	addpd	%xmm8,   %xmm1

  1090. 	SHUFPD_1 %xmm1,  %xmm0

  1091. 

  1092. 	addpd	-16 * SIZE(Y), %xmm0

  1093. 	movaps	%xmm0, -16 * SIZE(Y)

  1094. 	movsd	-10 * SIZE(X), %xmm0

  1095. 	movhps	 -9 * SIZE(X), %xmm0

  1096. 

  1097. 	pshufd	$0x4e,   %xmm2, %xmm8

  1098. 	mulpd	ALPHA_R, %xmm2

  1099. 	mulpd	ALPHA_I, %xmm8

  1100. 	addpd	%xmm8,   %xmm2

  1101. 	SHUFPD_1 %xmm2,  %xmm1

  1102. 

  1103. 	addpd	-14 * SIZE(Y), %xmm1

  1104. 	movaps	%xmm1, -14 * SIZE(Y)

  1105. 	movsd	 -8 * SIZE(X), %xmm1

  1106. 	movhps	 -7 * SIZE(X), %xmm1

  1107. 

  1108. #ifdef PREFETCH

  1109. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  1110. #endif

  1111. 

  1112. 	pshufd	$0x4e,   %xmm3, %xmm8

  1113. 	mulpd	ALPHA_R, %xmm3

  1114. 	mulpd	ALPHA_I, %xmm8

  1115. 	addpd	%xmm8,   %xmm3

  1116. 	SHUFPD_1 %xmm3,  %xmm2

  1117. 

  1118. 	addpd	-12 * SIZE(Y), %xmm2

  1119. 	movaps	%xmm2, -12 * SIZE(Y)

  1120. 	movsd	 -6 * SIZE(X), %xmm2

  1121. 	movhps	 -5 * SIZE(X), %xmm2

  1122. 

  1123. 	pshufd	$0x4e,   %xmm0, %xmm8

  1124. 	mulpd	ALPHA_R, %xmm0

  1125. 	mulpd	ALPHA_I, %xmm8

  1126. 	addpd	%xmm8,   %xmm0

  1127. 	SHUFPD_1 %xmm0,  %xmm3

  1128. 

  1129. 	addpd	-10 * SIZE(Y), %xmm3

  1130. 	movaps	%xmm3, -10 * SIZE(Y)

  1131. 	movsd	 -4 * SIZE(X), %xmm3

  1132. 	movhps	 -3 * SIZE(X), %xmm3

  1133. 

  1134. #if defined(PREFETCHW) && !defined(FETCH128)

  1135. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  1136. #endif

  1137. 

  1138. 	pshufd	$0x4e,   %xmm1, %xmm8

  1139. 	mulpd	ALPHA_R, %xmm1

  1140. 	mulpd	ALPHA_I, %xmm8

  1141. 	addpd	%xmm8,   %xmm1

  1142. 	SHUFPD_1 %xmm1,  %xmm0

  1143. 

  1144. 	addpd	 -8 * SIZE(Y), %xmm0

  1145. 	movaps	%xmm0,  -8 * SIZE(Y)

  1146. 	movsd	 -2 * SIZE(X), %xmm0

  1147. 	movhps	 -1 * SIZE(X), %xmm0

  1148. 

  1149. 	pshufd	$0x4e,   %xmm2, %xmm8

  1150. 	mulpd	ALPHA_R, %xmm2

  1151. 	mulpd	ALPHA_I, %xmm8

  1152. 	addpd	%xmm8,   %xmm2

  1153. 	SHUFPD_1 %xmm2,  %xmm1

  1154. 

  1155. 	addpd	 -6 * SIZE(Y), %xmm1

  1156. 	movaps	%xmm1,  -6 * SIZE(Y)

  1157. 	movsd	  0 * SIZE(X), %xmm1

  1158. 	movhps	  1 * SIZE(X), %xmm1

  1159. 

  1160. #if defined(PREFETCH) && !defined(FETCH128)

  1161. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  1162. #endif

  1163. 

  1164. 	pshufd	$0x4e,   %xmm3, %xmm8

  1165. 	mulpd	ALPHA_R, %xmm3

  1166. 	mulpd	ALPHA_I, %xmm8

  1167. 	addpd	%xmm8,   %xmm3

  1168. 	SHUFPD_1 %xmm3,  %xmm2

  1169. 

  1170. 	addpd	 -4 * SIZE(Y), %xmm2

  1171. 	movaps	%xmm2,  -4 * SIZE(Y)

  1172. 	movsd	  2 * SIZE(X), %xmm2

  1173. 	movhps	  3 * SIZE(X), %xmm2

  1174. 

  1175. 	pshufd	$0x4e,   %xmm0, %xmm8

  1176. 	mulpd	ALPHA_R, %xmm0

  1177. 	mulpd	ALPHA_I, %xmm8

  1178. 	addpd	%xmm8,   %xmm0

  1179. 	SHUFPD_1 %xmm0,  %xmm3

  1180. 

  1181. 	addpd	 -2 * SIZE(Y), %xmm3

  1182. 	movaps	%xmm3,  -2 * SIZE(Y)

  1183. 	movsd	  4 * SIZE(X), %xmm3

  1184. 	movhps	  5 * SIZE(X), %xmm3

  1185. 

  1186. 	subq	$-16 * SIZE, X

  1187. 	subq	$-16 * SIZE, Y

  1188. 	decq	%rax

  1189. 	jg	.L41

  1190. 	ALIGN_3

  1191. 

  1192. .L42:

  1193. 	pshufd	$0x4e,   %xmm1, %xmm8

  1194. 	mulpd	ALPHA_R, %xmm1

  1195. 	mulpd	ALPHA_I, %xmm8

  1196. 	addpd	%xmm8,   %xmm1

  1197. 	SHUFPD_1 %xmm1,  %xmm0

  1198. 

  1199. 	addpd	-16 * SIZE(Y), %xmm0

  1200. 	movaps	%xmm0, -16 * SIZE(Y)

  1201. 	movsd	-10 * SIZE(X), %xmm0

  1202. 	movhps	 -9 * SIZE(X), %xmm0

  1203. 

  1204. 	pshufd	$0x4e,   %xmm2, %xmm8

  1205. 	mulpd	ALPHA_R, %xmm2

  1206. 	mulpd	ALPHA_I, %xmm8

  1207. 	addpd	%xmm8,   %xmm2

  1208. 	SHUFPD_1 %xmm2,  %xmm1

  1209. 

  1210. 	addpd	-14 * SIZE(Y), %xmm1

  1211. 	movaps	%xmm1, -14 * SIZE(Y)

  1212. 	movsd	 -8 * SIZE(X), %xmm1

  1213. 	movhps	 -7 * SIZE(X), %xmm1

  1214. 

  1215. 	pshufd	$0x4e,   %xmm3, %xmm8

  1216. 	mulpd	ALPHA_R, %xmm3

  1217. 	mulpd	ALPHA_I, %xmm8

  1218. 	addpd	%xmm8,   %xmm3

  1219. 	SHUFPD_1 %xmm3,  %xmm2

  1220. 

  1221. 	addpd	-12 * SIZE(Y), %xmm2

  1222. 	movaps	%xmm2, -12 * SIZE(Y)

  1223. 	movsd	 -6 * SIZE(X), %xmm2

  1224. 	movhps	 -5 * SIZE(X), %xmm2

  1225. 

  1226. 	pshufd	$0x4e,   %xmm0, %xmm8

  1227. 	mulpd	ALPHA_R, %xmm0

  1228. 	mulpd	ALPHA_I, %xmm8

  1229. 	addpd	%xmm8,   %xmm0

  1230. 	SHUFPD_1 %xmm0,  %xmm3

  1231. 

  1232. 	addpd	-10 * SIZE(Y), %xmm3

  1233. 	movaps	%xmm3, -10 * SIZE(Y)

  1234. 	movsd	 -4 * SIZE(X), %xmm3

  1235. 	movhps	 -3 * SIZE(X), %xmm3

  1236. 

  1237. 	pshufd	$0x4e,   %xmm1, %xmm8

  1238. 	mulpd	ALPHA_R, %xmm1

  1239. 	mulpd	ALPHA_I, %xmm8

  1240. 	addpd	%xmm8,   %xmm1

  1241. 	SHUFPD_1 %xmm1,  %xmm0

  1242. 

  1243. 	addpd	 -8 * SIZE(Y), %xmm0

  1244. 	movaps	%xmm0,  -8 * SIZE(Y)

  1245. 	movsd	 -2 * SIZE(X), %xmm0

  1246. 	movhps	 -1 * SIZE(X), %xmm0

  1247. 

  1248. 	pshufd	$0x4e,   %xmm2, %xmm8

  1249. 	mulpd	ALPHA_R, %xmm2

  1250. 	mulpd	ALPHA_I, %xmm8

  1251. 	addpd	%xmm8,   %xmm2

  1252. 	SHUFPD_1 %xmm2,  %xmm1

  1253. 

  1254. 	addpd	 -6 * SIZE(Y), %xmm1

  1255. 	movaps	%xmm1,  -6 * SIZE(Y)

  1256. 

  1257. 	pshufd	$0x4e,   %xmm3, %xmm8

  1258. 	mulpd	ALPHA_R, %xmm3

  1259. 	mulpd	ALPHA_I, %xmm8

  1260. 	addpd	%xmm8,   %xmm3

  1261. 	SHUFPD_1 %xmm3,  %xmm2

  1262. 

  1263. 	addpd	 -4 * SIZE(Y), %xmm2

  1264. 	movaps	%xmm2,  -4 * SIZE(Y)

  1265. 

  1266. 	pshufd	$0x4e,   %xmm0, %xmm8

  1267. 	mulpd	ALPHA_R, %xmm0

  1268. 	mulpd	ALPHA_I, %xmm8

  1269. 	addpd	%xmm8,   %xmm0

  1270. 	SHUFPD_1 %xmm0,  %xmm3

  1271. 

  1272. 	addpd	 -2 * SIZE(Y), %xmm3

  1273. 	movaps	%xmm3,  -2 * SIZE(Y)

  1274. 

  1275. 	subq	$-16 * SIZE, X

  1276. 	subq	$-16 * SIZE, Y

  1277. 	ALIGN_3

  1278. 

  1279. .L45:

  1280. 	movq	M,  %rax

  1281. 	andq	$4, %rax

  1282. 	jle	.L46

  1283. 

  1284. 	movsd	-16 * SIZE(X), %xmm1

  1285. 	movhps	-15 * SIZE(X), %xmm1

  1286. 	movsd	-14 * SIZE(X), %xmm2

  1287. 	movhps	-13 * SIZE(X), %xmm2

  1288. 	movsd	-12 * SIZE(X), %xmm3

  1289. 	movhps	-11 * SIZE(X), %xmm3

  1290. 	movsd	-10 * SIZE(X), %xmm4

  1291. 	movhps	 -9 * SIZE(X), %xmm4

  1292. 

  1293. 	pshufd	$0x4e,   %xmm1, %xmm8

  1294. 	mulpd	ALPHA_R, %xmm1

  1295. 	mulpd	ALPHA_I, %xmm8

  1296. 	addpd	%xmm8,   %xmm1

  1297. 	SHUFPD_1 %xmm1,  %xmm0

  1298. 

  1299. 	addpd	-16 * SIZE(Y), %xmm0

  1300. 	movaps	%xmm0, -16 * SIZE(Y)

  1301. 

  1302. 	pshufd	$0x4e,   %xmm2, %xmm8

  1303. 	mulpd	ALPHA_R, %xmm2

  1304. 	mulpd	ALPHA_I, %xmm8

  1305. 	addpd	%xmm8,   %xmm2

  1306. 	SHUFPD_1 %xmm2,  %xmm1

  1307. 

  1308. 	addpd	-14 * SIZE(Y), %xmm1

  1309. 	movaps	%xmm1, -14 * SIZE(Y)

  1310. 

  1311. 	pshufd	$0x4e,   %xmm3, %xmm8

  1312. 	mulpd	ALPHA_R, %xmm3

  1313. 	mulpd	ALPHA_I, %xmm8

  1314. 	addpd	%xmm8,   %xmm3

  1315. 	SHUFPD_1 %xmm3,  %xmm2

  1316. 

  1317. 	addpd	-12 * SIZE(Y), %xmm2

  1318. 	movaps	%xmm2, -12 * SIZE(Y)

  1319. 

  1320. 	pshufd	$0x4e,   %xmm4, %xmm8

  1321. 	mulpd	ALPHA_R, %xmm4

  1322. 	mulpd	ALPHA_I, %xmm8

  1323. 	addpd	%xmm8,   %xmm4

  1324. 	SHUFPD_1 %xmm4,  %xmm3

  1325. 

  1326. 	addpd	-10 * SIZE(Y), %xmm3

  1327. 	movaps	%xmm3, -10 * SIZE(Y)

  1328. 	movaps	%xmm4,   %xmm0

  1329. 

  1330. 	addq	$8 * SIZE, X

  1331. 	addq	$8 * SIZE, Y

  1332. 	ALIGN_3

  1333. 

  1334. .L46:

  1335. 	movq	M,  %rax

  1336. 	andq	$2, %rax

  1337. 	jle	.L47

  1338. 

  1339. 	movsd	-16 * SIZE(X), %xmm1

  1340. 	movhps	-15 * SIZE(X), %xmm1

  1341. 	movsd	-14 * SIZE(X), %xmm2

  1342. 	movhps	-13 * SIZE(X), %xmm2

  1343. 

  1344. 	pshufd	$0x4e,   %xmm1, %xmm8

  1345. 	mulpd	ALPHA_R, %xmm1

  1346. 	mulpd	ALPHA_I, %xmm8

  1347. 	addpd	%xmm8,   %xmm1

  1348. 	SHUFPD_1 %xmm1,  %xmm0

  1349. 

  1350. 	addpd	-16 * SIZE(Y), %xmm0

  1351. 	movaps	%xmm0, -16 * SIZE(Y)

  1352. 

  1353. 	pshufd	$0x4e,   %xmm2, %xmm8

  1354. 	mulpd	ALPHA_R, %xmm2

  1355. 	mulpd	ALPHA_I, %xmm8

  1356. 	addpd	%xmm8,   %xmm2

  1357. 	SHUFPD_1 %xmm2,  %xmm1

  1358. 

  1359. 	addpd	-14 * SIZE(Y), %xmm1

  1360. 	movaps	%xmm1, -14 * SIZE(Y)

  1361. 	movaps	%xmm2,   %xmm0

  1362. 

  1363. 	addq	$4 * SIZE, X

  1364. 	addq	$4 * SIZE, Y

  1365. 	ALIGN_3

  1366. 

  1367. .L47:

  1368. 	movq	M,  %rax

  1369. 	andq	$1, %rax

  1370. 	jle	.L49

  1371. 

  1372. 	movsd	-16 * SIZE(X), %xmm1

  1373. 	movhps	-15 * SIZE(X), %xmm1

  1374. 

  1375. 	pshufd	$0x4e,   %xmm1, %xmm8

  1376. 	mulpd	ALPHA_R, %xmm1

  1377. 	mulpd	ALPHA_I, %xmm8

  1378. 	addpd	%xmm8,   %xmm1

  1379. 	SHUFPD_1 %xmm1,  %xmm0

  1380. 

  1381. 	addpd	-16 * SIZE(Y), %xmm0

  1382. 	movaps	%xmm0, -16 * SIZE(Y)

  1383. 	movaps	%xmm1,   %xmm0

  1384. 

  1385. 	addq	$2 * SIZE, X

  1386. 	addq	$2 * SIZE, Y

  1387. 	ALIGN_3

  1388. 

  1389. .L49:

  1390. 	SHUFPD_1 %xmm0,  %xmm0

  1391. 

  1392. 	addsd	-16 * SIZE(Y), %xmm0

  1393. 	movlps	%xmm0, -16 * SIZE(Y)

  1394. 	jmp	.L999

  1395. 	ALIGN_3

  1396. 

  1397. .L50:

  1398. #ifndef CONJ

  1399. 	movaps	%xmm0, %xmm14		#  a   0

  1400. 

  1401. 	pxor	%xmm15, %xmm15		#  0   0

  1402. 	subsd	%xmm1,  %xmm15		# -b   0

  1403. 

  1404. 	unpcklpd %xmm14, %xmm15		# -b   a

  1405. 	unpcklpd %xmm1,  %xmm14		#  a   b

  1406. #else

  1407. 	movaps	%xmm0, %xmm14		#  a   0

  1408. 	movaps	%xmm1, %xmm15		#  b   0

  1409. 

  1410. 	pxor	%xmm13, %xmm13		#  0   0

  1411. 	subsd	%xmm0,  %xmm13		# -a   0

  1412. 

  1413. 	unpcklpd %xmm13, %xmm15		#  b   -a

  1414. 	unpcklpd %xmm1,  %xmm14		#  a   b

  1415. #endif

  1416. 

  1417. 	movq	Y, YY

  1418. 	movq	M,  %rax

  1419. //If incx==0 || incy==0, avoid unloop and jump to end.

  1420. 	cmpq	$0, INCX

  1421. 	jne  	.L59

  1422. 	cmpq	$0, INCY

  1423. 	je	.L58

  1424. .L59:

  1425. 	sarq	$3, %rax

  1426. 	jle	.L55

  1427. 

  1428. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1429. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1430. 	addq	 INCX, X

  1431. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1432. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1433. 	addq	 INCX, X

  1434. 	MOVDDUP( 0 * SIZE, X, %xmm4)

  1435. 	MOVDDUP( 1 * SIZE, X, %xmm5)

  1436. 	addq	 INCX, X

  1437. 	MOVDDUP( 0 * SIZE, X, %xmm6)

  1438. 	MOVDDUP( 1 * SIZE, X, %xmm7)

  1439. 	addq	 INCX, X

  1440. 

  1441. 	movsd	 0 * SIZE(Y), %xmm8

  1442. 	movhpd	 1 * SIZE(Y), %xmm8

  1443. 	addq	 INCY, Y

  1444. 	movsd	 0 * SIZE(Y), %xmm9

  1445. 	movhpd	 1 * SIZE(Y), %xmm9

  1446. 	addq	 INCY, Y

  1447. 	movsd	 0 * SIZE(Y), %xmm10

  1448. 	movhpd	 1 * SIZE(Y), %xmm10

  1449. 	addq	 INCY, Y

  1450. 	movsd	 0 * SIZE(Y), %xmm11

  1451. 	movhpd	 1 * SIZE(Y), %xmm11

  1452. 	addq	 INCY, Y

  1453. 

  1454. 	mulpd	%xmm14, %xmm0

  1455. 	mulpd	%xmm14, %xmm2

  1456. 	mulpd	%xmm14, %xmm4

  1457. 	mulpd	%xmm14, %xmm6

  1458. 

  1459. 	decq	%rax

  1460. 	jle	.L52

  1461. 	ALIGN_3

  1462. 

  1463. .L51:

  1464. 	addpd	%xmm0, %xmm8

  1465. 	mulpd	%xmm15, %xmm1

  1466. 	addpd	%xmm2, %xmm9

  1467. 	mulpd	%xmm15, %xmm3

  1468. 	addpd	%xmm4, %xmm10

  1469. 	mulpd	%xmm15, %xmm5

  1470. 	addpd	%xmm6, %xmm11

  1471. 	mulpd	%xmm15, %xmm7

  1472. 

  1473. 	addpd	%xmm1, %xmm8

  1474. 	addpd	%xmm3, %xmm9

  1475. 	addpd	%xmm5, %xmm10

  1476. 	addpd	%xmm7, %xmm11

  1477. 

  1478. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1479. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1480. 	addq	 INCX, X

  1481. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1482. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1483. 	addq	 INCX, X

  1484. 	MOVDDUP( 0 * SIZE, X, %xmm4)

  1485. 	MOVDDUP( 1 * SIZE, X, %xmm5)

  1486. 	addq	 INCX, X

  1487. 	MOVDDUP( 0 * SIZE, X, %xmm6)

  1488. 	MOVDDUP( 1 * SIZE, X, %xmm7)

  1489. 	addq	 INCX, X

  1490. 

  1491. 	mulpd	%xmm14, %xmm0

  1492. 	mulpd	%xmm14, %xmm2

  1493. 	mulpd	%xmm14, %xmm4

  1494. 	mulpd	%xmm14, %xmm6

  1495. 

  1496. 	movlpd	%xmm8,   0 * SIZE(YY)

  1497. 	movhpd	%xmm8,   1 * SIZE(YY)

  1498. 	addq	 INCY, YY

  1499. 	movlpd	%xmm9,   0 * SIZE(YY)

  1500. 	movhpd	%xmm9,   1 * SIZE(YY)

  1501. 	addq	 INCY, YY

  1502. 	movlpd	%xmm10,  0 * SIZE(YY)

  1503. 	movhpd	%xmm10,  1 * SIZE(YY)

  1504. 	addq	 INCY, YY

  1505. 	movlpd	%xmm11,  0 * SIZE(YY)

  1506. 	movhpd	%xmm11,  1 * SIZE(YY)

  1507. 	addq	 INCY, YY

  1508. 

  1509. 	movsd	 0 * SIZE(Y), %xmm8

  1510. 	movhpd	 1 * SIZE(Y), %xmm8

  1511. 	addq	 INCY, Y

  1512. 	movsd	 0 * SIZE(Y), %xmm9

  1513. 	movhpd	 1 * SIZE(Y), %xmm9

  1514. 	addq	 INCY, Y

  1515. 	movsd	 0 * SIZE(Y), %xmm10

  1516. 	movhpd	 1 * SIZE(Y), %xmm10

  1517. 	addq	 INCY, Y

  1518. 	movsd	 0 * SIZE(Y), %xmm11

  1519. 	movhpd	 1 * SIZE(Y), %xmm11

  1520. 	addq	 INCY, Y

  1521. 

  1522. 	addpd	%xmm0, %xmm8

  1523. 	mulpd	%xmm15, %xmm1

  1524. 	addpd	%xmm2, %xmm9

  1525. 	mulpd	%xmm15, %xmm3

  1526. 	addpd	%xmm4, %xmm10

  1527. 	mulpd	%xmm15, %xmm5

  1528. 	addpd	%xmm6, %xmm11

  1529. 	mulpd	%xmm15, %xmm7

  1530. 

  1531. 	addpd	%xmm1, %xmm8

  1532. 	addpd	%xmm3, %xmm9

  1533. 	addpd	%xmm5, %xmm10

  1534. 	addpd	%xmm7, %xmm11

  1535. 

  1536. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1537. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1538. 	addq	 INCX, X

  1539. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1540. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1541. 	addq	 INCX, X

  1542. 	MOVDDUP( 0 * SIZE, X, %xmm4)

  1543. 	MOVDDUP( 1 * SIZE, X, %xmm5)

  1544. 	addq	 INCX, X

  1545. 	MOVDDUP( 0 * SIZE, X, %xmm6)

  1546. 	MOVDDUP( 1 * SIZE, X, %xmm7)

  1547. 	addq	 INCX, X

  1548. 

  1549. 	mulpd	%xmm14, %xmm0

  1550. 	mulpd	%xmm14, %xmm2

  1551. 	mulpd	%xmm14, %xmm4

  1552. 	mulpd	%xmm14, %xmm6

  1553. 

  1554. 	movlpd	%xmm8,   0 * SIZE(YY)

  1555. 	movhpd	%xmm8,   1 * SIZE(YY)

  1556. 	addq	 INCY, YY

  1557. 	movlpd	%xmm9,   0 * SIZE(YY)

  1558. 	movhpd	%xmm9,   1 * SIZE(YY)

  1559. 	addq	 INCY, YY

  1560. 	movlpd	%xmm10,  0 * SIZE(YY)

  1561. 	movhpd	%xmm10,  1 * SIZE(YY)

  1562. 	addq	 INCY, YY

  1563. 	movlpd	%xmm11,  0 * SIZE(YY)

  1564. 	movhpd	%xmm11,  1 * SIZE(YY)

  1565. 	addq	 INCY, YY

  1566. 

  1567. 	movsd	 0 * SIZE(Y), %xmm8

  1568. 	movhpd	 1 * SIZE(Y), %xmm8

  1569. 	addq	 INCY, Y

  1570. 	movsd	 0 * SIZE(Y), %xmm9

  1571. 	movhpd	 1 * SIZE(Y), %xmm9

  1572. 	addq	 INCY, Y

  1573. 	movsd	 0 * SIZE(Y), %xmm10

  1574. 	movhpd	 1 * SIZE(Y), %xmm10

  1575. 	addq	 INCY, Y

  1576. 	movsd	 0 * SIZE(Y), %xmm11

  1577. 	movhpd	 1 * SIZE(Y), %xmm11

  1578. 	addq	 INCY, Y

  1579. 

  1580. 	decq	%rax

  1581. 	jg	.L51

  1582. 	ALIGN_3

  1583. 

  1584. .L52:

  1585. 	addpd	%xmm0, %xmm8

  1586. 	mulpd	%xmm15, %xmm1

  1587. 	addpd	%xmm2, %xmm9

  1588. 	mulpd	%xmm15, %xmm3

  1589. 	addpd	%xmm4, %xmm10

  1590. 	mulpd	%xmm15, %xmm5

  1591. 	addpd	%xmm6, %xmm11

  1592. 	mulpd	%xmm15, %xmm7

  1593. 

  1594. 	addpd	%xmm1, %xmm8

  1595. 	addpd	%xmm3, %xmm9

  1596. 	addpd	%xmm5, %xmm10

  1597. 	addpd	%xmm7, %xmm11

  1598. 

  1599. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1600. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1601. 	addq	 INCX, X

  1602. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1603. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1604. 	addq	 INCX, X

  1605. 	MOVDDUP( 0 * SIZE, X, %xmm4)

  1606. 	MOVDDUP( 1 * SIZE, X, %xmm5)

  1607. 	addq	 INCX, X

  1608. 	MOVDDUP( 0 * SIZE, X, %xmm6)

  1609. 	MOVDDUP( 1 * SIZE, X, %xmm7)

  1610. 	addq	 INCX, X

  1611. 

  1612. 	mulpd	%xmm14, %xmm0

  1613. 	mulpd	%xmm14, %xmm2

  1614. 	mulpd	%xmm14, %xmm4

  1615. 	mulpd	%xmm14, %xmm6

  1616. 

  1617. 	movlpd	%xmm8,   0 * SIZE(YY)

  1618. 	movhpd	%xmm8,   1 * SIZE(YY)

  1619. 	addq	 INCY, YY

  1620. 	movlpd	%xmm9,   0 * SIZE(YY)

  1621. 	movhpd	%xmm9,   1 * SIZE(YY)

  1622. 	addq	 INCY, YY

  1623. 	movlpd	%xmm10,  0 * SIZE(YY)

  1624. 	movhpd	%xmm10,  1 * SIZE(YY)

  1625. 	addq	 INCY, YY

  1626. 	movlpd	%xmm11,  0 * SIZE(YY)

  1627. 	movhpd	%xmm11,  1 * SIZE(YY)

  1628. 	addq	 INCY, YY

  1629. 

  1630. 	movsd	 0 * SIZE(Y), %xmm8

  1631. 	movhpd	 1 * SIZE(Y), %xmm8

  1632. 	addq	 INCY, Y

  1633. 	movsd	 0 * SIZE(Y), %xmm9

  1634. 	movhpd	 1 * SIZE(Y), %xmm9

  1635. 	addq	 INCY, Y

  1636. 	movsd	 0 * SIZE(Y), %xmm10

  1637. 	movhpd	 1 * SIZE(Y), %xmm10

  1638. 	addq	 INCY, Y

  1639. 	movsd	 0 * SIZE(Y), %xmm11

  1640. 	movhpd	 1 * SIZE(Y), %xmm11

  1641. 	addq	 INCY, Y

  1642. 

  1643. 	addpd	%xmm0, %xmm8

  1644. 	mulpd	%xmm15, %xmm1

  1645. 	addpd	%xmm2, %xmm9

  1646. 	mulpd	%xmm15, %xmm3

  1647. 	addpd	%xmm4, %xmm10

  1648. 	mulpd	%xmm15, %xmm5

  1649. 	addpd	%xmm6, %xmm11

  1650. 	mulpd	%xmm15, %xmm7

  1651. 

  1652. 	addpd	%xmm1, %xmm8

  1653. 	addpd	%xmm3, %xmm9

  1654. 	addpd	%xmm5, %xmm10

  1655. 	addpd	%xmm7, %xmm11

  1656. 

  1657. 	movlpd	%xmm8,   0 * SIZE(YY)

  1658. 	movhpd	%xmm8,   1 * SIZE(YY)

  1659. 	addq	 INCY, YY

  1660. 	movlpd	%xmm9,   0 * SIZE(YY)

  1661. 	movhpd	%xmm9,   1 * SIZE(YY)

  1662. 	addq	 INCY, YY

  1663. 	movlpd	%xmm10,  0 * SIZE(YY)

  1664. 	movhpd	%xmm10,  1 * SIZE(YY)

  1665. 	addq	 INCY, YY

  1666. 	movlpd	%xmm11,  0 * SIZE(YY)

  1667. 	movhpd	%xmm11,  1 * SIZE(YY)

  1668. 	addq	 INCY, YY

  1669. 	ALIGN_3

  1670. 

  1671. .L55:

  1672. 	movq	M,  %rax

  1673. 	andq	$4, %rax

  1674. 	jle	.L56

  1675. 

  1676. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1677. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1678. 	addq	INCX, X

  1679. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1680. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1681. 	addq	INCX, X

  1682. 

  1683. 	MOVDDUP( 0 * SIZE, X, %xmm4)

  1684. 	MOVDDUP( 1 * SIZE, X, %xmm5)

  1685. 	addq	INCX, X

  1686. 	MOVDDUP( 0 * SIZE, X, %xmm6)

  1687. 	MOVDDUP( 1 * SIZE, X, %xmm7)

  1688. 	addq	INCX, X

  1689. 

  1690. 	movsd	 0 * SIZE(Y), %xmm8

  1691. 	movhpd	 1 * SIZE(Y), %xmm8

  1692. 	addq	 INCY, Y

  1693. 	movsd	 0 * SIZE(Y), %xmm9

  1694. 	movhpd	 1 * SIZE(Y), %xmm9

  1695. 	addq	 INCY, Y

  1696. 	movsd	 0 * SIZE(Y), %xmm10

  1697. 	movhpd	 1 * SIZE(Y), %xmm10

  1698. 	addq	 INCY, Y

  1699. 	movsd	 0 * SIZE(Y), %xmm11

  1700. 	movhpd	 1 * SIZE(Y), %xmm11

  1701. 	addq	 INCY, Y

  1702. 

  1703. 	mulpd	%xmm14, %xmm0

  1704. 	mulpd	%xmm14, %xmm2

  1705. 	mulpd	%xmm14, %xmm4

  1706. 	mulpd	%xmm14, %xmm6

  1707. 

  1708. 	addpd	%xmm0, %xmm8

  1709. 	mulpd	%xmm15, %xmm1

  1710. 	addpd	%xmm2, %xmm9

  1711. 	mulpd	%xmm15, %xmm3

  1712. 	addpd	%xmm4, %xmm10

  1713. 	mulpd	%xmm15, %xmm5

  1714. 	addpd	%xmm6, %xmm11

  1715. 	mulpd	%xmm15, %xmm7

  1716. 

  1717. 	addpd	%xmm1, %xmm8

  1718. 	addpd	%xmm3, %xmm9

  1719. 	addpd	%xmm5, %xmm10

  1720. 	addpd	%xmm7, %xmm11

  1721. 

  1722. 	movlpd	%xmm8,  0 * SIZE(YY)

  1723. 	movhpd	%xmm8,  1 * SIZE(YY)

  1724. 	addq	 INCY, YY

  1725. 	movlpd	%xmm9,  0 * SIZE(YY)

  1726. 	movhpd	%xmm9,  1 * SIZE(YY)

  1727. 	addq	 INCY, YY

  1728. 	movlpd	%xmm10, 0 * SIZE(YY)

  1729. 	movhpd	%xmm10, 1 * SIZE(YY)

  1730. 	addq	 INCY, YY

  1731. 	movlpd	%xmm11, 0 * SIZE(YY)

  1732. 	movhpd	%xmm11, 1 * SIZE(YY)

  1733. 	addq	 INCY, YY

  1734. 	ALIGN_3

  1735. 

  1736. .L56:

  1737. 	movq	M,  %rax

  1738. 	andq	$2, %rax

  1739. 	jle	.L57

  1740. 

  1741. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1742. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1743. 	addq	INCX, X

  1744. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1745. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1746. 	addq	INCX, X

  1747. 

  1748. 	movsd	 0 * SIZE(Y), %xmm8

  1749. 	movhpd	 1 * SIZE(Y), %xmm8

  1750. 	addq	 INCY, Y

  1751. 	movsd	 0 * SIZE(Y), %xmm9

  1752. 	movhpd	 1 * SIZE(Y), %xmm9

  1753. 	addq	 INCY, Y

  1754. 

  1755. 	mulpd	%xmm14, %xmm0

  1756. 	mulpd	%xmm14, %xmm2

  1757. 	mulpd	%xmm15, %xmm1

  1758. 	mulpd	%xmm15, %xmm3

  1759. 

  1760. 	addpd	%xmm0, %xmm8

  1761. 	addpd	%xmm2, %xmm9

  1762. 	addpd	%xmm1, %xmm8

  1763. 	addpd	%xmm3, %xmm9

  1764. 

  1765. 	movlpd	%xmm8,   0 * SIZE(YY)

  1766. 	movhpd	%xmm8,   1 * SIZE(YY)

  1767. 	addq	 INCY, YY

  1768. 	movlpd	%xmm9,   0 * SIZE(YY)

  1769. 	movhpd	%xmm9,   1 * SIZE(YY)

  1770. 	addq	 INCY, YY

  1771. 	ALIGN_3

  1772. 

  1773. .L57:

  1774. 	movq	M,  %rax

  1775. 	andq	$1, %rax

  1776. 	jle	.L999

  1777. 

  1778. .L58:

  1779. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1780. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1781. 

  1782. 	movsd	 0 * SIZE(Y), %xmm8

  1783. 	movhpd	 1 * SIZE(Y), %xmm8

  1784. 	mulpd	%xmm14, %xmm0

  1785. 	mulpd	%xmm15, %xmm1

  1786. 	addpd	%xmm0, %xmm8

  1787. 	addpd	%xmm1, %xmm8

  1788. 

  1789. 	movlpd	%xmm8,   0 * SIZE(YY)

  1790. 	movhpd	%xmm8,   1 * SIZE(YY)

  1791. 

  1792. 	decq %rax

  1793. 	jg	.L58

  1794. 	ALIGN_3

  1795. 

  1796. .L999:

  1797. 	xorq	%rax, %rax

  1798. 

  1799. 	RESTOREREGISTERS

  1800. 

  1801. 	ret

  1802. 

  1803. 	EPILOGUE