OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 549a9f1dbb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '549a9f1dbb'
${ noResults }
OpenBLAS/kernel/x86_64/zdot_sse2.S

1563 lines
32 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define N	ARG1	/* rdi */

  44. #define X	ARG2	/* rsi */

  45. #define INCX	ARG3	/* rdx */

  46. #define Y	ARG4	/* rcx */

  47. #define INCY	ARG5	/* r8  */

  48. #else

  49. #define RESULT_ADDRESS	ARG1    /*rcx*/

  50. #define N	ARG2	/* rdx */

  51. #define X	ARG3	/* r8 */

  52. #define INCX	ARG4	/* r9*/

  53. #define Y	%r10

  54. #define INCY	%r11

  55. #endif

  56. 

  57. #include "l1param.h"

  58. 

  59. #undef movsd

  60. 

  61. #ifndef OPTERON

  62. #define MOVLPS	movsd

  63. #else

  64. #define MOVLPS	movlps

  65. #endif

  66. 

  67. 

  68. 	PROLOGUE

  69. 	PROFCODE

  70. 

  71. #ifdef WINDOWS_ABI

  72. 	movq	40(%rsp), Y

  73. 	movq	48(%rsp), INCY

  74. #endif

  75. 

  76. 	SAVEREGISTERS

  77. 

  78. 	salq	$ZBASE_SHIFT, INCX

  79. 	salq	$ZBASE_SHIFT, INCY

  80. 

  81. 	xorps	%xmm0, %xmm0

  82. 	xorps	%xmm1, %xmm1

  83. 	xorps	%xmm2, %xmm2

  84. 	xorps	%xmm3, %xmm3

  85. 

  86. 	cmpq	$0, N

  87. 	jle	.L999

  88. 

  89. 	cmpq	$2 * SIZE, INCX

  90. 	jne	.L50

  91. 	cmpq	$2 * SIZE, INCY

  92. 	jne	.L50

  93. 

  94. 	subq	$-16 * SIZE, X

  95. 	subq	$-16 * SIZE, Y

  96. 

  97. 	testq	$SIZE, Y

  98. 	jne	.L30

  99. 

  100. 	testq	$SIZE, X

  101. 	jne	.L20

  102. 

  103. 	movq	N,  %rax

  104. 	sarq	$3, %rax

  105. 	jle	.L15

  106. 

  107. 	movaps	-16 * SIZE(X), %xmm4

  108. 	movaps	-14 * SIZE(X), %xmm5

  109. 	movaps	-16 * SIZE(Y), %xmm8

  110. 	movaps	-14 * SIZE(Y), %xmm9

  111. 	movaps	-12 * SIZE(X), %xmm6

  112. 	movaps	-10 * SIZE(X), %xmm7

  113. 	movaps	-12 * SIZE(Y), %xmm10

  114. 	movaps	-10 * SIZE(Y), %xmm11

  115. 

  116. 	decq	%rax

  117. 	jle	.L12

  118. 	ALIGN_3

  119. 

  120. .L11:

  121. #ifdef PREFETCH

  122. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  123. #endif

  124. 

  125. 	pshufd	$0x4e,  %xmm8, %xmm12

  126. 	mulpd	%xmm4,  %xmm8

  127. 	addpd	%xmm8,  %xmm0

  128. 	movaps	 -8 * SIZE(Y), %xmm8

  129. 	mulpd	%xmm4,  %xmm12

  130. 	movaps	 -8 * SIZE(X), %xmm4

  131. 	addpd	%xmm12, %xmm1

  132. 

  133. 	pshufd	$0x4e,  %xmm9, %xmm12

  134. 	mulpd	%xmm5,  %xmm9

  135. 	addpd	%xmm9,  %xmm2

  136. 	movaps	 -6 * SIZE(Y), %xmm9

  137. 	mulpd	%xmm5,  %xmm12

  138. 	movaps	 -6 * SIZE(X), %xmm5

  139. 	addpd	%xmm12, %xmm3

  140. 

  141. #ifdef PREFETCH

  142. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  143. #endif

  144. 

  145. 	pshufd	$0x4e,  %xmm10, %xmm12

  146. 	mulpd	%xmm6,  %xmm10

  147. 	addpd	%xmm10, %xmm0

  148. 	movaps	 -4 * SIZE(Y), %xmm10

  149. 	mulpd	%xmm6,  %xmm12

  150. 	movaps	 -4 * SIZE(X), %xmm6

  151. 	addpd	%xmm12, %xmm1

  152. 

  153. 	pshufd	$0x4e,  %xmm11, %xmm12

  154. 	mulpd	%xmm7,  %xmm11

  155. 	addpd	%xmm11, %xmm2

  156. 	movaps	 -2 * SIZE(Y), %xmm11

  157. 	mulpd	%xmm7,  %xmm12

  158. 	movaps	 -2 * SIZE(X), %xmm7

  159. 	addpd	%xmm12, %xmm3

  160. 

  161. #if defined(PREFETCH) && !defined(FETCH128)

  162. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  163. #endif

  164. 

  165. 	pshufd	$0x4e,  %xmm8, %xmm12

  166. 	mulpd	%xmm4,  %xmm8

  167. 	addpd	%xmm8,  %xmm0

  168. 	movaps	  0 * SIZE(Y), %xmm8

  169. 	mulpd	%xmm4,  %xmm12

  170. 	movaps	  0 * SIZE(X), %xmm4

  171. 	addpd	%xmm12, %xmm1

  172. 

  173. 	pshufd	$0x4e,  %xmm9, %xmm12

  174. 	mulpd	%xmm5,  %xmm9

  175. 	addpd	%xmm9,  %xmm2

  176. 	movaps	  2 * SIZE(Y), %xmm9

  177. 	mulpd	%xmm5,  %xmm12

  178. 	movaps	  2 * SIZE(X), %xmm5

  179. 	addpd	%xmm12, %xmm3

  180. 

  181. #if defined(PREFETCH) && !defined(FETCH128)

  182. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  183. #endif

  184. 

  185. 	pshufd	$0x4e,  %xmm10, %xmm12

  186. 	mulpd	%xmm6,  %xmm10

  187. 	addpd	%xmm10, %xmm0

  188. 	movaps	  4 * SIZE(Y), %xmm10

  189. 	mulpd	%xmm6,  %xmm12

  190. 	movaps	  4 * SIZE(X), %xmm6

  191. 	addpd	%xmm12, %xmm1

  192. 

  193. 	pshufd	$0x4e,  %xmm11, %xmm12

  194. 	mulpd	%xmm7,  %xmm11

  195. 	addpd	%xmm11, %xmm2

  196. 	movaps	  6 * SIZE(Y), %xmm11

  197. 	mulpd	%xmm7,  %xmm12

  198. 	movaps	  6 * SIZE(X), %xmm7

  199. 	addpd	%xmm12, %xmm3

  200. 

  201. 	subq	$-16 * SIZE, X

  202. 	subq	$-16 * SIZE, Y

  203. 

  204. 	decq	%rax

  205. 	jg	.L11

  206. 	ALIGN_3

  207. 

  208. .L12:

  209. 	pshufd	$0x4e,  %xmm8, %xmm12

  210. 	mulpd	%xmm4,  %xmm8

  211. 	addpd	%xmm8,  %xmm0

  212. 	movaps	 -8 * SIZE(Y), %xmm8

  213. 	mulpd	%xmm4,  %xmm12

  214. 	movaps	 -8 * SIZE(X), %xmm4

  215. 	addpd	%xmm12, %xmm1

  216. 

  217. 	pshufd	$0x4e,  %xmm9, %xmm12

  218. 	mulpd	%xmm5,  %xmm9

  219. 	addpd	%xmm9,  %xmm2

  220. 	movaps	 -6 * SIZE(Y), %xmm9

  221. 	mulpd	%xmm5,  %xmm12

  222. 	movaps	 -6 * SIZE(X), %xmm5

  223. 	addpd	%xmm12, %xmm3

  224. 

  225. 	pshufd	$0x4e,  %xmm10, %xmm12

  226. 	mulpd	%xmm6,  %xmm10

  227. 	addpd	%xmm10, %xmm0

  228. 	movaps	 -4 * SIZE(Y), %xmm10

  229. 	mulpd	%xmm6,  %xmm12

  230. 	movaps	 -4 * SIZE(X), %xmm6

  231. 	addpd	%xmm12, %xmm1

  232. 

  233. 	pshufd	$0x4e,  %xmm11, %xmm12

  234. 	mulpd	%xmm7,  %xmm11

  235. 	addpd	%xmm11, %xmm2

  236. 	movaps	 -2 * SIZE(Y), %xmm11

  237. 	mulpd	%xmm7,  %xmm12

  238. 	movaps	 -2 * SIZE(X), %xmm7

  239. 	addpd	%xmm12, %xmm3

  240. 

  241. 	pshufd	$0x4e,  %xmm8, %xmm12

  242. 	mulpd	%xmm4,  %xmm8

  243. 	addpd	%xmm8,  %xmm0

  244. 	mulpd	%xmm4,  %xmm12

  245. 	addpd	%xmm12, %xmm1

  246. 

  247. 	pshufd	$0x4e,  %xmm9, %xmm12

  248. 	mulpd	%xmm5,  %xmm9

  249. 	addpd	%xmm9,  %xmm2

  250. 	mulpd	%xmm5,  %xmm12

  251. 	addpd	%xmm12, %xmm3

  252. 

  253. 	pshufd	$0x4e,  %xmm10, %xmm12

  254. 	mulpd	%xmm6,  %xmm10

  255. 	addpd	%xmm10, %xmm0

  256. 	mulpd	%xmm6,  %xmm12

  257. 	addpd	%xmm12, %xmm1

  258. 

  259. 	pshufd	$0x4e,  %xmm11, %xmm12

  260. 	mulpd	%xmm7,  %xmm11

  261. 	addpd	%xmm11, %xmm2

  262. 	mulpd	%xmm7,  %xmm12

  263. 	addpd	%xmm12, %xmm3

  264. 

  265. 	subq	$-16 * SIZE, X

  266. 	subq	$-16 * SIZE, Y

  267. 	ALIGN_3

  268. 

  269. .L15:

  270. 	testq	$4, N

  271. 	jle	.L16

  272. 

  273. 	movaps	-16 * SIZE(X), %xmm4

  274. 	movaps	-16 * SIZE(Y), %xmm8

  275. 	movaps	-14 * SIZE(X), %xmm5

  276. 	movaps	-14 * SIZE(Y), %xmm9

  277. 

  278. 	pshufd	$0x4e,  %xmm8, %xmm12

  279. 	mulpd	%xmm4,  %xmm8

  280. 	addpd	%xmm8,  %xmm0

  281. 	mulpd	%xmm4,  %xmm12

  282. 	addpd	%xmm12, %xmm1

  283. 

  284. 	pshufd	$0x4e,  %xmm9, %xmm12

  285. 	mulpd	%xmm5,  %xmm9

  286. 	addpd	%xmm9,  %xmm2

  287. 	mulpd	%xmm5,  %xmm12

  288. 	addpd	%xmm12, %xmm3

  289. 

  290. 	movaps	-12 * SIZE(X), %xmm6

  291. 	movaps	-12 * SIZE(Y), %xmm10

  292. 	movaps	-10 * SIZE(X), %xmm7

  293. 	movaps	-10 * SIZE(Y), %xmm11

  294. 

  295. 	pshufd	$0x4e,  %xmm10, %xmm12

  296. 	mulpd	%xmm6,  %xmm10

  297. 	addpd	%xmm10, %xmm0

  298. 	mulpd	%xmm6,  %xmm12

  299. 	addpd	%xmm12, %xmm1

  300. 

  301. 	pshufd	$0x4e,  %xmm11, %xmm12

  302. 	mulpd	%xmm7,  %xmm11

  303. 	addpd	%xmm11, %xmm2

  304. 	mulpd	%xmm7,  %xmm12

  305. 	addpd	%xmm12, %xmm3

  306. 

  307. 	addq	$8 * SIZE, X

  308. 	addq	$8 * SIZE, Y

  309. 	ALIGN_3

  310. 

  311. .L16:

  312. 	testq	$2, N

  313. 	jle	.L17

  314. 

  315. 	movaps	-16 * SIZE(X), %xmm4

  316. 	movaps	-16 * SIZE(Y), %xmm8

  317. 	movaps	-14 * SIZE(X), %xmm5

  318. 	movaps	-14 * SIZE(Y), %xmm9

  319. 

  320. 	pshufd	$0x4e,  %xmm8, %xmm12

  321. 	mulpd	%xmm4,  %xmm8

  322. 	addpd	%xmm8,  %xmm0

  323. 	mulpd	%xmm4,  %xmm12

  324. 	addpd	%xmm12, %xmm1

  325. 

  326. 	pshufd	$0x4e,  %xmm9, %xmm12

  327. 	mulpd	%xmm5,  %xmm9

  328. 	addpd	%xmm9,  %xmm2

  329. 	mulpd	%xmm5,  %xmm12

  330. 	addpd	%xmm12, %xmm3

  331. 

  332. 	addq	$4 * SIZE, X

  333. 	addq	$4 * SIZE, Y

  334. 	ALIGN_3

  335. 

  336. .L17:

  337. 	testq	$1, N

  338. 	jle	.L98

  339. 

  340. 	movaps	-16 * SIZE(X), %xmm4

  341. 	movaps	-16 * SIZE(Y), %xmm8

  342. 

  343. 	pshufd	$0x4e,  %xmm8, %xmm12

  344. 	mulpd	%xmm4,  %xmm8

  345. 	addpd	%xmm8,  %xmm0

  346. 	mulpd	%xmm4,  %xmm12

  347. 	addpd	%xmm12, %xmm1

  348. 	jmp	.L98

  349. 	ALIGN_3

  350. 

  351. .L20:

  352. 	movq	N,  %rax

  353. 	sarq	$3, %rax

  354. 	jle	.L25

  355. 

  356. 	MOVLPS	-16 * SIZE(X), %xmm4

  357. 	movhps	-15 * SIZE(X), %xmm4

  358. 	MOVLPS	-14 * SIZE(X), %xmm5

  359. 	movhps	-13 * SIZE(X), %xmm5

  360. 	movaps	-16 * SIZE(Y), %xmm8

  361. 	movaps	-14 * SIZE(Y), %xmm9

  362. 	MOVLPS	-12 * SIZE(X), %xmm6

  363. 	movhps	-11 * SIZE(X), %xmm6

  364. 	MOVLPS	-10 * SIZE(X), %xmm7

  365. 	movhps	 -9 * SIZE(X), %xmm7

  366. 	movaps	-12 * SIZE(Y), %xmm10

  367. 	movaps	-10 * SIZE(Y), %xmm11

  368. 

  369. 	decq	%rax

  370. 	jle	.L22

  371. 	ALIGN_3

  372. 

  373. .L21:

  374. #ifdef PREFETCH

  375. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  376. #endif

  377. 

  378. 	pshufd	$0x4e,  %xmm8, %xmm12

  379. 	mulpd	%xmm4,  %xmm8

  380. 	addpd	%xmm8,  %xmm0

  381. 	movaps	 -8 * SIZE(Y), %xmm8

  382. 	mulpd	%xmm4,  %xmm12

  383. 	MOVLPS	 -8 * SIZE(X), %xmm4

  384. 	movhps	 -7 * SIZE(X), %xmm4

  385. 	addpd	%xmm12, %xmm1

  386. 

  387. 	pshufd	$0x4e,  %xmm9, %xmm12

  388. 	mulpd	%xmm5,  %xmm9

  389. 	addpd	%xmm9,  %xmm2

  390. 	movaps	 -6 * SIZE(Y), %xmm9

  391. 	mulpd	%xmm5,  %xmm12

  392. 	MOVLPS	 -6 * SIZE(X), %xmm5

  393. 	movhps	 -5 * SIZE(X), %xmm5

  394. 	addpd	%xmm12, %xmm3

  395. 

  396. #ifdef PREFETCH

  397. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  398. #endif

  399. 

  400. 	pshufd	$0x4e,  %xmm10, %xmm12

  401. 	mulpd	%xmm6,  %xmm10

  402. 	addpd	%xmm10, %xmm0

  403. 	movaps	 -4 * SIZE(Y), %xmm10

  404. 	mulpd	%xmm6,  %xmm12

  405. 	MOVLPS	 -4 * SIZE(X), %xmm6

  406. 	movhps	 -3 * SIZE(X), %xmm6

  407. 	addpd	%xmm12, %xmm1

  408. 

  409. 	pshufd	$0x4e,  %xmm11, %xmm12

  410. 	mulpd	%xmm7,  %xmm11

  411. 	addpd	%xmm11, %xmm2

  412. 	movaps	 -2 * SIZE(Y), %xmm11

  413. 	mulpd	%xmm7,  %xmm12

  414. 	MOVLPS	 -2 * SIZE(X), %xmm7

  415. 	movhps	 -1 * SIZE(X), %xmm7

  416. 	addpd	%xmm12, %xmm3

  417. 

  418. #if defined(PREFETCH) && !defined(FETCH128)

  419. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  420. #endif

  421. 

  422. 	pshufd	$0x4e,  %xmm8, %xmm12

  423. 	mulpd	%xmm4,  %xmm8

  424. 	addpd	%xmm8,  %xmm0

  425. 	movaps	  0 * SIZE(Y), %xmm8

  426. 	mulpd	%xmm4,  %xmm12

  427. 	MOVLPS	  0 * SIZE(X), %xmm4

  428. 	movhps	  1 * SIZE(X), %xmm4

  429. 	addpd	%xmm12, %xmm1

  430. 

  431. 	pshufd	$0x4e,  %xmm9, %xmm12

  432. 	mulpd	%xmm5,  %xmm9

  433. 	addpd	%xmm9,  %xmm2

  434. 	movaps	  2 * SIZE(Y), %xmm9

  435. 	mulpd	%xmm5,  %xmm12

  436. 	MOVLPS	  2 * SIZE(X), %xmm5

  437. 	movhps	  3 * SIZE(X), %xmm5

  438. 	addpd	%xmm12, %xmm3

  439. 

  440. #if defined(PREFETCH) && !defined(FETCH128)

  441. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  442. #endif

  443. 

  444. 	pshufd	$0x4e,  %xmm10, %xmm12

  445. 	mulpd	%xmm6,  %xmm10

  446. 	addpd	%xmm10, %xmm0

  447. 	movaps	  4 * SIZE(Y), %xmm10

  448. 	mulpd	%xmm6,  %xmm12

  449. 	MOVLPS	  4 * SIZE(X), %xmm6

  450. 	movhps	  5 * SIZE(X), %xmm6

  451. 	addpd	%xmm12, %xmm1

  452. 

  453. 	pshufd	$0x4e,  %xmm11, %xmm12

  454. 	mulpd	%xmm7,  %xmm11

  455. 	addpd	%xmm11, %xmm2

  456. 	movaps	  6 * SIZE(Y), %xmm11

  457. 	mulpd	%xmm7,  %xmm12

  458. 	MOVLPS	  6 * SIZE(X), %xmm7

  459. 	movhps	  7 * SIZE(X), %xmm7

  460. 	addpd	%xmm12, %xmm3

  461. 

  462. 	subq	$-16 * SIZE, X

  463. 	subq	$-16 * SIZE, Y

  464. 

  465. 	decq	%rax

  466. 	jg	.L21

  467. 	ALIGN_3

  468. 

  469. .L22:

  470. 

  471. 	pshufd	$0x4e,  %xmm8, %xmm12

  472. 	mulpd	%xmm4,  %xmm8

  473. 	addpd	%xmm8,  %xmm0

  474. 	movaps	 -8 * SIZE(Y), %xmm8

  475. 	mulpd	%xmm4,  %xmm12

  476. 	MOVLPS	 -8 * SIZE(X), %xmm4

  477. 	movhps	 -7 * SIZE(X), %xmm4

  478. 	addpd	%xmm12, %xmm1

  479. 

  480. 	pshufd	$0x4e,  %xmm9, %xmm12

  481. 	mulpd	%xmm5,  %xmm9

  482. 	addpd	%xmm9,  %xmm2

  483. 	movaps	 -6 * SIZE(Y), %xmm9

  484. 	mulpd	%xmm5,  %xmm12

  485. 	MOVLPS	 -6 * SIZE(X), %xmm5

  486. 	movhps	 -5 * SIZE(X), %xmm5

  487. 	addpd	%xmm12, %xmm3

  488. 

  489. 	pshufd	$0x4e,  %xmm10, %xmm12

  490. 	mulpd	%xmm6,  %xmm10

  491. 	addpd	%xmm10, %xmm0

  492. 	movaps	 -4 * SIZE(Y), %xmm10

  493. 	mulpd	%xmm6,  %xmm12

  494. 	MOVLPS	 -4 * SIZE(X), %xmm6

  495. 	movhps	 -3 * SIZE(X), %xmm6

  496. 	addpd	%xmm12, %xmm1

  497. 

  498. 	pshufd	$0x4e,  %xmm11, %xmm12

  499. 	mulpd	%xmm7,  %xmm11

  500. 	addpd	%xmm11, %xmm2

  501. 	movaps	 -2 * SIZE(Y), %xmm11

  502. 	mulpd	%xmm7,  %xmm12

  503. 	MOVLPS	 -2 * SIZE(X), %xmm7

  504. 	movhps	 -1 * SIZE(X), %xmm7

  505. 	addpd	%xmm12, %xmm3

  506. 

  507. 	pshufd	$0x4e,  %xmm8, %xmm12

  508. 	mulpd	%xmm4,  %xmm8

  509. 	addpd	%xmm8,  %xmm0

  510. 	mulpd	%xmm4,  %xmm12

  511. 	addpd	%xmm12, %xmm1

  512. 

  513. 	pshufd	$0x4e,  %xmm9, %xmm12

  514. 	mulpd	%xmm5,  %xmm9

  515. 	addpd	%xmm9,  %xmm2

  516. 	mulpd	%xmm5,  %xmm12

  517. 	addpd	%xmm12, %xmm3

  518. 

  519. 	pshufd	$0x4e,  %xmm10, %xmm12

  520. 	mulpd	%xmm6,  %xmm10

  521. 	addpd	%xmm10, %xmm0

  522. 	mulpd	%xmm6,  %xmm12

  523. 	addpd	%xmm12, %xmm1

  524. 

  525. 	pshufd	$0x4e,  %xmm11, %xmm12

  526. 	mulpd	%xmm7,  %xmm11

  527. 	addpd	%xmm11, %xmm2

  528. 	mulpd	%xmm7,  %xmm12

  529. 	addpd	%xmm12, %xmm3

  530. 

  531. 	subq	$-16 * SIZE, X

  532. 	subq	$-16 * SIZE, Y

  533. 	ALIGN_3

  534. 

  535. .L25:

  536. 	testq	$4, N

  537. 	jle	.L26

  538. 

  539. 	MOVLPS	-16 * SIZE(X), %xmm4

  540. 	movhps	-15 * SIZE(X), %xmm4

  541. 	movaps	-16 * SIZE(Y), %xmm8

  542. 

  543. 	pshufd	$0x4e,  %xmm8, %xmm12

  544. 	mulpd	%xmm4,  %xmm8

  545. 	addpd	%xmm8,  %xmm0

  546. 	mulpd	%xmm4,  %xmm12

  547. 	addpd	%xmm12, %xmm1

  548. 

  549. 	MOVLPS	-14 * SIZE(X), %xmm5

  550. 	movhps	-13 * SIZE(X), %xmm5

  551. 	movaps	-14 * SIZE(Y), %xmm9

  552. 

  553. 	pshufd	$0x4e,  %xmm9, %xmm12

  554. 	mulpd	%xmm5,  %xmm9

  555. 	addpd	%xmm9,  %xmm2

  556. 	mulpd	%xmm5,  %xmm12

  557. 	addpd	%xmm12, %xmm3

  558. 

  559. 	MOVLPS	-12 * SIZE(X), %xmm6

  560. 	movhps	-11 * SIZE(X), %xmm6

  561. 	movaps	-12 * SIZE(Y), %xmm10

  562. 

  563. 	pshufd	$0x4e,  %xmm10, %xmm12

  564. 	mulpd	%xmm6,  %xmm10

  565. 	addpd	%xmm10, %xmm0

  566. 	mulpd	%xmm6,  %xmm12

  567. 	addpd	%xmm12, %xmm1

  568. 

  569. 	MOVLPS	-10 * SIZE(X), %xmm7

  570. 	movhps	 -9 * SIZE(X), %xmm7

  571. 	movaps	-10 * SIZE(Y), %xmm11

  572. 

  573. 	pshufd	$0x4e,  %xmm11, %xmm12

  574. 	mulpd	%xmm7,  %xmm11

  575. 	addpd	%xmm11, %xmm2

  576. 	mulpd	%xmm7,  %xmm12

  577. 	addpd	%xmm12, %xmm3

  578. 

  579. 	addq	$8 * SIZE, X

  580. 	addq	$8 * SIZE, Y

  581. 	ALIGN_3

  582. 

  583. .L26:

  584. 	testq	$2, N

  585. 	jle	.L27

  586. 

  587. 	MOVLPS	-16 * SIZE(X), %xmm4

  588. 	movhps	-15 * SIZE(X), %xmm4

  589. 	movaps	-16 * SIZE(Y), %xmm8

  590. 

  591. 	pshufd	$0x4e,  %xmm8, %xmm12

  592. 	mulpd	%xmm4,  %xmm8

  593. 	addpd	%xmm8,  %xmm0

  594. 	mulpd	%xmm4,  %xmm12

  595. 	addpd	%xmm12, %xmm1

  596. 

  597. 	MOVLPS	-14 * SIZE(X), %xmm5

  598. 	movhps	-13 * SIZE(X), %xmm5

  599. 	movaps	-14 * SIZE(Y), %xmm9

  600. 

  601. 	pshufd	$0x4e,  %xmm9, %xmm12

  602. 	mulpd	%xmm5,  %xmm9

  603. 	addpd	%xmm9,  %xmm2

  604. 	mulpd	%xmm5,  %xmm12

  605. 	addpd	%xmm12, %xmm3

  606. 

  607. 	addq	$4 * SIZE, X

  608. 	addq	$4 * SIZE, Y

  609. 	ALIGN_3

  610. 

  611. .L27:

  612. 	testq	$1, N

  613. 	jle	.L98

  614. 

  615. 	MOVLPS	-16 * SIZE(X), %xmm4

  616. 	movhps	-15 * SIZE(X), %xmm4

  617. 	movaps	-16 * SIZE(Y), %xmm8

  618. 

  619. 	pshufd	$0x4e,  %xmm8, %xmm12

  620. 	mulpd	%xmm4,  %xmm8

  621. 	addpd	%xmm8,  %xmm0

  622. 	mulpd	%xmm4,  %xmm12

  623. 	addpd	%xmm12, %xmm1

  624. 	jmp	.L98

  625. 	ALIGN_3

  626. 

  627. .L30:

  628. 	testq	$SIZE, X

  629. 	jne	.L40

  630. 

  631. 	movq	N,  %rax

  632. 	sarq	$3, %rax

  633. 	jle	.L35

  634. 

  635. 	MOVLPS	-16 * SIZE(Y), %xmm4

  636. 	movhps	-15 * SIZE(Y), %xmm4

  637. 	MOVLPS	-14 * SIZE(Y), %xmm5

  638. 	movhps	-13 * SIZE(Y), %xmm5

  639. 	movaps	-16 * SIZE(X), %xmm8

  640. 	movaps	-14 * SIZE(X), %xmm9

  641. 	MOVLPS	-12 * SIZE(Y), %xmm6

  642. 	movhps	-11 * SIZE(Y), %xmm6

  643. 	MOVLPS	-10 * SIZE(Y), %xmm7

  644. 	movhps	 -9 * SIZE(Y), %xmm7

  645. 	movaps	-12 * SIZE(X), %xmm10

  646. 	movaps	-10 * SIZE(X), %xmm11

  647. 

  648. 	decq	%rax

  649. 	jle	.L32

  650. 	ALIGN_3

  651. 

  652. .L31:

  653. #ifdef PREFETCH

  654. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  655. #endif

  656. 

  657. 	pshufd	$0x4e,  %xmm8, %xmm12

  658. 	mulpd	%xmm4,  %xmm8

  659. 	addpd	%xmm8,  %xmm0

  660. 	movaps	 -8 * SIZE(X), %xmm8

  661. 	mulpd	%xmm4,  %xmm12

  662. 	MOVLPS	 -8 * SIZE(Y), %xmm4

  663. 	movhps	 -7 * SIZE(Y), %xmm4

  664. 	addpd	%xmm12, %xmm1

  665. 

  666. 	pshufd	$0x4e,  %xmm9, %xmm12

  667. 	mulpd	%xmm5,  %xmm9

  668. 	addpd	%xmm9,  %xmm2

  669. 	movaps	 -6 * SIZE(X), %xmm9

  670. 	mulpd	%xmm5,  %xmm12

  671. 	MOVLPS	 -6 * SIZE(Y), %xmm5

  672. 	movhps	 -5 * SIZE(Y), %xmm5

  673. 	addpd	%xmm12, %xmm3

  674. 

  675. #ifdef PREFETCH

  676. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  677. #endif

  678. 

  679. 	pshufd	$0x4e,  %xmm10, %xmm12

  680. 	mulpd	%xmm6,  %xmm10

  681. 	addpd	%xmm10, %xmm0

  682. 	movaps	 -4 * SIZE(X), %xmm10

  683. 	mulpd	%xmm6,  %xmm12

  684. 	MOVLPS	 -4 * SIZE(Y), %xmm6

  685. 	movhps	 -3 * SIZE(Y), %xmm6

  686. 	addpd	%xmm12, %xmm1

  687. 

  688. 	pshufd	$0x4e,  %xmm11, %xmm12

  689. 	mulpd	%xmm7,  %xmm11

  690. 	addpd	%xmm11, %xmm2

  691. 	movaps	 -2 * SIZE(X), %xmm11

  692. 	mulpd	%xmm7,  %xmm12

  693. 	MOVLPS	 -2 * SIZE(Y), %xmm7

  694. 	movhps	 -1 * SIZE(Y), %xmm7

  695. 	addpd	%xmm12, %xmm3

  696. 

  697. #if defined(PREFETCH) && !defined(FETCH128)

  698. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  699. #endif

  700. 

  701. 	pshufd	$0x4e,  %xmm8, %xmm12

  702. 	mulpd	%xmm4,  %xmm8

  703. 	addpd	%xmm8,  %xmm0

  704. 	movaps	  0 * SIZE(X), %xmm8

  705. 	mulpd	%xmm4,  %xmm12

  706. 	MOVLPS	  0 * SIZE(Y), %xmm4

  707. 	movhps	  1 * SIZE(Y), %xmm4

  708. 	addpd	%xmm12, %xmm1

  709. 

  710. 	pshufd	$0x4e,  %xmm9, %xmm12

  711. 	mulpd	%xmm5,  %xmm9

  712. 	addpd	%xmm9,  %xmm2

  713. 	movaps	  2 * SIZE(X), %xmm9

  714. 	mulpd	%xmm5,  %xmm12

  715. 	MOVLPS	  2 * SIZE(Y), %xmm5

  716. 	movhps	  3 * SIZE(Y), %xmm5

  717. 	addpd	%xmm12, %xmm3

  718. 

  719. #if defined(PREFETCH) && !defined(FETCH128)

  720. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  721. #endif

  722. 

  723. 	pshufd	$0x4e,  %xmm10, %xmm12

  724. 	mulpd	%xmm6,  %xmm10

  725. 	addpd	%xmm10, %xmm0

  726. 	movaps	  4 * SIZE(X), %xmm10

  727. 	mulpd	%xmm6,  %xmm12

  728. 	MOVLPS	  4 * SIZE(Y), %xmm6

  729. 	movhps	  5 * SIZE(Y), %xmm6

  730. 	addpd	%xmm12, %xmm1

  731. 

  732. 	pshufd	$0x4e,  %xmm11, %xmm12

  733. 	mulpd	%xmm7,  %xmm11

  734. 	addpd	%xmm11, %xmm2

  735. 	movaps	  6 * SIZE(X), %xmm11

  736. 	mulpd	%xmm7,  %xmm12

  737. 	MOVLPS	  6 * SIZE(Y), %xmm7

  738. 	movhps	  7 * SIZE(Y), %xmm7

  739. 	addpd	%xmm12, %xmm3

  740. 

  741. 	subq	$-16 * SIZE, X

  742. 	subq	$-16 * SIZE, Y

  743. 

  744. 	decq	%rax

  745. 	jg	.L31

  746. 	ALIGN_3

  747. 

  748. .L32:

  749. 

  750. 	pshufd	$0x4e,  %xmm8, %xmm12

  751. 	mulpd	%xmm4,  %xmm8

  752. 	addpd	%xmm8,  %xmm0

  753. 	movaps	 -8 * SIZE(X), %xmm8

  754. 	mulpd	%xmm4,  %xmm12

  755. 	MOVLPS	 -8 * SIZE(Y), %xmm4

  756. 	movhps	 -7 * SIZE(Y), %xmm4

  757. 	addpd	%xmm12, %xmm1

  758. 

  759. 	pshufd	$0x4e,  %xmm9, %xmm12

  760. 	mulpd	%xmm5,  %xmm9

  761. 	addpd	%xmm9,  %xmm2

  762. 	movaps	 -6 * SIZE(X), %xmm9

  763. 	mulpd	%xmm5,  %xmm12

  764. 	MOVLPS	 -6 * SIZE(Y), %xmm5

  765. 	movhps	 -5 * SIZE(Y), %xmm5

  766. 	addpd	%xmm12, %xmm3

  767. 

  768. 	pshufd	$0x4e,  %xmm10, %xmm12

  769. 	mulpd	%xmm6,  %xmm10

  770. 	addpd	%xmm10, %xmm0

  771. 	movaps	 -4 * SIZE(X), %xmm10

  772. 	mulpd	%xmm6,  %xmm12

  773. 	MOVLPS	 -4 * SIZE(Y), %xmm6

  774. 	movhps	 -3 * SIZE(Y), %xmm6

  775. 	addpd	%xmm12, %xmm1

  776. 

  777. 	pshufd	$0x4e,  %xmm11, %xmm12

  778. 	mulpd	%xmm7,  %xmm11

  779. 	addpd	%xmm11, %xmm2

  780. 	movaps	 -2 * SIZE(X), %xmm11

  781. 	mulpd	%xmm7,  %xmm12

  782. 	MOVLPS	 -2 * SIZE(Y), %xmm7

  783. 	movhps	 -1 * SIZE(Y), %xmm7

  784. 	addpd	%xmm12, %xmm3

  785. 

  786. 	pshufd	$0x4e,  %xmm8, %xmm12

  787. 	mulpd	%xmm4,  %xmm8

  788. 	addpd	%xmm8,  %xmm0

  789. 	mulpd	%xmm4,  %xmm12

  790. 	addpd	%xmm12, %xmm1

  791. 

  792. 	pshufd	$0x4e,  %xmm9, %xmm12

  793. 	mulpd	%xmm5,  %xmm9

  794. 	addpd	%xmm9,  %xmm2

  795. 	mulpd	%xmm5,  %xmm12

  796. 	addpd	%xmm12, %xmm3

  797. 

  798. 	pshufd	$0x4e,  %xmm10, %xmm12

  799. 	mulpd	%xmm6,  %xmm10

  800. 	addpd	%xmm10, %xmm0

  801. 	mulpd	%xmm6,  %xmm12

  802. 	addpd	%xmm12, %xmm1

  803. 

  804. 	pshufd	$0x4e,  %xmm11, %xmm12

  805. 	mulpd	%xmm7,  %xmm11

  806. 	addpd	%xmm11, %xmm2

  807. 	mulpd	%xmm7,  %xmm12

  808. 	addpd	%xmm12, %xmm3

  809. 

  810. 	subq	$-16 * SIZE, X

  811. 	subq	$-16 * SIZE, Y

  812. 	ALIGN_3

  813. 

  814. .L35:

  815. 	testq	$4, N

  816. 	jle	.L36

  817. 

  818. 	MOVLPS	-16 * SIZE(Y), %xmm4

  819. 	movhps	-15 * SIZE(Y), %xmm4

  820. 	movaps	-16 * SIZE(X), %xmm8

  821. 

  822. 	pshufd	$0x4e,  %xmm8, %xmm12

  823. 	mulpd	%xmm4,  %xmm8

  824. 	addpd	%xmm8,  %xmm0

  825. 	mulpd	%xmm4,  %xmm12

  826. 	addpd	%xmm12, %xmm1

  827. 

  828. 	MOVLPS	-14 * SIZE(Y), %xmm5

  829. 	movhps	-13 * SIZE(Y), %xmm5

  830. 	movaps	-14 * SIZE(X), %xmm9

  831. 

  832. 	pshufd	$0x4e,  %xmm9, %xmm12

  833. 	mulpd	%xmm5,  %xmm9

  834. 	addpd	%xmm9,  %xmm2

  835. 	mulpd	%xmm5,  %xmm12

  836. 	addpd	%xmm12, %xmm3

  837. 

  838. 	MOVLPS	-12 * SIZE(Y), %xmm6

  839. 	movhps	-11 * SIZE(Y), %xmm6

  840. 	movaps	-12 * SIZE(X), %xmm10

  841. 

  842. 	pshufd	$0x4e,  %xmm10, %xmm12

  843. 	mulpd	%xmm6,  %xmm10

  844. 	addpd	%xmm10, %xmm0

  845. 	mulpd	%xmm6,  %xmm12

  846. 	addpd	%xmm12, %xmm1

  847. 

  848. 	MOVLPS	-10 * SIZE(Y), %xmm7

  849. 	movhps	 -9 * SIZE(Y), %xmm7

  850. 	movaps	-10 * SIZE(X), %xmm11

  851. 

  852. 	pshufd	$0x4e,  %xmm11, %xmm12

  853. 	mulpd	%xmm7,  %xmm11

  854. 	addpd	%xmm11, %xmm2

  855. 	mulpd	%xmm7,  %xmm12

  856. 	addpd	%xmm12, %xmm3

  857. 

  858. 	addq	$8 * SIZE, X

  859. 	addq	$8 * SIZE, Y

  860. 	ALIGN_3

  861. 

  862. .L36:

  863. 	testq	$2, N

  864. 	jle	.L37

  865. 

  866. 	MOVLPS	-16 * SIZE(Y), %xmm4

  867. 	movhps	-15 * SIZE(Y), %xmm4

  868. 	movaps	-16 * SIZE(X), %xmm8

  869. 

  870. 	pshufd	$0x4e,  %xmm8, %xmm12

  871. 	mulpd	%xmm4,  %xmm8

  872. 	addpd	%xmm8,  %xmm0

  873. 	mulpd	%xmm4,  %xmm12

  874. 	addpd	%xmm12, %xmm1

  875. 

  876. 	MOVLPS	-14 * SIZE(Y), %xmm5

  877. 	movhps	-13 * SIZE(Y), %xmm5

  878. 	movaps	-14 * SIZE(X), %xmm9

  879. 

  880. 	pshufd	$0x4e,  %xmm9, %xmm12

  881. 	mulpd	%xmm5,  %xmm9

  882. 	addpd	%xmm9,  %xmm2

  883. 	mulpd	%xmm5,  %xmm12

  884. 	addpd	%xmm12, %xmm3

  885. 

  886. 	addq	$4 * SIZE, X

  887. 	addq	$4 * SIZE, Y

  888. 	ALIGN_3

  889. 

  890. .L37:

  891. 	SHUFPD_1 %xmm1, %xmm1

  892. 	SHUFPD_1 %xmm3, %xmm3

  893. 

  894. 	testq	$1, N

  895. 	jle	.L98

  896. 

  897. 	MOVLPS	-16 * SIZE(Y), %xmm4

  898. 	movhps	-15 * SIZE(Y), %xmm4

  899. 	movaps	-16 * SIZE(X), %xmm8

  900. 

  901. 	pshufd	$0x4e,  %xmm8, %xmm12

  902. 	mulpd	%xmm4,  %xmm8

  903. 	addpd	%xmm8,  %xmm0

  904. 	mulpd	%xmm4,  %xmm12

  905. 	SHUFPD_1 %xmm12, %xmm12

  906. 	addpd	%xmm12, %xmm1

  907. 	jmp	.L98

  908. 	ALIGN_3

  909. 

  910. .L40:

  911. 	movhps	-16 * SIZE(X), %xmm4

  912. 	addq	$SIZE, X

  913. 	movhps	-16 * SIZE(Y), %xmm8

  914. 	addq	$SIZE, Y

  915. 

  916. 	movq	N,  %rax

  917. 	sarq	$3, %rax

  918. 	jle	.L45

  919. 

  920. 	movaps	-16 * SIZE(X), %xmm5

  921. 	movaps	-16 * SIZE(Y), %xmm9

  922. 	movaps	-14 * SIZE(X), %xmm6

  923. 	movaps	-14 * SIZE(Y), %xmm10

  924. 	movaps	-12 * SIZE(X), %xmm7

  925. 	movaps	-12 * SIZE(Y), %xmm11

  926. 	decq	%rax

  927. 	jle	.L42

  928. 	ALIGN_3

  929. 

  930. .L41:

  931. #ifdef PREFETCH

  932. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  933. #endif

  934. 

  935. 	movsd	%xmm9,  %xmm8

  936. 	pshufd	$0x4e,  %xmm8, %xmm12

  937. 	movsd	%xmm5,  %xmm4

  938. 	mulpd	%xmm4,  %xmm8

  939. 	addpd	%xmm8,  %xmm0

  940. 	movaps	-10 * SIZE(Y), %xmm8

  941. 	mulpd	%xmm4,  %xmm12

  942. 	movaps	-10 * SIZE(X), %xmm4

  943. 	addpd	%xmm12, %xmm1

  944. 

  945. 	movsd	%xmm10, %xmm9

  946. 	pshufd	$0x4e,  %xmm9, %xmm12

  947. 	movsd	%xmm6,  %xmm5

  948. 	mulpd	%xmm5,  %xmm9

  949. 	addpd	%xmm9,  %xmm0

  950.  	movaps	 -8 * SIZE(Y), %xmm9

  951. 	mulpd	%xmm5,  %xmm12

  952. 	movaps	 -8 * SIZE(X), %xmm5

  953. 	addpd	%xmm12, %xmm1

  954. 

  955. #ifdef PREFETCH

  956. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  957. #endif

  958. 

  959. 	movsd	%xmm11, %xmm10

  960. 	pshufd	$0x4e,  %xmm10, %xmm12

  961. 	movsd	%xmm7,  %xmm6

  962. 	mulpd	%xmm6,  %xmm10

  963. 	addpd	%xmm10, %xmm0

  964. 	movaps	 -6 * SIZE(Y), %xmm10

  965. 	mulpd	%xmm6,  %xmm12

  966. 	movaps	 -6 * SIZE(X), %xmm6

  967. 	addpd	%xmm12, %xmm1

  968. 

  969. 	movsd	%xmm8, %xmm11

  970. 	pshufd	$0x4e,  %xmm11, %xmm12

  971. 	movsd	%xmm4,  %xmm7

  972. 	mulpd	%xmm7,  %xmm11

  973. 	addpd	%xmm11,  %xmm0

  974. 	movaps	 -4 * SIZE(Y), %xmm11

  975. 	mulpd	%xmm7,  %xmm12

  976. 	movaps	 -4 * SIZE(X), %xmm7

  977. 	addpd	%xmm12, %xmm1

  978. 

  979. #if defined(PREFETCH) && !defined(FETCH128)

  980. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  981. #endif

  982. 

  983. 	movsd	%xmm9,  %xmm8

  984. 	pshufd	$0x4e,  %xmm8, %xmm12

  985. 	movsd	%xmm5,  %xmm4

  986. 	mulpd	%xmm4,  %xmm8

  987. 	addpd	%xmm8,  %xmm0

  988. 	movaps	 -2 * SIZE(Y), %xmm8

  989. 	mulpd	%xmm4,  %xmm12

  990. 	movaps	 -2 * SIZE(X), %xmm4

  991. 	addpd	%xmm12, %xmm1

  992. 

  993. 	movsd	%xmm10, %xmm9

  994. 	pshufd	$0x4e,  %xmm9, %xmm12

  995. 	movsd	%xmm6,  %xmm5

  996. 	mulpd	%xmm5,  %xmm9

  997. 	addpd	%xmm9,  %xmm0

  998. 	movaps	  0 * SIZE(Y), %xmm9

  999. 	mulpd	%xmm5,  %xmm12

  1000. 	movaps	  0 * SIZE(X), %xmm5

  1001. 	addpd	%xmm12, %xmm1

  1002. 

  1003. #if defined(PREFETCH) && !defined(FETCH128)

  1004. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  1005. #endif

  1006. 

  1007. 	movsd	%xmm11, %xmm10

  1008. 	pshufd	$0x4e,  %xmm10, %xmm12

  1009. 	movsd	%xmm7,  %xmm6

  1010. 	mulpd	%xmm6,  %xmm10

  1011. 	addpd	%xmm10, %xmm0

  1012. 	movaps	  2 * SIZE(Y), %xmm10

  1013. 	mulpd	%xmm6,  %xmm12

  1014. 	movaps	  2 * SIZE(X), %xmm6

  1015. 	addpd	%xmm12, %xmm1

  1016. 

  1017. 	movsd	%xmm8, %xmm11

  1018. 	pshufd	$0x4e,  %xmm11, %xmm12

  1019. 	movsd	%xmm4,  %xmm7

  1020. 	mulpd	%xmm7,  %xmm11

  1021. 	addpd	%xmm11,  %xmm0

  1022. 	movaps	  4 * SIZE(Y), %xmm11

  1023. 	mulpd	%xmm7,  %xmm12

  1024. 	movaps	  4 * SIZE(X), %xmm7

  1025. 	addpd	%xmm12, %xmm1

  1026. 

  1027. 	subq	$-16 * SIZE, X

  1028. 	subq	$-16 * SIZE, Y

  1029. 

  1030. 	decq	%rax

  1031. 	jg	.L41

  1032. 	ALIGN_3

  1033. 

  1034. .L42:

  1035. 	movsd	%xmm9,  %xmm8

  1036. 	pshufd	$0x4e,  %xmm8, %xmm12

  1037. 	movsd	%xmm5,  %xmm4

  1038. 	mulpd	%xmm4,  %xmm8

  1039. 	addpd	%xmm8,  %xmm0

  1040. 	movaps	-10 * SIZE(Y), %xmm8

  1041. 	mulpd	%xmm4,  %xmm12

  1042. 	movaps	-10 * SIZE(X), %xmm4

  1043. 	addpd	%xmm12, %xmm1

  1044. 

  1045. 	movsd	%xmm10, %xmm9

  1046. 	pshufd	$0x4e,  %xmm9, %xmm12

  1047. 	movsd	%xmm6,  %xmm5

  1048. 	mulpd	%xmm5,  %xmm9

  1049. 	addpd	%xmm9,  %xmm0

  1050.  	movaps	 -8 * SIZE(Y), %xmm9

  1051. 	mulpd	%xmm5,  %xmm12

  1052. 	movaps	 -8 * SIZE(X), %xmm5

  1053. 	addpd	%xmm12, %xmm1

  1054. 

  1055. 	movsd	%xmm11, %xmm10

  1056. 	pshufd	$0x4e,  %xmm10, %xmm12

  1057. 	movsd	%xmm7,  %xmm6

  1058. 	mulpd	%xmm6,  %xmm10

  1059. 	addpd	%xmm10, %xmm0

  1060. 	movaps	 -6 * SIZE(Y), %xmm10

  1061. 	mulpd	%xmm6,  %xmm12

  1062. 	movaps	 -6 * SIZE(X), %xmm6

  1063. 	addpd	%xmm12, %xmm1

  1064. 

  1065. 	movsd	%xmm8, %xmm11

  1066. 	pshufd	$0x4e,  %xmm11, %xmm12

  1067. 	movsd	%xmm4,  %xmm7

  1068. 	mulpd	%xmm7,  %xmm11

  1069. 	addpd	%xmm11,  %xmm0

  1070. 	movaps	 -4 * SIZE(Y), %xmm11

  1071. 	mulpd	%xmm7,  %xmm12

  1072. 	movaps	 -4 * SIZE(X), %xmm7

  1073. 	addpd	%xmm12, %xmm1

  1074. 

  1075. 	movsd	%xmm9,  %xmm8

  1076. 	pshufd	$0x4e,  %xmm8, %xmm12

  1077. 	movsd	%xmm5,  %xmm4

  1078. 	mulpd	%xmm4,  %xmm8

  1079. 	addpd	%xmm8,  %xmm0

  1080. 	movaps	 -2 * SIZE(Y), %xmm8

  1081. 	mulpd	%xmm4,  %xmm12

  1082. 	movaps	 -2 * SIZE(X), %xmm4

  1083. 	addpd	%xmm12, %xmm1

  1084. 

  1085. 	movsd	%xmm10, %xmm9

  1086. 	pshufd	$0x4e,  %xmm9, %xmm12

  1087. 	movsd	%xmm6,  %xmm5

  1088. 	mulpd	%xmm5,  %xmm9

  1089. 	addpd	%xmm9,  %xmm0

  1090. 	mulpd	%xmm5,  %xmm12

  1091. 	addpd	%xmm12, %xmm1

  1092. 

  1093. 	movsd	%xmm11, %xmm10

  1094. 	pshufd	$0x4e,  %xmm10, %xmm12

  1095. 	movsd	%xmm7,  %xmm6

  1096. 	mulpd	%xmm6,  %xmm10

  1097. 	addpd	%xmm10, %xmm0

  1098. 	mulpd	%xmm6,  %xmm12

  1099. 	addpd	%xmm12, %xmm1

  1100. 

  1101. 	movsd	%xmm8, %xmm11

  1102. 	pshufd	$0x4e,  %xmm11, %xmm12

  1103. 	movsd	%xmm4,  %xmm7

  1104. 	mulpd	%xmm7,  %xmm11

  1105. 	addpd	%xmm11,  %xmm0

  1106. 	mulpd	%xmm7,  %xmm12

  1107. 	addpd	%xmm12, %xmm1

  1108. 

  1109. 	subq	$-16 * SIZE, X

  1110. 	subq	$-16 * SIZE, Y

  1111. 	ALIGN_3

  1112. 

  1113. .L45:

  1114. 	testq	$4, N

  1115. 	jle	.L46

  1116. 

  1117. 	movaps	-16 * SIZE(X), %xmm5

  1118. 	movaps	-16 * SIZE(Y), %xmm9

  1119. 	movaps	-14 * SIZE(X), %xmm6

  1120. 	movaps	-14 * SIZE(Y), %xmm10

  1121. 

  1122. 	movsd	%xmm9,  %xmm8

  1123. 	pshufd	$0x4e,  %xmm8, %xmm12

  1124. 	movsd	%xmm5,  %xmm4

  1125. 	mulpd	%xmm4,  %xmm8

  1126. 	addpd	%xmm8,  %xmm0

  1127. 	mulpd	%xmm4,  %xmm12

  1128. 	addpd	%xmm12, %xmm1

  1129. 

  1130. 	movaps	-12 * SIZE(X), %xmm7

  1131. 	movaps	-12 * SIZE(Y), %xmm11

  1132. 

  1133. 	movsd	%xmm10, %xmm9

  1134. 	pshufd	$0x4e,  %xmm9, %xmm12

  1135. 	movsd	%xmm6,  %xmm5

  1136. 	mulpd	%xmm5,  %xmm9

  1137. 	addpd	%xmm9,  %xmm0

  1138. 	mulpd	%xmm5,  %xmm12

  1139. 	addpd	%xmm12, %xmm1

  1140. 

  1141. 	movaps	-10 * SIZE(X), %xmm4

  1142. 	movaps	-10 * SIZE(Y), %xmm8

  1143. 

  1144. 	movsd	%xmm11, %xmm10

  1145. 	pshufd	$0x4e,  %xmm10, %xmm12

  1146. 	movsd	%xmm7,  %xmm6

  1147. 	mulpd	%xmm6,  %xmm10

  1148. 	addpd	%xmm10, %xmm0

  1149. 	mulpd	%xmm6,  %xmm12

  1150. 	addpd	%xmm12, %xmm1

  1151. 

  1152. 	movsd	%xmm8, %xmm11

  1153. 	pshufd	$0x4e,  %xmm11, %xmm12

  1154. 	movsd	%xmm4,  %xmm7

  1155. 	mulpd	%xmm7,  %xmm11

  1156. 	addpd	%xmm11,  %xmm0

  1157. 	mulpd	%xmm7,  %xmm12

  1158. 	addpd	%xmm12, %xmm1

  1159. 

  1160. 	addq	$8 * SIZE, X

  1161. 	addq	$8 * SIZE, Y

  1162. 	ALIGN_3

  1163. 

  1164. .L46:

  1165. 	testq	$2, N

  1166. 	jle	.L47

  1167. 

  1168. 	movaps	-16 * SIZE(X), %xmm5

  1169. 	movaps	-16 * SIZE(Y), %xmm9

  1170. 

  1171. 	movsd	%xmm9,  %xmm8

  1172. 	pshufd	$0x4e,  %xmm8, %xmm12

  1173. 	movsd	%xmm5,  %xmm4

  1174. 	mulpd	%xmm4,  %xmm8

  1175. 	addpd	%xmm8,  %xmm0

  1176. 	mulpd	%xmm4,  %xmm12

  1177. 	addpd	%xmm12, %xmm1

  1178. 

  1179. 	movaps	-14 * SIZE(X), %xmm6

  1180. 	movaps	-14 * SIZE(Y), %xmm10

  1181. 

  1182. 	movsd	%xmm10, %xmm9

  1183. 	pshufd	$0x4e,  %xmm9, %xmm12

  1184. 	movsd	%xmm6,  %xmm5

  1185. 	mulpd	%xmm5,  %xmm9

  1186. 	addpd	%xmm9,  %xmm0

  1187. 	mulpd	%xmm5,  %xmm12

  1188. 	addpd	%xmm12, %xmm1

  1189. 

  1190. 	movaps	%xmm6,  %xmm4

  1191. 	movaps	%xmm10, %xmm8

  1192. 

  1193. 	addq	$4 * SIZE, X

  1194. 	addq	$4 * SIZE, Y

  1195. 	ALIGN_3

  1196. 

  1197. .L47:

  1198. 	testq	$1, N

  1199. 	jle	.L48

  1200. 

  1201. 	movlps	-16 * SIZE(X), %xmm4

  1202. 	movlps	-16 * SIZE(Y), %xmm8

  1203. 

  1204. 	pshufd	$0x4e,  %xmm8, %xmm12

  1205. 	mulpd	%xmm4,  %xmm8

  1206. 	addpd	%xmm8,  %xmm0

  1207. 	mulpd	%xmm4,  %xmm12

  1208. 	addpd	%xmm12, %xmm1

  1209. 	ALIGN_3

  1210. 

  1211. .L48:

  1212. 	SHUFPD_1 %xmm0, %xmm0

  1213. 	SHUFPD_1 %xmm1, %xmm1

  1214. 	SHUFPD_1 %xmm2, %xmm2

  1215. 	SHUFPD_1 %xmm3, %xmm3

  1216. 	jmp	.L98

  1217. 	ALIGN_3

  1218. 

  1219. .L50:

  1220. 	movq	N,  %rax

  1221. 	sarq	$3, %rax

  1222. 	jle	.L55

  1223. 

  1224. 	MOVLPS	0 * SIZE(X), %xmm4

  1225. 	movhps	1 * SIZE(X), %xmm4

  1226. 	addq	INCX, X

  1227. 	MOVLPS	0 * SIZE(Y), %xmm8

  1228. 	movhps	1 * SIZE(Y), %xmm8

  1229. 	addq	INCY, Y

  1230. 

  1231. 	MOVLPS	0 * SIZE(X), %xmm5

  1232. 	movhps	1 * SIZE(X), %xmm5

  1233. 	addq	INCX, X

  1234. 	MOVLPS	0 * SIZE(Y), %xmm9

  1235. 	movhps	1 * SIZE(Y), %xmm9

  1236. 	addq	INCY, Y

  1237. 

  1238. 	MOVLPS	0 * SIZE(X), %xmm6

  1239. 	movhps	1 * SIZE(X), %xmm6

  1240. 	addq	INCX, X

  1241. 	MOVLPS	0 * SIZE(Y), %xmm10

  1242. 	movhps	1 * SIZE(Y), %xmm10

  1243. 	addq	INCY, Y

  1244. 

  1245. 	MOVLPS	0 * SIZE(X), %xmm7

  1246. 	movhps	1 * SIZE(X), %xmm7

  1247. 	addq	INCX, X

  1248. 	MOVLPS	0 * SIZE(Y), %xmm11

  1249. 	movhps	1 * SIZE(Y), %xmm11

  1250. 	addq	INCY, Y

  1251. 

  1252. 	decq	%rax

  1253. 	jle	.L54

  1254. 	ALIGN_3

  1255. 

  1256. .L53:

  1257. 	pshufd	$0x4e,  %xmm8, %xmm12

  1258. 	mulpd	%xmm4,  %xmm8

  1259. 	addpd	%xmm8,  %xmm0

  1260. 	MOVLPS	0 * SIZE(Y), %xmm8

  1261. 	movhps	1 * SIZE(Y), %xmm8

  1262. 	addq	INCY, Y

  1263. 	mulpd	%xmm4,  %xmm12

  1264. 	MOVLPS	0 * SIZE(X), %xmm4

  1265. 	movhps	1 * SIZE(X), %xmm4

  1266. 	addq	INCX, X

  1267. 	addpd	%xmm12, %xmm1

  1268. 

  1269. 	pshufd	$0x4e,  %xmm9, %xmm12

  1270. 	mulpd	%xmm5,  %xmm9

  1271. 	addpd	%xmm9,  %xmm2

  1272. 	MOVLPS	0 * SIZE(Y), %xmm9

  1273. 	movhps	1 * SIZE(Y), %xmm9

  1274. 	addq	INCY, Y

  1275. 	mulpd	%xmm5,  %xmm12

  1276. 	MOVLPS	0 * SIZE(X), %xmm5

  1277. 	movhps	1 * SIZE(X), %xmm5

  1278. 	addq	INCX, X

  1279. 	addpd	%xmm12, %xmm3

  1280. 

  1281. 	pshufd	$0x4e,  %xmm10, %xmm12

  1282. 	mulpd	%xmm6,  %xmm10

  1283. 	addpd	%xmm10, %xmm0

  1284. 	MOVLPS	0 * SIZE(Y), %xmm10

  1285. 	movhps	1 * SIZE(Y), %xmm10

  1286. 	addq	INCY, Y

  1287. 	mulpd	%xmm6,  %xmm12

  1288. 	MOVLPS	0 * SIZE(X), %xmm6

  1289. 	movhps	1 * SIZE(X), %xmm6

  1290. 	addq	INCX, X

  1291. 	addpd	%xmm12, %xmm1

  1292. 

  1293. 	pshufd	$0x4e,  %xmm11, %xmm12

  1294. 	mulpd	%xmm7,  %xmm11

  1295. 	addpd	%xmm11, %xmm2

  1296. 	MOVLPS	0 * SIZE(Y), %xmm11

  1297. 	movhps	1 * SIZE(Y), %xmm11

  1298. 	addq	INCY, Y

  1299. 	mulpd	%xmm7,  %xmm12

  1300. 	MOVLPS	0 * SIZE(X), %xmm7

  1301. 	movhps	1 * SIZE(X), %xmm7

  1302. 	addq	INCX, X

  1303. 	addpd	%xmm12, %xmm3

  1304. 

  1305. 	pshufd	$0x4e,  %xmm8, %xmm12

  1306. 	mulpd	%xmm4,  %xmm8

  1307. 	addpd	%xmm8,  %xmm0

  1308. 	MOVLPS	0 * SIZE(Y), %xmm8

  1309. 	movhps	1 * SIZE(Y), %xmm8

  1310. 	addq	INCY, Y

  1311. 

  1312. 	mulpd	%xmm4,  %xmm12

  1313. 	MOVLPS	0 * SIZE(X), %xmm4

  1314. 	movhps	1 * SIZE(X), %xmm4

  1315. 	addq	INCX, X

  1316. 	addpd	%xmm12, %xmm1

  1317. 

  1318. 	pshufd	$0x4e,  %xmm9, %xmm12

  1319. 	mulpd	%xmm5,  %xmm9

  1320. 	addpd	%xmm9,  %xmm2

  1321. 	MOVLPS	0 * SIZE(Y), %xmm9

  1322. 	movhps	1 * SIZE(Y), %xmm9

  1323. 	addq	INCY, Y

  1324. 

  1325. 	mulpd	%xmm5,  %xmm12

  1326. 	MOVLPS	0 * SIZE(X), %xmm5

  1327. 	movhps	1 * SIZE(X), %xmm5

  1328. 	addq	INCX, X

  1329. 	addpd	%xmm12, %xmm3

  1330. 

  1331. 	pshufd	$0x4e,  %xmm10, %xmm12

  1332. 	mulpd	%xmm6,  %xmm10

  1333. 	addpd	%xmm10, %xmm0

  1334. 	MOVLPS	0 * SIZE(Y), %xmm10

  1335. 	movhps	1 * SIZE(Y), %xmm10

  1336. 	addq	INCY, Y

  1337. 	mulpd	%xmm6,  %xmm12

  1338. 	MOVLPS	0 * SIZE(X), %xmm6

  1339. 	movhps	1 * SIZE(X), %xmm6

  1340. 	addq	INCX, X

  1341. 	addpd	%xmm12, %xmm1

  1342. 

  1343. 	pshufd	$0x4e,  %xmm11, %xmm12

  1344. 	mulpd	%xmm7,  %xmm11

  1345. 	addpd	%xmm11, %xmm2

  1346. 	MOVLPS	0 * SIZE(Y), %xmm11

  1347. 	movhps	1 * SIZE(Y), %xmm11

  1348. 	addq	INCY, Y

  1349. 	mulpd	%xmm7,  %xmm12

  1350. 	MOVLPS	0 * SIZE(X), %xmm7

  1351. 	movhps	1 * SIZE(X), %xmm7

  1352. 	addq	INCX, X

  1353. 	addpd	%xmm12, %xmm3

  1354. 

  1355. 	decq	%rax

  1356. 	jg	.L53

  1357. 	ALIGN_3

  1358. 

  1359. .L54:

  1360. 	pshufd	$0x4e,  %xmm8, %xmm12

  1361. 	mulpd	%xmm4,  %xmm8

  1362. 	addpd	%xmm8,  %xmm0

  1363. 	MOVLPS	0 * SIZE(Y), %xmm8

  1364. 	movhps	1 * SIZE(Y), %xmm8

  1365. 	addq	INCY, Y

  1366. 	mulpd	%xmm4,  %xmm12

  1367. 	MOVLPS	0 * SIZE(X), %xmm4

  1368. 	movhps	1 * SIZE(X), %xmm4

  1369. 	addq	INCX, X

  1370. 	addpd	%xmm12, %xmm1

  1371. 

  1372. 	pshufd	$0x4e,  %xmm9, %xmm12

  1373. 	mulpd	%xmm5,  %xmm9

  1374. 	addpd	%xmm9,  %xmm2

  1375. 	MOVLPS	0 * SIZE(Y), %xmm9

  1376. 	movhps	1 * SIZE(Y), %xmm9

  1377. 	addq	INCY, Y

  1378. 	mulpd	%xmm5,  %xmm12

  1379. 	MOVLPS	0 * SIZE(X), %xmm5

  1380. 	movhps	1 * SIZE(X), %xmm5

  1381. 	addq	INCX, X

  1382. 	addpd	%xmm12, %xmm3

  1383. 

  1384. 	pshufd	$0x4e,  %xmm10, %xmm12

  1385. 	mulpd	%xmm6,  %xmm10

  1386. 	addpd	%xmm10, %xmm0

  1387. 	MOVLPS	0 * SIZE(Y), %xmm10

  1388. 	movhps	1 * SIZE(Y), %xmm10

  1389. 	addq	INCY, Y

  1390. 	mulpd	%xmm6,  %xmm12

  1391. 	MOVLPS	0 * SIZE(X), %xmm6

  1392. 	movhps	1 * SIZE(X), %xmm6

  1393. 	addq	INCX, X

  1394. 	addpd	%xmm12, %xmm1

  1395. 

  1396. 	pshufd	$0x4e,  %xmm11, %xmm12

  1397. 	mulpd	%xmm7,  %xmm11

  1398. 	addpd	%xmm11, %xmm2

  1399. 	MOVLPS	0 * SIZE(Y), %xmm11

  1400. 	movhps	1 * SIZE(Y), %xmm11

  1401. 	addq	INCY, Y

  1402. 	mulpd	%xmm7,  %xmm12

  1403. 	MOVLPS	0 * SIZE(X), %xmm7

  1404. 	movhps	1 * SIZE(X), %xmm7

  1405. 	addq	INCX, X

  1406. 	addpd	%xmm12, %xmm3

  1407. 

  1408. 	pshufd	$0x4e,  %xmm8, %xmm12

  1409. 	mulpd	%xmm4,  %xmm8

  1410. 	addpd	%xmm8,  %xmm0

  1411. 	mulpd	%xmm4,  %xmm12

  1412. 	addpd	%xmm12, %xmm1

  1413. 

  1414. 	pshufd	$0x4e,  %xmm9, %xmm12

  1415. 	mulpd	%xmm5,  %xmm9

  1416. 	addpd	%xmm9,  %xmm2

  1417. 	mulpd	%xmm5,  %xmm12

  1418. 	addpd	%xmm12, %xmm3

  1419. 

  1420. 	pshufd	$0x4e,  %xmm10, %xmm12

  1421. 	mulpd	%xmm6,  %xmm10

  1422. 	addpd	%xmm10, %xmm0

  1423. 	mulpd	%xmm6,  %xmm12

  1424. 	addpd	%xmm12, %xmm1

  1425. 

  1426. 	pshufd	$0x4e,  %xmm11, %xmm12

  1427. 	mulpd	%xmm7,  %xmm11

  1428. 	addpd	%xmm11, %xmm2

  1429. 	mulpd	%xmm7,  %xmm12

  1430. 	addpd	%xmm12, %xmm3

  1431. 	ALIGN_3

  1432. 

  1433. .L55:

  1434. 	testq	$4, N

  1435. 	jle	.L56

  1436. 

  1437. 	MOVLPS	0 * SIZE(X), %xmm4

  1438. 	movhps	1 * SIZE(X), %xmm4

  1439. 	addq	INCX, X

  1440. 	MOVLPS	0 * SIZE(Y), %xmm8

  1441. 	movhps	1 * SIZE(Y), %xmm8

  1442. 	addq	INCY, Y

  1443. 

  1444. 	pshufd	$0x4e,  %xmm8, %xmm12

  1445. 	mulpd	%xmm4,  %xmm8

  1446. 	addpd	%xmm8,  %xmm0

  1447. 	mulpd	%xmm4,  %xmm12

  1448. 	addpd	%xmm12, %xmm1

  1449. 

  1450. 	MOVLPS	0 * SIZE(X), %xmm5

  1451. 	movhps	1 * SIZE(X), %xmm5

  1452. 	addq	INCX, X

  1453. 	MOVLPS	0 * SIZE(Y), %xmm9

  1454. 	movhps	1 * SIZE(Y), %xmm9

  1455. 	addq	INCY, Y

  1456. 

  1457. 	pshufd	$0x4e,  %xmm9, %xmm12

  1458. 	mulpd	%xmm5,  %xmm9

  1459. 	addpd	%xmm9,  %xmm2

  1460. 	mulpd	%xmm5,  %xmm12

  1461. 	addpd	%xmm12, %xmm3

  1462. 

  1463. 	MOVLPS	0 * SIZE(X), %xmm6

  1464. 	movhps	1 * SIZE(X), %xmm6

  1465. 	addq	INCX, X

  1466. 	MOVLPS	0 * SIZE(Y), %xmm10

  1467. 	movhps	1 * SIZE(Y), %xmm10

  1468. 	addq	INCY, Y

  1469. 

  1470. 	pshufd	$0x4e,  %xmm10, %xmm12

  1471. 	mulpd	%xmm6,  %xmm10

  1472. 	addpd	%xmm10, %xmm0

  1473. 	mulpd	%xmm6,  %xmm12

  1474. 	addpd	%xmm12, %xmm1

  1475. 

  1476. 	MOVLPS	0 * SIZE(X), %xmm7

  1477. 	movhps	1 * SIZE(X), %xmm7

  1478. 	addq	INCX, X

  1479. 	MOVLPS	0 * SIZE(Y), %xmm11

  1480. 	movhps	1 * SIZE(Y), %xmm11

  1481. 	addq	INCY, Y

  1482. 

  1483. 	pshufd	$0x4e,  %xmm11, %xmm12

  1484. 	mulpd	%xmm7,  %xmm11

  1485. 	addpd	%xmm11, %xmm2

  1486. 	mulpd	%xmm7,  %xmm12

  1487. 	addpd	%xmm12, %xmm3

  1488. 	ALIGN_3

  1489. 

  1490. .L56:

  1491. 	testq	$2, N

  1492. 	jle	.L57

  1493. 

  1494. 	MOVLPS	0 * SIZE(X), %xmm4

  1495. 	movhps	1 * SIZE(X), %xmm4

  1496. 	addq	INCX, X

  1497. 	MOVLPS	0 * SIZE(Y), %xmm8

  1498. 	movhps	1 * SIZE(Y), %xmm8

  1499. 	addq	INCY, Y

  1500. 

  1501. 	pshufd	$0x4e,  %xmm8, %xmm12

  1502. 	mulpd	%xmm4,  %xmm8

  1503. 	addpd	%xmm8,  %xmm0

  1504. 	mulpd	%xmm4,  %xmm12

  1505. 	addpd	%xmm12, %xmm1

  1506. 

  1507. 	MOVLPS	0 * SIZE(X), %xmm5

  1508. 	movhps	1 * SIZE(X), %xmm5

  1509. 	addq	INCX, X

  1510. 	MOVLPS	0 * SIZE(Y), %xmm9

  1511. 	movhps	1 * SIZE(Y), %xmm9

  1512. 	addq	INCY, Y

  1513. 

  1514. 	pshufd	$0x4e,  %xmm9, %xmm12

  1515. 	mulpd	%xmm5,  %xmm9

  1516. 	addpd	%xmm9,  %xmm2

  1517. 	mulpd	%xmm5,  %xmm12

  1518. 	addpd	%xmm12, %xmm3

  1519. 	ALIGN_3

  1520. 

  1521. .L57:

  1522. 	testq	$1, N

  1523. 	jle	.L98

  1524. 

  1525. 	MOVLPS	0 * SIZE(X), %xmm4

  1526. 	movhps	1 * SIZE(X), %xmm4

  1527. 	MOVLPS	0 * SIZE(Y), %xmm8

  1528. 	movhps	1 * SIZE(Y), %xmm8

  1529. 

  1530. 	pshufd	$0x4e,  %xmm8, %xmm12

  1531. 	mulpd	%xmm4,  %xmm8

  1532. 	addpd	%xmm8,  %xmm0

  1533. 	mulpd	%xmm4,  %xmm12

  1534. 	addpd	%xmm12, %xmm1

  1535. 	ALIGN_3

  1536. 

  1537. .L98:

  1538. 	addpd	%xmm2, %xmm0

  1539. 	addpd	%xmm3, %xmm1

  1540. 

  1541. 	pshufd	$0x4e, %xmm0, %xmm2

  1542. 	pshufd	$0x4e, %xmm1, %xmm3

  1543. 

  1544. .L999:

  1545. #ifndef CONJ

  1546. 	subsd	 %xmm2, %xmm0

  1547. 	addsd	 %xmm3, %xmm1

  1548. #else

  1549. 	addsd	 %xmm2, %xmm0

  1550. 	subsd	 %xmm3, %xmm1

  1551. #endif

  1552. 

  1553. #ifdef WINDOWS_ABI

  1554. 	movq	RESULT_ADDRESS,	%rax

  1555. 	movsd	%xmm0,	(%rax)

  1556. 	movsd	%xmm1,	8(%rax)

  1557. #endif

  1558. 

  1559. 	RESTOREREGISTERS

  1560. 	ret

  1561. 

  1562. 	EPILOGUE