OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfaaa975e6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfaaa975e6'
${ noResults }
OpenBLAS/kernel/x86/zdot_sse2.S

1557 lines
31 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	12

  43. #define ARGS     0

  44. 	

  45. #define RESULT		 4 + STACK + ARGS(%esp)

  46. #define STACK_N		 8 + STACK + ARGS(%esp)

  47. #define STACK_X		12 + STACK + ARGS(%esp)

  48. #define STACK_INCX	16 + STACK + ARGS(%esp)

  49. #define STACK_Y		20 + STACK + ARGS(%esp)

  50. #define STACK_INCY	24 + STACK + ARGS(%esp)

  51. 

  52. #define N	%ebx

  53. #define X	%esi

  54. #define INCX	%ecx

  55. #define Y	%edi

  56. #define INCY	%edx

  57. 

  58. #include "l1param.h"

  59. 

  60. #undef movsd

  61. 

  62. #ifndef OPTERON

  63. #define MOVLPS	movsd

  64. #else

  65. #define MOVLPS	movlps

  66. #endif

  67. 

  68. 	PROLOGUE

  69. 	PROFCODE

  70. 

  71. 	pushl	%edi

  72. 	pushl	%esi

  73. 	pushl	%ebx

  74. 

  75. 	movl	STACK_N,     N

  76. 	movl	STACK_X,     X

  77. 	movl	STACK_INCX,  INCX

  78. 	movl	STACK_Y,     Y

  79. 	movl	STACK_INCY,  INCY

  80. 

  81. 	sall	$ZBASE_SHIFT, INCX

  82. 	sall	$ZBASE_SHIFT, INCY

  83. 

  84. 	xorps	%xmm0, %xmm0

  85. 	xorps	%xmm1, %xmm1

  86. 

  87. 	cmpl	$0, N

  88. 	jle	.L999

  89. 

  90. 	cmpl	$2 * SIZE, INCX

  91. 	jne	.L50

  92. 	cmpl	$2 * SIZE, INCY

  93. 	jne	.L50

  94. 

  95. 	subl	$-16 * SIZE, X

  96. 	subl	$-16 * SIZE, Y

  97. 

  98. 	testl	$SIZE, Y

  99. 	jne	.L30

  100. 

  101. 	testl	$SIZE, X

  102. 	jne	.L20

  103. 

  104. 	movl	N,  %eax

  105. 	sarl	$3, %eax

  106. 	jle	.L15

  107. 

  108. 	movaps	-16 * SIZE(X), %xmm4

  109. 	movaps	-16 * SIZE(Y), %xmm6

  110. 	movaps	-14 * SIZE(X), %xmm5

  111. 	movaps	-14 * SIZE(Y), %xmm7

  112. 

  113. 	decl	%eax

  114. 	jle	.L12

  115. 	ALIGN_3

  116. 

  117. .L11:

  118. #ifdef PREFETCH

  119. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  120. #endif

  121. 

  122. 	pshufd	$0x4e, %xmm6, %xmm3 

  123. 	mulpd	%xmm4, %xmm6

  124. 	addpd	%xmm6, %xmm0

  125. 	movaps	-12 * SIZE(Y), %xmm6

  126. 	mulpd	%xmm4, %xmm3

  127. 	movaps	-12 * SIZE(X), %xmm4

  128. 	addpd	%xmm3, %xmm1

  129. 

  130. 	pshufd	$0x4e, %xmm7, %xmm3 

  131. 	mulpd	%xmm5, %xmm7

  132. 	addpd	%xmm7, %xmm0

  133. 	movaps	-10 * SIZE(Y), %xmm7

  134. 	mulpd	%xmm5, %xmm3

  135. 	movaps	-10 * SIZE(X), %xmm5

  136. 	addpd	%xmm3, %xmm1

  137. 

  138. #ifdef PREFETCH

  139. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  140. #endif

  141. 

  142. 	pshufd	$0x4e, %xmm6, %xmm3 

  143. 	mulpd	%xmm4, %xmm6

  144. 	addpd	%xmm6, %xmm0

  145. 	movaps	 -8 * SIZE(Y), %xmm6

  146. 	mulpd	%xmm4, %xmm3

  147. 	movaps	 -8 * SIZE(X), %xmm4

  148. 	addpd	%xmm3, %xmm1

  149. 

  150. 	pshufd	$0x4e, %xmm7, %xmm3 

  151. 	mulpd	%xmm5, %xmm7

  152. 	addpd	%xmm7, %xmm0

  153. 	movaps	 -6 * SIZE(Y), %xmm7

  154. 	mulpd	%xmm5, %xmm3

  155. 	movaps	 -6 * SIZE(X), %xmm5

  156. 	addpd	%xmm3, %xmm1

  157. 

  158. #if defined(PREFETCH) && !defined(FETCH128)

  159. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  160. #endif

  161. 

  162. 	pshufd	$0x4e, %xmm6, %xmm3 

  163. 	mulpd	%xmm4, %xmm6

  164. 	addpd	%xmm6, %xmm0

  165. 	movaps	 -4 * SIZE(Y), %xmm6

  166. 	mulpd	%xmm4, %xmm3

  167. 	movaps	 -4 * SIZE(X), %xmm4

  168. 	addpd	%xmm3, %xmm1

  169. 

  170. 	pshufd	$0x4e, %xmm7, %xmm3 

  171. 	mulpd	%xmm5, %xmm7

  172. 	addpd	%xmm7, %xmm0

  173. 	movaps	 -2 * SIZE(Y), %xmm7

  174. 	mulpd	%xmm5, %xmm3

  175. 	movaps	 -2 * SIZE(X), %xmm5

  176. 	addpd	%xmm3, %xmm1

  177. 

  178. #if defined(PREFETCH) && !defined(FETCH128)

  179. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  180. #endif

  181. 

  182. 	pshufd	$0x4e, %xmm6, %xmm3 

  183. 	mulpd	%xmm4, %xmm6

  184. 	addpd	%xmm6, %xmm0

  185. 	movaps	  0 * SIZE(Y), %xmm6

  186. 	mulpd	%xmm4, %xmm3

  187. 	movaps	  0 * SIZE(X), %xmm4

  188. 	addpd	%xmm3, %xmm1

  189. 

  190. 	pshufd	$0x4e, %xmm7, %xmm3 

  191. 	mulpd	%xmm5, %xmm7

  192. 	addpd	%xmm7, %xmm0

  193. 	movaps	  2 * SIZE(Y), %xmm7

  194. 	mulpd	%xmm5, %xmm3

  195. 	movaps	  2 * SIZE(X), %xmm5

  196. 	addpd	%xmm3, %xmm1

  197. 

  198. 	subl	$-16 * SIZE, X

  199. 	subl	$-16 * SIZE, Y

  200. 

  201. 	decl	%eax

  202. 	jg	.L11

  203. 	ALIGN_3

  204. 

  205. .L12:

  206. 	pshufd	$0x4e, %xmm6, %xmm3 

  207. 	mulpd	%xmm4, %xmm6

  208. 	addpd	%xmm6, %xmm0

  209. 	movaps	-12 * SIZE(Y), %xmm6

  210. 	mulpd	%xmm4, %xmm3

  211. 	movaps	-12 * SIZE(X), %xmm4

  212. 	addpd	%xmm3, %xmm1

  213. 

  214. 	pshufd	$0x4e, %xmm7, %xmm3 

  215. 	mulpd	%xmm5, %xmm7

  216. 	addpd	%xmm7, %xmm0

  217. 	movaps	-10 * SIZE(Y), %xmm7

  218. 	mulpd	%xmm5, %xmm3

  219. 	movaps	-10 * SIZE(X), %xmm5

  220. 	addpd	%xmm3, %xmm1

  221. 

  222. 	pshufd	$0x4e, %xmm6, %xmm3 

  223. 	mulpd	%xmm4, %xmm6

  224. 	addpd	%xmm6, %xmm0

  225. 	movaps	 -8 * SIZE(Y), %xmm6

  226. 	mulpd	%xmm4, %xmm3

  227. 	movaps	 -8 * SIZE(X), %xmm4

  228. 	addpd	%xmm3, %xmm1

  229. 

  230. 	pshufd	$0x4e, %xmm7, %xmm3 

  231. 	mulpd	%xmm5, %xmm7

  232. 	addpd	%xmm7, %xmm0

  233. 	movaps	 -6 * SIZE(Y), %xmm7

  234. 	mulpd	%xmm5, %xmm3

  235. 	movaps	 -6 * SIZE(X), %xmm5

  236. 	addpd	%xmm3, %xmm1

  237. 

  238. 	pshufd	$0x4e, %xmm6, %xmm3 

  239. 	mulpd	%xmm4, %xmm6

  240. 	addpd	%xmm6, %xmm0

  241. 	movaps	 -4 * SIZE(Y), %xmm6

  242. 	mulpd	%xmm4, %xmm3

  243. 	movaps	 -4 * SIZE(X), %xmm4

  244. 	addpd	%xmm3, %xmm1

  245. 

  246. 	pshufd	$0x4e, %xmm7, %xmm3 

  247. 	mulpd	%xmm5, %xmm7

  248. 	addpd	%xmm7, %xmm0

  249. 	movaps	 -2 * SIZE(Y), %xmm7

  250. 	mulpd	%xmm5, %xmm3

  251. 	movaps	 -2 * SIZE(X), %xmm5

  252. 	addpd	%xmm3, %xmm1

  253. 

  254. 	pshufd	$0x4e, %xmm6, %xmm3 

  255. 	mulpd	%xmm4, %xmm6

  256. 	addpd	%xmm6, %xmm0

  257. 	mulpd	%xmm4, %xmm3

  258. 	addpd	%xmm3, %xmm1

  259. 

  260. 	pshufd	$0x4e, %xmm7, %xmm3 

  261. 	mulpd	%xmm5, %xmm7

  262. 	addpd	%xmm7, %xmm0

  263. 	mulpd	%xmm5, %xmm3

  264. 	addpd	%xmm3, %xmm1

  265. 

  266. 	subl	$-16 * SIZE, X

  267. 	subl	$-16 * SIZE, Y

  268. 	ALIGN_3

  269. 

  270. .L15:

  271. 	testl	$4, N

  272. 	jle	.L16

  273. 

  274. 	movaps	-16 * SIZE(X), %xmm4

  275. 	movaps	-16 * SIZE(Y), %xmm6

  276. 	movaps	-14 * SIZE(X), %xmm5

  277. 	movaps	-14 * SIZE(Y), %xmm7

  278. 

  279. 	pshufd	$0x4e, %xmm6, %xmm3 

  280. 	mulpd	%xmm4, %xmm6

  281. 	addpd	%xmm6, %xmm0

  282. 	movaps	-12 * SIZE(Y), %xmm6

  283. 	mulpd	%xmm4, %xmm3

  284. 	movaps	-12 * SIZE(X), %xmm4

  285. 	addpd	%xmm3, %xmm1

  286. 

  287. 	pshufd	$0x4e, %xmm7, %xmm3 

  288. 	mulpd	%xmm5, %xmm7

  289. 	addpd	%xmm7, %xmm0

  290. 	movaps	-10 * SIZE(Y), %xmm7

  291. 	mulpd	%xmm5, %xmm3

  292. 	movaps	-10 * SIZE(X), %xmm5

  293. 	addpd	%xmm3, %xmm1

  294. 

  295. 	pshufd	$0x4e, %xmm6, %xmm3 

  296. 	mulpd	%xmm4, %xmm6

  297. 	addpd	%xmm6, %xmm0

  298. 	mulpd	%xmm4, %xmm3

  299. 	addpd	%xmm3, %xmm1

  300. 

  301. 	pshufd	$0x4e, %xmm7, %xmm3 

  302. 	mulpd	%xmm5, %xmm7

  303. 	addpd	%xmm7, %xmm0

  304. 	mulpd	%xmm5, %xmm3

  305. 	addpd	%xmm3, %xmm1

  306. 

  307. 	addl	$8 * SIZE, X

  308. 	addl	$8 * SIZE, Y

  309. 	ALIGN_3

  310. 

  311. .L16:

  312. 	testl	$2, N

  313. 	jle	.L17

  314. 

  315. 	movaps	-16 * SIZE(X), %xmm4

  316. 	movaps	-16 * SIZE(Y), %xmm6

  317. 	movaps	-14 * SIZE(X), %xmm5

  318. 	movaps	-14 * SIZE(Y), %xmm7

  319. 

  320. 	pshufd	$0x4e, %xmm6, %xmm3 

  321. 	mulpd	%xmm4, %xmm6

  322. 	addpd	%xmm6, %xmm0

  323. 	mulpd	%xmm4, %xmm3

  324. 	addpd	%xmm3, %xmm1

  325. 

  326. 	pshufd	$0x4e, %xmm7, %xmm3 

  327. 	mulpd	%xmm5, %xmm7

  328. 	addpd	%xmm7, %xmm0

  329. 	mulpd	%xmm5, %xmm3

  330. 	addpd	%xmm3, %xmm1

  331. 

  332. 	addl	$4 * SIZE, X

  333. 	addl	$4 * SIZE, Y

  334. 	ALIGN_3

  335. 

  336. .L17:

  337. 	testl	$1, N

  338. 	jle	.L98

  339. 

  340. 	movaps	-16 * SIZE(X), %xmm4

  341. 	movaps	-16 * SIZE(Y), %xmm6

  342. 

  343. 	pshufd	$0x4e,  %xmm6, %xmm3 

  344. 	mulpd	%xmm4,  %xmm6

  345. 	addpd	%xmm6,  %xmm0

  346. 	mulpd	%xmm4,  %xmm3

  347. 	addpd	%xmm3, %xmm1

  348. 	jmp	.L98

  349. 	ALIGN_3

  350. 

  351. .L20:

  352. 	movl	N,  %eax

  353. 	sarl	$3, %eax

  354. 	jle	.L25

  355. 

  356. 	MOVLPS	-16 * SIZE(X), %xmm4

  357. 	movhps	-15 * SIZE(X), %xmm4

  358. 	movaps	-16 * SIZE(Y), %xmm6

  359. 

  360. 	MOVLPS	-14 * SIZE(X), %xmm5

  361. 	movhps	-13 * SIZE(X), %xmm5

  362. 	movaps	-14 * SIZE(Y), %xmm7

  363. 

  364. 	decl	%eax

  365. 	jle	.L22

  366. 	ALIGN_3

  367. 

  368. .L21:

  369. #ifdef PREFETCH

  370. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  371. #endif

  372. 

  373. 	pshufd	$0x4e, %xmm6, %xmm3 

  374. 	mulpd	%xmm4, %xmm6

  375. 	addpd	%xmm6, %xmm0

  376. 	movaps	-12 * SIZE(Y), %xmm6

  377. 	mulpd	%xmm4, %xmm3

  378. 	MOVLPS	-12 * SIZE(X), %xmm4

  379. 	movhps	-11 * SIZE(X), %xmm4

  380. 	addpd	%xmm3, %xmm1

  381. 

  382. 	pshufd	$0x4e, %xmm7, %xmm3 

  383. 	mulpd	%xmm5, %xmm7

  384. 	addpd	%xmm7, %xmm0

  385. 	movaps	-10 * SIZE(Y), %xmm7

  386. 	mulpd	%xmm5, %xmm3

  387. 	MOVLPS	-10 * SIZE(X), %xmm5

  388. 	movhps	 -9 * SIZE(X), %xmm5

  389. 	addpd	%xmm3, %xmm1

  390. 

  391. #ifdef PREFETCH

  392. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  393. #endif

  394. 

  395. 	pshufd	$0x4e, %xmm6, %xmm3 

  396. 	mulpd	%xmm4, %xmm6

  397. 	addpd	%xmm6, %xmm0

  398. 	movaps	 -8 * SIZE(Y), %xmm6

  399. 	mulpd	%xmm4, %xmm3

  400. 	MOVLPS	 -8 * SIZE(X), %xmm4

  401. 	movhps	 -7 * SIZE(X), %xmm4

  402. 	addpd	%xmm3, %xmm1

  403. 

  404. 	pshufd	$0x4e, %xmm7, %xmm3 

  405. 	mulpd	%xmm5, %xmm7

  406. 	addpd	%xmm7, %xmm0

  407. 	movaps	 -6 * SIZE(Y), %xmm7

  408. 	mulpd	%xmm5, %xmm3

  409. 	MOVLPS	 -6 * SIZE(X), %xmm5

  410. 	movhps	 -5 * SIZE(X), %xmm5

  411. 	addpd	%xmm3, %xmm1

  412. 

  413. #if defined(PREFETCH) && !defined(FETCH128)

  414. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  415. #endif

  416. 

  417. 	pshufd	$0x4e, %xmm6, %xmm3 

  418. 	mulpd	%xmm4, %xmm6

  419. 	addpd	%xmm6, %xmm0

  420. 	movaps	 -4 * SIZE(Y), %xmm6

  421. 	mulpd	%xmm4, %xmm3

  422. 	MOVLPS	 -4 * SIZE(X), %xmm4

  423. 	movhps	 -3 * SIZE(X), %xmm4

  424. 	addpd	%xmm3, %xmm1

  425. 

  426. 	pshufd	$0x4e, %xmm7, %xmm3 

  427. 	mulpd	%xmm5, %xmm7

  428. 	addpd	%xmm7, %xmm0

  429. 	movaps	 -2 * SIZE(Y), %xmm7

  430. 	mulpd	%xmm5, %xmm3

  431. 	MOVLPS	 -2 * SIZE(X), %xmm5

  432. 	movhps	 -1 * SIZE(X), %xmm5

  433. 	addpd	%xmm3, %xmm1

  434. 

  435. #if defined(PREFETCH) && !defined(FETCH128)

  436. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  437. #endif

  438. 

  439. 	pshufd	$0x4e, %xmm6, %xmm3 

  440. 	mulpd	%xmm4, %xmm6

  441. 	addpd	%xmm6, %xmm0

  442. 	movaps	  0 * SIZE(Y), %xmm6

  443. 	mulpd	%xmm4, %xmm3

  444. 	MOVLPS	  0 * SIZE(X), %xmm4

  445. 	movhps	  1 * SIZE(X), %xmm4

  446. 	addpd	%xmm3, %xmm1

  447. 

  448. 	pshufd	$0x4e, %xmm7, %xmm3 

  449. 	mulpd	%xmm5, %xmm7

  450. 	addpd	%xmm7, %xmm0

  451. 	movaps	  2 * SIZE(Y), %xmm7

  452. 	mulpd	%xmm5, %xmm3

  453. 	MOVLPS	  2 * SIZE(X), %xmm5

  454. 	movhps	  3 * SIZE(X), %xmm5

  455. 	addpd	%xmm3, %xmm1

  456. 

  457. 	subl	$-16 * SIZE, X

  458. 	subl	$-16 * SIZE, Y

  459. 

  460. 	decl	%eax

  461. 	jg	.L21

  462. 	ALIGN_3

  463. 

  464. .L22:

  465. 	pshufd	$0x4e, %xmm6, %xmm3 

  466. 	mulpd	%xmm4, %xmm6

  467. 	addpd	%xmm6, %xmm0

  468. 	movaps	-12 * SIZE(Y), %xmm6

  469. 	mulpd	%xmm4, %xmm3

  470. 	MOVLPS	-12 * SIZE(X), %xmm4

  471. 	movhps	-11 * SIZE(X), %xmm4

  472. 	addpd	%xmm3, %xmm1

  473. 

  474. 	pshufd	$0x4e, %xmm7, %xmm3 

  475. 	mulpd	%xmm5, %xmm7

  476. 	addpd	%xmm7, %xmm0

  477. 	movaps	-10 * SIZE(Y), %xmm7

  478. 	mulpd	%xmm5, %xmm3

  479. 	MOVLPS	-10 * SIZE(X), %xmm5

  480. 	movhps	 -9 * SIZE(X), %xmm5

  481. 	addpd	%xmm3, %xmm1

  482. 

  483. 	pshufd	$0x4e, %xmm6, %xmm3 

  484. 	mulpd	%xmm4, %xmm6

  485. 	addpd	%xmm6, %xmm0

  486. 	movaps	 -8 * SIZE(Y), %xmm6

  487. 	mulpd	%xmm4, %xmm3

  488. 	MOVLPS	 -8 * SIZE(X), %xmm4

  489. 	movhps	 -7 * SIZE(X), %xmm4

  490. 	addpd	%xmm3, %xmm1

  491. 

  492. 	pshufd	$0x4e, %xmm7, %xmm3 

  493. 	mulpd	%xmm5, %xmm7

  494. 	addpd	%xmm7, %xmm0

  495. 	movaps	 -6 * SIZE(Y), %xmm7

  496. 	mulpd	%xmm5, %xmm3

  497. 	MOVLPS	 -6 * SIZE(X), %xmm5

  498. 	movhps	 -5 * SIZE(X), %xmm5

  499. 	addpd	%xmm3, %xmm1

  500. 

  501. 	pshufd	$0x4e, %xmm6, %xmm3 

  502. 	mulpd	%xmm4, %xmm6

  503. 	addpd	%xmm6, %xmm0

  504. 	movaps	 -4 * SIZE(Y), %xmm6

  505. 	mulpd	%xmm4, %xmm3

  506. 	MOVLPS	 -4 * SIZE(X), %xmm4

  507. 	movhps	 -3 * SIZE(X), %xmm4

  508. 	addpd	%xmm3, %xmm1

  509. 

  510. 	pshufd	$0x4e, %xmm7, %xmm3 

  511. 	mulpd	%xmm5, %xmm7

  512. 	addpd	%xmm7, %xmm0

  513. 	movaps	 -2 * SIZE(Y), %xmm7

  514. 	mulpd	%xmm5, %xmm3

  515. 	MOVLPS	 -2 * SIZE(X), %xmm5

  516. 	movhps	 -1 * SIZE(X), %xmm5

  517. 	addpd	%xmm3, %xmm1

  518. 

  519. 	pshufd	$0x4e, %xmm6, %xmm3 

  520. 	mulpd	%xmm4, %xmm6

  521. 	addpd	%xmm6, %xmm0

  522. 	mulpd	%xmm4, %xmm3

  523. 	addpd	%xmm3, %xmm1

  524. 

  525. 	pshufd	$0x4e, %xmm7, %xmm3 

  526. 	mulpd	%xmm5, %xmm7

  527. 	addpd	%xmm7, %xmm0

  528. 	mulpd	%xmm5, %xmm3

  529. 	addpd	%xmm3, %xmm1

  530. 

  531. 	subl	$-16 * SIZE, X

  532. 	subl	$-16 * SIZE, Y

  533. 	ALIGN_3

  534. 

  535. .L25:

  536. 	testl	$4, N

  537. 	jle	.L26

  538. 

  539. 	MOVLPS	-16 * SIZE(X), %xmm4

  540. 	movhps	-15 * SIZE(X), %xmm4

  541. 	movaps	-16 * SIZE(Y), %xmm6

  542. 

  543. 	MOVLPS	-14 * SIZE(X), %xmm5

  544. 	movhps	-13 * SIZE(X), %xmm5

  545. 	movaps	-14 * SIZE(Y), %xmm7

  546. 

  547. 	pshufd	$0x4e, %xmm6, %xmm3 

  548. 	mulpd	%xmm4, %xmm6

  549. 	addpd	%xmm6, %xmm0

  550. 	movaps	-12 * SIZE(Y), %xmm6

  551. 	mulpd	%xmm4, %xmm3

  552. 	MOVLPS	-12 * SIZE(X), %xmm4

  553. 	movhps	-11 * SIZE(X), %xmm4

  554. 	addpd	%xmm3, %xmm1

  555. 

  556. 	pshufd	$0x4e, %xmm7, %xmm3 

  557. 	mulpd	%xmm5, %xmm7

  558. 	addpd	%xmm7, %xmm0

  559. 	movaps	-10 * SIZE(Y), %xmm7

  560. 	mulpd	%xmm5, %xmm3

  561. 	MOVLPS	-10 * SIZE(X), %xmm5

  562. 	movhps	 -9 * SIZE(X), %xmm5

  563. 	addpd	%xmm3, %xmm1

  564. 

  565. 	pshufd	$0x4e, %xmm6, %xmm3 

  566. 	mulpd	%xmm4, %xmm6

  567. 	addpd	%xmm6, %xmm0

  568. 	mulpd	%xmm4, %xmm3

  569. 	addpd	%xmm3, %xmm1

  570. 

  571. 	pshufd	$0x4e, %xmm7, %xmm3 

  572. 	mulpd	%xmm5, %xmm7

  573. 	addpd	%xmm7, %xmm0

  574. 	mulpd	%xmm5, %xmm3

  575. 	addpd	%xmm3, %xmm1

  576. 

  577. 	addl	$8 * SIZE, X

  578. 	addl	$8 * SIZE, Y

  579. 	ALIGN_3

  580. 

  581. .L26:

  582. 	testl	$2, N

  583. 	jle	.L27

  584. 

  585. 	MOVLPS	-16 * SIZE(X), %xmm4

  586. 	movhps	-15 * SIZE(X), %xmm4

  587. 	movaps	-16 * SIZE(Y), %xmm6

  588. 

  589. 	pshufd	$0x4e, %xmm6, %xmm3 

  590. 	mulpd	%xmm4, %xmm6

  591. 	addpd	%xmm6, %xmm0

  592. 	mulpd	%xmm4, %xmm3

  593. 	addpd	%xmm3, %xmm1

  594. 

  595. 	MOVLPS	-14 * SIZE(X), %xmm5

  596. 	movhps	-13 * SIZE(X), %xmm5

  597. 	movaps	-14 * SIZE(Y), %xmm7

  598. 

  599. 	pshufd	$0x4e, %xmm7, %xmm3 

  600. 	mulpd	%xmm5, %xmm7

  601. 	addpd	%xmm7, %xmm0

  602. 	mulpd	%xmm5, %xmm3

  603. 	addpd	%xmm3, %xmm1

  604. 

  605. 	addl	$4 * SIZE, X

  606. 	addl	$4 * SIZE, Y

  607. 	ALIGN_3

  608. 

  609. .L27:

  610. 	testl	$1, N

  611. 	jle	.L98

  612. 

  613. 	MOVLPS	-16 * SIZE(X), %xmm4

  614. 	movhps	-15 * SIZE(X), %xmm4

  615. 	movaps	-16 * SIZE(Y), %xmm6

  616. 

  617. 	pshufd	$0x4e,  %xmm6, %xmm3 

  618. 	mulpd	%xmm4,  %xmm6

  619. 	addpd	%xmm6,  %xmm0

  620. 	mulpd	%xmm4,  %xmm3

  621. 	addpd	%xmm3, %xmm1

  622. 	jmp	.L98

  623. 	ALIGN_3

  624. 

  625. .L30:

  626. 	testl	$SIZE, X

  627. 	jne	.L40

  628. 

  629. 	movl	N,  %eax

  630. 	sarl	$3, %eax

  631. 	jle	.L35

  632. 

  633. 	MOVLPS	-16 * SIZE(Y), %xmm4

  634. 	movhps	-15 * SIZE(Y), %xmm4

  635. 	movaps	-16 * SIZE(X), %xmm6

  636. 

  637. 	MOVLPS	-14 * SIZE(Y), %xmm5

  638. 	movhps	-13 * SIZE(Y), %xmm5

  639. 	movaps	-14 * SIZE(X), %xmm7

  640. 

  641. 	decl	%eax

  642. 	jle	.L32

  643. 	ALIGN_3

  644. 

  645. .L31:

  646. #ifdef PREFETCH

  647. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  648. #endif

  649. 

  650. 	pshufd	$0x4e, %xmm6, %xmm3 

  651. 	mulpd	%xmm4, %xmm6

  652. 	addpd	%xmm6, %xmm0

  653. 	movaps	-12 * SIZE(X), %xmm6

  654. 	mulpd	%xmm4, %xmm3

  655. 	MOVLPS	-12 * SIZE(Y), %xmm4

  656. 	movhps	-11 * SIZE(Y), %xmm4

  657. 	addpd	%xmm3, %xmm1

  658. 

  659. 	pshufd	$0x4e, %xmm7, %xmm3 

  660. 	mulpd	%xmm5, %xmm7

  661. 	addpd	%xmm7, %xmm0

  662. 	movaps	-10 * SIZE(X), %xmm7

  663. 	mulpd	%xmm5, %xmm3

  664. 	MOVLPS	-10 * SIZE(Y), %xmm5

  665. 	movhps	 -9 * SIZE(Y), %xmm5

  666. 	addpd	%xmm3, %xmm1

  667. 

  668. #ifdef PREFETCH

  669. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  670. #endif

  671. 	pshufd	$0x4e, %xmm6, %xmm3 

  672. 	mulpd	%xmm4, %xmm6

  673. 	addpd	%xmm6, %xmm0

  674. 	movaps	 -8 * SIZE(X), %xmm6

  675. 	mulpd	%xmm4, %xmm3

  676. 	MOVLPS	 -8 * SIZE(Y), %xmm4

  677. 	movhps	 -7 * SIZE(Y), %xmm4

  678. 	addpd	%xmm3, %xmm1

  679. 

  680. 	pshufd	$0x4e, %xmm7, %xmm3 

  681. 	mulpd	%xmm5, %xmm7

  682. 	addpd	%xmm7, %xmm0

  683. 	movaps	 -6 * SIZE(X), %xmm7

  684. 	mulpd	%xmm5, %xmm3

  685. 	MOVLPS	 -6 * SIZE(Y), %xmm5

  686. 	movhps	 -5 * SIZE(Y), %xmm5

  687. 	addpd	%xmm3, %xmm1

  688. 

  689. #if defined(PREFETCH) && !defined(FETCH128)

  690. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  691. #endif

  692. 

  693. 	pshufd	$0x4e, %xmm6, %xmm3 

  694. 	mulpd	%xmm4, %xmm6

  695. 	addpd	%xmm6, %xmm0

  696. 	movaps	 -4 * SIZE(X), %xmm6

  697. 	mulpd	%xmm4, %xmm3

  698. 	MOVLPS	 -4 * SIZE(Y), %xmm4

  699. 	movhps	 -3 * SIZE(Y), %xmm4

  700. 	addpd	%xmm3, %xmm1

  701. 

  702. 	pshufd	$0x4e, %xmm7, %xmm3 

  703. 	mulpd	%xmm5, %xmm7

  704. 	addpd	%xmm7, %xmm0

  705. 	movaps	 -2 * SIZE(X), %xmm7

  706. 	mulpd	%xmm5, %xmm3

  707. 	MOVLPS	 -2 * SIZE(Y), %xmm5

  708. 	movhps	 -1 * SIZE(Y), %xmm5

  709. 	addpd	%xmm3, %xmm1

  710. 

  711. #if defined(PREFETCH) && !defined(FETCH128)

  712. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  713. #endif

  714. 

  715. 	pshufd	$0x4e, %xmm6, %xmm3 

  716. 	mulpd	%xmm4, %xmm6

  717. 	addpd	%xmm6, %xmm0

  718. 	movaps	  0 * SIZE(X), %xmm6

  719. 	mulpd	%xmm4, %xmm3

  720. 	MOVLPS	  0 * SIZE(Y), %xmm4

  721. 	movhps	  1 * SIZE(Y), %xmm4

  722. 	addpd	%xmm3, %xmm1

  723. 

  724. 	pshufd	$0x4e, %xmm7, %xmm3 

  725. 	mulpd	%xmm5, %xmm7

  726. 	addpd	%xmm7, %xmm0

  727. 	movaps	  2 * SIZE(X), %xmm7

  728. 	mulpd	%xmm5, %xmm3

  729. 	MOVLPS	  2 * SIZE(Y), %xmm5

  730. 	movhps	  3 * SIZE(Y), %xmm5

  731. 	addpd	%xmm3, %xmm1

  732. 

  733. 	subl	$-16 * SIZE, X

  734. 	subl	$-16 * SIZE, Y

  735. 

  736. 	decl	%eax

  737. 	jg	.L31

  738. 	ALIGN_3

  739. 

  740. .L32:

  741. 	pshufd	$0x4e, %xmm6, %xmm3 

  742. 	mulpd	%xmm4, %xmm6

  743. 	addpd	%xmm6, %xmm0

  744. 	movaps	-12 * SIZE(X), %xmm6

  745. 	mulpd	%xmm4, %xmm3

  746. 	MOVLPS	-12 * SIZE(Y), %xmm4

  747. 	movhps	-11 * SIZE(Y), %xmm4

  748. 	addpd	%xmm3, %xmm1

  749. 

  750. 	pshufd	$0x4e, %xmm7, %xmm3 

  751. 	mulpd	%xmm5, %xmm7

  752. 	addpd	%xmm7, %xmm0

  753. 	movaps	-10 * SIZE(X), %xmm7

  754. 	mulpd	%xmm5, %xmm3

  755. 	MOVLPS	-10 * SIZE(Y), %xmm5

  756. 	movhps	 -9 * SIZE(Y), %xmm5

  757. 	addpd	%xmm3, %xmm1

  758. 

  759. 	pshufd	$0x4e, %xmm6, %xmm3 

  760. 	mulpd	%xmm4, %xmm6

  761. 	addpd	%xmm6, %xmm0

  762. 	movaps	 -8 * SIZE(X), %xmm6

  763. 	mulpd	%xmm4, %xmm3

  764. 	MOVLPS	 -8 * SIZE(Y), %xmm4

  765. 	movhps	 -7 * SIZE(Y), %xmm4

  766. 	addpd	%xmm3, %xmm1

  767. 

  768. 	pshufd	$0x4e, %xmm7, %xmm3 

  769. 	mulpd	%xmm5, %xmm7

  770. 	addpd	%xmm7, %xmm0

  771. 	movaps	 -6 * SIZE(X), %xmm7

  772. 	mulpd	%xmm5, %xmm3

  773. 	MOVLPS	 -6 * SIZE(Y), %xmm5

  774. 	movhps	 -5 * SIZE(Y), %xmm5

  775. 	addpd	%xmm3, %xmm1

  776. 

  777. 	pshufd	$0x4e, %xmm6, %xmm3 

  778. 	mulpd	%xmm4, %xmm6

  779. 	addpd	%xmm6, %xmm0

  780. 	movaps	 -4 * SIZE(X), %xmm6

  781. 	mulpd	%xmm4, %xmm3

  782. 	MOVLPS	 -4 * SIZE(Y), %xmm4

  783. 	movhps	 -3 * SIZE(Y), %xmm4

  784. 	addpd	%xmm3, %xmm1

  785. 

  786. 	pshufd	$0x4e, %xmm7, %xmm3 

  787. 	mulpd	%xmm5, %xmm7

  788. 	addpd	%xmm7, %xmm0

  789. 	movaps	 -2 * SIZE(X), %xmm7

  790. 	mulpd	%xmm5, %xmm3

  791. 	MOVLPS	 -2 * SIZE(Y), %xmm5

  792. 	movhps	 -1 * SIZE(Y), %xmm5

  793. 	addpd	%xmm3, %xmm1

  794. 

  795. 	pshufd	$0x4e, %xmm6, %xmm3 

  796. 	mulpd	%xmm4, %xmm6

  797. 	addpd	%xmm6, %xmm0

  798. 	mulpd	%xmm4, %xmm3

  799. 	addpd	%xmm3, %xmm1

  800. 

  801. 	pshufd	$0x4e, %xmm7, %xmm3 

  802. 	mulpd	%xmm5, %xmm7

  803. 	addpd	%xmm7, %xmm0

  804. 	mulpd	%xmm5, %xmm3

  805. 	addpd	%xmm3, %xmm1

  806. 

  807. 	subl	$-16 * SIZE, X

  808. 	subl	$-16 * SIZE, Y

  809. 	ALIGN_3

  810. 

  811. .L35:

  812. 	testl	$4, N

  813. 	jle	.L36

  814. 

  815. 	MOVLPS	-16 * SIZE(Y), %xmm4

  816. 	movhps	-15 * SIZE(Y), %xmm4

  817. 	movaps	-16 * SIZE(X), %xmm6

  818. 

  819. 	MOVLPS	-14 * SIZE(Y), %xmm5

  820. 	movhps	-13 * SIZE(Y), %xmm5

  821. 	movaps	-14 * SIZE(X), %xmm7

  822. 

  823. 	pshufd	$0x4e, %xmm6, %xmm3 

  824. 	mulpd	%xmm4, %xmm6

  825. 	addpd	%xmm6, %xmm0

  826. 	movaps	-12 * SIZE(X), %xmm6

  827. 	mulpd	%xmm4, %xmm3

  828. 	MOVLPS	-12 * SIZE(Y), %xmm4

  829. 	movhps	-11 * SIZE(Y), %xmm4

  830. 	addpd	%xmm3, %xmm1

  831. 

  832. 	pshufd	$0x4e, %xmm7, %xmm3 

  833. 	mulpd	%xmm5, %xmm7

  834. 	addpd	%xmm7, %xmm0

  835. 	movaps	-10 * SIZE(X), %xmm7

  836. 	mulpd	%xmm5, %xmm3

  837. 	MOVLPS	-10 * SIZE(Y), %xmm5

  838. 	movhps	 -9 * SIZE(Y), %xmm5

  839. 	addpd	%xmm3, %xmm1

  840. 

  841. 	pshufd	$0x4e, %xmm6, %xmm3 

  842. 	mulpd	%xmm4, %xmm6

  843. 	addpd	%xmm6, %xmm0

  844. 	mulpd	%xmm4, %xmm3

  845. 	addpd	%xmm3, %xmm1

  846. 

  847. 	pshufd	$0x4e, %xmm7, %xmm3 

  848. 	mulpd	%xmm5, %xmm7

  849. 	addpd	%xmm7, %xmm0

  850. 	mulpd	%xmm5, %xmm3

  851. 	addpd	%xmm3, %xmm1

  852. 

  853. 	addl	$8 * SIZE, X

  854. 	addl	$8 * SIZE, Y

  855. 	ALIGN_3

  856. 

  857. .L36:

  858. 	testl	$2, N

  859. 	jle	.L37

  860. 

  861. 	MOVLPS	-16 * SIZE(Y), %xmm4

  862. 	movhps	-15 * SIZE(Y), %xmm4

  863. 	movaps	-16 * SIZE(X), %xmm6

  864. 

  865. 	pshufd	$0x4e, %xmm6, %xmm3 

  866. 	mulpd	%xmm4, %xmm6

  867. 	addpd	%xmm6, %xmm0

  868. 	mulpd	%xmm4, %xmm3

  869. 	addpd	%xmm3, %xmm1

  870. 

  871. 	MOVLPS	-14 * SIZE(Y), %xmm5

  872. 	movhps	-13 * SIZE(Y), %xmm5

  873. 	movaps	-14 * SIZE(X), %xmm7

  874. 

  875. 	pshufd	$0x4e, %xmm7, %xmm3 

  876. 	mulpd	%xmm5, %xmm7

  877. 	addpd	%xmm7, %xmm0

  878. 	mulpd	%xmm5, %xmm3

  879. 	addpd	%xmm3, %xmm1

  880. 

  881. 	addl	$4 * SIZE, X

  882. 	addl	$4 * SIZE, Y

  883. 	ALIGN_3

  884. 

  885. .L37:

  886. 	SHUFPD_1 %xmm1, %xmm1

  887. 	SHUFPD_1 %xmm3, %xmm3

  888. 

  889. 	testl	$1, N

  890. 	jle	.L98

  891. 

  892. 	MOVLPS	-16 * SIZE(Y), %xmm4

  893. 	movhps	-15 * SIZE(Y), %xmm4

  894. 	movaps	-16 * SIZE(X), %xmm6

  895. 

  896. 	pshufd	$0x4e,  %xmm6, %xmm3 

  897. 	mulpd	%xmm4,  %xmm6

  898. 	addpd	%xmm6,  %xmm0

  899. 	mulpd	%xmm4,  %xmm3

  900. 	SHUFPD_1 %xmm3, %xmm3

  901. 	addpd	%xmm3, %xmm1

  902. 	jmp	.L98

  903. 	ALIGN_3

  904. 

  905. .L40:

  906. 	movhps	-16 * SIZE(X), %xmm4

  907. 	addl	$SIZE, X

  908. 	movhps	-16 * SIZE(Y), %xmm6

  909. 	addl	$SIZE, Y

  910. 

  911. 	movl	N,  %eax

  912. 	sarl	$3, %eax

  913. 	jle	.L45

  914. 

  915. 	movaps	-16 * SIZE(X), %xmm5

  916. 	movaps	-16 * SIZE(Y), %xmm7

  917. 

  918. 	decl	%eax

  919. 	jle	.L42

  920. 	ALIGN_3

  921. 

  922. .L41:

  923. #ifdef PREFETCH

  924. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  925. #endif

  926. 

  927. #ifdef PREFETCH

  928. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  929. #endif

  930. 

  931. 

  932. #if defined(PREFETCH) && !defined(FETCH128)

  933. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  934. #endif

  935. 

  936. 

  937. #if defined(PREFETCH) && !defined(FETCH128)

  938. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  939. #endif

  940. 

  941. 	movsd	%xmm7, %xmm6

  942. 	pshufd	$0x4e, %xmm6, %xmm3 

  943. 	movsd	%xmm5, %xmm4

  944. 	mulpd	%xmm4, %xmm6

  945. 	addpd	%xmm6, %xmm0

  946. 	movaps	-14 * SIZE(Y), %xmm6

  947. 	mulpd	%xmm4, %xmm3

  948. 	movaps	-14 * SIZE(X), %xmm4

  949. 	addpd	%xmm3, %xmm1

  950. 

  951. 	movsd	%xmm6, %xmm7

  952. 	pshufd	$0x4e,  %xmm7, %xmm3 

  953. 	movsd	%xmm4,  %xmm5

  954. 	mulpd	%xmm5,  %xmm7

  955. 	addpd	%xmm7,  %xmm0

  956. 	movaps	-12 * SIZE(Y), %xmm7

  957. 	mulpd	%xmm5,  %xmm3

  958. 	movaps	-12 * SIZE(X), %xmm5

  959. 	addpd	%xmm3, %xmm1

  960. 

  961. 	movsd	%xmm7, %xmm6

  962. 	pshufd	$0x4e, %xmm6, %xmm3 

  963. 	movsd	%xmm5, %xmm4

  964. 	mulpd	%xmm4, %xmm6

  965. 	addpd	%xmm6, %xmm0

  966. 	movaps	-10 * SIZE(Y), %xmm6

  967. 	mulpd	%xmm4, %xmm3

  968. 	movaps	-10 * SIZE(X), %xmm4

  969. 	addpd	%xmm3, %xmm1

  970. 

  971. 	movsd	%xmm6, %xmm7

  972. 	pshufd	$0x4e,  %xmm7, %xmm3 

  973. 	movsd	%xmm4,  %xmm5

  974. 	mulpd	%xmm5,  %xmm7

  975. 	addpd	%xmm7,  %xmm0

  976. 	movaps	 -8 * SIZE(Y), %xmm7

  977. 	mulpd	%xmm5,  %xmm3

  978. 	movaps	 -8 * SIZE(X), %xmm5

  979. 	addpd	%xmm3, %xmm1

  980. 

  981. 	movsd	%xmm7, %xmm6

  982. 	pshufd	$0x4e, %xmm6, %xmm3 

  983. 	movsd	%xmm5, %xmm4

  984. 	mulpd	%xmm4, %xmm6

  985. 	addpd	%xmm6, %xmm0

  986. 	movaps	 -6 * SIZE(Y), %xmm6

  987. 	mulpd	%xmm4, %xmm3

  988. 	movaps	 -6 * SIZE(X), %xmm4

  989. 	addpd	%xmm3, %xmm1

  990. 

  991. 	movsd	%xmm6, %xmm7

  992. 	pshufd	$0x4e,  %xmm7, %xmm3 

  993. 	movsd	%xmm4,  %xmm5

  994. 	mulpd	%xmm5,  %xmm7

  995. 	addpd	%xmm7,  %xmm0

  996. 	movaps	 -4 * SIZE(Y), %xmm7

  997. 	mulpd	%xmm5,  %xmm3

  998. 	movaps	 -4 * SIZE(X), %xmm5

  999. 	addpd	%xmm3, %xmm1

  1000. 

  1001. 	movsd	%xmm7, %xmm6

  1002. 	pshufd	$0x4e, %xmm6, %xmm3 

  1003. 	movsd	%xmm5, %xmm4

  1004. 	mulpd	%xmm4, %xmm6

  1005. 	addpd	%xmm6, %xmm0

  1006. 	movaps	 -2 * SIZE(Y), %xmm6

  1007. 	mulpd	%xmm4, %xmm3

  1008. 	movaps	 -2 * SIZE(X), %xmm4

  1009. 	addpd	%xmm3, %xmm1

  1010. 

  1011. 	movsd	%xmm6, %xmm7

  1012. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1013. 	movsd	%xmm4,  %xmm5

  1014. 	mulpd	%xmm5,  %xmm7

  1015. 	addpd	%xmm7,  %xmm0

  1016. 	movaps	  0 * SIZE(Y), %xmm7

  1017. 	mulpd	%xmm5,  %xmm3

  1018. 	movaps	  0 * SIZE(X), %xmm5

  1019. 	addpd	%xmm3, %xmm1

  1020. 

  1021. 	subl	$-16 * SIZE, X

  1022. 	subl	$-16 * SIZE, Y

  1023. 

  1024. 	decl	%eax

  1025. 	jg	.L41

  1026. 	ALIGN_3

  1027. 

  1028. .L42:

  1029. 	movsd	%xmm7, %xmm6

  1030. 	pshufd	$0x4e, %xmm6, %xmm3 

  1031. 	movsd	%xmm5, %xmm4

  1032. 	mulpd	%xmm4, %xmm6

  1033. 	addpd	%xmm6, %xmm0

  1034. 	movaps	-14 * SIZE(Y), %xmm6

  1035. 	mulpd	%xmm4, %xmm3

  1036. 	movaps	-14 * SIZE(X), %xmm4

  1037. 	addpd	%xmm3, %xmm1

  1038. 

  1039. 	movsd	%xmm6, %xmm7

  1040. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1041. 	movsd	%xmm4,  %xmm5

  1042. 	mulpd	%xmm5,  %xmm7

  1043. 	addpd	%xmm7,  %xmm0

  1044. 	movaps	-12 * SIZE(Y), %xmm7

  1045. 	mulpd	%xmm5,  %xmm3

  1046. 	movaps	-12 * SIZE(X), %xmm5

  1047. 	addpd	%xmm3, %xmm1

  1048. 

  1049. 	movsd	%xmm7, %xmm6

  1050. 	pshufd	$0x4e, %xmm6, %xmm3 

  1051. 	movsd	%xmm5, %xmm4

  1052. 	mulpd	%xmm4, %xmm6

  1053. 	addpd	%xmm6, %xmm0

  1054. 	movaps	-10 * SIZE(Y), %xmm6

  1055. 	mulpd	%xmm4, %xmm3

  1056. 	movaps	-10 * SIZE(X), %xmm4

  1057. 	addpd	%xmm3, %xmm1

  1058. 

  1059. 	movsd	%xmm6, %xmm7

  1060. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1061. 	movsd	%xmm4,  %xmm5

  1062. 	mulpd	%xmm5,  %xmm7

  1063. 	addpd	%xmm7,  %xmm0

  1064. 	movaps	 -8 * SIZE(Y), %xmm7

  1065. 	mulpd	%xmm5,  %xmm3

  1066. 	movaps	 -8 * SIZE(X), %xmm5

  1067. 	addpd	%xmm3, %xmm1

  1068. 

  1069. 	movsd	%xmm7, %xmm6

  1070. 	pshufd	$0x4e, %xmm6, %xmm3 

  1071. 	movsd	%xmm5, %xmm4

  1072. 	mulpd	%xmm4, %xmm6

  1073. 	addpd	%xmm6, %xmm0

  1074. 	movaps	 -6 * SIZE(Y), %xmm6

  1075. 	mulpd	%xmm4, %xmm3

  1076. 	movaps	 -6 * SIZE(X), %xmm4

  1077. 	addpd	%xmm3, %xmm1

  1078. 

  1079. 	movsd	%xmm6, %xmm7

  1080. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1081. 	movsd	%xmm4,  %xmm5

  1082. 	mulpd	%xmm5,  %xmm7

  1083. 	addpd	%xmm7,  %xmm0

  1084. 	movaps	 -4 * SIZE(Y), %xmm7

  1085. 	mulpd	%xmm5,  %xmm3

  1086. 	movaps	 -4 * SIZE(X), %xmm5

  1087. 	addpd	%xmm3, %xmm1

  1088. 

  1089. 	movsd	%xmm7, %xmm6

  1090. 	pshufd	$0x4e, %xmm6, %xmm3 

  1091. 	movsd	%xmm5, %xmm4

  1092. 	mulpd	%xmm4, %xmm6

  1093. 	addpd	%xmm6, %xmm0

  1094. 	movaps	 -2 * SIZE(Y), %xmm6

  1095. 	mulpd	%xmm4, %xmm3

  1096. 	movaps	 -2 * SIZE(X), %xmm4

  1097. 	addpd	%xmm3, %xmm1

  1098. 

  1099. 	movsd	%xmm6, %xmm7

  1100. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1101. 	movsd	%xmm4,  %xmm5

  1102. 	mulpd	%xmm5,  %xmm7

  1103. 	addpd	%xmm7,  %xmm0

  1104. 	mulpd	%xmm5,  %xmm3

  1105. 	addpd	%xmm3, %xmm1

  1106. 

  1107. 	subl	$-16 * SIZE, X

  1108. 	subl	$-16 * SIZE, Y

  1109. 	ALIGN_3

  1110. 

  1111. .L45:

  1112. 	testl	$4, N

  1113. 	jle	.L46

  1114. 

  1115. 	movaps	-16 * SIZE(X), %xmm5

  1116. 	movaps	-16 * SIZE(Y), %xmm7

  1117. 

  1118. 	movsd	%xmm7, %xmm6

  1119. 	pshufd	$0x4e, %xmm6, %xmm3 

  1120. 	movsd	%xmm5, %xmm4

  1121. 	mulpd	%xmm4, %xmm6

  1122. 	addpd	%xmm6, %xmm0

  1123. 	movaps	-14 * SIZE(Y), %xmm6

  1124. 	mulpd	%xmm4, %xmm3

  1125. 	movaps	-14 * SIZE(X), %xmm4

  1126. 	addpd	%xmm3, %xmm1

  1127. 

  1128. 	movsd	%xmm6, %xmm7

  1129. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1130. 	movsd	%xmm4,  %xmm5

  1131. 	mulpd	%xmm5,  %xmm7

  1132. 	addpd	%xmm7,  %xmm0

  1133. 	movaps	-12 * SIZE(Y), %xmm7

  1134. 	mulpd	%xmm5,  %xmm3

  1135. 	movaps	-12 * SIZE(X), %xmm5

  1136. 	addpd	%xmm3, %xmm1

  1137. 

  1138. 	movsd	%xmm7, %xmm6

  1139. 	pshufd	$0x4e, %xmm6, %xmm3 

  1140. 	movsd	%xmm5, %xmm4

  1141. 	mulpd	%xmm4, %xmm6

  1142. 	addpd	%xmm6, %xmm0

  1143. 	movaps	-10 * SIZE(Y), %xmm6

  1144. 	mulpd	%xmm4, %xmm3

  1145. 	movaps	-10 * SIZE(X), %xmm4

  1146. 	addpd	%xmm3, %xmm1

  1147. 

  1148. 	movsd	%xmm6, %xmm7

  1149. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1150. 	movsd	%xmm4,  %xmm5

  1151. 	mulpd	%xmm5,  %xmm7

  1152. 	addpd	%xmm7,  %xmm0

  1153. 	mulpd	%xmm5,  %xmm3

  1154. 	addpd	%xmm3, %xmm1

  1155. 

  1156. 	addl	$8 * SIZE, X

  1157. 	addl	$8 * SIZE, Y

  1158. 	ALIGN_3

  1159. 

  1160. .L46:

  1161. 	testl	$2, N

  1162. 	jle	.L47

  1163. 

  1164. 	movaps	-16 * SIZE(X), %xmm5

  1165. 	movaps	-16 * SIZE(Y), %xmm7

  1166. 

  1167. 	movsd	%xmm7, %xmm6

  1168. 	pshufd	$0x4e, %xmm6, %xmm3 

  1169. 	movsd	%xmm5, %xmm4

  1170. 	mulpd	%xmm4, %xmm6

  1171. 	addpd	%xmm6, %xmm0

  1172. 	movaps	-14 * SIZE(Y), %xmm6

  1173. 	mulpd	%xmm4, %xmm3

  1174. 	movaps	-14 * SIZE(X), %xmm4

  1175. 	addpd	%xmm3, %xmm1

  1176. 

  1177. 	movsd	%xmm6, %xmm7

  1178. 	pshufd	$0x4e,  %xmm7, %xmm3 

  1179. 	movsd	%xmm4,  %xmm5

  1180. 	mulpd	%xmm5,  %xmm7

  1181. 	addpd	%xmm7,  %xmm0

  1182. 	mulpd	%xmm5,  %xmm3

  1183. 	addpd	%xmm3, %xmm1

  1184. 

  1185. 	addl	$4 * SIZE, X

  1186. 	addl	$4 * SIZE, Y

  1187. 	ALIGN_3

  1188. 

  1189. .L47:

  1190. 	testl	$1, N

  1191. 	jle	.L48

  1192. 

  1193. 	movlpd	-16 * SIZE(X), %xmm4

  1194. 	movlpd	-16 * SIZE(Y), %xmm6

  1195. 

  1196. 	pshufd	$0x4e,  %xmm6, %xmm3 

  1197. 	mulpd	%xmm4,  %xmm6

  1198. 	addpd	%xmm6,  %xmm0

  1199. 	mulpd	%xmm4,  %xmm3

  1200. 	addpd	%xmm3, %xmm1

  1201. 	ALIGN_3

  1202. 

  1203. .L48:

  1204. 	SHUFPD_1 %xmm0, %xmm0

  1205. 	SHUFPD_1 %xmm1, %xmm1

  1206. 	SHUFPD_1 %xmm2, %xmm2

  1207. 	SHUFPD_1 %xmm3, %xmm3

  1208. 	jmp	.L98	

  1209. 	ALIGN_3

  1210. 

  1211. .L50:

  1212. 	movl	N,  %eax

  1213. 	sarl	$3, %eax

  1214. 	jle	.L55

  1215. 

  1216. 	MOVLPS	0 * SIZE(X), %xmm4

  1217. 	movhps	1 * SIZE(X), %xmm4

  1218. 	addl	INCX, X

  1219. 	MOVLPS	0 * SIZE(Y), %xmm6

  1220. 	movhps	1 * SIZE(Y), %xmm6

  1221. 	addl	INCY, Y

  1222. 

  1223. 	MOVLPS	0 * SIZE(X), %xmm5

  1224. 	movhps	1 * SIZE(X), %xmm5

  1225. 	addl	INCX, X

  1226. 	MOVLPS	0 * SIZE(Y), %xmm7

  1227. 	movhps	1 * SIZE(Y), %xmm7

  1228. 	addl	INCY, Y

  1229. 

  1230. 	decl	%eax

  1231. 	jle	.L54

  1232. 	ALIGN_3

  1233. 

  1234. .L53:

  1235. 	pshufd	$0x4e, %xmm6, %xmm3 

  1236. 	mulpd	%xmm4, %xmm6

  1237. 	addpd	%xmm6, %xmm0

  1238. 	MOVLPS	0 * SIZE(Y), %xmm6

  1239. 	movhps	1 * SIZE(Y), %xmm6

  1240. 	addl	INCY, Y

  1241. 	mulpd	%xmm4, %xmm3

  1242. 	MOVLPS	0 * SIZE(X), %xmm4

  1243. 	movhps	1 * SIZE(X), %xmm4

  1244. 	addl	INCX, X

  1245. 	addpd	%xmm3, %xmm1

  1246. 

  1247. 	pshufd	$0x4e, %xmm7, %xmm3 

  1248. 	mulpd	%xmm5, %xmm7

  1249. 	addpd	%xmm7, %xmm0

  1250. 	MOVLPS	0 * SIZE(Y), %xmm7

  1251. 	movhps	1 * SIZE(Y), %xmm7

  1252. 	addl	INCY, Y

  1253. 	mulpd	%xmm5, %xmm3

  1254. 	MOVLPS	0 * SIZE(X), %xmm5

  1255. 	movhps	1 * SIZE(X), %xmm5

  1256. 	addl	INCX, X

  1257. 	addpd	%xmm3, %xmm1

  1258. 

  1259. 	pshufd	$0x4e, %xmm6, %xmm3 

  1260. 	mulpd	%xmm4, %xmm6

  1261. 	addpd	%xmm6, %xmm0

  1262. 	MOVLPS	0 * SIZE(Y), %xmm6

  1263. 	movhps	1 * SIZE(Y), %xmm6

  1264. 	addl	INCY, Y

  1265. 	mulpd	%xmm4, %xmm3

  1266. 	MOVLPS	0 * SIZE(X), %xmm4

  1267. 	movhps	1 * SIZE(X), %xmm4

  1268. 	addl	INCX, X

  1269. 	addpd	%xmm3, %xmm1

  1270. 

  1271. 	pshufd	$0x4e, %xmm7, %xmm3 

  1272. 	mulpd	%xmm5, %xmm7

  1273. 	addpd	%xmm7, %xmm0

  1274. 	MOVLPS	0 * SIZE(Y), %xmm7

  1275. 	movhps	1 * SIZE(Y), %xmm7

  1276. 	addl	INCY, Y

  1277. 	mulpd	%xmm5, %xmm3

  1278. 	MOVLPS	0 * SIZE(X), %xmm5

  1279. 	movhps	1 * SIZE(X), %xmm5

  1280. 	addl	INCX, X

  1281. 	addpd	%xmm3, %xmm1

  1282. 

  1283. 	pshufd	$0x4e, %xmm6, %xmm3 

  1284. 	mulpd	%xmm4, %xmm6

  1285. 	addpd	%xmm6, %xmm0

  1286. 	MOVLPS	0 * SIZE(Y), %xmm6

  1287. 	movhps	1 * SIZE(Y), %xmm6

  1288. 	addl	INCY, Y

  1289. 	mulpd	%xmm4, %xmm3

  1290. 	MOVLPS	0 * SIZE(X), %xmm4

  1291. 	movhps	1 * SIZE(X), %xmm4

  1292. 	addl	INCX, X

  1293. 	addpd	%xmm3, %xmm1

  1294. 

  1295. 	pshufd	$0x4e, %xmm7, %xmm3 

  1296. 	mulpd	%xmm5, %xmm7

  1297. 	addpd	%xmm7, %xmm0

  1298. 	MOVLPS	0 * SIZE(Y), %xmm7

  1299. 	movhps	1 * SIZE(Y), %xmm7

  1300. 	addl	INCY, Y

  1301. 	mulpd	%xmm5, %xmm3

  1302. 	MOVLPS	0 * SIZE(X), %xmm5

  1303. 	movhps	1 * SIZE(X), %xmm5

  1304. 	addl	INCX, X

  1305. 	addpd	%xmm3, %xmm1

  1306. 

  1307. 	pshufd	$0x4e, %xmm6, %xmm3 

  1308. 	mulpd	%xmm4, %xmm6

  1309. 	addpd	%xmm6, %xmm0

  1310. 	MOVLPS	0 * SIZE(Y), %xmm6

  1311. 	movhps	1 * SIZE(Y), %xmm6

  1312. 	addl	INCY, Y

  1313. 	mulpd	%xmm4, %xmm3

  1314. 	MOVLPS	0 * SIZE(X), %xmm4

  1315. 	movhps	1 * SIZE(X), %xmm4

  1316. 	addl	INCX, X

  1317. 	addpd	%xmm3, %xmm1

  1318. 

  1319. 	pshufd	$0x4e, %xmm7, %xmm3 

  1320. 	mulpd	%xmm5, %xmm7

  1321. 	addpd	%xmm7, %xmm0

  1322. 	MOVLPS	0 * SIZE(Y), %xmm7

  1323. 	movhps	1 * SIZE(Y), %xmm7

  1324. 	addl	INCY, Y

  1325. 	mulpd	%xmm5, %xmm3

  1326. 	MOVLPS	0 * SIZE(X), %xmm5

  1327. 	movhps	1 * SIZE(X), %xmm5

  1328. 	addl	INCX, X

  1329. 	addpd	%xmm3, %xmm1

  1330. 

  1331. 	decl	%eax

  1332. 	jg	.L53

  1333. 	ALIGN_3

  1334. 

  1335. .L54:

  1336. 	pshufd	$0x4e, %xmm6, %xmm3 

  1337. 	mulpd	%xmm4, %xmm6

  1338. 	addpd	%xmm6, %xmm0

  1339. 	MOVLPS	0 * SIZE(Y), %xmm6

  1340. 	movhps	1 * SIZE(Y), %xmm6

  1341. 	addl	INCY, Y

  1342. 	mulpd	%xmm4, %xmm3

  1343. 	MOVLPS	0 * SIZE(X), %xmm4

  1344. 	movhps	1 * SIZE(X), %xmm4

  1345. 	addl	INCX, X

  1346. 	addpd	%xmm3, %xmm1

  1347. 

  1348. 	pshufd	$0x4e, %xmm7, %xmm3 

  1349. 	mulpd	%xmm5, %xmm7

  1350. 	addpd	%xmm7, %xmm0

  1351. 	MOVLPS	0 * SIZE(Y), %xmm7

  1352. 	movhps	1 * SIZE(Y), %xmm7

  1353. 	addl	INCY, Y

  1354. 	mulpd	%xmm5, %xmm3

  1355. 	MOVLPS	0 * SIZE(X), %xmm5

  1356. 	movhps	1 * SIZE(X), %xmm5

  1357. 	addl	INCX, X

  1358. 	addpd	%xmm3, %xmm1

  1359. 

  1360. 	pshufd	$0x4e, %xmm6, %xmm3 

  1361. 	mulpd	%xmm4, %xmm6

  1362. 	addpd	%xmm6, %xmm0

  1363. 	MOVLPS	0 * SIZE(Y), %xmm6

  1364. 	movhps	1 * SIZE(Y), %xmm6

  1365. 	addl	INCY, Y

  1366. 	mulpd	%xmm4, %xmm3

  1367. 	MOVLPS	0 * SIZE(X), %xmm4

  1368. 	movhps	1 * SIZE(X), %xmm4

  1369. 	addl	INCX, X

  1370. 	addpd	%xmm3, %xmm1

  1371. 

  1372. 	pshufd	$0x4e, %xmm7, %xmm3 

  1373. 	mulpd	%xmm5, %xmm7

  1374. 	addpd	%xmm7, %xmm0

  1375. 	MOVLPS	0 * SIZE(Y), %xmm7

  1376. 	movhps	1 * SIZE(Y), %xmm7

  1377. 	addl	INCY, Y

  1378. 	mulpd	%xmm5, %xmm3

  1379. 	MOVLPS	0 * SIZE(X), %xmm5

  1380. 	movhps	1 * SIZE(X), %xmm5

  1381. 	addl	INCX, X

  1382. 	addpd	%xmm3, %xmm1

  1383. 

  1384. 	pshufd	$0x4e, %xmm6, %xmm3 

  1385. 	mulpd	%xmm4, %xmm6

  1386. 	addpd	%xmm6, %xmm0

  1387. 	MOVLPS	0 * SIZE(Y), %xmm6

  1388. 	movhps	1 * SIZE(Y), %xmm6

  1389. 	addl	INCY, Y

  1390. 	mulpd	%xmm4, %xmm3

  1391. 	MOVLPS	0 * SIZE(X), %xmm4

  1392. 	movhps	1 * SIZE(X), %xmm4

  1393. 	addl	INCX, X

  1394. 	addpd	%xmm3, %xmm1

  1395. 

  1396. 	pshufd	$0x4e, %xmm7, %xmm3 

  1397. 	mulpd	%xmm5, %xmm7

  1398. 	addpd	%xmm7, %xmm0

  1399. 	MOVLPS	0 * SIZE(Y), %xmm7

  1400. 	movhps	1 * SIZE(Y), %xmm7

  1401. 	addl	INCY, Y

  1402. 	mulpd	%xmm5, %xmm3

  1403. 	MOVLPS	0 * SIZE(X), %xmm5

  1404. 	movhps	1 * SIZE(X), %xmm5

  1405. 	addl	INCX, X

  1406. 	addpd	%xmm3, %xmm1

  1407. 

  1408. 	pshufd	$0x4e, %xmm6, %xmm3 

  1409. 	mulpd	%xmm4, %xmm6

  1410. 	addpd	%xmm6, %xmm0

  1411. 	mulpd	%xmm4, %xmm3

  1412. 	addpd	%xmm3, %xmm1

  1413. 

  1414. 	pshufd	$0x4e, %xmm7, %xmm3 

  1415. 	mulpd	%xmm5, %xmm7

  1416. 	addpd	%xmm7, %xmm0

  1417. 	mulpd	%xmm5, %xmm3

  1418. 	addpd	%xmm3, %xmm1

  1419. 	ALIGN_3

  1420. 

  1421. .L55:

  1422. 	testl	$4, N

  1423. 	jle	.L56

  1424. 

  1425. 	MOVLPS	0 * SIZE(X), %xmm4

  1426. 	movhps	1 * SIZE(X), %xmm4

  1427. 	addl	INCX, X

  1428. 	MOVLPS	0 * SIZE(Y), %xmm6

  1429. 	movhps	1 * SIZE(Y), %xmm6

  1430. 	addl	INCY, Y

  1431. 

  1432. 	MOVLPS	0 * SIZE(X), %xmm5

  1433. 	movhps	1 * SIZE(X), %xmm5

  1434. 	addl	INCX, X

  1435. 	MOVLPS	0 * SIZE(Y), %xmm7

  1436. 	movhps	1 * SIZE(Y), %xmm7

  1437. 	addl	INCY, Y

  1438. 

  1439. 	pshufd	$0x4e, %xmm6, %xmm3 

  1440. 	mulpd	%xmm4, %xmm6

  1441. 	addpd	%xmm6, %xmm0

  1442. 	MOVLPS	0 * SIZE(Y), %xmm6

  1443. 	movhps	1 * SIZE(Y), %xmm6

  1444. 	addl	INCY, Y

  1445. 	mulpd	%xmm4, %xmm3

  1446. 	MOVLPS	0 * SIZE(X), %xmm4

  1447. 	movhps	1 * SIZE(X), %xmm4

  1448. 	addl	INCX, X

  1449. 	addpd	%xmm3, %xmm1

  1450. 

  1451. 	pshufd	$0x4e, %xmm7, %xmm3 

  1452. 	mulpd	%xmm5, %xmm7

  1453. 	addpd	%xmm7, %xmm0

  1454. 	MOVLPS	0 * SIZE(Y), %xmm7

  1455. 	movhps	1 * SIZE(Y), %xmm7

  1456. 	addl	INCY, Y

  1457. 	mulpd	%xmm5, %xmm3

  1458. 	MOVLPS	0 * SIZE(X), %xmm5

  1459. 	movhps	1 * SIZE(X), %xmm5

  1460. 	addl	INCX, X

  1461. 	addpd	%xmm3, %xmm1

  1462. 

  1463. 	pshufd	$0x4e, %xmm6, %xmm3 

  1464. 	mulpd	%xmm4, %xmm6

  1465. 	addpd	%xmm6, %xmm0

  1466. 	mulpd	%xmm4, %xmm3

  1467. 	addpd	%xmm3, %xmm1

  1468. 

  1469. 	pshufd	$0x4e, %xmm7, %xmm3 

  1470. 	mulpd	%xmm5, %xmm7

  1471. 	addpd	%xmm7, %xmm0

  1472. 	mulpd	%xmm5, %xmm3

  1473. 	addpd	%xmm3, %xmm1

  1474. 	ALIGN_3

  1475. 

  1476. .L56:

  1477. 	testl	$2, N

  1478. 	jle	.L57

  1479. 

  1480. 	MOVLPS	0 * SIZE(X), %xmm4

  1481. 	movhps	1 * SIZE(X), %xmm4

  1482. 	addl	INCX, X

  1483. 	MOVLPS	0 * SIZE(Y), %xmm6

  1484. 	movhps	1 * SIZE(Y), %xmm6

  1485. 	addl	INCY, Y

  1486. 

  1487. 	pshufd	$0x4e, %xmm6, %xmm3 

  1488. 	mulpd	%xmm4, %xmm6

  1489. 	addpd	%xmm6, %xmm0

  1490. 	mulpd	%xmm4, %xmm3

  1491. 	addpd	%xmm3, %xmm1

  1492. 

  1493. 	MOVLPS	0 * SIZE(X), %xmm5

  1494. 	movhps	1 * SIZE(X), %xmm5

  1495. 	addl	INCX, X

  1496. 	MOVLPS	0 * SIZE(Y), %xmm7

  1497. 	movhps	1 * SIZE(Y), %xmm7

  1498. 	addl	INCY, Y

  1499. 

  1500. 	pshufd	$0x4e, %xmm7, %xmm3 

  1501. 	mulpd	%xmm5, %xmm7

  1502. 	addpd	%xmm7, %xmm0

  1503. 	mulpd	%xmm5, %xmm3

  1504. 	addpd	%xmm3, %xmm1

  1505. 	ALIGN_3

  1506. 

  1507. .L57:

  1508. 	testl	$1, N

  1509. 	jle	.L98

  1510. 

  1511. 	MOVLPS	0 * SIZE(X), %xmm4

  1512. 	movhps	1 * SIZE(X), %xmm4

  1513. 	MOVLPS	0 * SIZE(Y), %xmm6

  1514. 	movhps	1 * SIZE(Y), %xmm6

  1515. 

  1516. 	pshufd	$0x4e, %xmm6, %xmm3 

  1517. 	mulpd	%xmm4, %xmm6

  1518. 	addpd	%xmm6, %xmm0

  1519. 	mulpd	%xmm4, %xmm3

  1520. 	addpd	%xmm3, %xmm1

  1521. 	ALIGN_3

  1522. 

  1523. .L98:

  1524. 	pshufd	$0x4e, %xmm0, %xmm2

  1525. 	pshufd	$0x4e, %xmm1, %xmm3

  1526. 

  1527. #ifndef CONJ

  1528. 	subsd	 %xmm2, %xmm0

  1529. 	addsd	 %xmm3, %xmm1

  1530. #else

  1531. 	addsd	 %xmm2, %xmm0

  1532. 	subsd	 %xmm3, %xmm1

  1533. #endif

  1534. 

  1535. .L999:

  1536. 	movl	RESULT, %eax

  1537. 

  1538. 	MOVLPS	%xmm0, 0 * SIZE(%eax)

  1539. 	MOVLPS	%xmm1, 1 * SIZE(%eax)

  1540. 

  1541. 	popl	%ebx

  1542. 	popl	%esi

  1543. 	popl	%edi

  1544. #if defined(OS_WINNT) || defined(OS_CYGWIN_NT) || defined(OS_INTERIX)

  1545. #ifdef MS_ABI

  1546. /* For MingW GCC >= 4.7. It is compatible with MSVC ABI. http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36834 */

  1547. 	ret

  1548. #else

  1549. /* remove the hidden return value address from the stack. For MingW GCC < 4.7 */

  1550. 	ret	$0x4

  1551. #endif

  1552. #else

  1553. /*remove the hidden return value address from the stack on Linux.*/

  1554. 	ret $0x4

  1555. #endif

  1556. 	EPILOGUE