OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfaaa975e6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfaaa975e6'
${ noResults }
OpenBLAS/kernel/x86/zaxpy_sse2.S

1534 lines
32 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS	 0

  44. 	

  45. #define STACK_M		 4 + STACK + ARGS(%esp)

  46. #define STACK_ALPHA_R	16 + STACK + ARGS(%esp)

  47. #define STACK_ALPHA_I	24 + STACK + ARGS(%esp)

  48. #define STACK_X		32 + STACK + ARGS(%esp)

  49. #define STACK_INCX	36 + STACK + ARGS(%esp)

  50. #define STACK_Y		40 + STACK + ARGS(%esp)

  51. #define STACK_INCY	44 + STACK + ARGS(%esp)

  52. 

  53. #define M	%ebx

  54. #define X	%esi

  55. #define	INCX	%ecx

  56. #define Y	%edi

  57. #define INCY	%edx

  58. #define YY	%ebp

  59. 

  60. #define ALPHA_R	%xmm6

  61. #define ALPHA_I	%xmm7

  62. 

  63. #if defined(HAVE_SSE3) && !defined(CORE_OPTERON)

  64. #define MOVDDUP(a, b, c)	movddup	a(b), c

  65. #define MOVDDUP2(a, b, c)	movddup	a##b, c

  66. #else

  67. #define MOVDDUP(a, b, c)	movlpd	a(b), c;movhpd	a(b), c

  68. #define MOVDDUP2(a, b, c)	movlpd	a##b, c;movhpd	a##b, c

  69. #endif

  70. 

  71. #include "l1param.h"

  72. 

  73. 	PROLOGUE

  74. 

  75. 	pushl	%ebp

  76. 	pushl	%edi

  77. 	pushl	%esi

  78. 	pushl	%ebx

  79. 

  80. 	PROFCODE

  81. 

  82. 	movl	STACK_M,    M

  83. 	movsd	STACK_ALPHA_R, %xmm0

  84. 	movsd	STACK_ALPHA_I, %xmm1

  85. 	movl	STACK_X,    X

  86. 	movl	STACK_INCX, INCX

  87. 	movl	STACK_Y,    Y

  88. 	movl	STACK_INCY, INCY

  89. 

  90. 	sall	$ZBASE_SHIFT, INCX

  91. 	sall	$ZBASE_SHIFT, INCY

  92. 

  93. 	testl	M, M

  94. 	jle	.L999

  95. 

  96. 	cmpl	$2 * SIZE, INCX

  97. 	jne	.L50

  98. 	cmpl	$2 * SIZE, INCY

  99. 	jne	.L50

  100. 

  101. 	subl	$-16 * SIZE, X

  102. 	subl	$-16 * SIZE, Y

  103. 

  104. 	pcmpeqb	%xmm5, %xmm5

  105. 	psllq	$63,   %xmm5

  106. 

  107. #ifdef HAVE_SSE3

  108. 	movddup	%xmm0, ALPHA_R

  109. 	movddup	%xmm1, ALPHA_I

  110. #else

  111. 	pshufd	$0x44, %xmm0, ALPHA_R

  112. 	pshufd	$0x44, %xmm1, ALPHA_I

  113. #endif

  114. 

  115. #ifndef CONJ

  116. 	shufps	$0x0c, %xmm5, %xmm5	

  117. 	xorpd	%xmm5, ALPHA_I

  118. #else

  119. 	shufps	$0xc0, %xmm5, %xmm5	

  120. 	xorpd	%xmm5, ALPHA_R

  121. #endif

  122. 

  123. 	testl	$SIZE, Y

  124. 	jne	.L30

  125. 

  126. 	testl	$SIZE, X

  127. 	jne	.L20

  128. 

  129. 	movl	M,  %eax

  130. 	sarl	$3, %eax

  131. 	jle	.L15

  132. 

  133. 	movaps	-16 * SIZE(X), %xmm0

  134. 	movaps	-14 * SIZE(X), %xmm1

  135. 	movaps	-12 * SIZE(X), %xmm2

  136. 	movaps	-10 * SIZE(X), %xmm3

  137. 

  138. 	decl	%eax

  139. 	jle	.L12

  140. 	ALIGN_3

  141. 

  142. .L11:

  143. #ifdef PREFETCHW

  144. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  145. #endif

  146. 

  147. 	pshufd	$0x4e,   %xmm0, %xmm5

  148. 	mulpd	ALPHA_R, %xmm0

  149. 	mulpd	ALPHA_I, %xmm5

  150. 	addpd	-16 * SIZE(Y), %xmm0

  151. 	addpd	%xmm5,   %xmm0

  152. 	movaps	%xmm0, -16 * SIZE(Y)

  153. 	movaps	 -8 * SIZE(X), %xmm0

  154. 

  155. 	pshufd	$0x4e,   %xmm1, %xmm5

  156. 	mulpd	ALPHA_R, %xmm1

  157. 	mulpd	ALPHA_I, %xmm5

  158. 	addpd	-14 * SIZE(Y), %xmm1

  159. 	addpd	%xmm5,   %xmm1

  160. 	movaps	%xmm1, -14 * SIZE(Y)

  161. 	movaps	 -6 * SIZE(X), %xmm1

  162. 

  163. #ifdef PREFETCH

  164. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  165. #endif

  166. 

  167. 	pshufd	$0x4e,   %xmm2, %xmm5

  168. 	mulpd	ALPHA_R, %xmm2

  169. 	mulpd	ALPHA_I, %xmm5

  170. 	addpd	-12 * SIZE(Y), %xmm2

  171. 	addpd	%xmm5,   %xmm2

  172. 	movaps	%xmm2, -12 * SIZE(Y)

  173. 	movaps	 -4 * SIZE(X), %xmm2

  174. 

  175. 	pshufd	$0x4e,   %xmm3, %xmm5

  176. 	mulpd	ALPHA_R, %xmm3

  177. 	mulpd	ALPHA_I, %xmm5

  178. 	addpd	-10 * SIZE(Y), %xmm3

  179. 	addpd	%xmm5,   %xmm3

  180. 	movaps	%xmm3, -10 * SIZE(Y)

  181. 	movaps	 -2 * SIZE(X), %xmm3

  182. 

  183. #if defined(PREFETCHW) && !defined(FETCH128)

  184. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  185. #endif

  186. 

  187. 	pshufd	$0x4e,   %xmm0, %xmm5

  188. 	mulpd	ALPHA_R, %xmm0

  189. 	mulpd	ALPHA_I, %xmm5

  190. 	addpd	 -8 * SIZE(Y), %xmm0

  191. 	addpd	%xmm5,   %xmm0

  192. 	movaps	%xmm0,  -8 * SIZE(Y)

  193. 	movaps	  0 * SIZE(X), %xmm0

  194. 

  195. 	pshufd	$0x4e,   %xmm1, %xmm5

  196. 	mulpd	ALPHA_R, %xmm1

  197. 	mulpd	ALPHA_I, %xmm5

  198. 	addpd	 -6 * SIZE(Y), %xmm1

  199. 	addpd	%xmm5,   %xmm1

  200. 	movaps	%xmm1,  -6 * SIZE(Y)

  201. 	movaps	  2 * SIZE(X), %xmm1

  202. 

  203. #if defined(PREFETCH) && !defined(FETCH128)

  204. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  205. #endif

  206. 

  207. 	pshufd	$0x4e,   %xmm2, %xmm5

  208. 	mulpd	ALPHA_R, %xmm2

  209. 	mulpd	ALPHA_I, %xmm5

  210. 	addpd	 -4 * SIZE(Y), %xmm2

  211. 	addpd	%xmm5,   %xmm2

  212. 	movaps	%xmm2,  -4 * SIZE(Y)

  213. 	movaps	  4 * SIZE(X), %xmm2

  214. 

  215. 	pshufd	$0x4e,   %xmm3, %xmm5

  216. 	mulpd	ALPHA_R, %xmm3

  217. 	mulpd	ALPHA_I, %xmm5

  218. 	addpd	 -2 * SIZE(Y), %xmm3

  219. 	addpd	%xmm5,   %xmm3

  220. 	movaps	%xmm3,  -2 * SIZE(Y)

  221. 	movaps	  6 * SIZE(X), %xmm3

  222. 

  223. 	subl	$-16 * SIZE, X

  224. 	subl	$-16 * SIZE, Y

  225. 	decl	%eax

  226. 	jg	.L11

  227. 	ALIGN_3

  228. 

  229. .L12:

  230. 	pshufd	$0x4e,   %xmm0, %xmm5

  231. 	mulpd	ALPHA_R, %xmm0

  232. 	mulpd	ALPHA_I, %xmm5

  233. 	addpd	-16 * SIZE(Y), %xmm0

  234. 	addpd	%xmm5,   %xmm0

  235. 	movaps	%xmm0, -16 * SIZE(Y)

  236. 	movaps	 -8 * SIZE(X), %xmm0

  237. 

  238. 	pshufd	$0x4e,   %xmm1, %xmm5

  239. 	mulpd	ALPHA_R, %xmm1

  240. 	mulpd	ALPHA_I, %xmm5

  241. 	addpd	-14 * SIZE(Y), %xmm1

  242. 	addpd	%xmm5,   %xmm1

  243. 	movaps	%xmm1, -14 * SIZE(Y)

  244. 	movaps	 -6 * SIZE(X), %xmm1

  245. 

  246. 	pshufd	$0x4e,   %xmm2, %xmm5

  247. 	mulpd	ALPHA_R, %xmm2

  248. 	mulpd	ALPHA_I, %xmm5

  249. 	addpd	-12 * SIZE(Y), %xmm2

  250. 	addpd	%xmm5,   %xmm2

  251. 	movaps	%xmm2, -12 * SIZE(Y)

  252. 	movaps	 -4 * SIZE(X), %xmm2

  253. 

  254. 	pshufd	$0x4e,   %xmm3, %xmm5

  255. 	mulpd	ALPHA_R, %xmm3

  256. 	mulpd	ALPHA_I, %xmm5

  257. 	addpd	-10 * SIZE(Y), %xmm3

  258. 	addpd	%xmm5,   %xmm3

  259. 	movaps	%xmm3, -10 * SIZE(Y)

  260. 	movaps	 -2 * SIZE(X), %xmm3

  261. 

  262. 	pshufd	$0x4e,   %xmm0, %xmm5

  263. 	mulpd	ALPHA_R, %xmm0

  264. 	mulpd	ALPHA_I, %xmm5

  265. 	addpd	 -8 * SIZE(Y), %xmm0

  266. 	addpd	%xmm5,   %xmm0

  267. 	movaps	%xmm0,  -8 * SIZE(Y)

  268. 

  269. 	pshufd	$0x4e,   %xmm1, %xmm5

  270. 	mulpd	ALPHA_R, %xmm1

  271. 	mulpd	ALPHA_I, %xmm5

  272. 	addpd	 -6 * SIZE(Y), %xmm1

  273. 	addpd	%xmm5,   %xmm1

  274. 	movaps	%xmm1,  -6 * SIZE(Y)

  275. 

  276. 	pshufd	$0x4e,   %xmm2, %xmm5

  277. 	mulpd	ALPHA_R, %xmm2

  278. 	mulpd	ALPHA_I, %xmm5

  279. 	addpd	 -4 * SIZE(Y), %xmm2

  280. 	addpd	%xmm5,   %xmm2

  281. 	movaps	%xmm2,  -4 * SIZE(Y)

  282. 

  283. 	pshufd	$0x4e,   %xmm3, %xmm5

  284. 	mulpd	ALPHA_R, %xmm3

  285. 	mulpd	ALPHA_I, %xmm5

  286. 	addpd	 -2 * SIZE(Y), %xmm3

  287. 	addpd	%xmm5,   %xmm3

  288. 	movaps	%xmm3,  -2 * SIZE(Y)

  289. 

  290. 	subl	$-16 * SIZE, X

  291. 	subl	$-16 * SIZE, Y

  292. 	ALIGN_3

  293. 

  294. .L15:

  295. 	movl	M,  %eax

  296. 	andl	$4, %eax

  297. 	jle	.L16

  298. 

  299. 	movaps	-16 * SIZE(X), %xmm0

  300. 	movaps	-14 * SIZE(X), %xmm1

  301. 	movaps	-12 * SIZE(X), %xmm2

  302. 	movaps	-10 * SIZE(X), %xmm3

  303. 

  304. 	pshufd	$0x4e,   %xmm0, %xmm5

  305. 	mulpd	ALPHA_R, %xmm0

  306. 	mulpd	ALPHA_I, %xmm5

  307. 	addpd	-16 * SIZE(Y), %xmm0

  308. 	addpd	%xmm5,   %xmm0

  309. 	movaps	%xmm0, -16 * SIZE(Y)

  310. 

  311. 	pshufd	$0x4e,   %xmm1, %xmm5

  312. 	mulpd	ALPHA_R, %xmm1

  313. 	mulpd	ALPHA_I, %xmm5

  314. 	addpd	-14 * SIZE(Y), %xmm1

  315. 	addpd	%xmm5,   %xmm1

  316. 	movaps	%xmm1, -14 * SIZE(Y)

  317. 

  318. 	pshufd	$0x4e,   %xmm2, %xmm5

  319. 	mulpd	ALPHA_R, %xmm2

  320. 	mulpd	ALPHA_I, %xmm5

  321. 	addpd	-12 * SIZE(Y), %xmm2

  322. 	addpd	%xmm5,   %xmm2

  323. 	movaps	%xmm2, -12 * SIZE(Y)

  324. 

  325. 	pshufd	$0x4e,   %xmm3, %xmm5

  326. 	mulpd	ALPHA_R, %xmm3

  327. 	mulpd	ALPHA_I, %xmm5

  328. 	addpd	-10 * SIZE(Y), %xmm3

  329. 	addpd	%xmm5,   %xmm3

  330. 	movaps	%xmm3, -10 * SIZE(Y)

  331. 

  332. 	addl	$8 * SIZE, X

  333. 	addl	$8 * SIZE, Y

  334. 	ALIGN_3

  335. 

  336. .L16:

  337. 	movl	M,  %eax

  338. 	andl	$2, %eax

  339. 	jle	.L17

  340. 

  341. 	movaps	-16 * SIZE(X), %xmm0

  342. 	movaps	-14 * SIZE(X), %xmm1

  343. 

  344. 	pshufd	$0x4e,   %xmm0, %xmm5

  345. 	mulpd	ALPHA_R, %xmm0

  346. 	mulpd	ALPHA_I, %xmm5

  347. 	addpd	-16 * SIZE(Y), %xmm0

  348. 	addpd	%xmm5,   %xmm0

  349. 	movaps	%xmm0, -16 * SIZE(Y)

  350. 

  351. 	pshufd	$0x4e,   %xmm1, %xmm5

  352. 	mulpd	ALPHA_R, %xmm1

  353. 	mulpd	ALPHA_I, %xmm5

  354. 	addpd	-14 * SIZE(Y), %xmm1

  355. 	addpd	%xmm5,   %xmm1

  356. 	movaps	%xmm1, -14 * SIZE(Y)

  357. 

  358. 	addl	$4 * SIZE, X

  359. 	addl	$4 * SIZE, Y

  360. 	ALIGN_3

  361. 

  362. .L17:

  363. 	movl	M,  %eax

  364. 	andl	$1, %eax

  365. 	jle	.L999

  366. 

  367. 	movaps	-16 * SIZE(X), %xmm0

  368. 

  369. 	pshufd	$0x4e,   %xmm0, %xmm5

  370. 	mulpd	ALPHA_R, %xmm0

  371. 	mulpd	ALPHA_I, %xmm5

  372. 	addpd	-16 * SIZE(Y), %xmm0

  373. 	addpd	%xmm5,   %xmm0

  374. 	movaps	%xmm0, -16 * SIZE(Y)

  375. 	jmp	.L999

  376. 	ALIGN_3

  377. 

  378. .L20:

  379. 	movl	M,  %eax

  380. 	sarl	$3, %eax

  381. 	jle	.L25

  382. 

  383. 	movsd	-16 * SIZE(X), %xmm0

  384. 	movhps	-15 * SIZE(X), %xmm0

  385. 	movsd	-14 * SIZE(X), %xmm1

  386. 	movhps	-13 * SIZE(X), %xmm1

  387. 	movsd	-12 * SIZE(X), %xmm2

  388. 	movhps	-11 * SIZE(X), %xmm2

  389. 	movsd	-10 * SIZE(X), %xmm3

  390. 	movhps	 -9 * SIZE(X), %xmm3

  391. 

  392. 	decl	%eax

  393. 	jle	.L22

  394. 	ALIGN_3

  395. 

  396. .L21:

  397. #ifdef PREFETCHW

  398. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  399. #endif

  400. 

  401. 	pshufd	$0x4e,   %xmm0, %xmm5

  402. 	mulpd	ALPHA_R, %xmm0

  403. 	mulpd	ALPHA_I, %xmm5

  404. 	addpd	-16 * SIZE(Y), %xmm0

  405. 	addpd	%xmm5,   %xmm0

  406. 	movaps	%xmm0, -16 * SIZE(Y)

  407. 	movsd	 -8 * SIZE(X), %xmm0

  408. 	movhps	 -7 * SIZE(X), %xmm0

  409. 

  410. 	pshufd	$0x4e,   %xmm1, %xmm5

  411. 	mulpd	ALPHA_R, %xmm1

  412. 	mulpd	ALPHA_I, %xmm5

  413. 	addpd	-14 * SIZE(Y), %xmm1

  414. 	addpd	%xmm5,   %xmm1

  415. 	movaps	%xmm1, -14 * SIZE(Y)

  416. 	movsd	 -6 * SIZE(X), %xmm1

  417. 	movhps	 -5 * SIZE(X), %xmm1

  418. 

  419. #ifdef PREFETCH

  420. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  421. #endif

  422. 

  423. 	pshufd	$0x4e,   %xmm2, %xmm5

  424. 	mulpd	ALPHA_R, %xmm2

  425. 	mulpd	ALPHA_I, %xmm5

  426. 	addpd	-12 * SIZE(Y), %xmm2

  427. 	addpd	%xmm5,   %xmm2

  428. 	movaps	%xmm2, -12 * SIZE(Y)

  429. 	movsd	 -4 * SIZE(X), %xmm2

  430. 	movhps	 -3 * SIZE(X), %xmm2

  431. 

  432. 	pshufd	$0x4e,   %xmm3, %xmm5

  433. 	mulpd	ALPHA_R, %xmm3

  434. 	mulpd	ALPHA_I, %xmm5

  435. 	addpd	-10 * SIZE(Y), %xmm3

  436. 	addpd	%xmm5,   %xmm3

  437. 	movaps	%xmm3, -10 * SIZE(Y)

  438. 	movsd	 -2 * SIZE(X), %xmm3

  439. 	movhps	 -1 * SIZE(X), %xmm3

  440. 

  441. #if defined(PREFETCHW) && !defined(FETCH128)

  442. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  443. #endif

  444. 

  445. 	pshufd	$0x4e,   %xmm0, %xmm5

  446. 	mulpd	ALPHA_R, %xmm0

  447. 	mulpd	ALPHA_I, %xmm5

  448. 	addpd	 -8 * SIZE(Y), %xmm0

  449. 	addpd	%xmm5,   %xmm0

  450. 	movaps	%xmm0,  -8 * SIZE(Y)

  451. 	movsd	  0 * SIZE(X), %xmm0

  452. 	movhps	  1 * SIZE(X), %xmm0

  453. 

  454. 	pshufd	$0x4e,   %xmm1, %xmm5

  455. 	mulpd	ALPHA_R, %xmm1

  456. 	mulpd	ALPHA_I, %xmm5

  457. 	addpd	 -6 * SIZE(Y), %xmm1

  458. 	addpd	%xmm5,   %xmm1

  459. 	movaps	%xmm1,  -6 * SIZE(Y)

  460. 	movsd	  2 * SIZE(X), %xmm1

  461. 	movhps	  3 * SIZE(X), %xmm1

  462. 

  463. #if defined(PREFETCH) && !defined(FETCH128)

  464. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  465. #endif

  466. 

  467. 	pshufd	$0x4e,   %xmm2, %xmm5

  468. 	mulpd	ALPHA_R, %xmm2

  469. 	mulpd	ALPHA_I, %xmm5

  470. 	addpd	 -4 * SIZE(Y), %xmm2

  471. 	addpd	%xmm5,   %xmm2

  472. 	movaps	%xmm2,  -4 * SIZE(Y)

  473. 	movsd	  4 * SIZE(X), %xmm2

  474. 	movhps	  5 * SIZE(X), %xmm2

  475. 

  476. 	pshufd	$0x4e,   %xmm3, %xmm5

  477. 	mulpd	ALPHA_R, %xmm3

  478. 	mulpd	ALPHA_I, %xmm5

  479. 	addpd	 -2 * SIZE(Y), %xmm3

  480. 	addpd	%xmm5,   %xmm3

  481. 	movaps	%xmm3,  -2 * SIZE(Y)

  482. 	movsd	  6 * SIZE(X), %xmm3

  483. 	movhps	  7 * SIZE(X), %xmm3

  484. 

  485. 	subl	$-16 * SIZE, X

  486. 	subl	$-16 * SIZE, Y

  487. 	decl	%eax

  488. 	jg	.L21

  489. 	ALIGN_3

  490. 

  491. .L22:

  492. 	pshufd	$0x4e,   %xmm0, %xmm5

  493. 	mulpd	ALPHA_R, %xmm0

  494. 	mulpd	ALPHA_I, %xmm5

  495. 	addpd	-16 * SIZE(Y), %xmm0

  496. 	addpd	%xmm5,   %xmm0

  497. 	movaps	%xmm0, -16 * SIZE(Y)

  498. 	movsd	 -8 * SIZE(X), %xmm0

  499. 	movhps	 -7 * SIZE(X), %xmm0

  500. 

  501. 	pshufd	$0x4e,   %xmm1, %xmm5

  502. 	mulpd	ALPHA_R, %xmm1

  503. 	mulpd	ALPHA_I, %xmm5

  504. 	addpd	-14 * SIZE(Y), %xmm1

  505. 	addpd	%xmm5,   %xmm1

  506. 	movaps	%xmm1, -14 * SIZE(Y)

  507. 	movsd	 -6 * SIZE(X), %xmm1

  508. 	movhps	 -5 * SIZE(X), %xmm1

  509. 

  510. 	pshufd	$0x4e,   %xmm2, %xmm5

  511. 	mulpd	ALPHA_R, %xmm2

  512. 	mulpd	ALPHA_I, %xmm5

  513. 	addpd	-12 * SIZE(Y), %xmm2

  514. 	addpd	%xmm5,   %xmm2

  515. 	movaps	%xmm2, -12 * SIZE(Y)

  516. 	movsd	 -4 * SIZE(X), %xmm2

  517. 	movhps	 -3 * SIZE(X), %xmm2

  518. 

  519. 	pshufd	$0x4e,   %xmm3, %xmm5

  520. 	mulpd	ALPHA_R, %xmm3

  521. 	mulpd	ALPHA_I, %xmm5

  522. 	addpd	-10 * SIZE(Y), %xmm3

  523. 	addpd	%xmm5,   %xmm3

  524. 	movaps	%xmm3, -10 * SIZE(Y)

  525. 	movsd	 -2 * SIZE(X), %xmm3

  526. 	movhps	 -1 * SIZE(X), %xmm3

  527. 

  528. 	pshufd	$0x4e,   %xmm0, %xmm5

  529. 	mulpd	ALPHA_R, %xmm0

  530. 	mulpd	ALPHA_I, %xmm5

  531. 	addpd	 -8 * SIZE(Y), %xmm0

  532. 	addpd	%xmm5,   %xmm0

  533. 	movaps	%xmm0,  -8 * SIZE(Y)

  534. 

  535. 	pshufd	$0x4e,   %xmm1, %xmm5

  536. 	mulpd	ALPHA_R, %xmm1

  537. 	mulpd	ALPHA_I, %xmm5

  538. 	addpd	 -6 * SIZE(Y), %xmm1

  539. 	addpd	%xmm5,   %xmm1

  540. 	movaps	%xmm1,  -6 * SIZE(Y)

  541. 

  542. 	pshufd	$0x4e,   %xmm2, %xmm5

  543. 	mulpd	ALPHA_R, %xmm2

  544. 	mulpd	ALPHA_I, %xmm5

  545. 	addpd	 -4 * SIZE(Y), %xmm2

  546. 	addpd	%xmm5,   %xmm2

  547. 	movaps	%xmm2,  -4 * SIZE(Y)

  548. 

  549. 	pshufd	$0x4e,   %xmm3, %xmm5

  550. 	mulpd	ALPHA_R, %xmm3

  551. 	mulpd	ALPHA_I, %xmm5

  552. 	addpd	 -2 * SIZE(Y), %xmm3

  553. 	addpd	%xmm5,   %xmm3

  554. 	movaps	%xmm3,  -2 * SIZE(Y)

  555. 

  556. 	subl	$-16 * SIZE, X

  557. 	subl	$-16 * SIZE, Y

  558. 	ALIGN_3

  559. 

  560. .L25:

  561. 	movl	M,  %eax

  562. 	andl	$4, %eax

  563. 	jle	.L26

  564. 

  565. 	movsd	-16 * SIZE(X), %xmm0

  566. 	movhps	-15 * SIZE(X), %xmm0

  567. 	movsd	-14 * SIZE(X), %xmm1

  568. 	movhps	-13 * SIZE(X), %xmm1

  569. 

  570. 	pshufd	$0x4e,   %xmm0, %xmm5

  571. 	mulpd	ALPHA_R, %xmm0

  572. 	mulpd	ALPHA_I, %xmm5

  573. 	addpd	-16 * SIZE(Y), %xmm0

  574. 	addpd	%xmm5,   %xmm0

  575. 	movaps	%xmm0, -16 * SIZE(Y)

  576. 

  577. 	pshufd	$0x4e,   %xmm1, %xmm5

  578. 	mulpd	ALPHA_R, %xmm1

  579. 	mulpd	ALPHA_I, %xmm5

  580. 	addpd	-14 * SIZE(Y), %xmm1

  581. 	addpd	%xmm5,   %xmm1

  582. 	movaps	%xmm1, -14 * SIZE(Y)

  583. 

  584. 	movsd	-12 * SIZE(X), %xmm2

  585. 	movhps	-11 * SIZE(X), %xmm2

  586. 	movsd	-10 * SIZE(X), %xmm3

  587. 	movhps	 -9 * SIZE(X), %xmm3

  588. 

  589. 	pshufd	$0x4e,   %xmm2, %xmm5

  590. 	mulpd	ALPHA_R, %xmm2

  591. 	mulpd	ALPHA_I, %xmm5

  592. 	addpd	-12 * SIZE(Y), %xmm2

  593. 	addpd	%xmm5,   %xmm2

  594. 	movaps	%xmm2, -12 * SIZE(Y)

  595. 

  596. 	pshufd	$0x4e,   %xmm3, %xmm5

  597. 	mulpd	ALPHA_R, %xmm3

  598. 	mulpd	ALPHA_I, %xmm5

  599. 	addpd	-10 * SIZE(Y), %xmm3

  600. 	addpd	%xmm5,   %xmm3

  601. 	movaps	%xmm3, -10 * SIZE(Y)

  602. 

  603. 	addl	$8 * SIZE, X

  604. 	addl	$8 * SIZE, Y

  605. 	ALIGN_3

  606. 

  607. .L26:

  608. 	movl	M,  %eax

  609. 	andl	$2, %eax

  610. 	jle	.L27

  611. 

  612. 	movsd	-16 * SIZE(X), %xmm0

  613. 	movhps	-15 * SIZE(X), %xmm0

  614. 

  615. 	pshufd	$0x4e,   %xmm0, %xmm5

  616. 	mulpd	ALPHA_R, %xmm0

  617. 	mulpd	ALPHA_I, %xmm5

  618. 	addpd	-16 * SIZE(Y), %xmm0

  619. 	addpd	%xmm5,   %xmm0

  620. 	movaps	%xmm0, -16 * SIZE(Y)

  621. 

  622. 	movsd	-14 * SIZE(X), %xmm1

  623. 	movhps	-13 * SIZE(X), %xmm1

  624. 

  625. 	pshufd	$0x4e,   %xmm1, %xmm5

  626. 	mulpd	ALPHA_R, %xmm1

  627. 	mulpd	ALPHA_I, %xmm5

  628. 	addpd	-14 * SIZE(Y), %xmm1

  629. 	addpd	%xmm5,   %xmm1

  630. 	movaps	%xmm1, -14 * SIZE(Y)

  631. 

  632. 	addl	$4 * SIZE, X

  633. 	addl	$4 * SIZE, Y

  634. 	ALIGN_3

  635. 

  636. .L27:

  637. 	movl	M,  %eax

  638. 	andl	$1, %eax

  639. 	jle	.L999

  640. 

  641. 	movsd	-16 * SIZE(X), %xmm0

  642. 	movhps	-15 * SIZE(X), %xmm0

  643. 

  644. 	pshufd	$0x4e,   %xmm0, %xmm5

  645. 	mulpd	ALPHA_R, %xmm0

  646. 	mulpd	ALPHA_I, %xmm5

  647. 	addpd	-16 * SIZE(Y), %xmm0

  648. 	addpd	%xmm5,   %xmm0

  649. 	movaps	%xmm0, -16 * SIZE(Y)

  650. 	jmp	.L999

  651. 	ALIGN_3

  652. 

  653. .L30:

  654. 	testl	$SIZE, X

  655. 	jne	.L40

  656. 

  657. 	movaps	-16 * SIZE(X), %xmm1

  658. 

  659. 	pshufd	$0x4e,   %xmm1, %xmm5

  660. 	mulpd	ALPHA_R, %xmm1

  661. 	mulpd	ALPHA_I, %xmm5

  662. 	addpd	%xmm5,   %xmm1

  663. 	xorps	%xmm0,   %xmm0

  664. 	SHUFPD_1 %xmm1,  %xmm0

  665. 

  666. 	xorps	%xmm4,   %xmm4

  667. 	movhps	-16 * SIZE(Y), %xmm4

  668. 

  669. 	addpd	%xmm0,   %xmm4

  670. 	movhps	%xmm4, -16 * SIZE(Y)

  671. 	movaps	%xmm1,   %xmm0

  672. 

  673. 	addl	$2 * SIZE, X

  674. 	addl	$1 * SIZE, Y

  675. 	decl	M

  676. 	jle	.L39

  677. 

  678. 	movl	M,  %eax

  679. 	sarl	$3, %eax

  680. 	jle	.L35

  681. 

  682. 	movaps	-16 * SIZE(X), %xmm1

  683. 	movaps	-14 * SIZE(X), %xmm2

  684. 	movaps	-12 * SIZE(X), %xmm3

  685. 

  686. 	decl	%eax

  687. 	jle	.L32

  688. 	ALIGN_3

  689. 

  690. .L31:

  691. #ifdef PREFETCHW

  692. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  693. #endif

  694. 

  695. 	pshufd	$0x4e,   %xmm1, %xmm5

  696. 	mulpd	ALPHA_R, %xmm1

  697. 	mulpd	ALPHA_I, %xmm5

  698. 	addpd	%xmm5,   %xmm1

  699. 	SHUFPD_1 %xmm1,  %xmm0

  700. 	addpd	-16 * SIZE(Y), %xmm0

  701. 	movaps	%xmm0, -16 * SIZE(Y)

  702. 	movaps	-10 * SIZE(X), %xmm0

  703. 

  704. 	pshufd	$0x4e,   %xmm2, %xmm5

  705. 	mulpd	ALPHA_R, %xmm2

  706. 	mulpd	ALPHA_I, %xmm5

  707. 	addpd	%xmm5,   %xmm2

  708. 	SHUFPD_1 %xmm2,  %xmm1

  709. 	addpd	-14 * SIZE(Y), %xmm1

  710. 	movaps	%xmm1, -14 * SIZE(Y)

  711. 	movaps	 -8 * SIZE(X), %xmm1

  712. 

  713. #ifdef PREFETCH

  714. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  715. #endif

  716. 

  717. 	pshufd	$0x4e,   %xmm3, %xmm5

  718. 	mulpd	ALPHA_R, %xmm3

  719. 	mulpd	ALPHA_I, %xmm5

  720. 	addpd	%xmm5,   %xmm3

  721. 	SHUFPD_1 %xmm3,  %xmm2

  722. 	addpd	-12 * SIZE(Y), %xmm2

  723. 	movaps	%xmm2, -12 * SIZE(Y)

  724. 	movaps	 -6 * SIZE(X), %xmm2

  725. 

  726. 	pshufd	$0x4e,   %xmm0, %xmm5

  727. 	mulpd	ALPHA_R, %xmm0

  728. 	mulpd	ALPHA_I, %xmm5

  729. 	addpd	%xmm5,   %xmm0

  730. 	SHUFPD_1 %xmm0,  %xmm3

  731. 	addpd	-10 * SIZE(Y), %xmm3

  732. 	movaps	%xmm3, -10 * SIZE(Y)

  733. 	movaps	 -4 * SIZE(X), %xmm3

  734. 

  735. #if defined(PREFETCHW) && !defined(FETCH128)

  736. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  737. #endif

  738. 

  739. 	pshufd	$0x4e,   %xmm1, %xmm5

  740. 	mulpd	ALPHA_R, %xmm1

  741. 	mulpd	ALPHA_I, %xmm5

  742. 	addpd	%xmm5,   %xmm1

  743. 	SHUFPD_1 %xmm1,  %xmm0

  744. 	addpd	 -8 * SIZE(Y), %xmm0

  745. 	movaps	%xmm0,  -8 * SIZE(Y)

  746. 	movaps	 -2 * SIZE(X), %xmm0

  747. 

  748. 	pshufd	$0x4e,   %xmm2, %xmm5

  749. 	mulpd	ALPHA_R, %xmm2

  750. 	mulpd	ALPHA_I, %xmm5

  751. 	addpd	%xmm5,   %xmm2

  752. 	SHUFPD_1 %xmm2,  %xmm1

  753. 	addpd	 -6 * SIZE(Y), %xmm1

  754. 	movaps	%xmm1,  -6 * SIZE(Y)

  755. 	movaps	  0 * SIZE(X), %xmm1

  756. 

  757. #if defined(PREFETCH) && !defined(FETCH128)

  758. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  759. #endif

  760. 

  761. 	pshufd	$0x4e,   %xmm3, %xmm5

  762. 	mulpd	ALPHA_R, %xmm3

  763. 	mulpd	ALPHA_I, %xmm5

  764. 	addpd	%xmm5,   %xmm3

  765. 	SHUFPD_1 %xmm3,  %xmm2

  766. 	addpd	 -4 * SIZE(Y), %xmm2

  767. 	movaps	%xmm2,  -4 * SIZE(Y)

  768. 	movaps	  2 * SIZE(X), %xmm2

  769. 

  770. 	pshufd	$0x4e,   %xmm0, %xmm5

  771. 	mulpd	ALPHA_R, %xmm0

  772. 	mulpd	ALPHA_I, %xmm5

  773. 	addpd	%xmm5,   %xmm0

  774. 	SHUFPD_1 %xmm0,  %xmm3

  775. 	addpd	 -2 * SIZE(Y), %xmm3

  776. 	movaps	%xmm3,  -2 * SIZE(Y)

  777. 	movaps	  4 * SIZE(X), %xmm3

  778. 

  779. 	subl	$-16 * SIZE, X

  780. 	subl	$-16 * SIZE, Y

  781. 	decl	%eax

  782. 	jg	.L31

  783. 	ALIGN_3

  784. 

  785. .L32:

  786. 	pshufd	$0x4e,   %xmm1, %xmm5

  787. 	mulpd	ALPHA_R, %xmm1

  788. 	mulpd	ALPHA_I, %xmm5

  789. 	addpd	%xmm5,   %xmm1

  790. 	SHUFPD_1 %xmm1,  %xmm0

  791. 	addpd	-16 * SIZE(Y), %xmm0

  792. 	movaps	%xmm0, -16 * SIZE(Y)

  793. 	movaps	-10 * SIZE(X), %xmm0

  794. 

  795. 	pshufd	$0x4e,   %xmm2, %xmm5

  796. 	mulpd	ALPHA_R, %xmm2

  797. 	mulpd	ALPHA_I, %xmm5

  798. 	addpd	%xmm5,   %xmm2

  799. 	SHUFPD_1 %xmm2,  %xmm1

  800. 	addpd	-14 * SIZE(Y), %xmm1

  801. 	movaps	%xmm1, -14 * SIZE(Y)

  802. 	movaps	 -8 * SIZE(X), %xmm1

  803. 

  804. 	pshufd	$0x4e,   %xmm3, %xmm5

  805. 	mulpd	ALPHA_R, %xmm3

  806. 	mulpd	ALPHA_I, %xmm5

  807. 	addpd	%xmm5,   %xmm3

  808. 	SHUFPD_1 %xmm3,  %xmm2

  809. 	addpd	-12 * SIZE(Y), %xmm2

  810. 	movaps	%xmm2, -12 * SIZE(Y)

  811. 	movaps	 -6 * SIZE(X), %xmm2

  812. 

  813. 	pshufd	$0x4e,   %xmm0, %xmm5

  814. 	mulpd	ALPHA_R, %xmm0

  815. 	mulpd	ALPHA_I, %xmm5

  816. 	addpd	%xmm5,   %xmm0

  817. 	SHUFPD_1 %xmm0,  %xmm3

  818. 	addpd	-10 * SIZE(Y), %xmm3

  819. 	movaps	%xmm3, -10 * SIZE(Y)

  820. 	movaps	 -4 * SIZE(X), %xmm3

  821. 

  822. 	pshufd	$0x4e,   %xmm1, %xmm5

  823. 	mulpd	ALPHA_R, %xmm1

  824. 	mulpd	ALPHA_I, %xmm5

  825. 	addpd	%xmm5,   %xmm1

  826. 	SHUFPD_1 %xmm1,  %xmm0

  827. 	addpd	 -8 * SIZE(Y), %xmm0

  828. 	movaps	%xmm0,  -8 * SIZE(Y)

  829. 	movaps	 -2 * SIZE(X), %xmm0

  830. 

  831. 	pshufd	$0x4e,   %xmm2, %xmm5

  832. 	mulpd	ALPHA_R, %xmm2

  833. 	mulpd	ALPHA_I, %xmm5

  834. 	addpd	%xmm5,   %xmm2

  835. 	SHUFPD_1 %xmm2,  %xmm1

  836. 	addpd	 -6 * SIZE(Y), %xmm1

  837. 	movaps	%xmm1,  -6 * SIZE(Y)

  838. 

  839. 	pshufd	$0x4e,   %xmm3, %xmm5

  840. 	mulpd	ALPHA_R, %xmm3

  841. 	mulpd	ALPHA_I, %xmm5

  842. 	addpd	%xmm5,   %xmm3

  843. 	SHUFPD_1 %xmm3,  %xmm2

  844. 	addpd	 -4 * SIZE(Y), %xmm2

  845. 	movaps	%xmm2,  -4 * SIZE(Y)

  846. 

  847. 	pshufd	$0x4e,   %xmm0, %xmm5

  848. 	mulpd	ALPHA_R, %xmm0

  849. 	mulpd	ALPHA_I, %xmm5

  850. 	addpd	%xmm5,   %xmm0

  851. 	SHUFPD_1 %xmm0,  %xmm3

  852. 	addpd	 -2 * SIZE(Y), %xmm3

  853. 	movaps	%xmm3,  -2 * SIZE(Y)

  854. 

  855. 	subl	$-16 * SIZE, X

  856. 	subl	$-16 * SIZE, Y

  857. 	ALIGN_3

  858. 

  859. .L35:

  860. 	movl	M,  %eax

  861. 	andl	$4, %eax

  862. 	jle	.L36

  863. 

  864. 	movaps	-16 * SIZE(X), %xmm1

  865. 	movaps	-14 * SIZE(X), %xmm2

  866. 	movaps	-12 * SIZE(X), %xmm3

  867. 	movaps	-10 * SIZE(X), %xmm4

  868. 

  869. 	pshufd	$0x4e,   %xmm1, %xmm5

  870. 	mulpd	ALPHA_R, %xmm1

  871. 	mulpd	ALPHA_I, %xmm5

  872. 	addpd	%xmm5,   %xmm1

  873. 	SHUFPD_1 %xmm1,  %xmm0

  874. 	addpd	-16 * SIZE(Y), %xmm0

  875. 	movaps	%xmm0, -16 * SIZE(Y)

  876. 

  877. 	pshufd	$0x4e,   %xmm2, %xmm5

  878. 	mulpd	ALPHA_R, %xmm2

  879. 	mulpd	ALPHA_I, %xmm5

  880. 	addpd	%xmm5,   %xmm2

  881. 	SHUFPD_1 %xmm2,  %xmm1

  882. 	addpd	-14 * SIZE(Y), %xmm1

  883. 	movaps	%xmm1, -14 * SIZE(Y)

  884. 

  885. 	pshufd	$0x4e,   %xmm3, %xmm5

  886. 	mulpd	ALPHA_R, %xmm3

  887. 	mulpd	ALPHA_I, %xmm5

  888. 	addpd	%xmm5,   %xmm3

  889. 	SHUFPD_1 %xmm3,  %xmm2

  890. 	addpd	-12 * SIZE(Y), %xmm2

  891. 	movaps	%xmm2, -12 * SIZE(Y)

  892. 

  893. 	pshufd	$0x4e,   %xmm4, %xmm5

  894. 	mulpd	ALPHA_R, %xmm4

  895. 	mulpd	ALPHA_I, %xmm5

  896. 	addpd	%xmm5,   %xmm4

  897. 	SHUFPD_1 %xmm4,  %xmm3

  898. 	addpd	-10 * SIZE(Y), %xmm3

  899. 	movaps	%xmm3, -10 * SIZE(Y)

  900. 

  901. 	movaps	%xmm4, %xmm0

  902. 

  903. 	addl	$8 * SIZE, X

  904. 	addl	$8 * SIZE, Y

  905. 	ALIGN_3

  906. 

  907. .L36:

  908. 	movl	M,  %eax

  909. 	andl	$2, %eax

  910. 	jle	.L37

  911. 

  912. 	movaps	-16 * SIZE(X), %xmm1

  913. 	movaps	-14 * SIZE(X), %xmm2

  914. 

  915. 	pshufd	$0x4e,   %xmm1, %xmm5

  916. 	mulpd	ALPHA_R, %xmm1

  917. 	mulpd	ALPHA_I, %xmm5

  918. 	addpd	%xmm5,   %xmm1

  919. 	SHUFPD_1 %xmm1,  %xmm0

  920. 	addpd	-16 * SIZE(Y), %xmm0

  921. 	movaps	%xmm0, -16 * SIZE(Y)

  922. 

  923. 	pshufd	$0x4e,   %xmm2, %xmm5

  924. 	mulpd	ALPHA_R, %xmm2

  925. 	mulpd	ALPHA_I, %xmm5

  926. 	addpd	%xmm5,   %xmm2

  927. 	SHUFPD_1 %xmm2,  %xmm1

  928. 	addpd	-14 * SIZE(Y), %xmm1

  929. 	movaps	%xmm1, -14 * SIZE(Y)

  930. 

  931. 	movaps	%xmm2, %xmm0

  932. 

  933. 	addl	$4 * SIZE, X

  934. 	addl	$4 * SIZE, Y

  935. 	ALIGN_3

  936. 

  937. .L37:

  938. 	movl	M,  %eax

  939. 	andl	$1, %eax

  940. 	jle	.L39

  941. 

  942. 	movaps	-16 * SIZE(X), %xmm1

  943. 

  944. 	pshufd	$0x4e,   %xmm1, %xmm5

  945. 	mulpd	ALPHA_R, %xmm1

  946. 	mulpd	ALPHA_I, %xmm5

  947. 	addpd	%xmm5,   %xmm1

  948. 	SHUFPD_1 %xmm1,  %xmm0

  949. 

  950. 	addpd	-16 * SIZE(Y), %xmm0

  951. 	movaps	%xmm0, -16 * SIZE(Y)

  952. 	movaps	%xmm1,   %xmm0

  953. 

  954. 	addl	$2 * SIZE, X

  955. 	addl	$2 * SIZE, Y

  956. 	ALIGN_3

  957. 

  958. .L39:

  959. 	SHUFPD_1 %xmm0,  %xmm0

  960. 

  961. 	addsd	-16 * SIZE(Y), %xmm0

  962. 	movlps	%xmm0, -16 * SIZE(Y)

  963. 	jmp	.L999

  964. 	ALIGN_3

  965. 

  966. .L40:

  967. 	movsd	-16 * SIZE(X), %xmm1

  968. 	movhps	-15 * SIZE(X), %xmm1

  969. 

  970. 	pshufd	$0x4e,   %xmm1, %xmm5

  971. 	mulpd	ALPHA_R, %xmm1

  972. 	mulpd	ALPHA_I, %xmm5

  973. 	addpd	%xmm5,   %xmm1

  974. 	xorps	%xmm0,   %xmm0

  975. 	SHUFPD_1 %xmm1,  %xmm0

  976. 

  977. 	xorps	%xmm4,   %xmm4

  978. 	movhps	-16 * SIZE(Y), %xmm4

  979. 

  980. 	addpd	%xmm0,   %xmm4

  981. 	movhps	%xmm4, -16 * SIZE(Y)

  982. 	movaps	%xmm1,   %xmm0

  983. 

  984. 	addl	$2 * SIZE, X

  985. 	addl	$1 * SIZE, Y

  986. 	decl	M

  987. 	jle	.L49

  988. 

  989. 	movl	M,  %eax

  990. 	sarl	$3, %eax

  991. 	jle	.L45

  992. 

  993. 	movsd	-16 * SIZE(X), %xmm1

  994. 	movhps	-15 * SIZE(X), %xmm1

  995. 	movsd	-14 * SIZE(X), %xmm2

  996. 	movhps	-13 * SIZE(X), %xmm2

  997. 	movsd	-12 * SIZE(X), %xmm3

  998. 	movhps	-11 * SIZE(X), %xmm3

  999. 

  1000. 	decl	%eax

  1001. 	jle	.L42

  1002. 	ALIGN_3

  1003. 

  1004. .L41:

  1005. #ifdef PREFETCHW

  1006. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  1007. #endif

  1008. 

  1009. 	pshufd	$0x4e,   %xmm1, %xmm5

  1010. 	mulpd	ALPHA_R, %xmm1

  1011. 	mulpd	ALPHA_I, %xmm5

  1012. 	addpd	%xmm5,   %xmm1

  1013. 	SHUFPD_1 %xmm1,  %xmm0

  1014. 	addpd	-16 * SIZE(Y), %xmm0

  1015. 	movaps	%xmm0, -16 * SIZE(Y)

  1016. 	movsd	-10 * SIZE(X), %xmm0

  1017. 	movhps	 -9 * SIZE(X), %xmm0

  1018. 

  1019. 	pshufd	$0x4e,   %xmm2, %xmm5

  1020. 	mulpd	ALPHA_R, %xmm2

  1021. 	mulpd	ALPHA_I, %xmm5

  1022. 	addpd	%xmm5,   %xmm2

  1023. 	SHUFPD_1 %xmm2,  %xmm1

  1024. 	addpd	-14 * SIZE(Y), %xmm1

  1025. 	movaps	%xmm1, -14 * SIZE(Y)

  1026. 	movsd	 -8 * SIZE(X), %xmm1

  1027. 	movhps	 -7 * SIZE(X), %xmm1

  1028. 

  1029. #ifdef PREFETCH

  1030. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  1031. #endif

  1032. 

  1033. 	pshufd	$0x4e,   %xmm3, %xmm5

  1034. 	mulpd	ALPHA_R, %xmm3

  1035. 	mulpd	ALPHA_I, %xmm5

  1036. 	addpd	%xmm5,   %xmm3

  1037. 	SHUFPD_1 %xmm3,  %xmm2

  1038. 	addpd	-12 * SIZE(Y), %xmm2

  1039. 	movaps	%xmm2, -12 * SIZE(Y)

  1040. 	movsd	 -6 * SIZE(X), %xmm2

  1041. 	movhps	 -5 * SIZE(X), %xmm2

  1042. 

  1043. 	pshufd	$0x4e,   %xmm0, %xmm5

  1044. 	mulpd	ALPHA_R, %xmm0

  1045. 	mulpd	ALPHA_I, %xmm5

  1046. 	addpd	%xmm5,   %xmm0

  1047. 	SHUFPD_1 %xmm0,  %xmm3

  1048. 	addpd	-10 * SIZE(Y), %xmm3

  1049. 	movaps	%xmm3, -10 * SIZE(Y)

  1050. 	movsd	 -4 * SIZE(X), %xmm3

  1051. 	movhps	 -3 * SIZE(X), %xmm3

  1052. 

  1053. #if defined(PREFETCHW) && !defined(FETCH128)

  1054. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  1055. #endif

  1056. 

  1057. 	pshufd	$0x4e,   %xmm1, %xmm5

  1058. 	mulpd	ALPHA_R, %xmm1

  1059. 	mulpd	ALPHA_I, %xmm5

  1060. 	addpd	%xmm5,   %xmm1

  1061. 	SHUFPD_1 %xmm1,  %xmm0

  1062. 	addpd	 -8 * SIZE(Y), %xmm0

  1063. 	movaps	%xmm0,  -8 * SIZE(Y)

  1064. 	movsd	 -2 * SIZE(X), %xmm0

  1065. 	movhps	 -1 * SIZE(X), %xmm0

  1066. 

  1067. 	pshufd	$0x4e,   %xmm2, %xmm5

  1068. 	mulpd	ALPHA_R, %xmm2

  1069. 	mulpd	ALPHA_I, %xmm5

  1070. 	addpd	%xmm5,   %xmm2

  1071. 	SHUFPD_1 %xmm2,  %xmm1

  1072. 	addpd	 -6 * SIZE(Y), %xmm1

  1073. 	movaps	%xmm1,  -6 * SIZE(Y)

  1074. 	movsd	  0 * SIZE(X), %xmm1

  1075. 	movhps	  1 * SIZE(X), %xmm1

  1076. 

  1077. #if defined(PREFETCH) && !defined(FETCH128)

  1078. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  1079. #endif

  1080. 

  1081. 	pshufd	$0x4e,   %xmm3, %xmm5

  1082. 	mulpd	ALPHA_R, %xmm3

  1083. 	mulpd	ALPHA_I, %xmm5

  1084. 	addpd	%xmm5,   %xmm3

  1085. 	SHUFPD_1 %xmm3,  %xmm2

  1086. 	addpd	 -4 * SIZE(Y), %xmm2

  1087. 	movaps	%xmm2,  -4 * SIZE(Y)

  1088. 	movsd	  2 * SIZE(X), %xmm2

  1089. 	movhps	  3 * SIZE(X), %xmm2

  1090. 

  1091. 	pshufd	$0x4e,   %xmm0, %xmm5

  1092. 	mulpd	ALPHA_R, %xmm0

  1093. 	mulpd	ALPHA_I, %xmm5

  1094. 	addpd	%xmm5,   %xmm0

  1095. 	SHUFPD_1 %xmm0,  %xmm3

  1096. 	addpd	 -2 * SIZE(Y), %xmm3

  1097. 	movaps	%xmm3,  -2 * SIZE(Y)

  1098. 	movsd	  4 * SIZE(X), %xmm3

  1099. 	movhps	  5 * SIZE(X), %xmm3

  1100. 

  1101. 	subl	$-16 * SIZE, X

  1102. 	subl	$-16 * SIZE, Y

  1103. 	decl	%eax

  1104. 	jg	.L41

  1105. 	ALIGN_3

  1106. 

  1107. .L42:

  1108. 	pshufd	$0x4e,   %xmm1, %xmm5

  1109. 	mulpd	ALPHA_R, %xmm1

  1110. 	mulpd	ALPHA_I, %xmm5

  1111. 	addpd	%xmm5,   %xmm1

  1112. 	SHUFPD_1 %xmm1,  %xmm0

  1113. 	addpd	-16 * SIZE(Y), %xmm0

  1114. 	movaps	%xmm0, -16 * SIZE(Y)

  1115. 	movsd	-10 * SIZE(X), %xmm0

  1116. 	movhps	 -9 * SIZE(X), %xmm0

  1117. 

  1118. 	pshufd	$0x4e,   %xmm2, %xmm5

  1119. 	mulpd	ALPHA_R, %xmm2

  1120. 	mulpd	ALPHA_I, %xmm5

  1121. 	addpd	%xmm5,   %xmm2

  1122. 	SHUFPD_1 %xmm2,  %xmm1

  1123. 	addpd	-14 * SIZE(Y), %xmm1

  1124. 	movaps	%xmm1, -14 * SIZE(Y)

  1125. 	movsd	 -8 * SIZE(X), %xmm1

  1126. 	movhps	 -7 * SIZE(X), %xmm1

  1127. 

  1128. 	pshufd	$0x4e,   %xmm3, %xmm5

  1129. 	mulpd	ALPHA_R, %xmm3

  1130. 	mulpd	ALPHA_I, %xmm5

  1131. 	addpd	%xmm5,   %xmm3

  1132. 	SHUFPD_1 %xmm3,  %xmm2

  1133. 	addpd	-12 * SIZE(Y), %xmm2

  1134. 	movaps	%xmm2, -12 * SIZE(Y)

  1135. 	movsd	 -6 * SIZE(X), %xmm2

  1136. 	movhps	 -5 * SIZE(X), %xmm2

  1137. 

  1138. 	pshufd	$0x4e,   %xmm0, %xmm5

  1139. 	mulpd	ALPHA_R, %xmm0

  1140. 	mulpd	ALPHA_I, %xmm5

  1141. 	addpd	%xmm5,   %xmm0

  1142. 	SHUFPD_1 %xmm0,  %xmm3

  1143. 	addpd	-10 * SIZE(Y), %xmm3

  1144. 	movaps	%xmm3, -10 * SIZE(Y)

  1145. 	movsd	 -4 * SIZE(X), %xmm3

  1146. 	movhps	 -3 * SIZE(X), %xmm3

  1147. 

  1148. 	pshufd	$0x4e,   %xmm1, %xmm5

  1149. 	mulpd	ALPHA_R, %xmm1

  1150. 	mulpd	ALPHA_I, %xmm5

  1151. 	addpd	%xmm5,   %xmm1

  1152. 	SHUFPD_1 %xmm1,  %xmm0

  1153. 	addpd	 -8 * SIZE(Y), %xmm0

  1154. 	movaps	%xmm0,  -8 * SIZE(Y)

  1155. 	movsd	 -2 * SIZE(X), %xmm0

  1156. 	movhps	 -1 * SIZE(X), %xmm0

  1157. 

  1158. 	pshufd	$0x4e,   %xmm2, %xmm5

  1159. 	mulpd	ALPHA_R, %xmm2

  1160. 	mulpd	ALPHA_I, %xmm5

  1161. 	addpd	%xmm5,   %xmm2

  1162. 	SHUFPD_1 %xmm2,  %xmm1

  1163. 	addpd	 -6 * SIZE(Y), %xmm1

  1164. 	movaps	%xmm1,  -6 * SIZE(Y)

  1165. 

  1166. 	pshufd	$0x4e,   %xmm3, %xmm5

  1167. 	mulpd	ALPHA_R, %xmm3

  1168. 	mulpd	ALPHA_I, %xmm5

  1169. 	addpd	%xmm5,   %xmm3

  1170. 	SHUFPD_1 %xmm3,  %xmm2

  1171. 	addpd	 -4 * SIZE(Y), %xmm2

  1172. 	movaps	%xmm2,  -4 * SIZE(Y)

  1173. 

  1174. 	pshufd	$0x4e,   %xmm0, %xmm5

  1175. 	mulpd	ALPHA_R, %xmm0

  1176. 	mulpd	ALPHA_I, %xmm5

  1177. 	addpd	%xmm5,   %xmm0

  1178. 	SHUFPD_1 %xmm0,  %xmm3

  1179. 	addpd	 -2 * SIZE(Y), %xmm3

  1180. 	movaps	%xmm3,  -2 * SIZE(Y)

  1181. 

  1182. 	subl	$-16 * SIZE, X

  1183. 	subl	$-16 * SIZE, Y

  1184. 	ALIGN_3

  1185. 

  1186. .L45:

  1187. 	movl	M,  %eax

  1188. 	andl	$4, %eax

  1189. 	jle	.L46

  1190. 

  1191. 	movsd	-16 * SIZE(X), %xmm1

  1192. 	movhps	-15 * SIZE(X), %xmm1

  1193. 	movsd	-14 * SIZE(X), %xmm2

  1194. 	movhps	-13 * SIZE(X), %xmm2

  1195. 

  1196. 	pshufd	$0x4e,   %xmm1, %xmm5

  1197. 	mulpd	ALPHA_R, %xmm1

  1198. 	mulpd	ALPHA_I, %xmm5

  1199. 	addpd	%xmm5,   %xmm1

  1200. 	SHUFPD_1 %xmm1,  %xmm0

  1201. 	addpd	-16 * SIZE(Y), %xmm0

  1202. 	movaps	%xmm0, -16 * SIZE(Y)

  1203. 

  1204. 	pshufd	$0x4e,   %xmm2, %xmm5

  1205. 	mulpd	ALPHA_R, %xmm2

  1206. 	mulpd	ALPHA_I, %xmm5

  1207. 	addpd	%xmm5,   %xmm2

  1208. 	SHUFPD_1 %xmm2,  %xmm1

  1209. 	addpd	-14 * SIZE(Y), %xmm1

  1210. 	movaps	%xmm1, -14 * SIZE(Y)

  1211. 

  1212. 	movsd	-12 * SIZE(X), %xmm3

  1213. 	movhps	-11 * SIZE(X), %xmm3

  1214. 	movsd	-10 * SIZE(X), %xmm4

  1215. 	movhps	 -9 * SIZE(X), %xmm4

  1216. 

  1217. 	pshufd	$0x4e,   %xmm3, %xmm5

  1218. 	mulpd	ALPHA_R, %xmm3

  1219. 	mulpd	ALPHA_I, %xmm5

  1220. 	addpd	%xmm5,   %xmm3

  1221. 	SHUFPD_1 %xmm3,  %xmm2

  1222. 	addpd	-12 * SIZE(Y), %xmm2

  1223. 	movaps	%xmm2, -12 * SIZE(Y)

  1224. 

  1225. 	pshufd	$0x4e,   %xmm4, %xmm5

  1226. 	mulpd	ALPHA_R, %xmm4

  1227. 	mulpd	ALPHA_I, %xmm5

  1228. 	addpd	%xmm5,   %xmm4

  1229. 	SHUFPD_1 %xmm4,  %xmm3

  1230. 	addpd	-10 * SIZE(Y), %xmm3

  1231. 	movaps	%xmm3, -10 * SIZE(Y)

  1232. 

  1233. 	movaps	%xmm4, %xmm0

  1234. 

  1235. 	addl	$8 * SIZE, X

  1236. 	addl	$8 * SIZE, Y

  1237. 	ALIGN_3

  1238. 

  1239. .L46:

  1240. 	movl	M,  %eax

  1241. 	andl	$2, %eax

  1242. 	jle	.L47

  1243. 

  1244. 	movsd	-16 * SIZE(X), %xmm1

  1245. 	movhps	-15 * SIZE(X), %xmm1

  1246. 	movsd	-14 * SIZE(X), %xmm2

  1247. 	movhps	-13 * SIZE(X), %xmm2

  1248. 

  1249. 	pshufd	$0x4e,   %xmm1, %xmm5

  1250. 	mulpd	ALPHA_R, %xmm1

  1251. 	mulpd	ALPHA_I, %xmm5

  1252. 	addpd	%xmm5,   %xmm1

  1253. 	SHUFPD_1 %xmm1,  %xmm0

  1254. 	addpd	-16 * SIZE(Y), %xmm0

  1255. 	movaps	%xmm0, -16 * SIZE(Y)

  1256. 

  1257. 	pshufd	$0x4e,   %xmm2, %xmm5

  1258. 	mulpd	ALPHA_R, %xmm2

  1259. 	mulpd	ALPHA_I, %xmm5

  1260. 	addpd	%xmm5,   %xmm2

  1261. 	SHUFPD_1 %xmm2,  %xmm1

  1262. 	addpd	-14 * SIZE(Y), %xmm1

  1263. 	movaps	%xmm1, -14 * SIZE(Y)

  1264. 

  1265. 	movaps	%xmm2, %xmm0

  1266. 

  1267. 	addl	$4 * SIZE, X

  1268. 	addl	$4 * SIZE, Y

  1269. 	ALIGN_3

  1270. 

  1271. .L47:

  1272. 	movl	M,  %eax

  1273. 	andl	$1, %eax

  1274. 	jle	.L49

  1275. 

  1276. 	movsd	-16 * SIZE(X), %xmm1

  1277. 	movhps	-15 * SIZE(X), %xmm1

  1278. 

  1279. 	pshufd	$0x4e,   %xmm1, %xmm5

  1280. 	mulpd	ALPHA_R, %xmm1

  1281. 	mulpd	ALPHA_I, %xmm5

  1282. 	addpd	%xmm5,   %xmm1

  1283. 	SHUFPD_1 %xmm1,  %xmm0

  1284. 

  1285. 	addpd	-16 * SIZE(Y), %xmm0

  1286. 	movaps	%xmm0, -16 * SIZE(Y)

  1287. 	movaps	%xmm1, %xmm0

  1288. 	addl	$2 * SIZE, Y

  1289. 	ALIGN_3

  1290. 

  1291. .L49:

  1292. 	SHUFPD_1 %xmm0,  %xmm0

  1293. 

  1294. 	addsd	-16 * SIZE(Y), %xmm0

  1295. 	movlps	%xmm0, -16 * SIZE(Y)

  1296. 	jmp	.L999

  1297. 	ALIGN_3

  1298. 

  1299. .L50:

  1300. #ifndef CONJ

  1301. 	movaps	%xmm0, ALPHA_R

  1302. 

  1303. 	pxor	ALPHA_I, ALPHA_I

  1304. 	subsd	%xmm1,   ALPHA_I

  1305. 

  1306. 	unpcklpd ALPHA_R, ALPHA_I

  1307. 	unpcklpd %xmm1,   ALPHA_R

  1308. #else

  1309. 	movaps	%xmm0, ALPHA_R

  1310. 	movaps	%xmm1, ALPHA_I

  1311. 

  1312. 	pxor	%xmm5, %xmm5

  1313. 	subsd	%xmm0, %xmm5

  1314. 

  1315. 	unpcklpd %xmm5, ALPHA_I

  1316. 	unpcklpd %xmm1, ALPHA_R

  1317. #endif

  1318. 

  1319. 	movl	Y, YY

  1320. 	movl	M,  %eax

  1321. //If incx==0 || incy==0, avoid unloop and jump to end.

  1322. 	cmpl	$0, INCX

  1323. 	je  .L58

  1324. 	cmpl	$0, INCY

  1325. 	je	.L58

  1326. 

  1327. 	sarl	$2, %eax

  1328. 	jle	.L55

  1329. 

  1330. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1331. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1332. 	addl	INCX, X

  1333. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1334. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1335. 	addl	INCX, X

  1336. 

  1337. 	movsd	 0 * SIZE(Y), %xmm4

  1338. 	movhpd	 1 * SIZE(Y), %xmm4

  1339. 	addl	 INCY, Y

  1340. 	movsd	 0 * SIZE(Y), %xmm5

  1341. 	movhpd	 1 * SIZE(Y), %xmm5

  1342. 	addl	 INCY, Y

  1343. 

  1344. 	decl	%eax

  1345. 	jle	.L52

  1346. 	ALIGN_3

  1347. 

  1348. .L51:

  1349. 	mulpd	ALPHA_R, %xmm0

  1350. 	mulpd	ALPHA_R, %xmm2

  1351. 	mulpd	ALPHA_I, %xmm1

  1352. 	mulpd	ALPHA_I, %xmm3

  1353. 

  1354. 	addpd	%xmm0, %xmm4

  1355. 	addpd	%xmm2, %xmm5

  1356. 	addpd	%xmm1, %xmm4

  1357. 	addpd	%xmm3, %xmm5

  1358. 

  1359. 	movlpd	%xmm4,   0 * SIZE(YY)

  1360. 	movhpd	%xmm4,   1 * SIZE(YY)

  1361. 	addl	 INCY, YY

  1362. 	movlpd	%xmm5,   0 * SIZE(YY)

  1363. 	movhpd	%xmm5,   1 * SIZE(YY)

  1364. 	addl	 INCY, YY

  1365. 

  1366. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1367. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1368. 	addl	INCX, X

  1369. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1370. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1371. 	addl	INCX, X

  1372. 

  1373. 	movsd	 0 * SIZE(Y), %xmm4

  1374. 	movhpd	 1 * SIZE(Y), %xmm4

  1375. 	addl	 INCY, Y

  1376. 	movsd	 0 * SIZE(Y), %xmm5

  1377. 	movhpd	 1 * SIZE(Y), %xmm5

  1378. 	addl	 INCY, Y

  1379. 

  1380. 	mulpd	ALPHA_R, %xmm0

  1381. 	mulpd	ALPHA_R, %xmm2

  1382. 	mulpd	ALPHA_I, %xmm1

  1383. 	mulpd	ALPHA_I, %xmm3

  1384. 

  1385. 	addpd	%xmm0, %xmm4

  1386. 	addpd	%xmm2, %xmm5

  1387. 	addpd	%xmm1, %xmm4

  1388. 	addpd	%xmm3, %xmm5

  1389. 

  1390. 	movlpd	%xmm4,   0 * SIZE(YY)

  1391. 	movhpd	%xmm4,   1 * SIZE(YY)

  1392. 	addl	 INCY, YY

  1393. 	movlpd	%xmm5,   0 * SIZE(YY)

  1394. 	movhpd	%xmm5,   1 * SIZE(YY)

  1395. 	addl	 INCY, YY

  1396. 

  1397. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1398. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1399. 	addl	INCX, X

  1400. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1401. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1402. 	addl	INCX, X

  1403. 

  1404. 	movsd	 0 * SIZE(Y), %xmm4

  1405. 	movhpd	 1 * SIZE(Y), %xmm4

  1406. 	addl	 INCY, Y

  1407. 	movsd	 0 * SIZE(Y), %xmm5

  1408. 	movhpd	 1 * SIZE(Y), %xmm5

  1409. 	addl	 INCY, Y

  1410. 

  1411. 	decl	%eax

  1412. 	jg	.L51

  1413. 	ALIGN_3

  1414. 

  1415. .L52:

  1416. 	mulpd	ALPHA_R, %xmm0

  1417. 	mulpd	ALPHA_R, %xmm2

  1418. 	mulpd	ALPHA_I, %xmm1

  1419. 	mulpd	ALPHA_I, %xmm3

  1420. 

  1421. 	addpd	%xmm0, %xmm4

  1422. 	addpd	%xmm2, %xmm5

  1423. 	addpd	%xmm1, %xmm4

  1424. 	addpd	%xmm3, %xmm5

  1425. 

  1426. 	movlpd	%xmm4,   0 * SIZE(YY)

  1427. 	movhpd	%xmm4,   1 * SIZE(YY)

  1428. 	addl	 INCY, YY

  1429. 	movlpd	%xmm5,   0 * SIZE(YY)

  1430. 	movhpd	%xmm5,   1 * SIZE(YY)

  1431. 	addl	 INCY, YY

  1432. 

  1433. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1434. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1435. 	addl	INCX, X

  1436. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1437. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1438. 	addl	INCX, X

  1439. 

  1440. 	movsd	 0 * SIZE(Y), %xmm4

  1441. 	movhpd	 1 * SIZE(Y), %xmm4

  1442. 	addl	 INCY, Y

  1443. 	movsd	 0 * SIZE(Y), %xmm5

  1444. 	movhpd	 1 * SIZE(Y), %xmm5

  1445. 	addl	 INCY, Y

  1446. 

  1447. 	mulpd	ALPHA_R, %xmm0

  1448. 	mulpd	ALPHA_R, %xmm2

  1449. 	mulpd	ALPHA_I, %xmm1

  1450. 	mulpd	ALPHA_I, %xmm3

  1451. 

  1452. 	addpd	%xmm0, %xmm4

  1453. 	addpd	%xmm2, %xmm5

  1454. 	addpd	%xmm1, %xmm4

  1455. 	addpd	%xmm3, %xmm5

  1456. 

  1457. 	movlpd	%xmm4,   0 * SIZE(YY)

  1458. 	movhpd	%xmm4,   1 * SIZE(YY)

  1459. 	addl	 INCY, YY

  1460. 	movlpd	%xmm5,   0 * SIZE(YY)

  1461. 	movhpd	%xmm5,   1 * SIZE(YY)

  1462. 	addl	 INCY, YY

  1463. 	ALIGN_3

  1464. 

  1465. .L55:

  1466. 	movl	M,  %eax

  1467. 	andl	$2, %eax

  1468. 	jle	.L57

  1469. 

  1470. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1471. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1472. 	addl	INCX, X

  1473. 	MOVDDUP( 0 * SIZE, X, %xmm2)

  1474. 	MOVDDUP( 1 * SIZE, X, %xmm3)

  1475. 	addl	INCX, X

  1476. 

  1477. 	movsd	 0 * SIZE(Y), %xmm4

  1478. 	movhpd	 1 * SIZE(Y), %xmm4

  1479. 	addl	 INCY, Y

  1480. 	movsd	 0 * SIZE(Y), %xmm5

  1481. 	movhpd	 1 * SIZE(Y), %xmm5

  1482. 	addl	 INCY, Y

  1483. 

  1484. 	mulpd	ALPHA_R, %xmm0

  1485. 	mulpd	ALPHA_R, %xmm2

  1486. 	mulpd	ALPHA_I, %xmm1

  1487. 	mulpd	ALPHA_I, %xmm3

  1488. 

  1489. 	addpd	%xmm0, %xmm4

  1490. 	addpd	%xmm2, %xmm5

  1491. 	addpd	%xmm1, %xmm4

  1492. 	addpd	%xmm3, %xmm5

  1493. 

  1494. 	movlpd	%xmm4,   0 * SIZE(YY)

  1495. 	movhpd	%xmm4,   1 * SIZE(YY)

  1496. 	addl	 INCY, YY

  1497. 	movlpd	%xmm5,   0 * SIZE(YY)

  1498. 	movhpd	%xmm5,   1 * SIZE(YY)

  1499. 	addl	 INCY, YY

  1500. 	ALIGN_3

  1501. 

  1502. .L57:

  1503. 	movl	M,  %eax

  1504. 	andl	$1, %eax

  1505. 	jle	.L999

  1506. 

  1507. .L58:

  1508. 	MOVDDUP( 0 * SIZE, X, %xmm0)

  1509. 	MOVDDUP( 1 * SIZE, X, %xmm1)

  1510. 

  1511. 	movsd	 0 * SIZE(Y), %xmm4

  1512. 	movhpd	 1 * SIZE(Y), %xmm4

  1513. 	mulpd	ALPHA_R, %xmm0

  1514. 	mulpd	ALPHA_I, %xmm1

  1515. 	addpd	%xmm0, %xmm4

  1516. 	addpd	%xmm1, %xmm4

  1517. 

  1518. 	movlpd	%xmm4,   0 * SIZE(YY)

  1519. 	movhpd	%xmm4,   1 * SIZE(YY)

  1520. 

  1521. 			

  1522. 	decl %eax

  1523. 	jg	.L58

  1524. 	ALIGN_3

  1525. 

  1526. .L999:

  1527. 	popl	%ebx

  1528. 	popl	%esi

  1529. 	popl	%edi

  1530. 	popl	%ebp

  1531. 	ret

  1532. 

  1533. 	EPILOGUE