OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/zscal_sse2.S

1725 lines
32 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #else

  47. #define M	ARG1

  48. #define X	ARG2

  49. #define INCX	ARG3

  50. #endif

  51. 

  52. #define XX	%r10

  53. #define FLAG	%r11

  54. #define I	%rax

  55. 

  56. #include "l1param.h"

  57. 

  58. #if defined(NEHALEM) || defined(PENRYN) || defined(DUNNINGTON) || defined(BARCELONA) || defined(NANO)  || defined(SANDYBRIDGE)

  59. #define  USE_PSHUFD

  60. #else

  61. #define  USE_PSHUFD_HALF

  62. #endif

  63. 

  64. 	PROLOGUE

  65. 	PROFCODE

  66. 

  67. #ifdef WINDOWS_ABI

  68. 	movaps	%xmm3, %xmm0

  69. 	movsd	40(%rsp), %xmm1

  70. 	movq	48(%rsp), X

  71. 	movq	56(%rsp), INCX

  72. #endif

  73. 

  74. 	SAVEREGISTERS

  75. 

  76. 	salq	$ZBASE_SHIFT, INCX

  77. 	xor	FLAG, FLAG

  78. 

  79. 	testq	M, M

  80. 	jle	.L999

  81. 

  82. 	pxor	%xmm15, %xmm15

  83. 	comisd	%xmm0, %xmm15

  84. 	jne	.L100

  85. 

  86. 	comisd	%xmm1, %xmm15

  87. 	jne	.L100

  88. 

  89. /* Alpha == ZERO */

  90. 	cmpq	$2 * SIZE, INCX

  91. 	jne	.L20

  92. 

  93. /* INCX == 1 */

  94. 	testq	$SIZE, X

  95. 	je	.L05

  96. 

  97. 	movsd	%xmm15, 0 * SIZE(X)

  98. 	addq	$SIZE, X

  99. 	movq	$1, FLAG

  100. 	decq	M

  101. 	jle	.L19

  102. 	ALIGN_3

  103. .L05:

  104. 

  105. 	movq	M,  I	# rcx = n

  106. 	sarq	$3, I

  107. 	jle	.L12

  108. 	ALIGN_4

  109. 

  110. .L11:

  111. #ifdef PREFETCHW

  112. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  113. #endif

  114. 

  115. 	movaps	%xmm15,  0 * SIZE(X)

  116. 	movaps	%xmm15,  2 * SIZE(X)

  117. 	movaps	%xmm15,  4 * SIZE(X)

  118. 	movaps	%xmm15,  6 * SIZE(X)

  119. 

  120. #ifdef PREFETCHW

  121. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  122. #endif

  123. 

  124. 	movaps	%xmm15,  8 * SIZE(X)

  125. 	movaps	%xmm15, 10 * SIZE(X)

  126. 	movaps	%xmm15, 12 * SIZE(X)

  127. 	movaps	%xmm15, 14 * SIZE(X)

  128. 

  129. 	addq	$16 * SIZE, X

  130. 	decq	I

  131. 	jg	.L11

  132. 	ALIGN_4

  133. 

  134. .L12:

  135. 	testq	$4, M

  136. 	je	.L13

  137. 

  138. 	movaps	%xmm15,  0 * SIZE(X)

  139. 	movaps	%xmm15,  2 * SIZE(X)

  140. 	movaps	%xmm15,  4 * SIZE(X)

  141. 	movaps	%xmm15,  6 * SIZE(X)

  142. 	addq	$8 * SIZE, X

  143. 	ALIGN_3

  144. 

  145. .L13:

  146. 	testq  $2, M

  147. 	je    .L14

  148. 

  149. 	movaps	%xmm15,  0 * SIZE(X)

  150. 	movaps	%xmm15,  2 * SIZE(X)

  151. 	addq	$4 * SIZE, X

  152. 	ALIGN_3

  153. 

  154. .L14:

  155. 	testq  $1, M

  156. 	je    .L19

  157. 	movaps	%xmm15,  0 * SIZE(X)

  158. 	addq	$2 * SIZE, X

  159. 	ALIGN_3

  160. 

  161. .L19:

  162. 	testq  $1, FLAG

  163. 	je    .L999

  164. 

  165. 	movsd	%xmm15, 0 * SIZE(X)

  166. 	jmp	.L999

  167. 	ALIGN_4

  168. 

  169. /* incx != 1 */

  170. .L20:

  171. 	testq	$SIZE, X

  172. 	jne	.L30

  173. 

  174. /* Aligned Mode */

  175. 	movq	M,  I		# rcx = n

  176. 	sarq	$2, I

  177. 	jle	.L22

  178. 	ALIGN_4

  179. 

  180. .L21:

  181. #ifdef PREFETCHW

  182. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  183. #endif

  184. 

  185. 	movaps	%xmm15, (X)

  186. 	addq	INCX, X

  187. 	movaps	%xmm15, (X)

  188. 	addq	INCX, X

  189. 	movaps	%xmm15, (X)

  190. 	addq	INCX, X

  191. 	movaps	%xmm15, (X)

  192. 	addq	INCX, X

  193. 	decq	I

  194. 	jg	.L21

  195. 	ALIGN_4

  196. 

  197. .L22:

  198. 	testq	$3, M

  199. 	je	.L999

  200. 

  201. 	testq	$2, M

  202. 	je	.L23

  203. 

  204. 	movaps	%xmm15, (X)

  205. 	addq	INCX, X

  206. 	movaps	%xmm15, (X)

  207. 	addq	INCX, X

  208. 	ALIGN_3

  209. 

  210. .L23:

  211. 	testq	$1, M

  212. 	je	.L999

  213. 

  214. 	movaps	%xmm15, (X)

  215. 	jmp	.L999

  216. 	ALIGN_4

  217. 

  218. 

  219. /* Unaligned Mode */

  220. .L30:

  221. 	movq	M,  I		# rcx = n

  222. 	sarq	$2, I

  223. 	jle	.L32

  224. 	ALIGN_4

  225. 

  226. .L31:

  227. #ifdef PREFETCHW

  228. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  229. #endif

  230. 

  231. 	movlps	%xmm15, 0 * SIZE(X)

  232. 	movlps	%xmm15, 1 * SIZE(X)

  233. 	addq	INCX, X

  234. 	movlps	%xmm15, 0 * SIZE(X)

  235. 	movlps	%xmm15, 1 * SIZE(X)

  236. 	addq	INCX, X

  237. 	movlps	%xmm15, 0 * SIZE(X)

  238. 	movlps	%xmm15, 1 * SIZE(X)

  239. 	addq	INCX, X

  240. 	movlps	%xmm15, 0 * SIZE(X)

  241. 	movlps	%xmm15, 1 * SIZE(X)

  242. 	addq	INCX, X

  243. 	decq	I

  244. 	jg	.L31

  245. 	ALIGN_4

  246. 

  247. .L32:

  248. 	testq	$3, M

  249. 	je	.L999

  250. 

  251. 	testq	$2, M

  252. 	je	.L33

  253. 

  254. 	movlps	%xmm15, 0 * SIZE(X)

  255. 	movlps	%xmm15, 1 * SIZE(X)

  256. 	addq	INCX, X

  257. 	movlps	%xmm15, 0 * SIZE(X)

  258. 	movlps	%xmm15, 1 * SIZE(X)

  259. 	addq	INCX, X

  260. 	ALIGN_3

  261. 

  262. .L33:

  263. 	testq	$1, M

  264. 	je	.L999

  265. 

  266. 	movlps	%xmm15, 0 * SIZE(X)

  267. 	movlps	%xmm15, 1 * SIZE(X)

  268. 	jmp	.L999

  269. 	ALIGN_4

  270. 

  271. /* Alpha != ZERO */

  272. .L100:

  273. 	testq	$SIZE, X

  274. 	jne	.L200

  275. 

  276. #ifdef HAVE_SSE3

  277. 	movddup	%xmm0,  %xmm14

  278. #else

  279. 	pshufd	$0x44, %xmm0, %xmm14

  280. #endif

  281. 	pxor	%xmm15, %xmm15

  282. 	subsd	%xmm1,  %xmm15

  283. 	movlhps	%xmm1,  %xmm15

  284. 

  285. 	cmpq	$2 * SIZE, INCX

  286. 	jne	.L120

  287. 

  288. 	subq	$-16 * SIZE, X

  289. 

  290. 	movq	M,  I

  291. 	sarq	$3, I

  292. 	jle	.L115

  293. 

  294. 	movaps	-16 * SIZE(X), %xmm0

  295. 	movaps	-14 * SIZE(X), %xmm1

  296. 	movaps	-12 * SIZE(X), %xmm2

  297. 	movaps	-10 * SIZE(X), %xmm3

  298. 	movaps	 -8 * SIZE(X), %xmm4

  299. 	movaps	 -6 * SIZE(X), %xmm5

  300. 	movaps	 -4 * SIZE(X), %xmm6

  301. 	movaps	 -2 * SIZE(X), %xmm7

  302. 

  303. 	decq	  I

  304. 	jle	  .L112

  305. 	ALIGN_4

  306. 

  307. .L111:

  308. #ifdef PREFETCHW

  309. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  310. #endif

  311. 

  312. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  313. 	pshufd	$0x4e, %xmm0, %xmm8

  314. #else

  315. 	movsd	 -15 * SIZE(X), %xmm8

  316. 	movhps	 -16 * SIZE(X), %xmm8

  317. #endif

  318. 	mulpd	%xmm14, %xmm0

  319. 	mulpd	%xmm15, %xmm8

  320. 	addpd	%xmm8,  %xmm0

  321. 	movaps	%xmm0,  -16 * SIZE(X)

  322. 	movaps	  0 * SIZE(X), %xmm0

  323. 

  324. #ifdef USE_PSHUFD

  325. 	pshufd	$0x4e, %xmm1, %xmm8

  326. #else

  327. 	movsd	 -13 * SIZE(X), %xmm8

  328. 	movhps	 -14 * SIZE(X), %xmm8

  329. #endif

  330. 	mulpd	%xmm14, %xmm1

  331. 	mulpd	%xmm15, %xmm8

  332. 	addpd	%xmm8,  %xmm1

  333. 	movaps	%xmm1,  -14 * SIZE(X)

  334. 	movaps	  2 * SIZE(X), %xmm1

  335. 

  336. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  337. 	pshufd	$0x4e, %xmm2, %xmm8

  338. #else

  339. 	movsd	 -11 * SIZE(X), %xmm8

  340. 	movhps	 -12 * SIZE(X), %xmm8

  341. #endif

  342. 	mulpd	%xmm14, %xmm2

  343. 	mulpd	%xmm15, %xmm8

  344. 	addpd	%xmm8,  %xmm2

  345. 	movaps	%xmm2,  -12 * SIZE(X)

  346. 	movaps	  4 * SIZE(X), %xmm2

  347. 

  348. #ifdef USE_PSHUFD

  349. 	pshufd	$0x4e, %xmm3, %xmm8

  350. #else

  351. 	movsd	  -9 * SIZE(X), %xmm8

  352. 	movhps	 -10 * SIZE(X), %xmm8

  353. #endif

  354. 	mulpd	%xmm14, %xmm3

  355. 	mulpd	%xmm15, %xmm8

  356. 	addpd	%xmm8,  %xmm3

  357. 	movaps	%xmm3,  -10 * SIZE(X)

  358. 	movaps	  6 * SIZE(X), %xmm3

  359. 

  360. #ifdef PREFETCHW

  361. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  362. #endif

  363. 

  364. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  365. 	pshufd	$0x4e, %xmm4, %xmm8

  366. #else

  367. 	movsd	  -7 * SIZE(X), %xmm8

  368. 	movhps	  -8 * SIZE(X), %xmm8

  369. #endif

  370. 	mulpd	%xmm14, %xmm4

  371. 	mulpd	%xmm15, %xmm8

  372. 	addpd	%xmm8,  %xmm4

  373. 	movaps	%xmm4,   -8 * SIZE(X)

  374. 	movaps	  8 * SIZE(X), %xmm4

  375. 

  376. #ifdef USE_PSHUFD

  377. 	pshufd	$0x4e, %xmm5, %xmm8

  378. #else

  379. 	movsd	  -5 * SIZE(X), %xmm8

  380. 	movhps	  -6 * SIZE(X), %xmm8

  381. #endif

  382. 	mulpd	%xmm14, %xmm5

  383. 	mulpd	%xmm15, %xmm8

  384. 	addpd	%xmm8,  %xmm5

  385. 	movaps	%xmm5,   -6 * SIZE(X)

  386. 	movaps	 10 * SIZE(X), %xmm5

  387. 

  388. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  389. 	pshufd	$0x4e, %xmm6, %xmm8

  390. #else

  391. 	movsd	  -3 * SIZE(X), %xmm8

  392. 	movhps	  -4 * SIZE(X), %xmm8

  393. #endif

  394. 	mulpd	%xmm14, %xmm6

  395. 	mulpd	%xmm15, %xmm8

  396. 	addpd	%xmm8,  %xmm6

  397. 	movaps	%xmm6,   -4 * SIZE(X)

  398. 	movaps	 12 * SIZE(X), %xmm6

  399. 

  400. #ifdef USE_PSHUFD

  401. 	pshufd	$0x4e, %xmm7, %xmm8

  402. #else

  403. 	movsd	  -1 * SIZE(X), %xmm8

  404. 	movhps	  -2 * SIZE(X), %xmm8

  405. #endif

  406. 	mulpd	%xmm14, %xmm7

  407. 	mulpd	%xmm15, %xmm8

  408. 	addpd	%xmm8,  %xmm7

  409. 	movaps	%xmm7,   -2 * SIZE(X)

  410. 	movaps	 14 * SIZE(X), %xmm7

  411. 

  412. 	subq	$-16 * SIZE, X

  413. 	decq	I

  414. 	jg	.L111

  415. 	ALIGN_4

  416. 

  417. .L112:

  418. 	pshufd	$0x4e, %xmm0, %xmm8

  419. 	mulpd	%xmm14, %xmm0

  420. 	mulpd	%xmm15, %xmm8

  421. 	addpd	%xmm8,  %xmm0

  422. 	movaps	%xmm0,  -16 * SIZE(X)

  423. 

  424. 	pshufd	$0x4e, %xmm1, %xmm8

  425. 	mulpd	%xmm14, %xmm1

  426. 	mulpd	%xmm15, %xmm8

  427. 	addpd	%xmm8,  %xmm1

  428. 	movaps	%xmm1,  -14 * SIZE(X)

  429. 

  430. 	pshufd	$0x4e, %xmm2, %xmm8

  431. 	mulpd	%xmm14, %xmm2

  432. 	mulpd	%xmm15, %xmm8

  433. 	addpd	%xmm8,  %xmm2

  434. 	movaps	%xmm2,  -12 * SIZE(X)

  435. 

  436. 	pshufd	$0x4e, %xmm3, %xmm8

  437. 	mulpd	%xmm14, %xmm3

  438. 	mulpd	%xmm15, %xmm8

  439. 	addpd	%xmm8,  %xmm3

  440. 	movaps	%xmm3,  -10 * SIZE(X)

  441. 

  442. 	pshufd	$0x4e, %xmm4, %xmm8

  443. 	mulpd	%xmm14, %xmm4

  444. 	mulpd	%xmm15, %xmm8

  445. 	addpd	%xmm8,  %xmm4

  446. 	movaps	%xmm4,   -8 * SIZE(X)

  447. 

  448. 	pshufd	$0x4e, %xmm5, %xmm8

  449. 	mulpd	%xmm14, %xmm5

  450. 	mulpd	%xmm15, %xmm8

  451. 	addpd	%xmm8,  %xmm5

  452. 	movaps	%xmm5,   -6 * SIZE(X)

  453. 

  454. 	pshufd	$0x4e, %xmm6, %xmm8

  455. 	mulpd	%xmm14, %xmm6

  456. 	mulpd	%xmm15, %xmm8

  457. 	addpd	%xmm8,  %xmm6

  458. 	movaps	%xmm6,   -4 * SIZE(X)

  459. 

  460. 	pshufd	$0x4e, %xmm7, %xmm8

  461. 	mulpd	%xmm14, %xmm7

  462. 	mulpd	%xmm15, %xmm8

  463. 	addpd	%xmm8,  %xmm7

  464. 	movaps	%xmm7,   -2 * SIZE(X)

  465. 

  466. 	subq	$-16 * SIZE, X

  467. 	ALIGN_3

  468. 

  469. .L115:

  470. 	testq	$7, M

  471. 	je	.L999

  472. 

  473. 	testq	$4, M

  474. 	je	.L116

  475. 

  476. 	movaps	-16 * SIZE(X), %xmm0

  477. 	movaps	-14 * SIZE(X), %xmm1

  478. 

  479. 	pshufd	$0x4e, %xmm0, %xmm8

  480. 	mulpd	%xmm14, %xmm0

  481. 	mulpd	%xmm15, %xmm8

  482. 	addpd	%xmm8,  %xmm0

  483. 	movaps	%xmm0,  -16 * SIZE(X)

  484. 

  485. 	pshufd	$0x4e, %xmm1, %xmm8

  486. 	mulpd	%xmm14, %xmm1

  487. 	mulpd	%xmm15, %xmm8

  488. 	addpd	%xmm8,  %xmm1

  489. 	movaps	%xmm1,  -14 * SIZE(X)

  490. 

  491. 	movaps	-12 * SIZE(X), %xmm2

  492. 	movaps	-10 * SIZE(X), %xmm3

  493. 

  494. 	pshufd	$0x4e, %xmm2, %xmm8

  495. 	mulpd	%xmm14, %xmm2

  496. 	mulpd	%xmm15, %xmm8

  497. 	addpd	%xmm8,  %xmm2

  498. 	movaps	%xmm2,  -12 * SIZE(X)

  499. 

  500. 	pshufd	$0x4e, %xmm3, %xmm8

  501. 	mulpd	%xmm14, %xmm3

  502. 	mulpd	%xmm15, %xmm8

  503. 	addpd	%xmm8,  %xmm3

  504. 	movaps	%xmm3,  -10 * SIZE(X)

  505. 

  506. 	addq	$8 * SIZE, X

  507. 	ALIGN_3

  508. 

  509. .L116:

  510. 	testq	$2, M

  511. 	je	.L117

  512. 

  513. 	movaps	-16 * SIZE(X), %xmm0

  514. 	movaps	-14 * SIZE(X), %xmm1

  515. 

  516. 	pshufd	$0x4e, %xmm0, %xmm8

  517. 	mulpd	%xmm14, %xmm0

  518. 	mulpd	%xmm15, %xmm8

  519. 	addpd	%xmm8,  %xmm0

  520. 	movaps	%xmm0,  -16 * SIZE(X)

  521. 

  522. 	pshufd	$0x4e, %xmm1, %xmm8

  523. 	mulpd	%xmm14, %xmm1

  524. 	mulpd	%xmm15, %xmm8

  525. 	addpd	%xmm8,  %xmm1

  526. 	movaps	%xmm1,  -14 * SIZE(X)

  527. 

  528. 	addq	$4 * SIZE, X

  529. 	ALIGN_3

  530. 

  531. .L117:

  532. 	testq	$1, M

  533. 	je	.L999

  534. 

  535. 	movaps	-16 * SIZE(X), %xmm0

  536. 

  537. 	pshufd	$0x4e, %xmm0, %xmm8

  538. 	mulpd	%xmm14, %xmm0

  539. 	mulpd	%xmm15, %xmm8

  540. 	addpd	%xmm8,  %xmm0

  541. 

  542. 	movaps	%xmm0,  -16 * SIZE(X)

  543. 	jmp	.L999

  544. 	ALIGN_3

  545. 

  546. .L120:

  547. 	movq	X, XX

  548. 

  549. 	movq	M,  I

  550. 	sarq	$3, I

  551. 	jle	.L125

  552. 

  553. 	movaps	(X), %xmm0

  554. 	addq	INCX, X

  555. 	movaps	(X), %xmm1

  556. 	addq	INCX, X

  557. 	movaps	(X), %xmm2

  558. 	addq	INCX, X

  559. 	movaps	(X), %xmm3

  560. 	addq	INCX, X

  561. 	movaps	(X), %xmm4

  562. 	addq	INCX, X

  563. 	movaps	(X), %xmm5

  564. 	addq	INCX, X

  565. 	movaps	(X), %xmm6

  566. 	addq	INCX, X

  567. 	movaps	(X), %xmm7

  568. 	addq	INCX, X

  569. 

  570. 	decq	  I

  571. 	jle	  .L122

  572. 	ALIGN_4

  573. 

  574. .L121:

  575. #ifdef PREFETCHW

  576. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  577. #endif

  578. 

  579. 	pshufd	$0x4e, %xmm0, %xmm8

  580. 	mulpd	%xmm14, %xmm0

  581. 	mulpd	%xmm15, %xmm8

  582. 	addpd	%xmm8,  %xmm0

  583. 	movaps	%xmm0,  (XX)

  584. 	addq	INCX, XX

  585. 	movaps	(X), %xmm0

  586. 	addq	INCX, X

  587. 

  588. 	pshufd	$0x4e, %xmm1, %xmm8

  589. 	mulpd	%xmm14, %xmm1

  590. 	mulpd	%xmm15, %xmm8

  591. 	addpd	%xmm8,  %xmm1

  592. 	movaps	%xmm1,  (XX)

  593. 	addq	INCX, XX

  594. 	movaps	(X), %xmm1

  595. 	addq	INCX, X

  596. 

  597. 	pshufd	$0x4e, %xmm2, %xmm8

  598. 	mulpd	%xmm14, %xmm2

  599. 	mulpd	%xmm15, %xmm8

  600. 	addpd	%xmm8,  %xmm2

  601. 	movaps	%xmm2,  (XX)

  602. 	addq	INCX, XX

  603. 	movaps	(X), %xmm2

  604. 	addq	INCX, X

  605. 

  606. 	pshufd	$0x4e, %xmm3, %xmm8

  607. 	mulpd	%xmm14, %xmm3

  608. 	mulpd	%xmm15, %xmm8

  609. 	addpd	%xmm8,  %xmm3

  610. 	movaps	%xmm3,  (XX)

  611. 	addq	INCX, XX

  612. 	movaps	(X), %xmm3

  613. 	addq	INCX, X

  614. 

  615. #ifdef PREFETCHW

  616. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  617. #endif

  618. 

  619. 	pshufd	$0x4e, %xmm4, %xmm8

  620. 	mulpd	%xmm14, %xmm4

  621. 	mulpd	%xmm15, %xmm8

  622. 	addpd	%xmm8,  %xmm4

  623. 	movaps	%xmm4,  (XX)

  624. 	addq	INCX, XX

  625. 	movaps	(X), %xmm4

  626. 	addq	INCX, X

  627. 

  628. 	pshufd	$0x4e, %xmm5, %xmm8

  629. 	mulpd	%xmm14, %xmm5

  630. 	mulpd	%xmm15, %xmm8

  631. 	addpd	%xmm8,  %xmm5

  632. 	movaps	%xmm5,  (XX)

  633. 	addq	INCX, XX

  634. 	movaps	(X), %xmm5

  635. 	addq	INCX, X

  636. 

  637. 	pshufd	$0x4e, %xmm6, %xmm8

  638. 	mulpd	%xmm14, %xmm6

  639. 	mulpd	%xmm15, %xmm8

  640. 	addpd	%xmm8,  %xmm6

  641. 	movaps	%xmm6,  (XX)

  642. 	addq	INCX, XX

  643. 	movaps	(X), %xmm6

  644. 	addq	INCX, X

  645. 

  646. 	pshufd	$0x4e, %xmm7, %xmm8

  647. 	mulpd	%xmm14, %xmm7

  648. 	mulpd	%xmm15, %xmm8

  649. 	addpd	%xmm8,  %xmm7

  650. 	movaps	%xmm7,  (XX)

  651. 	addq	INCX, XX

  652. 	movaps	(X), %xmm7

  653. 	addq	INCX, X

  654. 

  655. 	decq	I

  656. 	jg	.L121

  657. 	ALIGN_4

  658. 

  659. .L122:

  660. 	pshufd	$0x4e, %xmm0, %xmm8

  661. 	mulpd	%xmm14, %xmm0

  662. 	mulpd	%xmm15, %xmm8

  663. 	addpd	%xmm8,  %xmm0

  664. 	movaps	%xmm0,  (XX)

  665. 	addq	INCX, XX

  666. 

  667. 	pshufd	$0x4e, %xmm1, %xmm8

  668. 	mulpd	%xmm14, %xmm1

  669. 	mulpd	%xmm15, %xmm8

  670. 	addpd	%xmm8,  %xmm1

  671. 	movaps	%xmm1,  (XX)

  672. 	addq	INCX, XX

  673. 

  674. 	pshufd	$0x4e, %xmm2, %xmm8

  675. 	mulpd	%xmm14, %xmm2

  676. 	mulpd	%xmm15, %xmm8

  677. 	addpd	%xmm8,  %xmm2

  678. 	movaps	%xmm2,  (XX)

  679. 	addq	INCX, XX

  680. 

  681. 	pshufd	$0x4e, %xmm3, %xmm8

  682. 	mulpd	%xmm14, %xmm3

  683. 	mulpd	%xmm15, %xmm8

  684. 	addpd	%xmm8,  %xmm3

  685. 	movaps	%xmm3,  (XX)

  686. 	addq	INCX, XX

  687. 

  688. 	pshufd	$0x4e, %xmm4, %xmm8

  689. 	mulpd	%xmm14, %xmm4

  690. 	mulpd	%xmm15, %xmm8

  691. 	addpd	%xmm8,  %xmm4

  692. 	movaps	%xmm4,  (XX)

  693. 	addq	INCX, XX

  694. 

  695. 	pshufd	$0x4e, %xmm5, %xmm8

  696. 	mulpd	%xmm14, %xmm5

  697. 	mulpd	%xmm15, %xmm8

  698. 	addpd	%xmm8,  %xmm5

  699. 	movaps	%xmm5,  (XX)

  700. 	addq	INCX, XX

  701. 

  702. 	pshufd	$0x4e, %xmm6, %xmm8

  703. 	mulpd	%xmm14, %xmm6

  704. 	mulpd	%xmm15, %xmm8

  705. 	addpd	%xmm8,  %xmm6

  706. 	movaps	%xmm6,  (XX)

  707. 	addq	INCX, XX

  708. 

  709. 	pshufd	$0x4e, %xmm7, %xmm8

  710. 	mulpd	%xmm14, %xmm7

  711. 	mulpd	%xmm15, %xmm8

  712. 	addpd	%xmm8,  %xmm7

  713. 	movaps	%xmm7,  (XX)

  714. 	addq	INCX, XX

  715. 	ALIGN_3

  716. 

  717. .L125:

  718. 	testq	$7, M

  719. 	je	.L999

  720. 

  721. 	testq	$4, M

  722. 	je	.L126

  723. 

  724. 	movaps	(X), %xmm0

  725. 	addq	INCX, X

  726. 	movaps	(X), %xmm1

  727. 	addq	INCX, X

  728. 

  729. 	pshufd	$0x4e, %xmm0, %xmm8

  730. 	mulpd	%xmm14, %xmm0

  731. 	mulpd	%xmm15, %xmm8

  732. 	addpd	%xmm8,  %xmm0

  733. 	movaps	%xmm0,  (XX)

  734. 	addq	INCX, XX

  735. 

  736. 	pshufd	$0x4e, %xmm1, %xmm8

  737. 	mulpd	%xmm14, %xmm1

  738. 	mulpd	%xmm15, %xmm8

  739. 	addpd	%xmm8,  %xmm1

  740. 	movaps	%xmm1,  (XX)

  741. 	addq	INCX, XX

  742. 

  743. 	movaps	(X), %xmm2

  744. 	addq	INCX, X

  745. 	movaps	(X), %xmm3

  746. 	addq	INCX, X

  747. 

  748. 	pshufd	$0x4e, %xmm2, %xmm8

  749. 	mulpd	%xmm14, %xmm2

  750. 	mulpd	%xmm15, %xmm8

  751. 	addpd	%xmm8,  %xmm2

  752. 	movaps	%xmm2,  (XX)

  753. 	addq	INCX, XX

  754. 

  755. 	pshufd	$0x4e, %xmm3, %xmm8

  756. 	mulpd	%xmm14, %xmm3

  757. 	mulpd	%xmm15, %xmm8

  758. 	addpd	%xmm8,  %xmm3

  759. 	movaps	%xmm3,  (XX)

  760. 	addq	INCX, XX

  761. 	ALIGN_3

  762. 

  763. .L126:

  764. 	testq	$2, M

  765. 	je	.L127

  766. 

  767. 	movaps	(X), %xmm0

  768. 	addq	INCX, X

  769. 	movaps	(X), %xmm1

  770. 	addq	INCX, X

  771. 

  772. 	pshufd	$0x4e, %xmm0, %xmm8

  773. 	mulpd	%xmm14, %xmm0

  774. 	mulpd	%xmm15, %xmm8

  775. 	addpd	%xmm8,  %xmm0

  776. 	movaps	%xmm0,  (XX)

  777. 	addq	INCX, XX

  778. 

  779. 	pshufd	$0x4e, %xmm1, %xmm8

  780. 	mulpd	%xmm14, %xmm1

  781. 	mulpd	%xmm15, %xmm8

  782. 	addpd	%xmm8,  %xmm1

  783. 	movaps	%xmm1,  (XX)

  784. 	addq	INCX, XX

  785. 	ALIGN_3

  786. 

  787. .L127:

  788. 	testq	$1, M

  789. 	je	.L999

  790. 

  791. 	movaps	(X), %xmm0

  792. 

  793. 	pshufd	$0x4e, %xmm0, %xmm8

  794. 	mulpd	%xmm14, %xmm0

  795. 	mulpd	%xmm15, %xmm8

  796. 	addpd	%xmm8,  %xmm0

  797. 

  798. 	movaps	%xmm0,  (XX)

  799. 	jmp	.L999

  800. 	ALIGN_3

  801. 

  802. .L200:

  803. 	cmpq	$2 * SIZE, INCX

  804. 	jne	.L220

  805. 

  806. #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE)

  807. 

  808. 	movddup	%xmm0,  %xmm14

  809. 	pxor	%xmm15, %xmm15

  810. 	subsd	%xmm1,  %xmm15

  811. 	movlhps	%xmm1,  %xmm15

  812. 	shufpd	$1, %xmm15, %xmm15

  813. 

  814. 	movhps	0 * SIZE(X), %xmm0

  815. 	movaps	1 * SIZE(X), %xmm1

  816. 	subq	$-16 * SIZE, X

  817. 

  818. 	unpckhpd %xmm0, %xmm0

  819. 	mulsd	%xmm14, %xmm0

  820. 	movaps	%xmm1,  %xmm8

  821. 	mulsd	%xmm15, %xmm8

  822. 	subsd	%xmm8,  %xmm0

  823. 	movlps	%xmm0, -16 * SIZE(X)

  824. 

  825. 	decq	M

  826. 

  827. 	movq	M,  I

  828. 	sarq	$3, I

  829. 	jle	.L205

  830. 

  831. 	movaps	-13 * SIZE(X), %xmm2

  832. 	movaps	-11 * SIZE(X), %xmm3

  833. 	movaps	 -9 * SIZE(X), %xmm4

  834. 

  835. 	decq	  I

  836. 	jle	  .L202

  837. 	ALIGN_4

  838. 

  839. .L201:

  840. #ifdef PREFETCHW

  841. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  842. #endif

  843. 

  844. 	movaps	%xmm1,  %xmm8

  845. 	SHUFPD_1 %xmm2, %xmm0

  846. 	mulpd	%xmm14, %xmm8

  847. 	mulpd	%xmm15, %xmm0

  848. 	addpd	%xmm8,  %xmm0

  849. 	movaps	%xmm0,  -15 * SIZE(X)

  850. 	movaps	 -7 * SIZE(X), %xmm5

  851. 

  852. 	movaps	%xmm2,  %xmm8

  853. 	SHUFPD_1 %xmm3, %xmm1

  854. 	mulpd	%xmm14, %xmm8

  855. 	mulpd	%xmm15, %xmm1

  856. 	addpd	%xmm8,  %xmm1

  857. 	movaps	%xmm1,  -13 * SIZE(X)

  858. 	movaps	 -5 * SIZE(X), %xmm6

  859. 

  860. 	movaps	%xmm3,  %xmm8

  861. 	SHUFPD_1 %xmm4, %xmm2

  862. 	mulpd	%xmm14, %xmm8

  863. 	mulpd	%xmm15, %xmm2

  864. 	addpd	%xmm8,  %xmm2

  865. 	movaps	%xmm2,  -11 * SIZE(X)

  866. 	movaps	 -3 * SIZE(X), %xmm7

  867. 

  868. 	movaps	%xmm4,  %xmm8

  869. 	SHUFPD_1 %xmm5, %xmm3

  870. 	mulpd	%xmm14, %xmm8

  871. 	mulpd	%xmm15, %xmm3

  872. 	addpd	%xmm8,  %xmm3

  873. 	movaps	%xmm3,   -9 * SIZE(X)

  874.  	movaps	 -1 * SIZE(X), %xmm0

  875. 

  876. #ifdef PREFETCHW

  877. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  878. #endif

  879. 

  880. 	movaps	%xmm5,  %xmm8

  881. 	SHUFPD_1 %xmm6, %xmm4

  882. 	mulpd	%xmm14, %xmm8

  883. 	mulpd	%xmm15, %xmm4

  884. 	addpd	%xmm8,  %xmm4

  885. 	movaps	%xmm4,   -7 * SIZE(X)

  886. 	movaps	  1 * SIZE(X), %xmm1

  887. 

  888. 	movaps	%xmm6,  %xmm8

  889. 	SHUFPD_1 %xmm7, %xmm5

  890. 	mulpd	%xmm14, %xmm8

  891. 	mulpd	%xmm15, %xmm5

  892. 	addpd	%xmm8,  %xmm5

  893. 	movaps	%xmm5,   -5 * SIZE(X)

  894. 	movaps	  3 * SIZE(X), %xmm2

  895. 

  896. 	movaps	%xmm7,  %xmm8

  897. 	SHUFPD_1 %xmm0, %xmm6

  898. 	mulpd	%xmm14, %xmm8

  899. 	mulpd	%xmm15, %xmm6

  900. 	addpd	%xmm8,  %xmm6

  901. 	movaps	%xmm6,   -3 * SIZE(X)

  902. 	movaps	  5 * SIZE(X), %xmm3

  903. 

  904. 	movaps	%xmm0,  %xmm8

  905. 	SHUFPD_1 %xmm1, %xmm7

  906. 	mulpd	%xmm14, %xmm8

  907. 	mulpd	%xmm15, %xmm7

  908. 	addpd	%xmm8,  %xmm7

  909. 	movaps	%xmm7,   -1 * SIZE(X)

  910. 	movaps	  7 * SIZE(X), %xmm4

  911. 

  912. 	subq	$-16 * SIZE, X

  913. 	decq	I

  914. 	jg	.L201

  915. 	ALIGN_4

  916. 

  917. .L202:

  918. 	movaps	%xmm1,  %xmm8

  919. 	SHUFPD_1 %xmm2, %xmm0

  920. 	mulpd	%xmm14, %xmm8

  921. 	mulpd	%xmm15, %xmm0

  922. 	addpd	%xmm8,  %xmm0

  923. 	movaps	%xmm0,  -15 * SIZE(X)

  924. 	movaps	 -7 * SIZE(X), %xmm5

  925. 

  926. 	movaps	%xmm2,  %xmm8

  927. 	SHUFPD_1 %xmm3, %xmm1

  928. 	mulpd	%xmm14, %xmm8

  929. 	mulpd	%xmm15, %xmm1

  930. 	addpd	%xmm8,  %xmm1

  931. 	movaps	%xmm1,  -13 * SIZE(X)

  932. 	movaps	 -5 * SIZE(X), %xmm6

  933. 

  934. 	movaps	%xmm3,  %xmm8

  935. 	SHUFPD_1 %xmm4, %xmm2

  936. 	mulpd	%xmm14, %xmm8

  937. 	mulpd	%xmm15, %xmm2

  938. 	addpd	%xmm8,  %xmm2

  939. 	movaps	%xmm2,  -11 * SIZE(X)

  940. 	movaps	 -3 * SIZE(X), %xmm7

  941. 

  942. 	movaps	%xmm4,  %xmm8

  943. 	SHUFPD_1 %xmm5, %xmm3

  944. 	mulpd	%xmm14, %xmm8

  945. 	mulpd	%xmm15, %xmm3

  946. 	addpd	%xmm8,  %xmm3

  947. 	movaps	%xmm3,   -9 * SIZE(X)

  948. 	movaps	 -1 * SIZE(X), %xmm0

  949. 

  950. 	movaps	%xmm5,  %xmm8

  951. 	SHUFPD_1 %xmm6, %xmm4

  952. 	mulpd	%xmm14, %xmm8

  953. 	mulpd	%xmm15, %xmm4

  954. 	addpd	%xmm8,  %xmm4

  955. 	movaps	%xmm4,   -7 * SIZE(X)

  956. 	movaps	  1 * SIZE(X), %xmm1

  957. 

  958. 	movaps	%xmm6,  %xmm8

  959. 	SHUFPD_1 %xmm7, %xmm5

  960. 	mulpd	%xmm14, %xmm8

  961. 	mulpd	%xmm15, %xmm5

  962. 	addpd	%xmm8,  %xmm5

  963. 	movaps	%xmm5,   -5 * SIZE(X)

  964. 

  965. 	movaps	%xmm7,  %xmm8

  966. 	SHUFPD_1 %xmm0, %xmm6

  967. 	mulpd	%xmm14, %xmm8

  968. 	mulpd	%xmm15, %xmm6

  969. 	addpd	%xmm8,  %xmm6

  970. 	movaps	%xmm6,   -3 * SIZE(X)

  971. 

  972. 	movaps	%xmm0,  %xmm8

  973. 	SHUFPD_1 %xmm1, %xmm7

  974. 	mulpd	%xmm14, %xmm8

  975. 	mulpd	%xmm15, %xmm7

  976. 	addpd	%xmm8,  %xmm7

  977. 	movaps	%xmm7,   -1 * SIZE(X)

  978. 

  979. 	subq	$-16 * SIZE, X

  980. 	ALIGN_3

  981. 

  982. .L205:

  983. 	testq	$4, M

  984. 	je	.L206

  985. 

  986. 	movaps	-13 * SIZE(X), %xmm2

  987. 

  988. 	movaps	%xmm1,  %xmm8

  989. 	SHUFPD_1 %xmm2, %xmm0

  990. 	mulpd	%xmm14, %xmm8

  991. 	mulpd	%xmm15, %xmm0

  992. 	addpd	%xmm8,  %xmm0

  993. 	movaps	%xmm0,  -15 * SIZE(X)

  994. 

  995. 	movaps	-11 * SIZE(X), %xmm3

  996. 

  997. 	movaps	%xmm2,  %xmm8

  998. 	SHUFPD_1 %xmm3, %xmm1

  999. 	mulpd	%xmm14, %xmm8

  1000. 	mulpd	%xmm15, %xmm1

  1001. 	addpd	%xmm8,  %xmm1

  1002. 	movaps	%xmm1,  -13 * SIZE(X)

  1003. 

  1004. 	movaps	 -9 * SIZE(X), %xmm0

  1005. 

  1006. 	movaps	%xmm3,  %xmm8

  1007. 	SHUFPD_1 %xmm0, %xmm2

  1008. 	mulpd	%xmm14, %xmm8

  1009. 	mulpd	%xmm15, %xmm2

  1010. 	addpd	%xmm8,  %xmm2

  1011. 	movaps	%xmm2,  -11 * SIZE(X)

  1012. 

  1013. 	movaps	 -7 * SIZE(X), %xmm1

  1014. 

  1015. 	movaps	%xmm0,  %xmm8

  1016. 	SHUFPD_1 %xmm1, %xmm3

  1017. 	mulpd	%xmm14, %xmm8

  1018. 	mulpd	%xmm15, %xmm3

  1019. 	addpd	%xmm8,  %xmm3

  1020. 	movaps	%xmm3,   -9 * SIZE(X)

  1021. 

  1022. 	addq	$8 * SIZE, X

  1023. 	ALIGN_3

  1024. 

  1025. .L206:

  1026. 	testq	$2, M

  1027. 	je	.L207

  1028. 

  1029. 	movaps	-13 * SIZE(X), %xmm2

  1030. 

  1031. 	movaps	%xmm1,  %xmm8

  1032. 	SHUFPD_1 %xmm2, %xmm0

  1033. 

  1034. 	mulpd	%xmm14, %xmm8

  1035. 	mulpd	%xmm15, %xmm0

  1036. 	addpd	%xmm8,  %xmm0

  1037. 	movaps	%xmm0,  -15 * SIZE(X)

  1038. 

  1039. 	movaps	-11 * SIZE(X), %xmm3

  1040. 

  1041. 	movaps	%xmm2,  %xmm8

  1042. 	SHUFPD_1 %xmm3, %xmm1

  1043. 

  1044. 	mulpd	%xmm14, %xmm8

  1045. 	mulpd	%xmm15, %xmm1

  1046. 	addpd	%xmm8,  %xmm1

  1047. 	movaps	%xmm1,  -13 * SIZE(X)

  1048. 

  1049. 	movaps	%xmm2, %xmm0

  1050. 	movaps	%xmm3, %xmm1

  1051. 

  1052. 	addq	$4 * SIZE, X

  1053. 	ALIGN_3

  1054. 

  1055. .L207:

  1056. 	testq	$1, M

  1057. 	je	.L208

  1058. 

  1059. 	movaps	-13 * SIZE(X), %xmm2

  1060. 

  1061. 	movaps	%xmm1,  %xmm8

  1062. 	SHUFPD_1 %xmm2, %xmm0

  1063. 

  1064. 	mulpd	%xmm14, %xmm8

  1065. 	mulpd	%xmm15, %xmm0

  1066. 	addpd	%xmm8,  %xmm0

  1067. 	movaps	%xmm0,  -15 * SIZE(X)

  1068. 

  1069. 	movaps	%xmm1, %xmm0

  1070. 	movaps	%xmm2, %xmm1

  1071. 	addq	$2 * SIZE, X

  1072. 	ALIGN_3

  1073. 

  1074. .L208:

  1075. 	unpckhpd %xmm0, %xmm0

  1076. 	mulsd	%xmm14, %xmm1

  1077. 	mulsd	%xmm15, %xmm0

  1078. 	addsd	%xmm1,  %xmm0

  1079. 	movlps	%xmm0, -15 * SIZE(X)

  1080. 	jmp	.L999

  1081. 	ALIGN_3

  1082. 

  1083. #else

  1084. 

  1085. 	movddup	%xmm0,  %xmm14

  1086. 	pxor	%xmm15, %xmm15

  1087. 	subsd	%xmm1,  %xmm15

  1088. 	movlhps	%xmm1,  %xmm15

  1089. 

  1090. 	subq	$-16 * SIZE, X

  1091. 

  1092. 	movq	M,  I

  1093. 	sarq	$3, I

  1094. 	jle	.L205

  1095. 

  1096. 	movsd	-16 * SIZE(X), %xmm0

  1097. 	movhps	-15 * SIZE(X), %xmm0

  1098. 	movsd	-14 * SIZE(X), %xmm1

  1099. 	movhps	-13 * SIZE(X), %xmm1

  1100. 	movsd	-12 * SIZE(X), %xmm2

  1101. 	movhps	-11 * SIZE(X), %xmm2

  1102. 	movsd	-10 * SIZE(X), %xmm3

  1103. 	movhps	 -9 * SIZE(X), %xmm3

  1104. 

  1105. 	movsd	 -8 * SIZE(X), %xmm4

  1106. 	movhps	 -7 * SIZE(X), %xmm4

  1107. 	movsd	 -6 * SIZE(X), %xmm5

  1108. 	movhps	 -5 * SIZE(X), %xmm5

  1109. 	movsd	 -4 * SIZE(X), %xmm6

  1110. 	movhps	 -3 * SIZE(X), %xmm6

  1111. 	movsd	 -2 * SIZE(X), %xmm7

  1112. 	movhps	 -1 * SIZE(X), %xmm7

  1113. 

  1114. 	decq	  I

  1115. 	jle	  .L202

  1116. 	ALIGN_4

  1117. 

  1118. .L201:

  1119. #ifdef PREFETCHW

  1120. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  1121. #endif

  1122. 

  1123. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  1124. 	pshufd	$0x4e, %xmm0, %xmm8

  1125. #else

  1126. 	movsd	 -15 * SIZE(X), %xmm8

  1127. 	movhps	 -16 * SIZE(X), %xmm8

  1128. #endif

  1129. 	mulpd	%xmm14, %xmm0

  1130. 	mulpd	%xmm15, %xmm8

  1131. 	addpd	%xmm8,  %xmm0

  1132. 	movlps	%xmm0,  -16 * SIZE(X)

  1133. 	movhps	%xmm0,  -15 * SIZE(X)

  1134. 	movsd	  0 * SIZE(X), %xmm0

  1135. 	movhps	  1 * SIZE(X), %xmm0

  1136. 

  1137. #ifdef USE_PSHUFD

  1138. 	pshufd	$0x4e, %xmm1, %xmm8

  1139. #else

  1140. 	movsd	 -13 * SIZE(X), %xmm8

  1141. 	movhps	 -14 * SIZE(X), %xmm8

  1142. #endif

  1143. 	mulpd	%xmm14, %xmm1

  1144. 	mulpd	%xmm15, %xmm8

  1145. 	addpd	%xmm8,  %xmm1

  1146. 	movlps	%xmm1,  -14 * SIZE(X)

  1147. 	movhps	%xmm1,  -13 * SIZE(X)

  1148. 	movsd	  2 * SIZE(X), %xmm1

  1149. 	movhps	  3 * SIZE(X), %xmm1

  1150. 

  1151. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  1152. 	pshufd	$0x4e, %xmm2, %xmm8

  1153. #else

  1154. 	movsd	 -11 * SIZE(X), %xmm8

  1155. 	movhps	 -12 * SIZE(X), %xmm8

  1156. #endif

  1157. 	mulpd	%xmm14, %xmm2

  1158. 	mulpd	%xmm15, %xmm8

  1159. 	addpd	%xmm8,  %xmm2

  1160. 	movlps	%xmm2,  -12 * SIZE(X)

  1161. 	movhps	%xmm2,  -11 * SIZE(X)

  1162. 	movsd	  4 * SIZE(X), %xmm2

  1163. 	movhps	  5 * SIZE(X), %xmm2

  1164. 

  1165. #ifdef USE_PSHUFD

  1166. 	pshufd	$0x4e, %xmm3, %xmm8

  1167. #else

  1168. 	movsd	  -9 * SIZE(X), %xmm8

  1169. 	movhps	 -10 * SIZE(X), %xmm8

  1170. #endif

  1171. 	mulpd	%xmm14, %xmm3

  1172. 	mulpd	%xmm15, %xmm8

  1173. 	addpd	%xmm8,  %xmm3

  1174. 	movlps	%xmm3,  -10 * SIZE(X)

  1175. 	movhps	%xmm3,   -9 * SIZE(X)

  1176. 	movsd	  6 * SIZE(X), %xmm3

  1177. 	movhps	  7 * SIZE(X), %xmm3

  1178. 

  1179. #ifdef PREFETCHW

  1180. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  1181. #endif

  1182. 

  1183. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  1184. 	pshufd	$0x4e, %xmm4, %xmm8

  1185. #else

  1186. 	movsd	  -7 * SIZE(X), %xmm8

  1187. 	movhps	  -8 * SIZE(X), %xmm8

  1188. #endif

  1189. 	mulpd	%xmm14, %xmm4

  1190. 	mulpd	%xmm15, %xmm8

  1191. 	addpd	%xmm8,  %xmm4

  1192. 	movlps	%xmm4,   -8 * SIZE(X)

  1193. 	movhps	%xmm4,   -7 * SIZE(X)

  1194. 	movsd	  8 * SIZE(X), %xmm4

  1195. 	movhps	  9 * SIZE(X), %xmm4

  1196. 

  1197. #ifdef USE_PSHUFD

  1198. 	pshufd	$0x4e, %xmm5, %xmm8

  1199. #else

  1200. 	movsd	  -5 * SIZE(X), %xmm8

  1201. 	movhps	  -6 * SIZE(X), %xmm8

  1202. #endif

  1203. 	mulpd	%xmm14, %xmm5

  1204. 	mulpd	%xmm15, %xmm8

  1205. 	addpd	%xmm8,  %xmm5

  1206. 	movlps	%xmm5,   -6 * SIZE(X)

  1207. 	movhps	%xmm5,   -5 * SIZE(X)

  1208. 	movsd	 10 * SIZE(X), %xmm5

  1209. 	movhps	 11 * SIZE(X), %xmm5

  1210. 

  1211. #if defined(USE_PSHUFD) || defined(USE_PSHUFD_HALF)

  1212. 	pshufd	$0x4e, %xmm6, %xmm8

  1213. #else

  1214. 	movsd	  -3 * SIZE(X), %xmm8

  1215. 	movhps	  -4 * SIZE(X), %xmm8

  1216. #endif

  1217. 	mulpd	%xmm14, %xmm6

  1218. 	mulpd	%xmm15, %xmm8

  1219. 	addpd	%xmm8,  %xmm6

  1220. 	movlps	%xmm6,   -4 * SIZE(X)

  1221. 	movhps	%xmm6,   -3 * SIZE(X)

  1222. 	movsd	 12 * SIZE(X), %xmm6

  1223. 	movhps	 13 * SIZE(X), %xmm6

  1224. 

  1225. #ifdef USE_PSHUFD

  1226. 	pshufd	$0x4e, %xmm7, %xmm8

  1227. #else

  1228. 	movsd	  -1 * SIZE(X), %xmm8

  1229. 	movhps	  -2 * SIZE(X), %xmm8

  1230. #endif

  1231. 	mulpd	%xmm14, %xmm7

  1232. 	mulpd	%xmm15, %xmm8

  1233. 	addpd	%xmm8,  %xmm7

  1234. 	movlps	%xmm7,   -2 * SIZE(X)

  1235. 	movhps	%xmm7,   -1 * SIZE(X)

  1236. 	movsd	 14 * SIZE(X), %xmm7

  1237. 	movhps	 15 * SIZE(X), %xmm7

  1238. 

  1239. 	subq	$-16 * SIZE, X

  1240. 	decq	I

  1241. 	jg	.L201

  1242. 	ALIGN_4

  1243. 

  1244. .L202:

  1245. 	pshufd	$0x4e, %xmm0, %xmm8

  1246. 	mulpd	%xmm14, %xmm0

  1247. 	mulpd	%xmm15, %xmm8

  1248. 	addpd	%xmm8,  %xmm0

  1249. 	movlps	%xmm0,  -16 * SIZE(X)

  1250. 	movhps	%xmm0,  -15 * SIZE(X)

  1251. 

  1252. 	pshufd	$0x4e, %xmm1, %xmm8

  1253. 	mulpd	%xmm14, %xmm1

  1254. 	mulpd	%xmm15, %xmm8

  1255. 	addpd	%xmm8,  %xmm1

  1256. 	movlps	%xmm1,  -14 * SIZE(X)

  1257. 	movhps	%xmm1,  -13 * SIZE(X)

  1258. 

  1259. 	pshufd	$0x4e, %xmm2, %xmm8

  1260. 	mulpd	%xmm14, %xmm2

  1261. 	mulpd	%xmm15, %xmm8

  1262. 	addpd	%xmm8,  %xmm2

  1263. 	movlps	%xmm2,  -12 * SIZE(X)

  1264. 	movhps	%xmm2,  -11 * SIZE(X)

  1265. 

  1266. 	pshufd	$0x4e, %xmm3, %xmm8

  1267. 	mulpd	%xmm14, %xmm3

  1268. 	mulpd	%xmm15, %xmm8

  1269. 	addpd	%xmm8,  %xmm3

  1270. 	movlps	%xmm3,  -10 * SIZE(X)

  1271. 	movhps	%xmm3,   -9 * SIZE(X)

  1272. 

  1273. 	pshufd	$0x4e, %xmm4, %xmm8

  1274. 	mulpd	%xmm14, %xmm4

  1275. 	mulpd	%xmm15, %xmm8

  1276. 	addpd	%xmm8,  %xmm4

  1277. 	movlps	%xmm4,   -8 * SIZE(X)

  1278. 	movhps	%xmm4,   -7 * SIZE(X)

  1279. 

  1280. 	pshufd	$0x4e, %xmm5, %xmm8

  1281. 	mulpd	%xmm14, %xmm5

  1282. 	mulpd	%xmm15, %xmm8

  1283. 	addpd	%xmm8,  %xmm5

  1284. 	movlps	%xmm5,   -6 * SIZE(X)

  1285. 	movhps	%xmm5,   -5 * SIZE(X)

  1286. 

  1287. 	pshufd	$0x4e, %xmm6, %xmm8

  1288. 	mulpd	%xmm14, %xmm6

  1289. 	mulpd	%xmm15, %xmm8

  1290. 	addpd	%xmm8,  %xmm6

  1291. 	movlps	%xmm6,   -4 * SIZE(X)

  1292. 	movhps	%xmm6,   -3 * SIZE(X)

  1293. 

  1294. 	pshufd	$0x4e, %xmm7, %xmm8

  1295. 	mulpd	%xmm14, %xmm7

  1296. 	mulpd	%xmm15, %xmm8

  1297. 	addpd	%xmm8,  %xmm7

  1298. 	movlps	%xmm7,   -2 * SIZE(X)

  1299. 	movhps	%xmm7,   -1 * SIZE(X)

  1300. 

  1301. 	subq	$-16 * SIZE, X

  1302. 	ALIGN_3

  1303. 

  1304. .L205:

  1305. 	testq	$7, M

  1306. 	je	.L999

  1307. 

  1308. 	testq	$4, M

  1309. 	je	.L206

  1310. 

  1311. 	movsd	-16 * SIZE(X), %xmm0

  1312. 	movhps	-15 * SIZE(X), %xmm0

  1313. 	movsd	-14 * SIZE(X), %xmm1

  1314. 	movhps	-13 * SIZE(X), %xmm1

  1315. 

  1316. 	pshufd	$0x4e, %xmm0, %xmm8

  1317. 	mulpd	%xmm14, %xmm0

  1318. 	mulpd	%xmm15, %xmm8

  1319. 	addpd	%xmm8,  %xmm0

  1320. 	movlps	%xmm0,  -16 * SIZE(X)

  1321. 	movhps	%xmm0,  -15 * SIZE(X)

  1322. 

  1323. 	pshufd	$0x4e, %xmm1, %xmm8

  1324. 	mulpd	%xmm14, %xmm1

  1325. 	mulpd	%xmm15, %xmm8

  1326. 	addpd	%xmm8,  %xmm1

  1327. 	movlps	%xmm1,  -14 * SIZE(X)

  1328. 	movhps	%xmm1,  -13 * SIZE(X)

  1329. 

  1330. 	movsd	-12 * SIZE(X), %xmm2

  1331. 	movhps	-11 * SIZE(X), %xmm2

  1332. 	movsd	-10 * SIZE(X), %xmm3

  1333. 	movhps	 -9 * SIZE(X), %xmm3

  1334. 

  1335. 	pshufd	$0x4e, %xmm2, %xmm8

  1336. 	mulpd	%xmm14, %xmm2

  1337. 	mulpd	%xmm15, %xmm8

  1338. 	addpd	%xmm8,  %xmm2

  1339. 	movlps	%xmm2,  -12 * SIZE(X)

  1340. 	movhps	%xmm2,  -11 * SIZE(X)

  1341. 

  1342. 	pshufd	$0x4e, %xmm3, %xmm8

  1343. 	mulpd	%xmm14, %xmm3

  1344. 	mulpd	%xmm15, %xmm8

  1345. 	addpd	%xmm8,  %xmm3

  1346. 	movlps	%xmm3,  -10 * SIZE(X)

  1347. 	movhps	%xmm3,   -9 * SIZE(X)

  1348. 

  1349. 	addq	$8 * SIZE, X

  1350. 	ALIGN_3

  1351. 

  1352. .L206:

  1353. 	testq	$2, M

  1354. 	je	.L207

  1355. 

  1356. 	movsd	-16 * SIZE(X), %xmm0

  1357. 	movhps	-15 * SIZE(X), %xmm0

  1358. 

  1359. 	pshufd	$0x4e, %xmm0, %xmm8

  1360. 	mulpd	%xmm14, %xmm0

  1361. 	mulpd	%xmm15, %xmm8

  1362. 	addpd	%xmm8,  %xmm0

  1363. 	movlps	%xmm0,  -16 * SIZE(X)

  1364. 	movhps	%xmm0,  -15 * SIZE(X)

  1365. 

  1366. 	movsd	-14 * SIZE(X), %xmm1

  1367. 	movhps	-13 * SIZE(X), %xmm1

  1368. 

  1369. 	pshufd	$0x4e, %xmm1, %xmm8

  1370. 	mulpd	%xmm14, %xmm1

  1371. 	mulpd	%xmm15, %xmm8

  1372. 	addpd	%xmm8,  %xmm1

  1373. 	movlps	%xmm1,  -14 * SIZE(X)

  1374. 	movhps	%xmm1,  -13 * SIZE(X)

  1375. 

  1376. 	addq	$4 * SIZE, X

  1377. 	ALIGN_3

  1378. 

  1379. .L207:

  1380. 	testq	$1, M

  1381. 	je	.L999

  1382. 

  1383. 	movsd	-16 * SIZE(X), %xmm0

  1384. 	movhps	-15 * SIZE(X), %xmm0

  1385. 

  1386. 	pshufd	$0x4e, %xmm0, %xmm8

  1387. 	mulpd	%xmm14, %xmm0

  1388. 	mulpd	%xmm15, %xmm8

  1389. 	addpd	%xmm8,  %xmm0

  1390. 

  1391. 	movlps	%xmm0,  -16 * SIZE(X)

  1392. 	movhps	%xmm0,  -15 * SIZE(X)

  1393. 	jmp	.L999

  1394. 	ALIGN_3

  1395. 

  1396. #endif

  1397. 

  1398. .L220:

  1399. 	movddup	%xmm0,  %xmm14

  1400. 	pxor	%xmm15, %xmm15

  1401. 	subsd	%xmm1,  %xmm15

  1402. 	movlhps	%xmm1,  %xmm15

  1403. 

  1404. 	movq	X, XX

  1405. 

  1406. 	movq	M,  I

  1407. 	sarq	$3, I

  1408. 	jle	.L225

  1409. 

  1410. 	movsd	 0 * SIZE(X), %xmm0

  1411. 	movhps	 1 * SIZE(X), %xmm0

  1412. 	addq	 INCX, X

  1413. 	movsd	 0 * SIZE(X), %xmm1

  1414. 	movhps	 1 * SIZE(X), %xmm1

  1415. 	addq	 INCX, X

  1416. 	movsd	 0 * SIZE(X), %xmm2

  1417. 	movhps	 1 * SIZE(X), %xmm2

  1418. 	addq	 INCX, X

  1419. 	movsd	 0 * SIZE(X), %xmm3

  1420. 	movhps	 1 * SIZE(X), %xmm3

  1421. 	addq	 INCX, X

  1422. 	movsd	 0 * SIZE(X), %xmm4

  1423. 	movhps	 1 * SIZE(X), %xmm4

  1424. 	addq	 INCX, X

  1425. 	movsd	 0 * SIZE(X), %xmm5

  1426. 	movhps	 1 * SIZE(X), %xmm5

  1427. 	addq	 INCX, X

  1428. 	movsd	 0 * SIZE(X), %xmm6

  1429. 	movhps	 1 * SIZE(X), %xmm6

  1430. 	addq	 INCX, X

  1431. 	movsd	 0 * SIZE(X), %xmm7

  1432. 	movhps	 1 * SIZE(X), %xmm7

  1433. 	addq	 INCX, X

  1434. 

  1435. 	decq	  I

  1436. 	jle	  .L222

  1437. 	ALIGN_4

  1438. 

  1439. .L221:

  1440. #ifdef PREFETCHW

  1441. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  1442. #endif

  1443. 

  1444. 	pshufd	$0x4e, %xmm0, %xmm8

  1445. 	mulpd	%xmm14, %xmm0

  1446. 	mulpd	%xmm15, %xmm8

  1447. 	addpd	%xmm8,  %xmm0

  1448. 	movlps	%xmm0,   0 * SIZE(XX)

  1449. 	movhps	%xmm0,   1 * SIZE(XX)

  1450. 	addq	 INCX, XX

  1451. 

  1452. 	movsd	 0 * SIZE(X), %xmm0

  1453. 	movhps	 1 * SIZE(X), %xmm0

  1454. 	addq	 INCX, X

  1455. 

  1456. 	pshufd	$0x4e, %xmm1, %xmm8

  1457. 	mulpd	%xmm14, %xmm1

  1458. 	mulpd	%xmm15, %xmm8

  1459. 	addpd	%xmm8,  %xmm1

  1460. 	movlps	%xmm1,   0 * SIZE(XX)

  1461. 	movhps	%xmm1,   1 * SIZE(XX)

  1462. 	addq	 INCX, XX

  1463. 

  1464. 	movsd	 0 * SIZE(X), %xmm1

  1465. 	movhps	 1 * SIZE(X), %xmm1

  1466. 	addq	 INCX, X

  1467. 

  1468. 	pshufd	$0x4e, %xmm2, %xmm8

  1469. 	mulpd	%xmm14, %xmm2

  1470. 	mulpd	%xmm15, %xmm8

  1471. 	addpd	%xmm8,  %xmm2

  1472. 	movlps	%xmm2,   0 * SIZE(XX)

  1473. 	movhps	%xmm2,   1 * SIZE(XX)

  1474. 	addq	 INCX, XX

  1475. 

  1476. 	movsd	 0 * SIZE(X), %xmm2

  1477. 	movhps	 1 * SIZE(X), %xmm2

  1478. 	addq	 INCX, X

  1479. 

  1480. 	pshufd	$0x4e, %xmm3, %xmm8

  1481. 	mulpd	%xmm14, %xmm3

  1482. 	mulpd	%xmm15, %xmm8

  1483. 	addpd	%xmm8,  %xmm3

  1484. 	movlps	%xmm3,   0 * SIZE(XX)

  1485. 	movhps	%xmm3,   1 * SIZE(XX)

  1486. 	addq	 INCX, XX

  1487. 

  1488. 	movsd	 0 * SIZE(X), %xmm3

  1489. 	movhps	 1 * SIZE(X), %xmm3

  1490. 	addq	 INCX, X

  1491. 

  1492. #ifdef PREFETCHW

  1493. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  1494. #endif

  1495. 

  1496. 	pshufd	$0x4e, %xmm4, %xmm8

  1497. 	mulpd	%xmm14, %xmm4

  1498. 	mulpd	%xmm15, %xmm8

  1499. 	addpd	%xmm8,  %xmm4

  1500. 	movlps	%xmm4,   0 * SIZE(XX)

  1501. 	movhps	%xmm4,   1 * SIZE(XX)

  1502. 	addq	 INCX, XX

  1503. 

  1504. 	movsd	 0 * SIZE(X), %xmm4

  1505. 	movhps	 1 * SIZE(X), %xmm4

  1506. 	addq	 INCX, X

  1507. 

  1508. 	pshufd	$0x4e, %xmm5, %xmm8

  1509. 	mulpd	%xmm14, %xmm5

  1510. 	mulpd	%xmm15, %xmm8

  1511. 	addpd	%xmm8,  %xmm5

  1512. 	movlps	%xmm5,   0 * SIZE(XX)

  1513. 	movhps	%xmm5,   1 * SIZE(XX)

  1514. 	addq	 INCX, XX

  1515. 

  1516. 	movsd	 0 * SIZE(X), %xmm5

  1517. 	movhps	 1 * SIZE(X), %xmm5

  1518. 	addq	 INCX, X

  1519. 

  1520. 	pshufd	$0x4e, %xmm6, %xmm8

  1521. 	mulpd	%xmm14, %xmm6

  1522. 	mulpd	%xmm15, %xmm8

  1523. 	addpd	%xmm8,  %xmm6

  1524. 	movlps	%xmm6,   0 * SIZE(XX)

  1525. 	movhps	%xmm6,   1 * SIZE(XX)

  1526. 	addq	 INCX, XX

  1527. 

  1528. 	movsd	 0 * SIZE(X), %xmm6

  1529. 	movhps	 1 * SIZE(X), %xmm6

  1530. 	addq	 INCX, X

  1531. 

  1532. 	pshufd	$0x4e, %xmm7, %xmm8

  1533. 	mulpd	%xmm14, %xmm7

  1534. 	mulpd	%xmm15, %xmm8

  1535. 	addpd	%xmm8,  %xmm7

  1536. 	movlps	%xmm7,   0 * SIZE(XX)

  1537. 	movhps	%xmm7,   1 * SIZE(XX)

  1538. 	addq	 INCX, XX

  1539. 

  1540. 	movsd	 0 * SIZE(X), %xmm7

  1541. 	movhps	 1 * SIZE(X), %xmm7

  1542. 	addq	 INCX, X

  1543. 

  1544. 

  1545. 	decq	I

  1546. 	jg	.L221

  1547. 	ALIGN_4

  1548. 

  1549. .L222:

  1550. 	pshufd	$0x4e, %xmm0, %xmm8

  1551. 	mulpd	%xmm14, %xmm0

  1552. 	mulpd	%xmm15, %xmm8

  1553. 	addpd	%xmm8,  %xmm0

  1554. 	movlps	%xmm0,   0 * SIZE(XX)

  1555. 	movhps	%xmm0,   1 * SIZE(XX)

  1556. 	addq	 INCX, XX

  1557. 

  1558. 	pshufd	$0x4e, %xmm1, %xmm8

  1559. 	mulpd	%xmm14, %xmm1

  1560. 	mulpd	%xmm15, %xmm8

  1561. 	addpd	%xmm8,  %xmm1

  1562. 	movlps	%xmm1,   0 * SIZE(XX)

  1563. 	movhps	%xmm1,   1 * SIZE(XX)

  1564. 	addq	 INCX, XX

  1565. 

  1566. 	pshufd	$0x4e, %xmm2, %xmm8

  1567. 	mulpd	%xmm14, %xmm2

  1568. 	mulpd	%xmm15, %xmm8

  1569. 	addpd	%xmm8,  %xmm2

  1570. 	movlps	%xmm2,   0 * SIZE(XX)

  1571. 	movhps	%xmm2,   1 * SIZE(XX)

  1572. 	addq	 INCX, XX

  1573. 

  1574. 	pshufd	$0x4e, %xmm3, %xmm8

  1575. 	mulpd	%xmm14, %xmm3

  1576. 	mulpd	%xmm15, %xmm8

  1577. 	addpd	%xmm8,  %xmm3

  1578. 	movlps	%xmm3,   0 * SIZE(XX)

  1579. 	movhps	%xmm3,   1 * SIZE(XX)

  1580. 	addq	 INCX, XX

  1581. 

  1582. 	pshufd	$0x4e, %xmm4, %xmm8

  1583. 	mulpd	%xmm14, %xmm4

  1584. 	mulpd	%xmm15, %xmm8

  1585. 	addpd	%xmm8,  %xmm4

  1586. 	movlps	%xmm4,   0 * SIZE(XX)

  1587. 	movhps	%xmm4,   1 * SIZE(XX)

  1588. 	addq	 INCX, XX

  1589. 

  1590. 	pshufd	$0x4e, %xmm5, %xmm8

  1591. 	mulpd	%xmm14, %xmm5

  1592. 	mulpd	%xmm15, %xmm8

  1593. 	addpd	%xmm8,  %xmm5

  1594. 	movlps	%xmm5,   0 * SIZE(XX)

  1595. 	movhps	%xmm5,   1 * SIZE(XX)

  1596. 	addq	 INCX, XX

  1597. 

  1598. 	pshufd	$0x4e, %xmm6, %xmm8

  1599. 	mulpd	%xmm14, %xmm6

  1600. 	mulpd	%xmm15, %xmm8

  1601. 	addpd	%xmm8,  %xmm6

  1602. 	movlps	%xmm6,   0 * SIZE(XX)

  1603. 	movhps	%xmm6,   1 * SIZE(XX)

  1604. 	addq	 INCX, XX

  1605. 

  1606. 	pshufd	$0x4e, %xmm7, %xmm8

  1607. 	mulpd	%xmm14, %xmm7

  1608. 	mulpd	%xmm15, %xmm8

  1609. 	addpd	%xmm8,  %xmm7

  1610. 	movlps	%xmm7,   0 * SIZE(XX)

  1611. 	movhps	%xmm7,   1 * SIZE(XX)

  1612. 	addq	 INCX, XX

  1613. 	ALIGN_3

  1614. 

  1615. .L225:

  1616. 	testq	$7, M

  1617. 	je	.L999

  1618. 

  1619. 	testq	$4, M

  1620. 	je	.L226

  1621. 

  1622. 	movsd	 0 * SIZE(X), %xmm0

  1623. 	movhps	 1 * SIZE(X), %xmm0

  1624. 	addq	 INCX, X

  1625. 

  1626. 	pshufd	$0x4e, %xmm0, %xmm8

  1627. 	mulpd	%xmm14, %xmm0

  1628. 	mulpd	%xmm15, %xmm8

  1629. 	addpd	%xmm8,  %xmm0

  1630. 	movlps	%xmm0,   0 * SIZE(XX)

  1631. 	movhps	%xmm0,   1 * SIZE(XX)

  1632. 	addq	 INCX, XX

  1633. 

  1634. 	movsd	 0 * SIZE(X), %xmm1

  1635. 	movhps	 1 * SIZE(X), %xmm1

  1636. 	addq	 INCX, X

  1637. 

  1638. 	pshufd	$0x4e, %xmm1, %xmm8

  1639. 	mulpd	%xmm14, %xmm1

  1640. 	mulpd	%xmm15, %xmm8

  1641. 	addpd	%xmm8,  %xmm1

  1642. 	movlps	%xmm1,   0 * SIZE(XX)

  1643. 	movhps	%xmm1,   1 * SIZE(XX)

  1644. 	addq	 INCX, XX

  1645. 

  1646. 	movsd	 0 * SIZE(X), %xmm2

  1647. 	movhps	 1 * SIZE(X), %xmm2

  1648. 	addq	 INCX, X

  1649. 

  1650. 	pshufd	$0x4e, %xmm2, %xmm8

  1651. 	mulpd	%xmm14, %xmm2

  1652. 	mulpd	%xmm15, %xmm8

  1653. 	addpd	%xmm8,  %xmm2

  1654. 	movlps	%xmm2,   0 * SIZE(XX)

  1655. 	movhps	%xmm2,   1 * SIZE(XX)

  1656. 	addq	 INCX, XX

  1657. 

  1658. 	movsd	 0 * SIZE(X), %xmm3

  1659. 	movhps	 1 * SIZE(X), %xmm3

  1660. 	addq	 INCX, X

  1661. 

  1662. 	pshufd	$0x4e, %xmm3, %xmm8

  1663. 	mulpd	%xmm14, %xmm3

  1664. 	mulpd	%xmm15, %xmm8

  1665. 	addpd	%xmm8,  %xmm3

  1666. 	movlps	%xmm3,   0 * SIZE(XX)

  1667. 	movhps	%xmm3,   1 * SIZE(XX)

  1668. 	addq	 INCX, XX

  1669. 	ALIGN_3

  1670. 

  1671. .L226:

  1672. 	testq	$2, M

  1673. 	je	.L227

  1674. 

  1675. 	movsd	 0 * SIZE(X), %xmm0

  1676. 	movhps	 1 * SIZE(X), %xmm0

  1677. 	addq	 INCX, X

  1678. 

  1679. 	pshufd	$0x4e, %xmm0, %xmm8

  1680. 	mulpd	%xmm14, %xmm0

  1681. 	mulpd	%xmm15, %xmm8

  1682. 	addpd	%xmm8,  %xmm0

  1683. 	movlps	%xmm0,   0 * SIZE(XX)

  1684. 	movhps	%xmm0,   1 * SIZE(XX)

  1685. 	addq	 INCX, XX

  1686. 

  1687. 	movsd	 0 * SIZE(X), %xmm1

  1688. 	movhps	 1 * SIZE(X), %xmm1

  1689. 	addq	 INCX, X

  1690. 

  1691. 	pshufd	$0x4e, %xmm1, %xmm8

  1692. 	mulpd	%xmm14, %xmm1

  1693. 	mulpd	%xmm15, %xmm8

  1694. 	addpd	%xmm8,  %xmm1

  1695. 	movlps	%xmm1,   0 * SIZE(XX)

  1696. 	movhps	%xmm1,   1 * SIZE(XX)

  1697. 	addq	 INCX, XX

  1698. 	ALIGN_3

  1699. 

  1700. .L227:

  1701. 	testq	$1, M

  1702. 	je	.L999

  1703. 

  1704. 	movsd	 0 * SIZE(X), %xmm0

  1705. 	movhps	 1 * SIZE(X), %xmm0

  1706. 

  1707. 	pshufd	$0x4e, %xmm0, %xmm8

  1708. 	mulpd	%xmm14, %xmm0

  1709. 	mulpd	%xmm15, %xmm8

  1710. 	addpd	%xmm8,  %xmm0

  1711. 

  1712. 	movlps	%xmm0,  0 * SIZE(XX)

  1713. 	movhps	%xmm0,  1 * SIZE(XX)

  1714. 	ALIGN_3

  1715. 

  1716. .L999:

  1717. 	xorq	%rax, %rax

  1718. 

  1719. 	RESTOREREGISTERS

  1720. 

  1721. 	ret

  1722. 

  1723. 	EPILOGUE