OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: de139337b8
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'de139337b8'
${ noResults }
OpenBLAS/kernel/x86/zscal_sse.S

1390 lines
25 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS     0

  44. 

  45. #define STACK_M	 	 4 + STACK + ARGS(%esp)

  46. #define STACK_ALPHA_R	16 + STACK + ARGS(%esp)

  47. #define STACK_ALPHA_I	20 + STACK + ARGS(%esp)

  48. #define STACK_X		24 + STACK + ARGS(%esp)

  49. #define STACK_INCX	28 + STACK + ARGS(%esp)

  50. 

  51. #define M	%ebx

  52. #define X	%ecx

  53. #define INCX	%edx

  54. #define I	%esi

  55. #define XX	%edi

  56. #define FLAG	%ebp

  57. 

  58. #if defined(NEHALEM) || defined(PENRYN) || defined(DUNNINGTON) || defined(SANDYBRIDGE)

  59. #define  USE_PSHUFD

  60. #else

  61. #define USE_PSHUFD_HALF

  62. #endif

  63. 

  64. #include "l1param.h"

  65. 

  66. 	PROLOGUE

  67. 	PROFCODE

  68. 

  69. 	pushl	%edi

  70. 	pushl	%esi

  71. 	pushl	%ebx

  72. 	pushl	%ebp

  73. 

  74. 	movl	STACK_M,     M

  75. 	movl	STACK_X,     X

  76. 	movl	STACK_INCX,  INCX

  77. 

  78. 	movss	STACK_ALPHA_R, %xmm0

  79. 	movss	STACK_ALPHA_I, %xmm1

  80. 

  81. 	sall	$ZBASE_SHIFT, INCX

  82. 	xor	FLAG, FLAG

  83. 

  84. 	testl	M, M

  85. 	jle	.L999

  86. 

  87. 	xorps	%xmm7, %xmm7

  88. 	comiss	%xmm0, %xmm7

  89. 	jne	.L100		# Alpha_r != ZERO

  90. 

  91. 	comiss	%xmm1, %xmm7

  92. 	jne	.L100		# Alpha_i != ZERO

  93. 

  94. /* Alpha == ZERO */

  95. 	cmpl	$2 * SIZE, INCX

  96. 	jne	.L50

  97. 

  98. /* INCX == 1 */

  99. 	cmpl	$3, M

  100. 	jle	.L13

  101. 

  102. 	testl	$4, X

  103. 	je	.L05

  104. 	movss	%xmm7, 0 * SIZE(X)

  105. 	addl	$SIZE, X

  106. 	movl	$1, FLAG

  107. 	decl	M

  108. 	ALIGN_3

  109. 

  110. .L05:

  111. 	testl	$8, X

  112. 	je	.L06

  113. 

  114. 	movlps	%xmm7, 0 * SIZE(X)

  115. 	addl	$2 * SIZE, X

  116. 	subl	$1, M

  117. 	ALIGN_3

  118. .L06:

  119. 

  120. 	movl	M,  I	# rcx = n

  121. 	sarl	$3, I

  122. 	jle	.L12

  123. 	ALIGN_4

  124. 

  125. .L11:

  126. #ifdef PREFETCHW

  127. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  128. #endif

  129. 

  130. 	movaps	%xmm7,  0 * SIZE(X)

  131. 	movaps	%xmm7,  4 * SIZE(X)

  132. 	movaps	%xmm7,  8 * SIZE(X)

  133. 	movaps	%xmm7, 12 * SIZE(X)

  134. 	addl	$16 * SIZE, X

  135. 	decl	I

  136. 	jg	.L11

  137. 	ALIGN_4

  138. 

  139. .L12:

  140. 	testl	$7, M

  141. 	je	.L19

  142. 	testl	$4, M

  143. 	je	.L13

  144. 

  145. 	movaps	%xmm7,  0 * SIZE(X)

  146. 	movaps	%xmm7,  4 * SIZE(X)

  147. 	addl	$8 * SIZE, X

  148. 	ALIGN_3

  149. 

  150. .L13:

  151. 	testl  $2, M

  152. 	je    .L14

  153. 

  154. 	movlps	%xmm7,  0 * SIZE(X)

  155. 	movhps	%xmm7,  2 * SIZE(X)

  156. 	addl	$4 * SIZE, X

  157. 	ALIGN_3

  158. 

  159. .L14:

  160. 	testl  $1, M

  161. 	je    .L19

  162. 

  163. 	movlps	%xmm7,  0 * SIZE(X)

  164. 	addl	$2 * SIZE, X

  165. 	ALIGN_3

  166. 

  167. .L19:

  168. 	testl  $1, FLAG

  169. 	je    .L999

  170. 

  171. 	movss	%xmm7, 0 * SIZE(X)

  172. 	jmp	.L999

  173. 	ALIGN_4

  174. 

  175. /* incx != 1 */

  176. .L50:

  177. 	movl	M,  I		# rcx = n

  178. 	sarl	$2, I

  179. 	jle	.L52

  180. 	ALIGN_4

  181. 

  182. .L51:

  183. #ifdef PREFETCHW

  184. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  185. #endif

  186. 

  187. 	movsd	%xmm7, 0 * SIZE(X)

  188. 	addl	INCX, X

  189. 	movsd	%xmm7, 0 * SIZE(X)

  190. 	addl	INCX, X

  191. 	movsd	%xmm7, 0 * SIZE(X)

  192. 	addl	INCX, X

  193. 	movsd	%xmm7, 0 * SIZE(X)

  194. 	addl	INCX, X

  195. 	decl	I

  196. 	jg	.L51

  197. 	ALIGN_4

  198. 

  199. .L52:

  200. 	testl	$2, M

  201. 	je	.L53

  202. 

  203. 	movsd	%xmm7, 0 * SIZE(X)

  204. 	addl	INCX, X

  205. 	movsd	%xmm7, 0 * SIZE(X)

  206. 	addl	INCX, X

  207. 	ALIGN_3

  208. 

  209. .L53:

  210. 	testl	$1, M

  211. 	je	.L999

  212. 

  213. 	movsd	%xmm7, 0 * SIZE(X)

  214. 	jmp	.L999

  215. 	ALIGN_4

  216. 

  217. /* Alpha != ZERO */

  218. 

  219. .L100:

  220. 	testl	$SIZE, X

  221. 	jne	.L130

  222. 

  223. 	cmpl	$2 * SIZE, INCX

  224. 	jne	.L120

  225. 

  226. 	movaps	%xmm0, %xmm6

  227. 	shufps	$0, %xmm6, %xmm6

  228. 	shufps	$0, %xmm1, %xmm1

  229. 	subps	  %xmm1,   %xmm7

  230. 	unpcklps  %xmm1,   %xmm7

  231. 

  232. 	subl	$-32 * SIZE, X

  233. 

  234. 	testl	$2 * SIZE, X

  235. 	je	.L105

  236. 

  237. 	movsd	-32 * SIZE(X), %xmm0

  238. 

  239. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  240. 	mulps	 %xmm6, %xmm0

  241. 	mulps	 %xmm7, %xmm5

  242. 	addps	 %xmm5, %xmm0

  243. 

  244. 	movlps	%xmm0,  -32 * SIZE(X)

  245. 	addl	$2 * SIZE, X

  246. 	decl	M

  247. 	jle	.L999

  248. 	ALIGN_3

  249. 

  250. .L105:

  251. 	movl	M,  I

  252. 	sarl	$4, I

  253. 	jle	.L115

  254. 

  255. 	movaps	-32 * SIZE(X), %xmm0

  256. 	movaps	-28 * SIZE(X), %xmm1

  257. 	movaps	-24 * SIZE(X), %xmm2

  258. 	movaps	-20 * SIZE(X), %xmm3

  259. 

  260. 	decl	I

  261. 	jle	.L112

  262. 	ALIGN_4

  263. 

  264. .L111:

  265. #ifdef PREFETCHW

  266. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  267. #endif

  268. 

  269. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  270. 	mulps	 %xmm6, %xmm0

  271. 	mulps	 %xmm7, %xmm5

  272. 	addps	 %xmm5, %xmm0

  273. 	movaps	%xmm0,  -32 * SIZE(X)

  274. 	movaps	-16 * SIZE(X), %xmm0

  275. 

  276. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  277. 	mulps	 %xmm6, %xmm1

  278. 	mulps	 %xmm7, %xmm5

  279. 	addps	 %xmm5, %xmm1

  280. 	movaps	%xmm1,  -28 * SIZE(X)

  281. 	movaps	-12 * SIZE(X), %xmm1

  282. 

  283. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  284. 	mulps	 %xmm6, %xmm2

  285. 	mulps	 %xmm7, %xmm5

  286. 	addps	 %xmm5, %xmm2

  287. 	movaps	%xmm2,  -24 * SIZE(X)

  288. 	movaps	 -8 * SIZE(X), %xmm2

  289. 

  290. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  291. 	mulps	 %xmm6, %xmm3

  292. 	mulps	 %xmm7, %xmm5

  293. 	addps	 %xmm5, %xmm3

  294. 	movaps	%xmm3,  -20 * SIZE(X)

  295. 	movaps	 -4 * SIZE(X), %xmm3

  296. 

  297. #ifdef PREFETCHW

  298. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  299. #endif

  300. 

  301. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  302. 	mulps	 %xmm6, %xmm0

  303. 	mulps	 %xmm7, %xmm5

  304. 	addps	 %xmm5, %xmm0

  305. 	movaps	%xmm0,  -16 * SIZE(X)

  306. 	movaps	  0 * SIZE(X), %xmm0

  307. 

  308. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  309. 	mulps	 %xmm6, %xmm1

  310. 	mulps	 %xmm7, %xmm5

  311. 	addps	 %xmm5, %xmm1

  312. 	movaps	%xmm1,  -12 * SIZE(X)

  313. 	movaps	  4 * SIZE(X), %xmm1

  314. 

  315. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  316. 	mulps	 %xmm6, %xmm2

  317. 	mulps	 %xmm7, %xmm5

  318. 	addps	 %xmm5, %xmm2

  319. 	movaps	%xmm2,   -8 * SIZE(X)

  320. 	movaps	  8 * SIZE(X), %xmm2

  321. 

  322. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  323. 	mulps	 %xmm6, %xmm3

  324. 	mulps	 %xmm7, %xmm5

  325. 	addps	 %xmm5, %xmm3

  326. 	movaps	%xmm3,   -4 * SIZE(X)

  327. 	movaps	 12 * SIZE(X), %xmm3

  328. 

  329. 	subl	$-32 * SIZE, X

  330. 	decl	I

  331. 	jg	.L111

  332. 	ALIGN_4

  333. 

  334. .L112:

  335. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  336. 	mulps	 %xmm6, %xmm0

  337. 	mulps	 %xmm7, %xmm5

  338. 	addps	 %xmm5, %xmm0

  339. 	movaps	%xmm0,  -32 * SIZE(X)

  340. 	movaps	-16 * SIZE(X), %xmm0

  341. 

  342. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  343. 	mulps	 %xmm6, %xmm1

  344. 	mulps	 %xmm7, %xmm5

  345. 	addps	 %xmm5, %xmm1

  346. 	movaps	%xmm1,  -28 * SIZE(X)

  347. 	movaps	-12 * SIZE(X), %xmm1

  348. 

  349. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  350. 	mulps	 %xmm6, %xmm2

  351. 	mulps	 %xmm7, %xmm5

  352. 	addps	 %xmm5, %xmm2

  353. 	movaps	%xmm2,  -24 * SIZE(X)

  354. 	movaps	 -8 * SIZE(X), %xmm2

  355. 

  356. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  357. 	mulps	 %xmm6, %xmm3

  358. 	mulps	 %xmm7, %xmm5

  359. 	addps	 %xmm5, %xmm3

  360. 	movaps	%xmm3,  -20 * SIZE(X)

  361. 	movaps	 -4 * SIZE(X), %xmm3

  362. 

  363. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  364. 	mulps	 %xmm6, %xmm0

  365. 	mulps	 %xmm7, %xmm5

  366. 	addps	 %xmm5, %xmm0

  367. 	movaps	%xmm0,  -16 * SIZE(X)

  368. 

  369. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  370. 	mulps	 %xmm6, %xmm1

  371. 	mulps	 %xmm7, %xmm5

  372. 	addps	 %xmm5, %xmm1

  373. 	movaps	%xmm1,  -12 * SIZE(X)

  374. 

  375. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  376. 	mulps	 %xmm6, %xmm2

  377. 	mulps	 %xmm7, %xmm5

  378. 	addps	 %xmm5, %xmm2

  379. 	movaps	%xmm2,   -8 * SIZE(X)

  380. 

  381. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  382. 	mulps	 %xmm6, %xmm3

  383. 	mulps	 %xmm7, %xmm5

  384. 	addps	 %xmm5, %xmm3

  385. 	movaps	%xmm3,   -4 * SIZE(X)

  386. 

  387. 	subl	$-32 * SIZE, X

  388. 	ALIGN_4

  389. 

  390. .L115:

  391. 	testl	$8, M

  392. 	je	.L116

  393. 

  394. 	movaps	-32 * SIZE(X), %xmm0

  395. 	movaps	-28 * SIZE(X), %xmm1

  396. 

  397. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  398. 	mulps	 %xmm6, %xmm0

  399. 	mulps	 %xmm7, %xmm5

  400. 	addps	 %xmm5, %xmm0

  401. 	movaps	%xmm0,  -32 * SIZE(X)

  402. 

  403. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  404. 	mulps	 %xmm6, %xmm1

  405. 	mulps	 %xmm7, %xmm5

  406. 	addps	 %xmm5, %xmm1

  407. 	movaps	%xmm1,  -28 * SIZE(X)

  408. 

  409. 	movaps	-24 * SIZE(X), %xmm2

  410. 	movaps	-20 * SIZE(X), %xmm3

  411. 

  412. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  413. 	mulps	 %xmm6, %xmm2

  414. 	mulps	 %xmm7, %xmm5

  415. 	addps	 %xmm5, %xmm2

  416. 	movaps	%xmm2,  -24 * SIZE(X)

  417. 

  418. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  419. 	mulps	 %xmm6, %xmm3

  420. 	mulps	 %xmm7, %xmm5

  421. 	addps	 %xmm5, %xmm3

  422. 	movaps	%xmm3,  -20 * SIZE(X)

  423. 

  424. 	addl	$16 * SIZE, X

  425. 	ALIGN_3

  426. 

  427. .L116:

  428. 	testl	$4, M

  429. 	je	.L117

  430. 

  431. 	movaps	-32 * SIZE(X), %xmm0

  432. 	movaps	-28 * SIZE(X), %xmm1

  433. 

  434. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  435. 	mulps	 %xmm6, %xmm0

  436. 	mulps	 %xmm7, %xmm5

  437. 	addps	 %xmm5, %xmm0

  438. 	movaps	%xmm0,  -32 * SIZE(X)

  439. 

  440. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  441. 	mulps	 %xmm6, %xmm1

  442. 	mulps	 %xmm7, %xmm5

  443. 	addps	 %xmm5, %xmm1

  444. 	movaps	%xmm1,  -28 * SIZE(X)

  445. 

  446. 	addl	$8 * SIZE, X

  447. 	ALIGN_3

  448. 

  449. .L117:

  450. 	testl	$2, M

  451. 	je	.L118

  452. 

  453. 	movaps	-32 * SIZE(X), %xmm0

  454. 

  455. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  456. 	mulps	 %xmm6, %xmm0

  457. 	mulps	 %xmm7, %xmm5

  458. 	addps	 %xmm5, %xmm0

  459. 	movaps	%xmm0,  -32 * SIZE(X)

  460. 

  461. 	addl	$4 * SIZE, X

  462. 	ALIGN_3

  463. 

  464. .L118:

  465. 	testl	$1, M

  466. 	je	.L999

  467. 

  468. 	movsd	-32 * SIZE(X), %xmm0

  469. 

  470. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  471. 	mulps	 %xmm6, %xmm0

  472. 	mulps	 %xmm7, %xmm5

  473. 	addps	 %xmm5, %xmm0

  474. 

  475. 	movlps	%xmm0,  -32 * SIZE(X)

  476. 	jmp	.L999

  477. 	ALIGN_3

  478. 

  479. .L120:

  480. 	PSHUFD2($0, %xmm0, %xmm6)

  481. 	PSHUFD2($0, %xmm1, %xmm1)

  482. 	subps	  %xmm1,   %xmm7

  483. 	unpcklps  %xmm1,   %xmm7

  484. 

  485. 	movl	X, XX

  486. 

  487. 	movl	M,  I

  488. 	sarl	$3, I

  489. 	jle	.L125

  490. 

  491. 	movsd	(X), %xmm0

  492. 	addl	INCX, X

  493. 	movhps	(X), %xmm0

  494. 	addl	INCX, X

  495. 

  496. 	movsd	(X), %xmm1

  497. 	addl	INCX, X

  498. 	movhps	(X), %xmm1

  499. 	addl	INCX, X

  500. 

  501. 	movsd	(X), %xmm2

  502. 	addl	INCX, X

  503. 	movhps	(X), %xmm2

  504. 	addl	INCX, X

  505. 

  506. 	movsd	(X), %xmm3

  507. 	addl	INCX, X

  508. 	movhps	(X), %xmm3

  509. 	addl	INCX, X

  510. 

  511. 	decl	I

  512. 	jle	.L122

  513. 	ALIGN_4

  514. 

  515. .L121:

  516. #ifdef PREFETCHW

  517. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  518. #endif

  519. 

  520. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  521. 	mulps	 %xmm6, %xmm0

  522. 	mulps	 %xmm7, %xmm5

  523. 	addps	 %xmm5,  %xmm0

  524. 

  525. 	movlps	%xmm0,  (XX)

  526. 	addl	INCX, XX

  527. 	movhps	%xmm0,  (XX)

  528. 	addl	INCX, XX

  529. 

  530. 	movsd	(X), %xmm0

  531. 	addl	INCX, X

  532. 	movhps	(X), %xmm0

  533. 	addl	INCX, X

  534. 

  535. 	PSHUFD2( $0xb1,  %xmm1, %xmm5)

  536. 	mulps	 %xmm6, %xmm1

  537. 	mulps	 %xmm7, %xmm5

  538. 	addps	 %xmm5,  %xmm1

  539. 

  540. 	movlps	%xmm1,  (XX)

  541. 	addl	INCX, XX

  542. 	movhps	%xmm1,  (XX)

  543. 	addl	INCX, XX

  544. 

  545. 	movsd	(X), %xmm1

  546. 	addl	INCX, X

  547. 	movhps	(X), %xmm1

  548. 	addl	INCX, X

  549. 

  550. 	PSHUFD2( $0xb1,  %xmm2, %xmm5)

  551. 	mulps	 %xmm6, %xmm2

  552. 	mulps	 %xmm7, %xmm5

  553. 	addps	 %xmm5,  %xmm2

  554. 

  555. 	movlps	%xmm2,  (XX)

  556. 	addl	INCX, XX

  557. 	movhps	%xmm2,  (XX)

  558. 	addl	INCX, XX

  559. 

  560. 	movsd	(X), %xmm2

  561. 	addl	INCX, X

  562. 	movhps	(X), %xmm2

  563. 	addl	INCX, X

  564. 

  565. 	PSHUFD2( $0xb1,  %xmm3, %xmm5)

  566. 	mulps	 %xmm6, %xmm3

  567. 	mulps	 %xmm7, %xmm5

  568. 	addps	 %xmm5,  %xmm3

  569. 

  570. 	movlps	%xmm3,  (XX)

  571. 	addl	INCX, XX

  572. 	movhps	%xmm3,  (XX)

  573. 	addl	INCX, XX

  574. 

  575. 	movsd	(X), %xmm3

  576. 	addl	INCX, X

  577. 	movhps	(X), %xmm3

  578. 	addl	INCX, X

  579. 

  580. 	decl	I

  581. 	jg	.L121

  582. 	ALIGN_4

  583. 

  584. .L122:

  585. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  586. 	mulps	 %xmm6, %xmm0

  587. 	mulps	 %xmm7, %xmm5

  588. 	addps	 %xmm5,  %xmm0

  589. 

  590. 	movlps	%xmm0,  (XX)

  591. 	addl	INCX, XX

  592. 	movhps	%xmm0,  (XX)

  593. 	addl	INCX, XX

  594. 

  595. 	PSHUFD2( $0xb1,  %xmm1, %xmm5)

  596. 	mulps	 %xmm6, %xmm1

  597. 	mulps	 %xmm7, %xmm5

  598. 	addps	 %xmm5,  %xmm1

  599. 

  600. 	movlps	%xmm1,  (XX)

  601. 	addl	INCX, XX

  602. 	movhps	%xmm1,  (XX)

  603. 	addl	INCX, XX

  604. 

  605. 	PSHUFD2( $0xb1,  %xmm2, %xmm5)

  606. 	mulps	 %xmm6, %xmm2

  607. 	mulps	 %xmm7, %xmm5

  608. 	addps	 %xmm5,  %xmm2

  609. 

  610. 	movlps	%xmm2,  (XX)

  611. 	addl	INCX, XX

  612. 	movhps	%xmm2,  (XX)

  613. 	addl	INCX, XX

  614. 

  615. 	PSHUFD2( $0xb1,  %xmm3, %xmm5)

  616. 	mulps	 %xmm6, %xmm3

  617. 	mulps	 %xmm7, %xmm5

  618. 	addps	 %xmm5,  %xmm3

  619. 

  620. 	movlps	%xmm3,  (XX)

  621. 	addl	INCX, XX

  622. 	movhps	%xmm3,  (XX)

  623. 	addl	INCX, XX

  624. 	ALIGN_4

  625. 

  626. .L125:

  627. 	testl	$4, M

  628. 	je	.L127

  629. 

  630. 	movsd	(X), %xmm0

  631. 	addl	INCX, X

  632. 	movhps	(X), %xmm0

  633. 	addl	INCX, X

  634. 

  635. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  636. 	mulps	 %xmm6, %xmm0

  637. 	mulps	 %xmm7, %xmm5

  638. 	addps	 %xmm5,  %xmm0

  639. 

  640. 	movlps	%xmm0,  (XX)

  641. 	addl	INCX, XX

  642. 	movhps	%xmm0,  (XX)

  643. 	addl	INCX, XX

  644. 

  645. 	movsd	(X), %xmm1

  646. 	addl	INCX, X

  647. 	movhps	(X), %xmm1

  648. 	addl	INCX, X

  649. 

  650. 	PSHUFD2( $0xb1,  %xmm1, %xmm5)

  651. 	mulps	 %xmm6, %xmm1

  652. 	mulps	 %xmm7, %xmm5

  653. 	addps	 %xmm5,  %xmm1

  654. 

  655. 	movlps	%xmm1,  (XX)

  656. 	addl	INCX, XX

  657. 	movhps	%xmm1,  (XX)

  658. 	addl	INCX, XX

  659. 	ALIGN_3

  660. 

  661. .L127:

  662. 	testl	$2, M

  663. 	je	.L128

  664. 

  665. 	movsd	(X), %xmm0

  666. 	addl	INCX, X

  667. 	movhps	(X), %xmm0

  668. 	addl	INCX, X

  669. 

  670. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  671. 	mulps	 %xmm6, %xmm0

  672. 	mulps	 %xmm7, %xmm5

  673. 	addps	 %xmm5,  %xmm0

  674. 

  675. 	movlps	%xmm0,  (XX)

  676. 	addl	INCX, XX

  677. 	movhps	%xmm0,  (XX)

  678. 	addl	INCX, XX

  679. 	ALIGN_3

  680. 

  681. .L128:

  682. 	testl	$1, M

  683. 	je	.L999

  684. 

  685. 	movsd	(X), %xmm0

  686. 

  687. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  688. 	mulps	 %xmm6, %xmm0

  689. 	mulps	 %xmm7, %xmm5

  690. 	addps	 %xmm5,  %xmm0

  691. 

  692. 	movlps	%xmm0,  (XX)

  693. 	jmp	.L999

  694. 	ALIGN_3

  695. 

  696. .L130:

  697. 	cmpl	$2 * SIZE, INCX

  698. 	jne	.L120

  699. 

  700. #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) &&  !defined(SANDYBRIDGE)

  701. 

  702. 	PSHUFD2($0, %xmm0, %xmm6)

  703. 	PSHUFD2($0, %xmm1, %xmm1)

  704. 	subps	  %xmm1,   %xmm7

  705. 	unpcklps  %xmm1,   %xmm7

  706. 

  707. 	subl	$-31 * SIZE, X

  708. 

  709. 	testl	$2 * SIZE, X

  710. 	je	.L130x

  711. 

  712. 	movsd	-31 * SIZE(X), %xmm0

  713. 

  714. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  715. 	mulps	 %xmm6, %xmm0

  716. 	mulps	 %xmm7, %xmm5

  717. 	addps	 %xmm5, %xmm0

  718. 

  719. 	movlps	%xmm0,  -31 * SIZE(X)

  720. 	addl	$2 * SIZE, X

  721. 	decl	M

  722. 	jle	.L999

  723. 	ALIGN_3

  724. 

  725. .L130x:

  726. 	shufps	$0xb1, %xmm7, %xmm7

  727. 

  728. 	movaps	-32 * SIZE(X), %xmm0

  729. 	movaps	%xmm0, %xmm4

  730. 

  731. 	movl	M,  I

  732. 	sarl	$4, I

  733. 	jle	.L135

  734. 

  735. 	movaps	-28 * SIZE(X), %xmm1

  736. 

  737. 

  738. 	decl	I

  739. 	jle	.L132

  740. 	ALIGN_4

  741. 

  742. .L131:

  743. #ifdef PREFETCHW

  744. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  745. #endif

  746. 

  747. 	movss	%xmm1, %xmm0

  748. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  749. 	mulps	%xmm6, %xmm0

  750. 	mulps	%xmm7, %xmm5

  751. 	addps	%xmm5, %xmm0

  752. 

  753. 	movaps	%xmm0, %xmm2

  754. 	movss	%xmm4, %xmm0

  755. 	movaps	%xmm0,  -32 * SIZE(X)

  756. 

  757. 	movaps	-24 * SIZE(X), %xmm0

  758. 

  759. 	movss	%xmm0, %xmm1

  760. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  761. 	mulps	%xmm6, %xmm1

  762. 	mulps	%xmm7, %xmm5

  763. 	addps	%xmm5, %xmm1

  764. 

  765. 	movaps	%xmm1, %xmm4

  766. 	movss	%xmm2, %xmm1

  767. 	movaps	%xmm1,  -28 * SIZE(X)

  768. 

  769. 	movaps	-20 * SIZE(X), %xmm1

  770. 

  771. 	movss	%xmm1, %xmm0

  772. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  773. 	mulps	%xmm6, %xmm0

  774. 	mulps	%xmm7, %xmm5

  775. 	addps	%xmm5, %xmm0

  776. 

  777. 	movaps	%xmm0, %xmm2

  778. 	movss	%xmm4, %xmm0

  779. 	movaps	%xmm0,  -24 * SIZE(X)

  780. 

  781. 	movaps	-16 * SIZE(X), %xmm0

  782. 

  783. 	movss	%xmm0, %xmm1

  784. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  785. 	mulps	%xmm6, %xmm1

  786. 	mulps	%xmm7, %xmm5

  787. 	addps	%xmm5, %xmm1

  788. 

  789. 	movaps	%xmm1, %xmm4

  790. 	movss	%xmm2, %xmm1

  791. 	movaps	%xmm1,  -20 * SIZE(X)

  792. 

  793. 	movaps	-12 * SIZE(X), %xmm1

  794. 

  795. #ifdef PREFETCHW

  796. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  797. #endif

  798. 

  799. 	movss	%xmm1, %xmm0

  800. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  801. 	mulps	%xmm6, %xmm0

  802. 	mulps	%xmm7, %xmm5

  803. 	addps	%xmm5, %xmm0

  804. 

  805. 	movaps	%xmm0, %xmm2

  806. 	movss	%xmm4, %xmm0

  807. 	movaps	%xmm0,  -16 * SIZE(X)

  808. 

  809. 	movaps	 -8 * SIZE(X), %xmm0

  810. 

  811. 	movss	%xmm0, %xmm1

  812. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  813. 	mulps	%xmm6, %xmm1

  814. 	mulps	%xmm7, %xmm5

  815. 	addps	%xmm5, %xmm1

  816. 

  817. 	movaps	%xmm1, %xmm4

  818. 	movss	%xmm2, %xmm1

  819. 	movaps	%xmm1,  -12 * SIZE(X)

  820. 

  821. 	movaps	 -4 * SIZE(X), %xmm1

  822. 

  823. 	movss	%xmm1, %xmm0

  824. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  825. 	mulps	%xmm6, %xmm0

  826. 	mulps	%xmm7, %xmm5

  827. 	addps	%xmm5, %xmm0

  828. 

  829. 	movaps	%xmm0, %xmm2

  830. 	movss	%xmm4, %xmm0

  831. 	movaps	%xmm0,   -8 * SIZE(X)

  832. 

  833. 	movaps	  0 * SIZE(X), %xmm0

  834. 

  835. 	movss	%xmm0, %xmm1

  836. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  837. 	mulps	%xmm6, %xmm1

  838. 	mulps	%xmm7, %xmm5

  839. 	addps	%xmm5, %xmm1

  840. 

  841. 	movaps	%xmm1, %xmm4

  842. 	movss	%xmm2, %xmm1

  843. 	movaps	%xmm1,   -4 * SIZE(X)

  844. 

  845. 	movaps	  4 * SIZE(X), %xmm1

  846. 

  847. 	subl	$-32 * SIZE, X

  848. 	decl	I

  849. 	jg	.L131

  850. 	ALIGN_4

  851. 

  852. .L132:

  853. 	movss	%xmm1, %xmm0

  854. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  855. 	mulps	%xmm6, %xmm0

  856. 	mulps	%xmm7, %xmm5

  857. 	addps	%xmm5, %xmm0

  858. 

  859. 	movaps	%xmm0, %xmm2

  860. 	movss	%xmm4, %xmm0

  861. 	movaps	%xmm0,  -32 * SIZE(X)

  862. 

  863. 	movaps	-24 * SIZE(X), %xmm0

  864. 

  865. 	movss	%xmm0, %xmm1

  866. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  867. 	mulps	%xmm6, %xmm1

  868. 	mulps	%xmm7, %xmm5

  869. 	addps	%xmm5, %xmm1

  870. 

  871. 	movaps	%xmm1, %xmm4

  872. 	movss	%xmm2, %xmm1

  873. 	movaps	%xmm1,  -28 * SIZE(X)

  874. 

  875. 	movaps	-20 * SIZE(X), %xmm1

  876. 

  877. 	movss	%xmm1, %xmm0

  878. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  879. 	mulps	%xmm6, %xmm0

  880. 	mulps	%xmm7, %xmm5

  881. 	addps	%xmm5, %xmm0

  882. 

  883. 	movaps	%xmm0, %xmm2

  884. 	movss	%xmm4, %xmm0

  885. 	movaps	%xmm0,  -24 * SIZE(X)

  886. 

  887. 	movaps	-16 * SIZE(X), %xmm0

  888. 

  889. 	movss	%xmm0, %xmm1

  890. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  891. 	mulps	%xmm6, %xmm1

  892. 	mulps	%xmm7, %xmm5

  893. 	addps	%xmm5, %xmm1

  894. 

  895. 	movaps	%xmm1, %xmm4

  896. 	movss	%xmm2, %xmm1

  897. 	movaps	%xmm1,  -20 * SIZE(X)

  898. 

  899. 	movaps	-12 * SIZE(X), %xmm1

  900. 

  901. 	movss	%xmm1, %xmm0

  902. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  903. 	mulps	%xmm6, %xmm0

  904. 	mulps	%xmm7, %xmm5

  905. 	addps	%xmm5, %xmm0

  906. 

  907. 	movaps	%xmm0, %xmm2

  908. 	movss	%xmm4, %xmm0

  909. 	movaps	%xmm0,  -16 * SIZE(X)

  910. 

  911. 	movaps	 -8 * SIZE(X), %xmm0

  912. 

  913. 	movss	%xmm0, %xmm1

  914. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  915. 	mulps	%xmm6, %xmm1

  916. 	mulps	%xmm7, %xmm5

  917. 	addps	%xmm5, %xmm1

  918. 

  919. 	movaps	%xmm1, %xmm4

  920. 	movss	%xmm2, %xmm1

  921. 	movaps	%xmm1,  -12 * SIZE(X)

  922. 

  923. 	movaps	 -4 * SIZE(X), %xmm1

  924. 

  925. 	movss	%xmm1, %xmm0

  926. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  927. 	mulps	%xmm6, %xmm0

  928. 	mulps	%xmm7, %xmm5

  929. 	addps	%xmm5, %xmm0

  930. 

  931. 	movaps	%xmm0, %xmm2

  932. 	movss	%xmm4, %xmm0

  933. 	movaps	%xmm0,   -8 * SIZE(X)

  934. 

  935. 	movaps	  0 * SIZE(X), %xmm0

  936. 

  937. 	movss	%xmm0, %xmm1

  938. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  939. 	mulps	%xmm6, %xmm1

  940. 	mulps	%xmm7, %xmm5

  941. 	addps	%xmm5, %xmm1

  942. 

  943. 	movaps	%xmm1, %xmm4

  944. 	movss	%xmm2, %xmm1

  945. 	movaps	%xmm1,   -4 * SIZE(X)

  946. 

  947. 	subl	$-32 * SIZE, X

  948. 	ALIGN_4

  949. 

  950. .L135:

  951. 	testl	$8, M

  952. 	je	.L136

  953. 

  954. 	movaps	-28 * SIZE(X), %xmm1

  955. 

  956. 	movss	%xmm1, %xmm0

  957. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  958. 	mulps	%xmm6, %xmm0

  959. 	mulps	%xmm7, %xmm5

  960. 	addps	%xmm5, %xmm0

  961. 

  962. 	movaps	%xmm0, %xmm2

  963. 	movss	%xmm4, %xmm0

  964. 	movaps	%xmm0,  -32 * SIZE(X)

  965. 

  966. 	movaps	-24 * SIZE(X), %xmm0

  967. 

  968. 	movss	%xmm0, %xmm1

  969. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  970. 	mulps	%xmm6, %xmm1

  971. 	mulps	%xmm7, %xmm5

  972. 	addps	%xmm5, %xmm1

  973. 

  974. 	movaps	%xmm1, %xmm4

  975. 	movss	%xmm2, %xmm1

  976. 	movaps	%xmm1,  -28 * SIZE(X)

  977. 

  978. 	movaps	-20 * SIZE(X), %xmm1

  979. 

  980. 	movss	%xmm1, %xmm0

  981. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  982. 	mulps	%xmm6, %xmm0

  983. 	mulps	%xmm7, %xmm5

  984. 	addps	%xmm5, %xmm0

  985. 

  986. 	movaps	%xmm0, %xmm2

  987. 	movss	%xmm4, %xmm0

  988. 	movaps	%xmm0,  -24 * SIZE(X)

  989. 

  990. 	movaps	-16 * SIZE(X), %xmm0

  991. 

  992. 	movss	%xmm0, %xmm1

  993. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  994. 	mulps	%xmm6, %xmm1

  995. 	mulps	%xmm7, %xmm5

  996. 	addps	%xmm5, %xmm1

  997. 

  998. 	movaps	%xmm1, %xmm4

  999. 	movss	%xmm2, %xmm1

  1000. 	movaps	%xmm1,  -20 * SIZE(X)

  1001. 

  1002. 	addl	$16 * SIZE, X

  1003. 	ALIGN_3

  1004. 

  1005. .L136:

  1006. 	testl	$4, M

  1007. 	je	.L137

  1008. 

  1009. 	movaps	-28 * SIZE(X), %xmm1

  1010. 

  1011. 	movss	%xmm1, %xmm0

  1012. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  1013. 	mulps	%xmm6, %xmm0

  1014. 	mulps	%xmm7, %xmm5

  1015. 	addps	%xmm5, %xmm0

  1016. 

  1017. 	movaps	%xmm0, %xmm2

  1018. 	movss	%xmm4, %xmm0

  1019. 	movaps	%xmm0,  -32 * SIZE(X)

  1020. 

  1021. 	movaps	-24 * SIZE(X), %xmm0

  1022. 

  1023. 	movss	%xmm0, %xmm1

  1024. 	PSHUFD2($0x1b, %xmm1, %xmm5)

  1025. 	mulps	%xmm6, %xmm1

  1026. 	mulps	%xmm7, %xmm5

  1027. 	addps	%xmm5, %xmm1

  1028. 

  1029. 	movaps	%xmm1, %xmm4

  1030. 	movss	%xmm2, %xmm1

  1031. 	movaps	%xmm1,  -28 * SIZE(X)

  1032. 

  1033. 	addl	$8 * SIZE, X

  1034. 	ALIGN_3

  1035. 

  1036. .L137:

  1037. 	testl	$2, M

  1038. 	je	.L138

  1039. 

  1040. 	movaps	-28 * SIZE(X), %xmm1

  1041. 

  1042. 	movss	%xmm1, %xmm0

  1043. 	PSHUFD2($0x1b, %xmm0, %xmm5)

  1044. 	mulps	%xmm6, %xmm0

  1045. 	mulps	%xmm7, %xmm5

  1046. 	addps	%xmm5, %xmm0

  1047. 

  1048. 	movaps	%xmm0, %xmm2

  1049. 	movss	%xmm4, %xmm0

  1050. 	movaps	%xmm0,  -32 * SIZE(X)

  1051. 	movaps	%xmm2, %xmm4

  1052. 	movaps	%xmm1, %xmm0

  1053. 

  1054. 	addl	$4 * SIZE, X

  1055. 	ALIGN_3

  1056. 

  1057. .L138:

  1058. 	movss	%xmm4, -32 * SIZE(X)

  1059. 

  1060. 	testl	$1, M

  1061. 	je	.L999

  1062. 

  1063. 	PSHUFD2( $0x1b,  %xmm0, %xmm5)

  1064. 	mulps	 %xmm6, %xmm0

  1065. 	mulps	 %xmm7, %xmm5

  1066. 	addps	 %xmm5,  %xmm0

  1067. 

  1068. 	PSHUFD1( $0x39, %xmm0)

  1069. 

  1070. 	movlps	%xmm0,  -31 * SIZE(X)

  1071. 	jmp	.L999

  1072. 	ALIGN_3

  1073. 

  1074. 

  1075. #else

  1076. 

  1077. 	PSHUFD2($0, %xmm0, %xmm6)

  1078. 	PSHUFD2($0, %xmm1, %xmm1)

  1079. 	subps	  %xmm1,   %xmm7

  1080. 	unpcklps  %xmm1,   %xmm7

  1081. 

  1082. 	subl	$-32 * SIZE, X

  1083. 

  1084. 	testl	$2 * SIZE, X

  1085. 	je	.L130x

  1086. 

  1087. #ifdef movsd

  1088. 	xorps	%xmm0, %xmm0

  1089. #endif

  1090. 	movsd	-32 * SIZE(X), %xmm0

  1091. 

  1092. 	PSHUFD2( $0xb1,  %xmm0, %xmm5)

  1093. 	mulps	 %xmm6, %xmm0

  1094. 	mulps	 %xmm7, %xmm5

  1095. 	addps	 %xmm5,  %xmm0

  1096. 

  1097. 	movlps	%xmm0,  -32 * SIZE(X)

  1098. 	addl	$2 * SIZE, X

  1099. 	decl	M

  1100. 	jle	.L999

  1101. 	ALIGN_3

  1102. 

  1103. .L130x:

  1104. 	movl	M,  I

  1105. 	sarl	$4, I

  1106. 	jle	.L135

  1107. 

  1108. 	movsd	-32 * SIZE(X), %xmm0

  1109. 	movhps	-30 * SIZE(X), %xmm0

  1110. 	movsd	-28 * SIZE(X), %xmm1

  1111. 	movhps	-26 * SIZE(X), %xmm1

  1112. 	movsd	-24 * SIZE(X), %xmm2

  1113. 	movhps	-22 * SIZE(X), %xmm2

  1114. 	movsd	-20 * SIZE(X), %xmm3

  1115. 	movhps	-18 * SIZE(X), %xmm3

  1116. 

  1117. 	decl	I

  1118. 	jle	.L132

  1119. 	ALIGN_4

  1120. 

  1121. .L131:

  1122. #ifdef PREFETCHW

  1123. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  1124. #endif

  1125. 

  1126. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1127. 	mulps	 %xmm6, %xmm0

  1128. 	mulps	 %xmm7, %xmm5

  1129. 	addps	 %xmm5, %xmm0

  1130. 	movlps	%xmm0,  -32 * SIZE(X)

  1131. 	movhps	%xmm0,  -30 * SIZE(X)

  1132. 	movsd	-16 * SIZE(X), %xmm0

  1133. 	movhps	-14 * SIZE(X), %xmm0

  1134. 

  1135. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1136. 	mulps	 %xmm6, %xmm1

  1137. 	mulps	 %xmm7, %xmm5

  1138. 	addps	 %xmm5, %xmm1

  1139. 	movlps	%xmm1,  -28 * SIZE(X)

  1140. 	movhps	%xmm1,  -26 * SIZE(X)

  1141. 	movsd	-12 * SIZE(X), %xmm1

  1142. 	movhps	-10 * SIZE(X), %xmm1

  1143. 

  1144. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  1145. 	mulps	 %xmm6, %xmm2

  1146. 	mulps	 %xmm7, %xmm5

  1147. 	addps	 %xmm5, %xmm2

  1148. 	movlps	%xmm2,  -24 * SIZE(X)

  1149. 	movhps	%xmm2,  -22 * SIZE(X)

  1150. 	movsd	 -8 * SIZE(X), %xmm2

  1151. 	movhps	 -6 * SIZE(X), %xmm2

  1152. 

  1153. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  1154. 	mulps	 %xmm6, %xmm3

  1155. 	mulps	 %xmm7, %xmm5

  1156. 	addps	 %xmm5, %xmm3

  1157. 	movlps	%xmm3,  -20 * SIZE(X)

  1158. 	movhps	%xmm3,  -18 * SIZE(X)

  1159. 	movsd	 -4 * SIZE(X), %xmm3

  1160. 	movhps	 -2 * SIZE(X), %xmm3

  1161. 

  1162. #ifdef PREFETCHW

  1163. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  1164. #endif

  1165. 

  1166. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1167. 	mulps	 %xmm6, %xmm0

  1168. 	mulps	 %xmm7, %xmm5

  1169. 	addps	 %xmm5, %xmm0

  1170. 	movlps	%xmm0,  -16 * SIZE(X)

  1171. 	movhps	%xmm0,  -14 * SIZE(X)

  1172. 	movsd	  0 * SIZE(X), %xmm0

  1173. 	movhps	  2 * SIZE(X), %xmm0

  1174. 

  1175. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1176. 	mulps	 %xmm6, %xmm1

  1177. 	mulps	 %xmm7, %xmm5

  1178. 	addps	 %xmm5, %xmm1

  1179. 	movlps	%xmm1,  -12 * SIZE(X)

  1180. 	movhps	%xmm1,  -10 * SIZE(X)

  1181. 	movsd	  4 * SIZE(X), %xmm1

  1182. 	movhps	  6 * SIZE(X), %xmm1

  1183. 

  1184. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  1185. 	mulps	 %xmm6, %xmm2

  1186. 	mulps	 %xmm7, %xmm5

  1187. 	addps	 %xmm5, %xmm2

  1188. 	movlps	%xmm2,   -8 * SIZE(X)

  1189. 	movhps	%xmm2,   -6 * SIZE(X)

  1190. 	movsd	  8 * SIZE(X), %xmm2

  1191. 	movhps	 10 * SIZE(X), %xmm2

  1192. 

  1193. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  1194. 	mulps	 %xmm6, %xmm3

  1195. 	mulps	 %xmm7, %xmm5

  1196. 	addps	 %xmm5, %xmm3

  1197. 	movlps	%xmm3,   -4 * SIZE(X)

  1198. 	movhps	%xmm3,   -2 * SIZE(X)

  1199. 	movsd	 12 * SIZE(X), %xmm3

  1200. 	movhps	 14 * SIZE(X), %xmm3

  1201. 

  1202. 	subl	$-32 * SIZE, X

  1203. 	decl	I

  1204. 	jg	.L131

  1205. 	ALIGN_4

  1206. 

  1207. .L132:

  1208. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1209. 	mulps	 %xmm6, %xmm0

  1210. 	mulps	 %xmm7, %xmm5

  1211. 	addps	 %xmm5, %xmm0

  1212. 	movlps	%xmm0,  -32 * SIZE(X)

  1213. 	movhps	%xmm0,  -30 * SIZE(X)

  1214. 	movsd	-16 * SIZE(X), %xmm0

  1215. 	movhps	-14 * SIZE(X), %xmm0

  1216. 

  1217. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1218. 	mulps	 %xmm6, %xmm1

  1219. 	mulps	 %xmm7, %xmm5

  1220. 	addps	 %xmm5, %xmm1

  1221. 	movlps	%xmm1,  -28 * SIZE(X)

  1222. 	movhps	%xmm1,  -26 * SIZE(X)

  1223. 	movsd	-12 * SIZE(X), %xmm1

  1224. 	movhps	-10 * SIZE(X), %xmm1

  1225. 

  1226. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  1227. 	mulps	 %xmm6, %xmm2

  1228. 	mulps	 %xmm7, %xmm5

  1229. 	addps	 %xmm5, %xmm2

  1230. 	movlps	%xmm2,  -24 * SIZE(X)

  1231. 	movhps	%xmm2,  -22 * SIZE(X)

  1232. 	movsd	 -8 * SIZE(X), %xmm2

  1233. 	movhps	 -6 * SIZE(X), %xmm2

  1234. 

  1235. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  1236. 	mulps	 %xmm6, %xmm3

  1237. 	mulps	 %xmm7, %xmm5

  1238. 	addps	 %xmm5, %xmm3

  1239. 	movlps	%xmm3,  -20 * SIZE(X)

  1240. 	movhps	%xmm3,  -18 * SIZE(X)

  1241. 	movsd	 -4 * SIZE(X), %xmm3

  1242. 	movhps	 -2 * SIZE(X), %xmm3

  1243. 

  1244. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1245. 	mulps	 %xmm6, %xmm0

  1246. 	mulps	 %xmm7, %xmm5

  1247. 	addps	 %xmm5, %xmm0

  1248. 	movlps	%xmm0,  -16 * SIZE(X)

  1249. 	movhps	%xmm0,  -14 * SIZE(X)

  1250. 

  1251. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1252. 	mulps	 %xmm6, %xmm1

  1253. 	mulps	 %xmm7, %xmm5

  1254. 	addps	 %xmm5, %xmm1

  1255. 	movlps	%xmm1,  -12 * SIZE(X)

  1256. 	movhps	%xmm1,  -10 * SIZE(X)

  1257. 

  1258. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  1259. 	mulps	 %xmm6, %xmm2

  1260. 	mulps	 %xmm7, %xmm5

  1261. 	addps	 %xmm5, %xmm2

  1262. 	movlps	%xmm2,   -8 * SIZE(X)

  1263. 	movhps	%xmm2,   -6 * SIZE(X)

  1264. 

  1265. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  1266. 	mulps	 %xmm6, %xmm3

  1267. 	mulps	 %xmm7, %xmm5

  1268. 	addps	 %xmm5, %xmm3

  1269. 	movlps	%xmm3,   -4 * SIZE(X)

  1270. 	movhps	%xmm3,   -2 * SIZE(X)

  1271. 

  1272. 	subl	$-32 * SIZE, X

  1273. 	ALIGN_4

  1274. 

  1275. .L135:

  1276. 	testl	$8, M

  1277. 	je	.L136

  1278. 

  1279. 	movsd	-32 * SIZE(X), %xmm0

  1280. 	movhps	-30 * SIZE(X), %xmm0

  1281. 

  1282. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1283. 	mulps	 %xmm6, %xmm0

  1284. 	mulps	 %xmm7, %xmm5

  1285. 	addps	 %xmm5, %xmm0

  1286. 	movlps	%xmm0,  -32 * SIZE(X)

  1287. 	movhps	%xmm0,  -30 * SIZE(X)

  1288. 

  1289. 	movsd	-28 * SIZE(X), %xmm1

  1290. 	movhps	-26 * SIZE(X), %xmm1

  1291. 

  1292. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1293. 	mulps	 %xmm6, %xmm1

  1294. 	mulps	 %xmm7, %xmm5

  1295. 	addps	 %xmm5, %xmm1

  1296. 	movlps	%xmm1,  -28 * SIZE(X)

  1297. 	movhps	%xmm1,  -26 * SIZE(X)

  1298. 

  1299. 	movsd	-24 * SIZE(X), %xmm2

  1300. 	movhps	-22 * SIZE(X), %xmm2

  1301. 

  1302. 	PSHUFD2( $0xb1, %xmm2, %xmm5)

  1303. 	mulps	 %xmm6, %xmm2

  1304. 	mulps	 %xmm7, %xmm5

  1305. 	addps	 %xmm5, %xmm2

  1306. 	movlps	%xmm2,  -24 * SIZE(X)

  1307. 	movhps	%xmm2,  -22 * SIZE(X)

  1308. 

  1309. 	movsd	-20 * SIZE(X), %xmm3

  1310. 	movhps	-18 * SIZE(X), %xmm3

  1311. 

  1312. 	PSHUFD2( $0xb1, %xmm3, %xmm5)

  1313. 	mulps	 %xmm6, %xmm3

  1314. 	mulps	 %xmm7, %xmm5

  1315. 	addps	 %xmm5, %xmm3

  1316. 	movlps	%xmm3,  -20 * SIZE(X)

  1317. 	movhps	%xmm3,  -18 * SIZE(X)

  1318. 

  1319. 	addl	$16 * SIZE, X

  1320. 	ALIGN_3

  1321. 

  1322. .L136:

  1323. 	testl	$4, M

  1324. 	je	.L137

  1325. 

  1326. 	movsd	-32 * SIZE(X), %xmm0

  1327. 	movhps	-30 * SIZE(X), %xmm0

  1328. 	movsd	-28 * SIZE(X), %xmm1

  1329. 	movhps	-26 * SIZE(X), %xmm1

  1330. 

  1331. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1332. 	mulps	 %xmm6, %xmm0

  1333. 	mulps	 %xmm7, %xmm5

  1334. 	addps	 %xmm5, %xmm0

  1335. 	movlps	%xmm0,  -32 * SIZE(X)

  1336. 	movhps	%xmm0,  -30 * SIZE(X)

  1337. 

  1338. 	PSHUFD2( $0xb1, %xmm1, %xmm5)

  1339. 	mulps	 %xmm6, %xmm1

  1340. 	mulps	 %xmm7, %xmm5

  1341. 	addps	 %xmm5, %xmm1

  1342. 	movlps	%xmm1,  -28 * SIZE(X)

  1343. 	movhps	%xmm1,  -26 * SIZE(X)

  1344. 

  1345. 	addl	$8 * SIZE, X

  1346. 	ALIGN_3

  1347. 

  1348. .L137:

  1349. 	testl	$2, M

  1350. 	je	.L138

  1351. 

  1352. 	movsd	-32 * SIZE(X), %xmm0

  1353. 	movhps	-30 * SIZE(X), %xmm0

  1354. 

  1355. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1356. 	mulps	 %xmm6, %xmm0

  1357. 	mulps	 %xmm7, %xmm5

  1358. 	addps	 %xmm5, %xmm0

  1359. 	movlps	%xmm0,  -32 * SIZE(X)

  1360. 	movhps	%xmm0,  -30 * SIZE(X)

  1361. 

  1362. 	addl	$4 * SIZE, X

  1363. 	ALIGN_3

  1364. 

  1365. .L138:

  1366. 	testl	$1, M

  1367. 	je	.L999

  1368. 

  1369. 	movsd	-32 * SIZE(X), %xmm0

  1370. 

  1371. 	PSHUFD2( $0xb1, %xmm0, %xmm5)

  1372. 	mulps	 %xmm6, %xmm0

  1373. 	mulps	 %xmm7, %xmm5

  1374. 	addps	 %xmm5, %xmm0

  1375. 

  1376. 	movlps	%xmm0,  -32 * SIZE(X)

  1377. 	ALIGN_3

  1378. #endif

  1379. 

  1380. .L999:

  1381. 	xorl	%eax, %eax

  1382. 	popl	%ebp

  1383. 	popl	%ebx

  1384. 	popl	%esi

  1385. 	popl	%edi

  1386. 

  1387. 	ret

  1388. 

  1389. 	EPILOGUE