OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: fb7c53c5e5
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'fb7c53c5e5'
${ noResults }
OpenBLAS/kernel/x86_64/zscal_sse.S

1360 lines
25 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #else

  47. #define M	ARG1

  48. #define X	ARG2

  49. #define INCX	ARG3

  50. #endif

  51. 

  52. #define XX	%r10

  53. #define FLAG	%r11

  54. #define I	%rax

  55. 

  56. #include "l1param.h"

  57. 

  58. 	PROLOGUE

  59. 	PROFCODE

  60. 

  61. #ifdef WINDOWS_ABI

  62. 	movaps	%xmm3, %xmm0

  63. 	movsd	40(%rsp), %xmm1

  64. 	movq	48(%rsp), X

  65. 	movq	56(%rsp), INCX

  66. #endif

  67. 

  68. 	SAVEREGISTERS

  69. 

  70. 	salq	$ZBASE_SHIFT, INCX

  71. 	xor	FLAG, FLAG

  72. 

  73. 	testq	M, M

  74. 	jle	.L999

  75. 

  76. 	pxor	%xmm15, %xmm15

  77. 	comiss	%xmm0, %xmm15

  78. 	jne	.L100		# Alpha_r != ZERO

  79. 	jp	.L100		# Alpha_r == NAN	

  80. 	comiss	%xmm1, %xmm15

  81. 	jne	.L100		# Alpha_i != ZERO

  82. 

  83. /* Alpha == ZERO */

  84. 	cmpq	$2 * SIZE, INCX

  85. 	jne	.L50

  86. 

  87. /* INCX == 1 */

  88. 	cmpq	$3, M

  89. 	jle	.L13

  90. 

  91. 	testq	$4, X

  92. 	je	.L05

  93. 	movss	%xmm15, 0 * SIZE(X)

  94. 	addq	$SIZE, X

  95. 	movq	$1, FLAG

  96. 	decq	M

  97. 	ALIGN_3

  98. 

  99. .L05:

  100. 	testq	$8, X

  101. 	je	.L06

  102. 

  103. 	movlps	%xmm15, 0 * SIZE(X)

  104. 	addq	$2 * SIZE, X

  105. 	subq	$1, M

  106. 	ALIGN_3

  107. .L06:

  108. 

  109. 	movq	M,  I	# rcx = n

  110. 	sarq	$3, I

  111. 	jle	.L12

  112. 	ALIGN_4

  113. 

  114. .L11:

  115. #ifdef PREFETCHW

  116. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  117. #endif

  118. 

  119. 	movaps	%xmm15,  0 * SIZE(X)

  120. 	movaps	%xmm15,  4 * SIZE(X)

  121. 	movaps	%xmm15,  8 * SIZE(X)

  122. 	movaps	%xmm15, 12 * SIZE(X)

  123. 	addq	$16 * SIZE, X

  124. 	decq	I

  125. 	jg	.L11

  126. 	ALIGN_4

  127. 

  128. .L12:

  129. 	testq	$7, M

  130. 	je	.L19

  131. 	testq	$4, M

  132. 	je	.L13

  133. 

  134. 	movaps	%xmm15,  0 * SIZE(X)

  135. 	movaps	%xmm15,  4 * SIZE(X)

  136. 	addq	$8 * SIZE, X

  137. 	ALIGN_3

  138. 

  139. .L13:

  140. 	testq  $2, M

  141. 	je    .L14

  142. 

  143. 	movlps	%xmm15,  0 * SIZE(X)

  144. 	movhps	%xmm15,  2 * SIZE(X)

  145. 	addq	$4 * SIZE, X

  146. 	ALIGN_3

  147. 

  148. .L14:

  149. 	testq  $1, M

  150. 	je    .L19

  151. 

  152. 	movlps	%xmm15,  0 * SIZE(X)

  153. 	addq	$2 * SIZE, X

  154. 	ALIGN_3

  155. 

  156. .L19:

  157. 	testq  $1, FLAG

  158. 	je    .L999

  159. 

  160. 	movss	%xmm15, 0 * SIZE(X)

  161. 	jmp	.L999

  162. 	ALIGN_4

  163. 

  164. /* incx != 1 */

  165. .L50:

  166. 	movq	M,  I		# rcx = n

  167. 	sarq	$2, I

  168. 	jle	.L52

  169. 	ALIGN_4

  170. 

  171. .L51:

  172. #ifdef PREFETCHW

  173. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  174. #endif

  175. 

  176. 	movsd	%xmm15, 0 * SIZE(X)

  177. 	addq	INCX, X

  178. 	movsd	%xmm15, 0 * SIZE(X)

  179. 	addq	INCX, X

  180. 	movsd	%xmm15, 0 * SIZE(X)

  181. 	addq	INCX, X

  182. 	movsd	%xmm15, 0 * SIZE(X)

  183. 	addq	INCX, X

  184. 	decq	I

  185. 	jg	.L51

  186. 	ALIGN_4

  187. 

  188. .L52:

  189. 	testq	$2, M

  190. 	je	.L53

  191. 

  192. 	movsd	%xmm15, 0 * SIZE(X)

  193. 	addq	INCX, X

  194. 	movsd	%xmm15, 0 * SIZE(X)

  195. 	addq	INCX, X

  196. 	ALIGN_3

  197. 

  198. .L53:

  199. 	testq	$1, M

  200. 	je	.L999

  201. 

  202. 	movsd	%xmm15, 0 * SIZE(X)

  203. 	jmp	.L999

  204. 	ALIGN_4

  205. 

  206. /* Alpha != ZERO */

  207. 

  208. .L100:

  209. 	testq	$SIZE, X

  210. 	jne	.L130

  211. 

  212. 	cmpq	$2 * SIZE, INCX

  213. 	jne	.L120

  214. 

  215. 	pshufd	$0, %xmm0, %xmm14

  216. 	pshufd	$0, %xmm1, %xmm1

  217. 	subps	  %xmm1,   %xmm15

  218. 	unpcklps  %xmm1,   %xmm15

  219. 

  220. 	subq	$-32 * SIZE, X

  221. 

  222. 	testq	$2 * SIZE, X

  223. 	je	.L105

  224. 

  225. 	movsd	-32 * SIZE(X), %xmm0

  226. 

  227. 	pshufd	 $0xb1,  %xmm0, %xmm8

  228. 	mulps	 %xmm14, %xmm0

  229. 	mulps	 %xmm15, %xmm8

  230. 	addps	 %xmm8,  %xmm0

  231. 

  232. 	movlps	%xmm0,  -32 * SIZE(X)

  233. 	addq	$2 * SIZE, X

  234. 	decq	M

  235. 	jle	.L999

  236. 	ALIGN_3

  237. 

  238. .L105:

  239. 	movq	M,  I

  240. 	sarq	$4, I

  241. 	jle	.L115

  242. 

  243. 	movaps	-32 * SIZE(X), %xmm0

  244. 	movaps	-28 * SIZE(X), %xmm1

  245. 	movaps	-24 * SIZE(X), %xmm2

  246. 	movaps	-20 * SIZE(X), %xmm3

  247. 	movaps	-16 * SIZE(X), %xmm4

  248. 	movaps	-12 * SIZE(X), %xmm5

  249. 	movaps	 -8 * SIZE(X), %xmm6

  250. 	movaps	 -4 * SIZE(X), %xmm7

  251. 

  252. 	decq	I

  253. 	jle	.L112

  254. 	ALIGN_4

  255. 

  256. .L111:

  257. #ifdef PREFETCHW

  258. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  259. #endif

  260. 

  261. 	pshufd	 $0xb1,  %xmm0, %xmm8

  262. 	mulps	 %xmm14, %xmm0

  263. 	mulps	 %xmm15, %xmm8

  264. 	addps	 %xmm8,  %xmm0

  265. 	movaps	%xmm0,  -32 * SIZE(X)

  266. 	movaps	  0 * SIZE(X), %xmm0

  267. 

  268. 	pshufd	 $0xb1,  %xmm1, %xmm8

  269. 	mulps	 %xmm14, %xmm1

  270. 	mulps	 %xmm15, %xmm8

  271. 	addps	 %xmm8,  %xmm1

  272. 	movaps	%xmm1,  -28 * SIZE(X)

  273. 	movaps	  4 * SIZE(X), %xmm1

  274. 

  275. 	pshufd	 $0xb1,  %xmm2, %xmm8

  276. 	mulps	 %xmm14, %xmm2

  277. 	mulps	 %xmm15, %xmm8

  278. 	addps	 %xmm8,  %xmm2

  279. 	movaps	%xmm2,  -24 * SIZE(X)

  280. 	movaps	  8 * SIZE(X), %xmm2

  281. 

  282. 	pshufd	 $0xb1,  %xmm3, %xmm8

  283. 	mulps	 %xmm14, %xmm3

  284. 	mulps	 %xmm15, %xmm8

  285. 	addps	 %xmm8,  %xmm3

  286. 	movaps	%xmm3,  -20 * SIZE(X)

  287. 	movaps	 12 * SIZE(X), %xmm3

  288. 

  289. #ifdef PREFETCHW

  290. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  291. #endif

  292. 

  293. 	pshufd	 $0xb1,  %xmm4, %xmm8

  294. 	mulps	 %xmm14, %xmm4

  295. 	mulps	 %xmm15, %xmm8

  296. 	addps	 %xmm8,  %xmm4

  297. 	movaps	%xmm4,  -16 * SIZE(X)

  298. 	movaps	 16 * SIZE(X), %xmm4

  299. 

  300. 	pshufd	 $0xb1,  %xmm5, %xmm8

  301. 	mulps	 %xmm14, %xmm5

  302. 	mulps	 %xmm15, %xmm8

  303. 	addps	 %xmm8,  %xmm5

  304. 	movaps	%xmm5,  -12 * SIZE(X)

  305. 	movaps	 20 * SIZE(X), %xmm5

  306. 

  307. 	pshufd	 $0xb1,  %xmm6, %xmm8

  308. 	mulps	 %xmm14, %xmm6

  309. 	mulps	 %xmm15, %xmm8

  310. 	addps	 %xmm8,  %xmm6

  311. 	movaps	%xmm6,   -8 * SIZE(X)

  312. 	movaps	 24 * SIZE(X), %xmm6

  313. 

  314. 	pshufd	 $0xb1,  %xmm7, %xmm8

  315. 	mulps	 %xmm14, %xmm7

  316. 	mulps	 %xmm15, %xmm8

  317. 	addps	 %xmm8,  %xmm7

  318. 	movaps	%xmm7,   -4 * SIZE(X)

  319. 	movaps	 28 * SIZE(X), %xmm7

  320. 

  321. 	subq	$-32 * SIZE, X

  322. 	decq	I

  323. 	jg	.L111

  324. 	ALIGN_4

  325. 

  326. .L112:

  327. 	pshufd	 $0xb1,  %xmm0, %xmm8

  328. 	mulps	 %xmm14, %xmm0

  329. 	mulps	 %xmm15, %xmm8

  330. 	addps	 %xmm8,  %xmm0

  331. 	movaps	%xmm0,  -32 * SIZE(X)

  332. 

  333. 	pshufd	 $0xb1,  %xmm1, %xmm8

  334. 	mulps	 %xmm14, %xmm1

  335. 	mulps	 %xmm15, %xmm8

  336. 	addps	 %xmm8,  %xmm1

  337. 	movaps	%xmm1,  -28 * SIZE(X)

  338. 

  339. 	pshufd	 $0xb1,  %xmm2, %xmm8

  340. 	mulps	 %xmm14, %xmm2

  341. 	mulps	 %xmm15, %xmm8

  342. 	addps	 %xmm8,  %xmm2

  343. 	movaps	%xmm2,  -24 * SIZE(X)

  344. 

  345. 	pshufd	 $0xb1,  %xmm3, %xmm8

  346. 	mulps	 %xmm14, %xmm3

  347. 	mulps	 %xmm15, %xmm8

  348. 	addps	 %xmm8,  %xmm3

  349. 	movaps	%xmm3,  -20 * SIZE(X)

  350. 

  351. 	pshufd	 $0xb1,  %xmm4, %xmm8

  352. 	mulps	 %xmm14, %xmm4

  353. 	mulps	 %xmm15, %xmm8

  354. 	addps	 %xmm8,  %xmm4

  355. 	movaps	%xmm4,  -16 * SIZE(X)

  356. 

  357. 	pshufd	 $0xb1,  %xmm5, %xmm8

  358. 	mulps	 %xmm14, %xmm5

  359. 	mulps	 %xmm15, %xmm8

  360. 	addps	 %xmm8,  %xmm5

  361. 	movaps	%xmm5,  -12 * SIZE(X)

  362. 

  363. 	pshufd	 $0xb1,  %xmm6, %xmm8

  364. 	mulps	 %xmm14, %xmm6

  365. 	mulps	 %xmm15, %xmm8

  366. 	addps	 %xmm8,  %xmm6

  367. 	movaps	%xmm6,   -8 * SIZE(X)

  368. 

  369. 	pshufd	 $0xb1,  %xmm7, %xmm8

  370. 	mulps	 %xmm14, %xmm7

  371. 	mulps	 %xmm15, %xmm8

  372. 	addps	 %xmm8,  %xmm7

  373. 	movaps	%xmm7,   -4 * SIZE(X)

  374. 

  375. 	subq	$-32 * SIZE, X

  376. 	ALIGN_4

  377. 

  378. .L115:

  379. 	testq	$8, M

  380. 	je	.L116

  381. 

  382. 	movaps	-32 * SIZE(X), %xmm0

  383. 	movaps	-28 * SIZE(X), %xmm1

  384. 

  385. 	pshufd	 $0xb1,  %xmm0, %xmm8

  386. 	mulps	 %xmm14, %xmm0

  387. 	mulps	 %xmm15, %xmm8

  388. 	addps	 %xmm8,  %xmm0

  389. 	movaps	%xmm0,  -32 * SIZE(X)

  390. 

  391. 	pshufd	 $0xb1,  %xmm1, %xmm8

  392. 	mulps	 %xmm14, %xmm1

  393. 	mulps	 %xmm15, %xmm8

  394. 	addps	 %xmm8,  %xmm1

  395. 	movaps	%xmm1,  -28 * SIZE(X)

  396. 

  397. 	movaps	-24 * SIZE(X), %xmm2

  398. 	movaps	-20 * SIZE(X), %xmm3

  399. 

  400. 	pshufd	 $0xb1,  %xmm2, %xmm8

  401. 	mulps	 %xmm14, %xmm2

  402. 	mulps	 %xmm15, %xmm8

  403. 	addps	 %xmm8,  %xmm2

  404. 	movaps	%xmm2,  -24 * SIZE(X)

  405. 

  406. 	pshufd	 $0xb1,  %xmm3, %xmm8

  407. 	mulps	 %xmm14, %xmm3

  408. 	mulps	 %xmm15, %xmm8

  409. 	addps	 %xmm8,  %xmm3

  410. 	movaps	%xmm3,  -20 * SIZE(X)

  411. 

  412. 	addq	$16 * SIZE, X

  413. 	ALIGN_3

  414. 

  415. .L116:

  416. 	testq	$4, M

  417. 	je	.L117

  418. 

  419. 	movaps	-32 * SIZE(X), %xmm0

  420. 	movaps	-28 * SIZE(X), %xmm1

  421. 

  422. 	pshufd	 $0xb1,  %xmm0, %xmm8

  423. 	mulps	 %xmm14, %xmm0

  424. 	mulps	 %xmm15, %xmm8

  425. 	addps	 %xmm8,  %xmm0

  426. 	movaps	%xmm0,  -32 * SIZE(X)

  427. 

  428. 	pshufd	 $0xb1,  %xmm1, %xmm8

  429. 	mulps	 %xmm14, %xmm1

  430. 	mulps	 %xmm15, %xmm8

  431. 	addps	 %xmm8,  %xmm1

  432. 	movaps	%xmm1,  -28 * SIZE(X)

  433. 

  434. 	addq	$8 * SIZE, X

  435. 	ALIGN_3

  436. 

  437. .L117:

  438. 	testq	$2, M

  439. 	je	.L118

  440. 

  441. 	movaps	-32 * SIZE(X), %xmm0

  442. 

  443. 	pshufd	 $0xb1,  %xmm0, %xmm8

  444. 	mulps	 %xmm14, %xmm0

  445. 	mulps	 %xmm15, %xmm8

  446. 	addps	 %xmm8,  %xmm0

  447. 	movaps	%xmm0,  -32 * SIZE(X)

  448. 

  449. 	addq	$4 * SIZE, X

  450. 	ALIGN_3

  451. 

  452. .L118:

  453. 	testq	$1, M

  454. 	je	.L999

  455. 

  456. 	movsd	-32 * SIZE(X), %xmm0

  457. 

  458. 	pshufd	 $0xb1,  %xmm0, %xmm8

  459. 	mulps	 %xmm14, %xmm0

  460. 	mulps	 %xmm15, %xmm8

  461. 	addps	 %xmm8,  %xmm0

  462. 

  463. 	movlps	%xmm0,  -32 * SIZE(X)

  464. 	jmp	.L999

  465. 	ALIGN_3

  466. 

  467. .L120:

  468. 	pshufd	$0, %xmm0, %xmm14

  469. 	pshufd	$0, %xmm1, %xmm1

  470. 	subps	  %xmm1,   %xmm15

  471. 	unpcklps  %xmm1,   %xmm15

  472. 

  473. 	movq	X, XX

  474. 

  475. 	movq	M,  I

  476. 	sarq	$3, I

  477. 	jle	.L125

  478. 

  479. 	movsd	(X), %xmm0

  480. 	addq	INCX, X

  481. 	movhps	(X), %xmm0

  482. 	addq	INCX, X

  483. 

  484. 	movsd	(X), %xmm1

  485. 	addq	INCX, X

  486. 	movhps	(X), %xmm1

  487. 	addq	INCX, X

  488. 

  489. 	movsd	(X), %xmm2

  490. 	addq	INCX, X

  491. 	movhps	(X), %xmm2

  492. 	addq	INCX, X

  493. 

  494. 	movsd	(X), %xmm3

  495. 	addq	INCX, X

  496. 	movhps	(X), %xmm3

  497. 	addq	INCX, X

  498. 

  499. 	decq	I

  500. 	jle	.L122

  501. 	ALIGN_4

  502. 

  503. .L121:

  504. #ifdef PREFETCHW

  505. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  506. #endif

  507. 

  508. 	pshufd	 $0xb1,  %xmm0, %xmm8

  509. 	mulps	 %xmm14, %xmm0

  510. 	mulps	 %xmm15, %xmm8

  511. 	addps	 %xmm8,  %xmm0

  512. 

  513. 	movlps	%xmm0,  (XX)

  514. 	addq	INCX, XX

  515. 	movhps	%xmm0,  (XX)

  516. 	addq	INCX, XX

  517. 

  518. 	movsd	(X), %xmm0

  519. 	addq	INCX, X

  520. 	movhps	(X), %xmm0

  521. 	addq	INCX, X

  522. 

  523. 	pshufd	 $0xb1,  %xmm1, %xmm8

  524. 	mulps	 %xmm14, %xmm1

  525. 	mulps	 %xmm15, %xmm8

  526. 	addps	 %xmm8,  %xmm1

  527. 

  528. 	movlps	%xmm1,  (XX)

  529. 	addq	INCX, XX

  530. 	movhps	%xmm1,  (XX)

  531. 	addq	INCX, XX

  532. 

  533. 	movsd	(X), %xmm1

  534. 	addq	INCX, X

  535. 	movhps	(X), %xmm1

  536. 	addq	INCX, X

  537. 

  538. 	pshufd	 $0xb1,  %xmm2, %xmm8

  539. 	mulps	 %xmm14, %xmm2

  540. 	mulps	 %xmm15, %xmm8

  541. 	addps	 %xmm8,  %xmm2

  542. 

  543. 	movlps	%xmm2,  (XX)

  544. 	addq	INCX, XX

  545. 	movhps	%xmm2,  (XX)

  546. 	addq	INCX, XX

  547. 

  548. 	movsd	(X), %xmm2

  549. 	addq	INCX, X

  550. 	movhps	(X), %xmm2

  551. 	addq	INCX, X

  552. 

  553. 	pshufd	 $0xb1,  %xmm3, %xmm8

  554. 	mulps	 %xmm14, %xmm3

  555. 	mulps	 %xmm15, %xmm8

  556. 	addps	 %xmm8,  %xmm3

  557. 

  558. 	movlps	%xmm3,  (XX)

  559. 	addq	INCX, XX

  560. 	movhps	%xmm3,  (XX)

  561. 	addq	INCX, XX

  562. 

  563. 	movsd	(X), %xmm3

  564. 	addq	INCX, X

  565. 	movhps	(X), %xmm3

  566. 	addq	INCX, X

  567. 

  568. 	decq	I

  569. 	jg	.L121

  570. 	ALIGN_4

  571. 

  572. .L122:

  573. 	pshufd	 $0xb1,  %xmm0, %xmm8

  574. 	mulps	 %xmm14, %xmm0

  575. 	mulps	 %xmm15, %xmm8

  576. 	addps	 %xmm8,  %xmm0

  577. 

  578. 	movlps	%xmm0,  (XX)

  579. 	addq	INCX, XX

  580. 	movhps	%xmm0,  (XX)

  581. 	addq	INCX, XX

  582. 

  583. 	pshufd	 $0xb1,  %xmm1, %xmm8

  584. 	mulps	 %xmm14, %xmm1

  585. 	mulps	 %xmm15, %xmm8

  586. 	addps	 %xmm8,  %xmm1

  587. 

  588. 	movlps	%xmm1,  (XX)

  589. 	addq	INCX, XX

  590. 	movhps	%xmm1,  (XX)

  591. 	addq	INCX, XX

  592. 

  593. 	pshufd	 $0xb1,  %xmm2, %xmm8

  594. 	mulps	 %xmm14, %xmm2

  595. 	mulps	 %xmm15, %xmm8

  596. 	addps	 %xmm8,  %xmm2

  597. 

  598. 	movlps	%xmm2,  (XX)

  599. 	addq	INCX, XX

  600. 	movhps	%xmm2,  (XX)

  601. 	addq	INCX, XX

  602. 

  603. 	pshufd	 $0xb1,  %xmm3, %xmm8

  604. 	mulps	 %xmm14, %xmm3

  605. 	mulps	 %xmm15, %xmm8

  606. 	addps	 %xmm8,  %xmm3

  607. 

  608. 	movlps	%xmm3,  (XX)

  609. 	addq	INCX, XX

  610. 	movhps	%xmm3,  (XX)

  611. 	addq	INCX, XX

  612. 	ALIGN_4

  613. 

  614. .L125:

  615. 	testq	$4, M

  616. 	je	.L127

  617. 

  618. 	movsd	(X), %xmm0

  619. 	addq	INCX, X

  620. 	movhps	(X), %xmm0

  621. 	addq	INCX, X

  622. 

  623. 	pshufd	 $0xb1,  %xmm0, %xmm8

  624. 	mulps	 %xmm14, %xmm0

  625. 	mulps	 %xmm15, %xmm8

  626. 	addps	 %xmm8,  %xmm0

  627. 

  628. 	movlps	%xmm0,  (XX)

  629. 	addq	INCX, XX

  630. 	movhps	%xmm0,  (XX)

  631. 	addq	INCX, XX

  632. 

  633. 	movsd	(X), %xmm1

  634. 	addq	INCX, X

  635. 	movhps	(X), %xmm1

  636. 	addq	INCX, X

  637. 

  638. 	pshufd	 $0xb1,  %xmm1, %xmm8

  639. 	mulps	 %xmm14, %xmm1

  640. 	mulps	 %xmm15, %xmm8

  641. 	addps	 %xmm8,  %xmm1

  642. 

  643. 	movlps	%xmm1,  (XX)

  644. 	addq	INCX, XX

  645. 	movhps	%xmm1,  (XX)

  646. 	addq	INCX, XX

  647. 	ALIGN_3

  648. 

  649. .L127:

  650. 	testq	$2, M

  651. 	je	.L128

  652. 

  653. 	movsd	(X), %xmm0

  654. 	addq	INCX, X

  655. 	movhps	(X), %xmm0

  656. 	addq	INCX, X

  657. 

  658. 	pshufd	 $0xb1,  %xmm0, %xmm8

  659. 	mulps	 %xmm14, %xmm0

  660. 	mulps	 %xmm15, %xmm8

  661. 	addps	 %xmm8,  %xmm0

  662. 

  663. 	movlps	%xmm0,  (XX)

  664. 	addq	INCX, XX

  665. 	movhps	%xmm0,  (XX)

  666. 	addq	INCX, XX

  667. 	ALIGN_3

  668. 

  669. .L128:

  670. 	testq	$1, M

  671. 	je	.L999

  672. 

  673. 	movsd	(X), %xmm0

  674. 

  675. 	pshufd	 $0xb1,  %xmm0, %xmm8

  676. 	mulps	 %xmm14, %xmm0

  677. 	mulps	 %xmm15, %xmm8

  678. 	addps	 %xmm8,  %xmm0

  679. 

  680. 	movlps	%xmm0,  (XX)

  681. 	jmp	.L999

  682. 	ALIGN_3

  683. 

  684. .L130:

  685. 	cmpq	$2 * SIZE, INCX

  686. 	jne	.L120

  687. 

  688. #if defined(ALIGNED_ACCESS) && !defined(NEHALEM) && !defined(SANDYBRIDGE)

  689. 

  690. 	pshufd	$0, %xmm0, %xmm14

  691. 	pshufd	$0, %xmm1, %xmm1

  692. 	subps	  %xmm1,   %xmm15

  693. 	unpcklps  %xmm1,   %xmm15

  694. 

  695. 	subq	$-31 * SIZE, X

  696. 

  697. 	testq	$2 * SIZE, X

  698. 	je	.L130x

  699. 

  700. 	movsd	-31 * SIZE(X), %xmm0

  701. 

  702. 	pshufd	 $0xb1,  %xmm0, %xmm8

  703. 	mulps	 %xmm14, %xmm0

  704. 	mulps	 %xmm15, %xmm8

  705. 	addps	 %xmm8,  %xmm0

  706. 

  707. 	movlps	%xmm0,  -31 * SIZE(X)

  708. 	addq	$2 * SIZE, X

  709. 	decq	M

  710. 	jle	.L999

  711. 	ALIGN_3

  712. 

  713. .L130x:

  714. 	shufps	$0xb1, %xmm15, %xmm15

  715. 

  716. 	movaps	-32 * SIZE(X), %xmm0

  717. 	movaps	%xmm0, %xmm9

  718. 

  719. 	movq	M,  I

  720. 	sarq	$4, I

  721. 	jle	.L135

  722. 

  723. 	movaps	-28 * SIZE(X), %xmm1

  724. 	movaps	-24 * SIZE(X), %xmm2

  725. 	movaps	-20 * SIZE(X), %xmm3

  726. 	movaps	-16 * SIZE(X), %xmm4

  727. 	movaps	-12 * SIZE(X), %xmm5

  728. 	movaps	 -8 * SIZE(X), %xmm6

  729. 	movaps	 -4 * SIZE(X), %xmm7

  730. 

  731. 	decq	I

  732. 	jle	.L132

  733. 	ALIGN_4

  734. 

  735. .L131:

  736. #ifdef PREFETCHW

  737. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  738. #endif

  739. 

  740. 	movss	%xmm1,  %xmm0

  741. 	pshufd	$0x1b,  %xmm0, %xmm8

  742. 	mulps	%xmm14, %xmm0

  743. 	mulps	%xmm15, %xmm8

  744. 	addps	%xmm8,  %xmm0

  745. 	movaps	%xmm0,  %xmm10

  746. 	movss	%xmm9,  %xmm0

  747. 	movaps	%xmm0,  -32 * SIZE(X)

  748. 

  749. 	movaps	  0 * SIZE(X), %xmm0

  750. 

  751. 	movss	%xmm2,  %xmm1

  752. 	pshufd	$0x1b,  %xmm1, %xmm8

  753. 	mulps	%xmm14, %xmm1

  754. 	mulps	%xmm15, %xmm8

  755. 	addps	%xmm8,  %xmm1

  756. 	movaps	%xmm1,  %xmm9

  757. 	movss	%xmm10,  %xmm1

  758. 	movaps	%xmm1,  -28 * SIZE(X)

  759. 

  760. 	movaps	  4 * SIZE(X), %xmm1

  761. 

  762. 	movss	%xmm3,  %xmm2

  763. 	pshufd	$0x1b,  %xmm2, %xmm8

  764. 	mulps	%xmm14, %xmm2

  765. 	mulps	%xmm15, %xmm8

  766. 	addps	%xmm8,  %xmm2

  767. 	movaps	%xmm2,  %xmm10

  768. 	movss	%xmm9,  %xmm2

  769. 	movaps	%xmm2,  -24 * SIZE(X)

  770. 

  771. 	movaps	  8 * SIZE(X), %xmm2

  772. 

  773. 	movss	%xmm4,  %xmm3

  774. 	pshufd	$0x1b,  %xmm3, %xmm8

  775. 	mulps	%xmm14, %xmm3

  776. 	mulps	%xmm15, %xmm8

  777. 	addps	%xmm8,  %xmm3

  778. 	movaps	%xmm3,  %xmm9

  779. 	movss	%xmm10,  %xmm3

  780. 	movaps	%xmm3,  -20 * SIZE(X)

  781. 

  782. 	movaps	 12 * SIZE(X), %xmm3

  783. 

  784. #ifdef PREFETCHW

  785. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  786. #endif

  787. 

  788. 	movss	%xmm5,  %xmm4

  789. 	pshufd	$0x1b,  %xmm4, %xmm8

  790. 	mulps	%xmm14, %xmm4

  791. 	mulps	%xmm15, %xmm8

  792. 	addps	%xmm8,  %xmm4

  793. 	movaps	%xmm4,  %xmm10

  794. 	movss	%xmm9,  %xmm4

  795. 	movaps	%xmm4,  -16 * SIZE(X)

  796. 

  797. 	movaps	 16 * SIZE(X), %xmm4

  798. 

  799. 	movss	%xmm6,  %xmm5

  800. 	pshufd	$0x1b,  %xmm5, %xmm8

  801. 	mulps	%xmm14, %xmm5

  802. 	mulps	%xmm15, %xmm8

  803. 	addps	%xmm8,  %xmm5

  804. 	movaps	%xmm5,  %xmm9

  805. 	movss	%xmm10,  %xmm5

  806. 	movaps	%xmm5,  -12 * SIZE(X)

  807. 

  808. 	movaps	 20 * SIZE(X), %xmm5

  809. 

  810. 	movss	%xmm7,  %xmm6

  811. 	pshufd	$0x1b,  %xmm6, %xmm8

  812. 	mulps	%xmm14, %xmm6

  813. 	mulps	%xmm15, %xmm8

  814. 	addps	%xmm8,  %xmm6

  815. 	movaps	%xmm6,  %xmm10

  816. 	movss	%xmm9,  %xmm6

  817. 	movaps	%xmm6,   -8 * SIZE(X)

  818. 

  819. 	movaps	 24 * SIZE(X), %xmm6

  820. 

  821. 	movss	%xmm0,  %xmm7

  822. 	pshufd	$0x1b,  %xmm7, %xmm8

  823. 	mulps	%xmm14, %xmm7

  824. 	mulps	%xmm15, %xmm8

  825. 	addps	%xmm8,  %xmm7

  826. 	movaps	%xmm7,  %xmm9

  827. 	movss	%xmm10,  %xmm7

  828. 	movaps	%xmm7,   -4 * SIZE(X)

  829. 

  830. 	movaps	 28 * SIZE(X), %xmm7

  831. 

  832. 	subq	$-32 * SIZE, X

  833. 	decq	I

  834. 	jg	.L131

  835. 	ALIGN_4

  836. 

  837. .L132:

  838. 	movss	%xmm1,  %xmm0

  839. 	pshufd	$0x1b,  %xmm0, %xmm8

  840. 	mulps	%xmm14, %xmm0

  841. 	mulps	%xmm15, %xmm8

  842. 	addps	%xmm8,  %xmm0

  843. 	movaps	%xmm0,  %xmm10

  844. 	movss	%xmm9,  %xmm0

  845. 	movaps	%xmm0,  -32 * SIZE(X)

  846. 

  847. 	movaps	   0 * SIZE(X), %xmm0

  848. 

  849. 	movss	%xmm2,  %xmm1

  850. 	pshufd	$0x1b,  %xmm1, %xmm8

  851. 	mulps	%xmm14, %xmm1

  852. 	mulps	%xmm15, %xmm8

  853. 	addps	%xmm8,  %xmm1

  854. 	movaps	%xmm1,  %xmm9

  855. 	movss	%xmm10,  %xmm1

  856. 	movaps	%xmm1,  -28 * SIZE(X)

  857. 

  858. 	movss	%xmm3,  %xmm2

  859. 	pshufd	$0x1b,  %xmm2, %xmm8

  860. 	mulps	%xmm14, %xmm2

  861. 	mulps	%xmm15, %xmm8

  862. 	addps	%xmm8,  %xmm2

  863. 	movaps	%xmm2,  %xmm10

  864. 	movss	%xmm9,  %xmm2

  865. 	movaps	%xmm2,  -24 * SIZE(X)

  866. 

  867. 	movss	%xmm4,  %xmm3

  868. 	pshufd	$0x1b,  %xmm3, %xmm8

  869. 	mulps	%xmm14, %xmm3

  870. 	mulps	%xmm15, %xmm8

  871. 	addps	%xmm8,  %xmm3

  872. 	movaps	%xmm3,  %xmm9

  873. 	movss	%xmm10,  %xmm3

  874. 	movaps	%xmm3,  -20 * SIZE(X)

  875. 

  876. 	movss	%xmm5,  %xmm4

  877. 	pshufd	$0x1b,  %xmm4, %xmm8

  878. 	mulps	%xmm14, %xmm4

  879. 	mulps	%xmm15, %xmm8

  880. 	addps	%xmm8,  %xmm4

  881. 	movaps	%xmm4,  %xmm10

  882. 	movss	%xmm9,  %xmm4

  883. 	movaps	%xmm4,  -16 * SIZE(X)

  884. 

  885. 	movss	%xmm6,  %xmm5

  886. 	pshufd	$0x1b,  %xmm5, %xmm8

  887. 	mulps	%xmm14, %xmm5

  888. 	mulps	%xmm15, %xmm8

  889. 	addps	%xmm8,  %xmm5

  890. 	movaps	%xmm5,  %xmm9

  891. 	movss	%xmm10,  %xmm5

  892. 	movaps	%xmm5,  -12 * SIZE(X)

  893. 

  894. 	movss	%xmm7,  %xmm6

  895. 	pshufd	$0x1b,  %xmm6, %xmm8

  896. 	mulps	%xmm14, %xmm6

  897. 	mulps	%xmm15, %xmm8

  898. 	addps	%xmm8,  %xmm6

  899. 	movaps	%xmm6,  %xmm10

  900. 	movss	%xmm9,  %xmm6

  901. 	movaps	%xmm6,   -8 * SIZE(X)

  902. 

  903. 	movss	%xmm0,  %xmm7

  904. 	pshufd	$0x1b,  %xmm7, %xmm8

  905. 	mulps	%xmm14, %xmm7

  906. 	mulps	%xmm15, %xmm8

  907. 	addps	%xmm8,  %xmm7

  908. 	movaps	%xmm7,  %xmm9

  909. 	movss	%xmm10,  %xmm7

  910. 	movaps	%xmm7,   -4 * SIZE(X)

  911. 

  912. 	subq	$-32 * SIZE, X

  913. 	ALIGN_4

  914. 

  915. .L135:

  916. 	testq	$8, M

  917. 	je	.L136

  918. 

  919. 	movaps	-28 * SIZE(X), %xmm1

  920. 

  921. 	movss	%xmm1,  %xmm0

  922. 	pshufd	$0x1b,  %xmm0, %xmm8

  923. 	mulps	%xmm14, %xmm0

  924. 	mulps	%xmm15, %xmm8

  925. 	addps	%xmm8,  %xmm0

  926. 

  927. 	movaps	%xmm0,  %xmm10

  928. 	movss	%xmm9,  %xmm0

  929. 	movaps	%xmm0,  -32 * SIZE(X)

  930. 

  931. 	movaps	-24 * SIZE(X), %xmm2

  932. 

  933. 	movss	%xmm2,  %xmm1

  934. 	pshufd	$0x1b,  %xmm1, %xmm8

  935. 	mulps	%xmm14, %xmm1

  936. 	mulps	%xmm15, %xmm8

  937. 	addps	%xmm8,  %xmm1

  938. 

  939. 	movaps	%xmm1,  %xmm9

  940. 	movss	%xmm10,  %xmm1

  941. 	movaps	%xmm1,  -28 * SIZE(X)

  942. 

  943. 	movaps	-20 * SIZE(X), %xmm3

  944. 

  945. 	movss	%xmm3,  %xmm2

  946. 	pshufd	$0x1b,  %xmm2, %xmm8

  947. 	mulps	%xmm14, %xmm2

  948. 	mulps	%xmm15, %xmm8

  949. 	addps	%xmm8,  %xmm2

  950. 

  951. 	movaps	%xmm2,  %xmm10

  952. 	movss	%xmm9,  %xmm2

  953. 	movaps	%xmm2,  -24 * SIZE(X)

  954. 

  955. 	movaps	-16 * SIZE(X), %xmm0

  956. 

  957. 	movss	%xmm0,  %xmm3

  958. 	pshufd	$0x1b,  %xmm3, %xmm8

  959. 	mulps	%xmm14, %xmm3

  960. 	mulps	%xmm15, %xmm8

  961. 	addps	%xmm8,  %xmm3

  962. 

  963. 	movaps	%xmm3,  %xmm9

  964. 	movss	%xmm10,  %xmm3

  965. 	movaps	%xmm3,  -20 * SIZE(X)

  966. 

  967. 	addq	$16 * SIZE, X

  968. 	ALIGN_3

  969. 

  970. .L136:

  971. 	testq	$4, M

  972. 	je	.L137

  973. 

  974. 	movaps	-28 * SIZE(X), %xmm1

  975. 

  976. 	movss	%xmm1,  %xmm0

  977. 	pshufd	$0x1b,  %xmm0, %xmm8

  978. 	mulps	%xmm14, %xmm0

  979. 	mulps	%xmm15, %xmm8

  980. 	addps	%xmm8,  %xmm0

  981. 

  982. 	movaps	%xmm0,  %xmm10

  983. 	movss	%xmm9,  %xmm0

  984. 	movaps	%xmm0,  -32 * SIZE(X)

  985. 

  986. 	movaps	-24 * SIZE(X), %xmm2

  987. 

  988. 	movss	%xmm2,  %xmm1

  989. 	pshufd	$0x1b,  %xmm1, %xmm8

  990. 	mulps	%xmm14, %xmm1

  991. 	mulps	%xmm15, %xmm8

  992. 	addps	%xmm8,  %xmm1

  993. 

  994. 	movaps	%xmm1,  %xmm9

  995. 	movss	%xmm10,  %xmm1

  996. 	movaps	%xmm1,  -28 * SIZE(X)

  997. 

  998. 	movaps	%xmm2,  %xmm0

  999. 

  1000. 	addq	$8 * SIZE, X

  1001. 	ALIGN_3

  1002. 

  1003. .L137:

  1004. 	testq	$2, M

  1005. 	je	.L138

  1006. 

  1007. 	movaps	-28 * SIZE(X), %xmm1

  1008. 

  1009. 	movss	%xmm1,  %xmm0

  1010. 	pshufd	$0x1b,  %xmm0, %xmm8

  1011. 	mulps	%xmm14, %xmm0

  1012. 	mulps	%xmm15, %xmm8

  1013. 	addps	%xmm8,  %xmm0

  1014. 

  1015. 	movaps	%xmm0,  %xmm10

  1016. 	movss	%xmm9,  %xmm0

  1017. 	movaps	%xmm0,  -32 * SIZE(X)

  1018. 	movaps	%xmm10, %xmm9

  1019. 	movaps	%xmm1,  %xmm0

  1020. 

  1021. 	addq	$4 * SIZE, X

  1022. 	ALIGN_3

  1023. 

  1024. .L138:

  1025. 	movss	%xmm9, -32 * SIZE(X)

  1026. 

  1027. 	testq	$1, M

  1028. 	je	.L999

  1029. 

  1030. 	pshufd	 $0x1b,  %xmm0, %xmm8

  1031. 	mulps	 %xmm14, %xmm0

  1032. 	mulps	 %xmm15, %xmm8

  1033. 	addps	 %xmm8,  %xmm0

  1034. 

  1035. 	pshufd	 $0x39, %xmm0, %xmm0

  1036. 

  1037. 	movlps	%xmm0,  -31 * SIZE(X)

  1038. 	jmp	.L999

  1039. 	ALIGN_3

  1040. 

  1041. 

  1042. #else

  1043. 

  1044. 	pshufd	$0, %xmm0, %xmm14

  1045. 	pshufd	$0, %xmm1, %xmm1

  1046. 	subps	  %xmm1,   %xmm15

  1047. 	unpcklps  %xmm1,   %xmm15

  1048. 

  1049. 	subq	$-32 * SIZE, X

  1050. 

  1051. 	testq	$2 * SIZE, X

  1052. 	je	.L130x

  1053. 

  1054. 	movsd	-32 * SIZE(X), %xmm0

  1055. 

  1056. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1057. 	mulps	 %xmm14, %xmm0

  1058. 	mulps	 %xmm15, %xmm8

  1059. 	addps	 %xmm8,  %xmm0

  1060. 

  1061. 	movlps	%xmm0,  -32 * SIZE(X)

  1062. 	addq	$2 * SIZE, X

  1063. 	decq	M

  1064. 	jle	.L999

  1065. 	ALIGN_3

  1066. 

  1067. .L130x:

  1068. 	movq	M,  I

  1069. 	sarq	$4, I

  1070. 	jle	.L135

  1071. 

  1072. 	movsd	-32 * SIZE(X), %xmm0

  1073. 	movhps	-30 * SIZE(X), %xmm0

  1074. 	movsd	-28 * SIZE(X), %xmm1

  1075. 	movhps	-26 * SIZE(X), %xmm1

  1076. 	movsd	-24 * SIZE(X), %xmm2

  1077. 	movhps	-22 * SIZE(X), %xmm2

  1078. 	movsd	-20 * SIZE(X), %xmm3

  1079. 	movhps	-18 * SIZE(X), %xmm3

  1080. 	movsd	-16 * SIZE(X), %xmm4

  1081. 	movhps	-14 * SIZE(X), %xmm4

  1082. 	movsd	-12 * SIZE(X), %xmm5

  1083. 	movhps	-10 * SIZE(X), %xmm5

  1084. 	movsd	 -8 * SIZE(X), %xmm6

  1085. 	movhps	 -6 * SIZE(X), %xmm6

  1086. 	movsd	 -4 * SIZE(X), %xmm7

  1087. 	movhps	 -2 * SIZE(X), %xmm7

  1088. 

  1089. 	decq	I

  1090. 	jle	.L132

  1091. 	ALIGN_4

  1092. 

  1093. .L131:

  1094. #ifdef PREFETCHW

  1095. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  1096. #endif

  1097. 

  1098. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1099. 	mulps	 %xmm14, %xmm0

  1100. 	mulps	 %xmm15, %xmm8

  1101. 	addps	 %xmm8,  %xmm0

  1102. 	movlps	%xmm0,  -32 * SIZE(X)

  1103. 	movhps	%xmm0,  -30 * SIZE(X)

  1104. 

  1105. 	movsd	  0 * SIZE(X), %xmm0

  1106. 	movhps	  2 * SIZE(X), %xmm0

  1107. 

  1108. 	pshufd	 $0xb1,  %xmm1, %xmm8

  1109. 	mulps	 %xmm14, %xmm1

  1110. 	mulps	 %xmm15, %xmm8

  1111. 	addps	 %xmm8,  %xmm1

  1112. 	movlps	%xmm1,  -28 * SIZE(X)

  1113. 	movhps	%xmm1,  -26 * SIZE(X)

  1114. 

  1115. 	movsd	  4 * SIZE(X), %xmm1

  1116. 	movhps	  6 * SIZE(X), %xmm1

  1117. 

  1118. 	pshufd	 $0xb1,  %xmm2, %xmm8

  1119. 	mulps	 %xmm14, %xmm2

  1120. 	mulps	 %xmm15, %xmm8

  1121. 	addps	 %xmm8,  %xmm2

  1122. 	movlps	%xmm2,  -24 * SIZE(X)

  1123. 	movhps	%xmm2,  -22 * SIZE(X)

  1124. 

  1125. 	movsd	  8 * SIZE(X), %xmm2

  1126. 	movhps	 10 * SIZE(X), %xmm2

  1127. 

  1128. 	pshufd	 $0xb1,  %xmm3, %xmm8

  1129. 	mulps	 %xmm14, %xmm3

  1130. 	mulps	 %xmm15, %xmm8

  1131. 	addps	 %xmm8,  %xmm3

  1132. 	movlps	%xmm3,  -20 * SIZE(X)

  1133. 	movhps	%xmm3,  -18 * SIZE(X)

  1134. 

  1135. 	movsd	 12 * SIZE(X), %xmm3

  1136. 	movhps	 14 * SIZE(X), %xmm3

  1137. 

  1138. #ifdef PREFETCHW

  1139. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  1140. #endif

  1141. 

  1142. 	pshufd	 $0xb1,  %xmm4, %xmm8

  1143. 	mulps	 %xmm14, %xmm4

  1144. 	mulps	 %xmm15, %xmm8

  1145. 	addps	 %xmm8,  %xmm4

  1146. 	movlps	%xmm4,  -16 * SIZE(X)

  1147. 	movhps	%xmm4,  -14 * SIZE(X)

  1148. 

  1149. 	movsd	 16 * SIZE(X), %xmm4

  1150. 	movhps	 18 * SIZE(X), %xmm4

  1151. 

  1152. 	pshufd	 $0xb1,  %xmm5, %xmm8

  1153. 	mulps	 %xmm14, %xmm5

  1154. 	mulps	 %xmm15, %xmm8

  1155. 	addps	 %xmm8,  %xmm5

  1156. 	movlps	%xmm5,  -12 * SIZE(X)

  1157. 	movhps	%xmm5,  -10 * SIZE(X)

  1158. 

  1159. 	movsd	 20 * SIZE(X), %xmm5

  1160. 	movhps	 22 * SIZE(X), %xmm5

  1161. 

  1162. 	pshufd	 $0xb1,  %xmm6, %xmm8

  1163. 	mulps	 %xmm14, %xmm6

  1164. 	mulps	 %xmm15, %xmm8

  1165. 	addps	 %xmm8,  %xmm6

  1166. 	movlps	%xmm6,   -8 * SIZE(X)

  1167. 	movhps	%xmm6,   -6 * SIZE(X)

  1168. 

  1169. 	movsd	 24 * SIZE(X), %xmm6

  1170. 	movhps	 26 * SIZE(X), %xmm6

  1171. 

  1172. 	pshufd	 $0xb1,  %xmm7, %xmm8

  1173. 	mulps	 %xmm14, %xmm7

  1174. 	mulps	 %xmm15, %xmm8

  1175. 	addps	 %xmm8,  %xmm7

  1176. 	movlps	%xmm7,   -4 * SIZE(X)

  1177. 	movhps	%xmm7,   -2 * SIZE(X)

  1178. 

  1179. 	movsd	 28 * SIZE(X), %xmm7

  1180. 	movhps	 30 * SIZE(X), %xmm7

  1181. 

  1182. 	subq	$-32 * SIZE, X

  1183. 	decq	I

  1184. 	jg	.L131

  1185. 	ALIGN_4

  1186. 

  1187. .L132:

  1188. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1189. 	mulps	 %xmm14, %xmm0

  1190. 	mulps	 %xmm15, %xmm8

  1191. 	addps	 %xmm8,  %xmm0

  1192. 	movlps	%xmm0,  -32 * SIZE(X)

  1193. 	movhps	%xmm0,  -30 * SIZE(X)

  1194. 

  1195. 	pshufd	 $0xb1,  %xmm1, %xmm8

  1196. 	mulps	 %xmm14, %xmm1

  1197. 	mulps	 %xmm15, %xmm8

  1198. 	addps	 %xmm8,  %xmm1

  1199. 	movlps	%xmm1,  -28 * SIZE(X)

  1200. 	movhps	%xmm1,  -26 * SIZE(X)

  1201. 

  1202. 	pshufd	 $0xb1,  %xmm2, %xmm8

  1203. 	mulps	 %xmm14, %xmm2

  1204. 	mulps	 %xmm15, %xmm8

  1205. 	addps	 %xmm8,  %xmm2

  1206. 	movlps	%xmm2,  -24 * SIZE(X)

  1207. 	movhps	%xmm2,  -22 * SIZE(X)

  1208. 

  1209. 	pshufd	 $0xb1,  %xmm3, %xmm8

  1210. 	mulps	 %xmm14, %xmm3

  1211. 	mulps	 %xmm15, %xmm8

  1212. 	addps	 %xmm8,  %xmm3

  1213. 	movlps	%xmm3,  -20 * SIZE(X)

  1214. 	movhps	%xmm3,  -18 * SIZE(X)

  1215. 

  1216. 	pshufd	 $0xb1,  %xmm4, %xmm8

  1217. 	mulps	 %xmm14, %xmm4

  1218. 	mulps	 %xmm15, %xmm8

  1219. 	addps	 %xmm8,  %xmm4

  1220. 	movlps	%xmm4,  -16 * SIZE(X)

  1221. 	movhps	%xmm4,  -14 * SIZE(X)

  1222. 

  1223. 	pshufd	 $0xb1,  %xmm5, %xmm8

  1224. 	mulps	 %xmm14, %xmm5

  1225. 	mulps	 %xmm15, %xmm8

  1226. 	addps	 %xmm8,  %xmm5

  1227. 	movlps	%xmm5,  -12 * SIZE(X)

  1228. 	movhps	%xmm5,  -10 * SIZE(X)

  1229. 

  1230. 	pshufd	 $0xb1,  %xmm6, %xmm8

  1231. 	mulps	 %xmm14, %xmm6

  1232. 	mulps	 %xmm15, %xmm8

  1233. 	addps	 %xmm8,  %xmm6

  1234. 	movlps	%xmm6,   -8 * SIZE(X)

  1235. 	movhps	%xmm6,   -6 * SIZE(X)

  1236. 

  1237. 	pshufd	 $0xb1,  %xmm7, %xmm8

  1238. 	mulps	 %xmm14, %xmm7

  1239. 	mulps	 %xmm15, %xmm8

  1240. 	addps	 %xmm8,  %xmm7

  1241. 	movlps	%xmm7,   -4 * SIZE(X)

  1242. 	movhps	%xmm7,   -2 * SIZE(X)

  1243. 

  1244. 	subq	$-32 * SIZE, X

  1245. 	ALIGN_4

  1246. 

  1247. .L135:

  1248. 	testq	$8, M

  1249. 	je	.L136

  1250. 

  1251. 	movsd	-32 * SIZE(X), %xmm0

  1252. 	movhps	-30 * SIZE(X), %xmm0

  1253. 

  1254. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1255. 	mulps	 %xmm14, %xmm0

  1256. 	mulps	 %xmm15, %xmm8

  1257. 	addps	 %xmm8,  %xmm0

  1258. 	movlps	%xmm0,  -32 * SIZE(X)

  1259. 	movhps	%xmm0,  -30 * SIZE(X)

  1260. 

  1261. 	movsd	-28 * SIZE(X), %xmm1

  1262. 	movhps	-26 * SIZE(X), %xmm1

  1263. 

  1264. 	pshufd	 $0xb1,  %xmm1, %xmm8

  1265. 	mulps	 %xmm14, %xmm1

  1266. 	mulps	 %xmm15, %xmm8

  1267. 	addps	 %xmm8,  %xmm1

  1268. 	movlps	%xmm1,  -28 * SIZE(X)

  1269. 	movhps	%xmm1,  -26 * SIZE(X)

  1270. 

  1271. 	movsd	-24 * SIZE(X), %xmm2

  1272. 	movhps	-22 * SIZE(X), %xmm2

  1273. 

  1274. 	pshufd	 $0xb1,  %xmm2, %xmm8

  1275. 	mulps	 %xmm14, %xmm2

  1276. 	mulps	 %xmm15, %xmm8

  1277. 	addps	 %xmm8,  %xmm2

  1278. 	movlps	%xmm2,  -24 * SIZE(X)

  1279. 	movhps	%xmm2,  -22 * SIZE(X)

  1280. 

  1281. 	movsd	-20 * SIZE(X), %xmm3

  1282. 	movhps	-18 * SIZE(X), %xmm3

  1283. 

  1284. 	pshufd	 $0xb1,  %xmm3, %xmm8

  1285. 	mulps	 %xmm14, %xmm3

  1286. 	mulps	 %xmm15, %xmm8

  1287. 	addps	 %xmm8,  %xmm3

  1288. 	movlps	%xmm3,  -20 * SIZE(X)

  1289. 	movhps	%xmm3,  -18 * SIZE(X)

  1290. 

  1291. 	addq	$16 * SIZE, X

  1292. 	ALIGN_3

  1293. 

  1294. .L136:

  1295. 	testq	$4, M

  1296. 	je	.L137

  1297. 

  1298. 	movsd	-32 * SIZE(X), %xmm0

  1299. 	movhps	-30 * SIZE(X), %xmm0

  1300. 	movsd	-28 * SIZE(X), %xmm1

  1301. 	movhps	-26 * SIZE(X), %xmm1

  1302. 

  1303. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1304. 	mulps	 %xmm14, %xmm0

  1305. 	mulps	 %xmm15, %xmm8

  1306. 	addps	 %xmm8,  %xmm0

  1307. 	movlps	%xmm0,  -32 * SIZE(X)

  1308. 	movhps	%xmm0,  -30 * SIZE(X)

  1309. 

  1310. 	pshufd	 $0xb1,  %xmm1, %xmm8

  1311. 	mulps	 %xmm14, %xmm1

  1312. 	mulps	 %xmm15, %xmm8

  1313. 	addps	 %xmm8,  %xmm1

  1314. 	movlps	%xmm1,  -28 * SIZE(X)

  1315. 	movhps	%xmm1,  -26 * SIZE(X)

  1316. 

  1317. 	addq	$8 * SIZE, X

  1318. 	ALIGN_3

  1319. 

  1320. .L137:

  1321. 	testq	$2, M

  1322. 	je	.L138

  1323. 

  1324. 	movsd	-32 * SIZE(X), %xmm0

  1325. 	movhps	-30 * SIZE(X), %xmm0

  1326. 

  1327. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1328. 	mulps	 %xmm14, %xmm0

  1329. 	mulps	 %xmm15, %xmm8

  1330. 	addps	 %xmm8,  %xmm0

  1331. 	movlps	%xmm0,  -32 * SIZE(X)

  1332. 	movhps	%xmm0,  -30 * SIZE(X)

  1333. 

  1334. 	addq	$4 * SIZE, X

  1335. 	ALIGN_3

  1336. 

  1337. .L138:

  1338. 	testq	$1, M

  1339. 	je	.L999

  1340. 

  1341. 	movsd	-32 * SIZE(X), %xmm0

  1342. 

  1343. 	pshufd	 $0xb1,  %xmm0, %xmm8

  1344. 	mulps	 %xmm14, %xmm0

  1345. 	mulps	 %xmm15, %xmm8

  1346. 	addps	 %xmm8,  %xmm0

  1347. 

  1348. 	movlps	%xmm0,  -32 * SIZE(X)

  1349. 	ALIGN_3

  1350. #endif

  1351. 

  1352. .L999:

  1353. 	xorq	%rax, %rax

  1354. 

  1355. 	RESTOREREGISTERS

  1356. 

  1357. 	ret

  1358. 

  1359. 	EPILOGUE