OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 549a9f1dbb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '549a9f1dbb'
${ noResults }
OpenBLAS/kernel/x86_64/izamax_sse2.S

598 lines
11 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define M	ARG1	/* rdi */

  43. #define X	ARG2	/* rsi */

  44. #define INCX	ARG3	/* rdx */

  45. 

  46. #define RET	%rax

  47. #define I	ARG4

  48. #define XX	%r10

  49. #define	MM	%r11

  50. 

  51. #ifdef USE_MIN

  52. #define maxpd	minpd

  53. #define maxsd	minsd

  54. #endif

  55. 

  56. #include "l1param.h"

  57. 

  58. 	PROLOGUE

  59. 	PROFCODE

  60. 

  61. 	SAVEREGISTERS

  62. 

  63. 	pxor	%xmm0, %xmm0

  64. 	xor	RET, RET

  65. 	testq	M, M

  66. 	jle	.L999

  67. 	testq	INCX, INCX

  68. 	jle	.L999

  69. 

  70. 	salq	$ZBASE_SHIFT, INCX

  71. 	movq	M, MM

  72. 	movq	X, XX

  73. 

  74. 	pcmpeqb	%xmm15, %xmm15

  75. 	psrlq	$1, %xmm15

  76. 

  77. 	movsd	0 * SIZE(X), %xmm0

  78. 	movsd	1 * SIZE(X), %xmm1

  79. 	addq	INCX, X

  80. 	decq	M

  81. 	andpd	 %xmm15, %xmm0

  82. 	andpd	 %xmm15, %xmm1

  83. 	addpd	 %xmm1, %xmm0

  84. 	unpcklpd %xmm0, %xmm0

  85. 	movapd	 %xmm0, %xmm1

  86. 	movapd	 %xmm0, %xmm2

  87. 	movapd	 %xmm0, %xmm3

  88. 	cmpq	$2 * SIZE, INCX

  89. 	jne	.L60

  90. 

  91. 	movq	M,  I

  92. 	sarq	$3, I

  93. 	jle	.L25

  94. 	ALIGN_4

  95. 

  96. .L21:

  97. #ifdef PREFETCH

  98. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  99. #endif

  100. 

  101. 	movsd	0 * SIZE(X), %xmm4

  102. 	movsd	1 * SIZE(X), %xmm5

  103. 	movhpd	2 * SIZE(X), %xmm4

  104. 	movhpd	3 * SIZE(X), %xmm5

  105. 	movsd	4 * SIZE(X), %xmm6

  106. 	movsd	5 * SIZE(X), %xmm7

  107. 	movhpd	6 * SIZE(X), %xmm6

  108. 	movhpd	7 * SIZE(X), %xmm7

  109. 

  110. 	andpd	%xmm15, %xmm4

  111. 	andpd	%xmm15, %xmm5

  112. 	andpd	%xmm15, %xmm6

  113. 	andpd	%xmm15, %xmm7

  114. 

  115. 	addpd	%xmm5,  %xmm4

  116. 	addpd	%xmm7,  %xmm6

  117. 	maxpd	%xmm4, %xmm0

  118. 	maxpd	%xmm6, %xmm1

  119. 

  120. #ifdef PREFETCH

  121. 	PREFETCH (PREFETCHSIZE + 64) - PREOFFSET(X)

  122. #endif

  123. 

  124. 	movsd	 8 * SIZE(X), %xmm4

  125. 	movsd	 9 * SIZE(X), %xmm5

  126. 	movhpd	10 * SIZE(X), %xmm4

  127. 	movhpd	11 * SIZE(X), %xmm5

  128. 	movsd	12 * SIZE(X), %xmm6

  129. 	movsd	13 * SIZE(X), %xmm7

  130. 	movhpd	14 * SIZE(X), %xmm6

  131. 	movhpd	15 * SIZE(X), %xmm7

  132. 

  133. 	andpd	%xmm15, %xmm4

  134. 	andpd	%xmm15, %xmm5

  135. 	andpd	%xmm15, %xmm6

  136. 	andpd	%xmm15, %xmm7

  137. 

  138. 	addpd	%xmm5,  %xmm4

  139. 	addpd	%xmm7,  %xmm6

  140. 	maxpd	%xmm4, %xmm2

  141. 	maxpd	%xmm6, %xmm3

  142. 

  143. 	addq	$16 * SIZE, X

  144. 	decq	I

  145. 	jg	.L21

  146. 	ALIGN_4

  147. 

  148. .L25:

  149. 	andq	$7,  M

  150. 	jle	.L30

  151. 

  152. 	testq	$4, M

  153. 	je	.L26

  154. 

  155. 	movsd	0 * SIZE(X), %xmm4

  156. 	movsd	1 * SIZE(X), %xmm5

  157. 	movhpd	2 * SIZE(X), %xmm4

  158. 	movhpd	3 * SIZE(X), %xmm5

  159. 	movsd	4 * SIZE(X), %xmm6

  160. 	movsd	5 * SIZE(X), %xmm7

  161. 	movhpd	6 * SIZE(X), %xmm6

  162. 	movhpd	7 * SIZE(X), %xmm7

  163. 

  164. 	andpd	%xmm15, %xmm4

  165. 	andpd	%xmm15, %xmm5

  166. 	andpd	%xmm15, %xmm6

  167. 	andpd	%xmm15, %xmm7

  168. 

  169. 	addpd	%xmm5,  %xmm4

  170. 	addpd	%xmm7,  %xmm6

  171. 	maxpd	%xmm4, %xmm0

  172. 	maxpd	%xmm6, %xmm1

  173. 

  174. 	addq	$8 * SIZE, X

  175. 	ALIGN_3

  176. 

  177. .L26:

  178. 	testq	$2, M

  179. 	je	.L27

  180. 

  181. 	movsd	0 * SIZE(X), %xmm4

  182. 	movsd	1 * SIZE(X), %xmm5

  183. 	movhpd	2 * SIZE(X), %xmm4

  184. 	movhpd	3 * SIZE(X), %xmm5

  185. 	addq	$4 * SIZE, X

  186. 

  187. 	andpd	%xmm15, %xmm4

  188. 	andpd	%xmm15, %xmm5

  189. 	addpd	%xmm5,  %xmm4

  190. 	maxpd	%xmm4, %xmm0

  191. 	ALIGN_3

  192. 

  193. .L27:

  194. 	testq	$1, M

  195. 	je	.L30

  196. 

  197. 	movsd	0 * SIZE(X), %xmm4

  198. 	movsd	1 * SIZE(X), %xmm5

  199. 	andpd	%xmm15, %xmm4

  200. 	andpd	%xmm15, %xmm5

  201. 	addpd	%xmm5, %xmm4

  202. 	maxsd	%xmm4, %xmm2

  203. 	ALIGN_4

  204. 

  205. .L30:

  206. 	movq	XX, X

  207. 	movq	MM, M

  208. 

  209. 	maxpd	%xmm1, %xmm0

  210. 	maxpd	%xmm3, %xmm2

  211. 	maxpd	%xmm2, %xmm0

  212. 	movapd	%xmm0, %xmm1

  213. 	unpckhpd %xmm0, %xmm0

  214. 	maxsd	%xmm1, %xmm0

  215. 	unpcklpd %xmm0, %xmm0

  216. 

  217. 	movq	M,  I

  218. 	sarq	$2, I

  219. 	jle	.L35

  220. 	ALIGN_4

  221. 

  222. .L31:

  223. #ifdef PREFETCH

  224. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  225. #endif

  226. 

  227. 	movsd	0 * SIZE(X), %xmm1

  228. 	movsd	1 * SIZE(X), %xmm2

  229. 	movhpd	2 * SIZE(X), %xmm1

  230. 	movhpd	3 * SIZE(X), %xmm2

  231. 	movsd	4 * SIZE(X), %xmm3

  232. 	movsd	5 * SIZE(X), %xmm4

  233. 	movhpd	6 * SIZE(X), %xmm3

  234. 	movhpd	7 * SIZE(X), %xmm4

  235. 

  236. 	andpd	%xmm15, %xmm1

  237. 	andpd	%xmm15, %xmm2

  238. 	andpd	%xmm15, %xmm3

  239. 	andpd	%xmm15, %xmm4

  240. 

  241. 	addpd	%xmm2,  %xmm1

  242. 	addpd	%xmm4,  %xmm3

  243. 

  244. 	cmpeqpd	%xmm0, %xmm1

  245. 	cmpeqpd	%xmm0, %xmm3

  246. 

  247. 	orpd	%xmm3, %xmm1

  248. #ifndef C_SUN

  249. 	movmskpd %xmm1, %r11

  250. #else

  251. 	.byte	0x66

  252. 	.long	0xd9500f4c

  253. #endif

  254. 	testq	 $3, %r11

  255. 	jne	 .L33

  256. 

  257. 	addq	$8 * SIZE, X

  258. 	addq	$4, RET

  259. 	decq	I

  260. 	jg	.L31

  261. 	jmp	.L35

  262. 	ALIGN_4

  263. 

  264. .L33:

  265. 	movsd	0 * SIZE(X), %xmm1

  266. 	movsd	1 * SIZE(X), %xmm2

  267. 	movsd	2 * SIZE(X), %xmm3

  268. 	movsd	3 * SIZE(X), %xmm4

  269. 	movsd	4 * SIZE(X), %xmm5

  270. 	movsd	5 * SIZE(X), %xmm6

  271. 	movsd	6 * SIZE(X), %xmm7

  272. 	movsd	7 * SIZE(X), %xmm8

  273. 	addq	$8 * SIZE, X

  274. 

  275. 	andpd	%xmm15, %xmm1

  276. 	andpd	%xmm15, %xmm2

  277. 	andpd	%xmm15, %xmm3

  278. 	andpd	%xmm15, %xmm4

  279. 	andpd	%xmm15, %xmm5

  280. 	andpd	%xmm15, %xmm6

  281. 	andpd	%xmm15, %xmm7

  282. 	andpd	%xmm15, %xmm8

  283. 

  284. 	addpd	%xmm2,  %xmm1

  285. 	addpd	%xmm4,  %xmm3

  286. 	addpd	%xmm6,  %xmm5

  287. 	addpd	%xmm8,  %xmm7

  288. 

  289. 	incq	RET

  290. 	comisd	%xmm0, %xmm1

  291. 	je	.L999

  292. 	incq	RET

  293. 	comisd	%xmm0, %xmm3

  294. 	je	.L999

  295. 	incq	RET

  296. 	comisd	%xmm0, %xmm5

  297. 	je	.L999

  298. 	incq	RET

  299. 	comisd	%xmm0, %xmm7

  300. 	je	.L999

  301. 	ALIGN_3

  302. 

  303. .L35:

  304. 	testq	$2, M

  305. 	je	.L36

  306. 

  307. 	movsd	0 * SIZE(X), %xmm1

  308. 	movsd	1 * SIZE(X), %xmm2

  309. 	movsd	2 * SIZE(X), %xmm3

  310. 	movsd	3 * SIZE(X), %xmm4

  311. 	addq	$4 * SIZE, X

  312. 

  313. 	andpd	%xmm15, %xmm1

  314. 	andpd	%xmm15, %xmm2

  315. 	andpd	%xmm15, %xmm3

  316. 	andpd	%xmm15, %xmm4

  317. 

  318. 	addpd	%xmm2,  %xmm1

  319. 	addpd	%xmm4,  %xmm3

  320. 	incq	RET

  321. 	comisd	%xmm0, %xmm1

  322. 	je	.L999

  323. 	incq	RET

  324. 	comisd	%xmm0, %xmm3

  325. 	je	.L999

  326. 	ALIGN_3

  327. 

  328. .L36:

  329. 	incq	RET

  330. 	jmp	.L999

  331. 	ALIGN_3

  332. 

  333. .L60:

  334. 	movq	M,  I

  335. 	sarq	$3, I

  336. 	jle	.L65

  337. 	ALIGN_4

  338. 

  339. .L61:

  340. #ifdef PREFETCH

  341. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  342. #endif

  343. 

  344. 	movsd	0 * SIZE(X), %xmm4

  345. 	movsd	1 * SIZE(X), %xmm5

  346. 	addq	INCX, X

  347. 	movhpd	0 * SIZE(X), %xmm4

  348. 	movhpd	1 * SIZE(X), %xmm5

  349. 	addq	INCX, X

  350. 	movsd	0 * SIZE(X), %xmm6

  351. 	movsd	1 * SIZE(X), %xmm7

  352. 	addq	INCX, X

  353. 	movhpd	0 * SIZE(X), %xmm6

  354. 	movhpd	1 * SIZE(X), %xmm7

  355. 	addq	INCX, X

  356. 

  357. 	andpd	%xmm15, %xmm4

  358. 	andpd	%xmm15, %xmm5

  359. 	andpd	%xmm15, %xmm6

  360. 	andpd	%xmm15, %xmm7

  361. 	addpd	%xmm5, %xmm4

  362. 	addpd	%xmm7, %xmm6

  363. 	maxpd	%xmm4, %xmm0

  364. 	maxpd	%xmm6, %xmm1

  365. 

  366. #ifdef PREFETCH

  367. 	PREFETCH (PREFETCHSIZE + 64) - PREOFFSET(X)

  368. #endif

  369. 

  370. 	movsd	0 * SIZE(X), %xmm4

  371. 	movsd	1 * SIZE(X), %xmm5

  372. 	addq	INCX, X

  373. 	movhpd	0 * SIZE(X), %xmm4

  374. 	movhpd	1 * SIZE(X), %xmm5

  375. 	addq	INCX, X

  376. 	movsd	0 * SIZE(X), %xmm6

  377. 	movsd	1 * SIZE(X), %xmm7

  378. 	addq	INCX, X

  379. 	movhpd	0 * SIZE(X), %xmm6

  380. 	movhpd	1 * SIZE(X), %xmm7

  381. 	addq	INCX, X

  382. 

  383. 	andpd	%xmm15, %xmm4

  384. 	andpd	%xmm15, %xmm5

  385. 	andpd	%xmm15, %xmm6

  386. 	andpd	%xmm15, %xmm7

  387. 	addpd	%xmm5, %xmm4

  388. 	addpd	%xmm7, %xmm6

  389. 	maxpd	%xmm4, %xmm2

  390. 	maxpd	%xmm6, %xmm3

  391. 

  392. 	decq	I

  393. 	jg	.L61

  394. 	ALIGN_4

  395. 

  396. .L65:

  397. 	andq	$7,  M

  398. 	jle	.L70

  399. 

  400. 	testq	$4, M

  401. 	je	.L66

  402. 

  403. 	movsd	0 * SIZE(X), %xmm4

  404. 	movsd	1 * SIZE(X), %xmm5

  405. 	addq	INCX, X

  406. 	movhpd	0 * SIZE(X), %xmm4

  407. 	movhpd	1 * SIZE(X), %xmm5

  408. 	addq	INCX, X

  409. 	movsd	0 * SIZE(X), %xmm6

  410. 	movsd	1 * SIZE(X), %xmm7

  411. 	addq	INCX, X

  412. 	movhpd	0 * SIZE(X), %xmm6

  413. 	movhpd	1 * SIZE(X), %xmm7

  414. 	addq	INCX, X

  415. 

  416. 	andpd	%xmm15, %xmm4

  417. 	andpd	%xmm15, %xmm5

  418. 	andpd	%xmm15, %xmm6

  419. 	andpd	%xmm15, %xmm7

  420. 	addpd	%xmm5, %xmm4

  421. 	addpd	%xmm7, %xmm6

  422. 	maxpd	%xmm4, %xmm0

  423. 	maxpd	%xmm6, %xmm1

  424. 	ALIGN_3

  425. 

  426. .L66:

  427. 	testq	$2, M

  428. 	je	.L67

  429. 

  430. 	movsd	0 * SIZE(X), %xmm4

  431. 	movsd	1 * SIZE(X), %xmm5

  432. 	addq	INCX, X

  433. 	movhpd	0 * SIZE(X), %xmm4

  434. 	movhpd	1 * SIZE(X), %xmm5

  435. 	addq	INCX, X

  436. 	andpd	%xmm15, %xmm4

  437. 	andpd	%xmm15, %xmm5

  438. 	addpd	%xmm5, %xmm4

  439. 	maxpd	%xmm4, %xmm2

  440. 	ALIGN_3

  441. 

  442. .L67:

  443. 	testq	$1, M

  444. 	je	.L70

  445. 

  446. 	movsd	0 * SIZE(X), %xmm4

  447. 	movsd	1 * SIZE(X), %xmm5

  448. 	andpd	%xmm15, %xmm4

  449. 	andpd	%xmm15, %xmm5

  450. 	addpd	%xmm5, %xmm4

  451. 	maxsd	%xmm4, %xmm3

  452. 	ALIGN_3

  453. 

  454. .L70:

  455. 	movq	XX, X

  456. 	movq	MM, M

  457. 

  458. 	maxpd	%xmm1, %xmm0

  459. 	maxpd	%xmm3, %xmm2

  460. 	maxpd	%xmm2, %xmm0

  461. 	movapd	%xmm0, %xmm1

  462. 	unpckhpd %xmm0, %xmm0

  463. 	maxsd	%xmm1, %xmm0

  464. 	unpcklpd %xmm0, %xmm0

  465. 

  466. 	movq	M,  I

  467. 	sarq	$2, I

  468. 	jle	.L75

  469. 	ALIGN_4

  470. 

  471. .L71:

  472. #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  473. 	prefetch	PREFETCHSIZE * SIZE(X)

  474. #endif

  475. 

  476. #ifdef PENTIUM4

  477. 	prefetchnta	PREFETCHSIZE * SIZE(X)

  478. #endif

  479. 

  480. 	movsd	0 * SIZE(X), %xmm1

  481. 	movsd	1 * SIZE(X), %xmm2

  482. 	addq	INCX, X

  483. 	movhpd	0 * SIZE(X), %xmm1

  484. 	movhpd	1 * SIZE(X), %xmm2

  485. 	addq	INCX, X

  486. 	movsd	0 * SIZE(X), %xmm3

  487. 	movsd	1 * SIZE(X), %xmm4

  488. 	addq	INCX, X

  489. 	movhpd	0 * SIZE(X), %xmm3

  490. 	movhpd	1 * SIZE(X), %xmm4

  491. 	addq	INCX, X

  492. 

  493. 	andpd	%xmm15, %xmm1

  494. 	andpd	%xmm15, %xmm2

  495. 	andpd	%xmm15, %xmm3

  496. 	andpd	%xmm15, %xmm4

  497. 

  498. 	addpd	%xmm2,  %xmm1

  499. 	addpd	%xmm4,  %xmm3

  500. 

  501. 	cmpeqpd	%xmm0, %xmm1

  502. 	cmpeqpd	%xmm0, %xmm3

  503. 

  504. 	orpd	%xmm3, %xmm1

  505. #ifndef C_SUN

  506. 	movmskpd %xmm1, %r11

  507. #else

  508. 	.byte	0x66

  509. 	.long	0xd9500f4c

  510. #endif

  511. 	testq	 $3, %r11

  512. 	jne	 .L73

  513. 

  514. 	addq	$4, RET

  515. 	decq	I

  516. 	jg	.L71

  517. 	jmp	.L75

  518. 	ALIGN_4

  519. 

  520. .L73:

  521. 	subq	INCX, X

  522. 	movsd	0 * SIZE(X), %xmm7

  523. 	movsd	1 * SIZE(X), %xmm8

  524. 	subq	INCX, X

  525. 	movsd	0 * SIZE(X), %xmm5

  526. 	movsd	1 * SIZE(X), %xmm6

  527. 	subq	INCX, X

  528. 	movsd	0 * SIZE(X), %xmm3

  529. 	movsd	1 * SIZE(X), %xmm4

  530. 	subq	INCX, X

  531. 	movsd	0 * SIZE(X), %xmm1

  532. 	movsd	1 * SIZE(X), %xmm2

  533. 

  534. 	andpd	%xmm15, %xmm1

  535. 	andpd	%xmm15, %xmm2

  536. 	andpd	%xmm15, %xmm3

  537. 	andpd	%xmm15, %xmm4

  538. 	andpd	%xmm15, %xmm5

  539. 	andpd	%xmm15, %xmm6

  540. 	andpd	%xmm15, %xmm7

  541. 	andpd	%xmm15, %xmm8

  542. 

  543. 	addpd	%xmm2,  %xmm1

  544. 	addpd	%xmm4,  %xmm3

  545. 	addpd	%xmm6,  %xmm5

  546. 	addpd	%xmm8,  %xmm7

  547. 

  548. 	incq	RET

  549. 	comisd	%xmm0, %xmm1

  550. 	je	.L999

  551. 	incq	RET

  552. 	comisd	%xmm0, %xmm3

  553. 	je	.L999

  554. 	incq	RET

  555. 	comisd	%xmm0, %xmm5

  556. 	je	.L999

  557. 	incq	RET

  558. 	comisd	%xmm0, %xmm7

  559. 	je	.L999

  560. 	ALIGN_3

  561. 

  562. .L75:

  563. 	testq	$2, M

  564. 	je	.L76

  565. 

  566. 	movsd	0 * SIZE(X), %xmm1

  567. 	movsd	1 * SIZE(X), %xmm2

  568. 	addq	INCX, X

  569. 	movsd	0 * SIZE(X), %xmm3

  570. 	movsd	1 * SIZE(X), %xmm4

  571. 	addq	INCX, X

  572. 

  573. 	andpd	%xmm15, %xmm1

  574. 	andpd	%xmm15, %xmm2

  575. 	andpd	%xmm15, %xmm3

  576. 	andpd	%xmm15, %xmm4

  577. 

  578. 	addpd	%xmm2,  %xmm1

  579. 	addpd	%xmm4,  %xmm3

  580. 	incq	RET

  581. 	comisd	%xmm0, %xmm1

  582. 	je	.L999

  583. 	incq	RET

  584. 	comisd	%xmm0, %xmm3

  585. 	je	.L999

  586. 	ALIGN_3

  587. 

  588. .L76:

  589. 	incq	RET

  590. 	ALIGN_4

  591. 

  592. .L999:

  593. 	RESTOREREGISTERS

  594. 

  595. 	ret

  596. 

  597. 	EPILOGUE