OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 821cbb2995
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '821cbb2995'
${ noResults }
OpenBLAS/kernel/x86/copy_sse2.S

656 lines
13 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	12

  43. #define ARGS	 0

  44. 	

  45. #define STACK_M		 4 + STACK + ARGS(%esp)

  46. #define STACK_X		 8 + STACK + ARGS(%esp)

  47. #define STACK_INCX	12 + STACK + ARGS(%esp)

  48. #define STACK_Y		16 + STACK + ARGS(%esp)

  49. #define STACK_INCY	20 + STACK + ARGS(%esp)

  50. 

  51. #define M	%ebx

  52. #define X	%esi

  53. #define INCX	%ecx

  54. #define Y	%edi

  55. #define INCY	%edx

  56. 

  57. #include "l1param.h"

  58. 

  59. #ifdef OPTERON

  60. #define LOAD(OFFSET, ADDR, REG)		xorps	REG, REG; addpd	OFFSET(ADDR), REG

  61. #else

  62. #define LOAD(OFFSET, ADDR, REG)		movaps	OFFSET(ADDR), REG

  63. #endif

  64. 

  65. 	PROLOGUE

  66. 	PROFCODE

  67. 

  68. 	pushl	%edi

  69. 	pushl	%esi

  70. 	pushl	%ebx

  71. 

  72. 	movl	STACK_M,     M

  73. 	movl	STACK_X,     X

  74. 	movl	STACK_INCX,  INCX

  75. 	movl	STACK_Y,     Y

  76. 	movl	STACK_INCY,  INCY

  77. 

  78. 	leal	(, INCX, SIZE), INCX

  79. 	leal	(, INCY, SIZE), INCY

  80. 

  81. 	cmpl	$SIZE, INCX

  82. 	jne	.L40

  83. 	cmpl	$SIZE, INCY

  84. 	jne	.L40

  85. 

  86. #ifdef ALIGNED_ACCESS

  87. 	testl	$SIZE, Y

  88. #else

  89. 	testl	$SIZE, X

  90. #endif

  91. 	je	.L10

  92. 

  93. 	movsd	(X), %xmm0

  94. 	movsd	%xmm0, (Y)

  95. 	addl	$1 * SIZE, X

  96. 	addl	$1 * SIZE, Y

  97. 	decl	M

  98. 	jle	.L19

  99. 	ALIGN_4

  100. 

  101. .L10:

  102. 	subl	$-16 * SIZE, X

  103. 	subl	$-16 * SIZE, Y

  104. 

  105. #ifdef ALIGNED_ACCESS

  106. 	testl	$SIZE, X

  107. #else

  108. 	testl	$SIZE, Y

  109. #endif

  110. 	jne	.L20

  111. 

  112. 	movl	M,  %eax

  113. 	sarl	$4, %eax

  114. 	jle	.L13

  115. 

  116. 	movaps	-16 * SIZE(X), %xmm0

  117. 	movaps	-14 * SIZE(X), %xmm1

  118. 	movaps	-12 * SIZE(X), %xmm2

  119. 	movaps	-10 * SIZE(X), %xmm3

  120. 	movaps	 -8 * SIZE(X), %xmm4

  121. 	movaps	 -6 * SIZE(X), %xmm5

  122. 	movaps	 -4 * SIZE(X), %xmm6

  123. 	movaps	 -2 * SIZE(X), %xmm7

  124. 

  125. 	decl	%eax

  126. 	jle .L12

  127. 	ALIGN_3

  128. 

  129. .L11:

  130. #ifdef PREFETCHW

  131. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  132. #endif

  133. 

  134. 	movaps	%xmm0, -16 * SIZE(Y)

  135. 	LOAD( 0 * SIZE, X, %xmm0)

  136. 	movaps	%xmm1, -14 * SIZE(Y)

  137. 	LOAD( 2 * SIZE, X, %xmm1)

  138. 

  139. #ifdef PREFETCH

  140. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  141. #endif

  142. 

  143. 	movaps	%xmm2, -12 * SIZE(Y)

  144. 	LOAD( 4 * SIZE, X, %xmm2)

  145. 	movaps	%xmm3, -10 * SIZE(Y)

  146. 	LOAD( 6 * SIZE, X, %xmm3)

  147. 

  148. #if defined(PREFETCHW) && !defined(FETCH128)

  149. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  150. #endif

  151. 

  152. 	movaps	%xmm4, -8 * SIZE(Y)

  153. 	LOAD( 8 * SIZE, X, %xmm4)

  154. 	movaps	%xmm5, -6 * SIZE(Y)

  155. 	LOAD(10 * SIZE, X, %xmm5)

  156. 

  157. #if defined(PREFETCH) && !defined(FETCH128)

  158. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  159. #endif

  160. 

  161. 	movaps	%xmm6, -4 * SIZE(Y)

  162. 	LOAD(12 * SIZE, X, %xmm6)

  163. 	movaps	%xmm7, -2 * SIZE(Y)

  164. 	LOAD(14 * SIZE, X, %xmm7)

  165. 

  166. 	subl	$-16 * SIZE, Y

  167. 	subl	$-16 * SIZE, X

  168. 	decl	%eax

  169. 	jg	.L11

  170. 	ALIGN_3

  171. 

  172. .L12:

  173. 	movaps	%xmm0, -16 * SIZE(Y)

  174. 	movaps	%xmm1, -14 * SIZE(Y)

  175. 	movaps	%xmm2, -12 * SIZE(Y)

  176. 	movaps	%xmm3, -10 * SIZE(Y)

  177. 	movaps	%xmm4,  -8 * SIZE(Y)

  178. 	movaps	%xmm5,  -6 * SIZE(Y)

  179. 	movaps	%xmm6,  -4 * SIZE(Y)

  180. 	movaps	%xmm7,  -2 * SIZE(Y)

  181. 

  182. 	subl	$-16 * SIZE, Y

  183. 	subl	$-16 * SIZE, X

  184. 	ALIGN_3

  185. 

  186. .L13:

  187. 	testl	$8, M

  188. 	jle	.L14

  189. 	ALIGN_3

  190. 

  191. 	movaps	-16 * SIZE(X), %xmm0

  192. 	movaps	-14 * SIZE(X), %xmm1

  193. 	movaps	-12 * SIZE(X), %xmm2

  194. 	movaps	-10 * SIZE(X), %xmm3

  195. 

  196. 	movaps	%xmm0, -16 * SIZE(Y)

  197. 	movaps	%xmm1, -14 * SIZE(Y)

  198. 	movaps	%xmm2, -12 * SIZE(Y)

  199. 	movaps	%xmm3, -10 * SIZE(Y)

  200. 

  201. 	addl	$8 * SIZE, X

  202. 	addl	$8 * SIZE, Y

  203. 	ALIGN_3

  204. 

  205. .L14:

  206. 	testl	$4, M

  207. 	jle	.L15

  208. 	ALIGN_3

  209. 

  210. 	movaps	-16 * SIZE(X), %xmm0

  211. 	movaps	-14 * SIZE(X), %xmm1

  212. 

  213. 	movaps	%xmm0, -16 * SIZE(Y)

  214. 	movaps	%xmm1, -14 * SIZE(Y)

  215. 

  216. 	addl	$4 * SIZE, X

  217. 	addl	$4 * SIZE, Y

  218. 	ALIGN_3

  219. 

  220. .L15:

  221. 	testl	$2, M

  222. 	jle	.L16

  223. 	ALIGN_3

  224. 

  225. 	movaps	-16 * SIZE(X), %xmm0

  226. 	movaps	%xmm0, -16 * SIZE(Y)

  227. 

  228. 	addl	$2 * SIZE, X

  229. 	addl	$2 * SIZE, Y

  230. 	ALIGN_3

  231. 

  232. .L16:

  233. 	testl	$1, M

  234. 	jle	.L19

  235. 	ALIGN_3

  236. 

  237. 	movsd	-16 * SIZE(X), %xmm0

  238. 	movsd	%xmm0, 	-16 * SIZE(Y)

  239. 	ALIGN_3

  240. 

  241. .L19:

  242. 	popl	%ebx

  243. 	popl	%esi

  244. 	popl	%edi

  245. 	ret

  246. 	ALIGN_3

  247. 

  248. .L20:

  249. #ifdef ALIGNED_ACCESS

  250. 

  251. 	movhps	-16 * SIZE(X), %xmm0

  252. 

  253. 	movl	M,  %eax

  254. 	sarl	$4, %eax

  255. 	jle	.L23

  256. 

  257. 	movaps	-15 * SIZE(X), %xmm1

  258. 	movaps	-13 * SIZE(X), %xmm2

  259. 	movaps	-11 * SIZE(X), %xmm3

  260. 	movaps	 -9 * SIZE(X), %xmm4

  261. 	movaps	 -7 * SIZE(X), %xmm5

  262. 	movaps	 -5 * SIZE(X), %xmm6

  263. 	movaps	 -3 * SIZE(X), %xmm7

  264. 

  265. 	decl	%eax

  266. 	jle .L22

  267. 	ALIGN_4

  268. 

  269. .L21:

  270. #ifdef PREFETCHW

  271. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  272. #endif

  273. 

  274. 	SHUFPD_1 %xmm1, %xmm0

  275. 	movaps	%xmm0, -16 * SIZE(Y)

  276. 	LOAD(-1 * SIZE, X, %xmm0)

  277. 

  278. 	SHUFPD_1 %xmm2, %xmm1

  279. 	movaps	%xmm1, -14 * SIZE(Y)

  280. 	LOAD( 1 * SIZE, X, %xmm1)

  281. 

  282. #ifdef PREFETCH

  283. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  284. #endif

  285. 

  286. 	SHUFPD_1 %xmm3, %xmm2

  287. 	movaps	%xmm2, -12 * SIZE(Y)

  288. 	LOAD( 3 * SIZE, X, %xmm2)

  289. 

  290. 	SHUFPD_1 %xmm4, %xmm3

  291. 	movaps	%xmm3, -10 * SIZE(Y)

  292. 	LOAD( 5 * SIZE, X, %xmm3)

  293. 

  294. #if defined(PREFETCHW) && !defined(FETCH128)

  295. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  296. #endif

  297. 

  298. 	SHUFPD_1 %xmm5, %xmm4

  299. 	movaps	%xmm4,  -8 * SIZE(Y)

  300. 	LOAD( 7 * SIZE, X, %xmm4)

  301. 

  302. 	SHUFPD_1 %xmm6, %xmm5

  303. 	movaps	%xmm5, -6 * SIZE(Y)

  304. 	LOAD( 9 * SIZE, X, %xmm5)

  305. 

  306. #if defined(PREFETCH) && !defined(FETCH128)

  307. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  308. #endif

  309. 

  310. 	SHUFPD_1 %xmm7, %xmm6

  311. 	movaps	%xmm6, -4 * SIZE(Y)

  312. 	LOAD(11 * SIZE, X, %xmm6)

  313. 

  314. 	SHUFPD_1 %xmm0, %xmm7

  315. 	movaps	%xmm7, -2 * SIZE(Y)

  316. 	LOAD(13 * SIZE, X, %xmm7)

  317. 

  318. 	subl	$-16 * SIZE, X

  319. 	subl	$-16 * SIZE, Y

  320. 	decl	%eax

  321. 	jg	.L21

  322. 	ALIGN_3

  323. 

  324. .L22:

  325. 	SHUFPD_1 %xmm1, %xmm0

  326. 	movaps	%xmm0, -16 * SIZE(Y)

  327. 	LOAD(-1 * SIZE, X, %xmm0)

  328. 

  329. 	SHUFPD_1 %xmm2, %xmm1

  330. 	movaps	%xmm1, -14 * SIZE(Y)

  331. 

  332. 	SHUFPD_1 %xmm3, %xmm2

  333. 	movaps	%xmm2, -12 * SIZE(Y)

  334. 	SHUFPD_1 %xmm4, %xmm3

  335. 	movaps	%xmm3, -10 * SIZE(Y)

  336. 

  337. 	SHUFPD_1 %xmm5, %xmm4

  338. 	movaps	%xmm4,  -8 * SIZE(Y)

  339. 	SHUFPD_1 %xmm6, %xmm5

  340. 	movaps	%xmm5,  -6 * SIZE(Y)

  341. 

  342. 	SHUFPD_1 %xmm7, %xmm6

  343. 	movaps	%xmm6,  -4 * SIZE(Y)

  344. 	SHUFPD_1 %xmm0, %xmm7

  345. 	movaps	%xmm7,  -2 * SIZE(Y)

  346. 

  347. 	subl	$-16 * SIZE, X

  348. 	subl	$-16 * SIZE, Y

  349. 	ALIGN_3

  350. 

  351. .L23:

  352. 	testl	$8, M

  353. 	jle	.L24

  354. 	ALIGN_3

  355. 

  356. 	movaps	-15 * SIZE(X), %xmm1

  357. 	movaps	-13 * SIZE(X), %xmm2

  358. 	movaps	-11 * SIZE(X), %xmm3

  359. 	movaps	 -9 * SIZE(X), %xmm4

  360. 

  361. 	SHUFPD_1 %xmm1, %xmm0

  362. 	movaps	%xmm0, -16 * SIZE(Y)

  363. 

  364. 	SHUFPD_1 %xmm2, %xmm1

  365. 	movaps	%xmm1, -14 * SIZE(Y)

  366. 

  367. 	SHUFPD_1 %xmm3, %xmm2

  368. 	movaps	%xmm2, -12 * SIZE(Y)

  369. 

  370. 	SHUFPD_1 %xmm4, %xmm3

  371. 	movaps	%xmm3, -10 * SIZE(Y)

  372. 

  373. 	movaps	%xmm4, %xmm0

  374. 

  375. 	addl	$8 * SIZE, X

  376. 	addl	$8 * SIZE, Y

  377. 	ALIGN_3

  378. 

  379. .L24:

  380. 	testl	$4, M

  381. 	jle	.L25

  382. 	ALIGN_3

  383. 

  384. 	movaps	-15 * SIZE(X), %xmm1

  385. 	movaps	-13 * SIZE(X), %xmm2

  386. 

  387. 	SHUFPD_1 %xmm1, %xmm0

  388. 	SHUFPD_1 %xmm2, %xmm1

  389. 

  390. 	movaps	%xmm0, -16 * SIZE(Y)

  391. 	movaps	%xmm1, -14 * SIZE(Y)

  392. 	movaps	%xmm2, %xmm0

  393. 

  394. 	addl	$4 * SIZE, X

  395. 	addl	$4 * SIZE, Y

  396. 	ALIGN_3

  397. 

  398. .L25:

  399. 	testl	$2, M

  400. 	jle	.L26

  401. 	ALIGN_3

  402. 

  403. 	movaps	-15 * SIZE(X), %xmm1

  404. 	SHUFPD_1 %xmm1, %xmm0

  405. 

  406. 	movaps	%xmm0, -16 * SIZE(Y)

  407. 

  408. 	addl	$2 * SIZE, X

  409. 	addl	$2 * SIZE, Y

  410. 	ALIGN_3

  411. 

  412. .L26:

  413. 	testl	$1, M

  414. 	jle	.L29

  415. 	ALIGN_3

  416. 

  417. 	movsd	-16 * SIZE(X), %xmm0

  418. 	movsd	%xmm0, 	-16 * SIZE(Y)

  419. 	ALIGN_3

  420. 

  421. .L29:

  422. 	popl	%ebx

  423. 	popl	%esi

  424. 	popl	%edi

  425. 	ret

  426. 	ALIGN_3

  427. 

  428. #else

  429. 

  430. 	movl	M,  %eax

  431. 	sarl	$4, %eax

  432. 	jle	.L23

  433. 

  434. 	movaps	-16 * SIZE(X), %xmm0

  435. 	movaps	-14 * SIZE(X), %xmm1

  436. 	movaps	-12 * SIZE(X), %xmm2

  437. 	movaps	-10 * SIZE(X), %xmm3

  438. 	movaps	 -8 * SIZE(X), %xmm4

  439. 	movaps	 -6 * SIZE(X), %xmm5

  440. 	movaps	 -4 * SIZE(X), %xmm6

  441. 	movaps	 -2 * SIZE(X), %xmm7

  442. 

  443. 	decl	%eax

  444. 	jle .L22

  445. 	ALIGN_3

  446. 

  447. .L21:

  448. #ifdef PREFETCHW

  449. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  450. #endif

  451. 

  452. 	movlps	%xmm0, -16 * SIZE(Y)

  453. 	movhps	%xmm0, -15 * SIZE(Y)

  454. 	LOAD( 0 * SIZE, X, %xmm0)

  455. 	movlps	%xmm1, -14 * SIZE(Y)

  456. 	movhps	%xmm1, -13 * SIZE(Y)

  457. 	LOAD( 2 * SIZE, X, %xmm1)

  458. 

  459. #ifdef PREFETCH

  460. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  461. #endif

  462. 

  463. 	movlps	%xmm2, -12 * SIZE(Y)

  464. 	movhps	%xmm2, -11 * SIZE(Y)

  465. 	LOAD( 4 * SIZE, X, %xmm2)

  466. 	movlps	%xmm3, -10 * SIZE(Y)

  467. 	movhps	%xmm3,  -9 * SIZE(Y)

  468. 	LOAD( 6 * SIZE, X, %xmm3)

  469. 

  470. #if defined(PREFETCHW) && !defined(FETCH128)

  471. 	PREFETCHW (PREFETCHSIZE +  64) - PREOFFSET(Y)

  472. #endif

  473. 

  474. 	movlps	%xmm4,  -8 * SIZE(Y)

  475. 	movhps	%xmm4,  -7 * SIZE(Y)

  476. 	LOAD( 8 * SIZE, X, %xmm4)

  477. 	movlps	%xmm5,  -6 * SIZE(Y)

  478. 	movhps	%xmm5,  -5 * SIZE(Y)

  479. 	LOAD(10 * SIZE, X, %xmm5)

  480. 

  481. #if defined(PREFETCH) && !defined(FETCH128)

  482. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  483. #endif

  484. 

  485. 	movlps	%xmm6,  -4 * SIZE(Y)

  486. 	movhps	%xmm6,  -3 * SIZE(Y)

  487. 	LOAD(12 * SIZE, X, %xmm6)

  488. 	movlps	%xmm7,  -2 * SIZE(Y)

  489. 	movhps	%xmm7,  -1 * SIZE(Y)

  490. 	LOAD(14 * SIZE, X, %xmm7)

  491. 

  492. 	subl	$-16 * SIZE, Y

  493. 	subl	$-16 * SIZE, X

  494. 	decl	%eax

  495. 	jg	.L21

  496. 	ALIGN_3

  497. 

  498. .L22:

  499. 	movlps	%xmm0, -16 * SIZE(Y)

  500. 	movhps	%xmm0, -15 * SIZE(Y)

  501. 	movlps	%xmm1, -14 * SIZE(Y)

  502. 	movhps	%xmm1, -13 * SIZE(Y)

  503. 	movlps	%xmm2, -12 * SIZE(Y)

  504. 	movhps	%xmm2, -11 * SIZE(Y)

  505. 	movlps	%xmm3, -10 * SIZE(Y)

  506. 	movhps	%xmm3,  -9 * SIZE(Y)

  507. 	movlps	%xmm4,  -8 * SIZE(Y)

  508. 	movhps	%xmm4,  -7 * SIZE(Y)

  509. 	movlps	%xmm5,  -6 * SIZE(Y)

  510. 	movhps	%xmm5,  -5 * SIZE(Y)

  511. 	movlps	%xmm6,  -4 * SIZE(Y)

  512. 	movhps	%xmm6,  -3 * SIZE(Y)

  513. 	movlps	%xmm7,  -2 * SIZE(Y)

  514. 	movhps	%xmm7,  -1 * SIZE(Y)

  515. 

  516. 	subl	$-16 * SIZE, Y

  517. 	subl	$-16 * SIZE, X

  518. 	ALIGN_3

  519. 

  520. .L23:

  521. 	testl	$8, M

  522. 	jle	.L24

  523. 	ALIGN_3

  524. 

  525. 	movaps	-16 * SIZE(X), %xmm0

  526. 	movlps	%xmm0, -16 * SIZE(Y)

  527. 	movhps	%xmm0, -15 * SIZE(Y)

  528. 	movaps	-14 * SIZE(X), %xmm1

  529. 	movlps	%xmm1, -14 * SIZE(Y)

  530. 	movhps	%xmm1, -13 * SIZE(Y)

  531. 	movaps	-12 * SIZE(X), %xmm2

  532. 	movlps	%xmm2, -12 * SIZE(Y)

  533. 	movhps	%xmm2, -11 * SIZE(Y)

  534. 	movaps	-10 * SIZE(X), %xmm3

  535. 	movlps	%xmm3, -10 * SIZE(Y)

  536. 	movhps	%xmm3,  -9 * SIZE(Y)

  537. 

  538. 	addl	$8 * SIZE, X

  539. 	addl	$8 * SIZE, Y

  540. 	ALIGN_3

  541. 

  542. .L24:

  543. 	testl	$4, M

  544. 	jle	.L25

  545. 	ALIGN_3

  546. 

  547. 	movaps	-16 * SIZE(X), %xmm0

  548. 	movlps	%xmm0, -16 * SIZE(Y)

  549. 	movhps	%xmm0, -15 * SIZE(Y)

  550. 	movaps	-14 * SIZE(X), %xmm1

  551. 	movlps	%xmm1, -14 * SIZE(Y)

  552. 	movhps	%xmm1, -13 * SIZE(Y)

  553. 

  554. 	addl	$4 * SIZE, X

  555. 	addl	$4 * SIZE, Y

  556. 	ALIGN_3

  557. 

  558. .L25:

  559. 	testl	$2, M

  560. 	jle	.L26

  561. 	ALIGN_3

  562. 

  563. 	movaps	-16 * SIZE(X), %xmm0

  564. 	movlps	%xmm0, -16 * SIZE(Y)

  565. 	movhps	%xmm0, -15 * SIZE(Y)

  566. 

  567. 	addl	$2 * SIZE, X

  568. 	addl	$2 * SIZE, Y

  569. 	ALIGN_3

  570. 

  571. .L26:

  572. 	testl	$1, M

  573. 	jle	.L29

  574. 	ALIGN_3

  575. 

  576. 	movsd	-16 * SIZE(X), %xmm0

  577. 	movsd	%xmm0, 	-16 * SIZE(Y)

  578. 	ALIGN_3

  579. 

  580. .L29:

  581. 	popl	%ebx

  582. 	popl	%esi

  583. 	popl	%edi

  584. 	ret

  585. 	ALIGN_3

  586. 

  587. #endif

  588. 

  589. .L40:

  590. 	movl	M,  %eax

  591. 	sarl	$3, %eax

  592. 	jle	.L45

  593. 	ALIGN_3

  594. 

  595. .L41:

  596. 	movsd	(X), %xmm0

  597. 	addl	INCX, X

  598. 	movhps	(X), %xmm0

  599. 	addl	INCX, X

  600. 	movsd	(X), %xmm1

  601. 	addl	INCX, X

  602. 	movhps	(X), %xmm1

  603. 	addl	INCX, X

  604. 	movsd	(X), %xmm2

  605. 	addl	INCX, X

  606. 	movhps	(X), %xmm2

  607. 	addl	INCX, X

  608. 	movsd	(X), %xmm3

  609. 	addl	INCX, X

  610. 	movhps	(X), %xmm3

  611. 	addl	INCX, X

  612. 

  613. 	movlps	%xmm0, (Y)

  614. 	addl	INCY, Y

  615. 	movhps	%xmm0, (Y)

  616. 	addl	INCY, Y

  617. 	movlps	%xmm1, (Y)

  618. 	addl	INCY, Y

  619. 	movhps	%xmm1, (Y)

  620. 	addl	INCY, Y

  621. 	movlps	%xmm2, (Y)

  622. 	addl	INCY, Y

  623. 	movhps	%xmm2, (Y)

  624. 	addl	INCY, Y

  625. 	movlps	%xmm3, (Y)

  626. 	addl	INCY, Y

  627. 	movhps	%xmm3, (Y)

  628. 	addl	INCY, Y

  629. 

  630. 	decl	%eax

  631. 	jg	.L41

  632. 	ALIGN_3

  633. 

  634. .L45:

  635. 	movl	M,  %eax

  636. 	andl	$7, %eax

  637. 	jle	.L47

  638. 	ALIGN_3

  639. 

  640. .L46:

  641. 	movsd	(X), %xmm0

  642. 	addl	INCX, X

  643. 	movlps	%xmm0, (Y)

  644. 	addl	INCY, Y

  645. 	decl	%eax

  646. 	jg	.L46

  647. 	ALIGN_3

  648. 

  649. .L47:

  650. 	popl	%ebx

  651. 	popl	%esi

  652. 	popl	%edi

  653. 	ret

  654. 

  655. 	EPILOGUE