OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 57ed58cefe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '57ed58cefe'
${ noResults }
OpenBLAS/kernel/x86/zswap_sse2.S

979 lines
18 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS     0

  44. 

  45. #define STACK_M		 4 + STACK + ARGS(%esp)

  46. #define STACK_X		32 + STACK + ARGS(%esp)

  47. #define STACK_INCX	36 + STACK + ARGS(%esp)

  48. #define STACK_Y		40 + STACK + ARGS(%esp)

  49. #define STACK_INCY	44 + STACK + ARGS(%esp)

  50. 

  51. #define M	%edx

  52. #define X	%esi

  53. #define Y	%edi

  54. #define INCX	%ebx

  55. #define INCY	%ecx

  56. 

  57. #include "l1param.h"

  58. 

  59. 	PROLOGUE

  60. 	PROFCODE

  61. 

  62. 	pushl	%ebp

  63. 	pushl	%edi

  64. 	pushl	%esi

  65. 	pushl	%ebx

  66. 

  67. 	movl	STACK_M,    M

  68. 	movl	STACK_X,    X

  69. 	movl	STACK_Y,    Y

  70. 	movl	STACK_INCX, INCX

  71. 	movl	STACK_INCY, INCY

  72. 

  73. 	sall	$ZBASE_SHIFT, INCX

  74. 	sall	$ZBASE_SHIFT, INCY

  75. 

  76. 	testl	M, M

  77. 	jle	.L19

  78. 

  79. 	cmpl	$2 * SIZE, INCX

  80. 	jne	.L50

  81. 	cmpl	$2 * SIZE, INCY

  82. 	jne	.L50

  83. 

  84. 	subl	$-16 * SIZE, X

  85. 	subl	$-16 * SIZE, Y

  86. 

  87. 	testl	$SIZE, Y

  88. 	jne	.L30

  89. 

  90. 	testl	$SIZE, X

  91. 	jne	.L20

  92. 

  93. 	movl	M,  %eax

  94. 	sarl	$3, %eax

  95. 	jle	.L13

  96. 	ALIGN_3

  97. 

  98. .L11:

  99. #ifdef PREFETCHW

  100. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  101. #endif

  102. 

  103. 	movaps	-16 * SIZE(X), %xmm0

  104. 	movaps	-16 * SIZE(Y), %xmm1

  105. 

  106. 	movaps	%xmm0, 	-16 * SIZE(Y)

  107. 	movaps	%xmm1, 	-16 * SIZE(X)

  108. 

  109. 	movaps	-14 * SIZE(X), %xmm0

  110. 	movaps	-14 * SIZE(Y), %xmm1

  111. 

  112. 	movaps	%xmm0, 	-14 * SIZE(Y)

  113. 	movaps	%xmm1, 	-14 * SIZE(X)

  114. 

  115. #ifdef PREFETCHW

  116. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  117. #endif

  118. 

  119. 	movaps	-12 * SIZE(X), %xmm0

  120. 	movaps	-12 * SIZE(Y), %xmm1

  121. 

  122. 	movaps	%xmm0, 	-12 * SIZE(Y)

  123. 	movaps	%xmm1, 	-12 * SIZE(X)

  124. 

  125. 	movaps	-10 * SIZE(X), %xmm0

  126. 	movaps	-10 * SIZE(Y), %xmm1

  127. 

  128. 	movaps	%xmm0, 	-10 * SIZE(Y)

  129. 	movaps	%xmm1, 	-10 * SIZE(X)

  130. 

  131. #ifdef PREFETCHW

  132. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  133. #endif

  134. 

  135. 	movaps	 -8 * SIZE(X), %xmm0

  136. 	movaps	 -8 * SIZE(Y), %xmm1

  137. 

  138. 	movaps	%xmm0, 	 -8 * SIZE(Y)

  139. 	movaps	%xmm1, 	 -8 * SIZE(X)

  140. 

  141. 	movaps	 -6 * SIZE(X), %xmm0

  142. 	movaps	 -6 * SIZE(Y), %xmm1

  143. 

  144. 	movaps	%xmm0, 	 -6 * SIZE(Y)

  145. 	movaps	%xmm1, 	 -6 * SIZE(X)

  146. 

  147. #ifdef PREFETCHW

  148. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  149. #endif

  150. 

  151. 	movaps	 -4 * SIZE(X), %xmm0

  152. 	movaps	 -4 * SIZE(Y), %xmm1

  153. 

  154. 	movaps	%xmm0, 	 -4 * SIZE(Y)

  155. 	movaps	%xmm1, 	 -4 * SIZE(X)

  156. 

  157. 	movaps	 -2 * SIZE(X), %xmm0

  158. 	movaps	 -2 * SIZE(Y), %xmm1

  159. 

  160. 	movaps	%xmm0, 	 -2 * SIZE(Y)

  161. 	movaps	%xmm1, 	 -2 * SIZE(X)

  162. 

  163. 	subl	$-16 * SIZE, Y

  164. 	subl	$-16 * SIZE, X

  165. 

  166. 	decl	%eax

  167. 	jg	.L11

  168. 	ALIGN_3

  169. 

  170. .L13:

  171. 	testl	$4, M

  172. 	jle	.L14

  173. 

  174. 	movaps	-16 * SIZE(X), %xmm0

  175. 	movaps	-16 * SIZE(Y), %xmm1

  176. 

  177. 	movaps	%xmm0, 	-16 * SIZE(Y)

  178. 	movaps	%xmm1, 	-16 * SIZE(X)

  179. 

  180. 	movaps	-14 * SIZE(X), %xmm0

  181. 	movaps	-14 * SIZE(Y), %xmm1

  182. 

  183. 	movaps	%xmm0, 	-14 * SIZE(Y)

  184. 	movaps	%xmm1, 	-14 * SIZE(X)

  185. 

  186. 	movaps	-12 * SIZE(X), %xmm0

  187. 	movaps	-12 * SIZE(Y), %xmm1

  188. 

  189. 	movaps	%xmm0, 	-12 * SIZE(Y)

  190. 	movaps	%xmm1, 	-12 * SIZE(X)

  191. 

  192. 	movaps	-10 * SIZE(X), %xmm0

  193. 	movaps	-10 * SIZE(Y), %xmm1

  194. 

  195. 	movaps	%xmm0, 	-10 * SIZE(Y)

  196.  	movaps	%xmm1, 	-10 * SIZE(X)

  197. 

  198. 	addl	$8 * SIZE, X

  199. 	addl	$8 * SIZE, Y

  200. 	ALIGN_3

  201. 

  202. .L14:

  203. 	testl	$2, M

  204. 	jle	.L15

  205. 

  206. 	movaps	-16 * SIZE(X), %xmm0

  207. 	movaps	-16 * SIZE(Y), %xmm1

  208. 

  209. 	movaps	%xmm0, 	-16 * SIZE(Y)

  210. 	movaps	%xmm1, 	-16 * SIZE(X)

  211. 

  212. 	movaps	-14 * SIZE(X), %xmm0

  213. 	movaps	-14 * SIZE(Y), %xmm1

  214. 

  215. 	movaps	%xmm0, 	-14 * SIZE(Y)

  216. 	movaps	%xmm1, 	-14 * SIZE(X)

  217. 

  218. 	addl	$4 * SIZE, X

  219. 	addl	$4 * SIZE, Y

  220. 	ALIGN_3

  221. 

  222. .L15:

  223. 	testl	$1, M

  224. 	jle	.L19

  225. 

  226. 	movaps	-16 * SIZE(X), %xmm0

  227. 	movaps	-16 * SIZE(Y), %xmm1

  228. 

  229. 	movaps	%xmm0, 	-16 * SIZE(Y)

  230. 	movaps	%xmm1, 	-16 * SIZE(X)

  231. 

  232. 	addl	$2 * SIZE, X

  233. 	addl	$2 * SIZE, Y

  234. 	ALIGN_3

  235. 

  236. .L19:

  237. 	xorl	%eax,%eax

  238. 

  239. 	popl	%ebx

  240. 	popl	%esi

  241. 	popl	%edi

  242. 	popl	%ebp

  243. 	ret

  244. 	ALIGN_3

  245. 

  246. .L20:

  247. 	movhps	 -16 * SIZE(X), %xmm0

  248. 	movaps	 -16 * SIZE(Y), %xmm1

  249. 

  250. 	movlps	%xmm1, -16 * SIZE(X)

  251. 	decl	M

  252. 	jle	.L29

  253. 

  254. 	movl	M,  %eax

  255. 	sarl	$3, %eax

  256. 	jle	.L23

  257. 	ALIGN_4

  258. 

  259. .L21:

  260. #ifdef PREFETCHW

  261. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  262. #endif

  263. 

  264. 	movaps	-15 * SIZE(X), %xmm2

  265. 	movaps	-14 * SIZE(Y), %xmm3

  266. 

  267. 	SHUFPD_1  %xmm2, %xmm0

  268. 	movaps	%xmm0, -16 * SIZE(Y)

  269. 	SHUFPD_1  %xmm3, %xmm1

  270. 	movaps	%xmm1, -15 * SIZE(X)

  271. 

  272. 	movaps	-13 * SIZE(X), %xmm0

  273. 	movaps	-12 * SIZE(Y), %xmm1

  274. 

  275. 	SHUFPD_1  %xmm0, %xmm2

  276. 	movaps	%xmm2, -14 * SIZE(Y)

  277. 	SHUFPD_1  %xmm1, %xmm3

  278. 	movaps	%xmm3, -13 * SIZE(X)

  279. 

  280. #ifdef PREFETCHW

  281. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  282. #endif

  283. 

  284. 	movaps	-11 * SIZE(X), %xmm2

  285. 	movaps	-10 * SIZE(Y), %xmm3

  286. 

  287. 	SHUFPD_1  %xmm2, %xmm0

  288. 	movaps	%xmm0, -12 * SIZE(Y)

  289. 	SHUFPD_1  %xmm3, %xmm1

  290. 	movaps	%xmm1, -11 * SIZE(X)

  291. 

  292. 	movaps	 -9 * SIZE(X), %xmm0

  293. 	movaps	 -8 * SIZE(Y), %xmm1

  294. 

  295. 	SHUFPD_1  %xmm0, %xmm2

  296. 	movaps	%xmm2, -10 * SIZE(Y)

  297. 	SHUFPD_1  %xmm1, %xmm3

  298. 	movaps	%xmm3,  -9 * SIZE(X)

  299. 

  300. #ifdef PREFETCHW

  301. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  302. #endif

  303. 

  304. 	movaps	 -7 * SIZE(X), %xmm2

  305. 	movaps	 -6 * SIZE(Y), %xmm3

  306. 

  307. 	SHUFPD_1  %xmm2, %xmm0

  308. 	movaps	%xmm0,  -8 * SIZE(Y)

  309. 	SHUFPD_1  %xmm3, %xmm1

  310. 	movaps	%xmm1,  -7 * SIZE(X)

  311. 

  312. 	movaps	 -5 * SIZE(X), %xmm0

  313. 	movaps	 -4 * SIZE(Y), %xmm1

  314. 

  315. 	SHUFPD_1  %xmm0, %xmm2

  316. 	movaps	%xmm2,  -6 * SIZE(Y)

  317. 	SHUFPD_1  %xmm1, %xmm3

  318. 	movaps	%xmm3,  -5 * SIZE(X)

  319. 

  320. #ifdef PREFETCHW

  321. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  322. #endif

  323. 

  324. 	movaps	 -3 * SIZE(X), %xmm2

  325. 	movaps	 -2 * SIZE(Y), %xmm3

  326. 

  327. 	SHUFPD_1  %xmm2, %xmm0

  328. 	movaps	%xmm0,  -4 * SIZE(Y)

  329. 	SHUFPD_1  %xmm3, %xmm1

  330. 	movaps	%xmm1,  -3 * SIZE(X)

  331. 

  332. 	movaps	 -1 * SIZE(X), %xmm0

  333. 	movaps	  0 * SIZE(Y), %xmm1

  334. 

  335. 	SHUFPD_1  %xmm0, %xmm2

  336. 	movaps	%xmm2,  -2 * SIZE(Y)

  337. 	SHUFPD_1  %xmm1, %xmm3

  338. 	movaps	%xmm3,  -1 * SIZE(X)

  339. 

  340. 	subl	$-16 * SIZE, X

  341. 	subl	$-16 * SIZE, Y

  342. 

  343. 	decl	%eax

  344. 	jg	.L21

  345. 	ALIGN_3

  346. 

  347. .L23:

  348. 	testl	$4, M

  349. 	jle	.L24

  350. 

  351. 	movaps	-15 * SIZE(X), %xmm2

  352. 	movaps	-14 * SIZE(Y), %xmm3

  353. 

  354. 	SHUFPD_1  %xmm2, %xmm0

  355. 	movaps	%xmm0, -16 * SIZE(Y)

  356. 	SHUFPD_1  %xmm3, %xmm1

  357. 	movaps	%xmm1, -15 * SIZE(X)

  358. 

  359. 	movaps	-13 * SIZE(X), %xmm0

  360. 	movaps	-12 * SIZE(Y), %xmm1

  361. 

  362. 	SHUFPD_1  %xmm0, %xmm2

  363. 	movaps	%xmm2, -14 * SIZE(Y)

  364. 	SHUFPD_1  %xmm1, %xmm3

  365. 	movaps	%xmm3, -13 * SIZE(X)

  366. 

  367. 	movaps	-11 * SIZE(X), %xmm2

  368. 	movaps	-10 * SIZE(Y), %xmm3

  369. 

  370. 	SHUFPD_1  %xmm2, %xmm0

  371. 	movaps	%xmm0, -12 * SIZE(Y)

  372. 	SHUFPD_1  %xmm3, %xmm1

  373. 	movaps	%xmm1, -11 * SIZE(X)

  374. 

  375. 	movaps	 -9 * SIZE(X), %xmm0

  376. 	movaps	 -8 * SIZE(Y), %xmm1

  377. 

  378. 	SHUFPD_1  %xmm0, %xmm2

  379. 	movaps	%xmm2, -10 * SIZE(Y)

  380. 	SHUFPD_1  %xmm1, %xmm3

  381. 	movaps	%xmm3,  -9 * SIZE(X)

  382. 

  383. 	addl	$8 * SIZE, X

  384. 	addl	$8 * SIZE, Y

  385. 	ALIGN_3

  386. 

  387. .L24:

  388. 	testl	$2, M

  389. 	jle	.L25

  390. 

  391. 	movaps	-15 * SIZE(X), %xmm2

  392. 	movaps	-14 * SIZE(Y), %xmm3

  393. 

  394. 	SHUFPD_1  %xmm2, %xmm0

  395. 	movaps	%xmm0, -16 * SIZE(Y)

  396. 	SHUFPD_1  %xmm3, %xmm1

  397. 	movaps	%xmm1, -15 * SIZE(X)

  398. 

  399. 	movaps	-13 * SIZE(X), %xmm0

  400. 	movaps	-12 * SIZE(Y), %xmm1

  401. 

  402. 	SHUFPD_1  %xmm0, %xmm2

  403. 	movaps	%xmm2, -14 * SIZE(Y)

  404. 	SHUFPD_1  %xmm1, %xmm3

  405. 	movaps	%xmm3, -13 * SIZE(X)

  406. 

  407. 	addl	$4 * SIZE, X

  408. 	addl	$4 * SIZE, Y

  409. 	ALIGN_3

  410. 

  411. .L25:

  412. 	testl	$1, M

  413. 	jle	.L29

  414. 

  415. 	movaps	-15 * SIZE(X), %xmm2

  416. 	movaps	-14 * SIZE(Y), %xmm3

  417. 

  418. 	SHUFPD_1  %xmm3, %xmm1

  419. 	movaps	%xmm1, -15 * SIZE(X)

  420. 	SHUFPD_1  %xmm2, %xmm0

  421. 	movaps	%xmm0, -16 * SIZE(Y)

  422. 

  423. 	movaps	%xmm2, %xmm0

  424. 	movaps	%xmm3, %xmm1

  425. 

  426. 	addl	$2 * SIZE, X

  427. 	addl	$2 * SIZE, Y

  428. 	ALIGN_3

  429. 

  430. .L29:

  431. 	movaps	-15 * SIZE(X), %xmm2

  432. 

  433. 	movhps	%xmm1, -15 * SIZE(X)

  434. 	SHUFPD_1  %xmm2, %xmm0

  435. 	movaps	%xmm0, -16 * SIZE(Y)

  436. 

  437. 	popl	%ebx

  438. 	popl	%esi

  439. 	popl	%edi

  440. 	popl	%ebp

  441. 	ret

  442. 	ALIGN_3

  443. 

  444. .L30:

  445. 	testl	$SIZE, X

  446. 	jne	.L40

  447. 

  448. 	movhps	 -16 * SIZE(Y), %xmm0

  449. 	movaps	 -16 * SIZE(X), %xmm1

  450. 

  451. 	movlps	%xmm1, -16 * SIZE(Y)

  452. 	decl	M

  453. 	jle	.L39

  454. 

  455. 	movl	M,  %eax

  456. 	sarl	$3, %eax

  457. 	jle	.L33

  458. 	ALIGN_4

  459. 

  460. .L31:

  461. #ifdef PREFETCHW

  462. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  463. #endif

  464. 

  465. 	movaps	-15 * SIZE(Y), %xmm2

  466. 	movaps	-14 * SIZE(X), %xmm3

  467. 

  468. 	SHUFPD_1  %xmm2, %xmm0

  469. 	movaps	%xmm0, -16 * SIZE(X)

  470. 	SHUFPD_1  %xmm3, %xmm1

  471. 	movaps	%xmm1, -15 * SIZE(Y)

  472. 

  473. 	movaps	-13 * SIZE(Y), %xmm0

  474. 	movaps	-12 * SIZE(X), %xmm1

  475. 

  476. 	SHUFPD_1  %xmm0, %xmm2

  477. 	movaps	%xmm2, -14 * SIZE(X)

  478. 	SHUFPD_1  %xmm1, %xmm3

  479. 	movaps	%xmm3, -13 * SIZE(Y)

  480. 

  481. #ifdef PREFETCHW

  482. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  483. #endif

  484. 

  485. 	movaps	-11 * SIZE(Y), %xmm2

  486. 	movaps	-10 * SIZE(X), %xmm3

  487. 

  488. 	SHUFPD_1  %xmm2, %xmm0

  489. 	movaps	%xmm0, -12 * SIZE(X)

  490. 	SHUFPD_1  %xmm3, %xmm1

  491. 	movaps	%xmm1, -11 * SIZE(Y)

  492. 

  493. 	movaps	 -9 * SIZE(Y), %xmm0

  494. 	movaps	 -8 * SIZE(X), %xmm1

  495. 

  496. 	SHUFPD_1  %xmm0, %xmm2

  497. 	movaps	%xmm2, -10 * SIZE(X)

  498. 	SHUFPD_1  %xmm1, %xmm3

  499. 	movaps	%xmm3,  -9 * SIZE(Y)

  500. 

  501. #ifdef PREFETCHW

  502. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  503. #endif

  504. 

  505. 	movaps	 -7 * SIZE(Y), %xmm2

  506. 	movaps	 -6 * SIZE(X), %xmm3

  507. 

  508. 	SHUFPD_1  %xmm2, %xmm0

  509. 	movaps	%xmm0,  -8 * SIZE(X)

  510. 	SHUFPD_1  %xmm3, %xmm1

  511. 	movaps	%xmm1,  -7 * SIZE(Y)

  512. 

  513. 	movaps	 -5 * SIZE(Y), %xmm0

  514. 	movaps	 -4 * SIZE(X), %xmm1

  515. 

  516. 	SHUFPD_1  %xmm0, %xmm2

  517. 	movaps	%xmm2,  -6 * SIZE(X)

  518. 	SHUFPD_1  %xmm1, %xmm3

  519. 	movaps	%xmm3,  -5 * SIZE(Y)

  520. 

  521. #ifdef PREFETCHW

  522. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  523. #endif

  524. 

  525. 	movaps	 -3 * SIZE(Y), %xmm2

  526. 	movaps	 -2 * SIZE(X), %xmm3

  527. 

  528. 	SHUFPD_1  %xmm2, %xmm0

  529. 	movaps	%xmm0,  -4 * SIZE(X)

  530. 	SHUFPD_1  %xmm3, %xmm1

  531. 	movaps	%xmm1,  -3 * SIZE(Y)

  532. 

  533. 	movaps	 -1 * SIZE(Y), %xmm0

  534. 	movaps	  0 * SIZE(X), %xmm1

  535. 

  536. 	SHUFPD_1  %xmm0, %xmm2

  537. 	movaps	%xmm2,  -2 * SIZE(X)

  538. 	SHUFPD_1  %xmm1, %xmm3

  539. 	movaps	%xmm3,  -1 * SIZE(Y)

  540. 

  541. 	subl	$-16 * SIZE, X

  542. 	subl	$-16 * SIZE, Y

  543. 

  544. 	decl	%eax

  545. 	jg	.L31

  546. 	ALIGN_3

  547. 

  548. .L33:

  549. 	testl	$4, M

  550. 	jle	.L34

  551. 

  552. 	movaps	-15 * SIZE(Y), %xmm2

  553. 	movaps	-14 * SIZE(X), %xmm3

  554. 

  555. 	SHUFPD_1  %xmm2, %xmm0

  556. 	movaps	%xmm0, -16 * SIZE(X)

  557. 	SHUFPD_1  %xmm3, %xmm1

  558. 	movaps	%xmm1, -15 * SIZE(Y)

  559. 

  560. 	movaps	-13 * SIZE(Y), %xmm0

  561. 	movaps	-12 * SIZE(X), %xmm1

  562. 

  563. 	SHUFPD_1  %xmm0, %xmm2

  564. 	movaps	%xmm2, -14 * SIZE(X)

  565. 	SHUFPD_1  %xmm1, %xmm3

  566. 	movaps	%xmm3, -13 * SIZE(Y)

  567. 

  568. 	movaps	-11 * SIZE(Y), %xmm2

  569. 	movaps	-10 * SIZE(X), %xmm3

  570. 

  571. 	SHUFPD_1  %xmm2, %xmm0

  572. 	movaps	%xmm0, -12 * SIZE(X)

  573. 	SHUFPD_1  %xmm3, %xmm1

  574. 	movaps	%xmm1, -11 * SIZE(Y)

  575. 

  576. 	movaps	 -9 * SIZE(Y), %xmm0

  577. 	movaps	 -8 * SIZE(X), %xmm1

  578. 

  579. 	SHUFPD_1  %xmm0, %xmm2

  580. 	movaps	%xmm2, -10 * SIZE(X)

  581. 	SHUFPD_1  %xmm1, %xmm3

  582. 	movaps	%xmm3,  -9 * SIZE(Y)

  583. 

  584. 	addl	$8 * SIZE, X

  585. 	addl	$8 * SIZE, Y

  586. 	ALIGN_3

  587. 

  588. .L34:

  589. 	testl	$2, M

  590. 	jle	.L35

  591. 

  592. 	movaps	-15 * SIZE(Y), %xmm2

  593. 	movaps	-14 * SIZE(X), %xmm3

  594. 

  595. 	SHUFPD_1  %xmm2, %xmm0

  596. 	movaps	%xmm0, -16 * SIZE(X)

  597. 	SHUFPD_1  %xmm3, %xmm1

  598. 	movaps	%xmm1, -15 * SIZE(Y)

  599. 

  600. 	movaps	-13 * SIZE(Y), %xmm0

  601. 	movaps	-12 * SIZE(X), %xmm1

  602. 

  603. 	SHUFPD_1  %xmm0, %xmm2

  604. 	movaps	%xmm2, -14 * SIZE(X)

  605. 	SHUFPD_1  %xmm1, %xmm3

  606. 	movaps	%xmm3, -13 * SIZE(Y)

  607. 

  608. 	addl	$4 * SIZE, X

  609. 	addl	$4 * SIZE, Y

  610. 	ALIGN_3

  611. 

  612. .L35:

  613. 	testl	$1, M

  614. 	jle	.L39

  615. 

  616. 	movaps	-15 * SIZE(Y), %xmm2

  617. 	movaps	-14 * SIZE(X), %xmm3

  618. 

  619. 	SHUFPD_1  %xmm3, %xmm1

  620. 	movaps	%xmm1, -15 * SIZE(Y)

  621. 	SHUFPD_1  %xmm2, %xmm0

  622. 	movaps	%xmm0, -16 * SIZE(X)

  623. 

  624. 	movaps	%xmm2, %xmm0

  625. 	movaps	%xmm3, %xmm1

  626. 

  627. 	addl	$2 * SIZE, X

  628. 	addl	$2 * SIZE, Y

  629. 	ALIGN_3

  630. 

  631. .L39:

  632. 	movaps	-15 * SIZE(Y), %xmm2

  633. 

  634. 	movhps	%xmm1, -15 * SIZE(Y)

  635. 	SHUFPD_1  %xmm2, %xmm0

  636. 	movaps	%xmm0, -16 * SIZE(X)

  637. 

  638. 	popl	%ebx

  639. 	popl	%esi

  640. 	popl	%edi

  641. 	popl	%ebp

  642. 	ret

  643. 	ALIGN_3

  644. 

  645. .L40:

  646. 	movsd	-16 * SIZE(X), %xmm0

  647. 	movsd	-16 * SIZE(Y), %xmm1

  648. 

  649. 	movlps	%xmm0, 	-16 * SIZE(Y)

  650. 	movlps	%xmm1, 	-16 * SIZE(X)

  651. 

  652. 	addl	$SIZE, X

  653. 	addl	$SIZE, Y

  654. 	decl	M

  655. 	jle	.L49

  656. 

  657. 	movl	M,  %eax

  658. 	sarl	$3, %eax

  659. 	jle	.L43

  660. 	ALIGN_3

  661. 

  662. .L41:

  663. #ifdef PREFETCHW

  664. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  665. #endif

  666. 

  667. 	movaps	-16 * SIZE(X), %xmm0

  668. 	movaps	-16 * SIZE(Y), %xmm1

  669. 

  670. 	movaps	%xmm0, 	-16 * SIZE(Y)

  671. 	movaps	%xmm1, 	-16 * SIZE(X)

  672. 

  673. 	movaps	-14 * SIZE(X), %xmm0

  674. 	movaps	-14 * SIZE(Y), %xmm1

  675. 

  676. 	movaps	%xmm0, 	-14 * SIZE(Y)

  677. 	movaps	%xmm1, 	-14 * SIZE(X)

  678. 

  679. #ifdef PREFETCHW

  680. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  681. #endif

  682. 

  683. 	movaps	-12 * SIZE(X), %xmm0

  684. 	movaps	-12 * SIZE(Y), %xmm1

  685. 

  686. 	movaps	%xmm0, 	-12 * SIZE(Y)

  687. 	movaps	%xmm1, 	-12 * SIZE(X)

  688. 

  689. 	movaps	-10 * SIZE(X), %xmm0

  690. 	movaps	-10 * SIZE(Y), %xmm1

  691. 

  692. 	movaps	%xmm0, 	-10 * SIZE(Y)

  693. 	movaps	%xmm1, 	-10 * SIZE(X)

  694. 

  695. #ifdef PREFETCHW

  696. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  697. #endif

  698. 

  699. 	movaps	 -8 * SIZE(X), %xmm0

  700. 	movaps	 -8 * SIZE(Y), %xmm1

  701. 

  702. 	movaps	%xmm0, 	 -8 * SIZE(Y)

  703. 	movaps	%xmm1, 	 -8 * SIZE(X)

  704. 

  705. 	movaps	 -6 * SIZE(X), %xmm0

  706. 	movaps	 -6 * SIZE(Y), %xmm1

  707. 

  708. 	movaps	%xmm0, 	 -6 * SIZE(Y)

  709. 	movaps	%xmm1, 	 -6 * SIZE(X)

  710. 

  711. #ifdef PREFETCHW

  712. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  713. #endif

  714. 

  715. 	movaps	 -4 * SIZE(X), %xmm0

  716. 	movaps	 -4 * SIZE(Y), %xmm1

  717. 

  718. 	movaps	%xmm0, 	 -4 * SIZE(Y)

  719. 	movaps	%xmm1, 	 -4 * SIZE(X)

  720. 

  721. 	movaps	 -2 * SIZE(X), %xmm0

  722. 	movaps	 -2 * SIZE(Y), %xmm1

  723. 

  724. 	movaps	%xmm0, 	 -2 * SIZE(Y)

  725. 	movaps	%xmm1, 	 -2 * SIZE(X)

  726. 

  727. 	subl	$-16 * SIZE, Y

  728. 	subl	$-16 * SIZE, X

  729. 

  730. 	decl	%eax

  731. 	jg	.L41

  732. 	ALIGN_3

  733. 

  734. .L43:

  735. 	testl	$4, M

  736. 	jle	.L44

  737. 

  738. 	movaps	-16 * SIZE(X), %xmm0

  739. 	movaps	-16 * SIZE(Y), %xmm1

  740. 

  741. 	movaps	%xmm0, 	-16 * SIZE(Y)

  742. 	movaps	%xmm1, 	-16 * SIZE(X)

  743. 

  744. 	movaps	-14 * SIZE(X), %xmm0

  745. 	movaps	-14 * SIZE(Y), %xmm1

  746. 

  747. 	movaps	%xmm0, 	-14 * SIZE(Y)

  748. 	movaps	%xmm1, 	-14 * SIZE(X)

  749. 

  750. 	movaps	-12 * SIZE(X), %xmm0

  751. 	movaps	-12 * SIZE(Y), %xmm1

  752. 

  753. 	movaps	%xmm0, 	-12 * SIZE(Y)

  754. 	movaps	%xmm1, 	-12 * SIZE(X)

  755. 

  756. 	movaps	-10 * SIZE(X), %xmm0

  757. 	movaps	-10 * SIZE(Y), %xmm1

  758. 

  759. 	movaps	%xmm0, 	-10 * SIZE(Y)

  760.  	movaps	%xmm1, 	-10 * SIZE(X)

  761. 

  762. 	addl	$8 * SIZE, X

  763. 	addl	$8 * SIZE, Y

  764. 	ALIGN_3

  765. 

  766. .L44:

  767. 	testl	$2, M

  768. 	jle	.L45

  769. 

  770. 	movaps	-16 * SIZE(X), %xmm0

  771. 	movaps	-16 * SIZE(Y), %xmm1

  772. 

  773. 	movaps	%xmm0, 	-16 * SIZE(Y)

  774. 	movaps	%xmm1, 	-16 * SIZE(X)

  775. 

  776. 	movaps	-14 * SIZE(X), %xmm0

  777. 	movaps	-14 * SIZE(Y), %xmm1

  778. 

  779. 	movaps	%xmm0, 	-14 * SIZE(Y)

  780. 	movaps	%xmm1, 	-14 * SIZE(X)

  781. 

  782. 	addl	$4 * SIZE, X

  783. 	addl	$4 * SIZE, Y

  784. 	ALIGN_3

  785. 

  786. .L45:

  787. 	testl	$1, M

  788. 	jle	.L49

  789. 

  790. 	movaps	-16 * SIZE(X), %xmm0

  791. 	movaps	-16 * SIZE(Y), %xmm1

  792. 

  793. 	movaps	%xmm0, 	-16 * SIZE(Y)

  794. 	movaps	%xmm1, 	-16 * SIZE(X)

  795. 

  796. 	addl	$2 * SIZE, X

  797. 	addl	$2 * SIZE, Y

  798. 	ALIGN_3

  799. 

  800. .L49:

  801. 	movsd	-16 * SIZE(X), %xmm0

  802. 	movsd	-16 * SIZE(Y), %xmm1

  803. 

  804. 	movlps	%xmm0, 	-16 * SIZE(Y)

  805. 	movlps	%xmm1, 	-16 * SIZE(X)

  806. 

  807. 	popl	%ebx

  808. 	popl	%esi

  809. 	popl	%edi

  810. 	popl	%ebp

  811. 	ret

  812. 	ALIGN_3

  813. 

  814. .L50:

  815. 	testl	$SIZE, X

  816. 	jne	.L60

  817. 	testl	$SIZE, Y

  818. 	jne	.L60

  819. 

  820. 	movl	M,  %eax

  821. 	sarl	$2, %eax

  822. 	jle	.L55

  823. 	ALIGN_3

  824. 

  825. .L51:

  826. 	movaps	(X), %xmm0

  827. 	movaps	(Y), %xmm1

  828. 

  829. 	movaps	%xmm1, (X)

  830. 	addl	INCX, X

  831. 	movaps	%xmm0, (Y)

  832. 	addl	INCY, Y

  833. 

  834. 	movaps	(X), %xmm0

  835. 	movaps	(Y), %xmm1

  836. 

  837. 	movaps	%xmm1, (X)

  838. 	addl	INCX, X

  839. 	movaps	%xmm0, (Y)

  840. 	addl	INCY, Y

  841. 

  842. 	movaps	(X), %xmm0

  843. 	movaps	(Y), %xmm1

  844. 

  845. 	movaps	%xmm1, (X)

  846. 	addl	INCX, X

  847. 	movaps	%xmm0, (Y)

  848. 	addl	INCY, Y

  849. 

  850. 	movaps	(X), %xmm0

  851. 	movaps	(Y), %xmm1

  852. 

  853. 	movaps	%xmm1, (X)

  854. 	addl	INCX, X

  855. 	movaps	%xmm0, (Y)

  856. 	addl	INCY, Y

  857. 

  858. 	decl	%eax

  859. 	jg	.L51

  860. 	ALIGN_3

  861. 

  862. .L55:

  863. 	movl	M,  %eax

  864. 	andl	$3, %eax

  865. 	jle	.L57

  866. 	ALIGN_3

  867. 

  868. .L56:

  869. 	movaps	(X), %xmm0

  870. 	movaps	(Y), %xmm1

  871. 

  872. 	movaps	%xmm1, (X)

  873. 	addl	INCX, X

  874. 	movaps	%xmm0, (Y)

  875. 	addl	INCY, Y

  876. 

  877. 	decl	%eax

  878. 	jg	.L56

  879. 	ALIGN_3

  880. 

  881. .L57:

  882. 	popl	%ebx

  883. 	popl	%esi

  884. 	popl	%edi

  885. 	popl	%ebp

  886. 	ret

  887. 	ALIGN_3

  888. 

  889. .L60:

  890. 	movl	M,  %eax

  891. 	sarl	$2, %eax

  892. 	jle	.L65

  893. 	ALIGN_3

  894. 

  895. .L61:

  896. 	movsd	0 * SIZE(X), %xmm0

  897. 	movhps	1 * SIZE(X), %xmm0

  898. 	movsd	0 * SIZE(Y), %xmm1

  899. 	movhps	1 * SIZE(Y), %xmm1

  900. 

  901. 	movlps	%xmm1, 0 * SIZE(X)

  902. 	movhps	%xmm1, 1 * SIZE(X)

  903. 	addl	INCX, X

  904. 	movlps	%xmm0, 0 * SIZE(Y)

  905. 	movhps	%xmm0, 1 * SIZE(Y)

  906. 	addl	INCY, Y

  907. 

  908. 	movsd	0 * SIZE(X), %xmm0

  909. 	movhps	1 * SIZE(X), %xmm0

  910. 	movsd	0 * SIZE(Y), %xmm1

  911. 	movhps	1 * SIZE(Y), %xmm1

  912. 

  913. 	movlps	%xmm1, 0 * SIZE(X)

  914. 	movhps	%xmm1, 1 * SIZE(X)

  915. 	addl	INCX, X

  916. 	movlps	%xmm0, 0 * SIZE(Y)

  917. 	movhps	%xmm0, 1 * SIZE(Y)

  918. 	addl	INCY, Y

  919. 

  920. 	movsd	0 * SIZE(X), %xmm0

  921. 	movhps	1 * SIZE(X), %xmm0

  922. 	movsd	0 * SIZE(Y), %xmm1

  923. 	movhps	1 * SIZE(Y), %xmm1

  924. 

  925. 	movlps	%xmm1, 0 * SIZE(X)

  926. 	movhps	%xmm1, 1 * SIZE(X)

  927. 	addl	INCX, X

  928. 	movlps	%xmm0, 0 * SIZE(Y)

  929. 	movhps	%xmm0, 1 * SIZE(Y)

  930. 	addl	INCY, Y

  931. 

  932. 	movsd	0 * SIZE(X), %xmm0

  933. 	movhps	1 * SIZE(X), %xmm0

  934. 	movsd	0 * SIZE(Y), %xmm1

  935. 	movhps	1 * SIZE(Y), %xmm1

  936. 

  937. 	movlps	%xmm1, 0 * SIZE(X)

  938. 	movhps	%xmm1, 1 * SIZE(X)

  939. 	addl	INCX, X

  940. 	movlps	%xmm0, 0 * SIZE(Y)

  941. 	movhps	%xmm0, 1 * SIZE(Y)

  942. 	addl	INCY, Y

  943. 

  944. 	decl	%eax

  945. 	jg	.L61

  946. 	ALIGN_3

  947. 

  948. .L65:

  949. 	movl	M,  %eax

  950. 	andl	$3, %eax

  951. 	jle	.L67

  952. 	ALIGN_3

  953. 

  954. .L66:

  955. 	movsd	0 * SIZE(X), %xmm0

  956. 	movhps	1 * SIZE(X), %xmm0

  957. 	movsd	0 * SIZE(Y), %xmm1

  958. 	movhps	1 * SIZE(Y), %xmm1

  959. 

  960. 	movlps	%xmm1, 0 * SIZE(X)

  961. 	movhps	%xmm1, 1 * SIZE(X)

  962. 	addl	INCX, X

  963. 	movlps	%xmm0, 0 * SIZE(Y)

  964. 	movhps	%xmm0, 1 * SIZE(Y)

  965. 	addl	INCY, Y

  966. 

  967. 	decl	%eax

  968. 	jg	.L66

  969. 	ALIGN_3

  970. 

  971. .L67:

  972. 	popl	%ebx

  973. 	popl	%esi

  974. 	popl	%edi

  975. 	popl	%ebp

  976. 	ret

  977. 

  978. 	EPILOGUE