OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/rot_sse2.S

991 lines
18 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define N	ARG1	/* rdi */

  43. #define X	ARG2	/* rsi */

  44. #define INCX	ARG3	/* rdx */

  45. #define Y	ARG4	/* rcx */

  46. #ifndef WINDOWS_ABI

  47. #define INCY	ARG5	/* r8  */

  48. #else

  49. #define INCY	%r10

  50. #endif

  51. 

  52. #define C	%xmm14

  53. #define S	%xmm15

  54. 

  55. #include "l1param.h"

  56. 

  57. 	PROLOGUE

  58. 	PROFCODE

  59. 

  60. #ifdef WINDOWS_ABI

  61. 	movq	40(%rsp), INCY

  62. 	movsd	48(%rsp), %xmm0

  63. 	movsd	56(%rsp), %xmm1

  64. #endif

  65. 

  66. 	SAVEREGISTERS

  67. 

  68. 	leaq	(, INCX, SIZE), INCX

  69. 	leaq	(, INCY, SIZE), INCY

  70. 

  71. 	pshufd	$0x44, %xmm0, C

  72. 	pshufd	$0x44, %xmm1, S

  73. 

  74. 	cmpq	$0, N

  75. 	jle	.L999

  76. 

  77. 	cmpq	$SIZE, INCX

  78. 	jne	.L50

  79. 	cmpq	$SIZE, INCY

  80. 	jne	.L50

  81. 

  82. 	testq	$SIZE, X

  83. 	je	.L10

  84. 

  85. 	movsd	0 * SIZE(Y), %xmm1

  86. 	movsd	0 * SIZE(X), %xmm0

  87. 

  88. 	movaps	%xmm1, %xmm2

  89. 	movaps	%xmm0, %xmm3

  90. 

  91. 	mulsd	C, %xmm0

  92. 	mulsd	S, %xmm1

  93. 

  94. 	mulsd	C, %xmm2

  95. 	mulsd	S, %xmm3

  96. 

  97. 	addsd	%xmm1, %xmm0

  98. 	subsd	%xmm3, %xmm2

  99. 

  100. 	movsd	%xmm0, 0 * SIZE(X)

  101. 	movsd	%xmm2, 0 * SIZE(Y)

  102. 

  103. 	addq	$1 * SIZE, X

  104. 	addq	$1 * SIZE, Y

  105. 	decq	N

  106. 	jle	.L999

  107. 	ALIGN_2

  108. 

  109. .L10:

  110. 	testq	$SIZE, Y

  111. 	jne	.L20

  112. 

  113. 	movq	N,  %rax

  114. 	sarq	$4, %rax

  115. 	jle	.L14

  116. 

  117. 	movaps	0 * SIZE(Y), %xmm1

  118. 	movaps	2 * SIZE(Y), %xmm3

  119. 	movaps	4 * SIZE(Y), %xmm9

  120. 	movaps	6 * SIZE(Y), %xmm11

  121. 

  122. 	movaps	0 * SIZE(X), %xmm0

  123. 	movaps	2 * SIZE(X), %xmm2

  124. 	movaps	4 * SIZE(X), %xmm8

  125. 	movaps	6 * SIZE(X), %xmm10

  126. 

  127. 	decq	%rax

  128. 	jle	.L12

  129. 	ALIGN_3

  130. 

  131. .L11:

  132. #ifdef PREFETCHW

  133. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  134. #endif

  135. 

  136. 	movaps	%xmm1, %xmm4

  137. 	mulpd	S, %xmm1

  138. 	movaps	%xmm3, %xmm6

  139. 	mulpd	S, %xmm3

  140. 	movaps	%xmm0, %xmm5

  141. 	mulpd	C, %xmm0

  142. 	movaps	%xmm2, %xmm7

  143. 	mulpd	C, %xmm2

  144. 

  145. 	mulpd	C, %xmm4

  146. 	mulpd	S, %xmm5

  147. 	mulpd	C, %xmm6

  148. 	mulpd	S, %xmm7

  149. 

  150. 	addpd	%xmm1, %xmm0

  151. 	movaps	 8 * SIZE(Y), %xmm1

  152. 	addpd	%xmm3, %xmm2

  153. 	movaps	10 * SIZE(Y), %xmm3

  154. 	subpd	%xmm5, %xmm4

  155. 	subpd	%xmm7, %xmm6

  156. 

  157. #ifdef PREFETCHW

  158. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  159. #endif

  160. 

  161. 	movaps	%xmm0, 0 * SIZE(X)

  162. 	movaps	 8 * SIZE(X), %xmm0

  163. 	movaps	%xmm2, 2 * SIZE(X)

  164. 	movaps	10 * SIZE(X), %xmm2

  165. 	movaps	%xmm4, 0 * SIZE(Y)

  166. 	movaps	%xmm6, 2 * SIZE(Y)

  167. 

  168. 	movaps	%xmm9, %xmm4

  169. 	mulpd	S, %xmm9

  170. 	movaps	%xmm8, %xmm5

  171. 	mulpd	C, %xmm8

  172. 	movaps	%xmm11, %xmm6

  173. 	mulpd	S, %xmm11

  174. 	movaps	%xmm10, %xmm7

  175. 	mulpd	C, %xmm10

  176. 

  177. 	mulpd	C, %xmm4

  178. 	mulpd	S, %xmm5

  179. 	mulpd	C, %xmm6

  180. 	mulpd	S, %xmm7

  181. 

  182. 	addpd	%xmm9,  %xmm8

  183. 	movaps	12 * SIZE(Y), %xmm9

  184. 	addpd	%xmm11, %xmm10

  185. 	movaps	14 * SIZE(Y), %xmm11

  186. 	subpd	%xmm5,  %xmm4

  187. 	subpd	%xmm7,  %xmm6

  188. 

  189. 	movaps	%xmm8, 4 * SIZE(X)

  190. 	movaps	12 * SIZE(X), %xmm8

  191. 	movaps	%xmm10,6 * SIZE(X)

  192. 	movaps	14 * SIZE(X), %xmm10

  193. 	movaps	%xmm4, 4 * SIZE(Y)

  194. 	movaps	%xmm6, 6 * SIZE(Y)

  195. 

  196. #ifdef PREFETCHW

  197. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  198. #endif

  199. 

  200. 	movaps	%xmm1, %xmm4

  201. 	mulpd	S, %xmm1

  202. 	movaps	%xmm3, %xmm6

  203. 	mulpd	S, %xmm3

  204. 	movaps	%xmm0, %xmm5

  205. 	mulpd	C, %xmm0

  206. 	movaps	%xmm2, %xmm7

  207. 	mulpd	C, %xmm2

  208. 

  209. 	mulpd	C, %xmm4

  210. 	mulpd	S, %xmm5

  211. 	mulpd	C, %xmm6

  212. 	mulpd	S, %xmm7

  213. 

  214. 	addpd	%xmm1, %xmm0

  215. 	movaps	16 * SIZE(Y), %xmm1

  216. 	addpd	%xmm3, %xmm2

  217. 	movaps	18 * SIZE(Y), %xmm3

  218. 	subpd	%xmm5, %xmm4

  219. 	subpd	%xmm7, %xmm6

  220. 

  221. 	movaps	%xmm0,  8 * SIZE(X)

  222. 	movaps	16 * SIZE(X), %xmm0

  223. 	movaps	%xmm2, 10 * SIZE(X)

  224. 	movaps	18 * SIZE(X), %xmm2

  225. 	movaps	%xmm4,  8 * SIZE(Y)

  226. 	movaps	%xmm6, 10 * SIZE(Y)

  227. 

  228. #ifdef PREFETCHW

  229. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  230. #endif

  231. 

  232. 	movaps	%xmm9, %xmm4

  233. 	mulpd	S, %xmm9

  234. 	movaps	%xmm8, %xmm5

  235. 	mulpd	C, %xmm8

  236. 	movaps	%xmm11, %xmm6

  237. 	mulpd	S, %xmm11

  238. 	movaps	%xmm10, %xmm7

  239. 	mulpd	C, %xmm10

  240. 

  241. 	mulpd	C, %xmm4

  242. 	mulpd	S, %xmm5

  243. 	mulpd	C, %xmm6

  244. 	mulpd	S, %xmm7

  245. 

  246. 	addpd	%xmm9,  %xmm8

  247. 	movaps	20 * SIZE(Y), %xmm9

  248. 	addpd	%xmm11, %xmm10

  249. 	movaps	22 * SIZE(Y), %xmm11

  250. 	subpd	%xmm5,  %xmm4

  251. 	subpd	%xmm7,  %xmm6

  252. 

  253. 	movaps	%xmm8,  12 * SIZE(X)

  254. 	movaps	20 * SIZE(X), %xmm8

  255. 	movaps	%xmm10, 14 * SIZE(X)

  256. 	movaps	22 * SIZE(X), %xmm10

  257. 	movaps	%xmm4,  12 * SIZE(Y)

  258. 	movaps	%xmm6,  14 * SIZE(Y)

  259. 

  260. 	addq	$16 * SIZE, X

  261. 	addq	$16 * SIZE, Y

  262. 

  263. 	decq	%rax

  264. 	jg	.L11

  265. 	ALIGN_3

  266. 

  267. .L12:

  268. 	movaps	%xmm1, %xmm4

  269. 	mulpd	S, %xmm1

  270. 	movaps	%xmm3, %xmm6

  271. 	mulpd	S, %xmm3

  272. 	movaps	%xmm0, %xmm5

  273. 	mulpd	C, %xmm0

  274. 	movaps	%xmm2, %xmm7

  275. 	mulpd	C, %xmm2

  276. 

  277. 	mulpd	C, %xmm4

  278. 	mulpd	S, %xmm5

  279. 	mulpd	C, %xmm6

  280. 	mulpd	S, %xmm7

  281. 

  282. 	addpd	%xmm1, %xmm0

  283. 	movaps	 8 * SIZE(Y), %xmm1

  284. 	addpd	%xmm3, %xmm2

  285. 	movaps	10 * SIZE(Y), %xmm3

  286. 	subpd	%xmm5, %xmm4

  287. 	subpd	%xmm7, %xmm6

  288. 

  289. 	movaps	%xmm0, 0 * SIZE(X)

  290. 	movaps	 8 * SIZE(X), %xmm0

  291. 	movaps	%xmm2, 2 * SIZE(X)

  292. 	movaps	10 * SIZE(X), %xmm2

  293. 

  294. 	movaps	%xmm4, 0 * SIZE(Y)

  295. 	movaps	%xmm6, 2 * SIZE(Y)

  296. 

  297. 	movaps	%xmm9, %xmm4

  298. 	mulpd	S, %xmm9

  299. 	movaps	%xmm8, %xmm5

  300. 	mulpd	C, %xmm8

  301. 	movaps	%xmm11, %xmm6

  302. 	mulpd	S, %xmm11

  303. 	movaps	%xmm10, %xmm7

  304. 	mulpd	C, %xmm10

  305. 

  306. 	mulpd	C, %xmm4

  307. 	mulpd	S, %xmm5

  308. 	mulpd	C, %xmm6

  309. 	mulpd	S, %xmm7

  310. 

  311. 	addpd	%xmm9,  %xmm8

  312. 	movaps	12 * SIZE(Y), %xmm9

  313. 	addpd	%xmm11, %xmm10

  314. 	movaps	14 * SIZE(Y), %xmm11

  315. 	subpd	%xmm5,  %xmm4

  316. 	subpd	%xmm7,  %xmm6

  317. 

  318. 	movaps	%xmm8, 4 * SIZE(X)

  319. 	movaps	12 * SIZE(X), %xmm8

  320. 	movaps	%xmm10,6 * SIZE(X)

  321. 	movaps	14 * SIZE(X), %xmm10

  322. 	movaps	%xmm4, 4 * SIZE(Y)

  323. 	movaps	%xmm6, 6 * SIZE(Y)

  324. 

  325. 	movaps	%xmm1, %xmm4

  326. 	mulpd	S, %xmm1

  327. 	movaps	%xmm3, %xmm6

  328. 	mulpd	S, %xmm3

  329. 	movaps	%xmm0, %xmm5

  330. 	mulpd	C, %xmm0

  331. 	movaps	%xmm2, %xmm7

  332. 	mulpd	C, %xmm2

  333. 

  334. 	mulpd	C, %xmm4

  335. 	mulpd	S, %xmm5

  336. 	mulpd	C, %xmm6

  337. 	mulpd	S, %xmm7

  338. 

  339. 	addpd	%xmm1, %xmm0

  340. 	addpd	%xmm3, %xmm2

  341. 	subpd	%xmm5, %xmm4

  342. 	subpd	%xmm7, %xmm6

  343. 

  344. 	movaps	%xmm0,  8 * SIZE(X)

  345. 	movaps	%xmm2, 10 * SIZE(X)

  346. 	movaps	%xmm4,  8 * SIZE(Y)

  347. 	movaps	%xmm6, 10 * SIZE(Y)

  348. 

  349. 	movaps	%xmm9, %xmm4

  350. 	mulpd	S, %xmm9

  351. 	movaps	%xmm8, %xmm5

  352. 	mulpd	C, %xmm8

  353. 	movaps	%xmm11, %xmm6

  354. 	mulpd	S, %xmm11

  355. 	movaps	%xmm10, %xmm7

  356. 	mulpd	C, %xmm10

  357. 

  358. 	mulpd	C, %xmm4

  359. 	mulpd	S, %xmm5

  360. 	mulpd	C, %xmm6

  361. 	mulpd	S, %xmm7

  362. 

  363. 	addpd	%xmm9,  %xmm8

  364. 	addpd	%xmm11, %xmm10

  365. 	subpd	%xmm5,  %xmm4

  366. 	subpd	%xmm7,  %xmm6

  367. 

  368. 	movaps	%xmm8,  12 * SIZE(X)

  369. 	movaps	%xmm10, 14 * SIZE(X)

  370. 	movaps	%xmm4,  12 * SIZE(Y)

  371. 	movaps	%xmm6,  14 * SIZE(Y)

  372. 

  373. 	addq	$16 * SIZE, X

  374. 	addq	$16 * SIZE, Y

  375. 	ALIGN_3

  376. 

  377. 

  378. .L14:

  379. 	testq	$15, N

  380. 	jle	.L999

  381. 

  382. 	testq	$8, N

  383. 	jle	.L15

  384. 

  385. 	movaps	0 * SIZE(Y), %xmm1

  386. 	movaps	0 * SIZE(X), %xmm0

  387. 	movaps	2 * SIZE(Y), %xmm3

  388. 	movaps	2 * SIZE(X), %xmm2

  389. 

  390. 	movaps	%xmm1, %xmm4

  391. 	movaps	%xmm0, %xmm5

  392. 	movaps	%xmm3, %xmm6

  393. 	movaps	%xmm2, %xmm7

  394. 

  395. 	mulpd	C, %xmm0

  396. 	mulpd	S, %xmm1

  397. 	mulpd	C, %xmm2

  398. 	mulpd	S, %xmm3

  399. 

  400. 	mulpd	C, %xmm4

  401. 	mulpd	S, %xmm5

  402. 	mulpd	C, %xmm6

  403. 	mulpd	S, %xmm7

  404. 

  405. 	addpd	%xmm1, %xmm0

  406. 	addpd	%xmm3, %xmm2

  407. 	subpd	%xmm5, %xmm4

  408. 	subpd	%xmm7, %xmm6

  409. 

  410. 	movaps	%xmm0, 0 * SIZE(X)

  411. 	movaps	%xmm2, 2 * SIZE(X)

  412. 	movaps	%xmm4, 0 * SIZE(Y)

  413. 	movaps	%xmm6, 2 * SIZE(Y)

  414. 

  415. 	movaps	4 * SIZE(Y), %xmm1

  416. 	movaps	4 * SIZE(X), %xmm0

  417. 	movaps	6 * SIZE(Y), %xmm3

  418. 	movaps	6 * SIZE(X), %xmm2

  419. 

  420. 	movaps	%xmm1, %xmm4

  421. 	movaps	%xmm0, %xmm5

  422. 	movaps	%xmm3, %xmm6

  423. 	movaps	%xmm2, %xmm7

  424. 

  425. 	mulpd	C, %xmm0

  426. 	mulpd	S, %xmm1

  427. 	mulpd	C, %xmm2

  428. 	mulpd	S, %xmm3

  429. 

  430. 	mulpd	C, %xmm4

  431. 	mulpd	S, %xmm5

  432. 	mulpd	C, %xmm6

  433. 	mulpd	S, %xmm7

  434. 

  435. 	addpd	%xmm1, %xmm0

  436. 	addpd	%xmm3, %xmm2

  437. 	subpd	%xmm5, %xmm4

  438. 	subpd	%xmm7, %xmm6

  439. 

  440. 	movaps	%xmm0, 4 * SIZE(X)

  441. 	movaps	%xmm2, 6 * SIZE(X)

  442. 	movaps	%xmm4, 4 * SIZE(Y)

  443. 	movaps	%xmm6, 6 * SIZE(Y)

  444. 

  445. 	addq	$8 * SIZE, X

  446. 	addq	$8 * SIZE, Y

  447. 	ALIGN_3

  448. 

  449. .L15:

  450. 	testq	$4, N

  451. 	jle	.L16

  452. 

  453. 	movaps	0 * SIZE(Y), %xmm1

  454. 	movaps	0 * SIZE(X), %xmm0

  455. 	movaps	2 * SIZE(Y), %xmm3

  456. 	movaps	2 * SIZE(X), %xmm2

  457. 

  458. 	movaps	%xmm1, %xmm4

  459. 	movaps	%xmm0, %xmm5

  460. 	movaps	%xmm3, %xmm6

  461. 	movaps	%xmm2, %xmm7

  462. 

  463. 	mulpd	C, %xmm0

  464. 	mulpd	S, %xmm1

  465. 	mulpd	C, %xmm2

  466. 	mulpd	S, %xmm3

  467. 

  468. 	mulpd	C, %xmm4

  469. 	mulpd	S, %xmm5

  470. 	mulpd	C, %xmm6

  471. 	mulpd	S, %xmm7

  472. 

  473. 	addpd	%xmm1, %xmm0

  474. 	addpd	%xmm3, %xmm2

  475. 	subpd	%xmm5, %xmm4

  476. 	subpd	%xmm7, %xmm6

  477. 

  478. 	movaps	%xmm0, 0 * SIZE(X)

  479. 	movaps	%xmm2, 2 * SIZE(X)

  480. 	movaps	%xmm4, 0 * SIZE(Y)

  481. 	movaps	%xmm6, 2 * SIZE(Y)

  482. 

  483. 	addq	$4 * SIZE, X

  484. 	addq	$4 * SIZE, Y

  485. 	ALIGN_3

  486. 

  487. .L16:

  488. 	testq	$2, N

  489. 	jle	.L17

  490. 

  491. 	movaps	0 * SIZE(Y), %xmm1

  492. 	movaps	0 * SIZE(X), %xmm0

  493. 

  494. 	movaps	%xmm1, %xmm2

  495. 	movaps	%xmm0, %xmm3

  496. 

  497. 	mulpd	C, %xmm0

  498. 	mulpd	S, %xmm1

  499. 

  500. 	mulpd	C, %xmm2

  501. 	mulpd	S, %xmm3

  502. 

  503. 	addpd	%xmm1, %xmm0

  504. 	subpd	%xmm3, %xmm2

  505. 

  506. 	movaps	%xmm0, 0 * SIZE(X)

  507. 	movaps	%xmm2, 0 * SIZE(Y)

  508. 

  509. 	addq	$2 * SIZE, X

  510. 	addq	$2 * SIZE, Y

  511. 	ALIGN_3

  512. 

  513. .L17:

  514. 	testq	$1, N

  515. 	jle	.L999

  516. 

  517. 	movsd	0 * SIZE(Y), %xmm1

  518. 	movsd	0 * SIZE(X), %xmm0

  519. 

  520. 	movaps	%xmm1, %xmm2

  521. 	movaps	%xmm0, %xmm3

  522. 

  523. 	mulsd	C, %xmm0

  524. 	mulsd	S, %xmm1

  525. 

  526. 	mulsd	C, %xmm2

  527. 	mulsd	S, %xmm3

  528. 

  529. 	addsd	%xmm1, %xmm0

  530. 	subsd	%xmm3, %xmm2

  531. 

  532. 	movsd	%xmm0, 0 * SIZE(X)

  533. 	movsd	%xmm2, 0 * SIZE(Y)

  534. 	jmp	.L999

  535. 	ALIGN_3

  536. 

  537. .L20:

  538. 	movaps	-1 * SIZE(Y), %xmm1

  539. 

  540. 	movq	N,  %rax

  541. 	sarq	$4, %rax

  542. 	jle	.L24

  543. 	ALIGN_3

  544. 

  545. .L21:

  546. #ifdef PREFETCHW

  547. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  548. #endif

  549. 

  550. 	movaps	1 * SIZE(Y), %xmm3

  551. 	movaps	3 * SIZE(Y), %xmm8

  552. 	movaps	0 * SIZE(X), %xmm0

  553. 	movaps	2 * SIZE(X), %xmm2

  554. 

  555. 	SHUFPD_1 %xmm3, %xmm1

  556. 	SHUFPD_1 %xmm8, %xmm3

  557. 

  558. 	movaps	%xmm1, %xmm4

  559. 	movaps	%xmm0, %xmm5

  560. 	movaps	%xmm3, %xmm6

  561. 	movaps	%xmm2, %xmm7

  562. 

  563. 	mulpd	C, %xmm0

  564. 	mulpd	S, %xmm1

  565. 	mulpd	C, %xmm2

  566. 	mulpd	S, %xmm3

  567. 

  568. 	mulpd	C, %xmm4

  569. 	mulpd	S, %xmm5

  570. 	mulpd	C, %xmm6

  571. 	mulpd	S, %xmm7

  572. 

  573. 	addpd	%xmm1, %xmm0

  574. 	addpd	%xmm3, %xmm2

  575. 	subpd	%xmm5, %xmm4

  576. 	subpd	%xmm7, %xmm6

  577. 

  578. 	movaps	%xmm0, 0 * SIZE(X)

  579. 	movaps	%xmm2, 2 * SIZE(X)

  580. 

  581. 	movlpd	%xmm4, 0 * SIZE(Y)

  582. 	movhps	%xmm4, 1 * SIZE(Y)

  583. 	movlpd	%xmm6, 2 * SIZE(Y)

  584. 	movhps	%xmm6, 3 * SIZE(Y)

  585. 

  586. #ifdef PREFETCHW

  587. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  588. #endif

  589. 

  590. 	movaps	5 * SIZE(Y), %xmm9

  591. 	movaps	7 * SIZE(Y), %xmm1

  592. 	movaps	4 * SIZE(X), %xmm0

  593. 	movaps	6 * SIZE(X), %xmm2

  594. 

  595. 	SHUFPD_1 %xmm9, %xmm8

  596. 	SHUFPD_1 %xmm1, %xmm9

  597. 

  598. 	movaps	%xmm8, %xmm4

  599. 	movaps	%xmm0, %xmm5

  600. 	movaps	%xmm9, %xmm6

  601. 	movaps	%xmm2, %xmm7

  602. 

  603. 	mulpd	C, %xmm0

  604. 	mulpd	S, %xmm8

  605. 	mulpd	C, %xmm2

  606. 	mulpd	S, %xmm9

  607. 

  608. 	mulpd	C, %xmm4

  609. 	mulpd	S, %xmm5

  610. 	mulpd	C, %xmm6

  611. 	mulpd	S, %xmm7

  612. 

  613. 	addpd	%xmm8, %xmm0

  614. 	addpd	%xmm9, %xmm2

  615. 	subpd	%xmm5, %xmm4

  616. 	subpd	%xmm7, %xmm6

  617. 

  618. 	movaps	%xmm0, 4 * SIZE(X)

  619. 	movaps	%xmm2, 6 * SIZE(X)

  620. 	movlpd	%xmm4, 4 * SIZE(Y)

  621. 	movhps	%xmm4, 5 * SIZE(Y)

  622. 	movlpd	%xmm6, 6 * SIZE(Y)

  623. 	movhps	%xmm6, 7 * SIZE(Y)

  624. 

  625. #ifdef PREFETCHW

  626. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(X)

  627. #endif

  628. 

  629. 	movaps	 9 * SIZE(Y), %xmm3

  630. 	movaps	11 * SIZE(Y), %xmm8

  631. 	movaps	 8 * SIZE(X), %xmm0

  632. 	movaps	10 * SIZE(X), %xmm2

  633. 

  634. 	SHUFPD_1 %xmm3, %xmm1

  635. 	SHUFPD_1 %xmm8, %xmm3

  636. 

  637. 	movaps	%xmm1, %xmm4

  638. 	movaps	%xmm0, %xmm5

  639. 	movaps	%xmm3, %xmm6

  640. 	movaps	%xmm2, %xmm7

  641. 

  642. 	mulpd	C, %xmm0

  643. 	mulpd	S, %xmm1

  644. 	mulpd	C, %xmm2

  645. 	mulpd	S, %xmm3

  646. 

  647. 	mulpd	C, %xmm4

  648. 	mulpd	S, %xmm5

  649. 	mulpd	C, %xmm6

  650. 	mulpd	S, %xmm7

  651. 

  652. 	addpd	%xmm1, %xmm0

  653. 	addpd	%xmm3, %xmm2

  654. 	subpd	%xmm5, %xmm4

  655. 	subpd	%xmm7, %xmm6

  656. 

  657. 	movaps	%xmm0,  8 * SIZE(X)

  658. 	movaps	%xmm2, 10 * SIZE(X)

  659. 	movlpd	%xmm4,  8 * SIZE(Y)

  660. 	movhps	%xmm4,  9 * SIZE(Y)

  661. 	movlpd	%xmm6, 10 * SIZE(Y)

  662. 	movhps	%xmm6, 11 * SIZE(Y)

  663. 

  664. #ifdef PREFETCHW

  665. 	PREFETCHW (PREFETCHSIZE + 64) - PREOFFSET(Y)

  666. #endif

  667. 

  668. 	movaps	13 * SIZE(Y), %xmm9

  669. 	movaps	15 * SIZE(Y), %xmm1

  670. 	movaps	12 * SIZE(X), %xmm0

  671. 	movaps	14 * SIZE(X), %xmm2

  672. 

  673. 	SHUFPD_1 %xmm9, %xmm8

  674. 	SHUFPD_1 %xmm1, %xmm9

  675. 

  676. 	movaps	%xmm8, %xmm4

  677. 	movaps	%xmm0, %xmm5

  678. 	movaps	%xmm9, %xmm6

  679. 	movaps	%xmm2, %xmm7

  680. 

  681. 	mulpd	C, %xmm0

  682. 	mulpd	S, %xmm8

  683. 	mulpd	C, %xmm2

  684. 	mulpd	S, %xmm9

  685. 

  686. 	mulpd	C, %xmm4

  687. 	mulpd	S, %xmm5

  688. 	mulpd	C, %xmm6

  689. 	mulpd	S, %xmm7

  690. 

  691. 	addpd	%xmm8, %xmm0

  692. 	addpd	%xmm9, %xmm2

  693. 	subpd	%xmm5, %xmm4

  694. 	subpd	%xmm7, %xmm6

  695. 

  696. 	movaps	%xmm0, 12 * SIZE(X)

  697. 	movaps	%xmm2, 14 * SIZE(X)

  698. 	movlpd	%xmm4, 12 * SIZE(Y)

  699. 	movhps	%xmm4, 13 * SIZE(Y)

  700. 	movlpd	%xmm6, 14 * SIZE(Y)

  701. 	movhps	%xmm6, 15 * SIZE(Y)

  702. 

  703. 	addq	$16 * SIZE, X

  704. 	addq	$16 * SIZE, Y

  705. 	decq	%rax

  706. 	jg	.L21

  707. 	ALIGN_3

  708. 

  709. .L24:

  710. 	testq	$15, N

  711. 	jle	.L999

  712. 

  713. 	testq	$8, N

  714. 	jle	.L25

  715. 

  716. 	movaps	1 * SIZE(Y), %xmm3

  717. 	movaps	3 * SIZE(Y), %xmm8

  718. 	movaps	0 * SIZE(X), %xmm0

  719. 	movaps	2 * SIZE(X), %xmm2

  720. 

  721. 	SHUFPD_1 %xmm3, %xmm1

  722. 	SHUFPD_1 %xmm8, %xmm3

  723. 

  724. 	movaps	%xmm1, %xmm4

  725. 	movaps	%xmm0, %xmm5

  726. 	movaps	%xmm3, %xmm6

  727. 	movaps	%xmm2, %xmm7

  728. 

  729. 	mulpd	C, %xmm0

  730. 	mulpd	S, %xmm1

  731. 	mulpd	C, %xmm2

  732. 	mulpd	S, %xmm3

  733. 

  734. 	mulpd	C, %xmm4

  735. 	mulpd	S, %xmm5

  736. 	mulpd	C, %xmm6

  737. 	mulpd	S, %xmm7

  738. 

  739. 	addpd	%xmm1, %xmm0

  740. 	addpd	%xmm3, %xmm2

  741. 	subpd	%xmm5, %xmm4

  742. 	subpd	%xmm7, %xmm6

  743. 

  744. 	movaps	%xmm0, 0 * SIZE(X)

  745. 	movaps	%xmm2, 2 * SIZE(X)

  746. 	movlpd	%xmm4, 0 * SIZE(Y)

  747. 	movhps	%xmm4, 1 * SIZE(Y)

  748. 	movlpd	%xmm6, 2 * SIZE(Y)

  749. 	movhps	%xmm6, 3 * SIZE(Y)

  750. 

  751. 	movaps	5 * SIZE(Y), %xmm9

  752. 	movaps	7 * SIZE(Y), %xmm1

  753. 	movaps	4 * SIZE(X), %xmm0

  754. 	movaps	6 * SIZE(X), %xmm2

  755. 

  756. 	SHUFPD_1 %xmm9, %xmm8

  757. 	SHUFPD_1 %xmm1, %xmm9

  758. 

  759. 	movaps	%xmm8, %xmm4

  760. 	movaps	%xmm0, %xmm5

  761. 	movaps	%xmm9, %xmm6

  762. 	movaps	%xmm2, %xmm7

  763. 

  764. 	mulpd	C, %xmm0

  765. 	mulpd	S, %xmm8

  766. 	mulpd	C, %xmm2

  767. 	mulpd	S, %xmm9

  768. 

  769. 	mulpd	C, %xmm4

  770. 	mulpd	S, %xmm5

  771. 	mulpd	C, %xmm6

  772. 	mulpd	S, %xmm7

  773. 

  774. 	addpd	%xmm8, %xmm0

  775. 	addpd	%xmm9, %xmm2

  776. 	subpd	%xmm5, %xmm4

  777. 	subpd	%xmm7, %xmm6

  778. 

  779. 	movaps	%xmm0, 4 * SIZE(X)

  780. 	movaps	%xmm2, 6 * SIZE(X)

  781. 	movlpd	%xmm4, 4 * SIZE(Y)

  782. 	movhps	%xmm4, 5 * SIZE(Y)

  783. 	movlpd	%xmm6, 6 * SIZE(Y)

  784. 	movhps	%xmm6, 7 * SIZE(Y)

  785. 

  786. 	addq	$8 * SIZE, X

  787. 	addq	$8 * SIZE, Y

  788. 	ALIGN_3

  789. 

  790. .L25:

  791. 	testq	$4, N

  792. 	jle	.L26

  793. 

  794. 	movaps	1 * SIZE(Y), %xmm3

  795. 	movaps	3 * SIZE(Y), %xmm8

  796. 	movaps	0 * SIZE(X), %xmm0

  797. 	movaps	2 * SIZE(X), %xmm2

  798. 

  799. 	SHUFPD_1 %xmm3, %xmm1

  800. 	SHUFPD_1 %xmm8, %xmm3

  801. 

  802. 	movaps	%xmm1, %xmm4

  803. 	movaps	%xmm0, %xmm5

  804. 	movaps	%xmm3, %xmm6

  805. 	movaps	%xmm2, %xmm7

  806. 

  807. 	mulpd	C, %xmm0

  808. 	mulpd	S, %xmm1

  809. 	mulpd	C, %xmm2

  810. 	mulpd	S, %xmm3

  811. 

  812. 	mulpd	C, %xmm4

  813. 	mulpd	S, %xmm5

  814. 	mulpd	C, %xmm6

  815. 	mulpd	S, %xmm7

  816. 

  817. 	addpd	%xmm1, %xmm0

  818. 	addpd	%xmm3, %xmm2

  819. 	subpd	%xmm5, %xmm4

  820. 	subpd	%xmm7, %xmm6

  821. 

  822. 	movaps	%xmm0, 0 * SIZE(X)

  823. 	movaps	%xmm2, 2 * SIZE(X)

  824. 	movlpd	%xmm4, 0 * SIZE(Y)

  825. 	movhps	%xmm4, 1 * SIZE(Y)

  826. 	movlpd	%xmm6, 2 * SIZE(Y)

  827. 	movhps	%xmm6, 3 * SIZE(Y)

  828. 	movaps	%xmm8, %xmm1

  829. 

  830. 	addq	$4 * SIZE, X

  831. 	addq	$4 * SIZE, Y

  832. 	ALIGN_3

  833. 

  834. .L26:

  835. 	testq	$2, N

  836. 	jle	.L27

  837. 

  838. 	movaps	1 * SIZE(Y), %xmm4

  839. 	movaps	0 * SIZE(X), %xmm0

  840. 

  841. 	SHUFPD_1 %xmm4, %xmm1

  842. 	movaps	%xmm1, %xmm2

  843. 	movaps	%xmm0, %xmm3

  844. 

  845. 	mulpd	C, %xmm0

  846. 	mulpd	S, %xmm1

  847. 

  848. 	mulpd	C, %xmm2

  849. 	mulpd	S, %xmm3

  850. 

  851. 	addpd	%xmm1, %xmm0

  852. 	subpd	%xmm3, %xmm2

  853. 

  854. 	movaps	%xmm0, 0 * SIZE(X)

  855. 	movlpd	%xmm2, 0 * SIZE(Y)

  856. 	movhps	%xmm2, 1 * SIZE(Y)

  857. 	movaps	%xmm4, %xmm1

  858. 

  859. 	addq	$2 * SIZE, X

  860. 	addq	$2 * SIZE, Y

  861. 	ALIGN_3

  862. 

  863. .L27:

  864. 	testq	$1, N

  865. 	jle	.L999

  866. 

  867. 	unpckhpd %xmm1, %xmm1

  868. 	movsd	0 * SIZE(X), %xmm0

  869. 

  870. 	movaps	%xmm1, %xmm2

  871. 	movaps	%xmm0, %xmm3

  872. 

  873. 	mulsd	C, %xmm0

  874. 	mulsd	S, %xmm1

  875. 

  876. 	mulsd	C, %xmm2

  877. 	mulsd	S, %xmm3

  878. 

  879. 	addsd	%xmm1, %xmm0

  880. 	subsd	%xmm3, %xmm2

  881. 

  882. 	movsd	%xmm0, 0 * SIZE(X)

  883. 	movsd	%xmm2, 0 * SIZE(Y)

  884. 	jmp	.L999

  885. 	ALIGN_3

  886. 

  887. 

  888. .L50:

  889. 	movq	N,  %rax

  890. 	cmpq	$0, INCX

  891. 	je  .L56

  892. 	cmpq	$0, INCY

  893. 	je  .L56

  894. 	sarq	$2, %rax

  895. 	jle	.L55

  896. 	ALIGN_3

  897. 

  898. .L53:

  899. 	movsd	(Y), %xmm1

  900. 	movhps	(Y, INCY), %xmm1

  901. 	movsd	(X), %xmm0

  902. 	movhps	(X, INCX), %xmm0

  903. 

  904. 	movaps	%xmm1, %xmm2

  905. 	movaps	%xmm0, %xmm3

  906. 

  907. 	mulpd	C, %xmm0

  908. 	mulpd	S, %xmm1

  909. 

  910. 	mulpd	C, %xmm2

  911. 	mulpd	S, %xmm3

  912. 

  913. 	addpd	%xmm1, %xmm0

  914. 	subpd	%xmm3, %xmm2

  915. 

  916. 	movlpd	%xmm0, (X)

  917. 	movhps	%xmm0, (X, INCX)

  918. 	movlpd	%xmm2, (Y)

  919. 	movhps	%xmm2, (Y, INCY)

  920. 

  921. 	leaq	(X, INCX, 2), X

  922. 	leaq	(Y, INCY, 2), Y

  923. 

  924. 	movsd	(Y), %xmm1

  925. 	movhps	(Y, INCY), %xmm1

  926. 	movsd	(X), %xmm0

  927. 	movhps	(X, INCX), %xmm0

  928. 

  929. 	movaps	%xmm1, %xmm2

  930. 	movaps	%xmm0, %xmm3

  931. 

  932. 	mulpd	C, %xmm0

  933. 	mulpd	S, %xmm1

  934. 

  935. 	mulpd	C, %xmm2

  936. 	mulpd	S, %xmm3

  937. 

  938. 	addpd	%xmm1, %xmm0

  939. 	subpd	%xmm3, %xmm2

  940. 

  941. 	movlpd	%xmm0, (X)

  942. 	movhps	%xmm0, (X, INCX)

  943. 	movlpd	%xmm2, (Y)

  944. 	movhps	%xmm2, (Y, INCY)

  945. 

  946. 	leaq	(X, INCX, 2), X

  947. 	leaq	(Y, INCY, 2), Y

  948. 

  949. 	decq	%rax

  950. 	jg	.L53

  951. 	ALIGN_3

  952. 

  953. .L55:

  954. 	movq	N,  %rax

  955. 	andq	$3, %rax

  956. 	jle	.L999

  957. 	ALIGN_3

  958. 

  959. .L56:

  960. 	movsd	(Y), %xmm1

  961. 	movsd	(X), %xmm0

  962. 

  963. 	movaps	%xmm1, %xmm2

  964. 	movaps	%xmm0, %xmm3

  965. 

  966. 	mulsd	C, %xmm0

  967. 	mulsd	S, %xmm1

  968. 

  969. 	mulsd	C, %xmm2

  970. 	mulsd	S, %xmm3

  971. 

  972. 	addsd	%xmm1, %xmm0

  973. 	subsd	%xmm3, %xmm2

  974. 

  975. 	movsd	%xmm0, (X)

  976. 	movsd	%xmm2, (Y)

  977. 

  978. 	addq	INCX, X

  979. 	addq	INCY, Y

  980. 

  981. 	decq	%rax

  982. 	jg	.L56

  983. 	ALIGN_3

  984. 

  985. .L999:

  986. 	RESTOREREGISTERS

  987. 

  988. 	ret

  989. 

  990. 	EPILOGUE