OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 886cbaf4e4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '886cbaf4e4'
${ noResults }
OpenBLAS/kernel/x86_64/zaxpy_atom.S

676 lines
14 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #define Y	ARG6

  47. #define INCY	ARG2

  48. #else

  49. #define M	ARG1

  50. #define X	ARG2

  51. #define INCX	ARG3

  52. #define Y	ARG4

  53. #define INCY	%r10

  54. #endif

  55. 

  56. #define	YY	%r11

  57. #define ALPHA_R	%xmm14

  58. #define ALPHA_I	%xmm15

  59. 

  60. #include "l1param.h"

  61. 

  62. 	PROLOGUE

  63. 	PROFCODE

  64. 

  65. #ifndef WINDOWS_ABI

  66. #ifndef XDOUBLE

  67. 	movq	 8(%rsp), INCY

  68. #else

  69. 	movq	40(%rsp), INCY

  70. #endif

  71. #else

  72. 	movaps	%xmm3, %xmm0

  73. 	movsd	40(%rsp), %xmm1	

  74. 

  75. 	movq	48(%rsp), X

  76. 	movq	56(%rsp), INCX

  77. 	movq	64(%rsp), Y

  78. 	movq	72(%rsp), INCY

  79. #endif

  80. 

  81. 	SAVEREGISTERS

  82. 	

  83. #ifndef CONJ

  84. #define ADD1	subsd

  85. #define ADD2	addsd

  86. #else

  87. #define ADD1	addsd

  88. #define ADD2	subsd

  89. #endif

  90. 

  91. 	salq	$ZBASE_SHIFT, INCX

  92. 	movaps	%xmm0, ALPHA_R

  93. 	salq	$ZBASE_SHIFT, INCY

  94. 	movaps	%xmm1, ALPHA_I

  95. 

  96. 	testq	M, M

  97. 	jle	.L999

  98. 	

  99. 	cmpq	$2 * SIZE, INCX

  100. 	jne	.L20

  101. 	cmpq	$2 * SIZE, INCY

  102. 	jne	.L20

  103. 

  104. 	movq	M,  %rax

  105. 	sarq	$2, %rax

  106. 	jle	.L15

  107. 

  108. 	movsd	 0 * SIZE(X), %xmm0

  109. 	movsd	 1 * SIZE(X), %xmm1

  110. 	movsd	 0 * SIZE(Y), %xmm8

  111. 	movsd	 1 * SIZE(Y), %xmm9

  112. 

  113. 	movsd	 2 * SIZE(X), %xmm4

  114. 	movsd	 3 * SIZE(X), %xmm5

  115. 	movsd	 2 * SIZE(Y), %xmm10

  116. 	movsd	 3 * SIZE(Y), %xmm11

  117. 

  118. 	movaps	 %xmm0,  %xmm2

  119. 	mulsd	ALPHA_R, %xmm0

  120. 	movaps	 %xmm1,  %xmm3

  121. 	mulsd	ALPHA_R, %xmm1

  122. 	mulsd	ALPHA_I, %xmm3

  123. 	mulsd	ALPHA_I, %xmm2

  124. 

  125. 	movaps	 %xmm4,  %xmm6

  126. 	mulsd	ALPHA_R, %xmm4

  127. 	addsd	%xmm0, %xmm8

  128. 	movsd	 4 * SIZE(X), %xmm0

  129. 

  130. 	movaps	 %xmm5,  %xmm7

  131. 	mulsd	ALPHA_R, %xmm5

  132. 	ADD2	%xmm1, %xmm9

  133. 	movsd	 5 * SIZE(X), %xmm1

  134. 

  135. 	decq	%rax

  136. 	jle	.L12

  137. 	ALIGN_3

  138. 

  139. .L11:

  140. #ifdef PREFETCH

  141. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  142. #endif

  143. 

  144. 	mulsd	ALPHA_I, %xmm7

  145. 	movsd	 4 * SIZE(Y), %xmm12

  146. 	ADD1	%xmm3, %xmm8

  147. 

  148. 	mulsd	ALPHA_I, %xmm6

  149. 	movsd	 5 * SIZE(Y), %xmm13

  150. 	addsd	%xmm2, %xmm9

  151. 

  152. 	addsd	%xmm4, %xmm10

  153. 	movsd	 6 * SIZE(X), %xmm4

  154. 	movaps	 %xmm0,  %xmm2

  155. 	mulsd	ALPHA_R, %xmm0

  156. 

  157. 	ADD2	%xmm5, %xmm11

  158. 	movsd	 7 * SIZE(X), %xmm5

  159. 	movaps	 %xmm1,  %xmm3

  160. 	mulsd	ALPHA_R, %xmm1

  161. 

  162. 	ADD1	%xmm7, %xmm10

  163. 	movsd	%xmm8,   0 * SIZE(Y)

  164. 	mulsd	ALPHA_I, %xmm3

  165. 

  166. 	addsd	%xmm6, %xmm11

  167. 	movsd	%xmm9,   1 * SIZE(Y)

  168. 	mulsd	ALPHA_I, %xmm2

  169. 

  170. 	movaps	 %xmm4,  %xmm6

  171. 	movsd	%xmm10,  2 * SIZE(Y)

  172. 	mulsd	ALPHA_R, %xmm4

  173. 	movsd	 6 * SIZE(Y), %xmm10

  174. 	addsd	%xmm0, %xmm12

  175. 	movsd	 8 * SIZE(X), %xmm0

  176. 

  177. #ifdef PREFETCHW

  178. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  179. #endif

  180. 

  181. 	movaps	 %xmm5,  %xmm7

  182. 	movsd	%xmm11,  3 * SIZE(Y)

  183. 	mulsd	ALPHA_R, %xmm5

  184. 	movsd	 7 * SIZE(Y), %xmm11

  185. 	ADD2	%xmm1, %xmm13

  186. 	movsd	 9 * SIZE(X), %xmm1

  187. 

  188. 	mulsd	ALPHA_I, %xmm7

  189. 	movsd	 8 * SIZE(Y), %xmm8

  190. 	ADD1	%xmm3, %xmm12

  191. 

  192. 	mulsd	ALPHA_I, %xmm6

  193. 	movsd	 9 * SIZE(Y), %xmm9

  194. 	addsd	%xmm2, %xmm13

  195. 

  196. 	movaps	 %xmm0,  %xmm2

  197. 	mulsd	ALPHA_R, %xmm0

  198. 	addsd	%xmm4, %xmm10

  199. 	movsd	10 * SIZE(X), %xmm4

  200. 

  201. 	movaps	 %xmm1,  %xmm3

  202. 	mulsd	ALPHA_R, %xmm1

  203. 	ADD2	%xmm5, %xmm11

  204. 	movsd	11 * SIZE(X), %xmm5

  205. 

  206. 	mulsd	ALPHA_I, %xmm3

  207. 	movsd	%xmm12,   4 * SIZE(Y)

  208. 	ADD1	%xmm7, %xmm10

  209. 

  210. 	mulsd	ALPHA_I, %xmm2

  211. 	movsd	%xmm13,   5 * SIZE(Y)

  212. 	addsd	%xmm6, %xmm11

  213. 

  214. 	movaps	 %xmm4,  %xmm6

  215. 	movsd	%xmm10,  6 * SIZE(Y)

  216. 	mulsd	ALPHA_R, %xmm4

  217. 	addsd	%xmm0, %xmm8

  218. 	movsd	10 * SIZE(Y), %xmm10

  219. 	movsd	12 * SIZE(X), %xmm0

  220. 

  221. 	movaps	 %xmm5,  %xmm7

  222. 	movsd	%xmm11,  7 * SIZE(Y)

  223. 	mulsd	ALPHA_R, %xmm5

  224. 	movsd	11 * SIZE(Y), %xmm11

  225. 	ADD2	%xmm1, %xmm9

  226. 	movsd	13 * SIZE(X), %xmm1

  227. 

  228. 	addq	$8 * SIZE, X

  229. 	addq	$8 * SIZE, Y

  230. 	decq	%rax

  231. 	jg	.L11

  232. 	ALIGN_3

  233. 

  234. .L12:

  235. 	mulsd	ALPHA_I, %xmm7

  236. 	movsd	 4 * SIZE(Y), %xmm12

  237. 	ADD1	%xmm3, %xmm8

  238. 

  239. 	mulsd	ALPHA_I, %xmm6

  240. 	movsd	 5 * SIZE(Y), %xmm13

  241. 	addsd	%xmm2, %xmm9

  242. 

  243. 	addsd	%xmm4, %xmm10

  244. 	movsd	 6 * SIZE(X), %xmm4

  245. 	movaps	 %xmm0,  %xmm2

  246. 	mulsd	ALPHA_R, %xmm0

  247. 

  248. 	ADD2	%xmm5, %xmm11

  249. 	movsd	 7 * SIZE(X), %xmm5

  250. 	movaps	 %xmm1,  %xmm3

  251. 	mulsd	ALPHA_R, %xmm1

  252. 

  253. 	ADD1	%xmm7, %xmm10

  254. 	movsd	%xmm8,   0 * SIZE(Y)

  255. 	mulsd	ALPHA_I, %xmm3

  256. 

  257. 	addsd	%xmm6, %xmm11

  258. 	movsd	%xmm9,   1 * SIZE(Y)

  259. 	mulsd	ALPHA_I, %xmm2

  260. 

  261. 	movaps	 %xmm4,  %xmm6

  262. 	movsd	%xmm10,  2 * SIZE(Y)

  263. 	mulsd	ALPHA_R, %xmm4

  264. 	movsd	 6 * SIZE(Y), %xmm10

  265. 	addsd	%xmm0, %xmm12

  266. 

  267. 	movaps	 %xmm5,  %xmm7

  268. 	movsd	%xmm11,  3 * SIZE(Y)

  269. 	mulsd	ALPHA_R, %xmm5

  270. 	ADD2	%xmm1, %xmm13

  271. 	movsd	 7 * SIZE(Y), %xmm11

  272. 

  273. 	mulsd	ALPHA_I, %xmm7

  274. 	ADD1	%xmm3, %xmm12

  275. 

  276. 	mulsd	ALPHA_I, %xmm6

  277. 	addsd	%xmm2, %xmm13

  278. 

  279. 	movaps	 %xmm0,  %xmm2

  280. 	mulsd	ALPHA_R, %xmm0

  281. 	addsd	%xmm4, %xmm10

  282. 

  283. 	movaps	 %xmm1,  %xmm3

  284. 	mulsd	ALPHA_R, %xmm1

  285. 	ADD2	%xmm5, %xmm11

  286. 

  287. 	mulsd	ALPHA_I, %xmm3

  288. 	ADD1	%xmm7, %xmm10

  289. 

  290. 	addsd	%xmm6, %xmm11

  291. 	mulsd	ALPHA_I, %xmm2

  292. 

  293. 	movsd	%xmm12,  4 * SIZE(Y)

  294. 	movsd	%xmm13,  5 * SIZE(Y)

  295. 	movsd	%xmm10,  6 * SIZE(Y)

  296. 	movsd	%xmm11,  7 * SIZE(Y)

  297. 

  298. 	addq	$8 * SIZE, X

  299. 	addq	$8 * SIZE, Y

  300. 	ALIGN_3

  301. 

  302. .L15:

  303. 	movq	M,  %rax

  304. 	andq	$2, %rax

  305. 	jle	.L17

  306. 

  307. 	movsd	 0 * SIZE(X), %xmm0

  308. 	movsd	 1 * SIZE(X), %xmm1

  309. 	movsd	 2 * SIZE(X), %xmm4

  310. 	movsd	 3 * SIZE(X), %xmm5

  311. 

  312. 	movaps	 %xmm0,  %xmm2

  313. 	movsd	 0 * SIZE(Y), %xmm8

  314. 	mulsd	ALPHA_R, %xmm0

  315. 	movaps	 %xmm1,  %xmm3

  316. 	movsd	 1 * SIZE(Y), %xmm9

  317. 	mulsd	ALPHA_R, %xmm1

  318. 	movsd	 2 * SIZE(Y), %xmm10

  319. 	mulsd	ALPHA_I, %xmm3

  320. 	movsd	 3 * SIZE(Y), %xmm11

  321. 	mulsd	ALPHA_I, %xmm2

  322. 

  323. 	movaps	 %xmm4,  %xmm6

  324. 	mulsd	ALPHA_R, %xmm4

  325. 	addsd	%xmm0, %xmm8

  326. 

  327. 	movaps	 %xmm5,  %xmm7

  328. 	mulsd	ALPHA_R, %xmm5

  329. 	ADD2	%xmm1, %xmm9

  330. 

  331. 	mulsd	ALPHA_I, %xmm7

  332. 	ADD1	%xmm3, %xmm8

  333. 

  334. 	mulsd	ALPHA_I, %xmm6

  335. 	addsd	%xmm2, %xmm9

  336. 

  337. 	addsd	%xmm4, %xmm10

  338. 	movsd	%xmm8,   0 * SIZE(Y)

  339. 	ADD2	%xmm5, %xmm11

  340. 	movsd	%xmm9,   1 * SIZE(Y)

  341. 	ADD1	%xmm7, %xmm10

  342. 	addsd	%xmm6, %xmm11

  343. 

  344. 	movsd	%xmm10,   2 * SIZE(Y)

  345. 	movsd	%xmm11,   3 * SIZE(Y)

  346. 

  347. 	addq	$4 * SIZE, X

  348. 	addq	$4 * SIZE, Y

  349. 	ALIGN_3

  350. 

  351. .L17:

  352. 	movq	M,  %rax

  353. 	andq	$1, %rax

  354. 	jle	.L999

  355. 

  356. 	movsd	 0 * SIZE(X), %xmm0

  357. 	movsd	 1 * SIZE(X), %xmm1

  358. 	movsd	 0 * SIZE(Y), %xmm8

  359. 	movsd	 1 * SIZE(Y), %xmm9

  360. 

  361. 	movaps	 %xmm0,  %xmm2

  362. 	mulsd	ALPHA_R, %xmm0

  363. 	movaps	 %xmm1,  %xmm3

  364. 	mulsd	ALPHA_R, %xmm1

  365. 	mulsd	ALPHA_I, %xmm3

  366. 	mulsd	ALPHA_I, %xmm2

  367. 

  368. 	addsd	%xmm0, %xmm8

  369. 	ADD2	%xmm1, %xmm9

  370. 	ADD1	%xmm3, %xmm8

  371. 	addsd	%xmm2, %xmm9

  372. 

  373. 	movsd	%xmm8,   0 * SIZE(Y)

  374. 	movsd	%xmm9,   1 * SIZE(Y)

  375. 	jmp	.L999

  376. 	ALIGN_3

  377. 

  378. .L20:

  379. 	movq	Y, YY

  380. 

  381. 	movq	M,  %rax

  382. 	sarq	$2, %rax

  383. 	jle	.L25

  384. 

  385. 	movsd	 0 * SIZE(X), %xmm0

  386. 	movsd	 1 * SIZE(X), %xmm1

  387. 	addq	 INCX, X

  388. 	movsd	 0 * SIZE(Y), %xmm8

  389. 	movsd	 1 * SIZE(Y), %xmm9

  390. 	addq	 INCY, Y

  391. 

  392. 	movsd	 0 * SIZE(X), %xmm4

  393. 	movsd	 1 * SIZE(X), %xmm5

  394. 	addq	 INCX, X

  395. 	movsd	 0 * SIZE(Y), %xmm10

  396. 	movsd	 1 * SIZE(Y), %xmm11

  397. 	addq	 INCY, Y

  398. 

  399. 	movaps	 %xmm0,  %xmm2

  400. 	mulsd	ALPHA_R, %xmm0

  401. 	movaps	 %xmm1,  %xmm3

  402. 	mulsd	ALPHA_R, %xmm1

  403. 	mulsd	ALPHA_I, %xmm3

  404. 	mulsd	ALPHA_I, %xmm2

  405. 

  406. 	movaps	 %xmm4,  %xmm6

  407. 	mulsd	ALPHA_R, %xmm4

  408. 	addsd	%xmm0, %xmm8

  409. 	movsd	 0 * SIZE(X), %xmm0

  410. 

  411. 	movaps	 %xmm5,  %xmm7

  412. 	mulsd	ALPHA_R, %xmm5

  413. 	ADD2	%xmm1, %xmm9

  414. 	movsd	 1 * SIZE(X), %xmm1

  415. 	addq	 INCX, X

  416. 

  417. 	decq	%rax

  418. 	jle	.L22

  419. 	ALIGN_3

  420. 

  421. .L21:

  422. 	mulsd	ALPHA_I, %xmm7

  423. 	movsd	 0 * SIZE(Y), %xmm12

  424. 	ADD1	%xmm3, %xmm8

  425. 

  426. 	mulsd	ALPHA_I, %xmm6

  427. 	movsd	 1 * SIZE(Y), %xmm13

  428. 	addsd	%xmm2, %xmm9

  429. 	addq	 INCY, Y

  430. 

  431. 	addsd	%xmm4, %xmm10

  432. 	movsd	 0 * SIZE(X), %xmm4

  433. 	movaps	 %xmm0,  %xmm2

  434. 	mulsd	ALPHA_R, %xmm0

  435. 

  436. 	ADD2	%xmm5, %xmm11

  437. 	movsd	 1 * SIZE(X), %xmm5

  438. 	movaps	 %xmm1,  %xmm3

  439. 	addq	 INCX, X

  440. 	mulsd	ALPHA_R, %xmm1

  441. 

  442. 	ADD1	%xmm7, %xmm10

  443. 	movsd	%xmm8,   0 * SIZE(YY)

  444. 	mulsd	ALPHA_I, %xmm3

  445. 

  446. 	addsd	%xmm6, %xmm11

  447. 	movsd	%xmm9,   1 * SIZE(YY)

  448. 	mulsd	ALPHA_I, %xmm2

  449. 	addq	 INCY, YY

  450. 

  451. 	movaps	 %xmm4,  %xmm6

  452. 	movsd	%xmm10,  0 * SIZE(YY)

  453. 	mulsd	ALPHA_R, %xmm4

  454. 	movsd	 0 * SIZE(Y), %xmm10

  455. 	addsd	%xmm0, %xmm12

  456. 	movsd	 0 * SIZE(X), %xmm0

  457. 

  458. 	movaps	 %xmm5,  %xmm7

  459. 	movsd	%xmm11,  1 * SIZE(YY)

  460. 	addq	 INCY, YY

  461. 	mulsd	ALPHA_R, %xmm5

  462. 	movsd	 1 * SIZE(Y), %xmm11

  463. 	addq	 INCY, Y

  464. 	ADD2	%xmm1, %xmm13

  465. 	movsd	 1 * SIZE(X), %xmm1

  466. 	addq	 INCX, X

  467. 

  468. 	mulsd	ALPHA_I, %xmm7

  469. 	movsd	 0 * SIZE(Y), %xmm8

  470. 	ADD1	%xmm3, %xmm12

  471. 

  472. 	mulsd	ALPHA_I, %xmm6

  473. 	movsd	 1 * SIZE(Y), %xmm9

  474. 	addsd	%xmm2, %xmm13

  475. 	addq	 INCY, Y

  476. 

  477. 	movaps	 %xmm0,  %xmm2

  478. 	mulsd	ALPHA_R, %xmm0

  479. 	addsd	%xmm4, %xmm10

  480. 	movsd	 0 * SIZE(X), %xmm4

  481. 

  482. 	movaps	 %xmm1,  %xmm3

  483. 	mulsd	ALPHA_R, %xmm1

  484. 	ADD2	%xmm5, %xmm11

  485. 	movsd	 1 * SIZE(X), %xmm5

  486. 	addq	 INCX, X

  487. 

  488. 	mulsd	ALPHA_I, %xmm3

  489. 	movsd	%xmm12,   0 * SIZE(YY)

  490. 	ADD1	%xmm7, %xmm10

  491. 

  492. 	mulsd	ALPHA_I, %xmm2

  493. 	movsd	%xmm13,   1 * SIZE(YY)

  494. 	addsd	%xmm6, %xmm11

  495. 	addq	 INCY, YY

  496. 

  497. 	movaps	 %xmm4,  %xmm6

  498. 	movsd	%xmm10,  0 * SIZE(YY)

  499. 	mulsd	ALPHA_R, %xmm4

  500. 	addsd	%xmm0, %xmm8

  501. 	movsd	 0 * SIZE(Y), %xmm10

  502. 	movsd	 0 * SIZE(X), %xmm0

  503. 

  504. 	movaps	 %xmm5,  %xmm7

  505. 	movsd	%xmm11,  1 * SIZE(YY)

  506. 	addq	 INCY, YY

  507. 	mulsd	ALPHA_R, %xmm5

  508. 	movsd	 1 * SIZE(Y), %xmm11

  509. 	addq	 INCY, Y

  510. 	ADD2	%xmm1, %xmm9

  511. 	movsd	 1 * SIZE(X), %xmm1

  512. 	addq	 INCX, X

  513. 

  514. 	decq	%rax

  515. 	jg	.L21

  516. 	ALIGN_3

  517. 

  518. .L22:

  519. 	mulsd	ALPHA_I, %xmm7

  520. 	movsd	 0 * SIZE(Y), %xmm12

  521. 	ADD1	%xmm3, %xmm8

  522. 

  523. 	mulsd	ALPHA_I, %xmm6

  524. 	movsd	 1 * SIZE(Y), %xmm13

  525. 	addsd	%xmm2, %xmm9

  526. 	addq	 INCY, Y

  527. 

  528. 	addsd	%xmm4, %xmm10

  529. 	movsd	 0 * SIZE(X), %xmm4

  530. 	movaps	 %xmm0,  %xmm2

  531. 	mulsd	ALPHA_R, %xmm0

  532. 

  533. 	ADD2	%xmm5, %xmm11

  534. 	movsd	 1 * SIZE(X), %xmm5

  535. 	movaps	 %xmm1,  %xmm3

  536. 	addq	 INCX, X

  537. 	mulsd	ALPHA_R, %xmm1

  538. 

  539. 	ADD1	%xmm7, %xmm10

  540. 	movsd	%xmm8,   0 * SIZE(YY)

  541. 	mulsd	ALPHA_I, %xmm3

  542. 

  543. 	addsd	%xmm6, %xmm11

  544. 	movsd	%xmm9,   1 * SIZE(YY)

  545. 	mulsd	ALPHA_I, %xmm2

  546. 	addq	 INCY, YY

  547. 

  548. 	movaps	 %xmm4,  %xmm6

  549. 	movsd	%xmm10,  0 * SIZE(YY)

  550. 	mulsd	ALPHA_R, %xmm4

  551. 	movsd	 0 * SIZE(Y), %xmm10

  552. 	addsd	%xmm0, %xmm12

  553. 

  554. 	movaps	 %xmm5,  %xmm7

  555. 	movsd	%xmm11,  1 * SIZE(YY)

  556. 	mulsd	ALPHA_R, %xmm5

  557. 	addq	 INCY, YY

  558. 	ADD2	%xmm1, %xmm13

  559. 	movsd	 1 * SIZE(Y), %xmm11

  560. 

  561. 	mulsd	ALPHA_I, %xmm7

  562. 	addq	 INCY, Y

  563. 	ADD1	%xmm3, %xmm12

  564. 

  565. 	mulsd	ALPHA_I, %xmm6

  566. 	addsd	%xmm2, %xmm13

  567. 

  568. 	movaps	 %xmm0,  %xmm2

  569. 	mulsd	ALPHA_R, %xmm0

  570. 	addsd	%xmm4, %xmm10

  571. 

  572. 	movaps	 %xmm1,  %xmm3

  573. 	mulsd	ALPHA_R, %xmm1

  574. 	ADD2	%xmm5, %xmm11

  575. 

  576. 	mulsd	ALPHA_I, %xmm3

  577. 	ADD1	%xmm7, %xmm10

  578. 

  579. 	addsd	%xmm6, %xmm11

  580. 	mulsd	ALPHA_I, %xmm2

  581. 

  582. 	movsd	%xmm12,  0 * SIZE(YY)

  583. 	movsd	%xmm13,  1 * SIZE(YY)

  584. 	addq	 INCY, YY

  585. 	movsd	%xmm10,  0 * SIZE(YY)

  586. 	movsd	%xmm11,  1 * SIZE(YY)

  587. 	addq	 INCY, YY

  588. 	ALIGN_3

  589. 

  590. .L25:

  591. 	movq	M,  %rax

  592. 	andq	$2, %rax

  593. 	jle	.L27

  594. 

  595. 	movsd	 0 * SIZE(X), %xmm0

  596. 	movsd	 1 * SIZE(X), %xmm1

  597. 	addq	 INCX, X

  598. 	movsd	 0 * SIZE(X), %xmm4

  599. 	movsd	 1 * SIZE(X), %xmm5

  600. 	addq	 INCX, X

  601. 

  602. 	movaps	 %xmm0,  %xmm2

  603. 	movsd	 0 * SIZE(Y), %xmm8

  604. 	mulsd	ALPHA_R, %xmm0

  605. 	movaps	 %xmm1,  %xmm3

  606. 	movsd	 1 * SIZE(Y), %xmm9

  607. 	addq	 INCY, Y

  608. 	mulsd	ALPHA_R, %xmm1

  609. 	movsd	 0 * SIZE(Y), %xmm10

  610. 	mulsd	ALPHA_I, %xmm3

  611. 	movsd	 1 * SIZE(Y), %xmm11

  612. 	mulsd	ALPHA_I, %xmm2

  613. 	addq	 INCY, Y

  614. 

  615. 	movaps	 %xmm4,  %xmm6

  616. 	mulsd	ALPHA_R, %xmm4

  617. 	addsd	%xmm0, %xmm8

  618. 

  619. 	movaps	 %xmm5,  %xmm7

  620. 	mulsd	ALPHA_R, %xmm5

  621. 	ADD2	%xmm1, %xmm9

  622. 

  623. 	mulsd	ALPHA_I, %xmm7

  624. 	ADD1	%xmm3, %xmm8

  625. 

  626. 	mulsd	ALPHA_I, %xmm6

  627. 	addsd	%xmm2, %xmm9

  628. 

  629. 	addsd	%xmm4, %xmm10

  630. 	movsd	%xmm8,   0 * SIZE(YY)

  631. 	ADD2	%xmm5, %xmm11

  632. 	movsd	%xmm9,   1 * SIZE(YY)

  633. 	ADD1	%xmm7, %xmm10

  634. 	addq	 INCY, YY

  635. 	addsd	%xmm6, %xmm11

  636. 

  637. 	movsd	%xmm10,  0 * SIZE(YY)

  638. 	movsd	%xmm11,  1 * SIZE(YY)

  639. 	addq	 INCY, YY

  640. 	ALIGN_3

  641. 

  642. .L27:

  643. 	movq	M,  %rax

  644. 	andq	$1, %rax

  645. 	jle	.L999

  646. 

  647. 	movsd	 0 * SIZE(X), %xmm0

  648. 	movsd	 1 * SIZE(X), %xmm1

  649. 	movsd	 0 * SIZE(Y), %xmm8

  650. 	movsd	 1 * SIZE(Y), %xmm9

  651. 

  652. 	movaps	 %xmm0,  %xmm2

  653. 	mulsd	ALPHA_R, %xmm0

  654. 	movaps	 %xmm1,  %xmm3

  655. 	mulsd	ALPHA_R, %xmm1

  656. 	mulsd	ALPHA_I, %xmm3

  657. 	mulsd	ALPHA_I, %xmm2

  658. 

  659. 	addsd	%xmm0, %xmm8

  660. 	ADD2	%xmm1, %xmm9

  661. 	ADD1	%xmm3, %xmm8

  662. 	addsd	%xmm2, %xmm9

  663. 

  664. 	movsd	%xmm8,   0 * SIZE(YY)

  665. 	movsd	%xmm9,   1 * SIZE(YY)

  666. 	ALIGN_3

  667. 

  668. .L999:

  669. 	xorq	%rax, %rax

  670. 

  671. 	RESTOREREGISTERS

  672. 

  673. 	ret

  674. 

  675. 	EPILOGUE