OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/zgemv_n_dup.S

1501 lines
30 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. #include "l2param.h"

  42. 

  43. #ifndef WINDOWS_ABI

  44. 

  45. #define STACKSIZE	64

  46. 

  47. #define OLD_INCX	 8 + STACKSIZE(%rsp)

  48. #define OLD_Y		16 + STACKSIZE(%rsp)

  49. #define OLD_INCY	24 + STACKSIZE(%rsp)

  50. #define OLD_BUFFER	32 + STACKSIZE(%rsp)

  51. #define ALPHA_R		48	      (%rsp)

  52. #define ALPHA_I		56	      (%rsp)

  53. 

  54. #define M	  %rdi

  55. #define N	  %rsi

  56. #define A	  %rcx

  57. #define LDA	  %r8

  58. #define X	  %r9

  59. #define INCX	  %rdx

  60. #define Y	  %rbp

  61. #define INCY	  %r10

  62. 

  63. #else

  64. 

  65. #define STACKSIZE	256

  66. 

  67. #define OLD_ALPHA_I	 40 + STACKSIZE(%rsp)

  68. #define OLD_A		 48 + STACKSIZE(%rsp)

  69. #define OLD_LDA		 56 + STACKSIZE(%rsp)

  70. #define OLD_X		 64 + STACKSIZE(%rsp)

  71. #define OLD_INCX	 72 + STACKSIZE(%rsp)

  72. #define OLD_Y		 80 + STACKSIZE(%rsp)

  73. #define OLD_INCY	 88 + STACKSIZE(%rsp)

  74. #define OLD_BUFFER	 96 + STACKSIZE(%rsp)

  75. #define ALPHA_R		224	       (%rsp)

  76. #define ALPHA_I		232	       (%rsp)

  77. 

  78. #define M	  %rcx

  79. #define N	  %rdx

  80. #define A	  %r8

  81. #define LDA	  %r9

  82. #define X	  %rdi

  83. #define INCX	  %rsi

  84. #define Y	  %rbp

  85. #define INCY	  %r10

  86. 

  87. #endif

  88. 

  89. #define I	%rax

  90. #define A1	%r12

  91. #define A2	%r13

  92. 

  93. #define Y1	%r14

  94. #define BUFFER	%r15

  95. 

  96. #define J	%r11

  97. 

  98. #undef SUBPD

  99. 

  100. #if (!defined(CONJ) && !defined(XCONJ)) || (defined(CONJ) && defined(XCONJ))

  101. #define SUBPD	   subpd

  102. #else

  103. #define SUBPD	   addpd

  104. #endif

  105. 

  106. 	PROLOGUE

  107. 	PROFCODE

  108. 

  109. 	subq	$STACKSIZE, %rsp

  110. 	movq	%rbx,  0(%rsp)

  111. 	movq	%rbp,  8(%rsp)

  112. 	movq	%r12, 16(%rsp)

  113. 	movq	%r13, 24(%rsp)

  114. 	movq	%r14, 32(%rsp)

  115. 	movq	%r15, 40(%rsp)

  116. 

  117. #ifdef WINDOWS_ABI

  118. 	movq	%rdi,    48(%rsp)

  119. 	movq	%rsi,    56(%rsp)

  120. 	movups	%xmm6,   64(%rsp)

  121. 	movups	%xmm7,   80(%rsp)

  122. 	movups	%xmm8,   96(%rsp)

  123. 	movups	%xmm9,  112(%rsp)

  124. 	movups	%xmm10, 128(%rsp)

  125. 	movups	%xmm11, 144(%rsp)

  126. 	movups	%xmm12, 160(%rsp)

  127. 	movups	%xmm13, 176(%rsp)

  128. 	movups	%xmm14, 192(%rsp)

  129. 	movups	%xmm15, 208(%rsp)

  130. 

  131. 	movq	OLD_A,     A

  132. 	movq	OLD_LDA,   LDA

  133. 	movq	OLD_X,     X

  134. 

  135. 	movapd	%xmm3,       %xmm0

  136. 	movsd	OLD_ALPHA_I, %xmm1

  137. #endif

  138. 

  139. 	movq	OLD_INCX,  INCX

  140. 	movq	OLD_Y,     Y

  141. 	movq	OLD_INCY,  INCY

  142. 	movq	OLD_BUFFER, BUFFER

  143. 

  144. 	salq	$ZBASE_SHIFT,   LDA

  145. 	salq	$ZBASE_SHIFT,   INCX

  146. 	salq	$ZBASE_SHIFT,   INCY

  147. 

  148. 	movlps	%xmm0, ALPHA_R

  149. 	movlps	%xmm1, ALPHA_I

  150. 

  151. 	subq	$-16 * SIZE, A

  152. 

  153. 	testq	M, M

  154. 	jle	.L999

  155. 	testq	N, N

  156. 	jle	.L999

  157. 	ALIGN_3

  158. 

  159. 	movq	BUFFER, Y1

  160. 

  161. 	xorps	%xmm4, %xmm4

  162. 

  163. 	movq	M,   %rax

  164. 	addq	$8,  %rax

  165. 	sarq	$3,  %rax

  166. 	ALIGN_3

  167. 

  168. .L01:

  169. 	movaps	%xmm4,  0 * SIZE(Y1)

  170. 	movaps	%xmm4,  2 * SIZE(Y1)

  171. 	movaps	%xmm4,  4 * SIZE(Y1)

  172. 	movaps	%xmm4,  6 * SIZE(Y1)

  173. 	movaps	%xmm4,  8 * SIZE(Y1)

  174. 	movaps	%xmm4, 10 * SIZE(Y1)

  175. 	movaps	%xmm4, 12 * SIZE(Y1)

  176. 	movaps	%xmm4, 14 * SIZE(Y1)

  177. 

  178. 	subq	$-16 * SIZE, Y1

  179. 	decq	%rax

  180. 	jg	.L01

  181. 	ALIGN_3

  182. 

  183. .L10:

  184. #if GEMV_UNROLL >= 4

  185. 

  186. 	cmpq	$4, N

  187. 	jl	.L20

  188. 	ALIGN_3

  189. 

  190. .L11:

  191. 	subq	$4, N

  192. 

  193. 	leaq	16 * SIZE(BUFFER), Y1

  194. 	movq	A,  A1

  195. 	leaq	(A,  LDA, 2), A2

  196. 	leaq	(A,  LDA, 4), A

  197. 

  198. 	movddup	0 * SIZE(X), %xmm8

  199. 	movddup	1 * SIZE(X), %xmm9

  200. 	addq	INCX, X

  201. 	movddup	0 * SIZE(X), %xmm10

  202. 	movddup	1 * SIZE(X), %xmm11

  203. 	addq	INCX, X

  204. 	movddup	0 * SIZE(X), %xmm12

  205. 	movddup	1 * SIZE(X), %xmm13

  206. 	addq	INCX, X

  207. 	movddup	0 * SIZE(X), %xmm14

  208. 	movddup	1 * SIZE(X), %xmm15

  209. 	addq	INCX, X

  210. 

  211. 	pcmpeqb	%xmm5, %xmm5

  212. 	psllq	$63,   %xmm5

  213. 	shufps	$0x40, %xmm5, %xmm5

  214. 

  215. 	movsd	ALPHA_R, %xmm6

  216. 	movhps	ALPHA_I, %xmm6

  217. 

  218. 	pshufd	$0x4e, %xmm6,  %xmm7

  219. 

  220. #ifndef XCONJ

  221. 	xorps	 %xmm5, %xmm7

  222. #else

  223. 	xorps	 %xmm5, %xmm6

  224. #endif

  225. 

  226. 	mulpd	 %xmm6, %xmm8

  227. 	mulpd	 %xmm7, %xmm9

  228. 	mulpd	 %xmm6, %xmm10

  229. 	mulpd	 %xmm7, %xmm11

  230. 

  231. 	mulpd	 %xmm6, %xmm12

  232. 	mulpd	 %xmm7, %xmm13

  233. 	mulpd	 %xmm6, %xmm14

  234. 	mulpd	 %xmm7, %xmm15

  235. 

  236. #ifndef XCONJ

  237. 	subpd	 %xmm9,  %xmm8

  238. 	subpd	 %xmm11, %xmm10

  239. 	subpd	 %xmm13, %xmm12

  240. 	subpd	 %xmm15, %xmm14

  241. #else

  242. 	addpd	 %xmm9,  %xmm8

  243. 	addpd	 %xmm11, %xmm10

  244. 	addpd	 %xmm13, %xmm12

  245. 	addpd	 %xmm15, %xmm14

  246. #endif

  247. 

  248. 	pshufd	 $0x4e, %xmm8,  %xmm9

  249. 	pshufd	 $0x4e, %xmm10, %xmm11

  250. 	pshufd	 $0x4e, %xmm12, %xmm13

  251. 	pshufd	 $0x4e, %xmm14, %xmm15

  252. 

  253. #ifndef XCONJ

  254. 	xorps	 %xmm5, %xmm9

  255. 	xorps	 %xmm5, %xmm11

  256. 	xorps	 %xmm5, %xmm13

  257. 	xorps	 %xmm5, %xmm15

  258. #else

  259. 	xorps	 %xmm5, %xmm8

  260. 	xorps	 %xmm5, %xmm10

  261. 	xorps	 %xmm5, %xmm12

  262. 	xorps	 %xmm5, %xmm14

  263. #endif

  264. 

  265. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  266. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  267. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  268. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  269. 	ALIGN_3

  270. 

  271. 	movq	M,   I

  272. 	sarq	$2,  I

  273. 	jle	.L15

  274. 

  275. 	movddup	-16 * SIZE(A1), %xmm4

  276. 	movddup	-14 * SIZE(A1), %xmm5

  277. 	movddup	-12 * SIZE(A1), %xmm6

  278. 	movddup	-10 * SIZE(A1), %xmm7

  279. 

  280. 	decq	 I

  281. 	jle	 .L14

  282. 	ALIGN_3

  283. 

  284. .L13:

  285. #ifdef PREFETCH

  286. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1)

  287. #endif

  288. 

  289. 	mulpd	 %xmm8,  %xmm4

  290. 	addpd	 %xmm4,  %xmm0

  291. 	movddup	-15 * SIZE(A1), %xmm4

  292. 	mulpd	 %xmm8,  %xmm5

  293. 	addpd	 %xmm5,  %xmm1

  294. 	movddup	-13 * SIZE(A1), %xmm5

  295. 	mulpd	 %xmm8,  %xmm6

  296. 	addpd	 %xmm6,  %xmm2

  297. 	movddup	-11 * SIZE(A1), %xmm6

  298. 	mulpd	 %xmm8,  %xmm7

  299. 	addpd	 %xmm7,  %xmm3

  300. 	movddup	 -9 * SIZE(A1), %xmm7

  301. 

  302. 	mulpd	 %xmm9,  %xmm4

  303. 	SUBPD	 %xmm4,  %xmm0

  304. 	movddup	-16 * SIZE(A1, LDA), %xmm4

  305. 	mulpd	 %xmm9,  %xmm5

  306. 	SUBPD	 %xmm5,  %xmm1

  307. 	movddup	-14 * SIZE(A1, LDA), %xmm5

  308. 	mulpd	 %xmm9,  %xmm6

  309. 	SUBPD	 %xmm6,  %xmm2

  310. 	movddup	-12 * SIZE(A1, LDA), %xmm6

  311. 	mulpd	 %xmm9,  %xmm7

  312. 	SUBPD	 %xmm7,  %xmm3

  313. 	movddup	-10 * SIZE(A1, LDA), %xmm7

  314. 

  315. #ifdef PREFETCH

  316. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A1, LDA)

  317. #endif

  318. 

  319. 	mulpd	 %xmm10, %xmm4

  320. 	addpd	 %xmm4,  %xmm0

  321. 	movddup	-15 * SIZE(A1, LDA), %xmm4

  322. 	mulpd	 %xmm10, %xmm5

  323. 	addpd	 %xmm5,  %xmm1

  324. 	movddup	-13 * SIZE(A1, LDA), %xmm5

  325. 	mulpd	 %xmm10, %xmm6

  326. 	addpd	 %xmm6,  %xmm2

  327. 	movddup	-11 * SIZE(A1, LDA), %xmm6

  328. 	mulpd	 %xmm10, %xmm7

  329. 	addpd	 %xmm7,  %xmm3

  330. 	movddup	 -9 * SIZE(A1, LDA), %xmm7

  331. 

  332. 	mulpd	 %xmm11, %xmm4

  333. 	SUBPD	 %xmm4,  %xmm0

  334. 	movddup	-16 * SIZE(A2), %xmm4

  335. 	mulpd	 %xmm11, %xmm5

  336. 	SUBPD	 %xmm5,  %xmm1

  337. 	movddup	-14 * SIZE(A2), %xmm5

  338. 	mulpd	 %xmm11, %xmm6

  339. 	SUBPD	 %xmm6,  %xmm2

  340. 	movddup	-12 * SIZE(A2), %xmm6

  341. 	mulpd	 %xmm11, %xmm7

  342. 	SUBPD	 %xmm7,  %xmm3

  343. 	movddup	-10 * SIZE(A2), %xmm7

  344. 

  345. #ifdef PREFETCH

  346. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2)

  347. #endif

  348. 

  349. 	mulpd	 %xmm12, %xmm4

  350. 	addpd	 %xmm4,  %xmm0

  351. 	movddup	-15 * SIZE(A2), %xmm4

  352. 	mulpd	 %xmm12, %xmm5

  353. 	addpd	 %xmm5,  %xmm1

  354. 	movddup	-13 * SIZE(A2), %xmm5

  355. 	mulpd	 %xmm12, %xmm6

  356. 	addpd	 %xmm6,  %xmm2

  357. 	movddup	-11 * SIZE(A2), %xmm6

  358. 	mulpd	 %xmm12, %xmm7

  359. 	addpd	 %xmm7,  %xmm3

  360. 	movddup	 -9 * SIZE(A2), %xmm7

  361. 

  362. 	mulpd	 %xmm13, %xmm4

  363. 	SUBPD	 %xmm4,  %xmm0

  364. 	movddup	-16 * SIZE(A2, LDA), %xmm4

  365. 	mulpd	 %xmm13, %xmm5

  366. 	SUBPD	 %xmm5,  %xmm1

  367. 	movddup	-14 * SIZE(A2, LDA), %xmm5

  368. 	mulpd	 %xmm13, %xmm6

  369. 	SUBPD	 %xmm6,  %xmm2

  370. 	movddup	-12 * SIZE(A2, LDA), %xmm6

  371. 	mulpd	 %xmm13, %xmm7

  372. 	SUBPD	 %xmm7,  %xmm3

  373. 	movddup	-10 * SIZE(A2, LDA), %xmm7

  374. 

  375. #ifdef PREFETCH

  376. 	PREFETCH	(PREFETCHSIZE) - 128 + PREOFFSET(A2, LDA)

  377. #endif

  378. 

  379. 	mulpd	 %xmm14, %xmm4

  380. 	addpd	 %xmm4,  %xmm0

  381. 	movddup	-15 * SIZE(A2, LDA), %xmm4

  382. 	mulpd	 %xmm14, %xmm5

  383. 	addpd	 %xmm5,  %xmm1

  384. 	movddup	-13 * SIZE(A2, LDA), %xmm5

  385. 	mulpd	 %xmm14, %xmm6

  386. 	addpd	 %xmm6,  %xmm2

  387. 	movddup	-11 * SIZE(A2, LDA), %xmm6

  388. 	mulpd	 %xmm14, %xmm7

  389. 	addpd	 %xmm7,  %xmm3

  390. 	movddup	 -9 * SIZE(A2, LDA), %xmm7

  391. 

  392. 	mulpd	 %xmm15, %xmm4

  393. 	SUBPD	 %xmm4,  %xmm0

  394. 	movddup	 -8 * SIZE(A1), %xmm4

  395. 	mulpd	 %xmm15, %xmm5

  396. 	SUBPD	 %xmm5,  %xmm1

  397. 	movddup	 -6 * SIZE(A1), %xmm5

  398. 	mulpd	 %xmm15, %xmm6

  399. 	SUBPD	 %xmm6,  %xmm2

  400. 	movddup	 -4 * SIZE(A1), %xmm6

  401. 	mulpd	 %xmm15, %xmm7

  402. 	SUBPD	 %xmm7,  %xmm3

  403. 	movddup	 -2 * SIZE(A1), %xmm7

  404. 

  405. #ifdef PREFETCHW

  406. 	PREFETCHW	(PREFETCHSIZE) - 128 + PREOFFSET(Y1)

  407. #endif

  408. 

  409. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  410. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  411. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  412. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  413. 

  414. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  415. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  416. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  417. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  418. 

  419. 	subq	 $-8 * SIZE, A1

  420. 	subq	 $-8 * SIZE, A2

  421. 	subq	 $-8 * SIZE, Y1

  422. 

  423. 	subq	 $1, I

  424. 	BRANCH

  425. 	jg	.L13

  426. 	ALIGN_3

  427. 

  428. .L14:

  429. 	mulpd	 %xmm8,  %xmm4

  430. 	addpd	 %xmm4,  %xmm0

  431. 	movddup	-15 * SIZE(A1), %xmm4

  432. 	mulpd	 %xmm8,  %xmm5

  433. 	addpd	 %xmm5,  %xmm1

  434. 	movddup	-13 * SIZE(A1), %xmm5

  435. 	mulpd	 %xmm8,  %xmm6

  436. 	addpd	 %xmm6,  %xmm2

  437. 	movddup	-11 * SIZE(A1), %xmm6

  438. 	mulpd	 %xmm8,  %xmm7

  439. 	addpd	 %xmm7,  %xmm3

  440. 	movddup	 -9 * SIZE(A1), %xmm7

  441. 

  442. 	mulpd	 %xmm9,  %xmm4

  443. 	SUBPD	 %xmm4,  %xmm0

  444. 	movddup	-16 * SIZE(A1, LDA), %xmm4

  445. 	mulpd	 %xmm9,  %xmm5

  446. 	SUBPD	 %xmm5,  %xmm1

  447. 	movddup	-14 * SIZE(A1, LDA), %xmm5

  448. 	mulpd	 %xmm9,  %xmm6

  449. 	SUBPD	 %xmm6,  %xmm2

  450. 	movddup	-12 * SIZE(A1, LDA), %xmm6

  451. 	mulpd	 %xmm9,  %xmm7

  452. 	SUBPD	 %xmm7,  %xmm3

  453. 	movddup	-10 * SIZE(A1, LDA), %xmm7

  454. 

  455. 	mulpd	 %xmm10, %xmm4

  456. 	addpd	 %xmm4,  %xmm0

  457. 	movddup	-15 * SIZE(A1, LDA), %xmm4

  458. 	mulpd	 %xmm10, %xmm5

  459. 	addpd	 %xmm5,  %xmm1

  460. 	movddup	-13 * SIZE(A1, LDA), %xmm5

  461. 	mulpd	 %xmm10, %xmm6

  462. 	addpd	 %xmm6,  %xmm2

  463. 	movddup	-11 * SIZE(A1, LDA), %xmm6

  464. 	mulpd	 %xmm10, %xmm7

  465. 	addpd	 %xmm7,  %xmm3

  466. 	movddup	 -9 * SIZE(A1, LDA), %xmm7

  467. 

  468. 	mulpd	 %xmm11, %xmm4

  469. 	SUBPD	 %xmm4,  %xmm0

  470. 	movddup	-16 * SIZE(A2), %xmm4

  471. 	mulpd	 %xmm11, %xmm5

  472. 	SUBPD	 %xmm5,  %xmm1

  473. 	movddup	-14 * SIZE(A2), %xmm5

  474. 	mulpd	 %xmm11, %xmm6

  475. 	SUBPD	 %xmm6,  %xmm2

  476. 	movddup	-12 * SIZE(A2), %xmm6

  477. 	mulpd	 %xmm11, %xmm7

  478. 	SUBPD	 %xmm7,  %xmm3

  479. 	movddup	-10 * SIZE(A2), %xmm7

  480. 

  481. 	mulpd	 %xmm12, %xmm4

  482. 	addpd	 %xmm4,  %xmm0

  483. 	movddup	-15 * SIZE(A2), %xmm4

  484. 	mulpd	 %xmm12, %xmm5

  485. 	addpd	 %xmm5,  %xmm1

  486. 	movddup	-13 * SIZE(A2), %xmm5

  487. 	mulpd	 %xmm12, %xmm6

  488. 	addpd	 %xmm6,  %xmm2

  489. 	movddup	-11 * SIZE(A2), %xmm6

  490. 	mulpd	 %xmm12, %xmm7

  491. 	addpd	 %xmm7,  %xmm3

  492. 	movddup	 -9 * SIZE(A2), %xmm7

  493. 

  494. 	mulpd	 %xmm13, %xmm4

  495. 	SUBPD	 %xmm4,  %xmm0

  496. 	movddup	-16 * SIZE(A2, LDA), %xmm4

  497. 	mulpd	 %xmm13, %xmm5

  498. 	SUBPD	 %xmm5,  %xmm1

  499. 	movddup	-14 * SIZE(A2, LDA), %xmm5

  500. 	mulpd	 %xmm13, %xmm6

  501. 	SUBPD	 %xmm6,  %xmm2

  502. 	movddup	-12 * SIZE(A2, LDA), %xmm6

  503. 	mulpd	 %xmm13, %xmm7

  504. 	SUBPD	 %xmm7,  %xmm3

  505. 	movddup	-10 * SIZE(A2, LDA), %xmm7

  506. 

  507. 	mulpd	 %xmm14, %xmm4

  508. 	addpd	 %xmm4,  %xmm0

  509. 	movddup	-15 * SIZE(A2, LDA), %xmm4

  510. 	mulpd	 %xmm14, %xmm5

  511. 	addpd	 %xmm5,  %xmm1

  512. 	movddup	-13 * SIZE(A2, LDA), %xmm5

  513. 	mulpd	 %xmm14, %xmm6

  514. 	addpd	 %xmm6,  %xmm2

  515. 	movddup	-11 * SIZE(A2, LDA), %xmm6

  516. 	mulpd	 %xmm14, %xmm7

  517. 	addpd	 %xmm7,  %xmm3

  518. 	movddup	 -9 * SIZE(A2, LDA), %xmm7

  519. 

  520. 	mulpd	 %xmm15, %xmm4

  521. 	SUBPD	 %xmm4,  %xmm0

  522. 	mulpd	 %xmm15, %xmm5

  523. 	SUBPD	 %xmm5,  %xmm1

  524. 	mulpd	 %xmm15, %xmm6

  525. 	SUBPD	 %xmm6,  %xmm2

  526. 	mulpd	 %xmm15, %xmm7

  527. 	SUBPD	 %xmm7,  %xmm3

  528. 

  529. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  530. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  531. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  532. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  533. 

  534. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  535. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  536. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  537. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  538. 

  539. 	subq	 $-8 * SIZE, A1

  540. 	subq	 $-8 * SIZE, A2

  541. 	subq	 $-8 * SIZE, Y1

  542. 	ALIGN_3

  543. 

  544. .L15:

  545. 	testq	$2, M

  546. 	je	.L17

  547. 

  548. 	movddup	-16 * SIZE(A1), %xmm4

  549. 	movddup	-15 * SIZE(A1), %xmm5

  550. 	movddup	-14 * SIZE(A1), %xmm6

  551. 	movddup	-13 * SIZE(A1), %xmm7

  552. 

  553. 	mulpd	 %xmm8,  %xmm4

  554. 	addpd	 %xmm4,  %xmm0

  555. 	movddup	-16 * SIZE(A1, LDA, 1), %xmm4

  556. 	mulpd	 %xmm8,  %xmm6

  557. 	addpd	 %xmm6,  %xmm1

  558. 	movddup	-14 * SIZE(A1, LDA, 1), %xmm6

  559. 

  560. 	mulpd	 %xmm9,  %xmm5

  561. 	SUBPD	 %xmm5,  %xmm0

  562. 	movddup	-15 * SIZE(A1, LDA, 1), %xmm5

  563. 	mulpd	 %xmm9,  %xmm7

  564. 	SUBPD	 %xmm7,  %xmm1

  565. 	movddup	-13 * SIZE(A1, LDA, 1), %xmm7

  566. 

  567. 	mulpd	 %xmm10, %xmm4

  568. 	addpd	 %xmm4,  %xmm0

  569. 	movddup	-16 * SIZE(A2), %xmm4

  570. 	mulpd	 %xmm10, %xmm6

  571. 	addpd	 %xmm6,  %xmm1

  572. 	movddup	-14 * SIZE(A2), %xmm6

  573. 

  574. 	mulpd	 %xmm11, %xmm5

  575. 	SUBPD	 %xmm5,  %xmm0

  576. 	movddup	-15 * SIZE(A2), %xmm5

  577. 	mulpd	 %xmm11, %xmm7

  578. 	SUBPD	 %xmm7,  %xmm1

  579. 	movddup	-13 * SIZE(A2), %xmm7

  580. 

  581. 	mulpd	 %xmm12, %xmm4

  582. 	addpd	 %xmm4,  %xmm0

  583. 	movddup	-16 * SIZE(A2, LDA, 1), %xmm4

  584. 	mulpd	 %xmm12, %xmm6

  585. 	addpd	 %xmm6,  %xmm1

  586. 	movddup	-14 * SIZE(A2, LDA, 1), %xmm6

  587. 

  588. 	mulpd	 %xmm13, %xmm5

  589. 	SUBPD	 %xmm5,  %xmm0

  590. 	movddup	-15 * SIZE(A2, LDA, 1), %xmm5

  591. 	mulpd	 %xmm13, %xmm7

  592. 	SUBPD	 %xmm7,  %xmm1

  593. 	movddup	-13 * SIZE(A2, LDA, 1), %xmm7

  594. 

  595. 	mulpd	 %xmm14, %xmm4

  596. 	addpd	 %xmm4,  %xmm0

  597. 	mulpd	 %xmm14, %xmm6

  598. 	addpd	 %xmm6,  %xmm1

  599. 

  600. 	mulpd	 %xmm15, %xmm5

  601. 	SUBPD	 %xmm5,  %xmm0

  602. 	mulpd	 %xmm15, %xmm7

  603. 	SUBPD	 %xmm7,  %xmm1

  604. 

  605. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  606. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  607. 	movaps	 %xmm2, %xmm0

  608. 

  609. 	addq	 $4 * SIZE, A1

  610. 	addq	 $4 * SIZE, A2

  611. 	addq	 $4 * SIZE, Y1

  612. 	ALIGN_3

  613. 

  614. .L17:

  615. 	testq	$1, M

  616. 	je	.L19

  617. 

  618. 	movddup	-16 * SIZE(A1), %xmm4

  619. 	movddup	-15 * SIZE(A1), %xmm5

  620. 	movddup	-16 * SIZE(A1, LDA, 1), %xmm6

  621. 	movddup	-15 * SIZE(A1, LDA, 1), %xmm7

  622. 

  623. 	mulpd	 %xmm8,  %xmm4

  624. 	addpd	 %xmm4,  %xmm0

  625. 	movddup	-16 * SIZE(A2), %xmm4

  626. 	mulpd	 %xmm9,  %xmm5

  627. 	SUBPD	 %xmm5,  %xmm0

  628. 	movddup	-15 * SIZE(A2), %xmm5

  629. 

  630. 	mulpd	 %xmm10, %xmm6

  631. 	addpd	 %xmm6,  %xmm0

  632. 	movddup	-16 * SIZE(A2, LDA, 1), %xmm6

  633. 	mulpd	 %xmm11, %xmm7

  634. 	SUBPD	 %xmm7,  %xmm0

  635. 	movddup	-15 * SIZE(A2, LDA, 1), %xmm7

  636. 

  637. 	mulpd	 %xmm12, %xmm4

  638. 	addpd	 %xmm4,  %xmm0

  639. 	mulpd	 %xmm13, %xmm5

  640. 	SUBPD	 %xmm5,  %xmm0

  641. 

  642. 	mulpd	 %xmm14, %xmm6

  643. 	addpd	 %xmm6,  %xmm0

  644. 	mulpd	 %xmm15, %xmm7

  645. 	SUBPD	 %xmm7,  %xmm0

  646. 

  647. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  648. 	ALIGN_3

  649. 

  650. .L19:

  651. 	cmpq	$4, N

  652. 	jge	.L11

  653. 	ALIGN_3

  654. 

  655. .L20:

  656. #endif

  657. 

  658. #if GEMV_UNROLL >= 2

  659. 

  660. 	cmpq	$2, N

  661. 	jl	.L30

  662. 

  663. #if GEMV_UNROLL == 2

  664. 	ALIGN_3

  665. 

  666. .L21:

  667. #endif

  668. 

  669. 	subq	$2, N

  670. 

  671. 	leaq	16 * SIZE(BUFFER), Y1

  672. 	movq	A,  A1

  673. 	leaq	(A,  LDA, 1), A2

  674. 	leaq	(A,  LDA, 2), A

  675. 

  676. 	movddup	0 * SIZE(X), %xmm8

  677. 	movddup	1 * SIZE(X), %xmm9

  678. 	addq	INCX, X

  679. 	movddup	0 * SIZE(X), %xmm10

  680. 	movddup	1 * SIZE(X), %xmm11

  681. 	addq	INCX, X

  682. 

  683. 	pcmpeqb	%xmm5, %xmm5

  684. 	psllq	$63,   %xmm5

  685. 	shufps	$0x40, %xmm5, %xmm5

  686. 

  687. 	movsd	ALPHA_R, %xmm6

  688. 	movhps	ALPHA_I, %xmm6

  689. 

  690. 	pshufd	$0x4e, %xmm6,  %xmm7

  691. 

  692. #ifndef XCONJ

  693. 	xorps	 %xmm5, %xmm7

  694. #else

  695. 	xorps	 %xmm5, %xmm6

  696. #endif

  697. 

  698. 	mulpd	 %xmm6, %xmm8

  699. 	mulpd	 %xmm7, %xmm9

  700. 	mulpd	 %xmm6, %xmm10

  701. 	mulpd	 %xmm7, %xmm11

  702. 

  703. #ifndef XCONJ

  704. 	subpd	 %xmm9,  %xmm8

  705. 	subpd	 %xmm11, %xmm10

  706. #else

  707. 	addpd	 %xmm9,  %xmm8

  708. 	addpd	 %xmm11, %xmm10

  709. #endif

  710. 

  711. 	pshufd	 $0x4e, %xmm8,  %xmm9

  712. 	pshufd	 $0x4e, %xmm10, %xmm11

  713. 

  714. #ifndef XCONJ

  715. 	xorps	 %xmm5, %xmm9

  716. 	xorps	 %xmm5, %xmm11

  717. #else

  718. 	xorps	 %xmm5, %xmm8

  719. 	xorps	 %xmm5, %xmm10

  720. #endif

  721. 

  722. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  723. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  724. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  725. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  726. 

  727. 	movq	M,   I

  728. 	sarq	$2,  I

  729. 	jle	.L25

  730. 

  731. 	movddup	-16 * SIZE(A1), %xmm4

  732. 	movddup	-14 * SIZE(A1), %xmm5

  733. 	movddup	-12 * SIZE(A1), %xmm6

  734. 	movddup	-10 * SIZE(A1), %xmm7

  735. 

  736. 	decq	 I

  737. 	jle	 .L24

  738. 	ALIGN_3

  739. 

  740. .L23:

  741. #ifdef PREFETCH

  742. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A2)

  743. #endif

  744. 

  745. 	mulpd	 %xmm8,  %xmm4

  746. 	addpd	 %xmm4,  %xmm0

  747. 	movddup	-15 * SIZE(A1), %xmm4

  748. 	mulpd	 %xmm8,  %xmm5

  749. 	addpd	 %xmm5,  %xmm1

  750. 	movddup	-13 * SIZE(A1), %xmm5

  751. 	mulpd	 %xmm8,  %xmm6

  752. 	addpd	 %xmm6,  %xmm2

  753. 	movddup	-11 * SIZE(A1), %xmm6

  754. 	mulpd	 %xmm8,  %xmm7

  755. 	addpd	 %xmm7,  %xmm3

  756. 	movddup	 -9 * SIZE(A1), %xmm7

  757. 

  758. 	mulpd	 %xmm9,  %xmm4

  759. 	SUBPD	 %xmm4,  %xmm0

  760. 	movddup	-16 * SIZE(A2), %xmm4

  761. 	mulpd	 %xmm9,  %xmm5

  762. 	SUBPD	 %xmm5,  %xmm1

  763. 	movddup	-14 * SIZE(A2), %xmm5

  764. 	mulpd	 %xmm9,  %xmm6

  765. 	SUBPD	 %xmm6,  %xmm2

  766. 	movddup	-12 * SIZE(A2), %xmm6

  767. 	mulpd	 %xmm9,  %xmm7

  768. 	SUBPD	 %xmm7,  %xmm3

  769. 	movddup	-10 * SIZE(A2), %xmm7

  770. 

  771. #ifdef PREFETCH

  772. 	PREFETCH	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(A1)

  773. #endif

  774. 

  775. 	mulpd	 %xmm10, %xmm4

  776. 	addpd	 %xmm4,  %xmm0

  777. 	movddup	-15 * SIZE(A2), %xmm4

  778. 	mulpd	 %xmm10, %xmm5

  779. 	addpd	 %xmm5,  %xmm1

  780. 	movddup	-13 * SIZE(A2), %xmm5

  781. 	mulpd	 %xmm10, %xmm6

  782. 	addpd	 %xmm6,  %xmm2

  783. 	movddup	-11 * SIZE(A2), %xmm6

  784. 	mulpd	 %xmm10, %xmm7

  785. 	addpd	 %xmm7,  %xmm3

  786. 	movddup	 -9 * SIZE(A2), %xmm7

  787. 

  788. 	mulpd	 %xmm11, %xmm4

  789. 	SUBPD	 %xmm4,  %xmm0

  790. 	movddup	 -8 * SIZE(A1), %xmm4

  791. 	mulpd	 %xmm11, %xmm5

  792. 	SUBPD	 %xmm5,  %xmm1

  793. 	movddup	 -6 * SIZE(A1), %xmm5

  794. 	mulpd	 %xmm11, %xmm6

  795. 	SUBPD	 %xmm6,  %xmm2

  796. 	movddup	 -4 * SIZE(A1), %xmm6

  797. 	mulpd	 %xmm11, %xmm7

  798. 	SUBPD	 %xmm7,  %xmm3

  799. 	movddup	 -2 * SIZE(A1), %xmm7

  800. 

  801. #ifdef PREFETCHW

  802. 	PREFETCHW	(PREFETCHSIZE) * 2 - 128 + PREOFFSET(Y1)

  803. #endif

  804. 

  805. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  806. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  807. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  808. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  809. 

  810. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  811. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  812. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  813. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  814. 

  815. 	subq	 $-8 * SIZE, A1

  816. 	subq	 $-8 * SIZE, A2

  817. 	subq	 $-8 * SIZE, Y1

  818. 

  819. 	subq	 $1, I

  820. 	BRANCH

  821. 	jg	.L23

  822. 	ALIGN_3

  823. 

  824. .L24:

  825. 	mulpd	 %xmm8,  %xmm4

  826. 	addpd	 %xmm4,  %xmm0

  827. 	movddup	-15 * SIZE(A1), %xmm4

  828. 	mulpd	 %xmm8,  %xmm5

  829. 	addpd	 %xmm5,  %xmm1

  830. 	movddup	-13 * SIZE(A1), %xmm5

  831. 	mulpd	 %xmm8,  %xmm6

  832. 	addpd	 %xmm6,  %xmm2

  833. 	movddup	-11 * SIZE(A1), %xmm6

  834. 	mulpd	 %xmm8,  %xmm7

  835. 	addpd	 %xmm7,  %xmm3

  836. 	movddup	 -9 * SIZE(A1), %xmm7

  837. 

  838. 	mulpd	 %xmm9,  %xmm4

  839. 	SUBPD	 %xmm4,  %xmm0

  840. 	movddup	-16 * SIZE(A2), %xmm4

  841. 	mulpd	 %xmm9,  %xmm5

  842. 	SUBPD	 %xmm5,  %xmm1

  843. 	movddup	-14 * SIZE(A2), %xmm5

  844. 	mulpd	 %xmm9,  %xmm6

  845. 	SUBPD	 %xmm6,  %xmm2

  846. 	movddup	-12 * SIZE(A2), %xmm6

  847. 	mulpd	 %xmm9,  %xmm7

  848. 	SUBPD	 %xmm7,  %xmm3

  849. 	movddup	-10 * SIZE(A2), %xmm7

  850. 

  851. 	mulpd	 %xmm10, %xmm4

  852. 	addpd	 %xmm4,  %xmm0

  853. 	movddup	-15 * SIZE(A2), %xmm4

  854. 	mulpd	 %xmm10, %xmm5

  855. 	addpd	 %xmm5,  %xmm1

  856. 	movddup	-13 * SIZE(A2), %xmm5

  857. 	mulpd	 %xmm10, %xmm6

  858. 	addpd	 %xmm6,  %xmm2

  859. 	movddup	-11 * SIZE(A2), %xmm6

  860. 	mulpd	 %xmm10, %xmm7

  861. 	addpd	 %xmm7,  %xmm3

  862. 	movddup	 -9 * SIZE(A2), %xmm7

  863. 

  864. 	mulpd	 %xmm11, %xmm4

  865. 	SUBPD	 %xmm4,  %xmm0

  866. 	mulpd	 %xmm11, %xmm5

  867. 	SUBPD	 %xmm5,  %xmm1

  868. 	mulpd	 %xmm11, %xmm6

  869. 	SUBPD	 %xmm6,  %xmm2

  870. 	mulpd	 %xmm11, %xmm7

  871. 	SUBPD	 %xmm7,  %xmm3

  872. 

  873. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  874. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  875. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  876. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  877. 

  878. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  879. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  880. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  881. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  882. 

  883. 	subq	 $-8 * SIZE, A1

  884. 	subq	 $-8 * SIZE, A2

  885. 	subq	 $-8 * SIZE, Y1

  886. 	ALIGN_3

  887. 

  888. .L25:

  889. 	testq	$2, M

  890. 	je	.L27

  891. 

  892. 	movddup	-16 * SIZE(A1), %xmm4

  893. 	movddup	-15 * SIZE(A1), %xmm5

  894. 	movddup	-14 * SIZE(A1), %xmm6

  895. 	movddup	-13 * SIZE(A1), %xmm7

  896. 

  897. 	mulpd	 %xmm8,  %xmm4

  898. 	addpd	 %xmm4,  %xmm0

  899. 	movddup	-16 * SIZE(A2), %xmm4

  900. 	mulpd	 %xmm8,  %xmm6

  901. 	addpd	 %xmm6,  %xmm1

  902. 	movddup	-14 * SIZE(A2), %xmm6

  903. 

  904. 	mulpd	 %xmm9,  %xmm5

  905. 	SUBPD	 %xmm5,  %xmm0

  906. 	movddup	-15 * SIZE(A2), %xmm5

  907. 	mulpd	 %xmm9,  %xmm7

  908. 	SUBPD	 %xmm7,  %xmm1

  909. 	movddup	-13 * SIZE(A2), %xmm7

  910. 

  911. 	mulpd	 %xmm10, %xmm4

  912. 	addpd	 %xmm4,  %xmm0

  913. 	mulpd	 %xmm10, %xmm6

  914. 	addpd	 %xmm6,  %xmm1

  915. 

  916. 	mulpd	 %xmm11, %xmm5

  917. 	SUBPD	 %xmm5,  %xmm0

  918. 	mulpd	 %xmm11, %xmm7

  919. 	SUBPD	 %xmm7,  %xmm1

  920. 

  921. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  922. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  923. 	movaps	 %xmm2, %xmm0

  924. 

  925. 	addq	 $4 * SIZE, A1

  926. 	addq	 $4 * SIZE, A2

  927. 	addq	 $4 * SIZE, Y1

  928. 	ALIGN_3

  929. 

  930. .L27:

  931. 	testq	$1, M

  932. #if GEMV_UNROLL == 2

  933. 	je	.L29

  934. #else

  935. 	je	.L30

  936. #endif

  937. 

  938. 	movddup	-16 * SIZE(A1), %xmm4

  939. 	movddup	-15 * SIZE(A1), %xmm5

  940. 	movddup	-16 * SIZE(A2), %xmm6

  941. 	movddup	-15 * SIZE(A2), %xmm7

  942. 

  943. 	mulpd	 %xmm8,  %xmm4

  944. 	addpd	 %xmm4,  %xmm0

  945. 	mulpd	 %xmm9,  %xmm5

  946. 	SUBPD	 %xmm5,  %xmm0

  947. 

  948. 	mulpd	 %xmm10, %xmm6

  949. 	addpd	 %xmm6,  %xmm0

  950. 	mulpd	 %xmm11, %xmm7

  951. 	SUBPD	 %xmm7,  %xmm0

  952. 

  953. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  954. 

  955. #if GEMV_UNROLL == 2

  956. 	ALIGN_3

  957. 

  958. .L29:

  959. 	cmpq	$2, N

  960. 	jge	.L21

  961. #endif

  962. 	ALIGN_3

  963. 

  964. .L30:

  965. #endif

  966. 

  967. 	cmpq	$1, N

  968. 	jl	.L980

  969. 

  970. #if GEMV_UNROLL == 1

  971. .L31:

  972. 	decq	N

  973. #endif

  974. 

  975. 	leaq	16 * SIZE(BUFFER), Y1

  976. 	movq	A,  A1

  977. #if GEMV_UNROLL == 1

  978. 	addq	LDA, A

  979. #endif

  980. 

  981. 	movddup	0 * SIZE(X), %xmm8

  982. 	movddup	1 * SIZE(X), %xmm9

  983. 	addq	INCX, X

  984. 

  985. 	pcmpeqb	%xmm5, %xmm5

  986. 	psllq	$63,   %xmm5

  987. 	shufps	$0x40, %xmm5, %xmm5

  988. 

  989. 	movsd	ALPHA_R, %xmm6

  990. 	movhps	ALPHA_I, %xmm6

  991. 

  992. 	pshufd	$0x4e, %xmm6,  %xmm7

  993. 

  994. #ifndef XCONJ

  995. 	xorps	 %xmm5, %xmm7

  996. #else

  997. 	xorps	 %xmm5, %xmm6

  998. #endif

  999. 

  1000. 	mulpd	 %xmm6, %xmm8

  1001. 	mulpd	 %xmm7, %xmm9

  1002. 

  1003. #ifndef XCONJ

  1004. 	subpd	 %xmm9,  %xmm8

  1005. #else

  1006. 	addpd	 %xmm9,  %xmm8

  1007. #endif

  1008. 

  1009. 	pshufd	 $0x4e, %xmm8,  %xmm9

  1010. 

  1011. #ifndef XCONJ

  1012. 	xorps	 %xmm5, %xmm9

  1013. #else

  1014. 	xorps	 %xmm5, %xmm8

  1015. #endif

  1016. 

  1017. 	MOVUPS_YL1(-16 * SIZE, Y1, %xmm0)

  1018. 	MOVUPS_YL1(-14 * SIZE, Y1, %xmm1)

  1019. 	MOVUPS_YL1(-12 * SIZE, Y1, %xmm2)

  1020. 	MOVUPS_YL1(-10 * SIZE, Y1, %xmm3)

  1021. 

  1022. 	movq	M,   I

  1023. 	sarq	$2,  I

  1024. 	jle	.L35

  1025. 

  1026. 	movddup	-16 * SIZE(A1), %xmm4

  1027. 	movddup	-14 * SIZE(A1), %xmm5

  1028. 	movddup	-12 * SIZE(A1), %xmm6

  1029. 	movddup	-10 * SIZE(A1), %xmm7

  1030. 

  1031. 	decq	 I

  1032. 	jle	 .L34

  1033. 	ALIGN_3

  1034. 

  1035. .L33:

  1036. #ifdef PREFETCH

  1037. 	PREFETCH	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(A1)

  1038. #endif

  1039. 

  1040. 	mulpd	 %xmm8,  %xmm4

  1041. 	addpd	 %xmm4,  %xmm0

  1042. 	movddup	-15 * SIZE(A1), %xmm4

  1043. 	mulpd	 %xmm8,  %xmm5

  1044. 	addpd	 %xmm5,  %xmm1

  1045. 	movddup	-13 * SIZE(A1), %xmm5

  1046. 	mulpd	 %xmm8,  %xmm6

  1047. 	addpd	 %xmm6,  %xmm2

  1048. 	movddup	-11 * SIZE(A1), %xmm6

  1049. 	mulpd	 %xmm8,  %xmm7

  1050. 	addpd	 %xmm7,  %xmm3

  1051. 	movddup	 -9 * SIZE(A1), %xmm7

  1052. 

  1053. 	mulpd	 %xmm9,  %xmm4

  1054. 	SUBPD	 %xmm4,  %xmm0

  1055. 	movddup	 -8 * SIZE(A1), %xmm4

  1056. 	mulpd	 %xmm9,  %xmm5

  1057. 	SUBPD	 %xmm5,  %xmm1

  1058. 	movddup	 -6 * SIZE(A1), %xmm5

  1059. 	mulpd	 %xmm9,  %xmm6

  1060. 	SUBPD	 %xmm6,  %xmm2

  1061. 	movddup	 -4 * SIZE(A1), %xmm6

  1062. 	mulpd	 %xmm9,  %xmm7

  1063. 	SUBPD	 %xmm7,  %xmm3

  1064. 	movddup	 -2 * SIZE(A1), %xmm7

  1065. 

  1066. #ifdef PREFETCHW

  1067. 	PREFETCHW	(PREFETCHSIZE) * 4 - 128 + PREOFFSET(Y1)

  1068. #endif

  1069. 

  1070. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1071. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1072. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1073. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1074. 

  1075. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1076. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1077. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1078. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1079. 

  1080. 	subq	 $-8 * SIZE, A1

  1081. 	subq	 $-8 * SIZE, Y1

  1082. 

  1083. 	subq	 $1, I

  1084. 	BRANCH

  1085. 	jg	.L33

  1086. 	ALIGN_3

  1087. 

  1088. .L34:

  1089. 	mulpd	 %xmm8,  %xmm4

  1090. 	addpd	 %xmm4,  %xmm0

  1091. 	movddup	-15 * SIZE(A1), %xmm4

  1092. 	mulpd	 %xmm8,  %xmm5

  1093. 	addpd	 %xmm5,  %xmm1

  1094. 	movddup	-13 * SIZE(A1), %xmm5

  1095. 	mulpd	 %xmm8,  %xmm6

  1096. 	addpd	 %xmm6,  %xmm2

  1097. 	movddup	-11 * SIZE(A1), %xmm6

  1098. 	mulpd	 %xmm8,  %xmm7

  1099. 	addpd	 %xmm7,  %xmm3

  1100. 	movddup	 -9 * SIZE(A1), %xmm7

  1101. 

  1102. 	mulpd	 %xmm9,  %xmm4

  1103. 	SUBPD	 %xmm4,  %xmm0

  1104. 	mulpd	 %xmm9,  %xmm5

  1105. 	SUBPD	 %xmm5,  %xmm1

  1106. 	mulpd	 %xmm9,  %xmm6

  1107. 	SUBPD	 %xmm6,  %xmm2

  1108. 	mulpd	 %xmm9,  %xmm7

  1109. 	SUBPD	 %xmm7,  %xmm3

  1110. 

  1111. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1112. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1113. 	MOVUPS_YS1(-12 * SIZE, Y1, %xmm2)

  1114. 	MOVUPS_YS1(-10 * SIZE, Y1, %xmm3)

  1115. 

  1116. 	MOVUPS_YL1( -8 * SIZE, Y1, %xmm0)

  1117. 	MOVUPS_YL1( -6 * SIZE, Y1, %xmm1)

  1118. 	MOVUPS_YL1( -4 * SIZE, Y1, %xmm2)

  1119. 	MOVUPS_YL1( -2 * SIZE, Y1, %xmm3)

  1120. 

  1121. 	subq	 $-8 * SIZE, A1

  1122. 	subq	 $-8 * SIZE, Y1

  1123. 	ALIGN_3

  1124. 

  1125. .L35:

  1126. 	testq	$2, M

  1127. 	je	.L37

  1128. 

  1129. 	movddup	-16 * SIZE(A1), %xmm4

  1130. 	movddup	-15 * SIZE(A1), %xmm5

  1131. 	movddup	-14 * SIZE(A1), %xmm6

  1132. 	movddup	-13 * SIZE(A1), %xmm7

  1133. 

  1134. 	mulpd	 %xmm8,  %xmm4

  1135. 	addpd	 %xmm4,  %xmm0

  1136. 	mulpd	 %xmm8,  %xmm6

  1137. 	addpd	 %xmm6,  %xmm1

  1138. 

  1139. 	mulpd	 %xmm9,  %xmm5

  1140. 	SUBPD	 %xmm5,  %xmm0

  1141. 	mulpd	 %xmm9,  %xmm7

  1142. 	SUBPD	 %xmm7,  %xmm1

  1143. 

  1144. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1145. 	MOVUPS_YS1(-14 * SIZE, Y1, %xmm1)

  1146. 	movaps	 %xmm2, %xmm0

  1147. 

  1148. 	addq	 $4 * SIZE, A1

  1149. 	addq	 $4 * SIZE, Y1

  1150. 	ALIGN_3

  1151. 

  1152. .L37:

  1153. 	testq	$1, M

  1154. #if GEMV_UNROLL == 1

  1155. 	je	.L39

  1156. #else

  1157. 	je	.L980

  1158. #endif

  1159. 

  1160. 	movddup	-16 * SIZE(A1), %xmm4

  1161. 	movddup	-15 * SIZE(A1), %xmm5

  1162. 

  1163. 	mulpd	 %xmm8,  %xmm4

  1164. 	addpd	 %xmm4,  %xmm0

  1165. 	mulpd	 %xmm9,  %xmm5

  1166. 	SUBPD	 %xmm5,  %xmm0

  1167. 

  1168. 	MOVUPS_YS1(-16 * SIZE, Y1, %xmm0)

  1169. 

  1170. #if GEMV_UNROLL == 1

  1171. 	ALIGN_3

  1172. .L39:

  1173. 	cmpq	$1, N

  1174. 	jge	.L31

  1175. #endif

  1176. 

  1177. .L980:

  1178. 	testq	$SIZE, Y

  1179. 	jne	.L990

  1180. 

  1181. 	movq	Y,  Y1

  1182. 

  1183. 	movq	M,  %rax

  1184. 	sarq	$3, %rax

  1185. 	jle	.L184

  1186. 	ALIGN_3

  1187. 

  1188. .L182:

  1189.  	movaps	 (Y), %xmm0

  1190. 	addq	INCY, Y

  1191. 	movaps	 (Y), %xmm1

  1192. 	addq	INCY, Y

  1193. 	movaps	 (Y), %xmm2

  1194. 	addq	INCY, Y

  1195. 	movaps	 (Y), %xmm3

  1196. 	addq	INCY, Y

  1197.  	movaps	 (Y), %xmm4

  1198. 	addq	INCY, Y

  1199. 	movaps	 (Y), %xmm5

  1200. 	addq	INCY, Y

  1201. 	movaps	 (Y), %xmm6

  1202. 	addq	INCY, Y

  1203. 	movaps	 (Y), %xmm7

  1204. 	addq	INCY, Y

  1205. 

  1206. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1207. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1208. 	addpd	 4 * SIZE(BUFFER), %xmm2

  1209. 	addpd	 6 * SIZE(BUFFER), %xmm3

  1210. 	addpd	 8 * SIZE(BUFFER), %xmm4

  1211. 	addpd	10 * SIZE(BUFFER), %xmm5

  1212. 	addpd	12 * SIZE(BUFFER), %xmm6

  1213. 	addpd	14 * SIZE(BUFFER), %xmm7

  1214. 

  1215. 	movaps	%xmm0,  (Y1)

  1216. 	addq	INCY, Y1

  1217. 	movaps	%xmm1,  (Y1)

  1218. 	addq	INCY, Y1

  1219. 	movaps	%xmm2,  (Y1)

  1220. 	addq	INCY, Y1

  1221. 	movaps	%xmm3,  (Y1)

  1222. 	addq	INCY, Y1

  1223. 	movaps	%xmm4,  (Y1)

  1224. 	addq	INCY, Y1

  1225. 	movaps	%xmm5,  (Y1)

  1226. 	addq	INCY, Y1

  1227. 	movaps	%xmm6,  (Y1)

  1228. 	addq	INCY, Y1

  1229. 	movaps	%xmm7,  (Y1)

  1230. 	addq	INCY, Y1

  1231. 

  1232. 	subq	$-16 * SIZE, BUFFER

  1233. 	decq	%rax

  1234. 	jg	.L182

  1235. 	ALIGN_3

  1236. 

  1237. .L184:

  1238. 	testq	$7, M

  1239. 	jle	.L999

  1240. 

  1241. 	testq	$4, M

  1242. 	jle	.L185

  1243. 

  1244.  	movaps	 (Y), %xmm0

  1245. 	addq	INCY, Y

  1246. 	movaps	 (Y), %xmm1

  1247. 	addq	INCY, Y

  1248. 	movaps	 (Y), %xmm2

  1249. 	addq	INCY, Y

  1250. 	movaps	 (Y), %xmm3

  1251. 	addq	INCY, Y

  1252. 

  1253. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1254. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1255. 	addpd	 4 * SIZE(BUFFER), %xmm2

  1256. 	addpd	 6 * SIZE(BUFFER), %xmm3

  1257. 

  1258. 	movaps	%xmm0,  (Y1)

  1259. 	addq	INCY, Y1

  1260. 	movaps	%xmm1,  (Y1)

  1261. 	addq	INCY, Y1

  1262. 	movaps	%xmm2,  (Y1)

  1263. 	addq	INCY, Y1

  1264. 	movaps	%xmm3,  (Y1)

  1265. 	addq	INCY, Y1

  1266. 

  1267. 	addq	$8 * SIZE, BUFFER

  1268. 	ALIGN_3

  1269. 

  1270. .L185:

  1271. 	testq	$2, M

  1272. 	jle	.L186

  1273. 

  1274.  	movaps	 (Y), %xmm0

  1275. 	addq	INCY, Y

  1276. 	movaps	 (Y), %xmm1

  1277. 	addq	INCY, Y

  1278. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1279. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1280. 

  1281. 	movaps	%xmm0,  (Y1)

  1282. 	addq	INCY, Y1

  1283. 	movaps	%xmm1,  (Y1)

  1284. 	addq	INCY, Y1

  1285. 

  1286. 	addq	$4 * SIZE, BUFFER

  1287. 	ALIGN_3

  1288. 

  1289. .L186:

  1290. 	testq	$1, M

  1291. 	jle	.L999

  1292. 

  1293.  	movaps	 (Y), %xmm0

  1294. 

  1295. 	addpd	 (BUFFER), %xmm0

  1296. 

  1297. 	movaps	%xmm0,  (Y1)

  1298. 	jmp	.L999

  1299. 	ALIGN_3

  1300. 

  1301. .L990:

  1302. 	movq	Y,  Y1

  1303. 

  1304. 	movq	M,  %rax

  1305. 	sarq	$3, %rax

  1306. 	jle	.L994

  1307. 	ALIGN_3

  1308. 

  1309. .L992:

  1310.  	movsd	 0 * SIZE(Y), %xmm0

  1311. 	movhpd	 1 * SIZE(Y), %xmm0

  1312. 	addq	INCY, Y

  1313. 

  1314. 	movsd	 0 * SIZE(Y), %xmm1

  1315. 	movhpd	 1 * SIZE(Y), %xmm1

  1316. 	addq	INCY, Y

  1317. 

  1318. 	movsd	 0 * SIZE(Y), %xmm2

  1319. 	movhpd	 1 * SIZE(Y), %xmm2

  1320. 	addq	INCY, Y

  1321. 

  1322. 	movsd	 0 * SIZE(Y), %xmm3

  1323. 	movhpd	 1 * SIZE(Y), %xmm3

  1324. 	addq	INCY, Y

  1325. 

  1326.  	movsd	 0 * SIZE(Y), %xmm4

  1327. 	movhpd	 1 * SIZE(Y), %xmm4

  1328. 	addq	INCY, Y

  1329. 

  1330. 	movsd	 0 * SIZE(Y), %xmm5

  1331. 	movhpd	 1 * SIZE(Y), %xmm5

  1332. 	addq	INCY, Y

  1333. 

  1334. 	movsd	 0 * SIZE(Y), %xmm6

  1335. 	movhpd	 1 * SIZE(Y), %xmm6

  1336. 	addq	INCY, Y

  1337. 

  1338. 	movsd	 0 * SIZE(Y), %xmm7

  1339. 	movhpd	 1 * SIZE(Y), %xmm7

  1340. 	addq	INCY, Y

  1341. 

  1342. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1343. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1344. 	addpd	 4 * SIZE(BUFFER), %xmm2

  1345. 	addpd	 6 * SIZE(BUFFER), %xmm3

  1346. 	addpd	 8 * SIZE(BUFFER), %xmm4

  1347. 	addpd	10 * SIZE(BUFFER), %xmm5

  1348. 	addpd	12 * SIZE(BUFFER), %xmm6

  1349. 	addpd	14 * SIZE(BUFFER), %xmm7

  1350. 

  1351. 	movlpd	%xmm0,  0 * SIZE(Y1)

  1352. 	movhpd	%xmm0,  1 * SIZE(Y1)

  1353. 	addq	INCY, Y1

  1354. 

  1355. 	movlpd	%xmm1,  0 * SIZE(Y1)

  1356. 	movhpd	%xmm1,  1 * SIZE(Y1)

  1357. 	addq	INCY, Y1

  1358. 

  1359. 	movlpd	%xmm2,  0 * SIZE(Y1)

  1360. 	movhpd	%xmm2,  1 * SIZE(Y1)

  1361. 	addq	INCY, Y1

  1362. 

  1363. 	movlpd	%xmm3,  0 * SIZE(Y1)

  1364. 	movhpd	%xmm3,  1 * SIZE(Y1)

  1365. 	addq	INCY, Y1

  1366. 

  1367. 	movlpd	%xmm4,  0 * SIZE(Y1)

  1368. 	movhpd	%xmm4,  1 * SIZE(Y1)

  1369. 	addq	INCY, Y1

  1370. 

  1371. 	movlpd	%xmm5,  0 * SIZE(Y1)

  1372. 	movhpd	%xmm5,  1 * SIZE(Y1)

  1373. 	addq	INCY, Y1

  1374. 

  1375. 	movlpd	%xmm6,  0 * SIZE(Y1)

  1376. 	movhpd	%xmm6,  1 * SIZE(Y1)

  1377. 	addq	INCY, Y1

  1378. 

  1379. 	movlpd	%xmm7,  0 * SIZE(Y1)

  1380. 	movhpd	%xmm7,  1 * SIZE(Y1)

  1381. 	addq	INCY, Y1

  1382. 

  1383. 	subq	$-16 * SIZE, BUFFER

  1384. 	decq	%rax

  1385. 	jg	.L992

  1386. 	ALIGN_3

  1387. 

  1388. .L994:

  1389. 	testq	$7, M

  1390. 	jle	.L999

  1391. 

  1392. 	testq	$4, M

  1393. 	jle	.L995

  1394. 

  1395.  	movsd	 0 * SIZE(Y), %xmm0

  1396. 	movhpd	 1 * SIZE(Y), %xmm0

  1397. 	addq	INCY, Y

  1398. 

  1399. 	movsd	 0 * SIZE(Y), %xmm1

  1400. 	movhpd	 1 * SIZE(Y), %xmm1

  1401. 	addq	INCY, Y

  1402. 

  1403. 	movsd	 0 * SIZE(Y), %xmm2

  1404. 	movhpd	 1 * SIZE(Y), %xmm2

  1405. 	addq	INCY, Y

  1406. 

  1407. 	movsd	 0 * SIZE(Y), %xmm3

  1408. 	movhpd	 1 * SIZE(Y), %xmm3

  1409. 	addq	INCY, Y

  1410. 

  1411. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1412. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1413. 	addpd	 4 * SIZE(BUFFER), %xmm2

  1414. 	addpd	 6 * SIZE(BUFFER), %xmm3

  1415. 

  1416. 	movlpd	%xmm0,  0 * SIZE(Y1)

  1417. 	movhpd	%xmm0,  1 * SIZE(Y1)

  1418. 	addq	INCY, Y1

  1419. 

  1420. 	movlpd	%xmm1,  0 * SIZE(Y1)

  1421. 	movhpd	%xmm1,  1 * SIZE(Y1)

  1422. 	addq	INCY, Y1

  1423. 

  1424. 	movlpd	%xmm2,  0 * SIZE(Y1)

  1425. 	movhpd	%xmm2,  1 * SIZE(Y1)

  1426. 	addq	INCY, Y1

  1427. 

  1428. 	movlpd	%xmm3,  0 * SIZE(Y1)

  1429. 	movhpd	%xmm3,  1 * SIZE(Y1)

  1430. 	addq	INCY, Y1

  1431. 

  1432. 	addq	$8 * SIZE, BUFFER

  1433. 	ALIGN_3

  1434. 

  1435. .L995:

  1436. 	testq	$2, M

  1437. 	jle	.L996

  1438. 

  1439.  	movsd	 0 * SIZE(Y), %xmm0

  1440. 	movhpd	 1 * SIZE(Y), %xmm0

  1441. 	addq	INCY, Y

  1442. 

  1443. 	movsd	 0 * SIZE(Y), %xmm1

  1444. 	movhpd	 1 * SIZE(Y), %xmm1

  1445. 	addq	INCY, Y

  1446. 

  1447. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1448. 	addpd	 2 * SIZE(BUFFER), %xmm1

  1449. 

  1450. 	movlpd	%xmm0,  0 * SIZE(Y1)

  1451. 	movhpd	%xmm0,  1 * SIZE(Y1)

  1452. 	addq	INCY, Y1

  1453. 

  1454. 	movlpd	%xmm1,  0 * SIZE(Y1)

  1455. 	movhpd	%xmm1,  1 * SIZE(Y1)

  1456. 	addq	INCY, Y1

  1457. 

  1458. 	addq	$4 * SIZE, BUFFER

  1459. 	ALIGN_3

  1460. 

  1461. .L996:

  1462. 	testq	$1, M

  1463. 	jle	.L999

  1464. 

  1465.  	movsd	 0 * SIZE(Y), %xmm0

  1466. 	movhpd	 1 * SIZE(Y), %xmm0

  1467. 

  1468. 	addpd	 0 * SIZE(BUFFER), %xmm0

  1469. 

  1470. 	movlpd	%xmm0,  0 * SIZE(Y1)

  1471. 	movhpd	%xmm0,  1 * SIZE(Y1)

  1472. 	ALIGN_3

  1473. 

  1474. .L999:

  1475. 	movq	  0(%rsp), %rbx

  1476. 	movq	  8(%rsp), %rbp

  1477. 	movq	 16(%rsp), %r12

  1478. 	movq	 24(%rsp), %r13

  1479. 	movq	 32(%rsp), %r14

  1480. 	movq	 40(%rsp), %r15

  1481. 

  1482. #ifdef WINDOWS_ABI

  1483. 	movq	 48(%rsp), %rdi

  1484. 	movq	 56(%rsp), %rsi

  1485. 	movups	 64(%rsp), %xmm6

  1486. 	movups	 80(%rsp), %xmm7

  1487. 	movups	 96(%rsp), %xmm8

  1488. 	movups	112(%rsp), %xmm9

  1489. 	movups	128(%rsp), %xmm10

  1490. 	movups	144(%rsp), %xmm11

  1491. 	movups	160(%rsp), %xmm12

  1492. 	movups	176(%rsp), %xmm13

  1493. 	movups	192(%rsp), %xmm14

  1494. 	movups	208(%rsp), %xmm15

  1495. #endif

  1496. 

  1497. 	addq	$STACKSIZE, %rsp

  1498. 	ret

  1499. 

  1500. 	EPILOGUE