OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 57ed58cefe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '57ed58cefe'
${ noResults }
OpenBLAS/kernel/x86/gemv_t_sse2.S

618 lines
12 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef PENTIUM4

  43. #define PREFETCH	prefetcht0

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCHSIZE	(8 * 2)

  46. #endif

  47. 

  48. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON) || defined(NEHALEM) || defined(SANDYBRIDGE)

  49. #define PREFETCH	prefetcht0

  50. #define PREFETCHW	prefetcht0

  51. #define PREFETCHSIZE	(8 * 7)

  52. #endif

  53. 

  54. #ifdef OPTERON

  55. #define PREFETCH	prefetchnta

  56. #define PREFETCHW	prefetchw

  57. #define PREFETCHSIZE	(8 * 3)

  58. #define movsd		movlps

  59. #endif

  60. 

  61. #ifdef BARCELONA

  62. #define PREFETCH	prefetchnta

  63. #define PREFETCHW	prefetchw

  64. #define PREFETCHSIZE	(8 * 5)

  65. #endif

  66. 

  67. #ifdef ATOM

  68. #define PREFETCH	prefetch

  69. #define PREFETCHW	prefetcht0

  70. #define PREFETCHSIZE	(8 * 6)

  71. #endif

  72. 

  73. #ifdef NANO

  74. #define PREFETCH	prefetcht0

  75. #define PREFETCHSIZE	(8 * 4)

  76. #endif

  77. 

  78. #define STACKSIZE	16

  79. #define ARGS	20

  80. 

  81. #define M		 4 + STACKSIZE+ARGS(%esp)

  82. #define N		 8 + STACKSIZE+ARGS(%esp)

  83. #define ALPHA		16 + STACKSIZE+ARGS(%esp)

  84. #define A		24 + STACKSIZE+ARGS(%esp)

  85. #define STACK_LDA	28 + STACKSIZE+ARGS(%esp)

  86. #define STACK_X		32 + STACKSIZE+ARGS(%esp)

  87. #define STACK_INCX	36 + STACKSIZE+ARGS(%esp)

  88. #define Y		40 + STACKSIZE+ARGS(%esp)

  89. #define STACK_INCY	44 + STACKSIZE+ARGS(%esp)

  90. #define BUFFER		48 + STACKSIZE+ARGS(%esp)

  91. 

  92. #define MMM	0+ARGS(%esp)

  93. #define AA	4+ARGS(%esp)

  94. #define XX 8+ARGS(%esp)

  95. 

  96. #define I	%eax

  97. #define J	%ebx

  98. 

  99. #define INCX	J

  100. #define INCY	%ecx

  101. 

  102. #define A1	%esi

  103. #define X	%edx

  104. #define Y1	%edi

  105. #define LDA	%ebp

  106. 

  107. 	PROLOGUE

  108. 

  109. 	subl	$ARGS,%esp

  110. 

  111. 	pushl	%ebp

  112. 	pushl	%edi

  113. 	pushl	%esi

  114. 	pushl	%ebx

  115. 

  116. 	PROFCODE

  117. 

  118. 

  119. 	movl	STACK_X,  X

  120. 	movl	X,XX

  121. 	movl	A,J

  122. 	movl	J,AA				# backup A

  123.     movl	M,J

  124. 	movl	J,MMM				# mov M to MMM

  125. .L0t:

  126. 	xorl	J,J

  127. 	addl	$1,J

  128. 	sall    $21,J                           # J=2^21*sizeof(double)=buffer size(16MB)

  129. 	subl    $4, J                           # Don't use last 4 double in the buffer.

  130. 	subl	J,MMM				# MMM=MMM-J

  131. 	movl	J,M

  132. 	jge		.L00t

  133. 	ALIGN_4

  134. 

  135. 	movl	MMM,%eax

  136. 	addl	J,%eax

  137. 	jle		.L999x

  138. 	movl	%eax,M

  139. 

  140. .L00t:

  141. 	movl	XX,%eax

  142. 	movl	%eax, X

  143. 

  144. 	movl	AA,%eax

  145. 	movl	%eax,A			 	# mov AA to A

  146. 

  147. 	movl	STACK_LDA,  LDA

  148. 	movl	STACK_INCX, INCX

  149. 	movl	STACK_INCY, INCY

  150. 

  151. 	leal	(,INCX, SIZE), INCX

  152. 	leal	(,INCY, SIZE), INCY

  153. 	leal	(,LDA,  SIZE), LDA

  154. 

  155. 

  156. 	subl	$-16 * SIZE, A

  157. 

  158. 	cmpl	$0, N

  159. 	jle	.L999

  160. 	cmpl	$0, M

  161. 	jle	.L999

  162. 

  163. 	movl	BUFFER, Y1

  164. 

  165. 	movl	M,  I

  166. 	sarl	$3, I

  167. 	jle	.L05

  168. 	ALIGN_4

  169. 

  170. .L02:

  171. 	movsd	(X), %xmm0

  172. 	addl	INCX, X

  173. 	movhpd	(X), %xmm0

  174. 	addl	INCX, X

  175. 

  176. 	movsd	(X), %xmm1

  177. 	addl	INCX, X

  178. 	movhpd	(X), %xmm1

  179. 	addl	INCX, X

  180. 

  181. 	movsd	(X), %xmm2

  182. 	addl	INCX, X

  183. 	movhpd	(X), %xmm2

  184. 	addl	INCX, X

  185. 

  186. 	movsd	(X), %xmm3

  187. 	addl	INCX, X

  188. 	movhpd	(X), %xmm3

  189. 	addl	INCX, X

  190. 

  191. 	movapd	%xmm0, 0 * SIZE(Y1)

  192. 	movapd	%xmm1, 2 * SIZE(Y1)

  193. 	movapd	%xmm2, 4 * SIZE(Y1)

  194. 	movapd	%xmm3, 6 * SIZE(Y1)

  195. 

  196. 	addl	$8 * SIZE, Y1

  197. 	decl	I

  198. 	jg	.L02

  199. 	ALIGN_4

  200. 

  201. .L05:

  202. 	movl	M,  I

  203. 	andl	$7, I

  204. 	jle	.L10

  205. 	ALIGN_2

  206. 

  207. .L06:

  208. 	movsd	(X), %xmm0

  209. 	addl	INCX, X

  210. 	movsd	%xmm0, 0 * SIZE(Y1)

  211. 	addl	$SIZE, Y1

  212. 	decl	I

  213. 	jg	.L06

  214. 	ALIGN_4

  215. 

  216. .L10:

  217. 	movl	Y, Y1

  218. 

  219. 	movl	N,  J

  220. 	sarl	$1, J

  221. 	jle	.L20

  222. 	ALIGN_3

  223. 

  224. .L11:

  225. 	movl	BUFFER, X

  226. 	addl	$16 * SIZE, X

  227. 

  228. 	movl	A, A1

  229. 	leal	(A1, LDA, 2), %eax

  230. 	movl	%eax, A

  231. 

  232. 	xorps	%xmm0, %xmm0

  233. 	xorps	%xmm1, %xmm1

  234. 

  235. 	movapd	-16 * SIZE(X), %xmm2

  236. 	movapd	-14 * SIZE(X), %xmm3

  237. 

  238. 	movl	M,   I

  239. 	sarl	$3,  I

  240. 	jle	.L15

  241. 

  242. 	movsd	-16 * SIZE(A1), %xmm4

  243. 	movhpd	-15 * SIZE(A1), %xmm4

  244. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  245. 	movhpd	-15 * SIZE(A1, LDA), %xmm5

  246. 

  247. 	movsd	-14 * SIZE(A1), %xmm6

  248. 	movhpd	-13 * SIZE(A1), %xmm6

  249. 	movsd	-14 * SIZE(A1, LDA), %xmm7

  250. 	movhpd	-13 * SIZE(A1, LDA), %xmm7

  251. 

  252. 	decl	I

  253. 	jle	.L13

  254. 	ALIGN_4

  255. 

  256. .L12:

  257. #ifdef PREFETCH

  258. 	PREFETCH PREFETCHSIZE * SIZE(A1)

  259. #endif

  260. 

  261. 	mulpd	%xmm2, %xmm4

  262. 	addpd	%xmm4, %xmm0

  263. 	movsd	-12 * SIZE(A1), %xmm4

  264. 	movhpd	-11 * SIZE(A1), %xmm4

  265. 	mulpd	%xmm2, %xmm5

  266. 	movapd	-12 * SIZE(X), %xmm2

  267. 	addpd	%xmm5, %xmm1

  268. 	movsd	-12 * SIZE(A1, LDA), %xmm5

  269. 	movhpd	-11 * SIZE(A1, LDA), %xmm5

  270. 

  271. 	mulpd	%xmm3, %xmm6

  272. 	addpd	%xmm6, %xmm0

  273. 	movsd	-10 * SIZE(A1), %xmm6

  274. 	movhpd	 -9 * SIZE(A1), %xmm6

  275. 	mulpd	%xmm3, %xmm7

  276. 	movapd	-10 * SIZE(X), %xmm3

  277. 	addpd	%xmm7, %xmm1

  278. 	movsd	-10 * SIZE(A1, LDA), %xmm7

  279. 	movhpd	 -9 * SIZE(A1, LDA), %xmm7

  280. 

  281. #ifdef PREFETCH

  282. 	PREFETCH PREFETCHSIZE * SIZE(A1, LDA)

  283. #endif

  284. 

  285. 	mulpd	%xmm2, %xmm4

  286. 	addpd	%xmm4, %xmm0

  287. 	movsd	 -8 * SIZE(A1), %xmm4

  288. 	movhpd	 -7 * SIZE(A1), %xmm4

  289. 	mulpd	%xmm2, %xmm5

  290. 	movapd	 -8 * SIZE(X), %xmm2

  291. 	addpd	%xmm5, %xmm1

  292. 	movsd	 -8 * SIZE(A1, LDA), %xmm5

  293. 	movhpd	 -7 * SIZE(A1, LDA), %xmm5

  294. 

  295. 	mulpd	%xmm3, %xmm6

  296. 	addpd	%xmm6, %xmm0

  297. 	movsd	 -6 * SIZE(A1), %xmm6

  298. 	movhpd	 -5 * SIZE(A1), %xmm6

  299. 	mulpd	%xmm3, %xmm7

  300. 	movapd	 -6 * SIZE(X), %xmm3

  301. 	addpd	%xmm7, %xmm1

  302. 	movsd	 -6 * SIZE(A1, LDA), %xmm7

  303. 	movhpd	 -5 * SIZE(A1, LDA), %xmm7

  304. 

  305. 	addl	$8 * SIZE, A1

  306. 	addl	$8 * SIZE, X

  307. 

  308. 	decl	I

  309. 	jg	.L12

  310. 	ALIGN_4

  311. 

  312. .L13:

  313. 	mulpd	%xmm2, %xmm4

  314. 	addpd	%xmm4, %xmm0

  315. 	movsd	-12 * SIZE(A1), %xmm4

  316. 	movhpd	-11 * SIZE(A1), %xmm4

  317. 	mulpd	%xmm2, %xmm5

  318. 	movapd	-12 * SIZE(X), %xmm2

  319. 	addpd	%xmm5, %xmm1

  320. 	movsd	-12 * SIZE(A1, LDA), %xmm5

  321. 	movhpd	-11 * SIZE(A1, LDA), %xmm5

  322. 

  323. 	mulpd	%xmm3, %xmm6

  324. 	addpd	%xmm6, %xmm0

  325. 	movsd	-10 * SIZE(A1), %xmm6

  326. 	movhpd	 -9 * SIZE(A1), %xmm6

  327. 	mulpd	%xmm3, %xmm7

  328. 	movapd	-10 * SIZE(X), %xmm3

  329. 	addpd	%xmm7, %xmm1

  330. 	movsd	-10 * SIZE(A1, LDA), %xmm7

  331. 	movhpd	 -9 * SIZE(A1, LDA), %xmm7

  332. 

  333. 	mulpd	%xmm2, %xmm4

  334. 	addpd	%xmm4, %xmm0

  335. 	mulpd	%xmm2, %xmm5

  336. 	movapd	 -8 * SIZE(X), %xmm2

  337. 	addpd	%xmm5, %xmm1

  338. 

  339. 	mulpd	%xmm3, %xmm6

  340. 	addpd	%xmm6, %xmm0

  341. 	mulpd	%xmm3, %xmm7

  342. 	movapd	 -6 * SIZE(X), %xmm3

  343. 	addpd	%xmm7, %xmm1

  344. 

  345. 	addl	$8 * SIZE, A1

  346. 	addl	$8 * SIZE, X

  347. 	ALIGN_4

  348. 

  349. .L15:

  350. 	testl	$4, M

  351. 	jle	.L16

  352. 

  353. 	movsd	-16 * SIZE(A1), %xmm4

  354. 	movhpd	-15 * SIZE(A1), %xmm4

  355. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  356. 	movhpd	-15 * SIZE(A1, LDA), %xmm5

  357. 

  358. 	movsd	-14 * SIZE(A1), %xmm6

  359. 	movhpd	-13 * SIZE(A1), %xmm6

  360. 	movsd	-14 * SIZE(A1, LDA), %xmm7

  361. 	movhpd	-13 * SIZE(A1, LDA), %xmm7

  362. 

  363. 	mulpd	%xmm2, %xmm4

  364. 	addpd	%xmm4, %xmm0

  365. 	mulpd	%xmm2, %xmm5

  366. 	movapd	-12 * SIZE(X), %xmm2

  367. 	addpd	%xmm5, %xmm1

  368. 

  369. 	mulpd	%xmm3, %xmm6

  370. 	addpd	%xmm6, %xmm0

  371. 	mulpd	%xmm3, %xmm7

  372. 	movapd	-10 * SIZE(X), %xmm3

  373. 	addpd	%xmm7, %xmm1

  374. 

  375. 	addl	$4 * SIZE, A1

  376. 	addl	$4 * SIZE, X

  377. 	ALIGN_4

  378. 

  379. .L16:

  380. 	testl	$2, M

  381. 	jle	.L17

  382. 

  383. 	movsd	-16 * SIZE(A1), %xmm4

  384. 	movhpd	-15 * SIZE(A1), %xmm4

  385. 

  386. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  387. 	movhpd	-15 * SIZE(A1, LDA), %xmm5

  388. 

  389. 	mulpd	%xmm2, %xmm4

  390. 	addpd	%xmm4, %xmm0

  391. 	mulpd	%xmm2, %xmm5

  392. 	addpd	%xmm5, %xmm1

  393. 	movapd	%xmm3, %xmm2

  394. 

  395. 	addl	$2 * SIZE, A1

  396. 	ALIGN_4

  397. 

  398. .L17:

  399. 	testl	$1, M

  400. 	jle	.L18

  401. 

  402. 	movsd	-16 * SIZE(A1), %xmm4

  403. 	mulsd	%xmm2, %xmm4

  404. 	addsd	%xmm4, %xmm0

  405. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  406. 	mulsd	%xmm2, %xmm5

  407. 	addsd	%xmm5, %xmm1

  408. 	ALIGN_4

  409. 

  410. .L18:

  411. #ifdef HAVE_SSE3

  412. 	haddpd	%xmm1, %xmm0

  413. #else

  414. 	movapd	%xmm0, %xmm2

  415. 	unpcklpd %xmm1, %xmm0

  416. 	unpckhpd %xmm1, %xmm2

  417. 

  418. 	addpd	%xmm2, %xmm0

  419. #endif

  420. 

  421. #ifdef HAVE_SSE3

  422.        movddup	ALPHA, %xmm7

  423. #else

  424.        movsd	ALPHA, %xmm7

  425.        unpcklpd	%xmm7, %xmm7

  426. #endif

  427. 

  428. 	mulpd	%xmm7, %xmm0

  429. 

  430. 	movsd	(Y1), %xmm4

  431. 	movhpd	(Y1, INCY), %xmm4

  432. 

  433. 	addpd	%xmm4, %xmm0

  434. 

  435. 	movlpd	%xmm0, (Y1)

  436. 	movhpd	%xmm0, (Y1, INCY)

  437. 	leal	(Y1, INCY, 2), Y1

  438. 

  439. 	decl	J

  440. 	jg	.L11

  441. 	ALIGN_4

  442. 

  443. .L20:

  444. 	testl	$1, N

  445. 	jle	.L999

  446. 

  447. 	movl	BUFFER, X

  448. 	addl	$16 * SIZE, X

  449. 

  450. 	movl	A, A1

  451. 	leal	(A1, LDA, 2), %eax

  452. 	movl	%eax, A

  453. 

  454. 	xorps	%xmm0, %xmm0

  455. 	xorps	%xmm1, %xmm1

  456. 

  457. 	movapd	-16 * SIZE(X), %xmm2

  458. 	movapd	-14 * SIZE(X), %xmm3

  459. 

  460. 	movl	M,   I

  461. 	sarl	$3,  I

  462. 	jle	.L25

  463. 

  464. 	movsd	-16 * SIZE(A1), %xmm4

  465. 	movhpd	-15 * SIZE(A1), %xmm4

  466. 	movsd	-14 * SIZE(A1), %xmm6

  467. 	movhpd	-13 * SIZE(A1), %xmm6

  468. 

  469. 	decl	I

  470. 	jle	.L23

  471. 	ALIGN_4

  472. 

  473. .L22:

  474. #ifdef PREFETCH

  475. 	PREFETCH PREFETCHSIZE * SIZE(A1)

  476. #endif

  477. 

  478. 	mulpd	%xmm2, %xmm4

  479. 	movapd	-12 * SIZE(X), %xmm2

  480. 	addpd	%xmm4, %xmm0

  481. 	movsd	-12 * SIZE(A1), %xmm4

  482. 	movhpd	-11 * SIZE(A1), %xmm4

  483. 

  484. 	mulpd	%xmm3, %xmm6

  485. 	movapd	-10 * SIZE(X), %xmm3

  486. 	addpd	%xmm6, %xmm0

  487. 	movsd	-10 * SIZE(A1), %xmm6

  488. 	movhpd	 -9 * SIZE(A1), %xmm6

  489. 

  490. 	mulpd	%xmm2, %xmm4

  491. 	movapd	 -8 * SIZE(X), %xmm2

  492. 	addpd	%xmm4, %xmm0

  493. 	movsd	 -8 * SIZE(A1), %xmm4

  494. 	movhpd	 -7 * SIZE(A1), %xmm4

  495. 

  496. 	mulpd	%xmm3, %xmm6

  497. 	movapd	 -6 * SIZE(X), %xmm3

  498. 	addpd	%xmm6, %xmm0

  499. 	movsd	 -6 * SIZE(A1), %xmm6

  500. 	movhpd	 -5 * SIZE(A1), %xmm6

  501. 

  502. 	addl	$8 * SIZE, A1

  503. 	addl	$8 * SIZE, X

  504. 

  505. 	decl	I

  506. 	jg	.L22

  507. 	ALIGN_4

  508. 

  509. .L23:

  510. 	mulpd	%xmm2, %xmm4

  511. 	movapd	-12 * SIZE(X), %xmm2

  512. 	addpd	%xmm4, %xmm0

  513. 	movsd	-12 * SIZE(A1), %xmm4

  514. 	movhpd	-11 * SIZE(A1), %xmm4

  515. 

  516. 	mulpd	%xmm3, %xmm6

  517. 	movapd	-10 * SIZE(X), %xmm3

  518. 	addpd	%xmm6, %xmm0

  519. 	movsd	-10 * SIZE(A1), %xmm6

  520. 	movhpd	 -9 * SIZE(A1), %xmm6

  521. 

  522. 	mulpd	%xmm2, %xmm4

  523. 	movapd	 -8 * SIZE(X), %xmm2

  524. 	addpd	%xmm4, %xmm0

  525. 

  526. 	mulpd	%xmm3, %xmm6

  527. 	movapd	 -6 * SIZE(X), %xmm3

  528. 	addpd	%xmm6, %xmm0

  529. 

  530. 	addl	$8 * SIZE, A1

  531. 	addl	$8 * SIZE, X

  532. 	ALIGN_4

  533. 

  534. .L25:

  535. 	testl	$4, M

  536. 	jle	.L26

  537. 

  538. 	movsd	-16 * SIZE(A1), %xmm4

  539. 	movhpd	-15 * SIZE(A1), %xmm4

  540. 	movsd	-14 * SIZE(A1), %xmm6

  541. 	movhpd	-13 * SIZE(A1), %xmm6

  542. 

  543. 	mulpd	%xmm2, %xmm4

  544. 	movapd	-12 * SIZE(X), %xmm2

  545. 	addpd	%xmm4, %xmm0

  546. 

  547. 	mulpd	%xmm3, %xmm6

  548. 	movapd	-10 * SIZE(X), %xmm3

  549. 	addpd	%xmm6, %xmm0

  550. 

  551. 	addl	$4 * SIZE, A1

  552. 	addl	$4 * SIZE, X

  553. 	ALIGN_4

  554. 

  555. .L26:

  556. 	testl	$2, M

  557. 	jle	.L27

  558. 

  559. 	movsd	-16 * SIZE(A1), %xmm4

  560. 	movhpd	-15 * SIZE(A1), %xmm4

  561. 

  562. 	mulpd	%xmm2, %xmm4

  563. 	addpd	%xmm4, %xmm0

  564. 	movapd	%xmm3, %xmm2

  565. 

  566. 	addl	$2 * SIZE, A1

  567. 	ALIGN_4

  568. 

  569. .L27:

  570. 	testl	$1, M

  571. 	jle	.L28

  572. 

  573. 	movsd	-16 * SIZE(A1), %xmm4

  574. 	mulsd	%xmm2, %xmm4

  575. 	addsd	%xmm4, %xmm0

  576. 	ALIGN_4

  577. 

  578. .L28:

  579. #ifdef HAVE_SSE3

  580. 	haddpd	%xmm1, %xmm0

  581. #else

  582. 	movapd	%xmm0, %xmm2

  583. 	unpcklpd %xmm1, %xmm0

  584. 	unpckhpd %xmm1, %xmm2

  585. 

  586. 	addsd	%xmm2, %xmm0

  587. #endif

  588. 

  589. 	movsd	ALPHA, %xmm7

  590. 

  591. 	mulpd	%xmm7, %xmm0

  592. 

  593. 	addsd	(Y1),  %xmm0

  594. 

  595. 	movlpd	%xmm0, (Y1)

  596. 	ALIGN_4

  597. 

  598. .L999:

  599. 	movl 	M,J

  600. 	leal 	(,J,SIZE),%eax

  601. 	addl	%eax,AA

  602. 	movl	STACK_INCX,INCX

  603. 	imull	INCX,%eax

  604. 	addl	%eax,XX

  605. 	jmp		.L0t

  606. 	ALIGN_4

  607. 

  608. .L999x:

  609. 	popl	%ebx

  610. 	popl	%esi

  611. 	popl	%edi

  612. 	popl	%ebp

  613. 

  614. 	addl	$ARGS,%esp

  615. 	ret

  616. 

  617. 	EPILOGUE