OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 3d6d026fe1
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3d6d026fe1'
${ noResults }
OpenBLAS/kernel/x86/gemv_t_atom.S

617 lines
12 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef ATOM

  43. #define PREFETCH	prefetchnta

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCHSIZE	(8 * 6)

  46. #endif

  47. 

  48. #define STACKSIZE	16

  49. 

  50. #define M		 4 + STACKSIZE(%esp)

  51. #define N		 8 + STACKSIZE(%esp)

  52. #define ALPHA		16 + STACKSIZE(%esp)

  53. #define A		24 + STACKSIZE(%esp)

  54. #define STACK_LDA	28 + STACKSIZE(%esp)

  55. #define STACK_X		32 + STACKSIZE(%esp)

  56. #define STACK_INCX	36 + STACKSIZE(%esp)

  57. #define Y		40 + STACKSIZE(%esp)

  58. #define STACK_INCY	44 + STACKSIZE(%esp)

  59. #define BUFFER		48 + STACKSIZE(%esp)

  60. 

  61. #define I	%eax

  62. #define J	%ebx

  63. 

  64. #define INCX	J

  65. #define INCY	%ecx

  66. 

  67. #define A1	%esi

  68. #define X	%edx

  69. #define Y1	%edi

  70. #define LDA	%ebp

  71. 

  72. 	PROLOGUE

  73. 

  74. 	pushl	%ebp

  75. 	pushl	%edi

  76. 	pushl	%esi

  77. 	pushl	%ebx

  78. 

  79. 	PROFCODE

  80. 

  81. 	movl	STACK_LDA,  LDA

  82. 	movl	STACK_X,    X

  83. 	movl	STACK_INCX, INCX

  84. 	movl	STACK_INCY, INCY

  85. 

  86. 	leal	(,INCX, SIZE), INCX

  87. 	leal	(,INCY, SIZE), INCY

  88. 	leal	(,LDA,  SIZE), LDA

  89. 

  90. 	subl	$-16 * SIZE, A

  91. 

  92. 	cmpl	$0, N

  93. 	jle	.L999

  94. 	cmpl	$0, M

  95. 	jle	.L999

  96. 

  97. 	movl	BUFFER, Y1

  98. 

  99. 	movl	M,  I

  100. 	sarl	$3, I

  101. 	jle	.L05

  102. 	ALIGN_4

  103. 

  104. .L02:

  105. 	movsd	(X), %xmm0

  106. 	addl	INCX, X

  107. 	movhpd	(X), %xmm0

  108. 	addl	INCX, X

  109. 

  110. 	movsd	(X), %xmm1

  111. 	addl	INCX, X

  112. 	movhpd	(X), %xmm1

  113. 	addl	INCX, X

  114. 

  115. 	movsd	(X), %xmm2

  116. 	addl	INCX, X

  117. 	movhpd	(X), %xmm2

  118. 	addl	INCX, X

  119. 

  120. 	movsd	(X), %xmm3

  121. 	addl	INCX, X

  122. 	movhpd	(X), %xmm3

  123. 	addl	INCX, X

  124. 

  125. 	movapd	%xmm0, 0 * SIZE(Y1)

  126. 	movapd	%xmm1, 2 * SIZE(Y1)

  127. 	movapd	%xmm2, 4 * SIZE(Y1)

  128. 	movapd	%xmm3, 6 * SIZE(Y1)

  129. 

  130. 	addl	$8 * SIZE, Y1

  131. 	decl	I

  132. 	jg	.L02

  133. 	ALIGN_4

  134. 

  135. .L05:

  136. 	movl	M,  I

  137. 	andl	$7, I

  138. 	jle	.L10

  139. 	ALIGN_2

  140. 

  141. .L06:

  142. 	movsd	(X), %xmm0

  143. 	addl	INCX, X

  144. 	movsd	%xmm0, 0 * SIZE(Y1)

  145. 	addl	$SIZE, Y1

  146. 	decl	I

  147. 	jg	.L06

  148. 	ALIGN_4

  149. 

  150. .L10:

  151. 	movl	Y, Y1

  152. 

  153. 	movl	N,  J

  154. 	sarl	$1, J

  155. 	jle	.L20

  156. 	ALIGN_3

  157. 

  158. .L11:

  159. 	movl	BUFFER, X

  160. 	addl	$16 * SIZE, X

  161. 

  162. 	movl	A, A1

  163. 	leal	(A1, LDA, 2), %eax

  164. 	movl	%eax, A

  165. 

  166. 	xorps	%xmm0, %xmm0

  167. 	xorps	%xmm1, %xmm1

  168. 

  169. 	movsd	-16 * SIZE(X), %xmm2

  170. 	movsd	-15 * SIZE(X), %xmm3

  171. 

  172. 	movl	M,   I

  173. 	sarl	$3,  I

  174. 	jle	.L15

  175. 

  176. 	movsd	-16 * SIZE(A1), %xmm4

  177. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  178. 	movsd	-15 * SIZE(A1), %xmm6

  179. 	movsd	-15 * SIZE(A1, LDA), %xmm7

  180. 

  181. 	mulsd	%xmm2, %xmm4

  182. 	mulsd	%xmm2, %xmm5

  183. 	movsd	-14 * SIZE(X), %xmm2

  184. 

  185. 	decl	I

  186. 	jle	.L13

  187. 	ALIGN_4

  188. 

  189. .L12:

  190. #ifdef PREFETCH

  191. 	PREFETCH PREFETCHSIZE * SIZE(A1)

  192. #endif

  193. 

  194. 	mulsd	%xmm3, %xmm6

  195. 	addsd	%xmm4, %xmm0

  196. 	movsd	-14 * SIZE(A1), %xmm4

  197. 	mulsd	%xmm3, %xmm7

  198. 	movsd	-13 * SIZE(X), %xmm3

  199. 	addsd	%xmm5, %xmm1

  200. 	movsd	-14 * SIZE(A1, LDA), %xmm5

  201. 

  202. 	mulsd	%xmm2, %xmm4

  203. 	addsd	%xmm6, %xmm0

  204. 	movsd	-13 * SIZE(A1), %xmm6

  205. 	mulsd	%xmm2, %xmm5

  206. 	movsd	-12 * SIZE(X), %xmm2

  207. 	addsd	%xmm7, %xmm1

  208. 	movsd	-13 * SIZE(A1, LDA), %xmm7

  209. 

  210. 	mulsd	%xmm3, %xmm6

  211. 	addsd	%xmm4, %xmm0

  212. 	movsd	-12 * SIZE(A1), %xmm4

  213. 	mulsd	%xmm3, %xmm7

  214. 	movsd	-11 * SIZE(X), %xmm3

  215. 	addsd	%xmm5, %xmm1

  216. 	movsd	-12 * SIZE(A1, LDA), %xmm5

  217. 

  218. 	mulsd	%xmm2, %xmm4

  219. 	addsd	%xmm6, %xmm0

  220. 	movsd	-11 * SIZE(A1), %xmm6

  221. 	mulsd	%xmm2, %xmm5

  222. 	movsd	-10 * SIZE(X), %xmm2

  223. 	addsd	%xmm7, %xmm1

  224. 	movsd	-11 * SIZE(A1, LDA), %xmm7

  225. 

  226. #ifdef PREFETCH

  227. 	PREFETCH PREFETCHSIZE * SIZE(A1, LDA)

  228. #endif

  229. 

  230. 	mulsd	%xmm3, %xmm6

  231. 	addsd	%xmm4, %xmm0

  232. 	movsd	-10 * SIZE(A1), %xmm4

  233. 	mulsd	%xmm3, %xmm7

  234. 	movsd	 -9 * SIZE(X), %xmm3

  235. 	addsd	%xmm5, %xmm1

  236. 	movsd	-10 * SIZE(A1, LDA), %xmm5

  237. 

  238. 	mulsd	%xmm2, %xmm4

  239. 	addsd	%xmm6, %xmm0

  240. 	movsd	 -9 * SIZE(A1), %xmm6

  241. 	mulsd	%xmm2, %xmm5

  242. 	movsd	 -8 * SIZE(X), %xmm2

  243. 	addsd	%xmm7, %xmm1

  244. 	movsd	 -9 * SIZE(A1, LDA), %xmm7

  245. 

  246. 	mulsd	%xmm3, %xmm6

  247. 	addsd	%xmm4, %xmm0

  248. 	movsd	 -8 * SIZE(A1), %xmm4

  249. 	mulsd	%xmm3, %xmm7

  250. 	movsd	 -7 * SIZE(X), %xmm3

  251. 	addsd	%xmm5, %xmm1

  252. 	movsd	 -8 * SIZE(A1, LDA), %xmm5

  253. 

  254. 	mulsd	%xmm2, %xmm4

  255. 	addsd	%xmm6, %xmm0

  256. 	movsd	 -7 * SIZE(A1), %xmm6

  257. 	mulsd	%xmm2, %xmm5

  258. 	movsd	 -6 * SIZE(X), %xmm2

  259. 	addsd	%xmm7, %xmm1

  260. 	movsd	 -7 * SIZE(A1, LDA), %xmm7

  261. 

  262. 	addl	$8 * SIZE, A1

  263. 	addl	$8 * SIZE, X

  264. 

  265. 	decl	I

  266. 	jg	.L12

  267. 	ALIGN_4

  268. 

  269. .L13:

  270. 	mulsd	%xmm3, %xmm6

  271. 	addsd	%xmm4, %xmm0

  272. 	movsd	-14 * SIZE(A1), %xmm4

  273. 	mulsd	%xmm3, %xmm7

  274. 	movsd	-13 * SIZE(X), %xmm3

  275. 	addsd	%xmm5, %xmm1

  276. 	movsd	-14 * SIZE(A1, LDA), %xmm5

  277. 

  278. 	mulsd	%xmm2, %xmm4

  279. 	addsd	%xmm6, %xmm0

  280. 	movsd	-13 * SIZE(A1), %xmm6

  281. 	mulsd	%xmm2, %xmm5

  282. 	movsd	-12 * SIZE(X), %xmm2

  283. 	addsd	%xmm7, %xmm1

  284. 	movsd	-13 * SIZE(A1, LDA), %xmm7

  285. 

  286. 	mulsd	%xmm3, %xmm6

  287. 	addsd	%xmm4, %xmm0

  288. 	movsd	-12 * SIZE(A1), %xmm4

  289. 	mulsd	%xmm3, %xmm7

  290. 	movsd	-11 * SIZE(X), %xmm3

  291. 	addsd	%xmm5, %xmm1

  292. 	movsd	-12 * SIZE(A1, LDA), %xmm5

  293. 

  294. 	mulsd	%xmm2, %xmm4

  295. 	addsd	%xmm6, %xmm0

  296. 	movsd	-11 * SIZE(A1), %xmm6

  297. 	mulsd	%xmm2, %xmm5

  298. 	movsd	-10 * SIZE(X), %xmm2

  299. 	addsd	%xmm7, %xmm1

  300. 	movsd	-11 * SIZE(A1, LDA), %xmm7

  301. 

  302. 	mulsd	%xmm3, %xmm6

  303. 	addsd	%xmm4, %xmm0

  304. 	movsd	-10 * SIZE(A1), %xmm4

  305. 	mulsd	%xmm3, %xmm7

  306. 	movsd	 -9 * SIZE(X), %xmm3

  307. 	addsd	%xmm5, %xmm1

  308. 	movsd	-10 * SIZE(A1, LDA), %xmm5

  309. 

  310. 	mulsd	%xmm2, %xmm4

  311. 	addsd	%xmm6, %xmm0

  312. 	movsd	 -9 * SIZE(A1), %xmm6

  313. 	mulsd	%xmm2, %xmm5

  314. 	movsd	 -8 * SIZE(X), %xmm2

  315. 	addsd	%xmm7, %xmm1

  316. 	movsd	 -9 * SIZE(A1, LDA), %xmm7

  317. 

  318. 	mulsd	%xmm3, %xmm6

  319. 	addsd	%xmm4, %xmm0

  320. 	mulsd	%xmm3, %xmm7

  321. 	movsd	 -7 * SIZE(X), %xmm3

  322. 	addsd	%xmm5, %xmm1

  323. 

  324. 	addsd	%xmm6, %xmm0

  325. 	addl	$8 * SIZE, A1

  326. 	addsd	%xmm7, %xmm1

  327. 	addl	$8 * SIZE, X

  328. 	ALIGN_4

  329. 

  330. .L15:

  331. 	testl	$4, M

  332. 	jle	.L16

  333. 

  334. 	movsd	-16 * SIZE(A1), %xmm4

  335. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  336. 

  337. 	movsd	-15 * SIZE(A1), %xmm6

  338. 	movsd	-15 * SIZE(A1, LDA), %xmm7

  339. 

  340. 	mulsd	%xmm2, %xmm4

  341. 	mulsd	%xmm2, %xmm5

  342. 	movsd	-14 * SIZE(X), %xmm2

  343. 

  344. 	mulsd	%xmm3, %xmm6

  345. 	addsd	%xmm4, %xmm0

  346. 	movsd	-14 * SIZE(A1), %xmm4

  347. 	mulsd	%xmm3, %xmm7

  348. 	movsd	-13 * SIZE(X), %xmm3

  349. 	addsd	%xmm5, %xmm1

  350. 	movsd	-14 * SIZE(A1, LDA), %xmm5

  351. 

  352. 	mulsd	%xmm2, %xmm4

  353. 	addsd	%xmm6, %xmm0

  354. 	movsd	-13 * SIZE(A1), %xmm6

  355. 	mulsd	%xmm2, %xmm5

  356. 	movsd	-12 * SIZE(X), %xmm2

  357. 	addsd	%xmm7, %xmm1

  358. 	movsd	-13 * SIZE(A1, LDA), %xmm7

  359. 

  360. 	mulsd	%xmm3, %xmm6

  361. 	addsd	%xmm4, %xmm0

  362. 	mulsd	%xmm3, %xmm7

  363. 	movsd	-11 * SIZE(X), %xmm3

  364. 	addsd	%xmm5, %xmm1

  365. 

  366. 	addsd	%xmm6, %xmm0

  367. 	addsd	%xmm7, %xmm1

  368. 

  369. 	addl	$4 * SIZE, A1

  370. 	addl	$4 * SIZE, X

  371. 	ALIGN_4

  372. 

  373. .L16:

  374. 	testl	$2, M

  375. 	jle	.L17

  376. 

  377. 	movsd	-16 * SIZE(A1), %xmm4

  378. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  379. 

  380. 	movsd	-15 * SIZE(A1), %xmm6

  381. 	movsd	-15 * SIZE(A1, LDA), %xmm7

  382. 

  383. 	mulsd	%xmm2, %xmm4

  384. 	mulsd	%xmm2, %xmm5

  385. 	movsd	-14 * SIZE(X), %xmm2

  386. 

  387. 	mulsd	%xmm3, %xmm6

  388. 	addsd	%xmm4, %xmm0

  389. 	mulsd	%xmm3, %xmm7

  390. 	addsd	%xmm5, %xmm1

  391. 

  392. 	addsd	%xmm6, %xmm0

  393. 	addsd	%xmm7, %xmm1

  394. 

  395. 	addl	$2 * SIZE, A1

  396. 	ALIGN_4

  397. 

  398. .L17:

  399. 	testl	$1, M

  400. 	jle	.L18

  401. 

  402. 	movsd	-16 * SIZE(A1), %xmm4

  403. 	movsd	-16 * SIZE(A1, LDA), %xmm5

  404. 

  405. 	mulsd	%xmm2, %xmm4

  406. 	mulsd	%xmm2, %xmm5

  407. 

  408. 	addsd	%xmm4, %xmm0

  409. 	addsd	%xmm5, %xmm1

  410. 	ALIGN_4

  411. 

  412. .L18:

  413. 	movsd	ALPHA, %xmm7

  414. 

  415. 	mulpd	%xmm7, %xmm0

  416. 	mulpd	%xmm7, %xmm1

  417. 

  418. 	addsd	(Y1), %xmm0

  419. 	addsd	(Y1, INCY), %xmm1

  420. 

  421. 	movsd	%xmm0, (Y1)

  422. 	movsd	%xmm1, (Y1, INCY)

  423. 	leal	(Y1, INCY, 2), Y1

  424. 

  425. 	decl	J

  426. 	jg	.L11

  427. 	ALIGN_4

  428. 

  429. .L20:

  430. 	testl	$1, N

  431. 	jle	.L999

  432. 

  433. 	movl	BUFFER, X

  434. 	addl	$16 * SIZE, X

  435. 

  436. 	movl	A, A1

  437. 	leal	(A1, LDA, 2), %eax

  438. 	movl	%eax, A

  439. 

  440. 	xorps	%xmm0, %xmm0

  441. 	xorps	%xmm1, %xmm1

  442. 

  443. 	movsd	-16 * SIZE(X), %xmm2

  444. 	movsd	-15 * SIZE(X), %xmm3

  445. 

  446. 	movl	M,   I

  447. 	sarl	$3,  I

  448. 	jle	.L25

  449. 

  450. 	movsd	-16 * SIZE(A1), %xmm4

  451. 	movsd	-15 * SIZE(A1), %xmm5

  452. 	movsd	-14 * SIZE(A1), %xmm6

  453. 	movsd	-13 * SIZE(A1), %xmm7

  454. 

  455. 	mulsd	%xmm2, %xmm4

  456. 	movsd	-14 * SIZE(X), %xmm2

  457. 	mulsd	%xmm3, %xmm5

  458. 	movsd	-13 * SIZE(X), %xmm3

  459. 

  460. 	decl	I

  461. 	jle	.L23

  462. 	ALIGN_4

  463. 

  464. .L22:

  465. #ifdef PREFETCH

  466. 	PREFETCH PREFETCHSIZE * SIZE(A1)

  467. #endif

  468. 

  469. 	mulsd	%xmm2, %xmm6

  470. 	movsd	-12 * SIZE(X), %xmm2

  471. 	addsd	%xmm4, %xmm0

  472. 	movsd	-12 * SIZE(A1), %xmm4

  473. 	mulsd	%xmm3, %xmm7

  474. 	movsd	-11 * SIZE(X), %xmm3

  475. 	addsd	%xmm5, %xmm1

  476. 	movsd	-11 * SIZE(A1), %xmm5

  477. 

  478. 	addsd	%xmm6, %xmm0

  479. 	movsd	-10 * SIZE(A1), %xmm6

  480. 	mulsd	%xmm2, %xmm4

  481. 	movsd	-10 * SIZE(X), %xmm2

  482. 	addsd	%xmm7, %xmm1

  483. 	movsd	 -9 * SIZE(A1), %xmm7

  484. 	mulsd	%xmm3, %xmm5

  485. 	movsd	 -9 * SIZE(X), %xmm3

  486. 

  487. 	mulsd	%xmm2, %xmm6

  488. 	movsd	 -8 * SIZE(X), %xmm2

  489. 	addsd	%xmm4, %xmm0

  490. 	movsd	 -8 * SIZE(A1), %xmm4

  491. 	mulsd	%xmm3, %xmm7

  492. 	movsd	 -7 * SIZE(X), %xmm3

  493. 	addsd	%xmm5, %xmm1

  494. 	movsd	 -7 * SIZE(A1), %xmm5

  495. 

  496. 	addsd	%xmm6, %xmm0

  497. 	movsd	 -6 * SIZE(A1), %xmm6

  498. 	mulsd	%xmm2, %xmm4

  499. 	movsd	 -6 * SIZE(X), %xmm2

  500. 	addsd	%xmm7, %xmm1

  501. 	movsd	 -5 * SIZE(A1), %xmm7

  502. 	mulsd	%xmm3, %xmm5

  503. 	movsd	 -5 * SIZE(X), %xmm3

  504. 

  505. 	addl	$8 * SIZE, A1

  506. 	addl	$8 * SIZE, X

  507. 

  508. 	decl	I

  509. 	jg	.L22

  510. 	ALIGN_4

  511. 

  512. .L23:

  513. 	mulsd	%xmm2, %xmm6

  514. 	movsd	-12 * SIZE(X), %xmm2

  515. 	addsd	%xmm4, %xmm0

  516. 	movsd	-12 * SIZE(A1), %xmm4

  517. 	mulsd	%xmm3, %xmm7

  518. 	movsd	-11 * SIZE(X), %xmm3

  519. 	addsd	%xmm5, %xmm1

  520. 	movsd	-11 * SIZE(A1), %xmm5

  521. 

  522. 	addsd	%xmm6, %xmm0

  523. 	movsd	-10 * SIZE(A1), %xmm6

  524. 	mulsd	%xmm2, %xmm4

  525. 	movsd	-10 * SIZE(X), %xmm2

  526. 	addsd	%xmm7, %xmm1

  527. 	movsd	 -9 * SIZE(A1), %xmm7

  528. 	mulsd	%xmm3, %xmm5

  529. 	movsd	 -9 * SIZE(X), %xmm3

  530. 

  531. 	mulsd	%xmm2, %xmm6

  532. 	movsd	 -8 * SIZE(X), %xmm2

  533. 	addsd	%xmm4, %xmm0

  534. 	mulsd	%xmm3, %xmm7

  535. 	movsd	 -7 * SIZE(X), %xmm3

  536. 	addsd	%xmm5, %xmm1

  537. 

  538. 	addsd	%xmm6, %xmm0

  539. 	addsd	%xmm7, %xmm1

  540. 

  541. 	addl	$8 * SIZE, A1

  542. 	addl	$8 * SIZE, X

  543. 	ALIGN_4

  544. 

  545. .L25:

  546. 	testl	$4, M

  547. 	jle	.L26

  548. 

  549. 	movsd	-16 * SIZE(A1), %xmm4

  550. 	movsd	-15 * SIZE(A1), %xmm5

  551. 	movsd	-14 * SIZE(A1), %xmm6

  552. 	movsd	-13 * SIZE(A1), %xmm7

  553. 

  554. 	mulsd	%xmm2, %xmm4

  555. 	movsd	-14 * SIZE(X), %xmm2

  556. 	mulsd	%xmm3, %xmm5

  557. 	movsd	-13 * SIZE(X), %xmm3

  558. 

  559. 	mulsd	%xmm2, %xmm6

  560. 	movsd	-12 * SIZE(X), %xmm2

  561. 	addsd	%xmm4, %xmm0

  562. 	mulsd	%xmm3, %xmm7

  563. 	movsd	-11 * SIZE(X), %xmm3

  564. 	addsd	%xmm5, %xmm1

  565. 

  566. 	addsd	%xmm6, %xmm0

  567. 	addsd	%xmm7, %xmm1

  568. 

  569. 	addl	$4 * SIZE, A1

  570. 	addl	$4 * SIZE, X

  571. 	ALIGN_4

  572. 

  573. .L26:

  574. 	testl	$2, M

  575. 	jle	.L27

  576. 

  577. 	movsd	-16 * SIZE(A1), %xmm4

  578. 	movsd	-15 * SIZE(A1), %xmm5

  579. 

  580. 	mulsd	%xmm2, %xmm4

  581. 	movsd	-14 * SIZE(X), %xmm2

  582. 	mulsd	%xmm3, %xmm5

  583. 	addsd	%xmm4, %xmm0

  584. 	addsd	%xmm5, %xmm1

  585. 

  586. 	addl	$2 * SIZE, A1

  587. 	ALIGN_4

  588. 

  589. .L27:

  590. 	testl	$1, M

  591. 	jle	.L28

  592. 

  593. 	movsd	-16 * SIZE(A1), %xmm4

  594. 	mulsd	%xmm2, %xmm4

  595. 	addsd	%xmm4, %xmm0

  596. 	ALIGN_4

  597. 

  598. .L28:

  599. 	movsd	ALPHA, %xmm7

  600. 	addsd	%xmm1, %xmm0

  601. 

  602. 	mulpd	%xmm7, %xmm0

  603. 

  604. 	addsd	(Y1),  %xmm0

  605. 

  606. 	movsd	%xmm0, (Y1)

  607. 	ALIGN_4

  608. 

  609. .L999:

  610. 	popl	%ebx

  611. 	popl	%esi

  612. 	popl	%edi

  613. 	popl	%ebp

  614. 	ret

  615. 

  616. 	EPILOGUE