OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 869a169c57
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '869a169c57'
${ noResults }
OpenBLAS/kernel/x86/gemv_n_atom.S

775 lines
16 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef ATOM

  43. #define PREFETCH	prefetchnta

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCHSIZE	(8 * 6)

  46. #endif

  47. 

  48. #define STACKSIZE	16

  49. 

  50. #define M		 4 + STACKSIZE(%esp)

  51. #define N		 8 + STACKSIZE(%esp)

  52. #define ALPHA		16 + STACKSIZE(%esp)

  53. #define A		24 + STACKSIZE(%esp)

  54. #define STACK_LDA	28 + STACKSIZE(%esp)

  55. #define STACK_X		32 + STACKSIZE(%esp)

  56. #define STACK_INCX	36 + STACKSIZE(%esp)

  57. #define Y		40 + STACKSIZE(%esp)

  58. #define STACK_INCY	44 + STACKSIZE(%esp)

  59. #define BUFFER		48 + STACKSIZE(%esp)

  60. 

  61. #define I	%eax

  62. #define J	%ebx

  63. 

  64. #define INCX	%ecx

  65. #define INCY	J

  66. 

  67. #define A1	%esi

  68. #define X	%edx

  69. #define Y1	%edi

  70. #define LDA	%ebp

  71. 

  72. 	PROLOGUE

  73. 

  74. 	pushl	%ebp

  75. 	pushl	%edi

  76. 	pushl	%esi

  77. 	pushl	%ebx

  78. 

  79. 	PROFCODE

  80. 

  81. 	movl	STACK_LDA,  LDA

  82. 	movl	STACK_X,    X

  83. 	movl	STACK_INCX, INCX

  84. 

  85. 	leal	(,INCX, SIZE), INCX

  86. 	leal	(,LDA,  SIZE), LDA

  87. 

  88. 	subl	$-16 * SIZE, A

  89. 

  90. 	cmpl	$0, N

  91. 	jle	.L999

  92. 	cmpl	$0, M

  93. 	jle	.L999

  94. 

  95. 	movl	BUFFER, Y1

  96. 

  97. 	pxor	%xmm7, %xmm7

  98. 

  99. 	movl	M,   %eax

  100. 	addl	$16, %eax

  101. 	sarl	$4,  %eax

  102. 	ALIGN_3

  103. 

  104. .L01:

  105. 	movapd	%xmm7,  0 * SIZE(Y1)

  106. 	movapd	%xmm7,  2 * SIZE(Y1)

  107. 	movapd	%xmm7,  4 * SIZE(Y1)

  108. 	movapd	%xmm7,  6 * SIZE(Y1)

  109. 	movapd	%xmm7,  8 * SIZE(Y1)

  110. 	movapd	%xmm7, 10 * SIZE(Y1)

  111. 	movapd	%xmm7, 12 * SIZE(Y1)

  112. 	movapd	%xmm7, 14 * SIZE(Y1)

  113. 	subl	$-16 * SIZE, Y1

  114. 	decl	%eax

  115. 	jg	.L01

  116. 	ALIGN_3

  117. 

  118. .L10:

  119. 	movl	N,  J

  120. 	sarl	$1, J

  121. 	jle	.L20

  122. 	ALIGN_3

  123. 

  124. .L11:

  125. 	movl	BUFFER, Y1

  126. 	addl	$16 * SIZE, Y1

  127. 

  128. 	movl	A,  A1

  129. 	leal	(A1,  LDA, 2), %eax

  130. 	movl	%eax, A

  131. 

  132. 	movsd	(X), %xmm6

  133. 	addl	INCX, X

  134. 	movsd	(X), %xmm7

  135. 	addl	INCX, X

  136. 

  137. 	movsd	ALPHA, %xmm0

  138. 

  139. 	mulsd	%xmm0, %xmm6

  140. 	mulsd	%xmm0, %xmm7

  141. 

  142. 	movsd	 -16 * SIZE(Y1), %xmm0

  143. 	movsd	 -15 * SIZE(Y1), %xmm1

  144. 

  145. 	movl	M,   I

  146. 	sarl	$3,  I

  147. 	jle	.L15

  148. 

  149. 	movsd	 -16 * SIZE(A1), %xmm2

  150. 	movsd	 -15 * SIZE(A1), %xmm3

  151. 	movsd	 -16 * SIZE(A1, LDA), %xmm4

  152. 	movsd	 -15 * SIZE(A1, LDA), %xmm5

  153. 

  154. 	mulsd	 %xmm6, %xmm2

  155. 	mulsd	 %xmm6, %xmm3

  156. 

  157. 	decl	 I

  158. 	jle	 .L14

  159. 	ALIGN_3

  160. 

  161. .L13:

  162. #ifdef PREFETCH

  163. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(A1)

  164. #endif

  165. 

  166. 	mulsd	 %xmm7, %xmm4

  167. 	addsd	 %xmm2, %xmm0

  168. 	movsd	 -14 * SIZE(A1), %xmm2

  169. 	mulsd	 %xmm7, %xmm5

  170. 	addsd	 %xmm3, %xmm1

  171. 	movsd	 -13 * SIZE(A1), %xmm3

  172. 

  173. 	addsd	 %xmm4, %xmm0

  174. 	movsd	 -14 * SIZE(A1, LDA), %xmm4

  175. 	mulsd	 %xmm6, %xmm2

  176. 	addsd	 %xmm5, %xmm1

  177. 	movsd	 -13 * SIZE(A1, LDA), %xmm5

  178. 	mulsd	 %xmm6, %xmm3

  179. 

  180. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  181. 	movsd	 -14 * SIZE(Y1), %xmm0

  182. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  183. 	movsd	 -13 * SIZE(Y1), %xmm1

  184. 

  185. 	mulsd	 %xmm7, %xmm4

  186. 	addsd	 %xmm2, %xmm0

  187. 	movsd	 -12 * SIZE(A1), %xmm2

  188. 	mulsd	 %xmm7, %xmm5

  189. 	addsd	 %xmm3, %xmm1

  190. 	movsd	 -11 * SIZE(A1), %xmm3

  191. 

  192. 	addsd	 %xmm4, %xmm0

  193. 	movsd	 -12 * SIZE(A1, LDA), %xmm4

  194. 	mulsd	 %xmm6, %xmm2

  195. 	addsd	 %xmm5, %xmm1

  196. 	movsd	 -11 * SIZE(A1, LDA), %xmm5

  197. 	mulsd	 %xmm6, %xmm3

  198. 

  199. 	movlpd	 %xmm0, -14 * SIZE(Y1)

  200. 	movsd	 -12 * SIZE(Y1), %xmm0

  201. 	movlpd	 %xmm1, -13 * SIZE(Y1)

  202. 	movsd	 -11 * SIZE(Y1), %xmm1

  203. 

  204. #ifdef PREFETCH

  205. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(A1, LDA)

  206. #endif

  207. 

  208. 	mulsd	 %xmm7, %xmm4

  209. 	addsd	 %xmm2, %xmm0

  210. 	movsd	 -10 * SIZE(A1), %xmm2

  211. 	mulsd	 %xmm7, %xmm5

  212. 	addsd	 %xmm3, %xmm1

  213. 	movsd	  -9 * SIZE(A1), %xmm3

  214. 

  215. 	addsd	 %xmm4, %xmm0

  216. 	movsd	 -10 * SIZE(A1, LDA), %xmm4

  217. 	mulsd	 %xmm6, %xmm2

  218. 	addsd	 %xmm5, %xmm1

  219. 	movsd	  -9 * SIZE(A1, LDA), %xmm5

  220. 	mulsd	 %xmm6, %xmm3

  221. 

  222. 	movlpd	 %xmm0, -12 * SIZE(Y1)

  223. 	movsd	 -10 * SIZE(Y1), %xmm0

  224. 	movlpd	 %xmm1, -11 * SIZE(Y1)

  225. 	movsd	  -9 * SIZE(Y1), %xmm1

  226. 

  227. 	mulsd	 %xmm7, %xmm4

  228. 	addsd	 %xmm2, %xmm0

  229. 	movsd	  -8 * SIZE(A1), %xmm2

  230. 	mulsd	 %xmm7, %xmm5

  231. 	addsd	 %xmm3, %xmm1

  232. 	movsd	  -7 * SIZE(A1), %xmm3

  233. 

  234. 	addsd	 %xmm4, %xmm0

  235. 	movsd	  -8 * SIZE(A1, LDA), %xmm4

  236. 	mulsd	 %xmm6, %xmm2

  237. 	addsd	 %xmm5, %xmm1

  238. 	movsd	  -7 * SIZE(A1, LDA), %xmm5

  239. 	mulsd	 %xmm6, %xmm3

  240. 

  241. 	movlpd	 %xmm0, -10 * SIZE(Y1)

  242. 	movsd	  -8 * SIZE(Y1), %xmm0

  243. 	movlpd	 %xmm1,  -9 * SIZE(Y1)

  244. 	movsd	  -7 * SIZE(Y1), %xmm1

  245. 

  246. 	subl	 $-8 * SIZE, A1

  247. 	subl	 $-8 * SIZE, Y1

  248. 

  249. 	subl	 $1, I

  250. 	BRANCH

  251. 	jg	.L13

  252. 	ALIGN_3

  253. 

  254. .L14:

  255. 	mulsd	 %xmm7, %xmm4

  256. 	addsd	 %xmm2, %xmm0

  257. 	movsd	 -14 * SIZE(A1), %xmm2

  258. 	mulsd	 %xmm7, %xmm5

  259. 	addsd	 %xmm3, %xmm1

  260. 	movsd	 -13 * SIZE(A1), %xmm3

  261. 

  262. 	addsd	 %xmm4, %xmm0

  263. 	movsd	 -14 * SIZE(A1, LDA), %xmm4

  264. 	mulsd	 %xmm6, %xmm2

  265. 	addsd	 %xmm5, %xmm1

  266. 	movsd	 -13 * SIZE(A1, LDA), %xmm5

  267. 	mulsd	 %xmm6, %xmm3

  268. 

  269. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  270. 	movsd	 -14 * SIZE(Y1), %xmm0

  271. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  272. 	movsd	 -13 * SIZE(Y1), %xmm1

  273. 

  274. 	mulsd	 %xmm7, %xmm4

  275. 	addsd	 %xmm2, %xmm0

  276. 	movsd	 -12 * SIZE(A1), %xmm2

  277. 	mulsd	 %xmm7, %xmm5

  278. 	addsd	 %xmm3, %xmm1

  279. 	movsd	 -11 * SIZE(A1), %xmm3

  280. 

  281. 	addsd	 %xmm4, %xmm0

  282. 	movsd	 -12 * SIZE(A1, LDA), %xmm4

  283. 	mulsd	 %xmm6, %xmm2

  284. 	addsd	 %xmm5, %xmm1

  285. 	movsd	 -11 * SIZE(A1, LDA), %xmm5

  286. 	mulsd	 %xmm6, %xmm3

  287. 

  288. 	movlpd	 %xmm0, -14 * SIZE(Y1)

  289. 	movsd	 -12 * SIZE(Y1), %xmm0

  290. 	movlpd	 %xmm1, -13 * SIZE(Y1)

  291. 	movsd	 -11 * SIZE(Y1), %xmm1

  292. 

  293. 	mulsd	 %xmm7, %xmm4

  294. 	addsd	 %xmm2, %xmm0

  295. 	movsd	 -10 * SIZE(A1), %xmm2

  296. 	mulsd	 %xmm7, %xmm5

  297. 	addsd	 %xmm3, %xmm1

  298. 	movsd	  -9 * SIZE(A1), %xmm3

  299. 

  300. 	addsd	 %xmm4, %xmm0

  301. 	movsd	 -10 * SIZE(A1, LDA), %xmm4

  302. 	mulsd	 %xmm6, %xmm2

  303. 	addsd	 %xmm5, %xmm1

  304. 	movsd	  -9 * SIZE(A1, LDA), %xmm5

  305. 	mulsd	 %xmm6, %xmm3

  306. 

  307. 	movlpd	 %xmm0, -12 * SIZE(Y1)

  308. 	movsd	 -10 * SIZE(Y1), %xmm0

  309. 	movlpd	 %xmm1, -11 * SIZE(Y1)

  310. 	movsd	  -9 * SIZE(Y1), %xmm1

  311. 

  312. 	mulsd	 %xmm7, %xmm4

  313. 	addsd	 %xmm2, %xmm0

  314. 	mulsd	 %xmm7, %xmm5

  315. 	addsd	 %xmm3, %xmm1

  316. 

  317. 	addsd	 %xmm4, %xmm0

  318. 	addsd	 %xmm5, %xmm1

  319. 

  320. 	movlpd	 %xmm0, -10 * SIZE(Y1)

  321. 	movsd	  -8 * SIZE(Y1), %xmm0

  322. 	movlpd	 %xmm1,  -9 * SIZE(Y1)

  323. 	movsd	  -7 * SIZE(Y1), %xmm1

  324. 

  325. 	subl	 $-8 * SIZE, A1

  326. 	subl	 $-8 * SIZE, Y1

  327. 	ALIGN_3

  328. 

  329. .L15:

  330. 	testl	$4, M

  331. 	je	.L16

  332. 

  333. 	movsd	 -16 * SIZE(A1), %xmm2

  334. 	movsd	 -15 * SIZE(A1), %xmm3

  335. 	movsd	 -16 * SIZE(A1, LDA), %xmm4

  336. 	movsd	 -15 * SIZE(A1, LDA), %xmm5

  337. 

  338. 	mulsd	 %xmm6, %xmm2

  339. 	mulsd	 %xmm6, %xmm3

  340. 

  341. 	mulsd	 %xmm7, %xmm4

  342. 	addsd	 %xmm2, %xmm0

  343. 	movsd	 -14 * SIZE(A1), %xmm2

  344. 	mulsd	 %xmm7, %xmm5

  345. 	addsd	 %xmm3, %xmm1

  346. 	movsd	 -13 * SIZE(A1), %xmm3

  347. 

  348. 	addsd	 %xmm4, %xmm0

  349. 	movsd	 -14 * SIZE(A1, LDA), %xmm4

  350. 	mulsd	 %xmm6, %xmm2

  351. 	addsd	 %xmm5, %xmm1

  352. 	movsd	 -13 * SIZE(A1, LDA), %xmm5

  353. 	mulsd	 %xmm6, %xmm3

  354. 

  355. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  356. 	movsd	 -14 * SIZE(Y1), %xmm0

  357. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  358. 	movsd	 -13 * SIZE(Y1), %xmm1

  359. 

  360. 	mulsd	 %xmm7, %xmm4

  361. 	addsd	 %xmm2, %xmm0

  362. 	mulsd	 %xmm7, %xmm5

  363. 	addsd	 %xmm3, %xmm1

  364. 

  365. 	addsd	 %xmm4, %xmm0

  366. 	addsd	 %xmm5, %xmm1

  367. 

  368. 	movlpd	 %xmm0, -14 * SIZE(Y1)

  369. 	movsd	 -12 * SIZE(Y1), %xmm0

  370. 	movlpd	 %xmm1, -13 * SIZE(Y1)

  371. 	movsd	 -11 * SIZE(Y1), %xmm1

  372. 

  373. 	addl	 $4 * SIZE, A1

  374. 	addl	 $4 * SIZE, Y1

  375. 	ALIGN_3

  376. 

  377. .L16:

  378. 	testl	$2, M

  379. 	je	.L17

  380. 

  381. 	movsd	 -16 * SIZE(A1), %xmm2

  382. 	movsd	 -15 * SIZE(A1), %xmm3

  383. 	movsd	 -16 * SIZE(A1, LDA), %xmm4

  384. 	movsd	 -15 * SIZE(A1, LDA), %xmm5

  385. 

  386. 	mulsd	 %xmm6, %xmm2

  387. 	mulsd	 %xmm6, %xmm3

  388. 

  389. 	mulsd	 %xmm7, %xmm4

  390. 	addsd	 %xmm2, %xmm0

  391. 	mulsd	 %xmm7, %xmm5

  392. 	addsd	 %xmm3, %xmm1

  393. 

  394. 	addsd	 %xmm4, %xmm0

  395. 	addsd	 %xmm5, %xmm1

  396. 

  397. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  398. 	movsd	 -14 * SIZE(Y1), %xmm0

  399. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  400. 

  401. 	addl	 $2 * SIZE, A1

  402. 	addl	 $2 * SIZE, Y1

  403. 	ALIGN_3

  404. 

  405. .L17:

  406. 	testl	$1, M

  407. 	je	.L19

  408. 

  409. 	movsd	 -16 * SIZE(A1), %xmm2

  410. 	movsd	 -16 * SIZE(A1, LDA), %xmm3

  411. 

  412. 	movsd	 -16 * SIZE(Y1), %xmm0

  413. 

  414. 	mulsd	 %xmm6, %xmm2

  415. 	addsd	 %xmm2, %xmm0

  416. 	mulsd	 %xmm7, %xmm3

  417. 	addsd	 %xmm3, %xmm0

  418. 

  419. 	movsd	 %xmm0, -16 * SIZE(Y1)

  420. 	ALIGN_3

  421. 

  422. .L19:

  423. 	decl	J

  424. 	jg	.L11

  425. 	ALIGN_4

  426. 

  427. .L20:

  428. 	testl	$1, N

  429. 	jle	.L990

  430. 

  431. 	movl	BUFFER, Y1

  432. 	addl	$16 * SIZE, Y1

  433. 

  434. 	movl	A,  A1

  435. 	leal	(A1,  LDA, 2), %eax

  436. 	movl	%eax, A

  437. 

  438. 	movsd	(X), %xmm6

  439. 	addl	INCX, X

  440. 	movsd	(X), %xmm7

  441. 	addl	INCX, X

  442. 

  443. 	movsd	ALPHA, %xmm0

  444. 

  445. 	mulsd	%xmm0, %xmm6

  446. 	mulsd	%xmm0, %xmm7

  447. 

  448. 	movsd	 -16 * SIZE(Y1), %xmm0

  449. 	movsd	 -15 * SIZE(Y1), %xmm1

  450. 	movsd	 -14 * SIZE(Y1), %xmm4

  451. 	movsd	 -13 * SIZE(Y1), %xmm5

  452. 

  453. 	movl	M,   I

  454. 	sarl	$3,  I

  455. 	jle	.L25

  456. 

  457. 	movsd	 -16 * SIZE(A1), %xmm2

  458. 	movsd	 -15 * SIZE(A1), %xmm3

  459. 

  460. 	mulsd	 %xmm6, %xmm2

  461. 	mulsd	 %xmm6, %xmm3

  462. 

  463. 	decl	 I

  464. 	jle	 .L24

  465. 	ALIGN_3

  466. 

  467. .L23:

  468. #ifdef PREFETCH

  469. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(A1)

  470. #endif

  471. 

  472. 	addsd	 %xmm2, %xmm0

  473. 	movsd	 -14 * SIZE(A1), %xmm2

  474. 	addsd	 %xmm3, %xmm1

  475. 	movsd	 -13 * SIZE(A1), %xmm3

  476. 

  477. 	mulsd	 %xmm6, %xmm2

  478. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  479. 	movsd	 -12 * SIZE(Y1), %xmm0

  480. 	mulsd	 %xmm6, %xmm3

  481. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  482. 	movsd	 -11 * SIZE(Y1), %xmm1

  483. 

  484. 	addsd	 %xmm2, %xmm4

  485. 	movsd	 -12 * SIZE(A1), %xmm2

  486. 	addsd	 %xmm3, %xmm5

  487. 	movsd	 -11 * SIZE(A1), %xmm3

  488. 

  489. 	mulsd	 %xmm6, %xmm2

  490. 	movlpd	 %xmm4, -14 * SIZE(Y1)

  491. 	movsd	 -10 * SIZE(Y1), %xmm4

  492. 	mulsd	 %xmm6, %xmm3

  493. 	movlpd	 %xmm5, -13 * SIZE(Y1)

  494. 	movsd	  -9 * SIZE(Y1), %xmm5

  495. 

  496. 	addsd	 %xmm2, %xmm0

  497. 	movsd	 -10 * SIZE(A1), %xmm2

  498. 	addsd	 %xmm3, %xmm1

  499. 	movsd	  -9 * SIZE(A1), %xmm3

  500. 

  501. 	mulsd	 %xmm6, %xmm2

  502. 	movlpd	 %xmm0, -12 * SIZE(Y1)

  503. 	movsd	  -8 * SIZE(Y1), %xmm0

  504. 	mulsd	 %xmm6, %xmm3

  505. 	movlpd	 %xmm1, -11 * SIZE(Y1)

  506. 	movsd	  -7 * SIZE(Y1), %xmm1

  507. 

  508. 	addsd	 %xmm2, %xmm4

  509. 	movsd	  -8 * SIZE(A1), %xmm2

  510. 	addsd	 %xmm3, %xmm5

  511. 	movsd	  -7 * SIZE(A1), %xmm3

  512. 

  513. 	mulsd	 %xmm6, %xmm2

  514. 	movlpd	 %xmm4, -10 * SIZE(Y1)

  515. 	movsd	  -6 * SIZE(Y1), %xmm4

  516. 	mulsd	 %xmm6, %xmm3

  517. 	movlpd	 %xmm5,  -9 * SIZE(Y1)

  518. 	movsd	  -5 * SIZE(Y1), %xmm5

  519. 

  520. 	subl	 $-8 * SIZE, A1

  521. 	subl	 $-8 * SIZE, Y1

  522. 

  523. 	subl	 $1, I

  524. 	BRANCH

  525. 	jg	.L23

  526. 	ALIGN_3

  527. 

  528. .L24:

  529. 	addsd	 %xmm2, %xmm0

  530. 	movsd	 -14 * SIZE(A1), %xmm2

  531. 	addsd	 %xmm3, %xmm1

  532. 	movsd	 -13 * SIZE(A1), %xmm3

  533. 

  534. 	mulsd	 %xmm6, %xmm2

  535. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  536. 	movsd	 -12 * SIZE(Y1), %xmm0

  537. 	mulsd	 %xmm6, %xmm3

  538. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  539. 	movsd	 -11 * SIZE(Y1), %xmm1

  540. 

  541. 	addsd	 %xmm2, %xmm4

  542. 	movsd	 -12 * SIZE(A1), %xmm2

  543. 	addsd	 %xmm3, %xmm5

  544. 	movsd	 -11 * SIZE(A1), %xmm3

  545. 

  546. 	mulsd	 %xmm6, %xmm2

  547. 	movlpd	 %xmm4, -14 * SIZE(Y1)

  548. 	movsd	 -10 * SIZE(Y1), %xmm4

  549. 	mulsd	 %xmm6, %xmm3

  550. 	movlpd	 %xmm5, -13 * SIZE(Y1)

  551. 	movsd	  -9 * SIZE(Y1), %xmm5

  552. 

  553. 	addsd	 %xmm2, %xmm0

  554. 	movsd	 -10 * SIZE(A1), %xmm2

  555. 	addsd	 %xmm3, %xmm1

  556. 	movsd	  -9 * SIZE(A1), %xmm3

  557. 

  558. 	mulsd	 %xmm6, %xmm2

  559. 	movlpd	 %xmm0, -12 * SIZE(Y1)

  560. 	mulsd	 %xmm6, %xmm3

  561. 	movlpd	 %xmm1, -11 * SIZE(Y1)

  562. 

  563. 	addsd	 %xmm2, %xmm4

  564. 	movsd	  -8 * SIZE(Y1), %xmm0

  565. 	addsd	 %xmm3, %xmm5

  566. 	movsd	  -7 * SIZE(Y1), %xmm1

  567. 

  568. 	movlpd	 %xmm4, -10 * SIZE(Y1)

  569. 	movsd	  -6 * SIZE(Y1), %xmm4

  570. 	movlpd	 %xmm5,  -9 * SIZE(Y1)

  571. 	movsd	  -5 * SIZE(Y1), %xmm5

  572. 

  573. 	subl	 $-8 * SIZE, A1

  574. 	subl	 $-8 * SIZE, Y1

  575. 	ALIGN_3

  576. 

  577. .L25:

  578. 	testl	$4, M

  579. 	je	.L26

  580. 

  581. 	movsd	 -16 * SIZE(A1), %xmm2

  582. 	movsd	 -15 * SIZE(A1), %xmm3

  583. 	mulsd	 %xmm6, %xmm2

  584. 	mulsd	 %xmm6, %xmm3

  585. 

  586. 	addsd	 %xmm2, %xmm0

  587. 	movsd	 -14 * SIZE(A1), %xmm2

  588. 	addsd	 %xmm3, %xmm1

  589. 	movsd	 -13 * SIZE(A1), %xmm3

  590. 

  591. 	mulsd	 %xmm6, %xmm2

  592. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  593. 	movsd	 -12 * SIZE(Y1), %xmm0

  594. 	mulsd	 %xmm6, %xmm3

  595. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  596. 	movsd	 -11 * SIZE(Y1), %xmm1

  597. 

  598. 	addsd	 %xmm2, %xmm4

  599. 	addsd	 %xmm3, %xmm5

  600. 

  601. 	movlpd	 %xmm4, -14 * SIZE(Y1)

  602. 	movlpd	 %xmm5, -13 * SIZE(Y1)

  603. 

  604. 	addl	 $4 * SIZE, A1

  605. 	addl	 $4 * SIZE, Y1

  606. 	ALIGN_3

  607. 

  608. .L26:

  609. 	testl	$2, M

  610. 	je	.L27

  611. 

  612. 	movsd	 -16 * SIZE(A1), %xmm2

  613. 	movsd	 -15 * SIZE(A1), %xmm3

  614. 

  615. 	mulsd	 %xmm6, %xmm2

  616. 	mulsd	 %xmm6, %xmm3

  617. 	addsd	 %xmm2, %xmm0

  618. 	addsd	 %xmm3, %xmm1

  619. 

  620. 	movlpd	 %xmm0, -16 * SIZE(Y1)

  621. 	movsd	 -14 * SIZE(Y1), %xmm0

  622. 	movlpd	 %xmm1, -15 * SIZE(Y1)

  623. 

  624. 	addl	 $2 * SIZE, A1

  625. 	addl	 $2 * SIZE, Y1

  626. 	ALIGN_3

  627. 

  628. .L27:

  629. 	testl	$1, M

  630. 	je	.L990

  631. 

  632. 	movsd	 -16 * SIZE(A1), %xmm2

  633. 	movsd	 -16 * SIZE(Y1), %xmm0

  634. 

  635. 	mulsd	 %xmm6, %xmm2

  636. 	addsd	 %xmm2, %xmm0

  637. 

  638. 	movsd	 %xmm0, -16 * SIZE(Y1)

  639. 	ALIGN_3

  640. 

  641. .L990:

  642. 	movl	Y,   Y1

  643. 	movl	BUFFER, X

  644. 	movl	Y1,  A1

  645. 

  646. 	movl	STACK_INCY, INCY

  647. 	sall	$BASE_SHIFT, INCY

  648. 

  649. 	movl	M,   %eax

  650. 	sarl	$3,  %eax

  651. 	jle	.L994

  652. 	ALIGN_3

  653. 

  654. .L992:

  655. 	movsd	(Y1), %xmm0

  656. 	addl	INCY, Y1

  657. 	movsd	(Y1), %xmm1

  658. 	addl	INCY, Y1

  659. 	movsd	(Y1), %xmm2

  660. 	addl	INCY, Y1

  661. 	movsd	(Y1), %xmm3

  662. 	addl	INCY, Y1

  663. 	movsd	(Y1), %xmm4

  664. 	addl	INCY, Y1

  665. 	movsd	(Y1), %xmm5

  666. 	addl	INCY, Y1

  667. 	movsd	(Y1), %xmm6

  668. 	addl	INCY, Y1

  669. 	movsd	(Y1), %xmm7

  670. 	addl	INCY, Y1

  671. 

  672. 	addsd	0 * SIZE(X), %xmm0

  673. 	addsd	1 * SIZE(X), %xmm1

  674. 	addsd	2 * SIZE(X), %xmm2

  675. 	addsd	3 * SIZE(X), %xmm3

  676. 	addsd	4 * SIZE(X), %xmm4

  677. 	addsd	5 * SIZE(X), %xmm5

  678. 	addsd	6 * SIZE(X), %xmm6

  679. 	addsd	7 * SIZE(X), %xmm7

  680. 

  681. 	movlpd	%xmm0, (A1)

  682. 	addl	INCY, A1

  683. 	movlpd	%xmm1, (A1)

  684. 	addl	INCY, A1

  685. 	movlpd	%xmm2, (A1)

  686. 	addl	INCY, A1

  687. 	movlpd	%xmm3, (A1)

  688. 	addl	INCY, A1

  689. 	movlpd	%xmm4, (A1)

  690. 	addl	INCY, A1

  691. 	movlpd	%xmm5, (A1)

  692. 	addl	INCY, A1

  693. 	movlpd	%xmm6, (A1)

  694. 	addl	INCY, A1

  695. 	movlpd	%xmm7, (A1)

  696. 	addl	INCY, A1

  697. 

  698. 	addl	$8 * SIZE, X

  699. 	decl	%eax

  700. 	jg	.L992

  701. 	ALIGN_3

  702. 

  703. .L994:

  704. 	testl	$7, M

  705. 	jle	.L999

  706. 

  707. 	testl	$4, M

  708. 	jle	.L995

  709. 

  710. 	movsd	(Y1), %xmm0

  711. 	addl	INCY, Y1

  712. 	movsd	(Y1), %xmm1

  713. 	addl	INCY, Y1

  714. 	movsd	(Y1), %xmm2

  715. 	addl	INCY, Y1

  716. 	movsd	(Y1), %xmm3

  717. 	addl	INCY, Y1

  718. 

  719. 	addsd	0 * SIZE(X), %xmm0

  720. 	addsd	1 * SIZE(X), %xmm1

  721. 	addsd	2 * SIZE(X), %xmm2

  722. 	addsd	3 * SIZE(X), %xmm3

  723. 

  724. 	movlpd	%xmm0, (A1)

  725. 	addl	INCY, A1

  726. 	movlpd	%xmm1, (A1)

  727. 	addl	INCY, A1

  728. 	movlpd	%xmm2, (A1)

  729. 	addl	INCY, A1

  730. 	movlpd	%xmm3, (A1)

  731. 	addl	INCY, A1

  732. 

  733. 	addl	$4 * SIZE, X

  734. 	ALIGN_3

  735. 

  736. .L995:

  737. 	testl	$2, M

  738. 	jle	.L996

  739. 

  740. 	movsd	(Y1), %xmm0

  741. 	addl	INCY, Y1

  742. 	movsd	(Y1), %xmm1

  743. 	addl	INCY, Y1

  744. 

  745. 	addsd	0 * SIZE(X), %xmm0

  746. 	addsd	1 * SIZE(X), %xmm1

  747. 

  748. 	movlpd	%xmm0, (A1)

  749. 	addl	INCY, A1

  750. 	movlpd	%xmm1, (A1)

  751. 	addl	INCY, A1

  752. 

  753. 	addl	$2 * SIZE, X

  754. 	ALIGN_3

  755. 

  756. .L996:

  757. 	testl	$1, M

  758. 	jle	.L999

  759. 

  760. 	movsd	(Y1), %xmm0

  761. 

  762. 	addsd	0 * SIZE(X), %xmm0

  763. 

  764. 	movlpd	%xmm0, (A1)

  765. 	ALIGN_3

  766. 

  767. .L999:

  768. 	popl	%ebx

  769. 	popl	%esi

  770. 	popl	%edi

  771. 	popl	%ebp

  772. 	ret

  773. 

  774. 	EPILOGUE