OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86/dot_sse2.S

729 lines
14 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	12

  43. #define ARGS     0

  44. 

  45. #define STACK_N		 4 + STACK + ARGS(%esp)

  46. #define STACK_X		 8 + STACK + ARGS(%esp)

  47. #define STACK_INCX	12 + STACK + ARGS(%esp)

  48. #define STACK_Y		16 + STACK + ARGS(%esp)

  49. #define STACK_INCY	20 + STACK + ARGS(%esp)

  50. 

  51. #define N	%ecx

  52. #define X	%esi

  53. #define INCX	%ebx

  54. #define Y	%edi

  55. #define INCY	%edx

  56. 

  57. #include "l1param.h"

  58. 

  59. 	PROLOGUE

  60. 	PROFCODE

  61. 

  62. 	pushl	%edi

  63. 	pushl	%esi

  64. 	pushl	%ebx

  65. 

  66. 	movl	STACK_N,     N

  67. 	movl	STACK_X,     X

  68. 	movl	STACK_INCX,  INCX

  69. 	movl	STACK_Y,     Y

  70. 	movl	STACK_INCY,  INCY

  71. 

  72. 	leal	(, INCX, SIZE), INCX

  73. 	leal	(, INCY, SIZE), INCY

  74. 

  75. 	xorps	%xmm0, %xmm0

  76. 	xorps	%xmm1, %xmm1

  77. 	xorps	%xmm2, %xmm2

  78. 	xorps	%xmm3, %xmm3

  79. 

  80. 	cmpl	$0, N

  81. 	jle	.L999

  82. 

  83. 	cmpl	$SIZE, INCX

  84. 	jne	.L50

  85. 	cmpl	$SIZE, INCY

  86. 	jne	.L50

  87. 

  88. 	subl	$-16 * SIZE, X

  89. 	subl	$-16 * SIZE, Y

  90. 

  91. 	testl	$SIZE, Y

  92. 	je	.L10

  93. 

  94. 	movsd	-16 * SIZE(X), %xmm0

  95. 	mulsd	-16 * SIZE(Y), %xmm0

  96. 	addl	$1 * SIZE, X

  97. 	addl	$1 * SIZE, Y

  98. 	decl	N

  99. 	ALIGN_2

  100. 

  101. .L10:

  102. 	testl	$SIZE, X

  103. 	jne	.L20

  104. 

  105. 	movl	N,  %eax

  106. 	sarl	$4, %eax

  107. 	jle	.L14

  108. 

  109. 	movaps	-16 * SIZE(X), %xmm4

  110. 	movaps	-14 * SIZE(X), %xmm5

  111. 	movaps	-12 * SIZE(X), %xmm6

  112. 	movaps	-10 * SIZE(X), %xmm7

  113. 

  114. 	decl	%eax

  115. 	jle	.L12

  116. 	ALIGN_3

  117. 

  118. .L11:

  119. #ifdef PREFETCH

  120. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  121. #endif

  122. 

  123. 	mulpd	-16 * SIZE(Y), %xmm4

  124. 	addpd	 %xmm4, %xmm0

  125. 	movaps	 -8 * SIZE(X), %xmm4

  126. 

  127. 	mulpd	-14 * SIZE(Y), %xmm5

  128. 	addpd	 %xmm5, %xmm1

  129. 	movaps	 -6 * SIZE(X), %xmm5

  130. 

  131. #ifdef PREFETCH

  132. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  133. #endif

  134. 

  135. 	mulpd	-12 * SIZE(Y), %xmm6

  136. 	addpd	 %xmm6, %xmm2

  137. 	movaps	 -4 * SIZE(X), %xmm6

  138. 

  139. 	mulpd	-10 * SIZE(Y), %xmm7

  140. 	addpd	 %xmm7, %xmm3

  141. 	movaps	 -2 * SIZE(X), %xmm7

  142. 

  143. #if defined(PREFETCH) && !defined(FETCH128)

  144. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  145. #endif

  146. 

  147. 	mulpd	 -8 * SIZE(Y), %xmm4

  148. 	addpd	 %xmm4, %xmm0

  149. 	movaps	  0 * SIZE(X), %xmm4

  150. 

  151. 	mulpd	 -6 * SIZE(Y), %xmm5

  152. 	addpd	 %xmm5, %xmm1

  153. 	movaps	  2 * SIZE(X), %xmm5

  154. 

  155. #if defined(PREFETCH) && !defined(FETCH128)

  156. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  157. #endif

  158. 

  159. 	mulpd	 -4 * SIZE(Y), %xmm6

  160. 	addpd	 %xmm6, %xmm2

  161. 	movaps	  4 * SIZE(X), %xmm6

  162. 

  163. 	mulpd	 -2 * SIZE(Y), %xmm7

  164. 	addpd	 %xmm7, %xmm3

  165. 	movaps	  6 * SIZE(X), %xmm7

  166. 

  167. 	subl	$-16 * SIZE, X

  168. 	subl	$-16 * SIZE, Y

  169. 

  170. 	decl	%eax

  171. 	jg	.L11

  172. 	ALIGN_3

  173. 

  174. .L12:

  175. 	mulpd	-16 * SIZE(Y), %xmm4

  176. 	addpd	 %xmm4, %xmm0

  177. 	movaps	 -8 * SIZE(X), %xmm4

  178. 

  179. 	mulpd	-14 * SIZE(Y), %xmm5

  180. 	addpd	 %xmm5, %xmm1

  181. 	movaps	 -6 * SIZE(X), %xmm5

  182. 

  183. 	mulpd	-12 * SIZE(Y), %xmm6

  184. 	addpd	 %xmm6, %xmm2

  185. 	movaps	 -4 * SIZE(X), %xmm6

  186. 

  187. 	mulpd	-10 * SIZE(Y), %xmm7

  188. 	addpd	 %xmm7, %xmm3

  189. 	movaps	 -2 * SIZE(X), %xmm7

  190. 

  191. 	mulpd	 -8 * SIZE(Y), %xmm4

  192. 	addpd	 %xmm4, %xmm0

  193. 	mulpd	 -6 * SIZE(Y), %xmm5

  194. 	addpd	 %xmm5, %xmm1

  195. 	mulpd	 -4 * SIZE(Y), %xmm6

  196. 	addpd	 %xmm6, %xmm2

  197. 	mulpd	 -2 * SIZE(Y), %xmm7

  198. 	addpd	 %xmm7, %xmm3

  199. 

  200. 	subl	$-16 * SIZE, X

  201. 	subl	$-16 * SIZE, Y

  202. 	ALIGN_3

  203. 

  204. .L14:

  205. 	testl	$15, N

  206. 	jle	.L999

  207. 

  208. 	testl	$8, N

  209. 	jle	.L15

  210. 

  211. 	movaps	-16 * SIZE(X), %xmm4

  212. 	movaps	-14 * SIZE(X), %xmm5

  213. 	movaps	-12 * SIZE(X), %xmm6

  214. 	movaps	-10 * SIZE(X), %xmm7

  215. 

  216. 	mulpd	-16 * SIZE(Y), %xmm4

  217. 	addpd	 %xmm4,  %xmm0

  218. 	mulpd	-14 * SIZE(Y), %xmm5

  219. 	addpd	 %xmm5,  %xmm1

  220. 	mulpd	-12 * SIZE(Y), %xmm6

  221. 	addpd	 %xmm6,  %xmm2

  222. 	mulpd	-10 * SIZE(Y), %xmm7

  223. 	addpd	 %xmm7,  %xmm3

  224. 

  225. 	addl	$8 * SIZE, X

  226. 	addl	$8 * SIZE, Y

  227. 	ALIGN_3

  228. 

  229. .L15:

  230. 	testl	$4, N

  231. 	jle	.L16

  232. 

  233. 	movaps	-16 * SIZE(X), %xmm4

  234. 	movaps	-14 * SIZE(X), %xmm5

  235. 

  236. 	mulpd	-16 * SIZE(Y), %xmm4

  237. 	addpd	 %xmm4,  %xmm0

  238. 	mulpd	-14 * SIZE(Y), %xmm5

  239. 	addpd	 %xmm5,  %xmm1

  240. 

  241. 	addl	$4 * SIZE, X

  242. 	addl	$4 * SIZE, Y

  243. 	ALIGN_3

  244. 

  245. .L16:

  246. 	testl	$2, N

  247. 	jle	.L17

  248. 

  249. 	movaps	-16 * SIZE(X), %xmm4

  250. 

  251. 	mulpd	-16 * SIZE(Y), %xmm4

  252. 	addpd	 %xmm4,  %xmm0

  253. 

  254. 	addl	$2 * SIZE, X

  255. 	addl	$2 * SIZE, Y

  256. 	ALIGN_3

  257. 

  258. .L17:

  259. 	testl	$1, N

  260. 	jle	.L999

  261. 

  262. 	movsd	-16 * SIZE(X), %xmm4

  263. 	mulsd	-16 * SIZE(Y), %xmm4

  264. 	addsd	%xmm4,  %xmm0

  265. 	jmp	.L999

  266. 	ALIGN_3

  267. 

  268. .L20:

  269. 

  270. #ifdef ALIGNED_ACCESS

  271. 

  272. 	movhps	-16 * SIZE(X), %xmm4

  273. 	addl	$SIZE, X

  274. 

  275. 	movl	N,  %eax

  276. 	sarl	$4, %eax

  277. 	jle	.L24

  278. 

  279. 	movaps	-16 * SIZE(X), %xmm5

  280. 	movaps	-14 * SIZE(X), %xmm6

  281. 	movaps	-12 * SIZE(X), %xmm7

  282. 

  283. 	decl	%eax

  284. 	jle	.L22

  285. 

  286. 	ALIGN_3

  287. 

  288. .L21:

  289. #ifdef PREFETCH

  290. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  291. #endif

  292. 

  293. 	SHUFPD_1 %xmm5, %xmm4

  294. 	mulpd	-16 * SIZE(Y), %xmm4

  295. 	addpd	 %xmm4,  %xmm0

  296. 	movaps	-10 * SIZE(X), %xmm4

  297. 

  298. 	SHUFPD_1 %xmm6, %xmm5

  299. 	mulpd	-14 * SIZE(Y), %xmm5

  300. 	addpd	 %xmm5,  %xmm1

  301. 	movaps	 -8 * SIZE(X), %xmm5

  302. 

  303. #ifdef PREFETCH

  304. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  305. #endif

  306. 

  307. 	SHUFPD_1 %xmm7, %xmm6

  308. 	mulpd	-12 * SIZE(Y), %xmm6

  309. 	addpd	 %xmm6,  %xmm2

  310. 	movaps	 -6 * SIZE(X), %xmm6

  311. 

  312. 	SHUFPD_1 %xmm4, %xmm7

  313. 	mulpd	-10 * SIZE(Y), %xmm7

  314. 	addpd	 %xmm7,  %xmm3

  315. 	movaps	 -4 * SIZE(X), %xmm7

  316. 

  317. #if defined(PREFETCH) && !defined(FETCH128)

  318. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  319. #endif

  320. 

  321. 	SHUFPD_1 %xmm5, %xmm4

  322. 	mulpd	 -8 * SIZE(Y), %xmm4

  323. 	addpd	 %xmm4,  %xmm0

  324. 	movaps	 -2 * SIZE(X), %xmm4

  325. 

  326. 	SHUFPD_1 %xmm6, %xmm5

  327. 	mulpd	 -6 * SIZE(Y), %xmm5

  328. 	addpd	 %xmm5,  %xmm1

  329. 	movaps	  0 * SIZE(X), %xmm5

  330. 

  331. #if defined(PREFETCH) && !defined(FETCH128)

  332. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  333. #endif

  334. 

  335. 	SHUFPD_1 %xmm7, %xmm6

  336. 	mulpd	 -4 * SIZE(Y), %xmm6

  337. 	addpd	 %xmm6, %xmm2

  338. 	movaps	  2 * SIZE(X), %xmm6

  339. 

  340. 	SHUFPD_1 %xmm4, %xmm7

  341. 	mulpd	 -2 * SIZE(Y), %xmm7

  342. 	addpd	 %xmm7, %xmm3

  343. 	movaps	  4 * SIZE(X), %xmm7

  344. 

  345. 	subl	$-16 * SIZE, X

  346. 	subl	$-16 * SIZE, Y

  347. 

  348. 	decl	%eax

  349. 	jg	.L21

  350. 	ALIGN_3

  351. 

  352. .L22:

  353. 	SHUFPD_1 %xmm5, %xmm4

  354. 	mulpd	-16 * SIZE(Y), %xmm4

  355. 	addpd	 %xmm4,  %xmm0

  356. 	movaps	-10 * SIZE(X), %xmm4

  357. 

  358. 	SHUFPD_1 %xmm6, %xmm5

  359. 	mulpd	-14 * SIZE(Y), %xmm5

  360. 	addpd	 %xmm5,  %xmm1

  361. 	movaps	 -8 * SIZE(X), %xmm5

  362. 

  363. 	SHUFPD_1 %xmm7, %xmm6

  364. 	mulpd	-12 * SIZE(Y), %xmm6

  365. 	addpd	 %xmm6,  %xmm2

  366. 	movaps	 -6 * SIZE(X), %xmm6

  367. 

  368. 	SHUFPD_1 %xmm4, %xmm7

  369. 	mulpd	-10 * SIZE(Y), %xmm7

  370. 	addpd	 %xmm7,  %xmm3

  371. 	movaps	 -4 * SIZE(X), %xmm7

  372. 

  373. 	SHUFPD_1 %xmm5, %xmm4

  374. 	mulpd	 -8 * SIZE(Y), %xmm4

  375. 	addpd	 %xmm4,  %xmm0

  376. 	movaps	 -2 * SIZE(X), %xmm4

  377. 

  378. 	SHUFPD_1 %xmm6, %xmm5

  379. 	mulpd	 -6 * SIZE(Y), %xmm5

  380. 	addpd	 %xmm5,  %xmm1

  381. 

  382. 	SHUFPD_1 %xmm7, %xmm6

  383. 	mulpd	 -4 * SIZE(Y), %xmm6

  384. 	addpd	 %xmm6, %xmm2

  385. 

  386. 	SHUFPD_1 %xmm4, %xmm7

  387. 	mulpd	 -2 * SIZE(Y), %xmm7

  388. 	addpd	 %xmm7, %xmm3

  389. 

  390. 	subl	$-16 * SIZE, X

  391. 	subl	$-16 * SIZE, Y

  392. 	ALIGN_3

  393. 

  394. .L24:

  395. 	testl	$15, N

  396. 	jle	.L999

  397. 

  398. 	testl	$8, N

  399. 	jle	.L25

  400. 

  401. 	movaps	-16 * SIZE(X), %xmm5

  402. 	movaps	-14 * SIZE(X), %xmm6

  403. 	movaps	-12 * SIZE(X), %xmm7

  404. 

  405. 	SHUFPD_1 %xmm5, %xmm4

  406. 	mulpd	-16 * SIZE(Y), %xmm4

  407. 	addpd	 %xmm4,  %xmm0

  408. 	movaps	-10 * SIZE(X), %xmm4

  409. 

  410. 	SHUFPD_1 %xmm6, %xmm5

  411. 	mulpd	-14 * SIZE(Y), %xmm5

  412. 	addpd	 %xmm5,  %xmm1

  413. 

  414. 	SHUFPD_1 %xmm7, %xmm6

  415. 	mulpd	-12 * SIZE(Y), %xmm6

  416. 	addpd	 %xmm6,  %xmm2

  417. 

  418. 	SHUFPD_1 %xmm4, %xmm7

  419. 	mulpd	-10 * SIZE(Y), %xmm7

  420. 	addpd	 %xmm7,  %xmm3

  421. 

  422. 	addl	$8 * SIZE, X

  423. 	addl	$8 * SIZE, Y

  424. 	ALIGN_3

  425. 

  426. .L25:

  427. 	testl	$4, N

  428. 	jle	.L26

  429. 

  430. 	movaps	-16 * SIZE(X), %xmm5

  431. 	movaps	-14 * SIZE(X), %xmm6

  432. 

  433. 	SHUFPD_1 %xmm5, %xmm4

  434. 	mulpd	-16 * SIZE(Y), %xmm4

  435. 	addpd	 %xmm4, %xmm0

  436. 	SHUFPD_1 %xmm6, %xmm5

  437. 	mulpd	-14 * SIZE(Y), %xmm5

  438. 	addpd	 %xmm5, %xmm1

  439. 	movapd	 %xmm6, %xmm4

  440. 

  441. 	addl	$4 * SIZE, X

  442. 	addl	$4 * SIZE, Y

  443. 	ALIGN_3

  444. 

  445. .L26:

  446. 	testl	$2, N

  447. 	jle	.L27

  448. 

  449. 	movaps	-16 * SIZE(X), %xmm5

  450. 

  451. 	SHUFPD_1 %xmm5, %xmm4

  452. 	mulpd	-16 * SIZE(Y), %xmm4

  453. 	addpd	 %xmm4,  %xmm0

  454. 	movapd	 %xmm5, %xmm4

  455. 

  456. 	addl	$2 * SIZE, X

  457. 	addl	$2 * SIZE, Y

  458. 	ALIGN_3

  459. 

  460. .L27:

  461. 	testl	$1, N

  462. 	jle	.L999

  463. 

  464. 	SHUFPD_1 %xmm4, %xmm4

  465. 	mulsd	-16 * SIZE(Y), %xmm4

  466. 	addsd	%xmm4,  %xmm0

  467. 	jmp	.L999

  468. 	ALIGN_3

  469. 

  470. #else

  471. 

  472. 	movl	N,  %eax

  473. 	sarl	$4, %eax

  474. 	jle	.L24

  475. 

  476. 	movlps	-16 * SIZE(X), %xmm4

  477. 	movhps	-15 * SIZE(X), %xmm4

  478. 	movlps	-14 * SIZE(X), %xmm5

  479. 	movhps	-13 * SIZE(X), %xmm5

  480. 	movlps	-12 * SIZE(X), %xmm6

  481. 	movhps	-11 * SIZE(X), %xmm6

  482. 	movlps	-10 * SIZE(X), %xmm7

  483. 	movhps	 -9 * SIZE(X), %xmm7

  484. 

  485. 	decl	%eax

  486. 	jle	.L22

  487. 

  488. 	ALIGN_3

  489. 

  490. .L21:

  491. #ifdef PREFETCH

  492. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  493. #endif

  494. 

  495. 	mulpd	-16 * SIZE(Y), %xmm4

  496. 	addpd	 %xmm4,  %xmm0

  497. 	movlps	 -8 * SIZE(X), %xmm4

  498. 	movhps	 -7 * SIZE(X), %xmm4

  499. 

  500. 	mulpd	-14 * SIZE(Y), %xmm5

  501. 	addpd	 %xmm5,  %xmm1

  502. 	movlps	 -6 * SIZE(X), %xmm5

  503. 	movhps	 -5 * SIZE(X), %xmm5

  504. 

  505. #ifdef PREFETCH

  506. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(Y)

  507. #endif

  508. 

  509. 	mulpd	-12 * SIZE(Y), %xmm6

  510. 	addpd	 %xmm6,  %xmm2

  511. 	movlps	 -4 * SIZE(X), %xmm6

  512. 	movhps	 -3 * SIZE(X), %xmm6

  513. 

  514. 	mulpd	-10 * SIZE(Y), %xmm7

  515. 	addpd	 %xmm7,  %xmm3

  516. 	movlps	 -2 * SIZE(X), %xmm7

  517. 	movhps	 -1 * SIZE(X), %xmm7

  518. 

  519. #if defined(PREFETCH) && !defined(FETCH128)

  520. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(X)

  521. #endif

  522. 

  523. 	mulpd	 -8 * SIZE(Y), %xmm4

  524. 	addpd	 %xmm4,  %xmm0

  525. 	movlps	  0 * SIZE(X), %xmm4

  526. 	movhps	  1 * SIZE(X), %xmm4

  527. 

  528. 	mulpd	 -6 * SIZE(Y), %xmm5

  529. 	addpd	 %xmm5,  %xmm1

  530. 	movlps	  2 * SIZE(X), %xmm5

  531. 	movhps	  3 * SIZE(X), %xmm5

  532. 

  533. #if defined(PREFETCH) && !defined(FETCH128)

  534. 	PREFETCH (PREFETCHSIZE +  64) - PREOFFSET(Y)

  535. #endif

  536. 

  537. 	mulpd	 -4 * SIZE(Y), %xmm6

  538. 	addpd	 %xmm6,  %xmm2

  539. 	movlps	  4 * SIZE(X), %xmm6

  540. 	movhps	  5 * SIZE(X), %xmm6

  541. 

  542. 	mulpd	 -2 * SIZE(Y), %xmm7

  543. 	addpd	 %xmm7,  %xmm3

  544. 	movlps	  6 * SIZE(X), %xmm7

  545. 	movhps	  7 * SIZE(X), %xmm7

  546. 

  547. 	subl	$-16 * SIZE, X

  548. 	subl	$-16 * SIZE, Y

  549. 

  550. 	decl	%eax

  551. 	jg	.L21

  552. 	ALIGN_3

  553. 

  554. .L22:

  555. 	mulpd	-16 * SIZE(Y), %xmm4

  556. 	addpd	 %xmm4,  %xmm0

  557. 	movlps	 -8 * SIZE(X), %xmm4

  558. 	movhps	 -7 * SIZE(X), %xmm4

  559. 

  560. 	mulpd	-14 * SIZE(Y), %xmm5

  561. 	addpd	 %xmm5,  %xmm1

  562. 	movlps	 -6 * SIZE(X), %xmm5

  563. 	movhps	 -5 * SIZE(X), %xmm5

  564. 

  565. 	mulpd	-12 * SIZE(Y), %xmm6

  566. 	addpd	 %xmm6,  %xmm2

  567. 	movlps	 -4 * SIZE(X), %xmm6

  568. 	movhps	 -3 * SIZE(X), %xmm6

  569. 

  570. 	mulpd	-10 * SIZE(Y), %xmm7

  571. 	addpd	 %xmm7,  %xmm3

  572. 	movlps	 -2 * SIZE(X), %xmm7

  573. 	movhps	 -1 * SIZE(X), %xmm7

  574. 

  575. 	mulpd	 -8 * SIZE(Y), %xmm4

  576. 	addpd	 %xmm4,  %xmm0

  577. 	mulpd	 -6 * SIZE(Y), %xmm5

  578. 	addpd	 %xmm5,  %xmm1

  579. 	mulpd	 -4 * SIZE(Y), %xmm6

  580. 	addpd	 %xmm6,  %xmm2

  581. 	mulpd	 -2 * SIZE(Y), %xmm7

  582. 	addpd	 %xmm7,  %xmm3

  583. 

  584. 	subl	$-16 * SIZE, X

  585. 	subl	$-16 * SIZE, Y

  586. 	ALIGN_3

  587. 

  588. .L24:

  589. 	testl	$15, N

  590. 	jle	.L999

  591. 

  592. 	testl	$8, N

  593. 	jle	.L25

  594. 

  595. 	movlps	-16 * SIZE(X), %xmm4

  596. 	movhps	-15 * SIZE(X), %xmm4

  597. 	movlps	-14 * SIZE(X), %xmm5

  598. 	movhps	-13 * SIZE(X), %xmm5

  599. 	movlps	-12 * SIZE(X), %xmm6

  600. 	movhps	-11 * SIZE(X), %xmm6

  601. 	movlps	-10 * SIZE(X), %xmm7

  602. 	movhps	 -9 * SIZE(X), %xmm7

  603. 

  604. 	mulpd	-16 * SIZE(Y), %xmm4

  605. 	addpd	 %xmm4,  %xmm0

  606. 	mulpd	-14 * SIZE(Y), %xmm5

  607. 	addpd	 %xmm5,  %xmm1

  608. 	mulpd	-12 * SIZE(Y), %xmm6

  609. 	addpd	 %xmm6,  %xmm2

  610. 	mulpd	-10 * SIZE(Y), %xmm7

  611. 	addpd	 %xmm7,  %xmm3

  612. 

  613. 	addl	$8 * SIZE, X

  614. 	addl	$8 * SIZE, Y

  615. 	ALIGN_3

  616. 

  617. .L25:

  618. 	testl	$4, N

  619. 	jle	.L26

  620. 

  621. 	movlps	-16 * SIZE(X), %xmm4

  622. 	movhps	-15 * SIZE(X), %xmm4

  623. 	movlps	-14 * SIZE(X), %xmm5

  624. 	movhps	-13 * SIZE(X), %xmm5

  625. 

  626. 	mulpd	-16 * SIZE(Y), %xmm4

  627. 	addpd	 %xmm4,  %xmm0

  628. 	mulpd	-14 * SIZE(Y), %xmm5

  629. 	addpd	 %xmm5,  %xmm1

  630. 

  631. 	addl	$4 * SIZE, X

  632. 	addl	$4 * SIZE, Y

  633. 	ALIGN_3

  634. 

  635. .L26:

  636. 	testl	$2, N

  637. 	jle	.L27

  638. 

  639. 	movlps	-16 * SIZE(X), %xmm4

  640. 	movhps	-15 * SIZE(X), %xmm4

  641. 

  642. 	mulpd	-16 * SIZE(Y), %xmm4

  643. 	addpd	 %xmm4,  %xmm0

  644. 

  645. 	addl	$2 * SIZE, X

  646. 	addl	$2 * SIZE, Y

  647. 	ALIGN_3

  648. 

  649. .L27:

  650. 	testl	$1, N

  651. 	jle	.L999

  652. 

  653. 	movsd	-16 * SIZE(X), %xmm4

  654. 	mulsd	-16 * SIZE(Y), %xmm4

  655. 	addsd	%xmm4,  %xmm0

  656. 	jmp	.L999

  657. 	ALIGN_3

  658. #endif

  659. 

  660. .L50:

  661. 	movl	N,  %eax

  662. 	sarl	$2, %eax

  663. 	jle	.L55

  664. 	ALIGN_3

  665. 

  666. .L53:

  667. 	movsd	(X), %xmm4

  668. 	addl	INCX, X

  669. 	mulsd	(Y), %xmm4

  670. 	addl	INCY, Y

  671. 	movsd	(X), %xmm5

  672. 	addl	INCX, X

  673. 	mulsd	(Y), %xmm5

  674. 	addl	INCY, Y

  675. 	movsd	(X), %xmm6

  676. 	addl	INCX, X

  677. 	mulsd	(Y), %xmm6

  678. 	addl	INCY, Y

  679. 	movsd	(X), %xmm7

  680. 	addl	INCX, X

  681. 	mulsd	(Y), %xmm7

  682. 	addl	INCY, Y

  683. 

  684. 	addsd	%xmm4, %xmm0

  685. 	addsd	%xmm5, %xmm1

  686. 	addsd	%xmm6, %xmm2

  687. 	addsd	%xmm7, %xmm3

  688. 

  689. 	decl	%eax

  690. 	jg	.L53

  691. 	ALIGN_3

  692. 

  693. .L55:

  694. 	movl	N,  %eax

  695. 	andl	$3, %eax

  696. 	jle	.L999

  697. 	ALIGN_3

  698. 

  699. .L56:

  700. 	movsd	(X), %xmm4

  701. 	addl	INCX, X

  702. 	mulsd	(Y), %xmm4

  703. 	addl	INCY, Y

  704. 	addsd	%xmm4, %xmm0

  705. 	decl	%eax

  706. 	jg	.L56

  707. 	ALIGN_3

  708. 

  709. .L999:

  710. 	addpd	%xmm1, %xmm0

  711. 	addpd	%xmm3, %xmm2

  712. 	addpd	%xmm2, %xmm0

  713. 

  714. #ifndef HAVE_SSE3

  715. 	pshufd	$0xe, %xmm0, %xmm1

  716. 	addsd	%xmm1, %xmm0

  717. #else

  718. 	haddpd	%xmm0, %xmm0

  719. #endif

  720. 	movlps	%xmm0, STACK_N

  721. 	fldl	STACK_N

  722. 

  723. 	popl	%ebx

  724. 	popl	%esi

  725. 	popl	%edi

  726. 	ret

  727. 

  728. 	EPILOGUE