OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: f66ca05b31
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'f66ca05b31'
${ noResults }
OpenBLAS/kernel/x86/gemm_kernel_8x1_sse2.S

879 lines
20 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if !defined(HAVE_SSE2) || !defined(HAVE_MMX)

  43. #error  You have to check your configuration.

  44. #endif

  45. 

  46. #define STACK	16

  47. #define ARGS	 0

  48. 

  49. #define STACK_M	 4 + STACK + ARGS(%esi)

  50. #define STACK_N	 8 + STACK + ARGS(%esi)

  51. #define STACK_K	12 + STACK + ARGS(%esi)

  52. #define STACK_ALPHA	16 + STACK + ARGS(%esi)

  53. #define STACK_A	24 + STACK + ARGS(%esi)

  54. #define STACK_B	28 + STACK + ARGS(%esi)

  55. #define STACK_C	32 + STACK + ARGS(%esi)

  56. #define STACK_LDC	36 + STACK + ARGS(%esi)

  57. 

  58. #define ALPHA	 0(%esp)

  59. #define K	16(%esp)

  60. #define N	20(%esp)

  61. #define M	24(%esp)

  62. #define A	28(%esp)

  63. #define C	36(%esp)

  64. #define J	44(%esp)

  65. #define OLD_STACK 48(%esp)

  66. #define BUFFER 128(%esp)

  67. 

  68. #define B	%edi

  69. #define LDC	%ebp

  70. 

  71. #define STACK_ALIGN	4096

  72. #define STACK_OFFSET	1024

  73. 

  74. #define AA	%edx

  75. #define BB	%ecx

  76. 

  77. #define KERNELMACRO(address) \

  78. 	mulpd	%xmm0, %xmm2; \

  79. 	mulpd	 2 * SIZE + (address) * SIZE(BB), %xmm0; \

  80. 	addpd	%xmm2, %xmm4; \

  81. 	movapd	 0 * SIZE + (address) * SIZE(BB), %xmm2; \

  82. 	addpd	%xmm0, %xmm5; \

  83. 	movapd	 2 * SIZE + (address) * SIZE(AA), %xmm0; \

  84. 	mulpd	%xmm0, %xmm2; \

  85. 	mulpd	 2 * SIZE + (address) * SIZE(BB), %xmm0; \

  86. 	addpd	%xmm2, %xmm6; \

  87. 	movapd	 4 * SIZE + (address) * SIZE(BB), %xmm2; \

  88. 	addpd	%xmm0, %xmm7; \

  89. 	movapd	 4 * SIZE + (address) * SIZE(AA), %xmm0; \

  90. 	mulpd	%xmm0, %xmm2; \

  91. 	mulpd	 6 * SIZE + (address) * SIZE(BB), %xmm0; \

  92. 	addpd	%xmm2, %xmm4; \

  93. 	movapd	 4 * SIZE + (address) * SIZE(BB), %xmm2; \

  94. 	addpd	%xmm0, %xmm5; \

  95. 	movapd	 6 * SIZE + (address) * SIZE(AA), %xmm0; \

  96. 	mulpd	%xmm0, %xmm2; \

  97. 	mulpd	 6 * SIZE + (address) * SIZE(BB), %xmm0; \

  98. 	addpd	%xmm2, %xmm6; \

  99. 	movapd	16 * SIZE + (address) * SIZE(BB), %xmm2; \

  100. 	addpd	%xmm0, %xmm7; \

  101. 	movapd	16 * SIZE + (address) * SIZE(AA), %xmm0; \

  102. 	mulpd	%xmm1, %xmm3; \

  103. 	mulpd	10 * SIZE + (address) * SIZE(BB), %xmm1; \

  104. 	addpd	%xmm3, %xmm4; \

  105. 	movapd	 8 * SIZE + (address) * SIZE(BB), %xmm3; \

  106. 	addpd	%xmm1, %xmm5; \

  107. 	movapd	10 * SIZE + (address) * SIZE(AA), %xmm1; \

  108. 	mulpd	%xmm1, %xmm3; \

  109. 	mulpd	10 * SIZE + (address) * SIZE(BB), %xmm1; \

  110. 	addpd	%xmm3, %xmm6; \

  111. 	movapd	12 * SIZE + (address) * SIZE(BB), %xmm3; \

  112. 	addpd	%xmm1, %xmm7; \

  113. 	movapd	12 * SIZE + (address) * SIZE(AA), %xmm1; \

  114. 	mulpd	%xmm1, %xmm3; \

  115. 	mulpd	14 * SIZE + (address) * SIZE(BB), %xmm1; \

  116. 	addpd	%xmm3, %xmm4; \

  117. 	movapd	12 * SIZE + (address) * SIZE(BB), %xmm3; \

  118. 	addpd	%xmm1, %xmm5; \

  119. 	movapd	14 * SIZE + (address) * SIZE(AA), %xmm1; \

  120. 	mulpd	%xmm1, %xmm3; \

  121. 	mulpd	14 * SIZE + (address) * SIZE(BB), %xmm1; \

  122. 	addpd	%xmm3, %xmm6; \

  123. 	movapd	24 * SIZE + (address) * SIZE(BB), %xmm3; \

  124. 	addpd	%xmm1, %xmm7; \

  125. 	movapd	24 * SIZE + (address) * SIZE(AA), %xmm1; \

  126. 	mulpd	%xmm0, %xmm2; \

  127. 	mulpd	18 * SIZE + (address) * SIZE(BB), %xmm0; \

  128. 	addpd	%xmm2, %xmm4; \

  129. 	movapd	16 * SIZE + (address) * SIZE(BB), %xmm2; \

  130. 	addpd	%xmm0, %xmm5; \

  131. 	movapd	18 * SIZE + (address) * SIZE(AA), %xmm0; \

  132. 	mulpd	%xmm0, %xmm2; \

  133. 	mulpd	18 * SIZE + (address) * SIZE(BB), %xmm0; \

  134. 	addpd	%xmm2, %xmm6; \

  135. 	movapd	20 * SIZE + (address) * SIZE(BB), %xmm2; \

  136. 	addpd	%xmm0, %xmm7; \

  137. 	movapd	20 * SIZE + (address) * SIZE(AA), %xmm0; \

  138. 	mulpd	%xmm0, %xmm2; \

  139. 	mulpd	22 * SIZE + (address) * SIZE(BB), %xmm0; \

  140. 	addpd	%xmm2, %xmm4; \

  141. 	movapd	20 * SIZE + (address) * SIZE(BB), %xmm2; \

  142. 	addpd	%xmm0, %xmm5; \

  143. 	movapd	22 * SIZE + (address) * SIZE(AA), %xmm0; \

  144. 	mulpd	%xmm0, %xmm2; \

  145. 	mulpd	22 * SIZE + (address) * SIZE(BB), %xmm0; \

  146. 	addpd	%xmm2, %xmm6; \

  147. 	movapd	32 * SIZE + (address) * SIZE(BB), %xmm2; \

  148. 	addpd	%xmm0, %xmm7; \

  149. 	movapd	32 * SIZE + (address) * SIZE(AA), %xmm0; \

  150. 	mulpd	%xmm1, %xmm3; \

  151. 	mulpd	26 * SIZE + (address) * SIZE(BB), %xmm1; \

  152. 	addpd	%xmm3, %xmm4; \

  153. 	movapd	24 * SIZE + (address) * SIZE(BB), %xmm3; \

  154. 	addpd	%xmm1, %xmm5; \

  155. 	movapd	26 * SIZE + (address) * SIZE(AA), %xmm1; \

  156. 	mulpd	%xmm1, %xmm3; \

  157. 	mulpd	26 * SIZE + (address) * SIZE(BB), %xmm1; \

  158. 	addpd	%xmm3, %xmm6; \

  159. 	movapd	28 * SIZE + (address) * SIZE(BB), %xmm3; \

  160. 	addpd	%xmm1, %xmm7; \

  161. 	movapd	28 * SIZE + (address) * SIZE(AA), %xmm1; \

  162. 	mulpd	%xmm1, %xmm3; \

  163. 	mulpd	30 * SIZE + (address) * SIZE(BB), %xmm1; \

  164. 	addpd	%xmm3, %xmm4; \

  165. 	movapd	28 * SIZE + (address) * SIZE(BB), %xmm3; \

  166. 	addpd	%xmm1, %xmm5; \

  167. 	movapd	30 * SIZE + (address) * SIZE(AA), %xmm1; \

  168. 	mulpd	%xmm1, %xmm3; \

  169. 	mulpd	30 * SIZE + (address) * SIZE(BB), %xmm1; \

  170. 	addpd	%xmm3, %xmm6; \

  171. 	movapd	40 * SIZE + (address) * SIZE(BB), %xmm3; \

  172. 	addpd	%xmm1, %xmm7; \

  173. 	movapd	40 * SIZE + (address) * SIZE(AA), %xmm1

  174. 

  175. 	PROLOGUE

  176. 

  177. 	pushl	%ebp

  178. 	pushl	%edi

  179. 	pushl	%esi

  180. 	pushl	%ebx

  181. 

  182. 	PROFCODE

  183. 

  184. 	EMMS

  185. 

  186. 	movl	%esp, %esi	# save old stack

  187. 

  188. 	subl	$128 + LOCAL_BUFFER_SIZE + STACK_OFFSET, %esp

  189. 	andl	$-STACK_ALIGN, %esp

  190. 	addl	$STACK_OFFSET, %esp

  191. 

  192.         STACK_TOUCHING

  193. 

  194. 	movd	STACK_M, %mm0

  195. 	movl	STACK_N, %eax

  196. 	movd	STACK_K, %mm1

  197. 	movd	STACK_A, %mm2

  198. 	movq	STACK_ALPHA,  %mm7

  199. 	movl	STACK_B, B

  200. 	movd	STACK_C, %mm3

  201. 	movl	STACK_LDC, LDC

  202. 

  203. 	movq	%mm7, 0 * SIZE + ALPHA

  204. 	movq	%mm7, 1 * SIZE + ALPHA

  205. 

  206. 	movd	%mm1, K

  207. 	movl	%eax, N

  208. 	movd	%mm0, M

  209. 	movd	%mm2, A

  210. 	movd	%mm3, C

  211. 	movl	%esi, OLD_STACK

  212. 

  213. 	leal	(, LDC, SIZE), LDC

  214. 

  215. 	test	%eax, %eax

  216. 	movl	%eax, J

  217. 	jle	.L999

  218. 	ALIGN_2

  219. 

  220. .L01:

  221. /* Copying to Sub Buffer */

  222. 	movl	K, %eax

  223. 	leal	BUFFER, %ecx

  224. 	sarl	$3, %eax

  225. 	jle	.L03

  226. 	ALIGN_4

  227. 

  228. .L02:

  229. 	prefetchnta	 96 * SIZE(B)

  230. 

  231. 	movsd	 0 * SIZE(B), %xmm0

  232. 	movsd	 1 * SIZE(B), %xmm1

  233. 	movsd	 2 * SIZE(B), %xmm2

  234. 	movsd	 3 * SIZE(B), %xmm3

  235. 	movsd	 4 * SIZE(B), %xmm4

  236. 	movsd	 5 * SIZE(B), %xmm5

  237. 	movsd	 6 * SIZE(B), %xmm6

  238. 	movsd	 7 * SIZE(B), %xmm7

  239. 

  240. 	unpcklpd  %xmm0, %xmm0

  241. 	unpcklpd  %xmm1, %xmm1

  242. 	unpcklpd  %xmm2, %xmm2

  243. 	unpcklpd  %xmm3, %xmm3

  244. 	unpcklpd  %xmm4, %xmm4

  245. 	unpcklpd  %xmm5, %xmm5

  246. 	unpcklpd  %xmm6, %xmm6

  247. 	unpcklpd  %xmm7, %xmm7

  248. 

  249. 	movapd	%xmm0,  0 * SIZE(%ecx)

  250. 	movapd	%xmm1,  2 * SIZE(%ecx)

  251. 	movapd	%xmm2,  4 * SIZE(%ecx)

  252. 	movapd	%xmm3,  6 * SIZE(%ecx)

  253. 	movapd	%xmm4,  8 * SIZE(%ecx)

  254. 	movapd	%xmm5, 10 * SIZE(%ecx)

  255. 	movapd	%xmm6, 12 * SIZE(%ecx)

  256. 	movapd	%xmm7, 14 * SIZE(%ecx)

  257. 

  258. 	addl	$ 8 * SIZE, B

  259. 	addl	$16 * SIZE, %ecx

  260. 	decl	%eax

  261. 	BRANCH

  262. 	jne	.L02

  263. 	ALIGN_2

  264. 

  265. .L03:

  266. 	movl	K, %eax

  267. 	andl	$7, %eax

  268. 	BRANCH

  269. 	jle	.L05

  270. 	ALIGN_2

  271. 

  272. .L04:

  273. 	movsd	 0 * SIZE(B), %xmm0

  274. 	unpcklpd  %xmm0, %xmm0

  275. 	movapd	%xmm0,  0 * SIZE(%ecx)

  276. 

  277. 	addl	$1 * SIZE, B

  278. 	addl	$2 * SIZE, %ecx

  279. 	decl	%eax

  280. 	jne	.L04

  281. 	ALIGN_4

  282. 

  283. .L05:

  284. 	movl	C, %esi		# coffset = c

  285. 	movl	A, %edx		# aoffset = a

  286. 	movl	M,  %ebx

  287. 	sarl	$3, %ebx	# i = (m >> 2)

  288. 	jle	.L20

  289. 	ALIGN_4

  290. 

  291. .L10:

  292. 	leal	BUFFER, %ecx	# boffset1 = boffset // different point

  293. 	movl	K, %eax

  294. 

  295. 	movapd	 0 * SIZE + BUFFER, %xmm2

  296. 	movapd	 0 * SIZE(%edx), %xmm0

  297. 	movapd	 8 * SIZE + BUFFER, %xmm3

  298. 	movapd	 8 * SIZE(%edx), %xmm1

  299. 

  300. 	pxor	%xmm4, %xmm4

  301. 	pxor	%xmm5, %xmm5

  302. 	pxor	%xmm6, %xmm6

  303. 	pxor	%xmm7, %xmm7

  304. 

  305. #if 0

  306. 	andl	$-8, %eax

  307. 	leal	(, %eax, 8), %eax

  308. 	je	.L12

  309. 

  310. 	KERNELMACRO(32 *  0)		# 0

  311. 	cmpl	$64 *  1, %eax

  312. 	jle	.L11

  313. 	KERNELMACRO(32 *  1)		# 1

  314. 	cmpl	$64 *  2, %eax

  315. 	jle	.L11

  316. 	KERNELMACRO(32 *  2)		# 2

  317. 	cmpl	$64 *  3, %eax

  318. 	jle	.L11

  319. 	KERNELMACRO(32 *  3)		# 3

  320. 	cmpl	$64 *  4, %eax

  321. 	jle	.L11

  322. 	KERNELMACRO(32 *  4)		# 4

  323. 	cmpl	$64 *  5, %eax

  324. 	jle	.L11

  325. 	KERNELMACRO(32 *  5)		# 5

  326. 	cmpl	$64 *  6, %eax

  327. 	jle	.L11

  328. 	KERNELMACRO(32 *  6)		# 6

  329. 	cmpl	$64 *  7, %eax

  330. 	jle	.L11

  331. 	KERNELMACRO(32 *  7)		# 7

  332. 	cmpl	$64 *  8, %eax

  333. 	jle	.L11

  334. 	KERNELMACRO(32 *  8)		# 8

  335. 	cmpl	$64 *  9, %eax

  336. 	jle	.L11

  337. 	KERNELMACRO(32 *  9)		# 9

  338. 	cmpl	$64 * 10, %eax

  339. 	jle	.L11

  340. 	KERNELMACRO(32 * 10)		# 10

  341. 	cmpl	$64 * 11, %eax

  342. 	jle	.L11

  343. 	KERNELMACRO(32 * 11)		# 11

  344. 	cmpl	$64 * 12, %eax

  345. 	jle	.L11

  346. 	KERNELMACRO(32 * 12)		# 12

  347. 	cmpl	$64 * 13, %eax

  348. 	jle	.L11

  349. 	KERNELMACRO(32 * 13)		# 13

  350. 	cmpl	$64 * 14, %eax

  351. 	jle	.L11

  352. 	KERNELMACRO(32 * 14)		# 14

  353. 	cmpl	$64 * 15, %eax

  354. 	jle	.L11

  355. 	movq	1 * SIZE(%esi), %mm0

  356. 	movq	1 * SIZE(%esi, LDC), %mm1

  357. 	KERNELMACRO(32 * 15)		# 15

  358. .L11:

  359. 	leal	(%edx, %eax, 4), %edx

  360. 	leal	(%ecx, %eax, 4), %ecx

  361. 

  362. #else

  363. 	movapd	 0 * SIZE(BB), %xmm0

  364. 	movapd	 8 * SIZE(BB), %xmm2

  365. 	movapd	 0 * SIZE(AA), %xmm1

  366. 	movapd	 8 * SIZE(AA), %xmm3

  367. 

  368. 	prefetchnta 8 * SIZE(%esi)

  369. 

  370. 	sarl	$3, %eax

  371. 	je	.L12

  372. 

  373. #define PRE 40

  374. 

  375. .L11:

  376. 	mulpd	%xmm0, %xmm1

  377. 	movd	(PRE + 0) * SIZE(AA), %mm0

  378. 	addpd	%xmm1, %xmm4

  379. 	movapd	 2 * SIZE(AA), %xmm1

  380. 	mulpd	%xmm0, %xmm1

  381. 	addpd	%xmm1, %xmm5

  382. 	movapd	 4 * SIZE(AA), %xmm1

  383. 	mulpd	%xmm0, %xmm1

  384. 	mulpd	 6 * SIZE(AA), %xmm0

  385. 	addpd	%xmm1, %xmm6

  386. 	movapd	16 * SIZE(AA), %xmm1

  387. 	movd	(PRE + 8) * SIZE(AA), %mm0

  388. 	addpd	%xmm0, %xmm7

  389. 	movapd	 2 * SIZE(BB), %xmm0

  390. 	mulpd	%xmm0, %xmm3

  391. 	addpd	%xmm3, %xmm4

  392. 	movapd	10 * SIZE(AA), %xmm3

  393. 	mulpd	%xmm0, %xmm3

  394. 	addpd	%xmm3, %xmm5

  395. 	movapd	12 * SIZE(AA), %xmm3

  396. 	mulpd	%xmm0, %xmm3

  397. 	mulpd	14 * SIZE(AA), %xmm0

  398. 	addpd	%xmm3, %xmm6

  399. 	movapd	24 * SIZE(AA), %xmm3

  400. 	movd	(PRE + 16) * SIZE(AA), %mm0

  401. 	addpd	%xmm0, %xmm7

  402. 	movapd	 4 * SIZE(BB), %xmm0

  403. 	mulpd	%xmm0, %xmm1

  404. 	addpd	%xmm1, %xmm4

  405. 	movapd	18 * SIZE(AA), %xmm1

  406. 	mulpd	%xmm0, %xmm1

  407. 	addpd	%xmm1, %xmm5

  408. 	movapd	20 * SIZE(AA), %xmm1

  409. 	mulpd	%xmm0, %xmm1

  410. 	mulpd	22 * SIZE(AA), %xmm0

  411. 	addpd	%xmm1, %xmm6

  412. 	movapd	32 * SIZE(AA), %xmm1

  413. 	movd	(PRE + 24) * SIZE(AA), %mm0

  414. 	addpd	%xmm0, %xmm7

  415. 	movapd	 6 * SIZE(BB), %xmm0

  416. 	mulpd	%xmm0, %xmm3

  417. 	addpd	%xmm3, %xmm4

  418. 	movapd	26 * SIZE(AA), %xmm3

  419. 	mulpd	%xmm0, %xmm3

  420. 	addpd	%xmm3, %xmm5

  421. 	movapd	28 * SIZE(AA), %xmm3

  422. 	mulpd	%xmm0, %xmm3

  423. 	mulpd	30 * SIZE(AA), %xmm0

  424. 	addpd	%xmm3, %xmm6

  425. 	movapd	40 * SIZE(AA), %xmm3

  426. 	movd	(PRE + 32) * SIZE(AA), %mm0

  427. 	addpd	%xmm0, %xmm7

  428. 	movapd	16 * SIZE(BB), %xmm0

  429. 	mulpd	%xmm2, %xmm1

  430. 	addpd	%xmm1, %xmm4

  431. 	movapd	34 * SIZE(AA), %xmm1

  432. 	mulpd	%xmm2, %xmm1

  433. 	addpd	%xmm1, %xmm5

  434. 	movapd	36 * SIZE(AA), %xmm1

  435. 	mulpd	%xmm2, %xmm1

  436. 	mulpd	38 * SIZE(AA), %xmm2

  437. 	addpd	%xmm1, %xmm6

  438. 	movapd	48 * SIZE(AA), %xmm1

  439. 	movd	(PRE + 40) * SIZE(AA), %mm0

  440. 	addpd	%xmm2, %xmm7

  441. 	movapd	10 * SIZE(BB), %xmm2

  442. 	mulpd	%xmm2, %xmm3

  443. 	addpd	%xmm3, %xmm4

  444. 	movapd	42 * SIZE(AA), %xmm3

  445. 	mulpd	%xmm2, %xmm3

  446. 	addpd	%xmm3, %xmm5

  447. 	movapd	44 * SIZE(AA), %xmm3

  448. 	mulpd	%xmm2, %xmm3

  449. 	mulpd	46 * SIZE(AA), %xmm2

  450. 	addpd	%xmm3, %xmm6

  451. 	movapd	56 * SIZE(AA), %xmm3

  452. 	movd	(PRE + 48) * SIZE(AA), %mm0

  453. 	addpd	%xmm2, %xmm7

  454. 	movapd	12 * SIZE(BB), %xmm2

  455. 	mulpd	%xmm2, %xmm1

  456. 	addpd	%xmm1, %xmm4

  457. 	movapd	50 * SIZE(AA), %xmm1

  458. 	mulpd	%xmm2, %xmm1

  459. 	addpd	%xmm1, %xmm5

  460. 	movapd	52 * SIZE(AA), %xmm1

  461. 	mulpd	%xmm2, %xmm1

  462. 	mulpd	54 * SIZE(AA), %xmm2

  463. 	addpd	%xmm1, %xmm6

  464. 	movapd	64 * SIZE(AA), %xmm1

  465. 	movd	(PRE + 56) * SIZE(AA), %mm0

  466. 	addpd	%xmm2, %xmm7

  467. 	movapd	14 * SIZE(BB), %xmm2

  468. 	mulpd	%xmm2, %xmm3

  469. 	addpd	%xmm3, %xmm4

  470. 	movapd	58 * SIZE(AA), %xmm3

  471. 	mulpd	%xmm2, %xmm3

  472. 	addpd	%xmm3, %xmm5

  473. 	movapd	60 * SIZE(AA), %xmm3

  474. 	mulpd	%xmm2, %xmm3

  475. 	mulpd	62 * SIZE(AA), %xmm2

  476. 	addpd	%xmm3, %xmm6

  477. 	movapd	72 * SIZE(AA), %xmm3

  478. 	addpd	%xmm2, %xmm7

  479. 	movapd	24 * SIZE(BB), %xmm2

  480. 

  481. 	addl   $64 * SIZE, AA

  482. 	addl   $16 * SIZE, BB

  483. 	decl   %eax

  484. 	jne    .L11

  485. #endif

  486. 

  487. .L12:

  488. 	movapd	ALPHA,  %xmm3

  489. 	movl	K, %eax

  490. 	andl	$7, %eax		# if (k & 1)

  491. 	BRANCH

  492. 	je .L14

  493. 

  494. .L13:

  495. 	movapd	 0 * SIZE(BB), %xmm0

  496. 	movapd	 0 * SIZE(AA), %xmm1

  497. 	mulpd	%xmm0, %xmm1

  498. 	addpd	%xmm1, %xmm4

  499. 	movapd	 2 * SIZE(AA), %xmm1

  500. 	mulpd	%xmm0, %xmm1

  501. 	addpd	%xmm1, %xmm5

  502. 	movapd	 4 * SIZE(AA), %xmm1

  503. 	mulpd	%xmm0, %xmm1

  504. 	addpd	%xmm1, %xmm6

  505. 	mulpd	 6 * SIZE(AA), %xmm0

  506. 	addpd	%xmm0, %xmm7

  507. 

  508. 	addl	$8 * SIZE, AA		# aoffset  += 8

  509. 	addl	$2 * SIZE, BB		# boffset1 += 8

  510. 	subl	$1, %eax

  511. 	jg	.L13

  512. 	ALIGN_4

  513. 

  514. .L14:

  515. 	mulpd	%xmm3, %xmm4

  516. 	mulpd	%xmm3, %xmm5

  517. 	mulpd	%xmm3, %xmm6

  518. 	mulpd	%xmm3, %xmm7

  519. 

  520. 	movsd	0 * SIZE(%esi), %xmm0

  521. 	movhpd	1 * SIZE(%esi), %xmm0

  522. 	movsd	2 * SIZE(%esi), %xmm1

  523. 	movhpd	3 * SIZE(%esi), %xmm1

  524. 	movsd	4 * SIZE(%esi), %xmm2

  525. 	movhpd	5 * SIZE(%esi), %xmm2

  526. 	movsd	6 * SIZE(%esi), %xmm3

  527. 	movhpd	7 * SIZE(%esi), %xmm3

  528. 

  529. 	addpd	%xmm0, %xmm4

  530. 	addpd	%xmm1, %xmm5

  531. 	addpd	%xmm2, %xmm6

  532. 	addpd	%xmm3, %xmm7

  533. 

  534. 	movsd	%xmm4, 0 * SIZE(%esi)

  535. 	movhpd	%xmm4, 1 * SIZE(%esi)

  536. 	movsd	%xmm5, 2 * SIZE(%esi)

  537. 	movhpd	%xmm5, 3 * SIZE(%esi)

  538. 	movsd	%xmm6, 4 * SIZE(%esi)

  539. 	movhpd	%xmm6, 5 * SIZE(%esi)

  540. 	movsd	%xmm7, 6 * SIZE(%esi)

  541. 	movhpd	%xmm7, 7 * SIZE(%esi)

  542. 

  543. 	addl	$8 * SIZE, %esi		# coffset += 4

  544. 	BRANCH

  545. 	decl	%ebx			# i --

  546. 	jg	.L10

  547. 	ALIGN_2

  548. 

  549. .L20:

  550. 	movl	M,  %ebx

  551. 	testl	$4, %ebx

  552. 	jle	.L30

  553. 

  554. 	leal	BUFFER, %ecx

  555. 	movl	K, %eax

  556. 

  557. 	movapd	 0 * SIZE + BUFFER, %xmm2

  558. 	movapd	 0 * SIZE(%edx), %xmm0

  559. 	movapd	 8 * SIZE + BUFFER, %xmm3

  560. 	movapd	 8 * SIZE(%edx), %xmm1

  561. 

  562. 	pxor	%xmm4, %xmm4

  563. 	pxor	%xmm5, %xmm5

  564. 	pxor	%xmm6, %xmm6

  565. 	pxor	%xmm7, %xmm7

  566. 

  567. 	sarl	$3, %eax

  568. 	je	.L22

  569. 

  570. .L21:

  571. 	movapd	 0 * SIZE(BB), %xmm0

  572. 	movapd	 0 * SIZE(AA), %xmm1

  573. 	mulpd	%xmm0, %xmm1

  574. 	addpd	%xmm1, %xmm4

  575. 	mulpd	 2 * SIZE(AA), %xmm0

  576. 	addpd	%xmm0, %xmm5

  577. 

  578. 	movapd	 2 * SIZE(BB), %xmm0

  579. 	movapd	 4 * SIZE(AA), %xmm1

  580. 	mulpd	%xmm0, %xmm1

  581. 	addpd	%xmm1, %xmm4

  582. 	mulpd	 6 * SIZE(AA), %xmm0

  583. 	addpd	%xmm0, %xmm5

  584. 

  585. 	movapd	 4 * SIZE(BB), %xmm0

  586. 	movapd	 8 * SIZE(AA), %xmm1

  587. 	mulpd	%xmm0, %xmm1

  588. 	addpd	%xmm1, %xmm4

  589. 	mulpd	10 * SIZE(AA), %xmm0

  590. 	addpd	%xmm0, %xmm5

  591. 

  592. 	movapd	 6 * SIZE(BB), %xmm0

  593. 	movapd	12 * SIZE(AA), %xmm1

  594. 	mulpd	%xmm0, %xmm1

  595. 	addpd	%xmm1, %xmm4

  596. 	mulpd	14 * SIZE(AA), %xmm0

  597. 	addpd	%xmm0, %xmm5

  598. 

  599. 	movapd	 8 * SIZE(BB), %xmm0

  600. 	movapd	16 * SIZE(AA), %xmm1

  601. 	mulpd	%xmm0, %xmm1

  602. 	addpd	%xmm1, %xmm4

  603. 	mulpd	18 * SIZE(AA), %xmm0

  604. 	addpd	%xmm0, %xmm5

  605. 

  606. 	movapd	10 * SIZE(BB), %xmm0

  607. 	movapd	20 * SIZE(AA), %xmm1

  608. 	mulpd	%xmm0, %xmm1

  609. 	addpd	%xmm1, %xmm4

  610. 	mulpd	22 * SIZE(AA), %xmm0

  611. 	addpd	%xmm0, %xmm5

  612. 

  613. 	movapd	12 * SIZE(BB), %xmm0

  614. 	movapd	24 * SIZE(AA), %xmm1

  615. 	mulpd	%xmm0, %xmm1

  616. 	addpd	%xmm1, %xmm4

  617. 	mulpd	26 * SIZE(AA), %xmm0

  618. 	addpd	%xmm0, %xmm5

  619. 

  620. 	movapd	14 * SIZE(BB), %xmm0

  621. 	movapd	28 * SIZE(AA), %xmm1

  622. 	mulpd	%xmm0, %xmm1

  623. 	addpd	%xmm1, %xmm4

  624. 	mulpd	30 * SIZE(AA), %xmm0

  625. 	addpd	%xmm0, %xmm5

  626. 

  627. 	addl   $32 * SIZE, AA

  628. 	addl   $16 * SIZE, BB

  629. 	decl   %eax

  630. 	jne    .L21

  631. 

  632. .L22:

  633. 	movapd	ALPHA,  %xmm3

  634. 	movl	K, %eax

  635. 	andl	$7, %eax

  636. 	BRANCH

  637. 	je .L24

  638. 

  639. .L23:

  640. 	movapd	 0 * SIZE(BB), %xmm0

  641. 	movapd	 0 * SIZE(AA), %xmm1

  642. 	mulpd	%xmm0, %xmm1

  643. 	addpd	%xmm1, %xmm4

  644. 	mulpd	 2 * SIZE(AA), %xmm0

  645. 	addpd	%xmm0, %xmm5

  646. 

  647. 	addl	$4 * SIZE, AA		# aoffset  += 8

  648. 	addl	$2 * SIZE, BB		# boffset1 += 8

  649. 	subl	$1, %eax

  650. 	jg	.L23

  651. 	ALIGN_4

  652. 

  653. .L24:

  654. 	mulpd	%xmm3, %xmm4

  655. 	mulpd	%xmm3, %xmm5

  656. 

  657. 	movsd	0 * SIZE(%esi), %xmm0

  658. 	movhpd	1 * SIZE(%esi), %xmm0

  659. 	movsd	2 * SIZE(%esi), %xmm1

  660. 	movhpd	3 * SIZE(%esi), %xmm1

  661. 

  662. 	addpd	%xmm0, %xmm4

  663. 	addpd	%xmm1, %xmm5

  664. 

  665. 	movsd	%xmm4, 0 * SIZE(%esi)

  666. 	movhpd	%xmm4, 1 * SIZE(%esi)

  667. 	movsd	%xmm5, 2 * SIZE(%esi)

  668. 	movhpd	%xmm5, 3 * SIZE(%esi)

  669. 	addl	$4 * SIZE, %esi		# coffset += 4

  670. 	ALIGN_4

  671. 

  672. .L30:

  673. 	movl	M,  %ebx

  674. 	testl	$2, %ebx

  675. 	jle	.L50

  676. 

  677. 	leal	BUFFER, %ecx

  678. 	movl	K, %eax

  679. 

  680. 	movapd	 0 * SIZE + BUFFER, %xmm2

  681. 	movapd	 0 * SIZE(AA), %xmm0

  682. 	movapd	 8 * SIZE + BUFFER, %xmm3

  683. 	movapd	 8 * SIZE(AA), %xmm1

  684. 

  685. 	pxor	%xmm4, %xmm4

  686. 	pxor	%xmm5, %xmm5

  687. 	pxor	%xmm6, %xmm6

  688. 	pxor	%xmm7, %xmm7

  689. 

  690. 	sarl	$3, %eax

  691. 	je	.L32

  692. 

  693. .L31:

  694. 	movapd	 0 * SIZE(BB), %xmm0

  695. 	movapd	 0 * SIZE(AA), %xmm1

  696. 	mulpd	%xmm0, %xmm1

  697. 	addpd	%xmm1, %xmm4

  698. 

  699. 	movapd	 2 * SIZE(BB), %xmm0

  700. 	movapd	 2 * SIZE(AA), %xmm1

  701. 	mulpd	%xmm0, %xmm1

  702. 	addpd	%xmm1, %xmm4

  703. 

  704. 	movapd	 4 * SIZE(BB), %xmm0

  705. 	movapd	 4 * SIZE(AA), %xmm1

  706. 	mulpd	%xmm0, %xmm1

  707. 	addpd	%xmm1, %xmm4

  708. 

  709. 	movapd	 6 * SIZE(BB), %xmm0

  710. 	movapd	 6 * SIZE(AA), %xmm1

  711. 	mulpd	%xmm0, %xmm1

  712. 	addpd	%xmm1, %xmm4

  713. 

  714. 	movapd	 8 * SIZE(BB), %xmm0

  715. 	movapd	 8 * SIZE(AA), %xmm1

  716. 	mulpd	%xmm0, %xmm1

  717. 	addpd	%xmm1, %xmm4

  718. 

  719. 	movapd	10 * SIZE(BB), %xmm0

  720. 	movapd	10 * SIZE(AA), %xmm1

  721. 	mulpd	%xmm0, %xmm1

  722. 	addpd	%xmm1, %xmm4

  723. 

  724. 	movapd	12 * SIZE(BB), %xmm0

  725. 	movapd	12 * SIZE(AA), %xmm1

  726. 	mulpd	%xmm0, %xmm1

  727. 	addpd	%xmm1, %xmm4

  728. 

  729. 	movapd	14 * SIZE(BB), %xmm0

  730. 	movapd	14 * SIZE(AA), %xmm1

  731. 	mulpd	%xmm0, %xmm1

  732. 	addpd	%xmm1, %xmm4

  733. 

  734. 	addl   $16 * SIZE, AA

  735. 	addl   $16 * SIZE, BB

  736. 	BRANCH

  737. 	decl   %eax

  738. 	jne    .L31

  739. 

  740. .L32:

  741. 	movapd	ALPHA,  %xmm3

  742. 	movl	K, %eax

  743. 	andl	$7, %eax		# if (k & 1)

  744. 	BRANCH

  745. 	je .L34

  746. 

  747. .L33:

  748. 	movapd	 0 * SIZE(BB), %xmm0

  749. 	movapd	 0 * SIZE(AA), %xmm1

  750. 	mulpd	%xmm0, %xmm1

  751. 	addpd	%xmm1, %xmm4

  752. 

  753. 	addl	$2 * SIZE, AA		# aoffset  += 8

  754. 	addl	$2 * SIZE, BB		# boffset1 += 8

  755. 	decl	%eax

  756. 	BRANCH

  757. 	jg	.L33

  758. 	ALIGN_4

  759. 

  760. .L34:

  761. 	mulpd	%xmm3, %xmm4

  762. 

  763. 	movsd	0 * SIZE(%esi), %xmm0

  764. 	movhpd	1 * SIZE(%esi), %xmm0

  765. 

  766. 	addpd	%xmm0, %xmm4

  767. 

  768. 	movsd	%xmm4, 0 * SIZE(%esi)

  769. 	movhpd	%xmm4, 1 * SIZE(%esi)

  770. 	addl	$2 * SIZE, %esi

  771. 	ALIGN_2

  772. 

  773. .L50:

  774. 	movl	M,  %ebx

  775. 	testl	$1, %ebx

  776. 	jle	.L99

  777. 

  778. 	leal	BUFFER, %ecx

  779. 	movl	K, %eax

  780. 

  781. 	movsd	 0 * SIZE + BUFFER, %xmm2

  782. 	movsd	 0 * SIZE(AA), %xmm0

  783. 	movsd	 8 * SIZE + BUFFER, %xmm3

  784. 	movsd	 4 * SIZE(AA), %xmm1

  785. 

  786. 	pxor	%xmm4, %xmm4

  787. 	pxor	%xmm5, %xmm5

  788. 	pxor	%xmm6, %xmm6

  789. 	pxor	%xmm7, %xmm7

  790. 

  791. 	sarl	$3, %eax

  792. 	je	.L52

  793. 

  794. .L51:

  795. 	movsd	 0 * SIZE(AA), %xmm0

  796. 	mulsd	 0 * SIZE(BB), %xmm0

  797. 	addsd	%xmm0, %xmm4

  798. 

  799. 	movsd	 1 * SIZE(AA), %xmm0

  800. 	mulsd	 2 * SIZE(BB), %xmm0

  801. 	addsd	%xmm0, %xmm4

  802. 

  803. 	movsd	 2 * SIZE(AA), %xmm0

  804. 	mulsd	 4 * SIZE(BB), %xmm0

  805. 	addsd	%xmm0, %xmm4

  806. 

  807. 	movsd	 3 * SIZE(AA), %xmm0

  808. 	mulsd	 6 * SIZE(BB), %xmm0

  809. 	addsd	%xmm0, %xmm4

  810. 

  811. 	movsd	 4 * SIZE(AA), %xmm0

  812. 	mulsd	 8 * SIZE(BB), %xmm0

  813. 	addsd	%xmm0, %xmm4

  814. 

  815. 	movsd	 5 * SIZE(AA), %xmm0

  816. 	mulsd	10 * SIZE(BB), %xmm0

  817. 	addsd	%xmm0, %xmm4

  818. 

  819. 	movsd	 6 * SIZE(AA), %xmm0

  820. 	mulsd	12 * SIZE(BB), %xmm0

  821. 	addsd	%xmm0, %xmm4

  822. 

  823. 	movsd	 7 * SIZE(AA), %xmm0

  824. 	mulsd	14 * SIZE(BB), %xmm0

  825. 	addsd	%xmm0, %xmm4

  826. 

  827. 	addl   $ 8 * SIZE, AA

  828. 	addl   $16 * SIZE, BB

  829. 	BRANCH

  830. 	decl   %eax

  831. 	jne    .L51

  832. 

  833. .L52:

  834. 	movsd	ALPHA,  %xmm3

  835. 	movl	K, %eax

  836. 	andl	$7, %eax		# if (k & 1)

  837. 	BRANCH

  838. 	je .L54

  839. 

  840. .L53:

  841. 	movsd	 0 * SIZE(AA), %xmm0

  842. 	mulsd	 0 * SIZE(BB), %xmm0

  843. 	addsd	%xmm0, %xmm4

  844. 

  845. 	addl	$1 * SIZE, AA		# aoffset  += 8

  846. 	addl	$2 * SIZE, BB		# boffset1 += 8

  847. 	decl	%eax

  848. 	BRANCH

  849. 	jg	.L53

  850. 	ALIGN_4

  851. 

  852. .L54:

  853. 	movsd	0 * SIZE(%esi), %xmm0

  854. 	mulsd	%xmm3, %xmm4

  855. 	addsd	%xmm0, %xmm4

  856. 	movsd	%xmm4, 0 * SIZE(%esi)

  857. 	ALIGN_2

  858. 

  859. .L99:

  860. 	addl	LDC, C

  861. 	decl	J			# j --

  862. 	jg	.L01

  863. 	ALIGN_2

  864. 

  865. .L999:

  866. 	movl	OLD_STACK, %esp

  867. 

  868. 	EMMS

  869. 

  870. 	popl	%ebx

  871. 	popl	%esi

  872. 	popl	%edi

  873. 	popl	%ebp

  874. 	ret

  875. 	ALIGN_2

  876. 

  877. 

  878. 	EPILOGUE