OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86/zgemm_kernel_4x1_core2.S

873 lines
19 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if !defined(HAVE_SSE) || !defined(HAVE_MMX)

  43. #error  You have to check your configuration.

  44. #endif

  45. 

  46. #define STACK	16

  47. #define ARGS	 0

  48. 

  49. #define STACK_M	 4 + STACK + ARGS(%esi)

  50. #define STACK_N	 8 + STACK + ARGS(%esi)

  51. #define STACK_K	12 + STACK + ARGS(%esi)

  52. #define STACK_ALPHA_R	16 + STACK + ARGS(%esi)

  53. #define STACK_ALPHA_I	20 + STACK + ARGS(%esi)

  54. #define STACK_A	24 + STACK + ARGS(%esi)

  55. #define STACK_B	28 + STACK + ARGS(%esi)

  56. #define STACK_C	32 + STACK + ARGS(%esi)

  57. #define STACK_LDC	36 + STACK + ARGS(%esi)

  58. #define STACK_OFFT	40 + STACK + ARGS(%esi)

  59. 

  60. #define ALPHA_R	16(%esp)

  61. #define ALPHA_I	32(%esp)

  62. #define K	48(%esp)

  63. #define N	52(%esp)

  64. #define M	56(%esp)

  65. #define A	60(%esp)

  66. #define C	64(%esp)

  67. #define J	68(%esp)

  68. #define OLD_STACK 72(%esp)

  69. #define TEMP    76(%esp)

  70. #define OFFSET  80(%esp)

  71. #define KK	84(%esp)

  72. #define KKK	88(%esp)

  73. #define BUFFER 128(%esp)

  74. 

  75. #define B	%edi

  76. #define LDC	%ebp

  77. #define C1	%esi

  78. 

  79. #define STACK_ALIGN	4096

  80. #define STACK_OFFSET	1024

  81. 

  82. #define PREFETCH_R    (8 * 16 + 0)

  83. #define PREFETCH_W    (PREFETCH_R * 2)

  84. 

  85. #define PREFETCHSIZE  (8 * 16 + 4)

  86. #define PREFETCH     prefetcht0

  87. 

  88. #define AA	%edx

  89. #define BB	%ecx

  90. 

  91. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  92.     defined(RN) || defined(RT) || defined(CN) || defined(CT)

  93. #define ADDSUB	addps

  94. #else

  95. #define ADDSUB	subps

  96. #endif

  97. 

  98. 	PROLOGUE

  99. 

  100. 	pushl	%ebp

  101. 	pushl	%edi

  102. 	pushl	%esi

  103. 	pushl	%ebx

  104. 

  105. 	PROFCODE

  106. 

  107. 	movl	%esp, %esi	# save old stack

  108. 

  109. 	subl	$128 + LOCAL_BUFFER_SIZE + STACK_OFFSET, %esp

  110. 	andl	$-STACK_ALIGN, %esp	# align stack

  111. 	addl	$STACK_OFFSET, %esp

  112. 

  113. 	STACK_TOUCHING

  114. 

  115. 	movd	STACK_M, %mm0

  116. 	movl	STACK_N, %eax

  117. 	movd	STACK_K, %mm1

  118. 	movd	STACK_A, %mm2

  119. 	movl	STACK_B, B

  120. 	movd	STACK_C, %mm3

  121. 	movl	STACK_LDC, LDC

  122. #ifdef TRMMKERNEL

  123. 	movd	STACK_OFFT, %mm4

  124. #endif

  125. 

  126. 	movd	%mm1, K

  127. 	movd	%mm0, M

  128. 	movl	%eax, N

  129. 	movd	%mm2, A

  130. 	movd	%mm3, C

  131. 	movl	%esi, OLD_STACK

  132. #ifdef TRMMKERNEL

  133. 	movd	%mm4, OFFSET

  134. 	movd	%mm4, KK

  135. #ifndef LEFT

  136. 	negl	KK

  137. #endif

  138. #endif

  139. 

  140. 	subl	$-32 * SIZE, A

  141. 	subl	$-32 * SIZE, B

  142. 

  143. 	leal	(, LDC, SIZE * 2), LDC

  144. 

  145. 	movss	STACK_ALPHA_R, %xmm0

  146. 	movss	STACK_ALPHA_I, %xmm1

  147. 

  148. 	pcmpeqb	%xmm7, %xmm7

  149. 	pslld	$31, %xmm7	# Generate mask

  150. 	shufps	$0, %xmm0, %xmm0

  151. 

  152. 	movaps	 %xmm0,  0 + ALPHA_R

  153. 	movss	 %xmm1,  4 + ALPHA_I

  154. 	movss	 %xmm1, 12 + ALPHA_I

  155. 	xorps	 %xmm7, %xmm1

  156. 	movss	 %xmm1,  0 + ALPHA_I

  157. 	movss	 %xmm1,  8 + ALPHA_I

  158. 

  159. 	movl	%eax, J			# j = n

  160. 	testl	%eax, %eax

  161. 	jle	.L999

  162. 

  163. .L01:

  164.  	leal	32 * SIZE + BUFFER, BB

  165. 

  166. #if defined(TRMMKERNEL) && defined(LEFT)

  167. 	movl	OFFSET, %eax

  168. 	movl	%eax, KK

  169. #endif

  170. 

  171. 	movl	K, %eax

  172. 	sarl	$2, %eax

  173. 	jle	.L03

  174. 

  175. .L02:

  176. 	prefetcht0	(PREFETCH_R + 0) * SIZE(B)

  177. 

  178. 	movss	-32 * SIZE(B), %xmm0

  179. 	movss	-31 * SIZE(B), %xmm1

  180. 	movss	-30 * SIZE(B), %xmm2

  181. 	movss	-29 * SIZE(B), %xmm3

  182. 	movss	-28 * SIZE(B), %xmm4

  183. 	movss	-27 * SIZE(B), %xmm5

  184. 	movss	-26 * SIZE(B), %xmm6

  185. 	movss	-25 * SIZE(B), %xmm7

  186. 

  187. 	prefetcht0	(PREFETCH_W +  0) * SIZE(BB)

  188. 	shufps	$0, %xmm0, %xmm0

  189. 	shufps	$0, %xmm1, %xmm1

  190. 	shufps	$0, %xmm2, %xmm2

  191. 	shufps	$0, %xmm3, %xmm3

  192. 	shufps	$0, %xmm4, %xmm4

  193. 	shufps	$0, %xmm5, %xmm5

  194. 	shufps	$0, %xmm6, %xmm6

  195. 	shufps	$0, %xmm7, %xmm7

  196. 

  197. 	prefetcht0	(PREFETCH_W + 16) * SIZE(BB)

  198. 	movaps	%xmm0,  -32 * SIZE(BB)

  199. 	movaps	%xmm1,  -28 * SIZE(BB)

  200. 	movaps	%xmm2,  -24 * SIZE(BB)

  201. 	movaps	%xmm3,  -20 * SIZE(BB)

  202. 	movaps	%xmm4,  -16 * SIZE(BB)

  203. 	movaps	%xmm5,  -12 * SIZE(BB)

  204. 	movaps	%xmm6,   -8 * SIZE(BB)

  205. 	movaps	%xmm7,   -4 * SIZE(BB)

  206. 

  207. 	addl	$  8 * SIZE, B

  208. 	subl	$-32 * SIZE, BB

  209. 	decl	%eax

  210. 	jne	.L02

  211. 

  212. .L03:

  213. 	movl	K, %eax

  214. 	andl	$3, %eax

  215. 	BRANCH

  216. 	jle	.L05

  217. 

  218. .L04:

  219. 	movss	-32 * SIZE(B), %xmm0

  220. 	movss	-31 * SIZE(B), %xmm1

  221. 

  222. 	shufps	$0, %xmm0, %xmm0

  223. 	shufps	$0, %xmm1, %xmm1

  224. 

  225. 	movaps	%xmm0,  -32 * SIZE(BB)

  226. 	movaps	%xmm1,  -28 * SIZE(BB)

  227. 	addl	$2 * SIZE, B

  228. 	addl	$8 * SIZE, BB

  229. 	decl	%eax

  230. 	jne	.L04

  231. 	ALIGN_4

  232. 

  233. .L05:

  234. 	movl	C, C1		# coffset = c

  235. 	movl	A, AA		# aoffset = a

  236. 	movl	M,  %ebx

  237. 	sarl	$2, %ebx	# i = (m >> 2)

  238. 	jle	.L20

  239. 	ALIGN_4

  240. 

  241. .L10:

  242. #if !defined(TRMMKERNEL) || \

  243. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  244. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  245. 

  246. 	leal	32 * SIZE + BUFFER, BB

  247. #else

  248. 	leal	32 * SIZE + BUFFER, BB

  249. 	movl	KK, %eax

  250. 	leal	(, %eax,   8), %eax

  251. 	leal	(AA, %eax, 4), AA

  252. 	leal	(BB, %eax, 4), BB /* because it's doubled */

  253. #endif

  254. 

  255. 	movaps	-32 * SIZE(AA), %xmm0

  256. 	pxor	%xmm4, %xmm4

  257. 	movaps	-32 * SIZE(BB), %xmm1

  258. 	pxor	%xmm5, %xmm5

  259. 	movapd	-16 * SIZE(AA), %xmm3

  260. 	pxor	%xmm6, %xmm6

  261. 	prefetcht0	7 * SIZE(C1)

  262. 	pxor	%xmm7, %xmm7

  263. 

  264. #ifndef TRMMKERNEL

  265. 	movl	K, %eax

  266. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  267. 	movl	K, %eax

  268. 	subl	KK, %eax

  269. 	movl	%eax, KKK

  270. #else

  271. 	movl	KK, %eax

  272. #ifdef LEFT

  273. 	addl	$4, %eax

  274. #else

  275. 	addl	$1, %eax

  276. #endif

  277. 	movl	%eax, KKK

  278. #endif

  279. 	sarl	$3, %eax

  280. 	je	.L15

  281. 	ALIGN_4

  282. 

  283. .L12:

  284. 	movaps	%xmm1,  %xmm2

  285. 	mulps	%xmm0,  %xmm1

  286. 	addps	%xmm1,  %xmm4

  287. 	movaps	-28 * SIZE(BB), %xmm1

  288. 	mulps	%xmm1,  %xmm0

  289. 	ADDSUB	%xmm0,  %xmm5

  290. 	movaps	-28 * SIZE(AA), %xmm0

  291. 	mulps	%xmm0,  %xmm2

  292. 	mulps	%xmm0,  %xmm1

  293. 	movaps	-24 * SIZE(AA), %xmm0

  294. 	addps	%xmm2,  %xmm6

  295. 	ADDSUB	%xmm1,  %xmm7

  296. 

  297. 	movaps	-24 * SIZE(BB), %xmm1

  298. 	movaps	%xmm1,  %xmm2

  299. 	mulps	%xmm0,  %xmm1

  300. 	addps	%xmm1,  %xmm4

  301. 	movaps	-20 * SIZE(BB), %xmm1

  302. 	mulps	%xmm1,  %xmm0

  303. 	ADDSUB	%xmm0,  %xmm5

  304. 	movaps	-20 * SIZE(AA), %xmm0

  305. 	mulps	%xmm0,  %xmm2

  306. 	mulps	%xmm0,  %xmm1

  307. 	movaps	  0 * SIZE(AA), %xmm0

  308. 	addps	%xmm2,  %xmm6

  309. 	ADDSUB	%xmm1,  %xmm7

  310. 

  311. 	movaps	-16 * SIZE(BB), %xmm1

  312. 	movaps	%xmm1,  %xmm2

  313. 	mulps	%xmm3,  %xmm1

  314. 	addps	%xmm1,  %xmm4

  315. 	movaps	-12 * SIZE(BB), %xmm1

  316. 	mulps	%xmm1,  %xmm3

  317. 	ADDSUB	%xmm3,  %xmm5

  318. 	movaps	-12 * SIZE(AA), %xmm3

  319. 	mulps	%xmm3,  %xmm2

  320. 	mulps	%xmm3,  %xmm1

  321. 	movaps	 -8 * SIZE(AA), %xmm3

  322. 	addps	%xmm2,  %xmm6

  323. 	ADDSUB	%xmm1,  %xmm7

  324. 

  325. 	movaps	 -8 * SIZE(BB), %xmm1

  326. 	movaps	%xmm1,  %xmm2

  327. 	mulps	%xmm3,  %xmm1

  328. 	addps	%xmm1,  %xmm4

  329. 	movaps	 -4 * SIZE(BB), %xmm1

  330. 	mulps	%xmm1,  %xmm3

  331. 	ADDSUB	%xmm3,  %xmm5

  332. 	movaps	 -4 * SIZE(AA), %xmm3

  333. 	mulps	%xmm3,  %xmm2

  334. 	mulps	%xmm3,  %xmm1

  335. 	movaps	 16 * SIZE(AA), %xmm3

  336. 	addps	%xmm2,  %xmm6

  337. 	ADDSUB	%xmm1,  %xmm7

  338. 	movaps	  0 * SIZE(BB), %xmm1

  339. 

  340. 	movaps	%xmm1,  %xmm2

  341. 	mulps	%xmm0,  %xmm1

  342. 	addps	%xmm1,  %xmm4

  343. 	movaps	  4 * SIZE(BB), %xmm1

  344. 	mulps	%xmm1,  %xmm0

  345. 	ADDSUB	%xmm0,  %xmm5

  346. 	movaps	  4 * SIZE(AA), %xmm0

  347. 	mulps	%xmm0,  %xmm2

  348. 	mulps	%xmm0,  %xmm1

  349. 	movaps	  8 * SIZE(AA), %xmm0

  350. 	addps	%xmm2,  %xmm6

  351. 	ADDSUB	%xmm1,  %xmm7

  352. 

  353. 	movaps	  8 * SIZE(BB), %xmm1

  354. 	movaps	%xmm1,  %xmm2

  355. 	mulps	%xmm0,  %xmm1

  356. 	addps	%xmm1,  %xmm4

  357. 	movaps	 12 * SIZE(BB), %xmm1

  358. 	mulps	%xmm1,  %xmm0

  359. 	ADDSUB	%xmm0,  %xmm5

  360. 	movaps	 12 * SIZE(AA), %xmm0

  361. 	mulps	%xmm0,  %xmm2

  362. 	mulps	%xmm0,  %xmm1

  363. 	movaps	 32 * SIZE(AA), %xmm0

  364. 	addps	%xmm2,  %xmm6

  365. 	ADDSUB	%xmm1,  %xmm7

  366. 

  367. 	movaps	 16 * SIZE(BB), %xmm1

  368. 	movaps	%xmm1,  %xmm2

  369. 	mulps	%xmm3,  %xmm1

  370. 	addps	%xmm1,  %xmm4

  371. 	movaps	 20 * SIZE(BB), %xmm1

  372. 	mulps	%xmm1,  %xmm3

  373. 	ADDSUB	%xmm3,  %xmm5

  374. 	movaps	 20 * SIZE(AA), %xmm3

  375. 	mulps	%xmm3,  %xmm2

  376. 	mulps	%xmm3,  %xmm1

  377. 	addps	%xmm2,  %xmm6

  378. 	movaps	 24 * SIZE(AA), %xmm3

  379. 	ADDSUB	%xmm1,  %xmm7

  380. 

  381. 	movaps	 24 * SIZE(BB), %xmm1

  382. 	movaps	%xmm1,  %xmm2

  383. 	mulps	%xmm3,  %xmm1

  384. 	addps	%xmm1,  %xmm4

  385. 	movaps	 28 * SIZE(BB), %xmm1

  386. 	mulps	%xmm1,  %xmm3

  387. 	ADDSUB	%xmm3,  %xmm5

  388. 	movaps	 28 * SIZE(AA), %xmm3

  389. 	mulps	%xmm3,  %xmm2

  390. 	mulps	%xmm3,  %xmm1

  391. 	subl   $-64 * SIZE, BB

  392. 	movaps	 48 * SIZE(AA), %xmm3

  393. 	subl   $-64 * SIZE, AA

  394. 	addps	%xmm2,  %xmm6

  395. 	ADDSUB	%xmm1,  %xmm7

  396. 	movaps	-32 * SIZE(BB), %xmm1

  397. 

  398. 

  399. 	decl   %eax

  400. 	jne    .L12

  401. 	ALIGN_4

  402. 

  403. .L15:

  404. #ifndef TRMMKERNEL

  405. 	movl	K, %eax

  406. #else

  407. 	movl	KKK, %eax

  408. #endif

  409. 	andl	$7, %eax		# if (k & 1)

  410. 	BRANCH

  411. 	je .L18

  412. 

  413. .L16:

  414. 	movaps	%xmm1,  %xmm2

  415. 	mulps	%xmm0,  %xmm1

  416. 	addps	%xmm1,  %xmm4

  417. 	movaps	-28 * SIZE(BB), %xmm1

  418. 	mulps	%xmm1,  %xmm0

  419. 	ADDSUB	%xmm0,  %xmm5

  420. 	movaps	-28 * SIZE(AA), %xmm0

  421. 	mulps	%xmm0,  %xmm2

  422. 	mulps	%xmm0,  %xmm1

  423. 	movaps	-24 * SIZE(AA), %xmm0

  424. 	addps	%xmm2,  %xmm6

  425. 	ADDSUB	%xmm1,  %xmm7

  426. 	movaps	-24 * SIZE(BB), %xmm1

  427. 

  428. 	addl	$8 * SIZE, AA

  429. 	addl	$8 * SIZE, BB

  430. 	decl	%eax

  431. 	jg	.L16

  432. 

  433. .L18:

  434. 	movaps	ALPHA_R, %xmm0

  435. 	movaps	ALPHA_I, %xmm1

  436. 

  437. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  438.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  439. 	shufps	$0xb1, %xmm5, %xmm5

  440. 	shufps	$0xb1, %xmm7, %xmm7

  441. 

  442. 	addsubps	%xmm5, %xmm4

  443. 	addsubps	%xmm7, %xmm6

  444. 

  445. 	movaps	%xmm4, %xmm5

  446. 	movaps	%xmm6, %xmm7

  447. 

  448. 	shufps	$0xb1, %xmm4, %xmm4

  449. 	shufps	$0xb1, %xmm6, %xmm6

  450. #else

  451. 	shufps	$0xb1, %xmm4, %xmm4

  452. 	shufps	$0xb1, %xmm6, %xmm6

  453. 

  454. 	addsubps	%xmm4, %xmm5

  455. 	addsubps	%xmm6, %xmm7

  456. 

  457. 	movaps	%xmm5, %xmm4

  458. 	movaps	%xmm7, %xmm6

  459. 

  460. 	shufps	$0xb1, %xmm5, %xmm5

  461. 	shufps	$0xb1, %xmm7, %xmm7

  462. #endif

  463. 

  464. 	mulps	%xmm0, %xmm5

  465. 	mulps	%xmm1, %xmm4

  466. 

  467. 	mulps	%xmm0, %xmm7

  468. 	mulps	%xmm1, %xmm6

  469. 

  470. 	addps	%xmm5, %xmm4

  471. 	addps	%xmm7, %xmm6

  472. 

  473. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  474. 	movsd	0 * SIZE(C1), %xmm2

  475. 	movhps	2 * SIZE(C1), %xmm2

  476. 	movsd	4 * SIZE(C1), %xmm3

  477. 	movhps	6 * SIZE(C1), %xmm3

  478. 

  479. 	addps	%xmm2, %xmm4

  480. 	addps	%xmm3, %xmm6

  481. #endif

  482. 

  483. 	movsd	%xmm4, 0 * SIZE(C1)

  484. 	movhps	%xmm4, 2 * SIZE(C1)

  485. 	movsd	%xmm6, 4 * SIZE(C1)

  486. 	movhps	%xmm6, 6 * SIZE(C1)

  487. 

  488. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  489.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  490. 	movl	K, %eax

  491. 	subl	KKK, %eax

  492. 	leal	(,%eax,    8), %eax

  493. 	leal	(AA, %eax, 4), AA

  494. 	leal	(BB, %eax, 4), BB

  495. #endif

  496. 

  497. #if defined(TRMMKERNEL) && defined(LEFT)

  498. 	addl	$4, KK

  499. #endif

  500. 

  501. 	addl	$8 * SIZE, C1

  502. 	decl	%ebx

  503. 	jg	.L10

  504. 	ALIGN_2

  505. 

  506. .L20:

  507. 	movl	M,  %ebx

  508. 	testl	$2, %ebx

  509. 	jle	.L30

  510. 

  511. #if !defined(TRMMKERNEL) || \

  512. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  513. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  514. 

  515. 	leal	32 * SIZE + BUFFER, BB

  516. #else

  517. 

  518. 	leal	32 * SIZE + BUFFER, BB

  519. 	movl	KK, %eax

  520. 	leal	(, %eax,   8), %eax

  521. 	leal	(AA, %eax, 2), AA

  522. 	leal	(BB, %eax, 4), BB /* because it's doubled */

  523. #endif

  524. 

  525. 	movaps	-32 * SIZE(AA), %xmm0

  526. 	pxor	%xmm4, %xmm4

  527. 	movaps	-32 * SIZE(BB), %xmm1

  528. 	pxor	%xmm5, %xmm5

  529. 	movaps	-16 * SIZE(AA), %xmm2

  530. 	pxor	%xmm6, %xmm6

  531. 	movaps	-16 * SIZE(BB), %xmm3

  532. 	pxor	%xmm7, %xmm7

  533. 

  534. #ifndef TRMMKERNEL

  535. 	movl	K, %eax

  536. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  537. 	movl	K, %eax

  538. 	subl	KK, %eax

  539. 	movl	%eax, KKK

  540. #else

  541. 	movl	KK, %eax

  542. #ifdef LEFT

  543. 	addl	$2, %eax

  544. #else

  545. 	addl	$1, %eax

  546. #endif

  547. 	movl	%eax, KKK

  548. #endif

  549. 	sarl	$3, %eax

  550. 	je	.L25

  551. 	ALIGN_4

  552. 

  553. .L22:

  554. 	mulps	%xmm0,  %xmm1

  555. 	mulps	-28 * SIZE(BB), %xmm0

  556. 	addps	%xmm1, %xmm4

  557. 	movaps	-24 * SIZE(BB), %xmm1

  558. 	ADDSUB	%xmm0, %xmm5

  559. 	movaps	-28 * SIZE(AA), %xmm0

  560. 	mulps	%xmm0,  %xmm1

  561. 	mulps	-20 * SIZE(BB), %xmm0

  562. 	addps	%xmm1, %xmm6

  563. 	movaps	  0 * SIZE(BB), %xmm1

  564. 	ADDSUB	%xmm0, %xmm7

  565. 	movaps	-24 * SIZE(AA), %xmm0

  566. 	mulps	%xmm0,  %xmm3

  567. 	mulps	-12 * SIZE(BB), %xmm0

  568. 	addps	%xmm3, %xmm4

  569. 	movaps	 -8 * SIZE(BB), %xmm3

  570. 	ADDSUB	%xmm0, %xmm5

  571. 	movaps	-20 * SIZE(AA), %xmm0

  572. 	mulps	%xmm0,  %xmm3

  573. 	mulps	 -4 * SIZE(BB), %xmm0

  574. 	addps	%xmm3, %xmm6

  575. 	movaps	 16 * SIZE(BB), %xmm3

  576. 	ADDSUB	%xmm0, %xmm7

  577. 	movaps	  0 * SIZE(AA), %xmm0

  578. 	mulps	%xmm2,  %xmm1

  579. 	mulps	  4 * SIZE(BB), %xmm2

  580. 	addps	%xmm1, %xmm4

  581. 	movaps	  8 * SIZE(BB), %xmm1

  582. 	ADDSUB	%xmm2, %xmm5

  583. 	movaps	-12 * SIZE(AA), %xmm2

  584. 	mulps	%xmm2,  %xmm1

  585. 	mulps	 12 * SIZE(BB), %xmm2

  586. 	addps	%xmm1, %xmm6

  587. 	movaps	 32 * SIZE(BB), %xmm1

  588. 	ADDSUB	%xmm2, %xmm7

  589. 	movaps	 -8 * SIZE(AA), %xmm2

  590. 	mulps	%xmm2,  %xmm3

  591. 	mulps	 20 * SIZE(BB), %xmm2

  592. 	addps	%xmm3, %xmm4

  593. 	movaps	 24 * SIZE(BB), %xmm3

  594. 	ADDSUB	%xmm2, %xmm5

  595. 	movaps	 -4 * SIZE(AA), %xmm2

  596. 	mulps	%xmm2,  %xmm3

  597. 	mulps	 28 * SIZE(BB), %xmm2

  598. 	addps	%xmm3, %xmm6

  599. 	movaps	 48 * SIZE(BB), %xmm3

  600. 	ADDSUB	%xmm2, %xmm7

  601. 	movaps	 16 * SIZE(AA), %xmm2

  602. 

  603. 	subl   $-32 * SIZE, AA

  604. 	addl   $ 64 * SIZE, BB

  605. 	decl	%eax

  606. 	jne	.L22

  607. 	ALIGN_2

  608. 

  609. .L25:

  610. #ifndef TRMMKERNEL

  611. 	movl	K, %eax

  612. #else

  613. 	movl	KKK, %eax

  614. #endif

  615. 	andl	$7, %eax		# if (k & 1)

  616. 	BRANCH

  617. 	je .L28

  618. 

  619. .L26:

  620. 	mulps	%xmm0,  %xmm1

  621. 	mulps	-28 * SIZE(BB), %xmm0

  622. 	addps	%xmm1, %xmm4

  623. 	movaps	-24 * SIZE(BB), %xmm1

  624. 	ADDSUB	%xmm0, %xmm5

  625. 	movaps	-28 * SIZE(AA), %xmm0

  626. 

  627. 	addl	$4 * SIZE, AA

  628. 	addl	$8 * SIZE, BB

  629. 	decl	%eax

  630. 	jg	.L26

  631. 

  632. .L28:

  633. 	addps	%xmm6, %xmm4

  634. 	addps	%xmm7, %xmm5

  635. 

  636. 	movaps	ALPHA_R, %xmm0

  637. 	movaps	ALPHA_I, %xmm1

  638. 

  639. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  640.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  641. 	shufps	$0xb1, %xmm5, %xmm5

  642. 	addsubps	%xmm5, %xmm4

  643. 	movaps	%xmm4, %xmm5

  644. 	shufps	$0xb1, %xmm4, %xmm4

  645. #else

  646. 	shufps	$0xb1, %xmm4, %xmm4

  647. 	addsubps	%xmm4, %xmm5

  648. 	movaps	%xmm5, %xmm4

  649. 	shufps	$0xb1, %xmm5, %xmm5

  650. #endif

  651. 

  652. 	mulps	%xmm0, %xmm5

  653. 	mulps	%xmm1, %xmm4

  654. 

  655. 	addps	%xmm5, %xmm4

  656. 

  657. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  658. 	movsd	0 * SIZE(C1), %xmm2

  659. 	movhps	2 * SIZE(C1), %xmm2

  660. 

  661. 	addps	%xmm2, %xmm4

  662. #endif

  663. 

  664. 	movsd	%xmm4, 0 * SIZE(C1)

  665. 	movhps	%xmm4, 2 * SIZE(C1)

  666. 

  667. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  668.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  669. 	movl	K, %eax

  670. 	subl	KKK, %eax

  671. 	leal	(,%eax,    8), %eax

  672. 	leal	(AA, %eax, 2), AA

  673. 	leal	(BB, %eax, 4), BB

  674. #endif

  675. 

  676. #if defined(TRMMKERNEL) && defined(LEFT)

  677. 	addl	$2, KK

  678. #endif

  679. 	addl	$4 * SIZE, C1

  680. 	ALIGN_2

  681. 

  682. .L30:

  683. 	testl	$1, %ebx

  684. 	jle	.L39

  685. 

  686. #if !defined(TRMMKERNEL) || \

  687. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  688. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  689. 

  690. 	leal	32 * SIZE + BUFFER, BB

  691. #else

  692. 

  693. 	leal	32 * SIZE + BUFFER, BB

  694. 	movl	KK, %eax

  695. 	leal	(, %eax,   8), %eax

  696. 	leal	(AA, %eax, 1), AA

  697. 	leal	(BB, %eax, 4), BB /* because it's doubled */

  698. #endif

  699. 

  700. 	movsd	-32 * SIZE(AA), %xmm0

  701. 	pxor	%xmm4, %xmm4

  702. 	movsd	-32 * SIZE(BB), %xmm1

  703. 	pxor	%xmm5, %xmm5

  704. 	movsd	-24 * SIZE(AA), %xmm2

  705. 	pxor	%xmm6, %xmm6

  706. 	movsd	-16 * SIZE(BB), %xmm3

  707. 	pxor	%xmm7, %xmm7

  708. 

  709. 

  710. #ifndef TRMMKERNEL

  711. 	movl	K, %eax

  712. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  713. 	movl	K, %eax

  714. 	subl	KK, %eax

  715. 	movl	%eax, KKK

  716. #else

  717. 	movl	KK, %eax

  718. 	addl	$1, %eax

  719. 	movl	%eax, KKK

  720. #endif

  721. 	sarl	$3, %eax

  722. 	je	.L35

  723. 	ALIGN_4

  724. 

  725. .L32:

  726. 	mulps	%xmm0,  %xmm1

  727. 	mulps	-28 * SIZE(BB), %xmm0

  728. 	addps	%xmm1, %xmm4

  729.  	movsd	-24 * SIZE(BB), %xmm1

  730. 	ADDSUB	%xmm0, %xmm5

  731. 	movsd	-30 * SIZE(AA), %xmm0

  732. 	mulps	%xmm0,  %xmm1

  733. 	mulps	-20 * SIZE(BB), %xmm0

  734. 	addps	%xmm1, %xmm6

  735. 	movsd	  0 * SIZE(BB), %xmm1

  736. 	ADDSUB	%xmm0, %xmm7

  737. 	movsd	-28 * SIZE(AA), %xmm0

  738. 	mulps	%xmm0,  %xmm3

  739. 	mulps	-12 * SIZE(BB), %xmm0

  740. 	addps	%xmm3, %xmm4

  741. 	movsd	 -8 * SIZE(BB), %xmm3

  742. 	ADDSUB	%xmm0, %xmm5

  743. 	movsd	-26 * SIZE(AA), %xmm0

  744. 	mulps	%xmm0,  %xmm3

  745. 	mulps	 -4 * SIZE(BB), %xmm0

  746. 	addps	%xmm3, %xmm6

  747. 	movsd	 16 * SIZE(BB), %xmm3

  748. 	ADDSUB	%xmm0, %xmm7

  749. 	movsd	-16 * SIZE(AA), %xmm0

  750. 	mulps	%xmm2,  %xmm1

  751. 	mulps	  4 * SIZE(BB), %xmm2

  752. 	addps	%xmm1, %xmm4

  753. 	movsd	  8 * SIZE(BB), %xmm1

  754. 	ADDSUB	%xmm2, %xmm5

  755. 	movsd	-22 * SIZE(AA), %xmm2

  756. 	mulps	%xmm2,  %xmm1

  757. 	mulps	 12 * SIZE(BB), %xmm2

  758. 	addps	%xmm1, %xmm6

  759. 	movsd	 32 * SIZE(BB), %xmm1

  760. 	ADDSUB	%xmm2, %xmm7

  761. 	movsd	-20 * SIZE(AA), %xmm2

  762. 	mulps	%xmm2,  %xmm3

  763. 	mulps	 20 * SIZE(BB), %xmm2

  764. 	addps	%xmm3, %xmm4

  765. 	movsd	 24 * SIZE(BB), %xmm3

  766. 	ADDSUB	%xmm2, %xmm5

  767. 	movsd	-18 * SIZE(AA), %xmm2

  768. 	mulps	%xmm2,  %xmm3

  769. 	mulps	 28 * SIZE(BB), %xmm2

  770. 	addps	%xmm3, %xmm6

  771. 	movsd	 48 * SIZE(BB), %xmm3

  772. 	ADDSUB	%xmm2, %xmm7

  773. 	movsd	 -8 * SIZE(AA), %xmm2

  774. 

  775. 	subl   $-16 * SIZE, AA

  776. 	addl   $ 64 * SIZE, BB

  777. 

  778. 	decl	%eax

  779. 	jne	.L32

  780. 	ALIGN_2

  781. 

  782. .L35:

  783. #ifndef TRMMKERNEL

  784. 	movl	K, %eax

  785. #else

  786. 	movl	KKK, %eax

  787. #endif

  788. 	andl	$7, %eax		# if (k & 1)

  789. 	BRANCH

  790. 	je .L38

  791. 

  792. .L36:

  793. 	mulps	%xmm0,  %xmm1

  794. 	mulps	-28 * SIZE(BB), %xmm0

  795. 	addps	%xmm1, %xmm4

  796. 	movsd	-24 * SIZE(BB), %xmm1

  797. 	ADDSUB	%xmm0, %xmm5

  798. 	movsd	-30 * SIZE(AA), %xmm0

  799. 

  800. 	addl	$2 * SIZE, AA

  801. 	addl	$8 * SIZE, BB

  802. 	decl	%eax

  803. 	jg	.L36

  804. 

  805. .L38:

  806. 	addps	%xmm6, %xmm4

  807. 	addps	%xmm7, %xmm5

  808. 

  809. 	movaps	ALPHA_R, %xmm0

  810. 	movaps	ALPHA_I, %xmm1

  811. 

  812. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  813.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  814. 	shufps	$0xb1, %xmm5, %xmm5

  815. 	addsubps	%xmm5, %xmm4

  816. 	movaps	%xmm4, %xmm5

  817. 	shufps	$0xb1, %xmm4, %xmm4

  818. #else

  819. 	shufps	$0xb1, %xmm4, %xmm4

  820. 	addsubps	%xmm4, %xmm5

  821. 	movaps	%xmm5, %xmm4

  822. 	shufps	$0xb1, %xmm5, %xmm5

  823. #endif

  824. 

  825. 	mulps	%xmm0, %xmm5

  826. 	mulps	%xmm1, %xmm4

  827. 

  828. 	addps	%xmm5, %xmm4

  829. 

  830. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  831. 	movsd	0 * SIZE(C1), %xmm2

  832. 	addps	%xmm2, %xmm4

  833. #endif

  834. 

  835. 	movsd	%xmm4, 0 * SIZE(C1)

  836. 

  837. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  838.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  839. 	movl	K, %eax

  840. 	subl	KKK, %eax

  841. 	leal	(,%eax,    8), %eax

  842. 	leal	(AA, %eax, 1), AA

  843. 	leal	(BB, %eax, 4), BB

  844. #endif

  845. 

  846. #if defined(TRMMKERNEL) && defined(LEFT)

  847. 	addl	$1, KK

  848. #endif

  849. 	ALIGN_2

  850. 

  851. .L39:

  852. #if defined(TRMMKERNEL) && !defined(LEFT)

  853. 	addl	$1, KK

  854. #endif

  855. 

  856. 	addl	LDC, C			# c += ldc

  857. 	decl	J			# j --

  858. 	jg	.L01

  859. 	ALIGN_2

  860. 

  861. .L999:

  862. 	movl	OLD_STACK, %esp

  863. 

  864. 	EMMS

  865. 

  866. 	popl	%ebx

  867. 	popl	%esi

  868. 	popl	%edi

  869. 	popl	%ebp

  870. 	ret

  871. 

  872. 	EPILOGUE