OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86/ztrsm_kernel_LT_1x2_sse3.S

966 lines
19 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS	16

  44. 

  45. #define M	 4 + STACK + ARGS(%esp)

  46. #define N	 8 + STACK + ARGS(%esp)

  47. #define K	12 + STACK + ARGS(%esp)

  48. #define ALPHA_R	16 + STACK + ARGS(%esp)

  49. #define ALPHA_I	24 + STACK + ARGS(%esp)

  50. #define A	32 + STACK + ARGS(%esp)

  51. #define ARG_B	36 + STACK + ARGS(%esp)

  52. #define C	40 + STACK + ARGS(%esp)

  53. #define ARG_LDC	44 + STACK + ARGS(%esp)

  54. #define OFFSET	48 + STACK + ARGS(%esp)

  55. 

  56. #define J	 0 + STACK(%esp)

  57. #define KK	 4 + STACK(%esp)

  58. #define KKK	 8 + STACK(%esp)

  59. #define AORIG	12 + STACK(%esp)

  60. 

  61. #ifdef PENTIUM4

  62. #define PREFETCH	prefetcht1

  63. #define PREFETCHSIZE 84

  64. #endif

  65. 

  66. #if defined(PENRYN) || defined(DUNNINGTON)

  67. #define PREFETCH	prefetcht1

  68. #define PREFETCHSIZE 84

  69. #endif

  70. 

  71. #ifdef PENTIUMM

  72. #define PREFETCH	prefetcht1

  73. #define PREFETCHSIZE 84

  74. #endif

  75. 

  76. #define AA	%edx

  77. #define BB	%ecx

  78. #define LDC	%ebp

  79. #define B	%edi

  80. #define CO1	%esi

  81. 

  82. #define ADDSUB	addpd

  83. 

  84. #define KERNEL1(address) \

  85. 	mulpd	 %xmm0, %xmm2; \

  86. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE + (address) * 1 * SIZE(AA); \

  87. 	addpd	 %xmm2, %xmm4; \

  88. 	movddup	 1 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  89. 	mulpd	 %xmm0, %xmm2; \

  90. 	ADDSUB	 %xmm2, %xmm5; \

  91. 	movddup	 2 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  92. 	mulpd	 %xmm0, %xmm2; \

  93. 	addpd	 %xmm2, %xmm6; \

  94. 	movddup	 3 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  95. 	mulpd	 %xmm0, %xmm2; \

  96. 	movapd	 2 * SIZE + (address) * 1 * SIZE(AA), %xmm0; \

  97. 	ADDSUB	 %xmm2, %xmm7; \

  98. 	movddup	 4 * SIZE + (address) * 2 * SIZE(BB), %xmm2

  99. 

  100. #define KERNEL2(address) \

  101. 	mulpd	 %xmm0, %xmm2; \

  102. 	addpd	 %xmm2, %xmm4; \

  103. 	movddup	 5 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  104. 	mulpd	 %xmm0, %xmm2; \

  105. 	ADDSUB	 %xmm2, %xmm5; \

  106. 	movddup	 6 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  107. 	mulpd	 %xmm0, %xmm2; \

  108. 	addpd	 %xmm2, %xmm6; \

  109. 	movddup	 7 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  110. 	mulpd	 %xmm0, %xmm2; \

  111. 	movapd	 4 * SIZE + (address) * 1 * SIZE(AA), %xmm0; \

  112. 	ADDSUB	 %xmm2, %xmm7; \

  113. 	movddup	16 * SIZE + (address) * 2 * SIZE(BB), %xmm2

  114. 

  115. #define KERNEL3(address) \

  116. 	mulpd	 %xmm0, %xmm3; \

  117. 	addpd	 %xmm3, %xmm4; \

  118. 	movddup	 9 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  119. 	mulpd	 %xmm0, %xmm3; \

  120. 	ADDSUB	 %xmm3, %xmm5; \

  121. 	movddup	10 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  122. 	mulpd	 %xmm0, %xmm3; \

  123. 	addpd	 %xmm3, %xmm6; \

  124. 	movddup	11 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  125. 	mulpd	 %xmm0, %xmm3; \

  126. 	movapd	 6 * SIZE + (address) * 1 * SIZE(AA), %xmm0; \

  127. 	ADDSUB	 %xmm3, %xmm7; \

  128. 	movddup	12 * SIZE + (address) * 2 * SIZE(BB), %xmm3

  129. 

  130. #define KERNEL4(address) \

  131. 	mulpd	 %xmm0, %xmm3; \

  132. 	addpd	 %xmm3, %xmm4; \

  133. 	movddup	13 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  134. 	mulpd	 %xmm0, %xmm3; \

  135. 	ADDSUB	 %xmm3, %xmm5; \

  136. 	movddup	14 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  137. 	mulpd	 %xmm0, %xmm3; \

  138. 	addpd	 %xmm3, %xmm6; \

  139. 	movddup	15 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  140. 	mulpd	 %xmm0, %xmm3; \

  141. 	movapd	16 * SIZE + (address) * 1 * SIZE(AA), %xmm0; \

  142. 	ADDSUB	 %xmm3, %xmm7; \

  143. 	movddup	24 * SIZE + (address) * 2 * SIZE(BB), %xmm3

  144. 

  145. #define KERNEL5(address) \

  146. 	mulpd	 %xmm1, %xmm2; \

  147. 	addpd	 %xmm2, %xmm4; \

  148. 	movddup	17 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  149. 	mulpd	 %xmm1, %xmm2; \

  150. 	ADDSUB	 %xmm2, %xmm5; \

  151. 	movddup	18 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  152. 	mulpd	 %xmm1, %xmm2; \

  153. 	addpd	 %xmm2, %xmm6; \

  154. 	movddup	19 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  155. 	mulpd	 %xmm1, %xmm2; \

  156. 	movapd	10 * SIZE + (address) * 1 * SIZE(AA), %xmm1; \

  157. 	ADDSUB	 %xmm2, %xmm7; \

  158. 	movddup	20 * SIZE + (address) * 2 * SIZE(BB), %xmm2

  159. 

  160. #define KERNEL6(address) \

  161. 	mulpd	 %xmm1, %xmm2; \

  162. 	addpd	 %xmm2, %xmm4; \

  163. 	movddup	21 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  164. 	mulpd	 %xmm1, %xmm2; \

  165. 	ADDSUB	 %xmm2, %xmm5; \

  166. 	movddup	22 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  167. 	mulpd	 %xmm1, %xmm2; \

  168. 	addpd	 %xmm2, %xmm6; \

  169. 	movddup	23 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  170. 	mulpd	 %xmm1, %xmm2; \

  171. 	movapd	12 * SIZE + (address) * 1 * SIZE(AA), %xmm1; \

  172. 	ADDSUB	 %xmm2, %xmm7

  173. 

  174. #define KERNEL7(address) \

  175. 	movddup	32 * SIZE + (address) * 2 * SIZE(BB), %xmm2; \

  176. 	mulpd	 %xmm1, %xmm3; \

  177. 	addpd	 %xmm3, %xmm4; \

  178. 	movddup	25 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  179. 	mulpd	 %xmm1, %xmm3; \

  180. 	ADDSUB	 %xmm3, %xmm5; \

  181. 	movddup	26 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  182. 	mulpd	 %xmm1, %xmm3; \

  183. 	addpd	 %xmm3, %xmm6; \

  184. 	movddup	27 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  185. 	mulpd	 %xmm1, %xmm3; \

  186. 	movapd	14 * SIZE + (address) * 1 * SIZE(AA), %xmm1; \

  187. 	ADDSUB	 %xmm3, %xmm7; \

  188. 	movddup	28 * SIZE + (address) * 2 * SIZE(BB), %xmm3

  189. 

  190. #define KERNEL8(address) \

  191. 	mulpd	 %xmm1, %xmm3; \

  192. 	addpd	 %xmm3, %xmm4; \

  193. 	movddup	29 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  194. 	mulpd	 %xmm1, %xmm3; \

  195. 	ADDSUB	 %xmm3, %xmm5; \

  196. 	movddup	30 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  197. 	mulpd	 %xmm1, %xmm3; \

  198. 	addpd	 %xmm3, %xmm6; \

  199. 	movddup	31 * SIZE + (address) * 2 * SIZE(BB), %xmm3; \

  200. 	mulpd	 %xmm1, %xmm3; \

  201. 	movapd	24 * SIZE + (address) * 1 * SIZE(AA), %xmm1; \

  202. 	ADDSUB	 %xmm3, %xmm7; \

  203. 	movddup	40 * SIZE + (address) * 2 * SIZE(BB), %xmm3

  204. 

  205. 	PROLOGUE

  206. 

  207. 	subl	$ARGS, %esp

  208. 

  209. 	pushl	%ebp

  210. 	pushl	%edi

  211. 	pushl	%esi

  212. 	pushl	%ebx

  213. 

  214. 	PROFCODE

  215. 

  216. 	movl	ARG_B,   B

  217. 	movl	ARG_LDC, LDC

  218. 	movl	OFFSET, %eax

  219. #ifdef RN

  220. 	negl	%eax

  221. #endif

  222. 	movl	%eax, KK

  223. 

  224. 	sall	$ZBASE_SHIFT, LDC

  225. 

  226. #ifdef LN

  227.        movl	M, %eax

  228.        sall	$ZBASE_SHIFT, %eax

  229.        addl	%eax, C

  230.        imull	K, %eax

  231.        addl	%eax, A

  232. #endif

  233. 

  234. #ifdef RT

  235.        movl	N, %eax

  236.        sall	$ZBASE_SHIFT, %eax

  237.        imull	K, %eax

  238.        addl	%eax, B

  239. 

  240.        movl	N, %eax

  241.        imull	LDC, %eax

  242.        addl	%eax, C

  243. #endif

  244. 

  245. #ifdef RT

  246.        movl	N, %eax

  247.        subl	OFFSET, %eax

  248.        movl	%eax, KK

  249. #endif

  250. 

  251. 	movl	N, %eax

  252. 	sarl	$1, %eax

  253. 	movl	%eax, J			# j = n

  254. 	jle	.L100

  255. 	ALIGN_4

  256. 

  257. .L01:

  258. #if defined(LT) || defined(RN)

  259. 	movl	A, AA

  260. #else

  261. 	movl	A, %eax

  262. 	movl	%eax, AORIG

  263. #endif

  264. 

  265. #ifdef RT

  266. 	movl	K, %eax

  267. 	sall	$1 + ZBASE_SHIFT, %eax

  268. 	subl	%eax, B

  269. #endif

  270. 

  271. 	leal	(, LDC, 2), %eax

  272. 

  273. #ifdef RT

  274. 	subl	%eax, C

  275. #endif

  276. 	movl	C, CO1

  277. #ifndef RT

  278. 	addl	%eax, C

  279. #endif

  280. 

  281. #ifdef LN

  282. 	movl	OFFSET, %eax

  283. 	addl	M, %eax

  284. 	movl	%eax, KK

  285. #endif

  286. 

  287. #ifdef LT

  288. 	movl	OFFSET, %eax

  289. 	movl	%eax, KK

  290. #endif

  291. 

  292. 	movl	M,  %ebx

  293. 	testl	%ebx, %ebx

  294. 	jle	.L100

  295. 	ALIGN_4

  296. 

  297. .L10:

  298. #ifdef LN

  299. 	movl	K, %eax

  300. 	sall	$ZBASE_SHIFT, %eax

  301. 	subl	%eax, AORIG

  302. #endif

  303. 

  304. #if defined(LN) || defined(RT)

  305. 	movl	KK, %eax

  306. 	movl	AORIG, AA

  307. 	sall	$ZBASE_SHIFT, %eax

  308. 	addl	%eax, AA

  309. #endif

  310. 

  311. 	movl	B, BB

  312. 

  313. #if defined(LN) || defined(RT)

  314. 	movl	KK, %eax

  315. 	sall	$1 + ZBASE_SHIFT, %eax

  316. 	addl	%eax, BB

  317. #endif

  318. 

  319. 	movapd	 0 * SIZE(AA), %xmm0

  320. 	pxor	%xmm4, %xmm4

  321. 	movapd	 8 * SIZE(AA), %xmm1

  322. 	pxor	%xmm5, %xmm5

  323. 	movddup	 0 * SIZE(BB), %xmm2

  324. 	pxor	%xmm6, %xmm6

  325. 	movddup	 8 * SIZE(BB), %xmm3

  326. 	pxor	%xmm7, %xmm7

  327. 

  328. #ifdef LN

  329. 	prefetchnta	-2 * SIZE(CO1)

  330. 	prefetchnta	-2 * SIZE(CO1, LDC, 1)

  331. #else

  332. 	prefetchnta	 2 * SIZE(CO1)

  333. 	prefetchnta	 2 * SIZE(CO1, LDC, 1)

  334. #endif

  335. 

  336. #if defined(LT) || defined(RN)

  337. 	movl	KK, %eax

  338. #else

  339. 	movl	K, %eax

  340. 	subl	KK, %eax

  341. #endif

  342. 	sarl	$3, %eax

  343. 	je	.L12

  344. 	ALIGN_4

  345. 

  346. .L11:

  347. 	KERNEL1(16  *  0)

  348. 	KERNEL2(16  *  0)

  349. 	KERNEL3(16  *  0)

  350. 	KERNEL4(16  *  0)

  351. 	KERNEL5(16  *  0)

  352. 	KERNEL6(16  *  0)

  353. 	KERNEL7(16  *  0)

  354. 	KERNEL8(16  *  0)

  355. 

  356. 	addl   $32 * SIZE, BB

  357. 	addl   $16 * SIZE, AA

  358. 	decl   %eax

  359. 	jne    .L11

  360. 	ALIGN_4

  361. 

  362. .L12:

  363. #if defined(LT) || defined(RN)

  364. 	movl	KK, %eax

  365. #else

  366. 	movl	K, %eax

  367. 	subl	KK, %eax

  368. #endif

  369. 	andl	$7, %eax		# if (k & 1)

  370. 	BRANCH

  371. 	je .L14

  372. 	ALIGN_4

  373. 

  374. .L13:

  375. 	mulpd	 %xmm0, %xmm2

  376. 	addpd	 %xmm2, %xmm4

  377. 	movddup	 1 * SIZE(BB), %xmm2

  378. 	mulpd	 %xmm0, %xmm2

  379. 	ADDSUB	 %xmm2, %xmm5

  380. 	movddup	 2 * SIZE(BB), %xmm2

  381. 	mulpd	 %xmm0, %xmm2

  382. 	addpd	 %xmm2, %xmm6

  383. 	movddup	 3 * SIZE(BB), %xmm2

  384. 	mulpd	 %xmm0, %xmm2

  385. 	movapd	 2 * SIZE(AA), %xmm0

  386. 	ADDSUB	 %xmm2, %xmm7

  387. 	movddup	 4 * SIZE(BB), %xmm2

  388. 

  389. 	addl	$2 * SIZE, AA

  390. 	addl	$4 * SIZE, BB

  391. 	decl	%eax

  392. 	jg	.L13

  393. 	ALIGN_4

  394. 

  395. .L14:

  396. #if defined(LN) || defined(RT)

  397. 	movl	KK, %eax

  398. #ifdef LN

  399. 	subl	$1, %eax

  400. #else

  401. 	subl	$2, %eax

  402. #endif

  403. 

  404. 	movl	AORIG, AA

  405. 	sall	$ZBASE_SHIFT, %eax

  406. 	leal	(AA, %eax, 1), AA

  407. 	leal	(B,  %eax, 2), BB

  408. #endif

  409. 

  410. 	pcmpeqb	%xmm1, %xmm1

  411. 	psllq	$63,   %xmm1

  412. 

  413. 	shufps	$0x40, %xmm1, %xmm1

  414. 

  415. 	SHUFPD_1 %xmm5, %xmm5

  416. 	SHUFPD_1 %xmm7, %xmm7

  417. 

  418. #ifndef CONJ

  419. 	xorpd	%xmm1, %xmm5

  420. 	xorpd	%xmm1, %xmm7

  421. 

  422. 	subpd	%xmm5, %xmm4

  423. 	subpd	%xmm7, %xmm6

  424. #else

  425. #if defined(LN) || defined(LT)

  426. 	xorpd	%xmm1, %xmm4

  427. 	xorpd	%xmm1, %xmm6

  428. #else

  429. 	xorpd	%xmm1, %xmm5

  430. 	xorpd	%xmm1, %xmm7

  431. #endif

  432. 	addpd	%xmm5, %xmm4

  433. 	addpd	%xmm7, %xmm6

  434. #endif

  435. 

  436. #if defined(LN) || defined(LT)

  437. 	movapd	 0 * SIZE(BB), %xmm5

  438. 	movapd	 2 * SIZE(BB), %xmm7

  439. 

  440. 	subpd	%xmm4,  %xmm5

  441. 	subpd	%xmm6,  %xmm7

  442. #else

  443. 	movapd	 0 * SIZE(AA), %xmm5

  444. 	movapd	 2 * SIZE(AA), %xmm7

  445. 

  446. 	subpd	%xmm4,  %xmm5

  447. 	subpd	%xmm6,  %xmm7

  448. #endif

  449. 

  450. #ifndef CONJ

  451. 	SHUFPD_1 %xmm1, %xmm1

  452. #endif

  453. 

  454. #if defined(LN) || defined(LT)

  455. 	movddup	 0 * SIZE(AA), %xmm2

  456. 	movddup	 1 * SIZE(AA), %xmm3

  457. 

  458. 	movapd	%xmm5, %xmm4

  459. 	movapd	%xmm7, %xmm6

  460. 

  461. 	SHUFPD_1 %xmm4, %xmm4

  462. 	SHUFPD_1 %xmm6, %xmm6

  463. 

  464. 	xorpd	 %xmm1, %xmm4

  465. 	xorpd	 %xmm1, %xmm6

  466. 

  467. 	mulpd	 %xmm2, %xmm5

  468. 	mulpd	 %xmm3, %xmm4

  469. 	mulpd	 %xmm2, %xmm7

  470. 	mulpd	 %xmm3, %xmm6

  471. 

  472. 	addpd	 %xmm4, %xmm5

  473. 	addpd	 %xmm6, %xmm7

  474. #endif

  475. 

  476. #ifdef RN

  477. 	movddup	 0 * SIZE(BB), %xmm2

  478. 	movddup	 1 * SIZE(BB), %xmm3

  479. 

  480. 	movapd	%xmm5, %xmm4

  481. 	SHUFPD_1 %xmm4, %xmm4

  482. 

  483. 	xorpd	 %xmm1, %xmm4

  484. 

  485. 	mulpd	 %xmm2, %xmm5

  486. 	mulpd	 %xmm3, %xmm4

  487. 

  488. 	addpd	 %xmm4, %xmm5

  489. 

  490. 	movddup	 2 * SIZE(BB), %xmm2

  491. 	movddup	 3 * SIZE(BB), %xmm3

  492. 

  493. 	movapd	 %xmm5, %xmm4

  494. 	movapd	 %xmm5, %xmm6

  495. 	SHUFPD_1 %xmm6, %xmm6

  496. 

  497. 	xorpd	 %xmm1, %xmm6

  498. 

  499. 	mulpd	 %xmm2, %xmm4

  500. 	mulpd	 %xmm3, %xmm6

  501. 

  502. 	subpd	 %xmm4, %xmm7

  503. 	subpd	 %xmm6, %xmm7

  504. 

  505. 	movddup	 6 * SIZE(BB), %xmm2

  506. 	movddup	 7 * SIZE(BB), %xmm3

  507. 

  508. 	movapd	 %xmm7, %xmm6

  509. 	SHUFPD_1 %xmm6, %xmm6

  510. 

  511. 	xorpd	 %xmm1, %xmm6

  512. 

  513. 	mulpd	 %xmm2, %xmm7

  514. 	mulpd	 %xmm3, %xmm6

  515. 

  516. 	addpd	 %xmm6, %xmm7

  517. #endif

  518. 

  519. #ifdef RT

  520. 	movddup	 6 * SIZE(BB), %xmm2

  521. 	movddup	 7 * SIZE(BB), %xmm3

  522. 

  523. 	movapd	 %xmm7, %xmm6

  524. 	SHUFPD_1 %xmm6, %xmm6

  525. 

  526. 	xorpd	 %xmm1, %xmm6

  527. 

  528. 	mulpd	 %xmm2, %xmm7

  529. 	mulpd	 %xmm3, %xmm6

  530. 

  531. 	addpd	 %xmm6, %xmm7

  532. 

  533. 	movddup	 4 * SIZE(BB), %xmm2

  534. 	movddup	 5 * SIZE(BB), %xmm3

  535. 

  536. 	movapd	 %xmm7, %xmm4

  537. 	movapd	 %xmm7, %xmm6

  538. 	SHUFPD_1 %xmm6, %xmm6

  539. 

  540. 	xorpd	 %xmm1, %xmm6

  541. 

  542. 	mulpd	 %xmm2, %xmm4

  543. 	mulpd	 %xmm3, %xmm6

  544. 

  545. 	subpd	 %xmm4, %xmm5

  546. 	subpd	 %xmm6, %xmm5

  547. 

  548. 	movddup	 0 * SIZE(BB), %xmm2

  549. 	movddup	 1 * SIZE(BB), %xmm3

  550. 

  551. 	movapd	 %xmm5, %xmm4

  552. 	SHUFPD_1 %xmm4, %xmm4

  553. 

  554. 	xorpd	 %xmm1, %xmm4

  555. 

  556. 	mulpd	 %xmm2, %xmm5

  557. 	mulpd	 %xmm3, %xmm4

  558. 

  559. 	addpd	 %xmm4, %xmm5

  560. #endif

  561. 

  562. #ifdef LN

  563. 	subl	$2 * SIZE, CO1

  564. #endif

  565. 

  566. 	movlpd	%xmm5,   0 * SIZE(CO1)

  567. 	movhpd	%xmm5,   1 * SIZE(CO1)

  568. 

  569. 	movlpd	%xmm7,   0 * SIZE(CO1, LDC)

  570. 	movhpd	%xmm7,   1 * SIZE(CO1, LDC)

  571. 

  572. #if defined(LN) || defined(LT)

  573. 	movapd	%xmm5,   0 * SIZE(BB)

  574. 	movapd	%xmm7,   2 * SIZE(BB)

  575. #else

  576. 	movapd	%xmm5,   0 * SIZE(AA)

  577. 	movapd	%xmm7,   2 * SIZE(AA)

  578. 

  579. #endif

  580. 

  581. #ifndef LN

  582. 	addl	$2 * SIZE, CO1

  583. #endif

  584. 

  585. #if defined(LT) || defined(RN)

  586. 	movl	K,  %eax

  587. 	subl	KK, %eax

  588. 	sall	$ZBASE_SHIFT, %eax

  589. 	addl	%eax, AA

  590. 	leal	(BB, %eax, 2), BB

  591. #endif

  592. 

  593. #ifdef LN

  594. 	subl	$1, KK

  595. #endif

  596. 

  597. #ifdef LT

  598. 	addl	$1, KK

  599. #endif

  600. 

  601. #ifdef RT

  602. 	movl	K, %eax

  603. 	sall	$ZBASE_SHIFT, %eax

  604. 	addl	%eax, AORIG

  605. #endif

  606. 

  607. 	decl	%ebx			# i --

  608. 	jg	.L10

  609. 	ALIGN_4

  610. 

  611. .L99:

  612. #ifdef LN

  613.        movl	K, %eax

  614.        sall	$1 + ZBASE_SHIFT, %eax

  615.        addl	%eax, B

  616. #endif

  617. 

  618. #if defined(LT) || defined(RN)

  619. 	movl	BB, B

  620. #endif

  621. 

  622. #ifdef RN

  623. 	addl	$2, KK

  624. #endif

  625. 

  626. #ifdef RT

  627. 	subl	$2, KK

  628. #endif

  629. 

  630. 	decl	J			# j --

  631. 	jg	.L01

  632. 	ALIGN_4

  633. 

  634. .L100:

  635. 	movl	N, %eax

  636. 	testl	$1, %eax

  637. 	jle	.L500

  638. 

  639. #if defined(LT) || defined(RN)

  640. 	movl	A, AA

  641. #else

  642. 	movl	A, %eax

  643. 	movl	%eax, AORIG

  644. #endif

  645. 

  646. #ifdef RT

  647. 	movl	K, %eax

  648. 	sall	$ZBASE_SHIFT, %eax

  649. 	subl	%eax, B

  650. #endif

  651. 

  652. #ifdef RT

  653. 	subl	LDC, C

  654. #endif

  655. 	movl	C, CO1

  656. #ifndef RT

  657. 	addl	LDC, C

  658. #endif

  659. 

  660. #ifdef LN

  661. 	movl	OFFSET, %eax

  662. 	addl	M, %eax

  663. 	movl	%eax, KK

  664. #endif

  665. 

  666. #ifdef LT

  667. 	movl	OFFSET, %eax

  668. 	movl	%eax, KK

  669. #endif

  670. 

  671. 	movl	M,  %ebx

  672. 	testl	%ebx, %ebx

  673. 	jle	.L500

  674. 	ALIGN_4

  675. 

  676. L110:

  677. #ifdef LN

  678. 	movl	K, %eax

  679. 	sall	$ZBASE_SHIFT, %eax

  680. 	subl	%eax, AORIG

  681. #endif

  682. 

  683. #if defined(LN) || defined(RT)

  684. 	movl	KK, %eax

  685. 	movl	AORIG, AA

  686. 	sall	$ZBASE_SHIFT, %eax

  687. 	addl	%eax, AA

  688. #endif

  689. 

  690. 	movl	B, BB

  691. 

  692. #if defined(LN) || defined(RT)

  693. 	movl	KK, %eax

  694. 	sall	$ZBASE_SHIFT, %eax

  695. 	addl	%eax, BB

  696. #endif

  697. 

  698. 	movapd	 0 * SIZE(AA), %xmm0

  699. 	pxor	%xmm4, %xmm4

  700. 	movapd	 8 * SIZE(AA), %xmm1

  701. 	pxor	%xmm5, %xmm5

  702. 	movddup	 0 * SIZE(BB), %xmm2

  703. 	pxor	%xmm6, %xmm6

  704. 	movddup	 8 * SIZE(BB), %xmm3

  705. 	pxor	%xmm7, %xmm7

  706. 

  707. #ifdef LN

  708. 	prefetchnta	-2 * SIZE(CO1)

  709. #else

  710. 	prefetchnta	 2 * SIZE(CO1)

  711. #endif

  712. 

  713. #if defined(LT) || defined(RN)

  714. 	movl	KK, %eax

  715. #else

  716. 	movl	K, %eax

  717. 	subl	KK, %eax

  718. #endif

  719. 	sarl	$3, %eax

  720. 	je	L112

  721. 	ALIGN_4

  722. 

  723. L111:

  724. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE(AA)

  725. 	mulpd	 %xmm0, %xmm2

  726. 	addpd	 %xmm2, %xmm4

  727. 	movddup	 1 * SIZE(BB), %xmm2

  728. 	mulpd	 %xmm0, %xmm2

  729. 	movapd	 2 * SIZE(AA), %xmm0

  730. 	ADDSUB	 %xmm2, %xmm5

  731. 	movddup	 2 * SIZE(BB), %xmm2

  732. 	mulpd	 %xmm0, %xmm2

  733. 	addpd	 %xmm2, %xmm6

  734. 	movddup	 3 * SIZE(BB), %xmm2

  735. 	mulpd	 %xmm0, %xmm2

  736. 	movapd	 4 * SIZE(AA), %xmm0

  737. 	ADDSUB	 %xmm2, %xmm7

  738. 	movddup	 4 * SIZE(BB), %xmm2

  739. 	mulpd	 %xmm0, %xmm2

  740. 	addpd	 %xmm2, %xmm4

  741. 	movddup	 5 * SIZE(BB), %xmm2

  742. 	mulpd	 %xmm0, %xmm2

  743. 	movapd	 6 * SIZE(AA), %xmm0

  744. 	ADDSUB	 %xmm2, %xmm5

  745. 	movddup	 6 * SIZE(BB), %xmm2

  746. 	mulpd	 %xmm0, %xmm2

  747. 	addpd	 %xmm2, %xmm6

  748. 	movddup	 7 * SIZE(BB), %xmm2

  749. 	mulpd	 %xmm0, %xmm2

  750. 	movapd	16 * SIZE(AA), %xmm0

  751. 	ADDSUB	 %xmm2, %xmm7

  752. 	movddup	16 * SIZE(BB), %xmm2

  753. 	mulpd	 %xmm1, %xmm3

  754. 	addpd	 %xmm3, %xmm4

  755. 	movddup	 9 * SIZE(BB), %xmm3

  756. 	mulpd	 %xmm1, %xmm3

  757. 	movapd	10 * SIZE(AA), %xmm1

  758. 	ADDSUB	 %xmm3, %xmm5

  759. 	movddup	10 * SIZE(BB), %xmm3

  760. 	mulpd	 %xmm1, %xmm3

  761. 	addpd	 %xmm3, %xmm6

  762. 	movddup	11 * SIZE(BB), %xmm3

  763. 	mulpd	 %xmm1, %xmm3

  764. 	movapd	12 * SIZE(AA), %xmm1

  765. 	ADDSUB	 %xmm3, %xmm7

  766. 	movddup	12 * SIZE(BB), %xmm3

  767. 	mulpd	 %xmm1, %xmm3

  768. 	addpd	 %xmm3, %xmm4

  769. 	movddup	13 * SIZE(BB), %xmm3

  770. 	mulpd	 %xmm1, %xmm3

  771. 	movapd	14 * SIZE(AA), %xmm1

  772. 	ADDSUB	 %xmm3, %xmm5

  773. 	movddup	14 * SIZE(BB), %xmm3

  774. 	mulpd	 %xmm1, %xmm3

  775. 	addpd	 %xmm3, %xmm6

  776. 	movddup	15 * SIZE(BB), %xmm3

  777. 	mulpd	 %xmm1, %xmm3

  778. 	movapd	24 * SIZE(AA), %xmm1

  779. 	ADDSUB	 %xmm3, %xmm7

  780. 	movddup	24 * SIZE(BB), %xmm3

  781. 

  782. 	addl   $16 * SIZE, AA

  783. 	addl   $16 * SIZE, BB

  784. 	decl   %eax

  785. 	jne    L111

  786. 	ALIGN_4

  787. 

  788. L112:

  789. #if defined(LT) || defined(RN)

  790. 	movl	KK, %eax

  791. #else

  792. 	movl	K, %eax

  793. 	subl	KK, %eax

  794. #endif

  795. 	andl	$7, %eax		# if (k & 1)

  796. 	BRANCH

  797. 	je L114

  798. 	ALIGN_4

  799. 

  800. L113:

  801. 	mulpd	 %xmm0, %xmm2

  802. 	addpd	 %xmm2, %xmm4

  803. 	movddup	 1 * SIZE(BB), %xmm2

  804. 	mulpd	 %xmm0, %xmm2

  805. 	movapd	 2 * SIZE(AA), %xmm0

  806. 	ADDSUB	 %xmm2, %xmm5

  807. 	movddup	 2 * SIZE(BB), %xmm2

  808. 

  809. 	addl	$2 * SIZE, AA

  810. 	addl	$2 * SIZE, BB

  811. 	decl	%eax

  812. 	jg	L113

  813. 	ALIGN_4

  814. 

  815. L114:

  816. 	addpd	%xmm6, %xmm4

  817. 	addpd	%xmm7, %xmm5

  818. 

  819. #if defined(LN) || defined(RT)

  820. 	movl	KK, %eax

  821. #ifdef LN

  822. 	subl	$1, %eax

  823. #else

  824. 	subl	$1, %eax

  825. #endif

  826. 

  827. 	movl	AORIG, AA

  828. 	sall	$ZBASE_SHIFT, %eax

  829. 	leal	(AA, %eax, 1), AA

  830. 	leal	(B,  %eax, 1), BB

  831. #endif

  832. 

  833. 	pcmpeqb	%xmm1, %xmm1

  834. 	psllq	$63,   %xmm1

  835. 

  836. 	shufps	$0x40, %xmm1, %xmm1

  837. 

  838. 	SHUFPD_1 %xmm5, %xmm5

  839. 

  840. #ifndef CONJ

  841. 	xorpd	%xmm1, %xmm5

  842. 

  843. 	subpd	%xmm5, %xmm4

  844. #else

  845. #if defined(LN) || defined(LT)

  846. 	xorpd	%xmm1, %xmm4

  847. #else

  848. 	xorpd	%xmm1, %xmm5

  849. #endif

  850. 	addpd	%xmm5, %xmm4

  851. #endif

  852. 

  853. #if defined(LN) || defined(LT)

  854. 	movapd	 0 * SIZE(BB), %xmm5

  855. 	subpd	%xmm4,  %xmm5

  856. #else

  857. 	movapd	 0 * SIZE(AA), %xmm5

  858. 	subpd	%xmm4,  %xmm5

  859. #endif

  860. 

  861. #ifndef CONJ

  862. 	SHUFPD_1 %xmm1, %xmm1

  863. #endif

  864. 

  865. #if defined(LN) || defined(LT)

  866. 	movddup	 0 * SIZE(AA), %xmm2

  867. 	movddup	 1 * SIZE(AA), %xmm3

  868. 

  869. 	movapd	 %xmm5, %xmm4

  870. 	SHUFPD_1 %xmm4, %xmm4

  871. 

  872. 	xorpd	 %xmm1, %xmm4

  873. 

  874. 	mulpd	 %xmm2, %xmm5

  875. 	mulpd	 %xmm3, %xmm4

  876. 

  877. 	addpd	 %xmm4, %xmm5

  878. #endif

  879. 

  880. #if defined(RN) || defined(RT)

  881. 	movddup	 0 * SIZE(BB), %xmm2

  882. 	movddup	 1 * SIZE(BB), %xmm3

  883. 

  884. 	movapd	 %xmm5, %xmm4

  885. 	SHUFPD_1 %xmm4, %xmm4

  886. 

  887. 	xorpd	 %xmm1, %xmm4

  888. 

  889. 	mulpd	 %xmm2, %xmm5

  890. 	mulpd	 %xmm3, %xmm4

  891. 

  892. 	addpd	 %xmm4, %xmm5

  893. #endif

  894. 

  895. #ifdef LN

  896. 	subl	$2 * SIZE, CO1

  897. #endif

  898. 

  899. 	movlpd	%xmm5,   0 * SIZE(CO1)

  900. 	movhpd	%xmm5,   1 * SIZE(CO1)

  901. 

  902. #if defined(LN) || defined(LT)

  903. 	movapd	%xmm5,   0 * SIZE(BB)

  904. #else

  905. 	movapd	%xmm5,   0 * SIZE(AA)

  906. #endif

  907. 

  908. #ifndef LN

  909. 	addl	$2 * SIZE, CO1

  910. #endif

  911. 

  912. #if defined(LT) || defined(RN)

  913. 	movl	K,  %eax

  914. 	subl	KK, %eax

  915. 	sall	$ZBASE_SHIFT, %eax

  916. 	addl	%eax, AA

  917. 	addl	%eax, BB

  918. #endif

  919. 

  920. #ifdef LN

  921. 	subl	$1, KK

  922. #endif

  923. 

  924. #ifdef LT

  925. 	addl	$1, KK

  926. #endif

  927. 

  928. #ifdef RT

  929. 	movl	K, %eax

  930. 	sall	$ZBASE_SHIFT, %eax

  931. 	addl	%eax, AORIG

  932. #endif

  933. 

  934. 	decl	%ebx			# i --

  935. 	jg	L110

  936. 

  937. #ifdef LN

  938.        movl	K, %eax

  939.        sall	$ZBASE_SHIFT, %eax

  940.        addl	%eax, B

  941. #endif

  942. 

  943. #if defined(LT) || defined(RN)

  944. 	movl	BB, B

  945. #endif

  946. 

  947. #ifdef RN

  948. 	addl	$1, KK

  949. #endif

  950. 

  951. #ifdef RT

  952. 	subl	$1, KK

  953. #endif

  954. 	ALIGN_4

  955. 

  956. .L500:

  957. 	popl	%ebx

  958. 	popl	%esi

  959. 	popl	%edi

  960. 	popl	%ebp

  961. 

  962. 	addl	$ARGS, %esp

  963. 	ret

  964. 

  965. 	EPILOGUE