OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: e740c4873d
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'e740c4873d'
${ noResults }
OpenBLAS/kernel/x86/ztrsm_kernel_LT_1x2_penryn.S

970 lines
17 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS	16

  44. 

  45. #define M	 4 + STACK + ARGS(%esp)

  46. #define N	 8 + STACK + ARGS(%esp)

  47. #define K	12 + STACK + ARGS(%esp)

  48. #define ALPHA_R	16 + STACK + ARGS(%esp)

  49. #define ALPHA_I	24 + STACK + ARGS(%esp)

  50. #define A	32 + STACK + ARGS(%esp)

  51. #define ARG_B	36 + STACK + ARGS(%esp)

  52. #define C	40 + STACK + ARGS(%esp)

  53. #define ARG_LDC	44 + STACK + ARGS(%esp)

  54. #define OFFSET	48 + STACK + ARGS(%esp)

  55. 

  56. #define J	 0 + STACK(%esp)

  57. #define KK	 4 + STACK(%esp)

  58. #define KKK	 8 + STACK(%esp)

  59. #define AORIG	12 + STACK(%esp)

  60. 

  61. #if defined(PENRYN) || defined(DUNNINGTON)

  62. #define PREFETCH	prefetcht1

  63. #define PREFETCHSIZE 84

  64. #endif

  65. 

  66. #if defined(NEHALEM) || defined(SANDYBRIDGE) || defined(HASWELL) || defined (SKYLAKEX) || defined (COOPERLAKE)

  67. #define PREFETCH	prefetcht1

  68. #define PREFETCHSIZE 84

  69. #endif

  70. 

  71. #ifdef NANO

  72. #define PREFETCH	prefetcht0

  73. #define PREFETCHSIZE (8 * 2)

  74. #endif

  75. 

  76. #define AA	%edx

  77. #define BB	%ecx

  78. #define LDC	%ebp

  79. #define B	%edi

  80. #define CO1	%esi

  81. 

  82. #define ADD1	  addpd

  83. #define ADD2	  addpd

  84. 

  85. 	PROLOGUE

  86. 

  87. 	subl	$ARGS, %esp

  88. 

  89. 	pushl	%ebp

  90. 	pushl	%edi

  91. 	pushl	%esi

  92. 	pushl	%ebx

  93. 

  94. 	PROFCODE

  95. 

  96. 	movl	ARG_B,   B

  97. 	movl	ARG_LDC, LDC

  98. 	movl	OFFSET, %eax

  99. #ifdef RN

  100. 	negl	%eax

  101. #endif

  102. 	movl	%eax, KK

  103. 

  104. 	movl	M,    %ebx

  105. 	testl	%ebx, %ebx

  106. 	jle	.L999

  107. 

  108. 	subl	$-16 * SIZE, A

  109. 	subl	$-16 * SIZE, B

  110. 

  111. 	sall	$ZBASE_SHIFT, LDC

  112. 

  113. #ifdef LN

  114.        movl	M, %eax

  115.        sall	$ZBASE_SHIFT, %eax

  116.        addl	%eax, C

  117.        imull	K, %eax

  118.        addl	%eax, A

  119. #endif

  120. 

  121. #ifdef RT

  122.        movl	N, %eax

  123.        sall	$ZBASE_SHIFT, %eax

  124.        imull	K, %eax

  125.        addl	%eax, B

  126. 

  127.        movl	N, %eax

  128.        imull	LDC, %eax

  129.        addl	%eax, C

  130. #endif

  131. 

  132. #ifdef RT

  133.        movl	N, %eax

  134.        subl	OFFSET, %eax

  135.        movl	%eax, KK

  136. #endif

  137. 

  138. 	movl	N, %eax

  139. 	sarl	$1, %eax

  140. 	movl	%eax, J			# j = n

  141. 	jle	.L100

  142. 	ALIGN_4

  143. 

  144. .L01:

  145. #if defined(LT) || defined(RN)

  146. 	movl	A, AA

  147. #else

  148. 	movl	A, %eax

  149. 	movl	%eax, AORIG

  150. #endif

  151. 

  152. #ifdef RT

  153. 	movl	K, %eax

  154. 	sall	$1 + ZBASE_SHIFT, %eax

  155. 	subl	%eax, B

  156. #endif

  157. 

  158. 	leal	(, LDC, 2), %eax

  159. 

  160. #ifdef RT

  161. 	subl	%eax, C

  162. #endif

  163. 	movl	C, CO1

  164. #ifndef RT

  165. 	addl	%eax, C

  166. #endif

  167. 

  168. #ifdef LN

  169. 	movl	OFFSET, %eax

  170. 	addl	M, %eax

  171. 	movl	%eax, KK

  172. #endif

  173. 

  174. #ifdef LT

  175. 	movl	OFFSET, %eax

  176. 	movl	%eax, KK

  177. #endif

  178. 

  179. 	movl	M,  %ebx

  180. 	ALIGN_4

  181. 

  182. .L10:

  183. #ifdef LN

  184. 	movl	K, %eax

  185. 	sall	$ZBASE_SHIFT, %eax

  186. 	subl	%eax, AORIG

  187. #endif

  188. 

  189. #if defined(LN) || defined(RT)

  190. 	movl	KK, %eax

  191. 	movl	AORIG, AA

  192. 	sall	$ZBASE_SHIFT, %eax

  193. 	addl	%eax, AA

  194. #endif

  195. 

  196. 	movl	B, BB

  197. 

  198. #if defined(LN) || defined(RT)

  199. 	movl	KK, %eax

  200. 	sall	$1 + ZBASE_SHIFT, %eax

  201. 	addl	%eax, BB

  202. #endif

  203. 

  204. 	movaps	-16 * SIZE(AA), %xmm0

  205. 	pxor	%xmm2, %xmm2

  206. 	movaps	-16 * SIZE(BB), %xmm1

  207. 	pxor	%xmm3, %xmm3

  208. 

  209. #ifdef LN

  210. 	pxor	%xmm4, %xmm4

  211. 	prefetcht0     -2 * SIZE(CO1)

  212. 	pxor	%xmm5, %xmm5

  213. 	prefetcht0     -2 * SIZE(CO1, LDC)

  214. #else

  215. 	pxor	%xmm4, %xmm4

  216. 	prefetcht0	1 * SIZE(CO1)

  217. 	pxor	%xmm5, %xmm5

  218. 	prefetcht0	1 * SIZE(CO1, LDC)

  219. #endif

  220. 	pxor	%xmm6, %xmm6

  221. 	pxor	%xmm7, %xmm7

  222. 

  223. #if defined(LT) || defined(RN)

  224. 	movl	KK, %eax

  225. #else

  226. 	movl	K, %eax

  227. 	subl	KK, %eax

  228. #endif

  229. 	sarl	$3, %eax

  230. 	je	.L15

  231. 	ALIGN_4

  232. 

  233. .L12:

  234. 	PREFETCH (PREFETCHSIZE +  0) * SIZE(AA)

  235. 

  236. 	ADD1	%xmm3, %xmm6

  237. 	movaps	-14 * SIZE(BB), %xmm3

  238. 	ADD2	%xmm2, %xmm7

  239. 	pshufd	$0x4e, %xmm1, %xmm2

  240. 	mulpd	%xmm0, %xmm1

  241. 	mulpd	%xmm0, %xmm2

  242. 

  243. 	ADD1	%xmm1, %xmm4

  244. 	movaps	-12 * SIZE(BB), %xmm1

  245. 	ADD2	%xmm2, %xmm5

  246. 	pshufd	$0x4e, %xmm3, %xmm2

  247. 	mulpd	%xmm0, %xmm3

  248. 	mulpd	%xmm0, %xmm2

  249. 	movaps	-14 * SIZE(AA), %xmm0

  250. 

  251. 	ADD1	%xmm3, %xmm6

  252. 	movaps	-10 * SIZE(BB), %xmm3

  253. 	ADD2	%xmm2, %xmm7

  254. 	pshufd	$0x4e, %xmm1, %xmm2

  255. 	mulpd	%xmm0, %xmm1

  256. 	mulpd	%xmm0, %xmm2

  257. 

  258. 	ADD1	%xmm1, %xmm4

  259. 	movaps	 -8 * SIZE(BB), %xmm1

  260. 	ADD2	%xmm2, %xmm5

  261. 	pshufd	$0x4e, %xmm3, %xmm2

  262. 	mulpd	%xmm0, %xmm3

  263. 	mulpd	%xmm0, %xmm2

  264. 	movaps	-12 * SIZE(AA), %xmm0

  265. 

  266. 	ADD1	%xmm3, %xmm6

  267. 	movaps	 -6 * SIZE(BB), %xmm3

  268. 	ADD2	%xmm2, %xmm7

  269. 	pshufd	$0x4e, %xmm1, %xmm2

  270. 	mulpd	%xmm0, %xmm1

  271. 	mulpd	%xmm0, %xmm2

  272. 

  273. 	ADD1	%xmm1, %xmm4

  274. 	movaps	 -4 * SIZE(BB), %xmm1

  275. 	ADD2	%xmm2, %xmm5

  276. 	pshufd	$0x4e, %xmm3, %xmm2

  277. 	mulpd	%xmm0, %xmm3

  278. 	mulpd	%xmm0, %xmm2

  279. 	movaps	-10 * SIZE(AA), %xmm0

  280. 

  281. 	ADD1	%xmm3, %xmm6

  282. 	movaps	 -2 * SIZE(BB), %xmm3

  283. 	ADD2	%xmm2, %xmm7

  284. 	pshufd	$0x4e, %xmm1, %xmm2

  285. 	mulpd	%xmm0, %xmm1

  286. 	mulpd	%xmm0, %xmm2

  287. 

  288. 	ADD1	%xmm1, %xmm4

  289. 	movaps	  0 * SIZE(BB), %xmm1

  290. 	ADD2	%xmm2, %xmm5

  291. 	pshufd	$0x4e, %xmm3, %xmm2

  292. 	mulpd	%xmm0, %xmm3

  293. 	mulpd	%xmm0, %xmm2

  294. 	movaps	 -8 * SIZE(AA), %xmm0

  295. 

  296. 	PREFETCH (PREFETCHSIZE +  8) * SIZE(AA)

  297. 

  298. 	ADD1	%xmm3, %xmm6

  299. 	movaps	  2 * SIZE(BB), %xmm3

  300. 	ADD2	%xmm2, %xmm7

  301. 	pshufd	$0x4e, %xmm1, %xmm2

  302. 	mulpd	%xmm0, %xmm1

  303. 	mulpd	%xmm0, %xmm2

  304. 

  305. 	ADD1	%xmm1, %xmm4

  306. 	movaps	  4 * SIZE(BB), %xmm1

  307. 	ADD2	%xmm2, %xmm5

  308. 	pshufd	$0x4e, %xmm3, %xmm2

  309. 	mulpd	%xmm0, %xmm3

  310. 	mulpd	%xmm0, %xmm2

  311. 	movaps	 -6 * SIZE(AA), %xmm0

  312. 

  313. 	ADD1	%xmm3, %xmm6

  314. 	movaps	  6 * SIZE(BB), %xmm3

  315. 	ADD2	%xmm2, %xmm7

  316. 	pshufd	$0x4e, %xmm1, %xmm2

  317. 	mulpd	%xmm0, %xmm1

  318. 	mulpd	%xmm0, %xmm2

  319. 

  320. 	ADD1	%xmm1, %xmm4

  321. 	movaps	  8 * SIZE(BB), %xmm1

  322. 	ADD2	%xmm2, %xmm5

  323. 	pshufd	$0x4e, %xmm3, %xmm2

  324. 	mulpd	%xmm0, %xmm3

  325. 	mulpd	%xmm0, %xmm2

  326. 	movaps	 -4 * SIZE(AA), %xmm0

  327. 

  328. 	ADD1	%xmm3, %xmm6

  329. 	movaps	 10 * SIZE(BB), %xmm3

  330. 	ADD2	%xmm2, %xmm7

  331. 	pshufd	$0x4e, %xmm1, %xmm2

  332. 	mulpd	%xmm0, %xmm1

  333. 	mulpd	%xmm0, %xmm2

  334. 

  335. 	ADD1	%xmm1, %xmm4

  336. 	movaps	 12 * SIZE(BB), %xmm1

  337. 	ADD2	%xmm2, %xmm5

  338. 	pshufd	$0x4e, %xmm3, %xmm2

  339. 	mulpd	%xmm0, %xmm3

  340. 	mulpd	%xmm0, %xmm2

  341. 	movaps	 -2 * SIZE(AA), %xmm0

  342. 

  343. 	ADD1	%xmm3, %xmm6

  344. 	movaps	 14 * SIZE(BB), %xmm3

  345. 	ADD2	%xmm2, %xmm7

  346. 	pshufd	$0x4e, %xmm1, %xmm2

  347. 	mulpd	%xmm0, %xmm1

  348. 	mulpd	%xmm0, %xmm2

  349. 

  350. 	ADD1	%xmm1, %xmm4

  351. 	movaps	 16 * SIZE(BB), %xmm1

  352. 	ADD2	%xmm2, %xmm5

  353. 	pshufd	$0x4e, %xmm3, %xmm2

  354. 	mulpd	%xmm0, %xmm3

  355. 	subl   $-32 * SIZE, BB

  356. 	mulpd	%xmm0, %xmm2

  357. 	movaps	  0 * SIZE(AA), %xmm0

  358. 

  359. 	subl   $-16 * SIZE, AA

  360. 

  361. 	subl   $1, %eax

  362. 	jne    .L12

  363. 	ALIGN_4

  364. 

  365. .L15:

  366. #if defined(LT) || defined(RN)

  367. 	movl	KK, %eax

  368. #else

  369. 	movl	K, %eax

  370. 	subl	KK, %eax

  371. #endif

  372. 	andl	$7, %eax		# if (k & 1)

  373. 	BRANCH

  374. 	je .L18

  375. 	ALIGN_4

  376. 

  377. .L16:

  378. 	ADD1	%xmm3, %xmm6

  379. 	movaps	-14 * SIZE(BB), %xmm3

  380. 	ADD2	%xmm2, %xmm7

  381. 	pshufd	$0x4e, %xmm1, %xmm2

  382. 	mulpd	%xmm0, %xmm1

  383. 	mulpd	%xmm0, %xmm2

  384. 

  385. 	ADD1	%xmm1, %xmm4

  386. 	movaps	-12 * SIZE(BB), %xmm1

  387. 	ADD2	%xmm2, %xmm5

  388. 	pshufd	$0x4e, %xmm3, %xmm2

  389. 	mulpd	%xmm0, %xmm3

  390. 	mulpd	%xmm0, %xmm2

  391. 

  392. 	movaps	-14 * SIZE(AA), %xmm0

  393. 

  394. 	addl	$2 * SIZE, AA

  395. 	addl	$4 * SIZE, BB

  396. 	decl	%eax

  397. 	jg	.L16

  398. 	ALIGN_4

  399. 

  400. .L18:

  401. #if defined(LN) || defined(RT)

  402. 	movl	KK, %eax

  403. #ifdef LN

  404. 	subl	$1, %eax

  405. #else

  406. 	subl	$2, %eax

  407. #endif

  408. 

  409. 	movl	AORIG, AA

  410. 	sall	$ZBASE_SHIFT, %eax

  411. 	leal	(AA, %eax, 1), AA

  412. 	leal	(B,  %eax, 2), BB

  413. #endif

  414. 

  415. 	ADD1	%xmm3, %xmm6

  416. 	pcmpeqb	%xmm1, %xmm1

  417. 	ADD2	%xmm2, %xmm7

  418. 	psllq	$63,   %xmm1

  419. 

  420. #ifndef CONJ

  421. 	pshufd	$0x40, %xmm1, %xmm0

  422. 	shufps	$0x04, %xmm1, %xmm1

  423. 

  424. 	pxor	%xmm0, %xmm4

  425. 	pxor	%xmm0, %xmm6

  426. #else

  427. #if defined(LN) || defined(LT)

  428. 	pshufd	$0x40, %xmm1, %xmm0

  429. #else

  430. 	pshufd	$0x04, %xmm1, %xmm0

  431. #endif

  432. 	shufps	$0x40, %xmm1, %xmm1

  433. 

  434. 	pxor	%xmm0, %xmm5

  435. 	pxor	%xmm0, %xmm7

  436. #endif

  437. 

  438. 	haddpd	%xmm5, %xmm4

  439. 	haddpd	%xmm7, %xmm6

  440. 

  441. 

  442. #if defined(LN) || defined(LT)

  443. 	movapd	-16 * SIZE(BB), %xmm5

  444. 	movapd	-14 * SIZE(BB), %xmm7

  445. 

  446. 	subpd	%xmm4,  %xmm5

  447. 	subpd	%xmm6,  %xmm7

  448. #else

  449. 	movapd	-16 * SIZE(AA), %xmm5

  450. 	movapd	-14 * SIZE(AA), %xmm7

  451. 

  452. 	subpd	%xmm4,  %xmm5

  453. 	subpd	%xmm6,  %xmm7

  454. #endif

  455. 

  456. #if defined(LN) || defined(LT)

  457. 	movddup	-16 * SIZE(AA), %xmm2

  458. 	movddup	-15 * SIZE(AA), %xmm3

  459. 

  460. 	pshufd	$0x4e, %xmm5, %xmm4

  461. 	pshufd	$0x4e, %xmm7, %xmm6

  462. 

  463. 	xorpd	 %xmm1, %xmm4

  464. 	xorpd	 %xmm1, %xmm6

  465. 

  466. 	mulpd	 %xmm2, %xmm5

  467. 	mulpd	 %xmm3, %xmm4

  468. 	mulpd	 %xmm2, %xmm7

  469. 	mulpd	 %xmm3, %xmm6

  470. 

  471. 	addpd	 %xmm4, %xmm5

  472. 	addpd	 %xmm6, %xmm7

  473. #endif

  474. 

  475. #ifdef RN

  476. 	movddup	-16 * SIZE(BB), %xmm2

  477. 	movddup	-15 * SIZE(BB), %xmm3

  478. 

  479. 	pshufd	$0x4e, %xmm5, %xmm4

  480. 

  481. 	xorpd	 %xmm1, %xmm4

  482. 

  483. 	mulpd	 %xmm2, %xmm5

  484. 	mulpd	 %xmm3, %xmm4

  485. 

  486. 	addpd	 %xmm4, %xmm5

  487. 

  488. 	movddup	-14 * SIZE(BB), %xmm2

  489. 	movddup -13 * SIZE(BB), %xmm3

  490. 

  491. 	movapd	 %xmm5, %xmm4

  492. 	pshufd	$0x4e, %xmm5, %xmm6

  493. 

  494. 	xorpd	 %xmm1, %xmm6

  495. 

  496. 	mulpd	 %xmm2, %xmm4

  497. 	mulpd	 %xmm3, %xmm6

  498. 

  499. 	subpd	 %xmm4, %xmm7

  500. 	subpd	 %xmm6, %xmm7

  501. 

  502. 	movddup	-10 * SIZE(BB), %xmm2

  503. 	movddup	 -9 * SIZE(BB), %xmm3

  504. 

  505. 	pshufd	$0x4e, %xmm7, %xmm6

  506. 

  507. 	xorpd	 %xmm1, %xmm6

  508. 

  509. 	mulpd	 %xmm2, %xmm7

  510. 	mulpd	 %xmm3, %xmm6

  511. 

  512. 	addpd	 %xmm6, %xmm7

  513. #endif

  514. 

  515. #ifdef RT

  516. 	movddup	-10 * SIZE(BB), %xmm2

  517. 	movddup	 -9 * SIZE(BB), %xmm3

  518. 

  519. 	pshufd	$0x4e, %xmm7, %xmm6

  520. 

  521. 	xorpd	 %xmm1, %xmm6

  522. 

  523. 	mulpd	 %xmm2, %xmm7

  524. 	mulpd	 %xmm3, %xmm6

  525. 

  526. 	addpd	 %xmm6, %xmm7

  527. 

  528. 	movddup	-12 * SIZE(BB), %xmm2

  529. 	movddup	-11 * SIZE(BB), %xmm3

  530. 

  531. 	movapd	 %xmm7, %xmm4

  532. 	pshufd	$0x4e, %xmm7, %xmm6

  533. 

  534. 	xorpd	 %xmm1, %xmm6

  535. 

  536. 	mulpd	 %xmm2, %xmm4

  537. 	mulpd	 %xmm3, %xmm6

  538. 

  539. 	subpd	 %xmm4, %xmm5

  540. 	subpd	 %xmm6, %xmm5

  541. 

  542. 	movddup	-16 * SIZE(BB), %xmm2

  543. 	movddup	-15 * SIZE(BB), %xmm3

  544. 

  545. 	pshufd	$0x4e, %xmm5, %xmm4

  546. 

  547. 	xorpd	 %xmm1, %xmm4

  548. 

  549. 	mulpd	 %xmm2, %xmm5

  550. 	mulpd	 %xmm3, %xmm4

  551. 

  552. 	addpd	 %xmm4, %xmm5

  553. #endif

  554. 

  555. #ifdef LN

  556. 	subl	$2 * SIZE, CO1

  557. #endif

  558. 

  559. 	movlpd	%xmm5,   0 * SIZE(CO1)

  560. 	movhpd	%xmm5,   1 * SIZE(CO1)

  561. 

  562. 	movlpd	%xmm7,   0 * SIZE(CO1, LDC)

  563. 	movhpd	%xmm7,   1 * SIZE(CO1, LDC)

  564. 

  565. #if defined(LN) || defined(LT)

  566. 	movapd	%xmm5,  -16 * SIZE(BB)

  567. 	movapd	%xmm7,  -14 * SIZE(BB)

  568. #else

  569. 	movapd	%xmm5,  -16 * SIZE(AA)

  570. 	movapd	%xmm7,  -14 * SIZE(AA)

  571. #endif

  572. 

  573. #ifndef LN

  574. 	addl	$2 * SIZE, CO1

  575. #endif

  576. 

  577. #if defined(LT) || defined(RN)

  578. 	movl	K,  %eax

  579. 	subl	KK, %eax

  580. 	sall	$ZBASE_SHIFT, %eax

  581. 	addl	%eax, AA

  582. 	leal	(BB, %eax, 2), BB

  583. #endif

  584. 

  585. #ifdef LN

  586. 	subl	$1, KK

  587. #endif

  588. 

  589. #ifdef LT

  590. 	addl	$1, KK

  591. #endif

  592. 

  593. #ifdef RT

  594. 	movl	K, %eax

  595. 	sall	$ZBASE_SHIFT, %eax

  596. 	addl	%eax, AORIG

  597. #endif

  598. 

  599. 	decl	%ebx			# i --

  600. 	jg	.L10

  601. 	ALIGN_4

  602. 

  603. .L99:

  604. #ifdef LN

  605.        movl	K, %eax

  606.        sall	$1 + ZBASE_SHIFT, %eax

  607.        addl	%eax, B

  608. #endif

  609. 

  610. #if defined(LT) || defined(RN)

  611. 	movl	BB, B

  612. #endif

  613. 

  614. #ifdef RN

  615. 	addl	$2, KK

  616. #endif

  617. 

  618. #ifdef RT

  619. 	subl	$2, KK

  620. #endif

  621. 

  622. 	decl	J			# j --

  623. 	jg	.L01

  624. 	ALIGN_4

  625. 

  626. .L100:

  627. 	movl	N, %eax

  628. 	testl	$1, %eax

  629. 	jle	.L999

  630. 

  631. #if defined(LT) || defined(RN)

  632. 	movl	A, AA

  633. #else

  634. 	movl	A, %eax

  635. 	movl	%eax, AORIG

  636. #endif

  637. 

  638. #ifdef RT

  639. 	movl	K, %eax

  640. 	sall	$ZBASE_SHIFT, %eax

  641. 	subl	%eax, B

  642. #endif

  643. 

  644. #ifdef RT

  645. 	subl	LDC, C

  646. #endif

  647. 	movl	C, CO1

  648. #ifndef RT

  649. 	addl	LDC, C

  650. #endif

  651. 

  652. #ifdef LN

  653. 	movl	OFFSET, %eax

  654. 	addl	M, %eax

  655. 	movl	%eax, KK

  656. #endif

  657. 

  658. #ifdef LT

  659. 	movl	OFFSET, %eax

  660. 	movl	%eax, KK

  661. #endif

  662. 

  663. 	movl	M,  %ebx

  664. 	ALIGN_4

  665. 

  666. L110:

  667. #ifdef LN

  668. 	movl	K, %eax

  669. 	sall	$ZBASE_SHIFT, %eax

  670. 	subl	%eax, AORIG

  671. #endif

  672. 

  673. #if defined(LN) || defined(RT)

  674. 	movl	KK, %eax

  675. 	movl	AORIG, AA

  676. 	sall	$ZBASE_SHIFT, %eax

  677. 	addl	%eax, AA

  678. #endif

  679. 

  680. 	movl	B, BB

  681. 

  682. #if defined(LN) || defined(RT)

  683. 	movl	KK, %eax

  684. 	sall	$ZBASE_SHIFT, %eax

  685. 	addl	%eax, BB

  686. #endif

  687. 

  688. 	movaps	-16 * SIZE(AA), %xmm0

  689. 	pxor	%xmm2, %xmm2

  690. 	movaps	-16 * SIZE(BB), %xmm1

  691. 	pxor	%xmm3, %xmm3

  692. 

  693. 	pxor	%xmm4, %xmm4

  694. #ifdef LN

  695. 	prefetcht0	-2 * SIZE(CO1)

  696. #else

  697. 	prefetcht0	 1 * SIZE(CO1)

  698. #endif

  699. 	pxor	%xmm5, %xmm5

  700. 	pxor	%xmm6, %xmm6

  701. 	pxor	%xmm7, %xmm7

  702. 

  703. #if defined(LT) || defined(RN)

  704. 	movl	KK, %eax

  705. #else

  706. 	movl	K, %eax

  707. 	subl	KK, %eax

  708. #endif

  709. 	sarl	$3, %eax

  710. 	je	L115

  711. 	ALIGN_4

  712. 

  713. L112:

  714. 	PREFETCH (PREFETCHSIZE +  0) * SIZE(AA)

  715. 

  716. 	pshufd	$0x4e, %xmm1, %xmm2

  717. 	mulpd	%xmm0, %xmm1

  718. 	mulpd	%xmm0, %xmm2

  719. 	movaps	-14 * SIZE(AA), %xmm0

  720. 

  721. 	ADD1	%xmm1, %xmm4

  722. 	movaps	-14 * SIZE(BB), %xmm1

  723. 	ADD2	%xmm2, %xmm5

  724. 

  725. 	pshufd	$0x4e, %xmm1, %xmm2

  726. 	mulpd	%xmm0, %xmm1

  727. 	mulpd	%xmm0, %xmm2

  728. 	movaps	-12 * SIZE(AA), %xmm0

  729. 

  730. 	ADD1	%xmm1, %xmm6

  731. 	movaps	-12 * SIZE(BB), %xmm1

  732. 	ADD2	%xmm2, %xmm7

  733. 

  734. 	pshufd	$0x4e, %xmm1, %xmm2

  735. 	mulpd	%xmm0, %xmm1

  736. 	mulpd	%xmm0, %xmm2

  737. 	movaps	-10 * SIZE(AA), %xmm0

  738. 

  739. 	ADD1	%xmm1, %xmm4

  740. 	movaps	-10 * SIZE(BB), %xmm1

  741. 	ADD2	%xmm2, %xmm5

  742. 

  743. 	pshufd	$0x4e, %xmm1, %xmm2

  744. 	mulpd	%xmm0, %xmm1

  745. 	mulpd	%xmm0, %xmm2

  746. 	movaps	 -8 * SIZE(AA), %xmm0

  747. 

  748. 	ADD1	%xmm1, %xmm6

  749. 	movaps	 -8 * SIZE(BB), %xmm1

  750. 	ADD2	%xmm2, %xmm7

  751. 

  752. 	PREFETCH (PREFETCHSIZE +  8) * SIZE(AA)

  753. 

  754. 	pshufd	$0x4e, %xmm1, %xmm2

  755. 	mulpd	%xmm0, %xmm1

  756. 	mulpd	%xmm0, %xmm2

  757. 	movaps	 -6 * SIZE(AA), %xmm0

  758. 

  759. 	ADD1	%xmm1, %xmm4

  760. 	movaps	 -6 * SIZE(BB), %xmm1

  761. 	ADD2	%xmm2, %xmm5

  762. 

  763. 	pshufd	$0x4e, %xmm1, %xmm2

  764. 	mulpd	%xmm0, %xmm1

  765. 	mulpd	%xmm0, %xmm2

  766. 	movaps	 -4 * SIZE(AA), %xmm0

  767. 

  768. 	ADD1	%xmm1, %xmm6

  769. 	movaps	 -4 * SIZE(BB), %xmm1

  770. 	ADD2	%xmm2, %xmm7

  771. 

  772. 	pshufd	$0x4e, %xmm1, %xmm2

  773. 	mulpd	%xmm0, %xmm1

  774. 	mulpd	%xmm0, %xmm2

  775. 	movaps	 -2 * SIZE(AA), %xmm0

  776. 

  777. 	ADD1	%xmm1, %xmm4

  778. 	movaps	 -2 * SIZE(BB), %xmm1

  779. 	ADD2	%xmm2, %xmm5

  780. 

  781. 	pshufd	$0x4e, %xmm1, %xmm2

  782. 	mulpd	%xmm0, %xmm1

  783. 	mulpd	%xmm0, %xmm2

  784. 	movaps	  0 * SIZE(AA), %xmm0

  785. 

  786. 	ADD1	%xmm1, %xmm6

  787. 	movaps	  0 * SIZE(BB), %xmm1

  788. 	ADD2	%xmm2, %xmm7

  789. 

  790. 	subl   $-16 * SIZE, AA

  791. 	subl   $-16 * SIZE, BB

  792. 

  793. 	subl   $1, %eax

  794. 	jne    L112

  795. 	ALIGN_4

  796. 

  797. L115:

  798. #if defined(LT) || defined(RN)

  799. 	movl	KK, %eax

  800. #else

  801. 	movl	K, %eax

  802. 	subl	KK, %eax

  803. #endif

  804. 	andl	$7, %eax		# if (k & 1)

  805. 	BRANCH

  806. 	je L118

  807. 	ALIGN_4

  808. 

  809. L116:

  810. 	pshufd	$0x4e, %xmm1, %xmm2

  811. 	mulpd	%xmm0, %xmm1

  812. 	mulpd	%xmm0, %xmm2

  813. 	movaps	-14 * SIZE(AA), %xmm0

  814. 

  815. 	ADD1	%xmm1, %xmm4

  816. 	movaps	-14 * SIZE(BB), %xmm1

  817. 	ADD2	%xmm2, %xmm5

  818. 

  819. 	addl	$2 * SIZE, AA

  820. 	addl	$2 * SIZE, BB

  821. 	decl	%eax

  822. 	jg	L116

  823. 	ALIGN_4

  824. 

  825. L118:

  826. #if defined(LN) || defined(RT)

  827. 	movl	KK, %eax

  828. #ifdef LN

  829. 	subl	$1, %eax

  830. #else

  831. 	subl	$1, %eax

  832. #endif

  833. 

  834. 	movl	AORIG, AA

  835. 	sall	$ZBASE_SHIFT, %eax

  836. 	leal	(AA, %eax, 1), AA

  837. 	leal	(B,  %eax, 1), BB

  838. #endif

  839. 

  840. 	addpd	%xmm6, %xmm4

  841. 	pcmpeqb	%xmm1, %xmm1

  842. 	addpd	%xmm7, %xmm5

  843. 	psllq	$63,   %xmm1

  844. 

  845. #ifndef CONJ

  846. 	pshufd	$0x40, %xmm1, %xmm0

  847. 	shufps	$0x04, %xmm1, %xmm1

  848. 

  849. 	pxor	%xmm0, %xmm4

  850. #else

  851. #if defined(LN) || defined(LT)

  852. 	pshufd	$0x40, %xmm1, %xmm0

  853. #else

  854. 	pshufd	$0x04, %xmm1, %xmm0

  855. #endif

  856. 	shufps	$0x40, %xmm1, %xmm1

  857. 

  858. 	pxor	%xmm0, %xmm5

  859. #endif

  860. 

  861. 	haddpd	%xmm5, %xmm4

  862. 

  863. #if defined(LN) || defined(LT)

  864. 	movapd	-16 * SIZE(BB), %xmm5

  865. 	subpd	%xmm4,  %xmm5

  866. #else

  867. 	movapd	-16 * SIZE(AA), %xmm5

  868. 	subpd	%xmm4,  %xmm5

  869. #endif

  870. 

  871. #if defined(LN) || defined(LT)

  872. 	movddup	-16 * SIZE(AA), %xmm2

  873. 	movddup	-15 * SIZE(AA), %xmm3

  874. 

  875. 	pshufd	$0x4e, %xmm5, %xmm4

  876. 

  877. 	xorpd	 %xmm1, %xmm4

  878. 

  879. 	mulpd	 %xmm2, %xmm5

  880. 	mulpd	 %xmm3, %xmm4

  881. 

  882. 	addpd	 %xmm4, %xmm5

  883. #endif

  884. 

  885. #if defined(RN) || defined(RT)

  886. 	movddup	-16 * SIZE(BB), %xmm2

  887. 	movddup	-15 * SIZE(BB), %xmm3

  888. 

  889. 	pshufd	$0x4e, %xmm5, %xmm4

  890. 

  891. 	xorpd	 %xmm1, %xmm4

  892. 

  893. 	mulpd	 %xmm2, %xmm5

  894. 	mulpd	 %xmm3, %xmm4

  895. 

  896. 	addpd	 %xmm4, %xmm5

  897. #endif

  898. 

  899. #ifdef LN

  900. 	subl	$2 * SIZE, CO1

  901. #endif

  902. 

  903. 	movlpd	%xmm5,   0 * SIZE(CO1)

  904. 	movhpd	%xmm5,   1 * SIZE(CO1)

  905. 

  906. #if defined(LN) || defined(LT)

  907. 	movapd	%xmm5, -16 * SIZE(BB)

  908. #else

  909. 	movapd	%xmm5, -16 * SIZE(AA)

  910. #endif

  911. 

  912. #ifndef LN

  913. 	addl	$2 * SIZE, CO1

  914. #endif

  915. 

  916. #if defined(LT) || defined(RN)

  917. 	movl	K,  %eax

  918. 	subl	KK, %eax

  919. 	sall	$ZBASE_SHIFT, %eax

  920. 	addl	%eax, AA

  921. 	addl	%eax, BB

  922. #endif

  923. 

  924. #ifdef LN

  925. 	subl	$1, KK

  926. #endif

  927. 

  928. #ifdef LT

  929. 	addl	$1, KK

  930. #endif

  931. 

  932. #ifdef RT

  933. 	movl	K, %eax

  934. 	sall	$ZBASE_SHIFT, %eax

  935. 	addl	%eax, AORIG

  936. #endif

  937. 

  938. 	decl	%ebx			# i --

  939. 	jg	L110

  940. 

  941. #ifdef LN

  942.        movl	K, %eax

  943.        sall	$ZBASE_SHIFT, %eax

  944.        addl	%eax, B

  945. #endif

  946. 

  947. #if defined(LT) || defined(RN)

  948. 	movl	BB, B

  949. #endif

  950. 

  951. #ifdef RN

  952. 	addl	$1, KK

  953. #endif

  954. 

  955. #ifdef RT

  956. 	subl	$1, KK

  957. #endif

  958. 	ALIGN_4

  959. 

  960. .L999:

  961. 	popl	%ebx

  962. 	popl	%esi

  963. 	popl	%edi

  964. 	popl	%ebp

  965. 

  966. 	addl	$ARGS, %esp

  967. 	ret

  968. 

  969. 	EPILOGUE