OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 66d9185ebe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '66d9185ebe'
${ noResults }
OpenBLAS/kernel/x86_64/ztrsm_kernel_LN_2x2_sse2.S

2279 lines
43 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define OLD_M	%rdi

  43. #define OLD_N	%rsi

  44. #define M	%r13

  45. #define N	%r14

  46. #define K	%rdx

  47. #define A	%rcx

  48. #define B	%r8

  49. #define C	%r9

  50. #define LDC	%r10

  51. 

  52. #define I	%r11

  53. #define J	%r12

  54. #define AO	%rdi

  55. #define BO	%rsi

  56. #define	CO1	%r15

  57. #define CO2	%rbp

  58. 

  59. #ifndef WINDOWS_ABI

  60. 

  61. #define STACKSIZE 64

  62. 

  63. #define OLD_LDC		 8 + STACKSIZE(%rsp)

  64. #define OLD_OFFSET	16 + STACKSIZE(%rsp)

  65. 

  66. #else

  67. 

  68. #define STACKSIZE 256

  69. 

  70. #define OLD_ALPHA_I	40 + STACKSIZE(%rsp)

  71. #define OLD_A		48 + STACKSIZE(%rsp)

  72. #define OLD_B		56 + STACKSIZE(%rsp)

  73. #define OLD_C		64 + STACKSIZE(%rsp)

  74. #define OLD_LDC		72 + STACKSIZE(%rsp)

  75. #define OLD_OFFSET	80 + STACKSIZE(%rsp)

  76. 

  77. #endif

  78. 

  79. #define POSINV	  0(%rsp)

  80. #define ALPHA_R	 16(%rsp)

  81. #define ALPHA_I	 32(%rsp)

  82. #define OFFSET	 40(%rsp)

  83. #define KK	 48(%rsp)

  84. #define KKK	 56(%rsp)

  85. #define AORIG    64(%rsp)

  86. #define BORIG	 72(%rsp)

  87. #define BUFFER	128(%rsp)

  88. 

  89. #if defined(OPTERON) || defined(BARCELONA) || defined(SHANGHAI) || defined(BOBCAT) || defined(BULLDOZER)

  90. #define PREFETCH     prefetch

  91. #define PREFETCHW    prefetchw

  92. #define PREFETCHNTA  prefetchnta

  93. #define PREFETCHSIZE (8 * 6 + 4)

  94. #endif

  95. 

  96. #ifdef GENERIC

  97. #define PREFETCH     prefetcht0

  98. #define PREFETCHW    prefetcht0

  99. #define PREFETCHNTA  prefetchnta

  100. #define PREFETCHSIZE (8 * 6 + 4)

  101. #endif

  102. 

  103. #define KERNEL1(xx) \

  104. 	mulpd	%xmm8, %xmm9 ;\

  105. 	addpd	%xmm9, %xmm0 ;\

  106. 	movapd	 0 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\

  107. 	mulpd	%xmm8, %xmm11 ;\

  108. 	PREFETCH	(PREFETCHSIZE     +  0) * SIZE + 1 * (xx) * SIZE(AO) ;\

  109. 	addpd	%xmm11, %xmm1 ;\

  110. 	movapd	 2 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  111. 	mulpd	%xmm8, %xmm13 ;\

  112. 	mulpd	 6 * SIZE + 2 * (xx) * SIZE(BO), %xmm8 ;\

  113. 	addpd	%xmm13, %xmm2 ;\

  114. 	movapd	 4 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  115. 	addpd	%xmm8, %xmm3 ;\

  116. 	movapd	 8 * SIZE + 1 * (xx) * SIZE(AO), %xmm8

  117. 

  118. #define KERNEL2(xx) \

  119. 	mulpd	%xmm10, %xmm9 ;\

  120. 	addpd	%xmm9, %xmm4 ;\

  121. 	movapd	16 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\

  122. 	mulpd	%xmm10, %xmm11 ;\

  123. 	addpd	%xmm11, %xmm5 ;\

  124. 	movapd	10 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  125. 	mulpd	%xmm10, %xmm13 ;\

  126. 	mulpd	 6 * SIZE + 2 * (xx) * SIZE(BO), %xmm10 ;\

  127. 	addpd	%xmm13, %xmm6 ;\

  128. 	movapd	12 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  129. 	addpd	%xmm10, %xmm7 ;\

  130. 	movapd	10 * SIZE + 1 * (xx) * SIZE(AO), %xmm10

  131. 

  132. #define KERNEL3(xx) \

  133. 	mulpd	%xmm12, %xmm15 ;\

  134. 	addpd	%xmm15, %xmm0 ;\

  135. 	movapd	 8 * SIZE + 2 * (xx) * SIZE(BO), %xmm15 ;\

  136. 	mulpd	%xmm12, %xmm11 ;\

  137. 	addpd	%xmm11, %xmm1 ;\

  138. 	movapd	10 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  139. 	mulpd	%xmm12, %xmm13 ;\

  140. 	mulpd	14 * SIZE + 2 * (xx) * SIZE(BO), %xmm12 ;\

  141. 	addpd	%xmm13, %xmm2 ;\

  142. 	movapd	12 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  143. 	addpd	%xmm12, %xmm3 ;\

  144. 	movapd	12 * SIZE + 1 * (xx) * SIZE(AO), %xmm12

  145. 

  146. #define KERNEL4(xx) \

  147. 	mulpd	%xmm14, %xmm15 ;\

  148. 	addpd	%xmm15, %xmm4 ;\

  149. 	movapd	24 * SIZE + 2 * (xx) * SIZE(BO), %xmm15 ;\

  150. 	mulpd	%xmm14, %xmm11 ;\

  151. 	addpd	%xmm11, %xmm5 ;\

  152. 	movapd	18 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  153. 	mulpd	%xmm14, %xmm13 ;\

  154. 	mulpd	14 * SIZE + 2 * (xx) * SIZE(BO), %xmm14 ;\

  155. 	addpd	%xmm13, %xmm6 ;\

  156. 	movapd	20 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  157. 	addpd	%xmm14, %xmm7 ;\

  158. 	movapd	14 * SIZE + 1 * (xx) * SIZE(AO), %xmm14

  159. 

  160. #define KERNEL5(xx) \

  161. 	mulpd	%xmm8, %xmm9 ;\

  162. 	addpd	%xmm9, %xmm0 ;\

  163. 	movapd	16 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\

  164. 	mulpd	%xmm8, %xmm11 ;\

  165. 	PREFETCH	(PREFETCHSIZE     +  8) * SIZE + 1 * (xx) * SIZE(AO) ;\

  166. 	addpd	%xmm11, %xmm1 ;\

  167. 	movapd	18 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  168. 	mulpd	%xmm8, %xmm13 ;\

  169. 	mulpd	22 * SIZE + 2 * (xx) * SIZE(BO), %xmm8 ;\

  170. 	addpd	%xmm13, %xmm2 ;\

  171. 	movapd	20 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  172. 	addpd	%xmm8, %xmm3 ;\

  173. 	movapd	16 * SIZE + 1 * (xx) * SIZE(AO), %xmm8

  174. 

  175. #define KERNEL6(xx) \

  176. 	mulpd	%xmm10, %xmm9 ;\

  177. 	addpd	%xmm9, %xmm4 ;\

  178. 	movapd	32 * SIZE + 2 * (xx) * SIZE(BO), %xmm9 ;\

  179. 	mulpd	%xmm10, %xmm11 ;\

  180. 	addpd	%xmm11, %xmm5 ;\

  181. 	movapd	26 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  182. 	mulpd	%xmm10, %xmm13 ;\

  183. 	mulpd	22 * SIZE + 2 * (xx) * SIZE(BO), %xmm10 ;\

  184. 	addpd	%xmm13, %xmm6 ;\

  185. 	movapd	28 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  186. 	addpd	%xmm10, %xmm7 ;\

  187. 	movapd	18 * SIZE + 1 * (xx) * SIZE(AO), %xmm10

  188. 

  189. #define KERNEL7(xx) \

  190. 	mulpd	%xmm12, %xmm15 ;\

  191. 	addpd	%xmm15, %xmm0 ;\

  192. 	movapd	24 * SIZE + 2 * (xx) * SIZE(BO), %xmm15 ;\

  193. 	mulpd	%xmm12, %xmm11 ;\

  194. 	addpd	%xmm11, %xmm1 ;\

  195. 	movapd	26 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  196. 	mulpd	%xmm12, %xmm13 ;\

  197. 	mulpd	30 * SIZE + 2 * (xx) * SIZE(BO), %xmm12 ;\

  198. 	addpd	%xmm13, %xmm2 ;\

  199. 	movapd	28 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  200. 	addpd	%xmm12, %xmm3 ;\

  201. 	movapd	20 * SIZE + 1 * (xx) * SIZE(AO), %xmm12

  202. 

  203. #define KERNEL8(xx) \

  204. 	mulpd	%xmm14, %xmm15 ;\

  205. 	addpd	%xmm15, %xmm4 ;\

  206. 	movapd	40 * SIZE + 2 * (xx) * SIZE(BO), %xmm15 ;\

  207. 	mulpd	%xmm14, %xmm11 ;\

  208. 	addpd	%xmm11, %xmm5 ;\

  209. 	movapd	34 * SIZE + 2 * (xx) * SIZE(BO), %xmm11 ;\

  210. 	mulpd	%xmm14, %xmm13 ;\

  211. 	mulpd	30 * SIZE + 2 * (xx) * SIZE(BO), %xmm14 ;\

  212. 	addpd	%xmm13, %xmm6 ;\

  213. 	movapd	36 * SIZE + 2 * (xx) * SIZE(BO), %xmm13 ;\

  214. 	addpd	%xmm14, %xmm7 ;\

  215. 	movapd	22 * SIZE + 1 * (xx) * SIZE(AO), %xmm14

  216. 

  217. #ifndef CONJ

  218. #define NN

  219. #else

  220. #if defined(LN) || defined(LT)

  221. #define CN

  222. #else

  223. #define NC

  224. #endif

  225. #endif

  226. 

  227. 	PROLOGUE

  228. 	PROFCODE

  229. 

  230. 	subq	$STACKSIZE, %rsp

  231. 

  232. 	movq	%rbx,  0(%rsp)

  233. 	movq	%rbp,  8(%rsp)

  234. 	movq	%r12, 16(%rsp)

  235. 	movq	%r13, 24(%rsp)

  236. 	movq	%r14, 32(%rsp)

  237. 	movq	%r15, 40(%rsp)

  238. 

  239. #ifdef WINDOWS_ABI

  240. 	movq	%rdi,    48(%rsp)

  241. 	movq	%rsi,    56(%rsp)

  242. 	movups	%xmm6,   64(%rsp)

  243. 	movups	%xmm7,   80(%rsp)

  244. 	movups	%xmm8,   96(%rsp)

  245. 	movups	%xmm9,  112(%rsp)

  246. 	movups	%xmm10, 128(%rsp)

  247. 	movups	%xmm11, 144(%rsp)

  248. 	movups	%xmm12, 160(%rsp)

  249. 	movups	%xmm13, 176(%rsp)

  250. 	movups	%xmm14, 192(%rsp)

  251. 	movups	%xmm15, 208(%rsp)

  252. 

  253. 	movq	ARG1,      OLD_M

  254. 	movq	ARG2,      OLD_N

  255. 	movq	ARG3,      K

  256. 	movq	OLD_A,     A

  257. 	movq	OLD_B,     B

  258. 	movq	OLD_C,     C

  259. 	movq	OLD_LDC,   LDC

  260. 	movsd	OLD_OFFSET, %xmm4

  261. 

  262. 	movaps	%xmm3, %xmm0

  263. 

  264. #else

  265. 	movq	OLD_LDC,   LDC

  266. 	movsd	OLD_OFFSET, %xmm4

  267. 

  268. #endif

  269. 

  270. 	movq	%rsp, %rbx	# save old stack

  271. 	subq	$128 + LOCAL_BUFFER_SIZE, %rsp

  272. 	andq	$-4096, %rsp	# align stack

  273. 

  274. 	STACK_TOUCHING

  275. 

  276. 	movq	OLD_M, M

  277. 	movq	OLD_N, N

  278. 

  279. 	pcmpeqb	%xmm15, %xmm15

  280. 	psllq	$63, %xmm15	# Generate mask

  281. 	pxor	%xmm2, %xmm2

  282. 

  283. 	movlpd	  %xmm2,  0 + POSINV

  284. 	movlpd	  %xmm15, 8 + POSINV

  285. 

  286. 	movlpd	%xmm4, OFFSET

  287. 	movlpd	%xmm4, KK

  288. 

  289. 	salq	$ZBASE_SHIFT, LDC

  290. 

  291. #ifdef LN

  292.        movq	M, %rax

  293.        salq	$ZBASE_SHIFT, %rax

  294.        addq	%rax, C

  295.        imulq	K, %rax

  296.        addq	%rax, A

  297. #endif

  298. 

  299. #ifdef RT

  300.        movq	N, %rax

  301.        salq	$ZBASE_SHIFT, %rax

  302.        imulq	K, %rax

  303.        addq	%rax, B

  304. 

  305.        movq	N, %rax

  306.        imulq	LDC, %rax

  307.        addq	%rax, C

  308. #endif

  309. 

  310. #ifdef RN

  311. 	negq	KK

  312. #endif

  313. 

  314. #ifdef RT

  315.        movq	N, %rax

  316.        subq	OFFSET, %rax

  317.        movq	%rax, KK

  318. #endif

  319. 

  320. 	movq	N,  J

  321. 	sarq	$1, J		# j = (n >> 2)

  322. 	jle	.L100

  323. 	ALIGN_4

  324. 

  325. .L01:

  326. #ifdef LN

  327. 	movq	OFFSET, %rax

  328. 	addq	M, %rax

  329. 	movq	%rax, KK

  330. #endif

  331. 

  332. /* Copying to Sub Buffer */

  333. 	leaq	BUFFER, BO

  334. 

  335. #ifdef RT

  336.        movq	K, %rax

  337.        salq	$1 + ZBASE_SHIFT, %rax

  338.        subq	%rax, B

  339. #endif

  340. 

  341. #if defined(LN) || defined(RT)

  342. 	movq	KK, %rax

  343. 	movq	B, BORIG

  344. 	salq	$ZBASE_SHIFT, %rax

  345. 	leaq	(B,  %rax, 2), B

  346. 	leaq	(BO, %rax, 4), BO

  347. #endif

  348. 

  349. #if defined(LT)

  350. 	movq	OFFSET, %rax

  351. 	movq	%rax, KK

  352. #endif

  353. 

  354. #if defined(LT) || defined(RN)

  355. 	movq	KK, %rax

  356. #else

  357. 	movq	K, %rax

  358. 	subq	KK, %rax

  359. #endif

  360. 	sarq	$2, %rax

  361. 	jle	.L03

  362. 

  363. 	addq	%rax, %rax

  364. 	ALIGN_4

  365. 

  366. .L02:

  367. 	PREFETCHNTA	 56 * SIZE(B)

  368. 

  369. 	movlpd	 0 * SIZE(B), %xmm0

  370. 	movlpd	 1 * SIZE(B), %xmm1

  371. 	movlpd	 2 * SIZE(B), %xmm2

  372. 	movlpd	 3 * SIZE(B), %xmm3

  373. 	movlpd	 4 * SIZE(B), %xmm4

  374. 	movlpd	 5 * SIZE(B), %xmm5

  375. 	movlpd	 6 * SIZE(B), %xmm6

  376. 	movlpd	 7 * SIZE(B), %xmm7

  377. 

  378. 	movlpd	%xmm0,  0 * SIZE(BO)

  379. 	movlpd	%xmm0,  1 * SIZE(BO)

  380. 	movlpd	%xmm1,  2 * SIZE(BO)

  381. 	movlpd	%xmm1,  3 * SIZE(BO)

  382. 	movlpd	%xmm2,  4 * SIZE(BO)

  383. 	movlpd	%xmm2,  5 * SIZE(BO)

  384. 	movlpd	%xmm3,  6 * SIZE(BO)

  385. 	movlpd	%xmm3,  7 * SIZE(BO)

  386. 	movlpd	%xmm4,  8 * SIZE(BO)

  387. 	movlpd	%xmm4,  9 * SIZE(BO)

  388. 	movlpd	%xmm5, 10 * SIZE(BO)

  389. 	movlpd	%xmm5, 11 * SIZE(BO)

  390. 	movlpd	%xmm6, 12 * SIZE(BO)

  391. 	movlpd	%xmm6, 13 * SIZE(BO)

  392. 	movlpd	%xmm7, 14 * SIZE(BO)

  393. 	movlpd	%xmm7, 15 * SIZE(BO)

  394. 

  395. 	subq	$-16 * SIZE, BO

  396. 	addq	$  8 * SIZE, B

  397. 	decq	%rax

  398. 	jne	.L02

  399. 	ALIGN_4

  400. 

  401. .L03:

  402. #if defined(LT) || defined(RN)

  403. 	movq	KK, %rax

  404. #else

  405. 	movq	K, %rax

  406. 	subq	KK, %rax

  407. #endif

  408. 	andq	$3, %rax

  409. 	BRANCH

  410. 	jle	.L05

  411. 	ALIGN_4

  412. 

  413. .L04:

  414. 	movlpd	 0 * SIZE(B), %xmm0

  415. 	movlpd	 1 * SIZE(B), %xmm1

  416. 	movlpd	 2 * SIZE(B), %xmm2

  417. 	movlpd	 3 * SIZE(B), %xmm3

  418. 

  419. 	movlpd	%xmm0,  0 * SIZE(BO)

  420. 	movlpd	%xmm0,  1 * SIZE(BO)

  421. 	movlpd	%xmm1,  2 * SIZE(BO)

  422. 	movlpd	%xmm1,  3 * SIZE(BO)

  423. 	movlpd	%xmm2,  4 * SIZE(BO)

  424. 	movlpd	%xmm2,  5 * SIZE(BO)

  425. 	movlpd	%xmm3,  6 * SIZE(BO)

  426. 	movlpd	%xmm3,  7 * SIZE(BO)

  427. 

  428. 	addq	$ 4 * SIZE, B

  429. 	addq	$ 8 * SIZE, BO

  430. 

  431. 	decq	%rax

  432. 	jne	.L04

  433. 	ALIGN_4

  434. 

  435. .L05:

  436. #if defined(LT) || defined(RN)

  437. 	movq	A, AO

  438. #else

  439. 	movq	A, AORIG

  440. #endif

  441. 

  442. #ifdef RT

  443.        leaq	(, LDC, 2), %rax

  444.        subq	%rax, C

  445. #endif

  446. 

  447. 	movq	C, CO1

  448. 	leaq	(C, LDC, 1), CO2

  449. 

  450. #ifndef RT

  451. 	leaq	(C, LDC, 2), C

  452. #endif

  453. 

  454. 	testq	$1, M

  455. 	jle	.L30

  456. 

  457. #ifdef LN

  458.        movq	K, %rax

  459.        salq	$0 + ZBASE_SHIFT, %rax

  460.        subq	%rax, AORIG

  461. #endif

  462. 

  463. #if defined(LN) || defined(RT)

  464. 	movq	KK, %rax

  465. 	movq	AORIG, AO

  466. 	salq	$ZBASE_SHIFT, %rax

  467. 	addq	%rax, AO

  468. #endif

  469. 

  470. 	leaq	BUFFER, BO

  471. 

  472. #if defined(LN) || defined(RT)

  473. 	movq	KK, %rax

  474. 	salq	$1 + ZBASE_SHIFT, %rax

  475. 	leaq	(BO, %rax, 2), BO

  476. #endif

  477. 

  478. 	pxor	%xmm0, %xmm0

  479. 	pxor	%xmm1, %xmm1

  480. 	pxor	%xmm2, %xmm2

  481. 	pxor	%xmm3, %xmm3

  482. 

  483. #if defined(LT) || defined(RN)

  484. 	movq	KK, %rax

  485. #else

  486. 	movq	K, %rax

  487. 	subq	KK, %rax

  488. #endif

  489. 	sarq	$2, %rax

  490. 	je	.L42

  491. 

  492. .L41:

  493. 	movapd	 0 * SIZE(AO), %xmm8

  494. 

  495. 	movapd	 0 * SIZE(BO), %xmm9

  496. 	mulpd	%xmm8, %xmm9

  497. 	addpd	%xmm9, %xmm0

  498. 

  499. 	movapd	 2 * SIZE(BO), %xmm9

  500. 	mulpd	%xmm8, %xmm9

  501. 	addpd	%xmm9, %xmm1

  502. 

  503. 	movapd	 4 * SIZE(BO), %xmm9

  504. 	mulpd	%xmm8, %xmm9

  505. 	addpd	%xmm9, %xmm2

  506. 

  507. 	movapd	 6 * SIZE(BO), %xmm9

  508. 	mulpd	%xmm8, %xmm9

  509. 	addpd	%xmm9, %xmm3

  510. 

  511. 	movapd	 2 * SIZE(AO), %xmm8

  512. 

  513. 	movapd	 8 * SIZE(BO), %xmm9

  514. 	mulpd	%xmm8, %xmm9

  515. 	addpd	%xmm9, %xmm0

  516. 

  517. 	movapd	10 * SIZE(BO), %xmm9

  518. 	mulpd	%xmm8, %xmm9

  519. 	addpd	%xmm9, %xmm1

  520. 

  521. 	movapd	12 * SIZE(BO), %xmm9

  522. 	mulpd	%xmm8, %xmm9

  523. 	addpd	%xmm9, %xmm2

  524. 

  525. 	movapd	14 * SIZE(BO), %xmm9

  526. 	mulpd	%xmm8, %xmm9

  527. 	addpd	%xmm9, %xmm3

  528. 

  529. 	movapd	 4 * SIZE(AO), %xmm8

  530. 

  531. 	movapd	16 * SIZE(BO), %xmm9

  532. 	mulpd	%xmm8, %xmm9

  533. 	addpd	%xmm9, %xmm0

  534. 

  535. 	movapd	18 * SIZE(BO), %xmm9

  536. 	mulpd	%xmm8, %xmm9

  537. 	addpd	%xmm9, %xmm1

  538. 

  539. 	movapd	20 * SIZE(BO), %xmm9

  540. 	mulpd	%xmm8, %xmm9

  541. 	addpd	%xmm9, %xmm2

  542. 

  543. 	movapd	22 * SIZE(BO), %xmm9

  544. 	mulpd	%xmm8, %xmm9

  545. 	addpd	%xmm9, %xmm3

  546. 

  547. 	movapd	 6 * SIZE(AO), %xmm8

  548. 

  549. 	movapd	24 * SIZE(BO), %xmm9

  550. 	mulpd	%xmm8, %xmm9

  551. 	addpd	%xmm9, %xmm0

  552. 

  553. 	movapd	26 * SIZE(BO), %xmm9

  554. 	mulpd	%xmm8, %xmm9

  555. 	addpd	%xmm9, %xmm1

  556. 

  557. 	movapd	28 * SIZE(BO), %xmm9

  558. 	mulpd	%xmm8, %xmm9

  559. 	addpd	%xmm9, %xmm2

  560. 

  561. 	movapd	30 * SIZE(BO), %xmm9

  562. 	mulpd	%xmm8, %xmm9

  563. 	addpd	%xmm9, %xmm3

  564. 

  565. 	addq   $ 8 * SIZE, AO

  566. 	addq   $32 * SIZE, BO

  567. 	decq   %rax

  568. 	jne    .L41

  569. 

  570. .L42:

  571. #if defined(LT) || defined(RN)

  572. 	movq	KK, %rax

  573. #else

  574. 	movq	K, %rax

  575. 	subq	KK, %rax

  576. #endif

  577. 	movapd	POSINV,  %xmm15

  578. 	andq	$3, %rax		# if (k & 1)

  579. 	BRANCH

  580. 	jle .L44

  581. 

  582. .L43:

  583. 	movapd	 0 * SIZE(AO), %xmm8

  584. 

  585. 	movapd	 0 * SIZE(BO), %xmm9

  586. 	mulpd	%xmm8, %xmm9

  587. 	addpd	%xmm9, %xmm0

  588. 

  589. 	movapd	 2 * SIZE(BO), %xmm9

  590. 	mulpd	%xmm8, %xmm9

  591. 	addpd	%xmm9, %xmm1

  592. 

  593. 	movapd	 4 * SIZE(BO), %xmm9

  594. 	mulpd	%xmm8, %xmm9

  595. 	addpd	%xmm9, %xmm2

  596. 

  597. 	movapd	 6 * SIZE(BO), %xmm9

  598. 	mulpd	%xmm8, %xmm9

  599. 	addpd	%xmm9, %xmm3

  600. 

  601. 	addq	$2 * SIZE, AO		# aoffset  += 4

  602. 	addq	$8 * SIZE, BO		# boffset1 += 8

  603. 

  604. 	decq	%rax

  605. 	jg	.L43

  606. 	ALIGN_4

  607. 

  608. .L44:

  609. #if defined(LN) || defined(RT)

  610. 	movq	KK, %rax

  611. #ifdef LN

  612. 	subq	$1, %rax

  613. #else

  614. 	subq	$2, %rax

  615. #endif

  616. 

  617. 	movq	AORIG, AO

  618. 	movq	BORIG, B

  619. 	leaq	BUFFER, BO

  620. 

  621. 	salq	$ZBASE_SHIFT, %rax

  622. 	leaq	(AO, %rax, 1), AO

  623. 	leaq	(B,  %rax, 2), B

  624. 	leaq	(BO, %rax, 4), BO

  625. #endif

  626. 

  627. 	SHUFPD_1 %xmm1, %xmm1

  628. 	SHUFPD_1 %xmm3, %xmm3

  629. 

  630. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  631.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  632. 	xorpd	%xmm15, %xmm1

  633. 	xorpd	%xmm15, %xmm3

  634. #else

  635. 	xorpd	%xmm15, %xmm0

  636. 	xorpd	%xmm15, %xmm2

  637. #endif

  638. 

  639. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  640.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  641. 	subpd	%xmm1, %xmm0

  642. 	subpd	%xmm3, %xmm2

  643. #else

  644. 	addpd	%xmm1, %xmm0

  645. 	addpd	%xmm3, %xmm2

  646. #endif

  647. 

  648. #if defined(LN) || defined(LT)

  649. 	movapd	 0 * SIZE(B), %xmm1

  650. 	movapd	 2 * SIZE(B), %xmm3

  651. 

  652. 	subpd	%xmm0,  %xmm1

  653. 	subpd	%xmm2,  %xmm3

  654. #else

  655. 	movapd	 0 * SIZE(AO), %xmm1

  656. 	movapd	 2 * SIZE(AO), %xmm3

  657. 

  658. 	subpd	%xmm0,  %xmm1

  659. 	subpd	%xmm2,  %xmm3

  660. #endif

  661. 

  662. #ifndef CONJ

  663. 	SHUFPD_1 %xmm15, %xmm15

  664. #endif

  665. 

  666. #if defined(LN) || defined(LT)

  667. 	movlpd	 0 * SIZE(AO), %xmm8

  668. 	movhpd	 0 * SIZE(AO), %xmm8

  669. 	movlpd	 1 * SIZE(AO), %xmm9

  670. 	movhpd	 1 * SIZE(AO), %xmm9

  671. 

  672. 	pshufd	$0x4e, %xmm1, %xmm0

  673. 	pshufd	$0x4e, %xmm3, %xmm2

  674. 

  675. 	xorpd	 %xmm15, %xmm0

  676. 	xorpd	 %xmm15, %xmm2

  677. 

  678. 	mulpd	 %xmm8, %xmm1

  679. 	mulpd	 %xmm9, %xmm0

  680. 	mulpd	 %xmm8, %xmm3

  681. 	mulpd	 %xmm9, %xmm2

  682. 

  683. 	addpd	 %xmm0, %xmm1

  684. 	addpd	 %xmm2, %xmm3

  685. #endif

  686. 

  687. #ifdef RN

  688. 	movlpd	 0 * SIZE(B), %xmm8

  689. 	movhpd	 0 * SIZE(B), %xmm8

  690. 	movlpd	 1 * SIZE(B), %xmm9

  691. 	movhpd	 1 * SIZE(B), %xmm9

  692. 	movlpd	 2 * SIZE(B), %xmm10

  693. 	movhpd	 2 * SIZE(B), %xmm10

  694. 	movlpd	 3 * SIZE(B), %xmm11

  695. 	movhpd	 3 * SIZE(B), %xmm11

  696. 	movlpd	 6 * SIZE(B), %xmm12

  697. 	movhpd	 6 * SIZE(B), %xmm12

  698. 	movlpd	 7 * SIZE(B), %xmm13

  699. 	movhpd	 7 * SIZE(B), %xmm13

  700. 

  701. 	pshufd	$0x4e, %xmm1, %xmm0

  702. 

  703. 	xorpd	 %xmm15, %xmm0

  704. 

  705. 	mulpd	 %xmm8, %xmm1

  706. 	mulpd	 %xmm9, %xmm0

  707. 

  708. 	addpd	 %xmm0, %xmm1

  709. 

  710. 	movapd	 %xmm1, %xmm0

  711. 	pshufd	 $0x4e, %xmm1, %xmm4

  712. 

  713. 	xorpd	 %xmm15, %xmm4

  714. 

  715. 	mulpd	 %xmm10, %xmm0

  716. 	mulpd	 %xmm11, %xmm4

  717. 

  718. 	subpd	 %xmm0, %xmm3

  719. 	subpd	 %xmm4, %xmm3

  720. 

  721. 	pshufd	$0x4e, %xmm3, %xmm2

  722. 

  723. 	xorpd	 %xmm15, %xmm2

  724. 

  725. 	mulpd	 %xmm12, %xmm3

  726. 	mulpd	 %xmm13, %xmm2

  727. 

  728. 	addpd	 %xmm2, %xmm3

  729. #endif

  730. 

  731. #ifdef RT

  732. 	movlpd	 6 * SIZE(B), %xmm8

  733. 	movhpd	 6 * SIZE(B), %xmm8

  734. 	movlpd	 7 * SIZE(B), %xmm9

  735. 	movhpd	 7 * SIZE(B), %xmm9

  736. 	movlpd	 4 * SIZE(B), %xmm10

  737. 	movhpd	 4 * SIZE(B), %xmm10

  738. 	movlpd	 5 * SIZE(B), %xmm11

  739. 	movhpd	 5 * SIZE(B), %xmm11

  740. 	movlpd	 0 * SIZE(B), %xmm12

  741. 	movhpd	 0 * SIZE(B), %xmm12

  742. 	movlpd	 1 * SIZE(B), %xmm13

  743. 	movhpd	 1 * SIZE(B), %xmm13

  744. 

  745. 	pshufd	$0x4e, %xmm3, %xmm2

  746. 

  747. 	xorpd	 %xmm15, %xmm2

  748. 

  749. 	mulpd	 %xmm8, %xmm3

  750. 	mulpd	 %xmm9, %xmm2

  751. 

  752. 	addpd	 %xmm2, %xmm3

  753. 

  754. 	movapd	 %xmm3, %xmm0

  755. 	pshufd	 $0x4e, %xmm3, %xmm4

  756. 

  757. 	xorpd	 %xmm15, %xmm4

  758. 

  759. 	mulpd	 %xmm10, %xmm0

  760. 	mulpd	 %xmm11, %xmm4

  761. 

  762. 	subpd	 %xmm0, %xmm1

  763. 	subpd	 %xmm4, %xmm1

  764. 

  765. 	pshufd	$0x4e, %xmm1, %xmm0

  766. 

  767. 	xorpd	 %xmm15, %xmm0

  768. 

  769. 	mulpd	 %xmm12, %xmm1

  770. 	mulpd	 %xmm13, %xmm0

  771. 

  772. 	addpd	 %xmm0, %xmm1

  773. #endif

  774. 

  775. #ifdef LN

  776. 	subq	$2 * SIZE, CO1

  777. 	subq	$2 * SIZE, CO2

  778. #endif

  779. 

  780. 	movsd	%xmm1,   0 * SIZE(CO1)

  781. 	movhpd	%xmm1,   1 * SIZE(CO1)

  782. 

  783. 	movsd	%xmm3,   0 * SIZE(CO2)

  784. 	movhpd	%xmm3,   1 * SIZE(CO2)

  785. 

  786. #if defined(LN) || defined(LT)

  787. 	movapd	%xmm1,   0 * SIZE(B)

  788. 	movapd	%xmm3,   2 * SIZE(B)

  789. 

  790. 	movlpd	%xmm1,   0 * SIZE(BO)

  791. 	movlpd	%xmm1,   1 * SIZE(BO)

  792. 	movhpd	%xmm1,   2 * SIZE(BO)

  793. 	movhpd	%xmm1,   3 * SIZE(BO)

  794. 	movlpd	%xmm3,   4 * SIZE(BO)

  795. 	movlpd	%xmm3,   5 * SIZE(BO)

  796. 	movhpd	%xmm3,   6 * SIZE(BO)

  797. 	movhpd	%xmm3,   7 * SIZE(BO)

  798. #else

  799. 	movapd	%xmm1,   0 * SIZE(AO)

  800. 	movapd	%xmm3,   2 * SIZE(AO)

  801. 

  802. #endif

  803. 

  804. #ifndef LN

  805. 	addq	$2 * SIZE, CO1

  806. 	addq	$2 * SIZE, CO2

  807. #endif

  808. 

  809. #if defined(LT) || defined(RN)

  810. 	movq	K,  %rax

  811. 	subq	KK, %rax

  812. 	salq	$ZBASE_SHIFT, %rax

  813. 	leaq	(AO, %rax, 1), AO

  814. #ifdef LT

  815. 	addq	$4 * SIZE, B

  816. #endif

  817. #endif

  818. 

  819. #ifdef LN

  820. 	subq	$1, KK

  821. 	movq	BORIG, B

  822. #endif

  823. 

  824. #ifdef LT

  825. 	addq	$1, KK

  826. #endif

  827. 

  828. #ifdef RT

  829. 	movq	K, %rax

  830. 	movq	BORIG, B

  831. 	salq	$0 + ZBASE_SHIFT, %rax

  832. 	addq	%rax, AORIG

  833. #endif

  834. 	ALIGN_4

  835. 

  836. .L30:

  837. 	movq	M,  I

  838. 	sarq	$1, I		# i = (m >> 2)

  839. 	jle	.L99

  840. 	ALIGN_4

  841. 

  842. .L10:

  843. #ifdef LN

  844.        movq	K, %rax

  845.        salq	$1 + ZBASE_SHIFT, %rax

  846.        subq	%rax, AORIG

  847. #endif

  848. 

  849. #if defined(LN) || defined(RT)

  850. 	movq	KK, %rax

  851. 	movq	AORIG, AO

  852. 	salq	$ZBASE_SHIFT, %rax

  853. 	leaq	(AO, %rax, 2), AO

  854. #endif

  855. 

  856. 	leaq	BUFFER, BO

  857. 

  858. #if defined(LN) || defined(RT)

  859. 	movq	KK, %rax

  860. 	salq	$1 + ZBASE_SHIFT, %rax

  861. 	leaq	(BO, %rax, 2), BO

  862. #endif

  863. 

  864. 	movapd	 0 * SIZE(AO), %xmm8

  865. 	pxor	%xmm0, %xmm0

  866. 	movapd	 2 * SIZE(AO), %xmm10

  867. 	pxor	%xmm1, %xmm1

  868. 	movapd	 4 * SIZE(AO), %xmm12

  869. 	pxor	%xmm2, %xmm2

  870. 	movapd	 6 * SIZE(AO), %xmm14

  871. 	pxor	%xmm3, %xmm3

  872. 

  873. 	movapd	 0 * SIZE(BO), %xmm9

  874. 	pxor	%xmm4, %xmm4

  875. 	movapd	 2 * SIZE(BO), %xmm11

  876. 	pxor	%xmm5, %xmm5

  877. 	movapd	 4 * SIZE(BO), %xmm13

  878. 	movapd	 8 * SIZE(BO), %xmm15

  879. 

  880. #ifdef LN

  881. 	PREFETCHW     -4 * SIZE(CO1)

  882. 	pxor	%xmm6, %xmm6

  883. 	PREFETCHW     -4 * SIZE(CO2)

  884. 	pxor	%xmm7, %xmm7

  885. #else

  886. 	PREFETCHW      4 * SIZE(CO1)

  887. 	pxor	%xmm6, %xmm6

  888. 	PREFETCHW      4 * SIZE(CO2)

  889. 	pxor	%xmm7, %xmm7

  890. #endif

  891. 

  892. #if defined(LT) || defined(RN)

  893. 	movq	KK, %rax

  894. #else

  895. 	movq	K, %rax

  896. 	subq	KK, %rax

  897. #endif

  898. 	andq	$-8, %rax

  899. 	salq	$4, %rax

  900. 	je	.L15

  901. .L1X:

  902. 	KERNEL1(16 *  0)

  903. 	KERNEL2(16 *  0)

  904. 	KERNEL3(16 *  0)

  905. 	KERNEL4(16 *  0)

  906. 	KERNEL5(16 *  0)

  907. 	KERNEL6(16 *  0)

  908. 	KERNEL7(16 *  0)

  909. 	KERNEL8(16 *  0)

  910. 	KERNEL1(16 *  1)

  911. 	KERNEL2(16 *  1)

  912. 	KERNEL3(16 *  1)

  913. 	KERNEL4(16 *  1)

  914. 	KERNEL5(16 *  1)

  915. 	KERNEL6(16 *  1)

  916. 	KERNEL7(16 *  1)

  917. 	KERNEL8(16 *  1)

  918. 	cmpq	$64 *  2, %rax

  919. 	jle	.L12

  920. 	KERNEL1(16 *  2)

  921. 	KERNEL2(16 *  2)

  922. 	KERNEL3(16 *  2)

  923. 	KERNEL4(16 *  2)

  924. 	KERNEL5(16 *  2)

  925. 	KERNEL6(16 *  2)

  926. 	KERNEL7(16 *  2)

  927. 	KERNEL8(16 *  2)

  928. 	KERNEL1(16 *  3)

  929. 	KERNEL2(16 *  3)

  930. 	KERNEL3(16 *  3)

  931. 	KERNEL4(16 *  3)

  932. 	KERNEL5(16 *  3)

  933. 	KERNEL6(16 *  3)

  934. 	KERNEL7(16 *  3)

  935. 	KERNEL8(16 *  3)

  936. 	cmpq	$64 *  4, %rax

  937. 	jle	.L12

  938. 	KERNEL1(16 *  4)

  939. 	KERNEL2(16 *  4)

  940. 	KERNEL3(16 *  4)

  941. 	KERNEL4(16 *  4)

  942. 	KERNEL5(16 *  4)

  943. 	KERNEL6(16 *  4)

  944. 	KERNEL7(16 *  4)

  945. 	KERNEL8(16 *  4)

  946. 	KERNEL1(16 *  5)

  947. 	KERNEL2(16 *  5)

  948. 	KERNEL3(16 *  5)

  949. 	KERNEL4(16 *  5)

  950. 	KERNEL5(16 *  5)

  951. 	KERNEL6(16 *  5)

  952. 	KERNEL7(16 *  5)

  953. 	KERNEL8(16 *  5)

  954. 	cmpq	$64 *  6, %rax

  955. 	jle	.L12

  956. 	KERNEL1(16 *  6)

  957. 	KERNEL2(16 *  6)

  958. 	KERNEL3(16 *  6)

  959. 	KERNEL4(16 *  6)

  960. 	KERNEL5(16 *  6)

  961. 	KERNEL6(16 *  6)

  962. 	KERNEL7(16 *  6)

  963. 	KERNEL8(16 *  6)

  964. 	KERNEL1(16 *  7)

  965. 	KERNEL2(16 *  7)

  966. 	KERNEL3(16 *  7)

  967. 	KERNEL4(16 *  7)

  968. 	KERNEL5(16 *  7)

  969. 	KERNEL6(16 *  7)

  970. 	KERNEL7(16 *  7)

  971. 	KERNEL8(16 *  7)

  972. 

  973. 	addq	$16 * 8  * SIZE, AO

  974. 	addq	$32 * 8  * SIZE, BO

  975. 	subq	$64 * 8, %rax

  976. 	jg	.L1X

  977. 

  978. .L12:

  979. 	leaq	(AO, %rax, 2), AO	# * 16

  980. 	leaq	(BO, %rax, 4), BO	# * 64

  981. 	ALIGN_4

  982. 

  983. .L15:

  984. #if defined(LT) || defined(RN)

  985. 	movq	KK, %rax

  986. #else

  987. 	movq	K, %rax

  988. 	subq	KK, %rax

  989. #endif

  990. 	movapd	POSINV,  %xmm15

  991. 	andq	$7, %rax		# if (k & 1)

  992. 	BRANCH

  993. 	je .L19

  994. 	ALIGN_4

  995. 

  996. .L16:

  997. 	mulpd	%xmm8, %xmm9

  998. 	addpd	%xmm9, %xmm0

  999. 	movapd	 2 * SIZE(BO), %xmm9

  1000. 	mulpd	%xmm8, %xmm9

  1001. 	addpd	%xmm9, %xmm1

  1002. 	movapd	 4 * SIZE(BO), %xmm9

  1003. 	mulpd	%xmm8, %xmm9

  1004. 	mulpd	 6 * SIZE(BO), %xmm8

  1005. 	addpd	%xmm9, %xmm2

  1006. 	movapd	 0 * SIZE(BO), %xmm9

  1007. 	addpd	%xmm8, %xmm3

  1008. 	movapd	 4 * SIZE(AO), %xmm8

  1009. 	mulpd	%xmm10, %xmm9

  1010. 	addpd	%xmm9, %xmm4

  1011. 	movapd	 2 * SIZE(BO), %xmm9

  1012. 	mulpd	%xmm10, %xmm9

  1013. 	addpd	%xmm9, %xmm5

  1014. 	movapd	 4 * SIZE(BO), %xmm9

  1015. 	mulpd	%xmm10, %xmm9

  1016. 	mulpd	 6 * SIZE(BO), %xmm10

  1017. 	addpd	%xmm9, %xmm6

  1018. 	movapd	 8 * SIZE(BO), %xmm9

  1019. 	addpd	%xmm10, %xmm7

  1020. 	movapd	 6 * SIZE(AO), %xmm10

  1021. 

  1022. 	addq	$4 * SIZE, AO		# aoffset  += 4

  1023. 	addq	$8 * SIZE, BO		# boffset1 += 8

  1024. 	decq	%rax

  1025. 	jg	.L16

  1026. 	ALIGN_4

  1027. 

  1028. .L19:

  1029. #if defined(LN) || defined(RT)

  1030. 	movq	KK, %rax

  1031. #ifdef LN

  1032. 	subq	$2, %rax

  1033. #else

  1034. 	subq	$2, %rax

  1035. #endif

  1036. 

  1037. 	movq	AORIG, AO

  1038. 	movq	BORIG, B

  1039. 	leaq	BUFFER, BO

  1040. 

  1041. 	salq	$ZBASE_SHIFT, %rax

  1042. 	leaq	(AO, %rax, 2), AO

  1043. 	leaq	(B,  %rax, 2), B

  1044. 	leaq	(BO, %rax, 4), BO

  1045. #endif

  1046. 

  1047. 	SHUFPD_1 %xmm1, %xmm1

  1048. 	SHUFPD_1 %xmm3, %xmm3

  1049. 	SHUFPD_1 %xmm5, %xmm5

  1050. 	SHUFPD_1 %xmm7, %xmm7

  1051. 

  1052. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1053.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1054. 	xorpd	%xmm15, %xmm1

  1055. 	xorpd	%xmm15, %xmm3

  1056. 	xorpd	%xmm15, %xmm5

  1057. 	xorpd	%xmm15, %xmm7

  1058. #else

  1059. 	xorpd	%xmm15, %xmm0

  1060. 	xorpd	%xmm15, %xmm2

  1061. 	xorpd	%xmm15, %xmm4

  1062. 	xorpd	%xmm15, %xmm6

  1063. #endif

  1064. 

  1065. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1066.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  1067. 	subpd	%xmm1, %xmm0

  1068. 	subpd	%xmm3, %xmm2

  1069. 	subpd	%xmm5, %xmm4

  1070. 	subpd	%xmm7, %xmm6

  1071. #else

  1072. 	addpd	%xmm1, %xmm0

  1073. 	addpd	%xmm3, %xmm2

  1074. 	addpd	%xmm5, %xmm4

  1075. 	addpd	%xmm7, %xmm6

  1076. #endif

  1077. 

  1078. #if defined(LN) || defined(LT)

  1079. 	movapd	 0 * SIZE(B), %xmm1

  1080. 	movapd	 2 * SIZE(B), %xmm3

  1081. 	movapd	 4 * SIZE(B), %xmm5

  1082. 	movapd	 6 * SIZE(B), %xmm7

  1083. 

  1084. 	subpd	%xmm0,  %xmm1

  1085. 	subpd	%xmm2,  %xmm3

  1086. 	subpd	%xmm4,  %xmm5

  1087. 	subpd	%xmm6,  %xmm7

  1088. #else

  1089. 	movapd	 0 * SIZE(AO), %xmm1

  1090. 	movapd	 2 * SIZE(AO), %xmm5

  1091. 	movapd	 4 * SIZE(AO), %xmm3

  1092. 	movapd	 6 * SIZE(AO), %xmm7

  1093. 

  1094. 	subpd	%xmm0,  %xmm1

  1095. 	subpd	%xmm2,  %xmm3

  1096. 	subpd	%xmm4,  %xmm5

  1097. 	subpd	%xmm6,  %xmm7

  1098. #endif

  1099. 

  1100. #ifndef CONJ

  1101. 	SHUFPD_1 %xmm15, %xmm15

  1102. #endif

  1103. 

  1104. #ifdef LN

  1105. 	movlpd	 6 * SIZE(AO), %xmm8

  1106. 	movhpd	 6 * SIZE(AO), %xmm8

  1107. 	movlpd	 7 * SIZE(AO), %xmm9

  1108. 	movhpd	 7 * SIZE(AO), %xmm9

  1109. 	movlpd	 4 * SIZE(AO), %xmm10

  1110. 	movhpd	 4 * SIZE(AO), %xmm10

  1111. 	movlpd	 5 * SIZE(AO), %xmm11

  1112. 	movhpd	 5 * SIZE(AO), %xmm11

  1113. 	movlpd	 0 * SIZE(AO), %xmm12

  1114. 	movhpd	 0 * SIZE(AO), %xmm12

  1115. 	movlpd	 1 * SIZE(AO), %xmm13

  1116. 	movhpd	 1 * SIZE(AO), %xmm13

  1117. 

  1118. 	pshufd	$0x4e, %xmm5, %xmm4

  1119. 	pshufd	$0x4e, %xmm7, %xmm6

  1120. 

  1121. 	xorpd	 %xmm15, %xmm4

  1122. 	xorpd	 %xmm15, %xmm6

  1123. 

  1124. 	mulpd	 %xmm8, %xmm5

  1125. 	mulpd	 %xmm9, %xmm4

  1126. 	mulpd	 %xmm8, %xmm7

  1127. 	mulpd	 %xmm9, %xmm6

  1128. 

  1129. 	addpd	 %xmm4, %xmm5

  1130. 	addpd	 %xmm6, %xmm7

  1131. 

  1132. 	movapd	 %xmm5, %xmm0

  1133. 	movapd	 %xmm7, %xmm2

  1134. 	pshufd	 $0x4e, %xmm5, %xmm4

  1135. 	pshufd	 $0x4e, %xmm7, %xmm6

  1136. 

  1137. 	xorpd	 %xmm15, %xmm4

  1138. 	xorpd	 %xmm15, %xmm6

  1139. 

  1140. 	mulpd	 %xmm10, %xmm0

  1141. 	mulpd	 %xmm10, %xmm2

  1142. 	mulpd	 %xmm11, %xmm4

  1143. 	mulpd	 %xmm11, %xmm6

  1144. 

  1145. 	subpd	 %xmm0, %xmm1

  1146. 	subpd	 %xmm2, %xmm3

  1147. 	subpd	 %xmm4, %xmm1

  1148. 	subpd	 %xmm6, %xmm3

  1149. 

  1150. 	pshufd	$0x4e, %xmm1, %xmm0

  1151. 	pshufd	$0x4e, %xmm3, %xmm2

  1152. 

  1153. 	xorpd	 %xmm15, %xmm0

  1154. 	xorpd	 %xmm15, %xmm2

  1155. 

  1156. 	mulpd	 %xmm12, %xmm1

  1157. 	mulpd	 %xmm13, %xmm0

  1158. 	mulpd	 %xmm12, %xmm3

  1159. 	mulpd	 %xmm13, %xmm2

  1160. 

  1161. 	addpd	 %xmm0, %xmm1

  1162. 	addpd	 %xmm2, %xmm3

  1163. #endif

  1164. 

  1165. #ifdef LT

  1166. 	movlpd	 0 * SIZE(AO), %xmm8

  1167. 	movhpd	 0 * SIZE(AO), %xmm8

  1168. 	movlpd	 1 * SIZE(AO), %xmm9

  1169. 	movhpd	 1 * SIZE(AO), %xmm9

  1170. 	movlpd	 2 * SIZE(AO), %xmm10

  1171. 	movhpd	 2 * SIZE(AO), %xmm10

  1172. 	movlpd	 3 * SIZE(AO), %xmm11

  1173. 	movhpd	 3 * SIZE(AO), %xmm11

  1174. 	movlpd	 6 * SIZE(AO), %xmm12

  1175. 	movhpd	 6 * SIZE(AO), %xmm12

  1176. 	movlpd	 7 * SIZE(AO), %xmm13

  1177. 	movhpd	 7 * SIZE(AO), %xmm13

  1178. 

  1179. 	pshufd	$0x4e, %xmm1, %xmm0

  1180. 	pshufd	$0x4e, %xmm3, %xmm2

  1181. 

  1182. 	xorpd	 %xmm15, %xmm0

  1183. 	xorpd	 %xmm15, %xmm2

  1184. 

  1185. 	mulpd	 %xmm8, %xmm1

  1186. 	mulpd	 %xmm9, %xmm0

  1187. 	mulpd	 %xmm8, %xmm3

  1188. 	mulpd	 %xmm9, %xmm2

  1189. 

  1190. 	addpd	 %xmm0, %xmm1

  1191. 	addpd	 %xmm2, %xmm3

  1192. 

  1193. 	movapd	 %xmm1, %xmm0

  1194. 	movapd	 %xmm3, %xmm2

  1195. 	pshufd	 $0x4e, %xmm1, %xmm4

  1196. 	pshufd	 $0x4e, %xmm3, %xmm6

  1197. 

  1198. 	xorpd	 %xmm15, %xmm4

  1199. 	xorpd	 %xmm15, %xmm6

  1200. 

  1201. 	mulpd	 %xmm10, %xmm0

  1202. 	mulpd	 %xmm10, %xmm2

  1203. 	mulpd	 %xmm11, %xmm4

  1204. 	mulpd	 %xmm11, %xmm6

  1205. 

  1206. 	subpd	 %xmm0, %xmm5

  1207. 	subpd	 %xmm2, %xmm7

  1208. 	subpd	 %xmm4, %xmm5

  1209. 	subpd	 %xmm6, %xmm7

  1210. 

  1211. 	pshufd	$0x4e, %xmm5, %xmm4

  1212. 	pshufd	$0x4e, %xmm7, %xmm6

  1213. 

  1214. 	xorpd	 %xmm15, %xmm4

  1215. 	xorpd	 %xmm15, %xmm6

  1216. 

  1217. 	mulpd	 %xmm12, %xmm5

  1218. 	mulpd	 %xmm13, %xmm4

  1219. 	mulpd	 %xmm12, %xmm7

  1220. 	mulpd	 %xmm13, %xmm6

  1221. 

  1222. 	addpd	 %xmm4, %xmm5

  1223. 	addpd	 %xmm6, %xmm7

  1224. #endif

  1225. 

  1226. #ifdef RN

  1227. 	movlpd	 0 * SIZE(B), %xmm8

  1228. 	movhpd	 0 * SIZE(B), %xmm8

  1229. 	movlpd	 1 * SIZE(B), %xmm9

  1230. 	movhpd	 1 * SIZE(B), %xmm9

  1231. 	movlpd	 2 * SIZE(B), %xmm10

  1232. 	movhpd	 2 * SIZE(B), %xmm10

  1233. 	movlpd	 3 * SIZE(B), %xmm11

  1234. 	movhpd	 3 * SIZE(B), %xmm11

  1235. 	movlpd	 6 * SIZE(B), %xmm12

  1236. 	movhpd	 6 * SIZE(B), %xmm12

  1237. 	movlpd	 7 * SIZE(B), %xmm13

  1238. 	movhpd	 7 * SIZE(B), %xmm13

  1239. 

  1240. 	pshufd	$0x4e, %xmm1, %xmm0

  1241. 	pshufd	$0x4e, %xmm5, %xmm4

  1242. 

  1243. 	xorpd	 %xmm15, %xmm0

  1244. 	xorpd	 %xmm15, %xmm4

  1245. 

  1246. 	mulpd	 %xmm8, %xmm1

  1247. 	mulpd	 %xmm9, %xmm0

  1248. 	mulpd	 %xmm8, %xmm5

  1249. 	mulpd	 %xmm9, %xmm4

  1250. 

  1251. 	addpd	 %xmm0, %xmm1

  1252. 	addpd	 %xmm4, %xmm5

  1253. 

  1254. 	movapd	 %xmm1, %xmm0

  1255. 	movapd	 %xmm5, %xmm2

  1256. 	pshufd	 $0x4e, %xmm1, %xmm4

  1257. 	pshufd	 $0x4e, %xmm5, %xmm6

  1258. 

  1259. 	xorpd	 %xmm15, %xmm4

  1260. 	xorpd	 %xmm15, %xmm6

  1261. 

  1262. 	mulpd	 %xmm10, %xmm0

  1263. 	mulpd	 %xmm10, %xmm2

  1264. 	mulpd	 %xmm11, %xmm4

  1265. 	mulpd	 %xmm11, %xmm6

  1266. 

  1267. 	subpd	 %xmm0, %xmm3

  1268. 	subpd	 %xmm2, %xmm7

  1269. 	subpd	 %xmm4, %xmm3

  1270. 	subpd	 %xmm6, %xmm7

  1271. 

  1272. 	pshufd	$0x4e, %xmm3, %xmm2

  1273. 	pshufd	$0x4e, %xmm7, %xmm6

  1274. 

  1275. 	xorpd	 %xmm15, %xmm2

  1276. 	xorpd	 %xmm15, %xmm6

  1277. 

  1278. 	mulpd	 %xmm12, %xmm3

  1279. 	mulpd	 %xmm13, %xmm2

  1280. 	mulpd	 %xmm12, %xmm7

  1281. 	mulpd	 %xmm13, %xmm6

  1282. 

  1283. 	addpd	 %xmm2, %xmm3

  1284. 	addpd	 %xmm6, %xmm7

  1285. #endif

  1286. 

  1287. #ifdef RT

  1288. 	movlpd	 6 * SIZE(B), %xmm8

  1289. 	movhpd	 6 * SIZE(B), %xmm8

  1290. 	movlpd	 7 * SIZE(B), %xmm9

  1291. 	movhpd	 7 * SIZE(B), %xmm9

  1292. 	movlpd	 4 * SIZE(B), %xmm10

  1293. 	movhpd	 4 * SIZE(B), %xmm10

  1294. 	movlpd	 5 * SIZE(B), %xmm11

  1295. 	movhpd	 5 * SIZE(B), %xmm11

  1296. 	movlpd	 0 * SIZE(B), %xmm12

  1297. 	movhpd	 0 * SIZE(B), %xmm12

  1298. 	movlpd	 1 * SIZE(B), %xmm13

  1299. 	movhpd	 1 * SIZE(B), %xmm13

  1300. 

  1301. 	pshufd	$0x4e, %xmm3, %xmm2

  1302. 	pshufd	$0x4e, %xmm7, %xmm6

  1303. 

  1304. 	xorpd	 %xmm15, %xmm2

  1305. 	xorpd	 %xmm15, %xmm6

  1306. 

  1307. 	mulpd	 %xmm8, %xmm3

  1308. 	mulpd	 %xmm9, %xmm2

  1309. 	mulpd	 %xmm8, %xmm7

  1310. 	mulpd	 %xmm9, %xmm6

  1311. 

  1312. 	addpd	 %xmm2, %xmm3

  1313. 	addpd	 %xmm6, %xmm7

  1314. 

  1315. 	movapd	 %xmm3, %xmm0

  1316. 	movapd	 %xmm7, %xmm2

  1317. 	pshufd	 $0x4e, %xmm3, %xmm4

  1318. 	pshufd	 $0x4e, %xmm7, %xmm6

  1319. 

  1320. 	xorpd	 %xmm15, %xmm4

  1321. 	xorpd	 %xmm15, %xmm6

  1322. 

  1323. 	mulpd	 %xmm10, %xmm0

  1324. 	mulpd	 %xmm10, %xmm2

  1325. 	mulpd	 %xmm11, %xmm4

  1326. 	mulpd	 %xmm11, %xmm6

  1327. 

  1328. 	subpd	 %xmm0, %xmm1

  1329. 	subpd	 %xmm2, %xmm5

  1330. 	subpd	 %xmm4, %xmm1

  1331. 	subpd	 %xmm6, %xmm5

  1332. 

  1333. 	pshufd	$0x4e, %xmm1, %xmm0

  1334. 	pshufd	$0x4e, %xmm5, %xmm4

  1335. 

  1336. 	xorpd	 %xmm15, %xmm0

  1337. 	xorpd	 %xmm15, %xmm4

  1338. 

  1339. 	mulpd	 %xmm12, %xmm1

  1340. 	mulpd	 %xmm13, %xmm0

  1341. 	mulpd	 %xmm12, %xmm5

  1342. 	mulpd	 %xmm13, %xmm4

  1343. 

  1344. 	addpd	 %xmm0, %xmm1

  1345. 	addpd	 %xmm4, %xmm5

  1346. #endif

  1347. 

  1348. #ifdef LN

  1349. 	subq	$4 * SIZE, CO1

  1350. 	subq	$4 * SIZE, CO2

  1351. #endif

  1352. 

  1353. 	movsd	%xmm1,   0 * SIZE(CO1)

  1354. 	movhpd	%xmm1,   1 * SIZE(CO1)

  1355. 	movsd	%xmm5,   2 * SIZE(CO1)

  1356. 	movhpd	%xmm5,   3 * SIZE(CO1)

  1357. 

  1358. 	movsd	%xmm3,   0 * SIZE(CO2)

  1359. 	movhpd	%xmm3,   1 * SIZE(CO2)

  1360. 	movsd	%xmm7,   2 * SIZE(CO2)

  1361. 	movhpd	%xmm7,   3 * SIZE(CO2)

  1362. 

  1363. #if defined(LN) || defined(LT)

  1364. 	movapd	%xmm1,   0 * SIZE(B)

  1365. 	movapd	%xmm3,   2 * SIZE(B)

  1366. 	movapd	%xmm5,   4 * SIZE(B)

  1367. 	movapd	%xmm7,   6 * SIZE(B)

  1368. 

  1369. 	movlpd	%xmm1,   0 * SIZE(BO)

  1370. 	movlpd	%xmm1,   1 * SIZE(BO)

  1371. 	movhpd	%xmm1,   2 * SIZE(BO)

  1372. 	movhpd	%xmm1,   3 * SIZE(BO)

  1373. 	movlpd	%xmm3,   4 * SIZE(BO)

  1374. 	movlpd	%xmm3,   5 * SIZE(BO)

  1375. 	movhpd	%xmm3,   6 * SIZE(BO)

  1376. 	movhpd	%xmm3,   7 * SIZE(BO)

  1377. 	movlpd	%xmm5,   8 * SIZE(BO)

  1378. 	movlpd	%xmm5,   9 * SIZE(BO)

  1379. 	movhpd	%xmm5,  10 * SIZE(BO)

  1380. 	movhpd	%xmm5,  11 * SIZE(BO)

  1381. 	movlpd	%xmm7,  12 * SIZE(BO)

  1382. 	movlpd	%xmm7,  13 * SIZE(BO)

  1383. 	movhpd	%xmm7,  14 * SIZE(BO)

  1384. 	movhpd	%xmm7,  15 * SIZE(BO)

  1385. #else

  1386. 	movapd	%xmm1,   0 * SIZE(AO)

  1387. 	movapd	%xmm5,   2 * SIZE(AO)

  1388. 	movapd	%xmm3,   4 * SIZE(AO)

  1389. 	movapd	%xmm7,   6 * SIZE(AO)

  1390. #endif

  1391. 

  1392. #ifndef LN

  1393. 	addq	$4 * SIZE, CO1

  1394. 	addq	$4 * SIZE, CO2

  1395. #endif

  1396. 

  1397. #if defined(LT) || defined(RN)

  1398. 	movq	K,  %rax

  1399. 	subq	KK, %rax

  1400. 	salq	$ZBASE_SHIFT, %rax

  1401. 	leaq	(AO, %rax, 2), AO

  1402. #ifdef LT

  1403. 	addq	$8 * SIZE, B

  1404. #endif

  1405. #endif

  1406. 

  1407. #ifdef LN

  1408. 	subq	$2, KK

  1409. 	movq	BORIG, B

  1410. #endif

  1411. 

  1412. #ifdef LT

  1413. 	addq	$2, KK

  1414. #endif

  1415. 

  1416. #ifdef RT

  1417. 	movq	K, %rax

  1418. 	movq	BORIG, B

  1419. 	salq	$1 + ZBASE_SHIFT, %rax

  1420. 	addq	%rax, AORIG

  1421. #endif

  1422. 

  1423. 	decq	I			# i --

  1424. 	jg	.L10

  1425. 	ALIGN_4

  1426. 

  1427. 

  1428. .L99:

  1429. #ifdef LN

  1430.        leaq	(, K, SIZE), %rax

  1431.        leaq	(B, %rax, 4), B

  1432. #endif

  1433. 

  1434. #if defined(LT) || defined(RN)

  1435. 	movq	K,  %rax

  1436. 	subq	KK, %rax

  1437. 	leaq	(,%rax, SIZE), %rax

  1438. 	leaq	(B,  %rax, 2 * COMPSIZE), B

  1439. #endif

  1440. 

  1441. #ifdef RN

  1442. 	addq	$2, KK

  1443. #endif

  1444. 

  1445. #ifdef RT

  1446. 	subq	$2, KK

  1447. #endif

  1448. 

  1449. 	decq	J			# j --

  1450. 	jg	.L01

  1451. 

  1452. .L100:

  1453. 	testq	$1, N

  1454. 	jle	.L999

  1455. 

  1456. .L101:

  1457. #ifdef LN

  1458. 	movq	OFFSET, %rax

  1459. 	addq	M, %rax

  1460. 	movq	%rax, KK

  1461. #endif

  1462. 

  1463. /* Copying to Sub Buffer */

  1464. 	leaq	BUFFER, BO

  1465. 

  1466. #ifdef RT

  1467.        movq	K, %rax

  1468.        salq	$0 + ZBASE_SHIFT, %rax

  1469.        subq	%rax, B

  1470. #endif

  1471. 

  1472. #if defined(LN) || defined(RT)

  1473. 	movq	KK, %rax

  1474. 	movq	B, BORIG

  1475. 	salq	$ZBASE_SHIFT, %rax

  1476. 	leaq	(B,  %rax, 1), B

  1477. 	leaq	(BO, %rax, 2), BO

  1478. #endif

  1479. 

  1480. #if defined(LT)

  1481. 	movq	OFFSET, %rax

  1482. 	movq	%rax, KK

  1483. #endif

  1484. 

  1485. #if defined(LT) || defined(RN)

  1486. 	movq	KK, %rax

  1487. #else

  1488. 	movq	K, %rax

  1489. 	subq	KK, %rax

  1490. #endif

  1491. 	sarq	$2, %rax

  1492. 	jle	.L103

  1493. 	ALIGN_4

  1494. 

  1495. .L102:

  1496. 	movlpd	 0 * SIZE(B), %xmm0

  1497. 	movlpd	 1 * SIZE(B), %xmm1

  1498. 	movlpd	 2 * SIZE(B), %xmm2

  1499. 	movlpd	 3 * SIZE(B), %xmm3

  1500. 	movlpd	 4 * SIZE(B), %xmm4

  1501. 	movlpd	 5 * SIZE(B), %xmm5

  1502. 	movlpd	 6 * SIZE(B), %xmm6

  1503. 	movlpd	 7 * SIZE(B), %xmm7

  1504. 

  1505. 	movlpd	%xmm0,  0 * SIZE(BO)

  1506. 	movlpd	%xmm0,  1 * SIZE(BO)

  1507. 	movlpd	%xmm1,  2 * SIZE(BO)

  1508. 	movlpd	%xmm1,  3 * SIZE(BO)

  1509. 	movlpd	%xmm2,  4 * SIZE(BO)

  1510. 	movlpd	%xmm2,  5 * SIZE(BO)

  1511. 	movlpd	%xmm3,  6 * SIZE(BO)

  1512. 	movlpd	%xmm3,  7 * SIZE(BO)

  1513. 	movlpd	%xmm4,  8 * SIZE(BO)

  1514. 	movlpd	%xmm4,  9 * SIZE(BO)

  1515. 	movlpd	%xmm5, 10 * SIZE(BO)

  1516. 	movlpd	%xmm5, 11 * SIZE(BO)

  1517. 	movlpd	%xmm6, 12 * SIZE(BO)

  1518. 	movlpd	%xmm6, 13 * SIZE(BO)

  1519. 	movlpd	%xmm7, 14 * SIZE(BO)

  1520. 	movlpd	%xmm7, 15 * SIZE(BO)

  1521. 

  1522. 	subq	$-16 * SIZE, BO

  1523. 	addq	$ 8 * SIZE, B

  1524. 	decq	%rax

  1525. 	jne	.L102

  1526. 	ALIGN_4

  1527. 

  1528. .L103:

  1529. #if defined(LT) || defined(RN)

  1530. 	movq	KK, %rax

  1531. #else

  1532. 	movq	K, %rax

  1533. 	subq	KK, %rax

  1534. #endif

  1535. 	andq	$3, %rax

  1536. 	BRANCH

  1537. 	jle	.L105

  1538. 	ALIGN_4

  1539. 

  1540. .L104:

  1541. 	movlpd	 0 * SIZE(B), %xmm0

  1542. 	movlpd	 1 * SIZE(B), %xmm1

  1543. 

  1544. 	movlpd	%xmm0,  0 * SIZE(BO)

  1545. 	movlpd	%xmm0,  1 * SIZE(BO)

  1546. 	movlpd	%xmm1,  2 * SIZE(BO)

  1547. 	movlpd	%xmm1,  3 * SIZE(BO)

  1548. 

  1549. 	addq	$4 * SIZE, BO

  1550. 	addq	$2 * SIZE, B

  1551. 	decq	%rax

  1552. 	jne	.L104

  1553. 	ALIGN_4

  1554. 

  1555. .L105:

  1556. #if defined(LT) || defined(RN)

  1557. 	movq	A, AO

  1558. #else

  1559. 	movq	A, AORIG

  1560. #endif

  1561. 

  1562. #ifdef RT

  1563.        subq	LDC, C

  1564. #endif

  1565. 

  1566. 	movq	C, CO1

  1567. #ifndef RT

  1568. 	addq	LDC, C

  1569. #endif

  1570. 

  1571. 	testq	$1, M

  1572. 	jle	.L130

  1573. 	ALIGN_4

  1574. 

  1575. .L140:

  1576. #ifdef LN

  1577.        movq	K, %rax

  1578.        salq	$0 + ZBASE_SHIFT, %rax

  1579.        subq	%rax, AORIG

  1580. #endif

  1581. 

  1582. #if defined(LN) || defined(RT)

  1583. 	movq	KK, %rax

  1584. 	movq	AORIG, AO

  1585. 	salq	$ZBASE_SHIFT, %rax

  1586. 	leaq	(AO, %rax, 1), AO

  1587. #endif

  1588. 

  1589. 	leaq	BUFFER, BO

  1590. 

  1591. #if defined(LN) || defined(RT)

  1592. 	movq	KK, %rax

  1593. 	salq	$0 + ZBASE_SHIFT, %rax

  1594. 	leaq	(BO, %rax, 2), BO

  1595. #endif

  1596. 

  1597. 	pxor	%xmm0, %xmm0

  1598. 	pxor	%xmm1, %xmm1

  1599. 	pxor	%xmm2, %xmm2

  1600. 	pxor	%xmm3, %xmm3

  1601. 

  1602. #if defined(LT) || defined(RN)

  1603. 	movq	KK, %rax

  1604. #else

  1605. 	movq	K, %rax

  1606. 	subq	KK, %rax

  1607. #endif

  1608. 	sarq	$2, %rax

  1609. 	je	.L142

  1610. 

  1611. .L141:

  1612. 	movapd	 0 * SIZE(AO), %xmm8

  1613. 	movapd	 0 * SIZE(BO), %xmm9

  1614. 	mulpd	%xmm8, %xmm9

  1615. 	addpd	%xmm9, %xmm0

  1616. 	mulpd	 2 * SIZE(BO), %xmm8

  1617. 	addpd	%xmm8, %xmm1

  1618. 

  1619. 	movapd	 2 * SIZE(AO), %xmm8

  1620. 	movapd	 4 * SIZE(BO), %xmm9

  1621. 	mulpd	%xmm8, %xmm9

  1622. 	addpd	%xmm9, %xmm2

  1623. 	mulpd	 6 * SIZE(BO), %xmm8

  1624. 	addpd	%xmm8, %xmm3

  1625. 

  1626. 	movapd	 4 * SIZE(AO), %xmm8

  1627. 	movapd	 8 * SIZE(BO), %xmm9

  1628. 	mulpd	%xmm8, %xmm9

  1629. 	addpd	%xmm9, %xmm0

  1630. 	mulpd	10 * SIZE(BO), %xmm8

  1631. 	addpd	%xmm8, %xmm1

  1632. 

  1633. 	movapd	 6 * SIZE(AO), %xmm8

  1634. 	movapd	12 * SIZE(BO), %xmm9

  1635. 	mulpd	%xmm8, %xmm9

  1636. 	addpd	%xmm9, %xmm2

  1637. 	mulpd	14 * SIZE(BO), %xmm8

  1638. 	addpd	%xmm8, %xmm3

  1639. 

  1640. 	addq   $8  * SIZE, AO

  1641. 	addq   $16 * SIZE, BO

  1642. 	decq   %rax

  1643. 	jne    .L141

  1644. 

  1645. .L142:

  1646. 	addpd	%xmm2, %xmm0

  1647. 	addpd	%xmm3, %xmm1

  1648. 

  1649. 	movapd	POSINV, %xmm15

  1650. 

  1651. #if defined(LT) || defined(RN)

  1652. 	movq	KK, %rax

  1653. #else

  1654. 	movq	K, %rax

  1655. 	subq	KK, %rax

  1656. #endif

  1657. 	andq	$3, %rax		# if (k & 1)

  1658. 	BRANCH

  1659. 	jle .L144

  1660. 

  1661. .L143:

  1662. 	movapd	 0 * SIZE(AO), %xmm8

  1663. 	movapd	 0 * SIZE(BO), %xmm9

  1664. 	mulpd	%xmm8, %xmm9

  1665. 	addpd	%xmm9, %xmm0

  1666. 	mulpd	 2 * SIZE(BO), %xmm8

  1667. 	addpd	%xmm8, %xmm1

  1668. 

  1669. 	addq	$2 * SIZE, AO		# aoffset  += 4

  1670. 	addq	$4 * SIZE, BO		# boffset1 += 8

  1671. 	decq	%rax

  1672. 	jg	.L143

  1673. 	ALIGN_4

  1674. 

  1675. .L144:

  1676. #if defined(LN) || defined(RT)

  1677. 	movq	KK, %rax

  1678. #ifdef LN

  1679. 	subq	$1, %rax

  1680. #else

  1681. 	subq	$1, %rax

  1682. #endif

  1683. 

  1684. 	movq	AORIG, AO

  1685. 	movq	BORIG, B

  1686. 	leaq	BUFFER, BO

  1687. 

  1688. 	salq	$ZBASE_SHIFT, %rax

  1689. 	leaq	(AO, %rax, 1), AO

  1690. 	leaq	(B,  %rax, 1), B

  1691. 	leaq	(BO, %rax, 2), BO

  1692. #endif

  1693. 

  1694. 	SHUFPD_1 %xmm1, %xmm1

  1695. 

  1696. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1697.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1698. 	xorpd	%xmm15, %xmm1

  1699. #else

  1700. 	xorpd	%xmm15, %xmm0

  1701. #endif

  1702. 

  1703. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1704.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  1705. 	subpd	%xmm1, %xmm0

  1706. #else

  1707. 	addpd	%xmm1, %xmm0

  1708. #endif

  1709. 

  1710. 

  1711. #if defined(LN) || defined(LT)

  1712. 	movapd	 0 * SIZE(B), %xmm1

  1713. 

  1714. 	subpd	%xmm0,  %xmm1

  1715. #else

  1716. 	movapd	 0 * SIZE(AO), %xmm1

  1717. 

  1718. 	subpd	%xmm0,  %xmm1

  1719. #endif

  1720. 

  1721. #ifndef CONJ

  1722. 	SHUFPD_1 %xmm15, %xmm15

  1723. #endif

  1724. 

  1725. #ifdef LN

  1726. 	movlpd	 0 * SIZE(AO), %xmm8

  1727. 	movhpd	 0 * SIZE(AO), %xmm8

  1728. 	movlpd	 1 * SIZE(AO), %xmm9

  1729. 	movhpd	 1 * SIZE(AO), %xmm9

  1730. 

  1731. 	pshufd	$0x4e, %xmm1, %xmm0

  1732. 	xorpd	 %xmm15, %xmm0

  1733. 

  1734. 	mulpd	 %xmm8, %xmm1

  1735. 	mulpd	 %xmm9, %xmm0

  1736. 

  1737. 	addpd	 %xmm0, %xmm1

  1738. #endif

  1739. 

  1740. #ifdef LT

  1741. 	movlpd	 0 * SIZE(AO), %xmm8

  1742. 	movhpd	 0 * SIZE(AO), %xmm8

  1743. 	movlpd	 1 * SIZE(AO), %xmm9

  1744. 	movhpd	 1 * SIZE(AO), %xmm9

  1745. 

  1746. 	pshufd	$0x4e, %xmm1, %xmm0

  1747. 

  1748. 	xorpd	 %xmm15, %xmm0

  1749. 

  1750. 	mulpd	 %xmm8, %xmm1

  1751. 	mulpd	 %xmm9, %xmm0

  1752. 

  1753. 	addpd	 %xmm0, %xmm1

  1754. #endif

  1755. 

  1756. #ifdef RN

  1757. 	movlpd	 0 * SIZE(B), %xmm8

  1758. 	movhpd	 0 * SIZE(B), %xmm8

  1759. 	movlpd	 1 * SIZE(B), %xmm9

  1760. 	movhpd	 1 * SIZE(B), %xmm9

  1761. 

  1762. 	pshufd	$0x4e, %xmm1, %xmm0

  1763. 

  1764. 	xorpd	 %xmm15, %xmm0

  1765. 

  1766. 	mulpd	 %xmm8, %xmm1

  1767. 	mulpd	 %xmm9, %xmm0

  1768. 

  1769. 	addpd	 %xmm0, %xmm1

  1770. #endif

  1771. 

  1772. #ifdef RT

  1773. 	movlpd	 0 * SIZE(B), %xmm8

  1774. 	movhpd	 0 * SIZE(B), %xmm8

  1775. 	movlpd	 1 * SIZE(B), %xmm9

  1776. 	movhpd	 1 * SIZE(B), %xmm9

  1777. 

  1778. 	pshufd	$0x4e, %xmm1, %xmm0

  1779. 

  1780. 	xorpd	 %xmm15, %xmm0

  1781. 

  1782. 	mulpd	 %xmm8, %xmm1

  1783. 	mulpd	 %xmm9, %xmm0

  1784. 

  1785. 	addpd	 %xmm0, %xmm1

  1786. #endif

  1787. 

  1788. #ifdef LN

  1789. 	subq	$2 * SIZE, CO1

  1790. #endif

  1791. 

  1792. 	movsd	%xmm1,   0 * SIZE(CO1)

  1793. 	movhpd	%xmm1,   1 * SIZE(CO1)

  1794. 

  1795. #if defined(LN) || defined(LT)

  1796. 	movapd	%xmm1,   0 * SIZE(B)

  1797. 

  1798. 	movlpd	%xmm1,   0 * SIZE(BO)

  1799. 	movlpd	%xmm1,   1 * SIZE(BO)

  1800. 	movhpd	%xmm1,   2 * SIZE(BO)

  1801. 	movhpd	%xmm1,   3 * SIZE(BO)

  1802. #else

  1803. 	movapd	%xmm1,   0 * SIZE(AO)

  1804. #endif

  1805. 

  1806. #ifndef LN

  1807. 	addq	$2 * SIZE, CO1

  1808. #endif

  1809. 

  1810. #if defined(LT) || defined(RN)

  1811. 	movq	K,  %rax

  1812. 	subq	KK, %rax

  1813. 	salq	$ZBASE_SHIFT, %rax

  1814. 	leaq	(AO, %rax, 1), AO

  1815. #ifdef LT

  1816. 	addq	$2 * SIZE, B

  1817. #endif

  1818. #endif

  1819. 

  1820. #ifdef LN

  1821. 	subq	$1, KK

  1822. 	movq	BORIG, B

  1823. #endif

  1824. 

  1825. #ifdef LT

  1826. 	addq	$1, KK

  1827. #endif

  1828. 

  1829. #ifdef RT

  1830. 	movq	K, %rax

  1831. 	movq	BORIG, B

  1832. 	salq	$0 + ZBASE_SHIFT, %rax

  1833. 	addq	%rax, AORIG

  1834. #endif

  1835. 	ALIGN_4

  1836. 

  1837. .L130:

  1838. 	movq	M,  I

  1839. 	sarq	$1, I		# i = (m >> 2)

  1840. 	jle	.L199

  1841. 	ALIGN_4

  1842. 

  1843. .L110:

  1844. #ifdef LN

  1845.        movq	K, %rax

  1846.        salq	$1 + ZBASE_SHIFT, %rax

  1847.        subq	%rax, AORIG

  1848. #endif

  1849. 

  1850. #if defined(LN) || defined(RT)

  1851. 	movq	KK, %rax

  1852. 	movq	AORIG, AO

  1853. 	salq	$ZBASE_SHIFT, %rax

  1854. 	leaq	(AO, %rax, 2), AO

  1855. #endif

  1856. 

  1857. 	leaq	BUFFER, BO

  1858. 

  1859. #if defined(LN) || defined(RT)

  1860. 	movq	KK, %rax

  1861. 	salq	$0 + ZBASE_SHIFT, %rax

  1862. 	leaq	(BO, %rax, 2), BO

  1863. #endif

  1864. 

  1865. 	pxor	%xmm0, %xmm0

  1866. 	pxor	%xmm1, %xmm1

  1867. 	pxor	%xmm4, %xmm4

  1868. 	pxor	%xmm5, %xmm5

  1869. #ifdef LN

  1870. 	PREFETCHW     -4 * SIZE(CO1)

  1871. #else

  1872. 	PREFETCHW      4 * SIZE(CO1)

  1873. #endif

  1874. 

  1875. 

  1876. #if defined(LT) || defined(RN)

  1877. 	movq	KK, %rax

  1878. #else

  1879. 	movq	K, %rax

  1880. 	subq	KK, %rax

  1881. #endif

  1882. 	sarq	$2, %rax

  1883. 	je	.L112

  1884. 

  1885. .L111:

  1886. 	movapd	 0 * SIZE(AO), %xmm8

  1887. 	movapd	 0 * SIZE(BO), %xmm9

  1888. 	mulpd	%xmm8, %xmm9

  1889. 	addpd	%xmm9, %xmm0

  1890. 	mulpd	 2 * SIZE(BO), %xmm8

  1891. 	addpd	%xmm8, %xmm1

  1892. 

  1893. 	movapd	 2 * SIZE(AO), %xmm8

  1894. 	movapd	 0 * SIZE(BO), %xmm9

  1895. 	mulpd	%xmm8, %xmm9

  1896. 	addpd	%xmm9, %xmm4

  1897. 	mulpd	 2 * SIZE(BO), %xmm8

  1898. 	addpd	%xmm8, %xmm5

  1899. 

  1900. 	movapd	 4 * SIZE(AO), %xmm8

  1901. 	movapd	 4 * SIZE(BO), %xmm9

  1902. 	mulpd	%xmm8, %xmm9

  1903. 	addpd	%xmm9, %xmm0

  1904. 	mulpd	 6 * SIZE(BO), %xmm8

  1905. 	addpd	%xmm8, %xmm1

  1906. 

  1907. 	movapd	 6 * SIZE(AO), %xmm8

  1908. 	movapd	 4 * SIZE(BO), %xmm9

  1909. 	mulpd	%xmm8, %xmm9

  1910. 	addpd	%xmm9, %xmm4

  1911. 	mulpd	 6 * SIZE(BO), %xmm8

  1912. 	addpd	%xmm8, %xmm5

  1913. 

  1914. 	movapd	 8 * SIZE(AO), %xmm8

  1915. 	movapd	 8 * SIZE(BO), %xmm9

  1916. 	mulpd	%xmm8, %xmm9

  1917. 	addpd	%xmm9, %xmm0

  1918. 	mulpd	10 * SIZE(BO), %xmm8

  1919. 	addpd	%xmm8, %xmm1

  1920. 

  1921. 	movapd	10 * SIZE(AO), %xmm8

  1922. 	movapd	 8 * SIZE(BO), %xmm9

  1923. 	mulpd	%xmm8, %xmm9

  1924. 	addpd	%xmm9, %xmm4

  1925. 	mulpd	10 * SIZE(BO), %xmm8

  1926. 	addpd	%xmm8, %xmm5

  1927. 

  1928. 	movapd	12 * SIZE(AO), %xmm8

  1929. 	movapd	12 * SIZE(BO), %xmm9

  1930. 	mulpd	%xmm8, %xmm9

  1931. 	addpd	%xmm9, %xmm0

  1932. 	mulpd	14 * SIZE(BO), %xmm8

  1933. 	addpd	%xmm8, %xmm1

  1934. 

  1935. 	movapd	14 * SIZE(AO), %xmm8

  1936. 	movapd	12 * SIZE(BO), %xmm9

  1937. 	mulpd	%xmm8, %xmm9

  1938. 	addpd	%xmm9, %xmm4

  1939. 	mulpd	14 * SIZE(BO), %xmm8

  1940. 	addpd	%xmm8, %xmm5

  1941. 

  1942. 	addq   $16 * SIZE, AO

  1943. 	addq   $16 * SIZE, BO

  1944. 	decq   %rax

  1945. 	jne    .L111

  1946. 	ALIGN_4

  1947. 

  1948. .L112:

  1949. #if defined(LT) || defined(RN)

  1950. 	movq	KK, %rax

  1951. #else

  1952. 	movq	K, %rax

  1953. 	subq	KK, %rax

  1954. #endif

  1955. 	movapd	POSINV,  %xmm15

  1956. 	andq	$3, %rax		# if (k & 1)

  1957. 	BRANCH

  1958. 	jle .L114

  1959. 

  1960. .L113:

  1961. 	movapd	 0 * SIZE(AO), %xmm8

  1962. 	movapd	 0 * SIZE(BO), %xmm9

  1963. 	mulpd	%xmm8, %xmm9

  1964. 	addpd	%xmm9, %xmm0

  1965. 	mulpd	 2 * SIZE(BO), %xmm8

  1966. 	addpd	%xmm8, %xmm1

  1967. 

  1968. 	movapd	 2 * SIZE(AO), %xmm8

  1969. 	movapd	 0 * SIZE(BO), %xmm9

  1970. 	mulpd	%xmm8, %xmm9

  1971. 	addpd	%xmm9, %xmm4

  1972. 	mulpd	 2 * SIZE(BO), %xmm8

  1973. 	addpd	%xmm8, %xmm5

  1974. 

  1975. 	addq	$4 * SIZE, AO		# aoffset  += 4

  1976. 	addq	$4 * SIZE, BO		# boffset1 += 8

  1977. 	decq	%rax

  1978. 	jg	.L113

  1979. 	ALIGN_4

  1980. 

  1981. .L114:

  1982. #if defined(LN) || defined(RT)

  1983. 	movq	KK, %rax

  1984. #ifdef LN

  1985. 	subq	$2, %rax

  1986. #else

  1987. 	subq	$1, %rax

  1988. #endif

  1989. 

  1990. 	movq	AORIG, AO

  1991. 	movq	BORIG, B

  1992. 	leaq	BUFFER, BO

  1993. 

  1994. 	salq	$ZBASE_SHIFT, %rax

  1995. 	leaq	(AO, %rax, 2), AO

  1996. 	leaq	(B,  %rax, 1), B

  1997. 	leaq	(BO, %rax, 2), BO

  1998. #endif

  1999. 

  2000. 	SHUFPD_1 %xmm1, %xmm1

  2001. 	SHUFPD_1 %xmm5, %xmm5

  2002. 

  2003. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  2004.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  2005. 	xorpd	%xmm15, %xmm1

  2006. 	xorpd	%xmm15, %xmm5

  2007. #else

  2008. 	xorpd	%xmm15, %xmm0

  2009. 	xorpd	%xmm15, %xmm4

  2010. #endif

  2011. 

  2012. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  2013.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  2014. 	subpd	%xmm1, %xmm0

  2015. 	subpd	%xmm5, %xmm4

  2016. #else

  2017. 	addpd	%xmm1, %xmm0

  2018. 	addpd	%xmm5, %xmm4

  2019. #endif

  2020. 

  2021. #if defined(LN) || defined(LT)

  2022. 	movapd	 0 * SIZE(B), %xmm1

  2023. 	movapd	 2 * SIZE(B), %xmm5

  2024. 

  2025. 	subpd	%xmm0,  %xmm1

  2026. 	subpd	%xmm4,  %xmm5

  2027. #else

  2028. 	movapd	 0 * SIZE(AO), %xmm1

  2029. 	movapd	 2 * SIZE(AO), %xmm5

  2030. 

  2031. 	subpd	%xmm0,  %xmm1

  2032. 	subpd	%xmm4,  %xmm5

  2033. #endif

  2034. 

  2035. #ifndef CONJ

  2036. 	SHUFPD_1 %xmm15, %xmm15

  2037. #endif

  2038. 

  2039. #ifdef LN

  2040. 	movlpd	 6 * SIZE(AO), %xmm8

  2041. 	movhpd	 6 * SIZE(AO), %xmm8

  2042. 	movlpd	 7 * SIZE(AO), %xmm9

  2043. 	movhpd	 7 * SIZE(AO), %xmm9

  2044. 	movlpd	 4 * SIZE(AO), %xmm10

  2045. 	movhpd	 4 * SIZE(AO), %xmm10

  2046. 	movlpd	 5 * SIZE(AO), %xmm11

  2047. 	movhpd	 5 * SIZE(AO), %xmm11

  2048. 	movlpd	 0 * SIZE(AO), %xmm12

  2049. 	movhpd	 0 * SIZE(AO), %xmm12

  2050. 	movlpd	 1 * SIZE(AO), %xmm13

  2051. 	movhpd	 1 * SIZE(AO), %xmm13

  2052. 

  2053. 	pshufd	$0x4e, %xmm5, %xmm4

  2054. 

  2055. 	xorpd	 %xmm15, %xmm4

  2056. 

  2057. 	mulpd	 %xmm8, %xmm5

  2058. 	mulpd	 %xmm9, %xmm4

  2059. 

  2060. 	addpd	 %xmm4, %xmm5

  2061. 

  2062. 	movapd	 %xmm5, %xmm0

  2063. 	pshufd	 $0x4e, %xmm5, %xmm4

  2064. 

  2065. 	xorpd	 %xmm15, %xmm4

  2066. 

  2067. 	mulpd	 %xmm10, %xmm0

  2068. 	mulpd	 %xmm11, %xmm4

  2069. 

  2070. 	subpd	 %xmm0, %xmm1

  2071. 	subpd	 %xmm4, %xmm1

  2072. 

  2073. 	pshufd	$0x4e, %xmm1, %xmm0

  2074. 

  2075. 	xorpd	 %xmm15, %xmm0

  2076. 

  2077. 	mulpd	 %xmm12, %xmm1

  2078. 	mulpd	 %xmm13, %xmm0

  2079. 

  2080. 	addpd	 %xmm0, %xmm1

  2081. #endif

  2082. 

  2083. #ifdef LT

  2084. 	movlpd	 0 * SIZE(AO), %xmm8

  2085. 	movhpd	 0 * SIZE(AO), %xmm8

  2086. 	movlpd	 1 * SIZE(AO), %xmm9

  2087. 	movhpd	 1 * SIZE(AO), %xmm9

  2088. 	movlpd	 2 * SIZE(AO), %xmm10

  2089. 	movhpd	 2 * SIZE(AO), %xmm10

  2090. 	movlpd	 3 * SIZE(AO), %xmm11

  2091. 	movhpd	 3 * SIZE(AO), %xmm11

  2092. 	movlpd	 6 * SIZE(AO), %xmm12

  2093. 	movhpd	 6 * SIZE(AO), %xmm12

  2094. 	movlpd	 7 * SIZE(AO), %xmm13

  2095. 	movhpd	 7 * SIZE(AO), %xmm13

  2096. 

  2097. 	pshufd	$0x4e, %xmm1, %xmm0

  2098. 

  2099. 	xorpd	 %xmm15, %xmm0

  2100. 

  2101. 	mulpd	 %xmm8, %xmm1

  2102. 	mulpd	 %xmm9, %xmm0

  2103. 

  2104. 	addpd	 %xmm0, %xmm1

  2105. 

  2106. 	movapd	 %xmm1, %xmm0

  2107. 	pshufd	 $0x4e, %xmm1, %xmm4

  2108. 

  2109. 	xorpd	 %xmm15, %xmm4

  2110. 

  2111. 	mulpd	 %xmm10, %xmm0

  2112. 	mulpd	 %xmm11, %xmm4

  2113. 

  2114. 	subpd	 %xmm0, %xmm5

  2115. 	subpd	 %xmm4, %xmm5

  2116. 

  2117. 	pshufd	$0x4e, %xmm5, %xmm4

  2118. 

  2119. 	xorpd	 %xmm15, %xmm4

  2120. 

  2121. 	mulpd	 %xmm12, %xmm5

  2122. 	mulpd	 %xmm13, %xmm4

  2123. 

  2124. 	addpd	 %xmm4, %xmm5

  2125. #endif

  2126. 

  2127. #ifdef RN

  2128. 	movlpd	 0 * SIZE(B), %xmm8

  2129. 	movhpd	 0 * SIZE(B), %xmm8

  2130. 	movlpd	 1 * SIZE(B), %xmm9

  2131. 	movhpd	 1 * SIZE(B), %xmm9

  2132. 

  2133. 	pshufd	$0x4e, %xmm1, %xmm0

  2134. 	pshufd	$0x4e, %xmm5, %xmm4

  2135. 

  2136. 	xorpd	 %xmm15, %xmm0

  2137. 	xorpd	 %xmm15, %xmm4

  2138. 

  2139. 	mulpd	 %xmm8, %xmm1

  2140. 	mulpd	 %xmm9, %xmm0

  2141. 	mulpd	 %xmm8, %xmm5

  2142. 	mulpd	 %xmm9, %xmm4

  2143. 

  2144. 	addpd	 %xmm0, %xmm1

  2145. 	addpd	 %xmm4, %xmm5

  2146. #endif

  2147. 

  2148. #ifdef RT

  2149. 	movlpd	 0 * SIZE(B), %xmm8

  2150. 	movhpd	 0 * SIZE(B), %xmm8

  2151. 	movlpd	 1 * SIZE(B), %xmm9

  2152. 	movhpd	 1 * SIZE(B), %xmm9

  2153. 

  2154. 	pshufd	$0x4e, %xmm1, %xmm0

  2155. 	pshufd	$0x4e, %xmm5, %xmm4

  2156. 

  2157. 	xorpd	 %xmm15, %xmm0

  2158. 	xorpd	 %xmm15, %xmm4

  2159. 

  2160. 	mulpd	 %xmm8, %xmm1

  2161. 	mulpd	 %xmm9, %xmm0

  2162. 	mulpd	 %xmm8, %xmm5

  2163. 	mulpd	 %xmm9, %xmm4

  2164. 

  2165. 	addpd	 %xmm0, %xmm1

  2166. 	addpd	 %xmm4, %xmm5

  2167. #endif

  2168. 

  2169. #ifdef LN

  2170. 	subq	$4 * SIZE, CO1

  2171. #endif

  2172. 

  2173. 	movsd	%xmm1,   0 * SIZE(CO1)

  2174. 	movhpd	%xmm1,   1 * SIZE(CO1)

  2175. 	movsd	%xmm5,   2 * SIZE(CO1)

  2176. 	movhpd	%xmm5,   3 * SIZE(CO1)

  2177. 

  2178. #if defined(LN) || defined(LT)

  2179. 	movapd	%xmm1,   0 * SIZE(B)

  2180. 	movapd	%xmm5,   2 * SIZE(B)

  2181. 

  2182. 	movlpd	%xmm1,   0 * SIZE(BO)

  2183. 	movlpd	%xmm1,   1 * SIZE(BO)

  2184. 	movhpd	%xmm1,   2 * SIZE(BO)

  2185. 	movhpd	%xmm1,   3 * SIZE(BO)

  2186. 	movlpd	%xmm5,   4 * SIZE(BO)

  2187. 	movlpd	%xmm5,   5 * SIZE(BO)

  2188. 	movhpd	%xmm5,   6 * SIZE(BO)

  2189. 	movhpd	%xmm5,   7 * SIZE(BO)

  2190. #else

  2191. 	movapd	%xmm1,   0 * SIZE(AO)

  2192. 	movapd	%xmm5,   2 * SIZE(AO)

  2193. #endif

  2194. 

  2195. #ifndef LN

  2196. 	addq	$4 * SIZE, CO1

  2197. #endif

  2198. 

  2199. #if defined(LT) || defined(RN)

  2200. 	movq	K,  %rax

  2201. 	subq	KK, %rax

  2202. 	salq	$ZBASE_SHIFT, %rax

  2203. 	leaq	(AO, %rax, 2), AO

  2204. #ifdef LT

  2205. 	addq	$4 * SIZE, B

  2206. #endif

  2207. #endif

  2208. 

  2209. #ifdef LN

  2210. 	subq	$2, KK

  2211. 	movq	BORIG, B

  2212. #endif

  2213. 

  2214. #ifdef LT

  2215. 	addq	$2, KK

  2216. #endif

  2217. 

  2218. #ifdef RT

  2219. 	movq	K, %rax

  2220. 	movq	BORIG, B

  2221. 	salq	$1 + ZBASE_SHIFT, %rax

  2222. 	addq	%rax, AORIG

  2223. #endif

  2224. 

  2225. 	decq	I			# i --

  2226. 	jg	.L110

  2227. 	ALIGN_4

  2228. 

  2229. .L199:

  2230. #ifdef LN

  2231.        leaq	(, K, SIZE), %rax

  2232.        leaq	(B, %rax, 2), B

  2233. #endif

  2234. 

  2235. #if defined(LT) || defined(RN)

  2236. 	movq	K,  %rax

  2237. 	subq	KK, %rax

  2238. 	leaq	(,%rax, SIZE), %rax

  2239. 	leaq	(B,  %rax, 1 * COMPSIZE), B

  2240. #endif

  2241. 

  2242. #ifdef RN

  2243. 	addq	$2, KK

  2244. #endif

  2245. 

  2246. #ifdef RT

  2247. 	subq	$2, KK

  2248. #endif

  2249. 	ALIGN_4

  2250. 

  2251. .L999:

  2252. 	movq	%rbx, %rsp

  2253. 	movq	  0(%rsp), %rbx

  2254. 	movq	  8(%rsp), %rbp

  2255. 	movq	 16(%rsp), %r12

  2256. 	movq	 24(%rsp), %r13

  2257. 	movq	 32(%rsp), %r14

  2258. 	movq	 40(%rsp), %r15

  2259. 

  2260. #ifdef WINDOWS_ABI

  2261. 	movq	 48(%rsp), %rdi

  2262. 	movq	 56(%rsp), %rsi

  2263. 	movups	 64(%rsp), %xmm6

  2264. 	movups	 80(%rsp), %xmm7

  2265. 	movups	 96(%rsp), %xmm8

  2266. 	movups	112(%rsp), %xmm9

  2267. 	movups	128(%rsp), %xmm10

  2268. 	movups	144(%rsp), %xmm11

  2269. 	movups	160(%rsp), %xmm12

  2270. 	movups	176(%rsp), %xmm13

  2271. 	movups	192(%rsp), %xmm14

  2272. 	movups	208(%rsp), %xmm15

  2273. #endif

  2274. 

  2275. 	addq	$STACKSIZE, %rsp

  2276. 	ret

  2277. 

  2278. 	EPILOGUE