OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cd8ac192a9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cd8ac192a9'
${ noResults }
OpenBLAS/kernel/x86_64/zgemm_kernel_2x2_sse3.S

1540 lines
36 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define M	%rdi

  43. #define N	%rsi

  44. #define K	%rdx

  45. 

  46. #define A	%rcx

  47. #define B	%r8

  48. #define C	%r9

  49. #define LDC	%r10

  50. 

  51. #define I	%r11

  52. #define J	%r12

  53. #define AO	%r13

  54. #define BO	%r14

  55. #define	CO1	%r15

  56. #define CO2	%rbx

  57. #define BB	%rbp

  58. 

  59. #ifndef WINDOWS_ABI

  60. 

  61. #define STACKSIZE 128

  62. 

  63. #define OLD_LDC		 8 + STACKSIZE(%rsp)

  64. #define OLD_OFFSET	16 + STACKSIZE(%rsp)

  65. 

  66. #define ALPHA_R	 48(%rsp)

  67. #define ALPHA_I	 56(%rsp)

  68. #define OFFSET	 64(%rsp)

  69. #define KKK	 72(%rsp)

  70. #define KK	 80(%rsp)

  71. 

  72. #else

  73. 

  74. #define STACKSIZE 512

  75. 

  76. #define OLD_ALPHA_I	40 + STACKSIZE(%rsp)

  77. #define OLD_A		48 + STACKSIZE(%rsp)

  78. #define OLD_B		56 + STACKSIZE(%rsp)

  79. #define OLD_C		64 + STACKSIZE(%rsp)

  80. #define OLD_LDC		72 + STACKSIZE(%rsp)

  81. #define OLD_OFFSET	80 + STACKSIZE(%rsp)

  82. 

  83. #define ALPHA_R	 224(%rsp)

  84. #define ALPHA_I	 232(%rsp)

  85. #define OFFSET	 240(%rsp)

  86. #define KKK	 248(%rsp)

  87. #define KK	 256(%rsp)

  88. #endif

  89. 

  90. #define PREFETCH     prefetcht1

  91. #define PREFETCHSIZE (16 * 12 + 3)

  92. #define PREFETCH_R    (4 *  4 + 0)

  93. 

  94. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  95. #define ADD1	  addpd

  96. #define ADD2	  addpd

  97. #elif defined(NR) || defined(NC) || defined(TR) || defined(TC)

  98. #define ADD1	  addpd

  99. #define ADD2	  subpd

  100. #elif defined(RN) || defined(RT) || defined(CN) || defined(CT)

  101. #define ADD1	  subpd

  102. #define ADD2	  addpd

  103. #else

  104. #define ADD1	  subpd

  105. #define ADD2	  subpd

  106. #endif

  107. 

  108. #define ADDSUB	subpd

  109. 

  110. #define KERNEL1(address) \

  111. 	mulpd	%xmm8, %xmm9;\

  112. 	PREFETCH  (PREFETCHSIZE +  0) * SIZE + (address) * 2 * SIZE(AO);\

  113. 	ADD1	%xmm9, %xmm0;\

  114. 	movddup	 1 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  115. 	mulpd	%xmm8, %xmm9;\

  116. 	ADD2	%xmm9, %xmm1;\

  117. 	movddup	 2 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  118. 	mulpd	%xmm8, %xmm9;\

  119. 	ADD1	%xmm9, %xmm2;\

  120. 	movddup	 3 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  121. 	mulpd	%xmm8, %xmm9;\

  122. 	movapd	 2 * SIZE + (address) * 2 * SIZE(AO), %xmm8;\

  123. 	ADD2	%xmm9, %xmm3;\

  124. 	movddup	 0 * SIZE + (address) * 2 * SIZE(BO), %xmm9

  125. 

  126. #define KERNEL2(address) \

  127. 	mulpd	%xmm8, %xmm9;\

  128. 	ADD1	%xmm9, %xmm4;\

  129. 	movddup	 1 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  130. 	mulpd	%xmm8, %xmm9;\

  131. 	ADD2	%xmm9, %xmm5;\

  132. 	movddup	 2 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  133. 	mulpd	%xmm8, %xmm9;\

  134. 	ADD1	%xmm9, %xmm6;\

  135. 	movddup	 3 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  136. 	mulpd	%xmm8, %xmm9;\

  137. 	movapd	 4 * SIZE + (address) * 2 * SIZE(AO), %xmm8;\

  138. 	ADD2	%xmm9, %xmm7;\

  139. 	movddup	 4 * SIZE + (address) * 2 * SIZE(BO), %xmm9

  140. 

  141. #define KERNEL3(address) \

  142. 	mulpd	%xmm8, %xmm9;\

  143. 	ADD1	%xmm9, %xmm0;\

  144. 	movddup	 5 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  145. 	mulpd	%xmm8, %xmm9;\

  146. 	ADD2	%xmm9, %xmm1;\

  147. 	movddup	 6 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  148. 	mulpd	%xmm8, %xmm9;\

  149. 	ADD1	%xmm9, %xmm2;\

  150. 	movddup	 7 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  151. 	mulpd	%xmm8, %xmm9;\

  152. 	movapd	 6 * SIZE + (address) * 2 * SIZE(AO), %xmm8;\

  153. 	ADD2	%xmm9, %xmm3;\

  154. 	movddup	 4 * SIZE + (address) * 2 * SIZE(BO), %xmm9

  155. 

  156. #define KERNEL4(address) \

  157. 	mulpd	%xmm8, %xmm9;\

  158. 	ADD1	%xmm9, %xmm4;\

  159. 	movddup	 5 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  160. 	mulpd	%xmm8, %xmm9;\

  161. 	ADD2	%xmm9, %xmm5;\

  162. 	movddup	 6 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  163. 	mulpd	%xmm8, %xmm9;\

  164. 	ADD1	%xmm9, %xmm6;\

  165. 	movddup	 7 * SIZE + (address) * 2 * SIZE(BO), %xmm9;\

  166. 	mulpd	%xmm8, %xmm9;\

  167. 	movapd	32 * SIZE + (address) * 2 * SIZE(AO), %xmm8;\

  168. 	ADD2	%xmm9, %xmm7;\

  169. 	movddup	32 * SIZE + (address) * 2 * SIZE(BO), %xmm9

  170. 

  171. #define KERNEL5(address) \

  172. 	mulpd	%xmm10, %xmm11;\

  173. 	ADD1	%xmm11, %xmm0;\

  174. 	movddup	 9 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  175. 	mulpd	%xmm10, %xmm11;\

  176. 	ADD2	%xmm11, %xmm1;\

  177. 	movddup	10 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  178. 	mulpd	%xmm10, %xmm11;\

  179. 	ADD1	%xmm11, %xmm2;\

  180. 	movddup	11 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  181. 	mulpd	%xmm10, %xmm11;\

  182. 	movapd	10 * SIZE + (address) * 2 * SIZE(AO), %xmm10;\

  183. 	ADD2	%xmm11, %xmm3;\

  184. 	movddup	 8 * SIZE + (address) * 2 * SIZE(BO), %xmm11

  185. 

  186. #define KERNEL6(address) \

  187. 	mulpd	%xmm10, %xmm11;\

  188. 	ADD1	%xmm11, %xmm4;\

  189. 	movddup	 9 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  190. 	mulpd	%xmm10, %xmm11;\

  191. 	ADD2	%xmm11, %xmm5;\

  192. 	movddup	10 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  193. 	mulpd	%xmm10, %xmm11;\

  194. 	ADD1	%xmm11, %xmm6;\

  195. 	movddup	11 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  196. 	mulpd	%xmm10, %xmm11;\

  197. 	movapd	12 * SIZE + (address) * 2 * SIZE(AO), %xmm10;\

  198. 	ADD2	%xmm11, %xmm7;\

  199. 	movddup	12 * SIZE + (address) * 2 * SIZE(BO), %xmm11

  200. 

  201. #define KERNEL7(address) \

  202. 	mulpd	%xmm10, %xmm11;\

  203. 	ADD1	%xmm11, %xmm0;\

  204. 	movddup	13 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  205. 	mulpd	%xmm10, %xmm11;\

  206. 	ADD2	%xmm11, %xmm1;\

  207. 	movddup	14 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  208. 	mulpd	%xmm10, %xmm11;\

  209. 	ADD1	%xmm11, %xmm2;\

  210. 	movddup	15 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  211. 	mulpd	%xmm10, %xmm11;\

  212. 	movapd	14 * SIZE + (address) * 2 * SIZE(AO), %xmm10;\

  213. 	ADD2	%xmm11, %xmm3;\

  214. 	movddup	12 * SIZE + (address) * 2 * SIZE(BO), %xmm11

  215. 

  216. #define KERNEL8(address) \

  217. 	mulpd	%xmm10, %xmm11;\

  218. 	ADD1	%xmm11, %xmm4;\

  219. 	movddup	13 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  220. 	mulpd	%xmm10, %xmm11;\

  221. 	ADD2	%xmm11, %xmm5;\

  222. 	movddup	14 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  223. 	mulpd	%xmm10, %xmm11;\

  224. 	ADD1	%xmm11, %xmm6;\

  225. 	movddup	15 * SIZE + (address) * 2 * SIZE(BO), %xmm11;\

  226. 	mulpd	%xmm10, %xmm11;\

  227. 	movapd	40 * SIZE + (address) * 2 * SIZE(AO), %xmm10;\

  228. 	ADD2	%xmm11, %xmm7;\

  229. 	movddup	40 * SIZE + (address) * 2 * SIZE(BO), %xmm11

  230. 

  231. #define KERNEL9(address) \

  232. 	mulpd	%xmm12, %xmm13;\

  233. 	PREFETCH  (PREFETCHSIZE + 16) * SIZE + (address) * 2 * SIZE(AO);\

  234. 	ADD1	%xmm13, %xmm0;\

  235. 	movddup	17 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  236. 	mulpd	%xmm12, %xmm13;\

  237. 	ADD2	%xmm13, %xmm1;\

  238. 	movddup	18 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  239. 	mulpd	%xmm12, %xmm13;\

  240. 	ADD1	%xmm13, %xmm2;\

  241. 	movddup	19 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  242. 	mulpd	%xmm12, %xmm13;\

  243. 	movapd	18 * SIZE + (address) * 2 * SIZE(AO), %xmm12;\

  244. 	ADD2	%xmm13, %xmm3;\

  245. 	movddup	16 * SIZE + (address) * 2 * SIZE(BO), %xmm13

  246. 

  247. #define KERNEL10(address) \

  248. 	mulpd	%xmm12, %xmm13;\

  249. 	ADD1	%xmm13, %xmm4;\

  250. 	movddup	17 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  251. 	mulpd	%xmm12, %xmm13;\

  252. 	ADD2	%xmm13, %xmm5;\

  253. 	movddup	18 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  254. 	mulpd	%xmm12, %xmm13;\

  255. 	ADD1	%xmm13, %xmm6;\

  256. 	movddup	19 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  257. 	mulpd	%xmm12, %xmm13;\

  258. 	movapd	20 * SIZE + (address) * 2 * SIZE(AO), %xmm12;\

  259. 	ADD2	%xmm13, %xmm7;\

  260. 	movddup	20 * SIZE + (address) * 2 * SIZE(BO), %xmm13

  261. 

  262. #define KERNEL11(address) \

  263. 	mulpd	%xmm12, %xmm13;\

  264. 	ADD1	%xmm13, %xmm0;\

  265. 	movddup	21 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  266. 	mulpd	%xmm12, %xmm13;\

  267. 	ADD2	%xmm13, %xmm1;\

  268. 	movddup	22 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  269. 	mulpd	%xmm12, %xmm13;\

  270. 	ADD1	%xmm13, %xmm2;\

  271. 	movddup	23 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  272. 	mulpd	%xmm12, %xmm13;\

  273. 	movapd	22 * SIZE + (address) * 2 * SIZE(AO), %xmm12;\

  274. 	ADD2	%xmm13, %xmm3;\

  275. 	movddup	20 * SIZE + (address) * 2 * SIZE(BO), %xmm13

  276. 

  277. #define KERNEL12(address) \

  278. 	mulpd	%xmm12, %xmm13;\

  279. 	ADD1	%xmm13, %xmm4;\

  280. 	movddup 21 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  281. 	mulpd	%xmm12, %xmm13;\

  282. 	ADD2	%xmm13, %xmm5;\

  283. 	movddup	22 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  284. 	mulpd	%xmm12, %xmm13;\

  285. 	ADD1	%xmm13, %xmm6;\

  286. 	movddup	23 * SIZE + (address) * 2 * SIZE(BO), %xmm13;\

  287. 	mulpd	%xmm12, %xmm13;\

  288. 	movapd	48 * SIZE + (address) * 2 * SIZE(AO), %xmm12;\

  289. 	ADD2	%xmm13, %xmm7;\

  290. 	movddup	48 * SIZE + (address) * 2 * SIZE(BO), %xmm13

  291. 

  292. #define KERNEL13(address) \

  293. 	mulpd	%xmm14, %xmm15;\

  294. 	ADD1	%xmm15, %xmm0;\

  295. 	movddup	25 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  296. 	mulpd	%xmm14, %xmm15;\

  297. 	ADD2	%xmm15, %xmm1;\

  298. 	movddup	26 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  299. 	mulpd	%xmm14, %xmm15;\

  300. 	ADD1	%xmm15, %xmm2;\

  301. 	movddup	27 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  302. 	mulpd	%xmm14, %xmm15;\

  303. 	movapd	26 * SIZE + (address) * 2 * SIZE(AO), %xmm14;\

  304. 	ADD2	%xmm15, %xmm3;\

  305. 	movddup	24 * SIZE + (address) * 2 * SIZE(BO), %xmm15

  306. 

  307. #define KERNEL14(address) \

  308. 	mulpd	%xmm14, %xmm15;\

  309. 	ADD1	%xmm15, %xmm4;\

  310. 	movddup	25 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  311. 	mulpd	%xmm14, %xmm15;\

  312. 	ADD2	%xmm15, %xmm5;\

  313. 	movddup	26 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  314. 	mulpd	%xmm14, %xmm15;\

  315. 	ADD1	%xmm15, %xmm6;\

  316. 	movddup	27 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  317. 	mulpd	%xmm14, %xmm15;\

  318. 	movapd	28 * SIZE + (address) * 2 * SIZE(AO), %xmm14;\

  319. 	ADD2	%xmm15, %xmm7;\

  320. 	movddup	28 * SIZE + (address) * 2 * SIZE(BO), %xmm15

  321. 

  322. #define KERNEL15(address) \

  323. 	mulpd	%xmm14, %xmm15;\

  324. 	ADD1	%xmm15, %xmm0;\

  325. 	movddup	29 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  326. 	mulpd	%xmm14, %xmm15;\

  327. 	ADD2	%xmm15, %xmm1;\

  328. 	movddup	30 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  329. 	mulpd	%xmm14, %xmm15;\

  330. 	ADD1	%xmm15, %xmm2;\

  331. 	movddup	31 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  332. 	mulpd	%xmm14, %xmm15;\

  333. 	movapd	30 * SIZE + (address) * 2 * SIZE(AO), %xmm14;\

  334. 	ADD2	%xmm15, %xmm3;\

  335. 	movddup	28 * SIZE + (address) * 2 * SIZE(BO), %xmm15

  336. 

  337. #define KERNEL16(address) \

  338. 	mulpd	%xmm14, %xmm15;\

  339. 	ADD1	%xmm15, %xmm4;\

  340. 	movddup	29 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  341. 	mulpd	%xmm14, %xmm15;\

  342. 	ADD2	%xmm15, %xmm5;\

  343. 	movddup	30 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  344. 	mulpd	%xmm14, %xmm15;\

  345. 	ADD1	%xmm15, %xmm6;\

  346. 	movddup	31 * SIZE + (address) * 2 * SIZE(BO), %xmm15;\

  347. 	mulpd	%xmm14, %xmm15;\

  348. 	movapd	56 * SIZE + (address) * 2 * SIZE(AO), %xmm14;\

  349. 	ADD2	%xmm15, %xmm7;\

  350. 	movddup	56 * SIZE + (address) * 2 * SIZE(BO), %xmm15

  351. 

  352. 	PROLOGUE

  353. 	PROFCODE

  354. 

  355. 	subq	$STACKSIZE, %rsp

  356. 

  357. 	movq	%rbx,  0(%rsp)

  358. 	movq	%rbp,  8(%rsp)

  359. 	movq	%r12, 16(%rsp)

  360. 	movq	%r13, 24(%rsp)

  361. 	movq	%r14, 32(%rsp)

  362. 	movq	%r15, 40(%rsp)

  363. 

  364. #ifdef WINDOWS_ABI

  365. 	movq	%rdi,    48(%rsp)

  366. 	movq	%rsi,    56(%rsp)

  367. 	movups	%xmm6,   64(%rsp)

  368. 	movups	%xmm7,   80(%rsp)

  369. 	movups	%xmm8,   96(%rsp)

  370. 	movups	%xmm9,  112(%rsp)

  371. 	movups	%xmm10, 128(%rsp)

  372. 	movups	%xmm11, 144(%rsp)

  373. 	movups	%xmm12, 160(%rsp)

  374. 	movups	%xmm13, 176(%rsp)

  375. 	movups	%xmm14, 192(%rsp)

  376. 	movups	%xmm15, 208(%rsp)

  377. 

  378. 	movq	ARG1,      M

  379. 	movq	ARG2,      N

  380. 	movq	ARG3,      K

  381. 	movq	OLD_A,     A

  382. 	movq	OLD_B,     B

  383. 	movq	OLD_C,     C

  384. 	movq	OLD_LDC,   LDC

  385. #ifdef TRMMKERNEL

  386. 	movsd	OLD_OFFSET,  %xmm4

  387. #endif

  388. 	movaps	%xmm3,       %xmm0

  389. 	movsd	OLD_ALPHA_I, %xmm1

  390. #else

  391. 	movq	OLD_LDC,   LDC

  392. #ifdef TRMMKERNEL

  393. 	movsd	OLD_OFFSET,  %xmm4

  394. #endif

  395. 

  396. #endif

  397. 

  398. 	movsd	 %xmm0, ALPHA_R

  399. 	movsd	 %xmm1, ALPHA_I

  400. 

  401. #ifdef TRMMKERNEL

  402. 	movsd	%xmm4, OFFSET

  403. 	movsd	%xmm4, KK

  404. #ifndef LEFT

  405. 	negq	KK

  406. #endif

  407. #endif

  408. 

  409. 	salq	$ZBASE_SHIFT, LDC

  410. 	movq	N,  J

  411. 	sarq	$1, J		# j = (n >> 2)

  412. 	jle	.L100

  413. 	ALIGN_4

  414. 

  415. .L01:

  416. #if defined(TRMMKERNEL) && defined(LEFT)

  417. 	movq	OFFSET, %rax

  418. 	movq	%rax, KK

  419. #endif

  420. 

  421. 	movq	C, CO1			# coffset1 = c

  422. 	leaq	(C, LDC, 1), CO2	# coffset2 = c + ldc

  423. 	movq	A, AO		# aoffset = a

  424. 

  425. 	movq	K, %rax

  426. 	salq	$ZBASE_SHIFT + 1, %rax

  427. 	leaq	(B, %rax), BB

  428. 

  429. 	movq	M,  I

  430. 	sarq	$1, I		# i = (m >> 2)

  431. 	jle	.L30

  432. 	ALIGN_4

  433. 

  434. .L10:

  435. #if !defined(TRMMKERNEL) || \

  436. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  437. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  438. 

  439. 	movq	B, BO

  440. #else

  441. 	movq	KK, %rax

  442. 	leaq	(, %rax, SIZE), %rax

  443. 	leaq	(AO, %rax, 4), AO

  444. 	leaq	(B,  %rax, 4), BO

  445. #endif

  446. 

  447. 	prefetcht0	 0 * SIZE(BB)

  448. 	subq	   $-8 * SIZE, BB

  449. 

  450. 	movapd	 0 * SIZE(AO), %xmm8

  451. 	pxor	%xmm0, %xmm0

  452. 	movddup	 0 * SIZE(BO), %xmm9

  453. 	pxor	%xmm1, %xmm1

  454. 	movapd	 8 * SIZE(AO), %xmm10

  455. 	pxor	%xmm2, %xmm2

  456. 	movddup	 8 * SIZE(BO), %xmm11

  457. 	pxor	%xmm3, %xmm3

  458. 

  459. 	movapd	16 * SIZE(AO), %xmm12

  460. 	pxor	%xmm4, %xmm4

  461. 	movddup 16 * SIZE(BO), %xmm13

  462. 	pxor	%xmm5, %xmm5

  463. 	movapd	24 * SIZE(AO), %xmm14

  464. 	pxor	%xmm6, %xmm6

  465. 	movddup	24 * SIZE(BO), %xmm15

  466. 	pxor	%xmm7, %xmm7

  467. 

  468. 	prefetchnta     3 * SIZE(CO1)

  469. 	prefetchnta     3 * SIZE(CO2)

  470. 

  471. #ifndef TRMMKERNEL

  472. 	movq	K, %rax

  473. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  474. 	movq	K, %rax

  475. 	subq	KK, %rax

  476. 	movq	%rax, KKK

  477. #else

  478. 	movq	KK, %rax

  479. #ifdef LEFT

  480. 	addq	$2, %rax

  481. #else

  482. 	addq	$2, %rax

  483. #endif

  484. 	movq	%rax, KKK

  485. #endif

  486. 

  487. 	andq	$-8, %rax

  488. 	salq	$4, %rax

  489. 	je	.L12

  490. 

  491. .L1X:

  492. 	KERNEL1 (16  *  0)

  493. 	KERNEL2 (16  *  0)

  494. 	KERNEL3 (16  *  0)

  495. 	KERNEL4 (16  *  0)

  496. 	KERNEL5 (16  *  0)

  497. 	KERNEL6 (16  *  0)

  498. 	KERNEL7 (16  *  0)

  499. 	KERNEL8 (16  *  0)

  500. 	KERNEL9 (16  *  0)

  501. 	KERNEL10(16  *  0)

  502. 	KERNEL11(16  *  0)

  503. 	KERNEL12(16  *  0)

  504. 	KERNEL13(16  *  0)

  505. 	KERNEL14(16  *  0)

  506. 	KERNEL15(16  *  0)

  507. 	KERNEL16(16  *  0)

  508. 	cmpq	$128 *  1, %rax

  509. 	NOBRANCH

  510. 	jle	.L11

  511. 	KERNEL1 (16  *  1)

  512. 	KERNEL2 (16  *  1)

  513. 	KERNEL3 (16  *  1)

  514. 	KERNEL4 (16  *  1)

  515. 	KERNEL5 (16  *  1)

  516. 	KERNEL6 (16  *  1)

  517. 	KERNEL7 (16  *  1)

  518. 	KERNEL8 (16  *  1)

  519. 	KERNEL9 (16  *  1)

  520. 	KERNEL10(16  *  1)

  521. 	KERNEL11(16  *  1)

  522. 	KERNEL12(16  *  1)

  523. 	KERNEL13(16  *  1)

  524. 	KERNEL14(16  *  1)

  525. 	KERNEL15(16  *  1)

  526. 	KERNEL16(16  *  1)

  527. 	cmpq	$128 *  2, %rax

  528. 	NOBRANCH

  529. 	jle	.L11

  530. 	KERNEL1 (16  *  2)

  531. 	KERNEL2 (16  *  2)

  532. 	KERNEL3 (16  *  2)

  533. 	KERNEL4 (16  *  2)

  534. 	KERNEL5 (16  *  2)

  535. 	KERNEL6 (16  *  2)

  536. 	KERNEL7 (16  *  2)

  537. 	KERNEL8 (16  *  2)

  538. 	KERNEL9 (16  *  2)

  539. 	KERNEL10(16  *  2)

  540. 	KERNEL11(16  *  2)

  541. 	KERNEL12(16  *  2)

  542. 	KERNEL13(16  *  2)

  543. 	KERNEL14(16  *  2)

  544. 	KERNEL15(16  *  2)

  545. 	KERNEL16(16  *  2)

  546. 	cmpq	$128 *  3, %rax

  547. 	NOBRANCH

  548. 	jle	.L11

  549. 	KERNEL1 (16  *  3)

  550. 	KERNEL2 (16  *  3)

  551. 	KERNEL3 (16  *  3)

  552. 	KERNEL4 (16  *  3)

  553. 	KERNEL5 (16  *  3)

  554. 	KERNEL6 (16  *  3)

  555. 	KERNEL7 (16  *  3)

  556. 	KERNEL8 (16  *  3)

  557. 	KERNEL9 (16  *  3)

  558. 	KERNEL10(16  *  3)

  559. 	KERNEL11(16  *  3)

  560. 	KERNEL12(16  *  3)

  561. 	KERNEL13(16  *  3)

  562. 	KERNEL14(16  *  3)

  563. 	KERNEL15(16  *  3)

  564. 	KERNEL16(16  *  3)

  565. 	cmpq	$128 *  4, %rax

  566. 	NOBRANCH

  567. 	jle	.L11

  568. 	KERNEL1 (16  *  4)

  569. 	KERNEL2 (16  *  4)

  570. 	KERNEL3 (16  *  4)

  571. 	KERNEL4 (16  *  4)

  572. 	KERNEL5 (16  *  4)

  573. 	KERNEL6 (16  *  4)

  574. 	KERNEL7 (16  *  4)

  575. 	KERNEL8 (16  *  4)

  576. 	KERNEL9 (16  *  4)

  577. 	KERNEL10(16  *  4)

  578. 	KERNEL11(16  *  4)

  579. 	KERNEL12(16  *  4)

  580. 	KERNEL13(16  *  4)

  581. 	KERNEL14(16  *  4)

  582. 	KERNEL15(16  *  4)

  583. 	KERNEL16(16  *  4)

  584. 	cmpq	$128 *  5, %rax

  585. 	NOBRANCH

  586. 	jle	.L11

  587. 	KERNEL1 (16  *  5)

  588. 	KERNEL2 (16  *  5)

  589. 	KERNEL3 (16  *  5)

  590. 	KERNEL4 (16  *  5)

  591. 	KERNEL5 (16  *  5)

  592. 	KERNEL6 (16  *  5)

  593. 	KERNEL7 (16  *  5)

  594. 	KERNEL8 (16  *  5)

  595. 	KERNEL9 (16  *  5)

  596. 	KERNEL10(16  *  5)

  597. 	KERNEL11(16  *  5)

  598. 	KERNEL12(16  *  5)

  599. 	KERNEL13(16  *  5)

  600. 	KERNEL14(16  *  5)

  601. 	KERNEL15(16  *  5)

  602. 	KERNEL16(16  *  5)

  603. 	cmpq	$128 *  6, %rax

  604. 	NOBRANCH

  605. 	jle	.L11

  606. 	KERNEL1 (16  *  6)

  607. 	KERNEL2 (16  *  6)

  608. 	KERNEL3 (16  *  6)

  609. 	KERNEL4 (16  *  6)

  610. 	KERNEL5 (16  *  6)

  611. 	KERNEL6 (16  *  6)

  612. 	KERNEL7 (16  *  6)

  613. 	KERNEL8 (16  *  6)

  614. 	KERNEL9 (16  *  6)

  615. 	KERNEL10(16  *  6)

  616. 	KERNEL11(16  *  6)

  617. 	KERNEL12(16  *  6)

  618. 	KERNEL13(16  *  6)

  619. 	KERNEL14(16  *  6)

  620. 	KERNEL15(16  *  6)

  621. 	KERNEL16(16  *  6)

  622. 	cmpq	$128 *  7, %rax

  623. 	NOBRANCH

  624. 	jle	.L11

  625. 	KERNEL1 (16  *  7)

  626. 	KERNEL2 (16  *  7)

  627. 	KERNEL3 (16  *  7)

  628. 	KERNEL4 (16  *  7)

  629. 	KERNEL5 (16  *  7)

  630. 	KERNEL6 (16  *  7)

  631. 	KERNEL7 (16  *  7)

  632. 	KERNEL8 (16  *  7)

  633. 	KERNEL9 (16  *  7)

  634. 	KERNEL10(16  *  7)

  635. 	KERNEL11(16  *  7)

  636. 	KERNEL12(16  *  7)

  637. 	KERNEL13(16  *  7)

  638. 	KERNEL14(16  *  7)

  639. 	KERNEL15(16  *  7)

  640. 	KERNEL16(16  *  7)

  641. 

  642. 	addq	$32 * 8  * SIZE, AO

  643. 	addq	$32 * 8  * SIZE, BO

  644. 	subq	$128 * 8, %rax

  645. 	jg	.L1X

  646. 

  647. .L11:

  648. 	leaq	(AO, %rax, 2), AO	# * 16

  649. 	leaq	(BO, %rax, 2), BO	# * 64

  650. 	ALIGN_4

  651. 

  652. .L12:

  653. #ifndef TRMMKERNEL

  654. 	movq	K, %rax

  655. #else

  656. 	movq	KKK, %rax

  657. #endif

  658. 	movddup	ALPHA_R, %xmm14

  659. 	movddup	ALPHA_I, %xmm15

  660. 	andq	$7, %rax		# if (k & 1)

  661. 	BRANCH

  662. 	je .L14

  663. 	ALIGN_4

  664. 

  665. .L13:

  666. 	mulpd	%xmm8, %xmm9

  667. 	movapd	 2 * SIZE(AO), %xmm10

  668. 	ADD1	%xmm9, %xmm0

  669. 	movddup	 1 * SIZE(BO), %xmm9

  670. 	mulpd	%xmm8, %xmm9

  671. 	movddup	 0 * SIZE(BO), %xmm11

  672. 	ADD2	%xmm9, %xmm1

  673. 	movddup	 2 * SIZE(BO), %xmm9

  674. 	mulpd	%xmm8, %xmm9

  675. 	ADD1	%xmm9, %xmm2

  676. 	movddup	 3 * SIZE(BO), %xmm9

  677. 	mulpd	%xmm8, %xmm9

  678. 	movapd	 4 * SIZE(AO), %xmm8

  679. 	ADD2	%xmm9, %xmm3

  680. 	movddup	 4 * SIZE(BO), %xmm9

  681. 	mulpd	%xmm10, %xmm11

  682. 	ADD1	%xmm11, %xmm4

  683. 	movddup	 1 * SIZE(BO), %xmm11

  684. 	mulpd	%xmm10, %xmm11

  685. 	ADD2	%xmm11, %xmm5

  686. 	movddup	 2 * SIZE(BO), %xmm11

  687. 	mulpd	%xmm10, %xmm11

  688. 	ADD1	%xmm11, %xmm6

  689. 	movddup	 3 * SIZE(BO), %xmm11

  690. 	mulpd	%xmm10, %xmm11

  691. 	ADD2	%xmm11, %xmm7

  692. 

  693. 	addq	$4 * SIZE, AO		# aoffset  += 4

  694. 	addq	$4 * SIZE, BO		# boffset1 += 8

  695. 	decq	%rax

  696. 	jg	.L13

  697. 	ALIGN_4

  698. 

  699. .L14:

  700. 	SHUFPD_1 %xmm1, %xmm1

  701. 	SHUFPD_1 %xmm3, %xmm3

  702. 	SHUFPD_1 %xmm5, %xmm5

  703. 	SHUFPD_1 %xmm7, %xmm7

  704. 

  705. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  706.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  707. 

  708. 	addsubpd	%xmm1, %xmm0

  709. 	addsubpd	%xmm3, %xmm2

  710. 	addsubpd	%xmm5, %xmm4

  711. 	addsubpd	%xmm7, %xmm6

  712. 

  713. 	movapd	%xmm0, %xmm1

  714. 	movapd	%xmm2, %xmm3

  715. 	movapd	%xmm4, %xmm5

  716. 	movapd	%xmm6, %xmm7

  717. #else

  718. 	addsubpd	%xmm0, %xmm1

  719. 	addsubpd	%xmm2, %xmm3

  720. 	addsubpd	%xmm4, %xmm5

  721. 	addsubpd	%xmm6, %xmm7

  722. 

  723. 	movapd		%xmm1, %xmm0

  724. 	movapd		%xmm3, %xmm2

  725. 	movapd		%xmm5, %xmm4

  726. 	movapd		%xmm7, %xmm6

  727. #endif

  728. 

  729. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  730. 	movsd	0 * SIZE(CO1), %xmm8

  731. 	movhpd	1 * SIZE(CO1), %xmm8

  732. 	movsd	2 * SIZE(CO1), %xmm10

  733. 	movhpd	3 * SIZE(CO1), %xmm10

  734. 

  735. 	movsd	0 * SIZE(CO2), %xmm9

  736. 	movhpd	1 * SIZE(CO2), %xmm9

  737. 	movsd	2 * SIZE(CO2), %xmm11

  738. 	movhpd	3 * SIZE(CO2), %xmm11

  739. #endif

  740. 

  741. 	SHUFPD_1 %xmm1, %xmm1

  742. 	SHUFPD_1 %xmm3, %xmm3

  743. 	SHUFPD_1 %xmm5, %xmm5

  744. 	SHUFPD_1 %xmm7, %xmm7

  745. 

  746. 	mulpd	%xmm14, %xmm0

  747. 	mulpd	%xmm14, %xmm2

  748. 	mulpd	%xmm14, %xmm4

  749. 	mulpd	%xmm14, %xmm6

  750. 

  751. 	mulpd	%xmm15, %xmm1

  752. 	mulpd	%xmm15, %xmm3

  753. 	mulpd	%xmm15, %xmm5

  754. 	mulpd	%xmm15, %xmm7

  755. 

  756. 	addsubpd	%xmm1, %xmm0

  757. 	addsubpd	%xmm3, %xmm2

  758. 	addsubpd	%xmm5, %xmm4

  759. 	addsubpd	%xmm7, %xmm6

  760. 

  761. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  762. 	addpd	%xmm8,  %xmm0

  763. 	addpd	%xmm9,  %xmm2

  764. 	addpd	%xmm10, %xmm4

  765. 	addpd	%xmm11, %xmm6

  766. #endif

  767. 

  768. 	movsd	%xmm0, 0 * SIZE(CO1)

  769. 	movhpd	%xmm0, 1 * SIZE(CO1)

  770. 	movsd	%xmm4, 2 * SIZE(CO1)

  771. 	movhpd	%xmm4, 3 * SIZE(CO1)

  772. 

  773. 	movsd	%xmm2, 0 * SIZE(CO2)

  774. 	movhpd	%xmm2, 1 * SIZE(CO2)

  775. 	movsd	%xmm6, 2 * SIZE(CO2)

  776. 	movhpd	%xmm6, 3 * SIZE(CO2)

  777. 

  778. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  779.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  780. 	movq	K, %rax

  781. 	subq	KKK, %rax

  782. 	leaq	(,%rax, SIZE), %rax

  783. 	leaq	(AO, %rax, 4), AO

  784. 	leaq	(BO, %rax, 4), BO

  785. #endif

  786. 

  787. #if defined(TRMMKERNEL) && defined(LEFT)

  788. 	addq	$2, KK

  789. #endif

  790. 

  791. 	addq	$4 * SIZE, CO1		# coffset += 4

  792. 	addq	$4 * SIZE, CO2		# coffset += 4

  793. 	decq	I			# i --

  794. 	jg	.L10

  795. 	ALIGN_4

  796. 

  797. .L30:

  798. 	testq	$1, M

  799. 	jle	.L99

  800. 

  801. .L40:

  802. #if !defined(TRMMKERNEL) || \

  803. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  804. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  805. 

  806. 	movq	B, BO

  807. #else

  808. 	movq	KK, %rax

  809. 	leaq	(, %rax, SIZE), %rax

  810. 	leaq	(AO, %rax, 2), AO

  811. 	leaq	(B,  %rax, 4), BO

  812. #endif

  813. 

  814. 	movapd	 0 * SIZE(AO), %xmm8

  815. 	pxor	%xmm0, %xmm0

  816. 	movddup	 0 * SIZE(BO), %xmm9

  817. 	pxor	%xmm1, %xmm1

  818. 	movapd	 8 * SIZE(AO), %xmm10

  819. 	pxor	%xmm2, %xmm2

  820. 	movddup	 8 * SIZE(BO), %xmm11

  821. 	pxor	%xmm3, %xmm3

  822. 

  823. #ifndef TRMMKERNEL

  824. 	movq	K, %rax

  825. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  826. 	movq	K, %rax

  827. 	subq	KK, %rax

  828. 	movq	%rax, KKK

  829. #else

  830. 	movq	KK, %rax

  831. #ifdef LEFT

  832. 	addq	$1, %rax

  833. #else

  834. 	addq	$2, %rax

  835. #endif

  836. 	movq	%rax, KKK

  837. #endif

  838. 	sarq	$3, %rax

  839. 	je	.L42

  840. 

  841. .L41:

  842. 	mulpd	%xmm8, %xmm9

  843. 	PREFETCH  (PREFETCHSIZE +  0) * SIZE(AO)

  844. 	ADD1	%xmm9, %xmm0

  845. 	movddup	 1 * SIZE(BO), %xmm9

  846. 	mulpd	%xmm8, %xmm9

  847. 	ADD2	%xmm9, %xmm1

  848. 	movddup	 2 * SIZE(BO), %xmm9

  849. 	mulpd	%xmm8, %xmm9

  850. 	ADD1	%xmm9, %xmm2

  851. 	movddup	 3 * SIZE(BO), %xmm9

  852. 	mulpd	%xmm8, %xmm9

  853. 	movapd	 2 * SIZE(AO), %xmm8

  854. 	ADD2	%xmm9, %xmm3

  855. 	movddup	 4 * SIZE(BO), %xmm9

  856. 	mulpd	%xmm8, %xmm9

  857. 	ADD1	%xmm9, %xmm0

  858. 	movddup	 5 * SIZE(BO), %xmm9

  859. 	mulpd	%xmm8, %xmm9

  860. 	ADD2	%xmm9, %xmm1

  861. 	movddup	 6 * SIZE(BO), %xmm9

  862. 	mulpd	%xmm8, %xmm9

  863. 	ADD1	%xmm9, %xmm2

  864. 	movddup	 7 * SIZE(BO), %xmm9

  865. 	mulpd	%xmm8, %xmm9

  866. 	movapd	 4 * SIZE(AO), %xmm8

  867. 	ADD2	%xmm9, %xmm3

  868. 	movddup	16 * SIZE(BO), %xmm9

  869. 	mulpd	%xmm8, %xmm11

  870. 	ADD1	%xmm11, %xmm0

  871. 	movddup	 9 * SIZE(BO), %xmm11

  872. 	mulpd	%xmm8, %xmm11

  873. 	ADD2	%xmm11, %xmm1

  874. 	movddup	10 * SIZE(BO), %xmm11

  875. 	mulpd	%xmm8, %xmm11

  876. 	ADD1	%xmm11, %xmm2

  877. 	movddup	11 * SIZE(BO), %xmm11

  878. 	mulpd	%xmm8, %xmm11

  879. 	movapd	 6 * SIZE(AO), %xmm8

  880. 	ADD2	%xmm11, %xmm3

  881. 	movddup	12 * SIZE(BO), %xmm11

  882. 	mulpd	%xmm8, %xmm11

  883. 	ADD1	%xmm11, %xmm0

  884. 	movddup	13 * SIZE(BO), %xmm11

  885. 	mulpd	%xmm8, %xmm11

  886. 	ADD2	%xmm11, %xmm1

  887. 	movddup	14 * SIZE(BO), %xmm11

  888. 	mulpd	%xmm8, %xmm11

  889. 	ADD1	%xmm11, %xmm2

  890. 	movddup	15 * SIZE(BO), %xmm11

  891. 	mulpd	%xmm8, %xmm11

  892. 	movapd	16 * SIZE(AO), %xmm8

  893. 	ADD2	%xmm11, %xmm3

  894. 	movddup	24 * SIZE(BO), %xmm11

  895. 	mulpd	%xmm10, %xmm9

  896. 	ADD1	%xmm9, %xmm0

  897. 	movddup	17 * SIZE(BO), %xmm9

  898. 	mulpd	%xmm10, %xmm9

  899. 	ADD2	%xmm9, %xmm1

  900. 	movddup	18 * SIZE(BO), %xmm9

  901. 	mulpd	%xmm10, %xmm9

  902. 	ADD1	%xmm9, %xmm2

  903. 	movddup	19 * SIZE(BO), %xmm9

  904. 	mulpd	%xmm10, %xmm9

  905. 	movapd	10 * SIZE(AO), %xmm10

  906. 	ADD2	%xmm9, %xmm3

  907. 	movddup	20 * SIZE(BO), %xmm9

  908. 	mulpd	%xmm10, %xmm9

  909. 	ADD1	%xmm9, %xmm0

  910. 	movddup	21 * SIZE(BO), %xmm9

  911. 	mulpd	%xmm10, %xmm9

  912. 	ADD2	%xmm9, %xmm1

  913. 	movddup	22 * SIZE(BO), %xmm9

  914. 	mulpd	%xmm10, %xmm9

  915. 	ADD1	%xmm9, %xmm2

  916. 	movddup	23 * SIZE(BO), %xmm9

  917. 	mulpd	%xmm10, %xmm9

  918. 	movapd	12 * SIZE(AO), %xmm10

  919. 	ADD2	%xmm9, %xmm3

  920. 	movddup	32 * SIZE(BO), %xmm9

  921. 	mulpd	%xmm10, %xmm11

  922. 	ADD1	%xmm11, %xmm0

  923. 	movddup	25 * SIZE(BO), %xmm11

  924. 	mulpd	%xmm10, %xmm11

  925. 	ADD2	%xmm11, %xmm1

  926. 	movddup	26 * SIZE(BO), %xmm11

  927. 	mulpd	%xmm10, %xmm11

  928. 	ADD1	%xmm11, %xmm2

  929. 	movddup	27 * SIZE(BO), %xmm11

  930. 	mulpd	%xmm10, %xmm11

  931. 	movapd	14 * SIZE(AO), %xmm10

  932. 	ADD2	%xmm11, %xmm3

  933. 	movddup	28 * SIZE(BO), %xmm11

  934. 	mulpd	%xmm10, %xmm11

  935. 	ADD1	%xmm11, %xmm0

  936. 	movddup	29 * SIZE(BO), %xmm11

  937. 	mulpd	%xmm10, %xmm11

  938. 	ADD2	%xmm11, %xmm1

  939. 	movddup	30 * SIZE(BO), %xmm11

  940. 	mulpd	%xmm10, %xmm11

  941. 	ADD1	%xmm11, %xmm2

  942. 	movddup	31 * SIZE(BO), %xmm11

  943. 	mulpd	%xmm10, %xmm11

  944. 	movapd	24 * SIZE(AO), %xmm10

  945. 	ADD2	%xmm11, %xmm3

  946. 	movddup	40 * SIZE(BO), %xmm11

  947. 

  948. 	addq   $16 * SIZE, AO

  949. 	addq   $32 * SIZE, BO

  950. 	decq   %rax

  951. 	jne    .L41

  952. 

  953. .L42:

  954. #ifndef TRMMKERNEL

  955. 	movq	K, %rax

  956. #else

  957. 	movq	KKK, %rax

  958. #endif

  959. 	movddup	ALPHA_R, %xmm14

  960. 	movddup	ALPHA_I, %xmm15

  961. 	andq	$7, %rax		# if (k & 1)

  962. 	BRANCH

  963. 	jle .L44

  964. 

  965. .L43:

  966. 	mulpd	%xmm8, %xmm9

  967. 	ADD1	%xmm9, %xmm0

  968. 	movddup	 1 * SIZE(BO), %xmm9

  969. 	mulpd	%xmm8, %xmm9

  970. 	ADD2	%xmm9, %xmm1

  971. 	movddup	 2 * SIZE(BO), %xmm9

  972. 	mulpd	%xmm8, %xmm9

  973. 	ADD1	%xmm9, %xmm2

  974. 	movddup	 3 * SIZE(BO), %xmm9

  975. 	mulpd	%xmm8, %xmm9

  976. 	movapd	 2 * SIZE(AO), %xmm8

  977. 	ADD2	%xmm9, %xmm3

  978. 	movddup	 4 * SIZE(BO), %xmm9

  979. 

  980. 	addq	$2 * SIZE, AO		# aoffset  += 4

  981. 	addq	$4 * SIZE, BO		# boffset1 += 8

  982. 	decq	%rax

  983. 	jg	.L43

  984. 	ALIGN_4

  985. 

  986. .L44:

  987. 	SHUFPD_1 %xmm1, %xmm1

  988. 	SHUFPD_1 %xmm3, %xmm3

  989. 

  990. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  991.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  992. 

  993. 	addsubpd	%xmm1, %xmm0

  994. 	addsubpd	%xmm3, %xmm2

  995. 

  996. 	movapd	%xmm0, %xmm1

  997. 	movapd	%xmm2, %xmm3

  998. #else

  999. 	addsubpd	%xmm0, %xmm1

  1000. 	addsubpd	%xmm2, %xmm3

  1001. 

  1002. 	movapd		%xmm1, %xmm0

  1003. 	movapd		%xmm3, %xmm2

  1004. #endif

  1005. 

  1006. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1007. 	movsd	0 * SIZE(CO1), %xmm8

  1008. 	movhpd	1 * SIZE(CO1), %xmm8

  1009. 

  1010. 	movsd	0 * SIZE(CO2), %xmm9

  1011. 	movhpd	1 * SIZE(CO2), %xmm9

  1012. #endif

  1013. 

  1014. 	SHUFPD_1 %xmm1, %xmm1

  1015. 	SHUFPD_1 %xmm3, %xmm3

  1016. 

  1017. 	mulpd	%xmm14, %xmm0

  1018. 	mulpd	%xmm14, %xmm2

  1019. 

  1020. 	mulpd	%xmm15, %xmm1

  1021. 	mulpd	%xmm15, %xmm3

  1022. 

  1023. 	addsubpd	%xmm1, %xmm0

  1024. 	addsubpd	%xmm3, %xmm2

  1025. 

  1026. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1027. 	addpd	%xmm8,  %xmm0

  1028. 	addpd	%xmm9,  %xmm2

  1029. #endif

  1030. 

  1031. 	movsd	%xmm0, 0 * SIZE(CO1)

  1032. 	movhpd	%xmm0, 1 * SIZE(CO1)

  1033. 	movsd	%xmm2, 0 * SIZE(CO2)

  1034. 	movhpd	%xmm2, 1 * SIZE(CO2)

  1035. 

  1036. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1037.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1038. 	movq	K, %rax

  1039. 	subq	KKK, %rax

  1040. 	leaq	(,%rax, SIZE), %rax

  1041. 	leaq	(AO, %rax, 2), AO

  1042. 	leaq	(BO, %rax, 4), BO

  1043. #endif

  1044. 

  1045. #if defined(TRMMKERNEL) && defined(LEFT)

  1046. 	addq	$1, KK

  1047. #endif

  1048. 	ALIGN_4

  1049. 

  1050. .L99:

  1051. #if defined(TRMMKERNEL) && !defined(LEFT)

  1052. 	addq	$2, KK

  1053. #endif

  1054. 

  1055. 	leaq	(C, LDC, 2), C		# c += 2 * ldc

  1056. 	movq	BO, B

  1057. 	decq	J			# j --

  1058. 	jg	.L01

  1059. 

  1060. .L100:

  1061. 	testq	$1, N

  1062. 	jle	.L999

  1063. 

  1064. .L101:

  1065. #if defined(TRMMKERNEL) && defined(LEFT)

  1066. 	movq	OFFSET, %rax

  1067. 	movq	%rax, KK

  1068. #endif

  1069. 

  1070. 	movq	C, CO1		# coffset1 = c

  1071. 	movq	A, AO		# aoffset = a

  1072. 

  1073. 	movq	M,  I

  1074. 	sarq	$1, I		# i = (m >> 2)

  1075. 	jle	.L130

  1076. 	ALIGN_4

  1077. 

  1078. .L110:

  1079. #if !defined(TRMMKERNEL) || \

  1080. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1081. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1082. 

  1083. 	movq	B, BO

  1084. #else

  1085. 	movq	KK, %rax

  1086. 	leaq	(, %rax, SIZE), %rax

  1087. 	leaq	(AO, %rax, 4), AO

  1088. 	leaq	(B,  %rax, 2), BO

  1089. #endif

  1090. 

  1091. 	movapd	 0 * SIZE(AO), %xmm8

  1092. 	pxor	%xmm0, %xmm0

  1093. 	movddup	 0 * SIZE(BO), %xmm9

  1094. 	pxor	%xmm1, %xmm1

  1095. 	movapd	 8 * SIZE(AO), %xmm10

  1096. 	pxor	%xmm4, %xmm4

  1097. 	movddup	 8 * SIZE(BO), %xmm11

  1098. 	pxor	%xmm5, %xmm5

  1099. 

  1100. 	prefetchnta     4 * SIZE(CO1)

  1101. 

  1102. #ifndef TRMMKERNEL

  1103. 	movq	K, %rax

  1104. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1105. 	movq	K, %rax

  1106. 	subq	KK, %rax

  1107. 	movq	%rax, KKK

  1108. #else

  1109. 	movq	KK, %rax

  1110. #ifdef LEFT

  1111. 	addq	$2, %rax

  1112. #else

  1113. 	addq	$1, %rax

  1114. #endif

  1115. 	movq	%rax, KKK

  1116. #endif

  1117. 	sarq	$3, %rax

  1118. 	je	.L112

  1119. 

  1120. .L111:

  1121. 	mulpd	%xmm8, %xmm9

  1122. 	PREFETCH  (PREFETCHSIZE +  0) * SIZE(AO)

  1123. 	ADD1	%xmm9, %xmm0

  1124. 	movddup	 1 * SIZE(BO), %xmm9

  1125. 	mulpd	%xmm8, %xmm9

  1126. 	movapd	 2 * SIZE(AO), %xmm8

  1127. 	ADD2	%xmm9, %xmm1

  1128. 	movddup	 0 * SIZE(BO), %xmm9

  1129. 	mulpd	%xmm8, %xmm9

  1130. 	ADD1	%xmm9, %xmm4

  1131. 	movddup	 1 * SIZE(BO), %xmm9

  1132. 	mulpd	%xmm8, %xmm9

  1133. 	movapd	 4 * SIZE(AO), %xmm8

  1134. 	ADD2	%xmm9, %xmm5

  1135. 	movddup	 2 * SIZE(BO), %xmm9

  1136. 	mulpd	%xmm8, %xmm9

  1137. 	ADD1	%xmm9, %xmm0

  1138. 	movddup	 3 * SIZE(BO), %xmm9

  1139. 	mulpd	%xmm8, %xmm9

  1140. 	movapd	 6 * SIZE(AO), %xmm8

  1141. 	ADD2	%xmm9, %xmm1

  1142. 	movddup	 2 * SIZE(BO), %xmm9

  1143. 	mulpd	%xmm8, %xmm9

  1144. 	ADD1	%xmm9, %xmm4

  1145. 	movddup	 3 * SIZE(BO), %xmm9

  1146. 	mulpd	%xmm8, %xmm9

  1147. 	movapd	16 * SIZE(AO), %xmm8

  1148. 	ADD2	%xmm9, %xmm5

  1149. 	movddup	 4 * SIZE(BO), %xmm9

  1150. 	mulpd	%xmm10, %xmm9

  1151. 	ADD1	%xmm9, %xmm0

  1152. 	movddup	 5 * SIZE(BO), %xmm9

  1153. 	mulpd	%xmm10, %xmm9

  1154. 	movapd	10 * SIZE(AO), %xmm10

  1155. 	ADD2	%xmm9, %xmm1

  1156. 	movddup	 4 * SIZE(BO), %xmm9

  1157. 	mulpd	%xmm10, %xmm9

  1158. 	ADD1	%xmm9, %xmm4

  1159. 	movddup	 5 * SIZE(BO), %xmm9

  1160. 	mulpd	%xmm10, %xmm9

  1161. 	movapd	12 * SIZE(AO), %xmm10

  1162. 	ADD2	%xmm9, %xmm5

  1163. 	movddup	 6 * SIZE(BO), %xmm9

  1164. 	mulpd	%xmm10, %xmm9

  1165. 	ADD1	%xmm9, %xmm0

  1166. 	movddup	 7 * SIZE(BO), %xmm9

  1167. 	mulpd	%xmm10, %xmm9

  1168. 	movapd	14 * SIZE(AO), %xmm10

  1169. 	ADD2	%xmm9, %xmm1

  1170. 	movddup	 6 * SIZE(BO), %xmm9

  1171. 	mulpd	%xmm10, %xmm9

  1172. 	ADD1	%xmm9, %xmm4

  1173. 	movddup	 7 * SIZE(BO), %xmm9

  1174. 	mulpd	%xmm10, %xmm9

  1175. 	movapd	40 * SIZE(AO), %xmm10

  1176. 	ADD2	%xmm9, %xmm5

  1177. 	movddup	16 * SIZE(BO), %xmm9

  1178. 	mulpd	%xmm8, %xmm11

  1179. 	PREFETCH  (PREFETCHSIZE + 16) * SIZE(AO)

  1180. 	ADD1	%xmm11, %xmm0

  1181. 	movddup	 9 * SIZE(BO), %xmm11

  1182. 	mulpd	%xmm8, %xmm11

  1183. 	movapd	18 * SIZE(AO), %xmm8

  1184. 	ADD2	%xmm11, %xmm1

  1185. 	movddup	 8 * SIZE(BO), %xmm11

  1186. 	mulpd	%xmm8, %xmm11

  1187. 	ADD1	%xmm11, %xmm4

  1188. 	movddup	 9 * SIZE(BO), %xmm11

  1189. 	mulpd	%xmm8, %xmm11

  1190. 	movapd	20 * SIZE(AO), %xmm8

  1191. 	ADD2	%xmm11, %xmm5

  1192. 	movddup	10 * SIZE(BO), %xmm11

  1193. 	mulpd	%xmm8, %xmm11

  1194. 	ADD1	%xmm11, %xmm0

  1195. 	movddup	11 * SIZE(BO), %xmm11

  1196. 	mulpd	%xmm8, %xmm11

  1197. 	movapd	22 * SIZE(AO), %xmm8

  1198. 	ADD2	%xmm11, %xmm1

  1199. 	movddup	10 * SIZE(BO), %xmm11

  1200. 	mulpd	%xmm8, %xmm11

  1201. 	ADD1	%xmm11, %xmm4

  1202. 	movddup	11 * SIZE(BO), %xmm11

  1203. 	mulpd	%xmm8, %xmm11

  1204. 	movapd	24 * SIZE(AO), %xmm8

  1205. 	ADD2	%xmm11, %xmm5

  1206. 	movddup	12 * SIZE(BO), %xmm11

  1207. 	mulpd	%xmm8, %xmm11

  1208. 	ADD1	%xmm11, %xmm0

  1209. 	movddup	13 * SIZE(BO), %xmm11

  1210. 	mulpd	%xmm8, %xmm11

  1211. 	movapd	26 * SIZE(AO), %xmm8

  1212. 	ADD2	%xmm11, %xmm1

  1213. 	movddup	12 * SIZE(BO), %xmm11

  1214. 	mulpd	%xmm8, %xmm11

  1215. 	ADD1	%xmm11, %xmm4

  1216. 	movddup	13 * SIZE(BO), %xmm11

  1217. 	mulpd	%xmm8, %xmm11

  1218. 	movapd	28 * SIZE(AO), %xmm8

  1219. 	ADD2	%xmm11, %xmm5

  1220. 	movddup	14 * SIZE(BO), %xmm11

  1221. 	mulpd	%xmm8, %xmm11

  1222. 	ADD1	%xmm11, %xmm0

  1223. 	movddup	15 * SIZE(BO), %xmm11

  1224. 	mulpd	%xmm8, %xmm11

  1225. 	movapd	30 * SIZE(AO), %xmm8

  1226. 	ADD2	%xmm11, %xmm1

  1227. 	movddup	14 * SIZE(BO), %xmm11

  1228. 	mulpd	%xmm8, %xmm11

  1229. 	ADD1	%xmm11, %xmm4

  1230. 	movddup	15 * SIZE(BO), %xmm11

  1231. 	mulpd	%xmm8, %xmm11

  1232. 	movapd	32 * SIZE(AO), %xmm8

  1233. 	ADD2	%xmm11, %xmm5

  1234. 	movddup	24 * SIZE(BO), %xmm11

  1235. 

  1236. 	addq   $32 * SIZE, AO

  1237. 	addq   $16 * SIZE, BO

  1238. 	decq   %rax

  1239. 	jne    .L111

  1240. 	ALIGN_4

  1241. 

  1242. .L112:

  1243. #ifndef TRMMKERNEL

  1244. 	movq	K, %rax

  1245. #else

  1246. 	movq	KKK, %rax

  1247. #endif

  1248. 	movddup	ALPHA_R, %xmm14

  1249. 	movddup	ALPHA_I, %xmm15

  1250. 	andq	$7, %rax		# if (k & 1)

  1251. 	BRANCH

  1252. 	jle .L114

  1253. 

  1254. .L113:

  1255. 	mulpd	%xmm8, %xmm9

  1256. 	movapd	 2 * SIZE(AO), %xmm10

  1257. 	ADD1	%xmm9, %xmm0

  1258. 	movddup	 1 * SIZE(BO), %xmm9

  1259. 	mulpd	%xmm8, %xmm9

  1260. 	movddup	 0 * SIZE(BO), %xmm11

  1261. 	ADD2	%xmm9, %xmm1

  1262. 	movddup	 2 * SIZE(BO), %xmm9

  1263. 	mulpd	%xmm10, %xmm11

  1264. 	movapd	 4 * SIZE(AO), %xmm8

  1265. 	ADD1	%xmm11, %xmm4

  1266. 	movddup	 1 * SIZE(BO), %xmm11

  1267. 	mulpd	%xmm10, %xmm11

  1268. 	ADD2	%xmm11, %xmm5

  1269. 

  1270. 	addq	$4 * SIZE, AO		# aoffset  += 4

  1271. 	addq	$2 * SIZE, BO		# boffset1 += 8

  1272. 	decq	%rax

  1273. 	jg	.L113

  1274. 	ALIGN_4

  1275. 

  1276. .L114:

  1277. 	SHUFPD_1 %xmm1, %xmm1

  1278. 	SHUFPD_1 %xmm5, %xmm5

  1279. 

  1280. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1281.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1282. 

  1283. 	addsubpd	%xmm1, %xmm0

  1284. 	addsubpd	%xmm5, %xmm4

  1285. 

  1286. 	movapd	%xmm0, %xmm1

  1287. 	movapd	%xmm4, %xmm5

  1288. #else

  1289. 	addsubpd	%xmm0, %xmm1

  1290. 	addsubpd	%xmm4, %xmm5

  1291. 

  1292. 	movapd		%xmm1, %xmm0

  1293. 	movapd		%xmm5, %xmm4

  1294. #endif

  1295. 

  1296. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1297. 	movsd	0 * SIZE(CO1), %xmm8

  1298. 	movhpd	1 * SIZE(CO1), %xmm8

  1299. 	movsd	2 * SIZE(CO1), %xmm10

  1300. 	movhpd	3 * SIZE(CO1), %xmm10

  1301. #endif

  1302. 

  1303. 	SHUFPD_1 %xmm1, %xmm1

  1304. 	SHUFPD_1 %xmm5, %xmm5

  1305. 

  1306. 	mulpd	%xmm14, %xmm0

  1307. 	mulpd	%xmm14, %xmm4

  1308. 

  1309. 	mulpd	%xmm15, %xmm1

  1310. 	mulpd	%xmm15, %xmm5

  1311. 

  1312. 	addsubpd	%xmm1, %xmm0

  1313. 	addsubpd	%xmm5, %xmm4

  1314. 

  1315. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1316. 	addpd	%xmm8,  %xmm0

  1317. 	addpd	%xmm10, %xmm4

  1318. #endif

  1319. 

  1320. 	movsd	%xmm0, 0 * SIZE(CO1)

  1321. 	movhpd	%xmm0, 1 * SIZE(CO1)

  1322. 	movsd	%xmm4, 2 * SIZE(CO1)

  1323. 	movhpd	%xmm4, 3 * SIZE(CO1)

  1324. 

  1325. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1326.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1327. 	movq	K, %rax

  1328. 	subq	KKK, %rax

  1329. 	leaq	(,%rax, SIZE), %rax

  1330. 	leaq	(AO, %rax, 4), AO

  1331. 	leaq	(BO, %rax, 2), BO

  1332. #endif

  1333. 

  1334. #if defined(TRMMKERNEL) && defined(LEFT)

  1335. 	addq	$2, KK

  1336. #endif

  1337. 

  1338. 	addq	$4 * SIZE, CO1		# coffset += 4

  1339. 	decq	I			# i --

  1340. 	jg	.L110

  1341. 	ALIGN_4

  1342. 

  1343. .L130:

  1344. 	testq	$1, M

  1345. 	jle	.L999

  1346. 	ALIGN_4

  1347. 

  1348. .L140:

  1349. #if !defined(TRMMKERNEL) || \

  1350. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1351. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1352. 

  1353. 	movq	B, BO

  1354. #else

  1355. 	movq	KK, %rax

  1356. 	leaq	(, %rax, SIZE), %rax

  1357. 	leaq	(AO, %rax, 2), AO

  1358. 	leaq	(B,  %rax, 2), BO

  1359. #endif

  1360. 

  1361. 	movapd	 0 * SIZE(AO), %xmm8

  1362. 	pxor	%xmm0, %xmm0

  1363. 	movddup	 0 * SIZE(BO), %xmm9

  1364. 	pxor	%xmm1, %xmm1

  1365. 	movapd	 8 * SIZE(AO), %xmm10

  1366. 	pxor	%xmm2, %xmm2

  1367. 	movddup	 8 * SIZE(BO), %xmm11

  1368. 	pxor	%xmm3, %xmm3

  1369. 

  1370. #ifndef TRMMKERNEL

  1371. 	movq	K, %rax

  1372. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1373. 	movq	K, %rax

  1374. 	subq	KK, %rax

  1375. 	movq	%rax, KKK

  1376. #else

  1377. 	movq	KK, %rax

  1378. #ifdef LEFT

  1379. 	addq	$1, %rax

  1380. #else

  1381. 	addq	$1, %rax

  1382. #endif

  1383. 	movq	%rax, KKK

  1384. #endif

  1385. 	sarq	$3, %rax

  1386. 	je	.L142

  1387. 

  1388. .L141:

  1389. 	mulpd	%xmm8, %xmm9

  1390. 	PREFETCH  (PREFETCHSIZE + 0) * SIZE(AO)

  1391. 	ADD1	%xmm9, %xmm0

  1392. 	movddup	 1 * SIZE(BO), %xmm9

  1393. 	mulpd	%xmm8, %xmm9

  1394. 	movapd	 2 * SIZE(AO), %xmm8

  1395. 	ADD2	%xmm9, %xmm1

  1396. 	movddup	 2 * SIZE(BO), %xmm9

  1397. 	mulpd	%xmm8, %xmm9

  1398. 	ADD1	%xmm9, %xmm2

  1399. 	movddup	 3 * SIZE(BO), %xmm9

  1400. 	mulpd	%xmm8, %xmm9

  1401. 	movapd	 4 * SIZE(AO), %xmm8

  1402. 	ADD2	%xmm9, %xmm3

  1403. 	movddup	 4 * SIZE(BO), %xmm9

  1404. 	mulpd	%xmm8, %xmm9

  1405. 	ADD1	%xmm9, %xmm0

  1406. 	movddup	 5 * SIZE(BO), %xmm9

  1407. 	mulpd	%xmm8, %xmm9

  1408. 	movapd	 6 * SIZE(AO), %xmm8

  1409. 	ADD2	%xmm9, %xmm1

  1410. 	movddup	 6 * SIZE(BO), %xmm9

  1411. 	mulpd	%xmm8, %xmm9

  1412. 	ADD1	%xmm9, %xmm2

  1413. 	movddup	 7 * SIZE(BO), %xmm9

  1414. 	mulpd	%xmm8, %xmm9

  1415. 	movapd	16 * SIZE(AO), %xmm8

  1416. 	ADD2	%xmm9, %xmm3

  1417. 	movddup	16 * SIZE(BO), %xmm9

  1418. 	mulpd	%xmm10, %xmm11

  1419. 	ADD1	%xmm11, %xmm0

  1420. 	movddup	 9 * SIZE(BO), %xmm11

  1421. 	mulpd	%xmm10, %xmm11

  1422. 	movapd	10 * SIZE(AO), %xmm10

  1423. 	ADD2	%xmm11, %xmm1

  1424. 	movddup	10 * SIZE(BO), %xmm11

  1425. 	mulpd	%xmm10, %xmm11

  1426. 	ADD1	%xmm11, %xmm2

  1427. 	movddup	11 * SIZE(BO), %xmm11

  1428. 	mulpd	%xmm10, %xmm11

  1429. 	movapd	12 * SIZE(AO), %xmm10

  1430. 	ADD2	%xmm11, %xmm3

  1431. 	movddup	12 * SIZE(BO), %xmm11

  1432. 	mulpd	%xmm10, %xmm11

  1433. 	ADD1	%xmm11, %xmm0

  1434. 	movddup	13 * SIZE(BO), %xmm11

  1435. 	mulpd	%xmm10, %xmm11

  1436. 	movapd	14 * SIZE(AO), %xmm10

  1437. 	ADD2	%xmm11, %xmm1

  1438. 	movddup	14 * SIZE(BO), %xmm11

  1439. 	mulpd	%xmm10, %xmm11

  1440. 	ADD1	%xmm11, %xmm2

  1441. 	movddup	15 * SIZE(BO), %xmm11

  1442. 	mulpd	%xmm10, %xmm11

  1443. 	movapd	24 * SIZE(AO), %xmm10

  1444. 	ADD2	%xmm11, %xmm3

  1445. 	movddup	24 * SIZE(BO), %xmm11

  1446. 

  1447. 	addq   $16 * SIZE, AO

  1448. 	addq   $16 * SIZE, BO

  1449. 	decq   %rax

  1450. 	jne    .L141

  1451. 

  1452. .L142:

  1453. #ifndef TRMMKERNEL

  1454. 	movq	K, %rax

  1455. #else

  1456. 	movq	KKK, %rax

  1457. #endif

  1458. 	movddup	ALPHA_R, %xmm14

  1459. 	movddup	ALPHA_I, %xmm15

  1460. 	andq	$7, %rax		# if (k & 1)

  1461. 	BRANCH

  1462. 	jle .L144

  1463. 

  1464. .L143:

  1465. 	mulpd	%xmm8, %xmm9

  1466. 	ADD1	%xmm9, %xmm0

  1467. 	movddup	 1 * SIZE(BO), %xmm9

  1468. 	mulpd	%xmm8, %xmm9

  1469. 	movapd	 2 * SIZE(AO), %xmm8

  1470. 	ADD2	%xmm9, %xmm1

  1471. 	movddup	 2 * SIZE(BO), %xmm9

  1472. 

  1473. 	addq	$2 * SIZE, AO		# aoffset  += 4

  1474. 	addq	$2 * SIZE, BO		# boffset1 += 8

  1475. 	decq	%rax

  1476. 	jg	.L143

  1477. 	ALIGN_4

  1478. 

  1479. .L144:

  1480. 	addpd	%xmm2, %xmm0

  1481. 	addpd	%xmm3, %xmm1

  1482. 

  1483. 	SHUFPD_1 %xmm1, %xmm1

  1484. 

  1485. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1486.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1487. 

  1488. 	addsubpd	%xmm1, %xmm0

  1489. 	movapd	%xmm0, %xmm1

  1490. #else

  1491. 	addsubpd	%xmm0, %xmm1

  1492. 	movapd		%xmm1, %xmm0

  1493. #endif

  1494. 

  1495. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1496. 	movsd	0 * SIZE(CO1), %xmm8

  1497. 	movhpd	1 * SIZE(CO1), %xmm8

  1498. #endif

  1499. 

  1500. 	SHUFPD_1 %xmm1, %xmm1

  1501. 	mulpd	%xmm14, %xmm0

  1502. 	mulpd	%xmm15, %xmm1

  1503. 	addsubpd	%xmm1, %xmm0

  1504. 

  1505. #if! defined(TRMMKERNEL) && !defined(BETAZERO)

  1506. 	addpd	%xmm8,  %xmm0

  1507. #endif

  1508. 

  1509. 	movsd	%xmm0, 0 * SIZE(CO1)

  1510. 	movhpd	%xmm0, 1 * SIZE(CO1)

  1511. 	ALIGN_4

  1512. 

  1513. .L999:

  1514. 	movq	  0(%rsp), %rbx

  1515. 	movq	  8(%rsp), %rbp

  1516. 	movq	 16(%rsp), %r12

  1517. 	movq	 24(%rsp), %r13

  1518. 	movq	 32(%rsp), %r14

  1519. 	movq	 40(%rsp), %r15

  1520. 

  1521. #ifdef WINDOWS_ABI

  1522. 	movq	 48(%rsp), %rdi

  1523. 	movq	 56(%rsp), %rsi

  1524. 	movups	 64(%rsp), %xmm6

  1525. 	movups	 80(%rsp), %xmm7

  1526. 	movups	 96(%rsp), %xmm8

  1527. 	movups	112(%rsp), %xmm9

  1528. 	movups	128(%rsp), %xmm10

  1529. 	movups	144(%rsp), %xmm11

  1530. 	movups	160(%rsp), %xmm12

  1531. 	movups	176(%rsp), %xmm13

  1532. 	movups	192(%rsp), %xmm14

  1533. 	movups	208(%rsp), %xmm15

  1534. #endif

  1535. 

  1536. 	addq	$STACKSIZE, %rsp

  1537. 	ret

  1538. 

  1539. 	EPILOGUE