OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 66d9185ebe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '66d9185ebe'
${ noResults }
OpenBLAS/kernel/x86_64/zgemm_kernel_2x2_barcelona.S

1424 lines
32 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define OLD_M	%rdi

  43. #define OLD_N	%rsi

  44. #define M	%r13

  45. #define N	%r14

  46. #define K	%rdx

  47. 

  48. #define A	%rcx

  49. #define B	%r8

  50. #define C	%r9

  51. #define LDC	%r10

  52. #define I	%r11

  53. #define AO	%rdi

  54. #define BO	%rsi

  55. #define	CO1	%rbp

  56. #define CO2	%rbx

  57. #define BB	%r12

  58. #define J	%r15

  59. 

  60. #ifndef WINDOWS_ABI

  61. 

  62. #define STACKSIZE 96

  63. 

  64. #define ALPHA_R	 48(%rsp)

  65. #define ALPHA_I	 56(%rsp)

  66. #define OFFSET	 64(%rsp)

  67. #define KK	 72(%rsp)

  68. #define KKK	 80(%rsp)

  69. 

  70. #else

  71. 

  72. #define STACKSIZE 320

  73. 

  74. #define OLD_ALPHA_I	40 + STACKSIZE(%rsp)

  75. #define OLD_A		48 + STACKSIZE(%rsp)

  76. #define OLD_B		56 + STACKSIZE(%rsp)

  77. #define OLD_C		64 + STACKSIZE(%rsp)

  78. #define OLD_LDC		72 + STACKSIZE(%rsp)

  79. #define OLD_OFFSET	80 + STACKSIZE(%rsp)

  80. 

  81. #define ALPHA_R	224(%rsp)

  82. #define ALPHA_I	232(%rsp)

  83. #define OFFSET	240(%rsp)

  84. #define KK	248(%rsp)

  85. #define KKK	256(%rsp)

  86. 

  87. #endif

  88. 

  89. #define movlpd	movsd

  90. #define movapd	movups

  91. #define movupd	movups

  92. 

  93. #if   defined(NN) || defined(NT) || defined(TN) || defined(TT)

  94. #define ADD1	addpd

  95. #define ADD2	addpd

  96. #elif defined(RN) || defined(RT) || defined(CN) || defined(CT)

  97. #define ADD1	subpd

  98. #define ADD2	addpd

  99. #elif defined(NR) || defined(NC) || defined(TR) || defined(TC)

  100. #define ADD1	addpd

  101. #define ADD2	subpd

  102. #else

  103. #define ADD1	subpd

  104. #define ADD2	subpd

  105. #endif

  106. 

  107. #define KERNEL1(xx) \

  108. 	mulpd	%xmm1, %xmm0 ;\

  109. 	ADD1	%xmm0, %xmm8 ;\

  110. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1 ;\

  111. 	movapd	%xmm2, %xmm0 ;\

  112. 	ADD1	%xmm1, %xmm12 ;\

  113. 	movddup	-14 * SIZE(BO, %rax, 4), %xmm1 ;\

  114. 	mulpd	%xmm3, %xmm2 ;\

  115. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3 ;\

  116. 	ADD2	%xmm2, %xmm9 ;\

  117. 	movapd	%xmm0, %xmm2 ;\

  118. 	ADD2	%xmm3, %xmm13 ;\

  119. 	movddup	-13 * SIZE(BO, %rax, 4), %xmm3 ;\

  120. 	mulpd	%xmm1, %xmm0 ;\

  121. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1 ;\

  122. 	ADD1	%xmm0, %xmm10 ;\

  123. 	movapd	-12 * SIZE(AO, %rax, 4), %xmm0 ;\

  124. 	ADD1	%xmm1, %xmm14 ;\

  125. 	movddup	-12 * SIZE(BO, %rax, 4), %xmm1 ;\

  126. 	mulpd	%xmm3, %xmm2 ;\

  127. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3 ;\

  128. 	ADD2	%xmm2, %xmm11 ;\

  129. 	ADD2	%xmm3, %xmm15 ;\

  130.  	movddup	-11 * SIZE(BO, %rax, 4), %xmm3 ;\

  131. 	movapd	%xmm0, %xmm2

  132. 

  133. #define KERNEL2(xx) \

  134. 	mulpd	%xmm1, %xmm0 ;\

  135. 	ADD1	%xmm0, %xmm8 ;\

  136. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm1 ;\

  137. 	movapd	%xmm2, %xmm0 ;\

  138. 	ADD1	%xmm1, %xmm12 ;\

  139. /*A*/	movapd	  (AO, %rax, 4), %xmm6 ;\

  140. 	movddup	-10 * SIZE(BO, %rax, 4), %xmm1 ;\

  141. 	mulpd	%xmm3, %xmm2 ;\

  142. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm3 ;\

  143. 	ADD2	%xmm2, %xmm9 ;\

  144. 	movapd	%xmm0, %xmm2 ;\

  145. 	ADD2	%xmm3, %xmm13 ;\

  146. 	movddup	 -9 * SIZE(BO, %rax, 4), %xmm3 ;\

  147. 	mulpd	%xmm1, %xmm0 ;\

  148. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm1 ;\

  149. 	ADD1	%xmm0, %xmm10 ;\

  150. 	ADD1	%xmm1, %xmm14 ;\

  151. /**/	movddup	  (BO, %rax, 4), %xmm1 ;\

  152. 	mulpd	%xmm3, %xmm2 ;\

  153. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm3 ;\

  154. 	ADD2	%xmm2, %xmm11 ;\

  155. 	ADD2	%xmm3, %xmm15 ;\

  156.  	movddup	 -7 * SIZE(BO, %rax, 4), %xmm3 ;\

  157. 	movapd	%xmm4, %xmm2

  158. 

  159. #define KERNEL3(xx) \

  160. 	mulpd	%xmm5, %xmm4 ;\

  161. 	ADD1	%xmm4, %xmm8 ;\

  162. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm5 ;\

  163. 	movapd	%xmm2, %xmm4 ;\

  164. 	ADD1	%xmm5, %xmm12 ;\

  165. 	movddup	 -6 * SIZE(BO, %rax, 4), %xmm5 ;\

  166. 	mulpd	%xmm3, %xmm2 ;\

  167. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm3 ;\

  168. 	ADD2	%xmm2, %xmm9 ;\

  169. 	movapd	%xmm4, %xmm2 ;\

  170. 	ADD2	%xmm3, %xmm13 ;\

  171. 	movddup	 -5 * SIZE(BO, %rax, 4), %xmm3 ;\

  172. 	mulpd	%xmm5, %xmm4 ;\

  173. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm5 ;\

  174. 	ADD1	%xmm4, %xmm10 ;\

  175. 	movapd	 -4 * SIZE(AO, %rax, 4), %xmm4 ;\

  176. 	ADD1	%xmm5, %xmm14 ;\

  177. 	movddup	 -4 * SIZE(BO, %rax, 4), %xmm5 ;\

  178. 	mulpd	%xmm3, %xmm2 ;\

  179. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm3 ;\

  180. 	ADD2	%xmm2, %xmm11 ;\

  181. 	ADD2	%xmm3, %xmm15 ;\

  182.  	movddup	 -3 * SIZE(BO, %rax, 4), %xmm3 ;\

  183. 	movapd	%xmm4, %xmm2

  184. 

  185. #define KERNEL4(xx) \

  186. 	mulpd	%xmm5, %xmm4 ;\

  187. 	ADD1	%xmm4, %xmm8 ;\

  188. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm5 ;\

  189. 	movapd	%xmm2, %xmm4 ;\

  190. 	ADD1	%xmm5, %xmm12 ;\

  191. /*A*/	movapd	  8 * SIZE(AO, %rax, 4), %xmm7 ;\

  192. 	movddup	 -2 * SIZE(BO, %rax, 4), %xmm5 ;\

  193. 	mulpd	%xmm3, %xmm2 ;\

  194. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm3 ;\

  195. 	ADD2	%xmm2, %xmm9 ;\

  196. 	movapd	%xmm4, %xmm2 ;\

  197. 	ADD2	%xmm3, %xmm13 ;\

  198. 	movddup	 -1 * SIZE(BO, %rax, 4), %xmm3 ;\

  199. 	mulpd	%xmm5, %xmm4 ;\

  200. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm5 ;\

  201. 	ADD1	%xmm4, %xmm10 ;\

  202. 	ADD1	%xmm5, %xmm14 ;\

  203. /**/	movddup	  8 * SIZE(BO, %rax, 4), %xmm5 ;\

  204. 	mulpd	%xmm3, %xmm2 ;\

  205. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm3 ;\

  206. 	ADD2	%xmm2, %xmm11 ;\

  207. 	ADD2	%xmm3, %xmm15 ;\

  208.  	movddup	  1 * SIZE(BO, %rax, 4), %xmm3 ;\

  209. 	movapd	%xmm6, %xmm2

  210. 

  211. #define KERNEL5(xx) \

  212. 	mulpd	%xmm1, %xmm6 ;\

  213. 	ADD1	%xmm6, %xmm8 ;\

  214. 	mulpd	  2 * SIZE(AO, %rax, 4), %xmm1 ;\

  215. 	movapd	%xmm2, %xmm6 ;\

  216. 	ADD1	%xmm1, %xmm12 ;\

  217. 	movddup	  2 * SIZE(BO, %rax, 4), %xmm1 ;\

  218. 	mulpd	%xmm3, %xmm2 ;\

  219. 	mulpd	  2 * SIZE(AO, %rax, 4), %xmm3 ;\

  220. 	ADD2	%xmm2, %xmm9 ;\

  221. 	movapd	%xmm6, %xmm2 ;\

  222. 	ADD2	%xmm3, %xmm13 ;\

  223. 	movddup	  3 * SIZE(BO, %rax, 4), %xmm3 ;\

  224. 	mulpd	%xmm1, %xmm6 ;\

  225. 	mulpd	  2 * SIZE(AO, %rax, 4), %xmm1 ;\

  226. 	ADD1	%xmm6, %xmm10 ;\

  227. 	movapd	  4 * SIZE(AO, %rax, 4), %xmm6 ;\

  228. 	ADD1	%xmm1, %xmm14 ;\

  229. 	movddup	  4 * SIZE(BO, %rax, 4), %xmm1 ;\

  230. 	mulpd	%xmm3, %xmm2 ;\

  231. 	mulpd	  2 * SIZE(AO, %rax, 4), %xmm3 ;\

  232. 	ADD2	%xmm2, %xmm11 ;\

  233. 	ADD2	%xmm3, %xmm15 ;\

  234.  	movddup	  5 * SIZE(BO, %rax, 4), %xmm3 ;\

  235. 	movapd	%xmm6, %xmm2

  236. 

  237. #define KERNEL6(xx) \

  238. 	mulpd	%xmm1, %xmm6 ;\

  239. 	ADD1	%xmm6, %xmm8 ;\

  240. 	mulpd	  6 * SIZE(AO, %rax, 4), %xmm1 ;\

  241. 	movapd	%xmm2, %xmm6 ;\

  242. 	ADD1	%xmm1, %xmm12 ;\

  243. /*A*/	movapd	 16 * SIZE(AO, %rax, 4), %xmm0 ;\

  244. 	movddup	  6 * SIZE(BO, %rax, 4), %xmm1 ;\

  245. 	mulpd	%xmm3, %xmm2 ;\

  246. 	mulpd	  6 * SIZE(AO, %rax, 4), %xmm3 ;\

  247. 	ADD2	%xmm2, %xmm9 ;\

  248. 	movapd	%xmm6, %xmm2 ;\

  249. 	ADD2	%xmm3, %xmm13 ;\

  250. 	movddup	  7 * SIZE(BO, %rax, 4), %xmm3 ;\

  251. 	mulpd	%xmm1, %xmm6 ;\

  252. 	mulpd	  6 * SIZE(AO, %rax, 4), %xmm1 ;\

  253. 	ADD1	%xmm6, %xmm10 ;\

  254.  	ADD1	%xmm1, %xmm14 ;\

  255. /**/	movddup	 16 * SIZE(BO, %rax, 4), %xmm1 ;\

  256. 	mulpd	%xmm3, %xmm2 ;\

  257. 	mulpd	  6 * SIZE(AO, %rax, 4), %xmm3 ;\

  258. 	ADD2	%xmm2, %xmm11 ;\

  259. 	ADD2	%xmm3, %xmm15 ;\

  260.  	movddup	  9 * SIZE(BO, %rax, 4), %xmm3 ;\

  261. 	movapd	%xmm7, %xmm2

  262. 

  263. #define KERNEL7(xx) \

  264. 	mulpd	%xmm5, %xmm7 ;\

  265. 	ADD1	%xmm7, %xmm8 ;\

  266. 	mulpd	 10 * SIZE(AO, %rax, 4), %xmm5 ;\

  267. 	movapd	%xmm2, %xmm7 ;\

  268. 	ADD1	%xmm5, %xmm12 ;\

  269. 	movddup	 10 * SIZE(BO, %rax, 4), %xmm5 ;\

  270. 	mulpd	%xmm3, %xmm2 ;\

  271. 	mulpd	 10 * SIZE(AO, %rax, 4), %xmm3 ;\

  272. 	ADD2	%xmm2, %xmm9 ;\

  273. 	movapd	%xmm7, %xmm2 ;\

  274. 	ADD2	%xmm3, %xmm13 ;\

  275. 	movddup	 11 * SIZE(BO, %rax, 4), %xmm3 ;\

  276. 	mulpd	%xmm5, %xmm7 ;\

  277. 	mulpd	 10 * SIZE(AO, %rax, 4), %xmm5 ;\

  278. 	ADD1	%xmm7, %xmm10 ;\

  279. 	movapd	 12 * SIZE(AO, %rax, 4), %xmm7 ;\

  280. 	ADD1	%xmm5, %xmm14 ;\

  281. 	movddup	 12 * SIZE(BO, %rax, 4), %xmm5 ;\

  282. 	mulpd	%xmm3, %xmm2 ;\

  283. 	mulpd	 10 * SIZE(AO, %rax, 4), %xmm3 ;\

  284. 	ADD2	%xmm2, %xmm11 ;\

  285. 	ADD2	%xmm3, %xmm15 ;\

  286.  	movddup	 13 * SIZE(BO, %rax, 4), %xmm3 ;\

  287. 	movapd	%xmm7, %xmm2

  288. 

  289. #define KERNEL8(xx) \

  290. 	mulpd	%xmm5, %xmm7 ;\

  291. 	ADD1	%xmm7, %xmm8 ;\

  292. 	mulpd	 14 * SIZE(AO, %rax, 4), %xmm5 ;\

  293. 	movapd	%xmm2, %xmm7 ;\

  294. 	ADD1	%xmm5, %xmm12 ;\

  295. /*A*/	movapd	 24 * SIZE(AO, %rax, 4), %xmm4 ;\

  296. 	movddup	 14 * SIZE(BO, %rax, 4), %xmm5 ;\

  297. 	mulpd	%xmm3, %xmm2 ;\

  298. 	mulpd	 14 * SIZE(AO, %rax, 4), %xmm3 ;\

  299. 	ADD2	%xmm2, %xmm9 ;\

  300. 	movapd	%xmm7, %xmm2 ;\

  301. 	ADD2	%xmm3, %xmm13 ;\

  302. 	movddup	 15 * SIZE(BO, %rax, 4), %xmm3 ;\

  303. 	mulpd	%xmm5, %xmm7 ;\

  304. 	mulpd	 14 * SIZE(AO, %rax, 4), %xmm5 ;\

  305. 	ADD1	%xmm7, %xmm10 ;\

  306. 	ADD1	%xmm5, %xmm14 ;\

  307. /**/	movddup	 24 * SIZE(BO, %rax, 4), %xmm5 ;\

  308. 	mulpd	%xmm3, %xmm2 ;\

  309. 	mulpd	 14 * SIZE(AO, %rax, 4), %xmm3 ;\

  310. 	ADD2	%xmm2, %xmm11 ;\

  311. 	ADD2	%xmm3, %xmm15 ;\

  312.  	movddup	 17 * SIZE(BO, %rax, 4), %xmm3 ;\

  313. 	movapd	%xmm0, %xmm2 ;\

  314. 	addq	$8 * SIZE, %rax ;\

  315. 

  316. #define KERNEL_SUB1(xx) \

  317. 	mulpd	%xmm1, %xmm0 ;\

  318. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1 ;\

  319. 	ADD1	%xmm0, %xmm8 ;\

  320. 	movapd	%xmm2, %xmm0 ;\

  321. 	ADD1	%xmm1, %xmm12 ;\

  322. 	movddup	-14 * SIZE(BO, %rax, 4), %xmm1 ;\

  323. 	mulpd	%xmm3, %xmm2 ;\

  324. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3 ;\

  325. 	ADD2	%xmm2, %xmm9 ;\

  326. 	movapd	%xmm0, %xmm2 ;\

  327. 	ADD2	%xmm3, %xmm13 ;\

  328. 	movddup	-13 * SIZE(BO, %rax, 4), %xmm3 ;\

  329. 	mulpd	%xmm1, %xmm0 ;\

  330. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1 ;\

  331. 	ADD1	%xmm0, %xmm10 ;\

  332. 	movapd	-12 * SIZE(AO, %rax, 4), %xmm0 ;\

  333. 	ADD1	%xmm1, %xmm14 ;\

  334. 	movddup	-12 * SIZE(BO, %rax, 4), %xmm1 ;\

  335. 	mulpd	%xmm3, %xmm2 ;\

  336. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3 ;\

  337. 	ADD2	%xmm2, %xmm11 ;\

  338. 	ADD2	%xmm3, %xmm15 ;\

  339.  	movddup	-11 * SIZE(BO, %rax, 4), %xmm3 ;\

  340. 	movapd	%xmm0, %xmm2

  341. 

  342. #define KERNEL_SUB2(xx) \

  343. 	mulpd	%xmm1, %xmm0 ;\

  344. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm1 ;\

  345. 	ADD1	%xmm0, %xmm8 ;\

  346. 	movapd	%xmm2, %xmm0 ;\

  347. 	ADD1	%xmm1, %xmm12 ;\

  348. 	movddup	-10 * SIZE(BO, %rax, 4), %xmm1 ;\

  349. 	mulpd	%xmm3, %xmm2 ;\

  350. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm3 ;\

  351. 	ADD2	%xmm2, %xmm9 ;\

  352. 	movapd	%xmm0, %xmm2 ;\

  353. 	ADD2	%xmm3, %xmm13 ;\

  354. 	movddup	 -9 * SIZE(BO, %rax, 4), %xmm3 ;\

  355. 	mulpd	%xmm1, %xmm0 ;\

  356. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm1 ;\

  357. 	ADD1	%xmm0, %xmm10 ;\

  358. 	movapd	  (AO, %rax, 4), %xmm0 ;\

  359. 	ADD1	%xmm1, %xmm14 ;\

  360. 	movddup	  (BO, %rax, 4), %xmm1 ;\

  361. 	mulpd	%xmm3, %xmm2 ;\

  362. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm3 ;\

  363. 	ADD2	%xmm2, %xmm11 ;\

  364. 	ADD2	%xmm3, %xmm15 ;\

  365.  	movddup	 -7 * SIZE(BO, %rax, 4), %xmm3 ;\

  366. 	movapd	%xmm4, %xmm2

  367. 

  368. #define KERNEL_SUB3(xx) \

  369. 	mulpd	%xmm5, %xmm4 ;\

  370. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm5 ;\

  371. 	ADD1	%xmm4, %xmm8 ;\

  372. 	movapd	%xmm2, %xmm4 ;\

  373. 	ADD1	%xmm5, %xmm12 ;\

  374. 	movddup	 -6 * SIZE(BO, %rax, 4), %xmm5 ;\

  375. 	mulpd	%xmm3, %xmm2 ;\

  376. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm3 ;\

  377. 	ADD2	%xmm2, %xmm9 ;\

  378. 	movapd	%xmm4, %xmm2 ;\

  379. 	ADD2	%xmm3, %xmm13 ;\

  380. 	movddup	 -5 * SIZE(BO, %rax, 4), %xmm3 ;\

  381. 	mulpd	%xmm5, %xmm4 ;\

  382. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm5 ;\

  383. 	ADD1	%xmm4, %xmm10 ;\

  384. 	movapd	 -4 * SIZE(AO, %rax, 4), %xmm4 ;\

  385. 	ADD1	%xmm5, %xmm14 ;\

  386. 	movddup	 -4 * SIZE(BO, %rax, 4), %xmm5 ;\

  387. 	mulpd	%xmm3, %xmm2 ;\

  388. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm3 ;\

  389. 	ADD2	%xmm2, %xmm11 ;\

  390. 	ADD2	%xmm3, %xmm15 ;\

  391.  	movddup	 -3 * SIZE(BO, %rax, 4), %xmm3 ;\

  392. 	movapd	%xmm4, %xmm2

  393. 

  394. #define KERNEL_SUB4(xx) \

  395. 	mulpd	%xmm5, %xmm4 ;\

  396. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm5 ;\

  397. 	ADD1	%xmm4, %xmm8 ;\

  398. 	movapd	%xmm2, %xmm4 ;\

  399. 	ADD1	%xmm5, %xmm12 ;\

  400. 	movddup	 -2 * SIZE(BO, %rax, 4), %xmm5 ;\

  401. 	mulpd	%xmm3, %xmm2 ;\

  402. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm3 ;\

  403. 	ADD2	%xmm2, %xmm9 ;\

  404. 	movapd	%xmm4, %xmm2 ;\

  405. 	ADD2	%xmm3, %xmm13 ;\

  406. 	movddup	 -1 * SIZE(BO, %rax, 4), %xmm3 ;\

  407. 	mulpd	%xmm5, %xmm4 ;\

  408. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm5 ;\

  409. 	ADD1	%xmm4, %xmm10 ;\

  410. 	ADD1	%xmm5, %xmm14 ;\

  411. 	mulpd	%xmm3, %xmm2 ;\

  412. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm3 ;\

  413. 	ADD2	%xmm2, %xmm11 ;\

  414. 	ADD2	%xmm3, %xmm15 ;\

  415.  	movddup	  1 * SIZE(BO, %rax, 4), %xmm3 ;\

  416. 	movapd	%xmm0, %xmm2

  417. 

  418. #if defined(OS_LINUX) && defined(CORE_BARCELONA) && !defined(TRMMKERNEL)

  419. 	.align 32768

  420. #endif

  421. 	PROLOGUE

  422. 	PROFCODE

  423. 

  424. 	subq	$STACKSIZE, %rsp

  425. 

  426. 	movq	%rbx,   (%rsp)

  427. 	movq	%rbp,  8(%rsp)

  428. 	movq	%r12, 16(%rsp)

  429. 	movq	%r13, 24(%rsp)

  430. 	movq	%r14, 32(%rsp)

  431. 	movq	%r15, 40(%rsp)

  432. 

  433. #ifdef WINDOWS_ABI

  434. 	movq	%rdi,    48(%rsp)

  435. 	movq	%rsi,    56(%rsp)

  436. 	movups	%xmm6,   64(%rsp)

  437. 	movups	%xmm7,   80(%rsp)

  438. 	movups	%xmm8,   96(%rsp)

  439. 	movups	%xmm9,  112(%rsp)

  440. 	movups	%xmm10, 128(%rsp)

  441. 	movups	%xmm11, 144(%rsp)

  442. 	movups	%xmm12, 160(%rsp)

  443. 	movups	%xmm13, 176(%rsp)

  444. 	movups	%xmm14, 192(%rsp)

  445. 	movups	%xmm15, 208(%rsp)

  446. 

  447. 	movq	ARG1,      OLD_M

  448. 	movq	ARG2,      OLD_N

  449. 	movq	ARG3,      K

  450. 	movq	OLD_A,     A

  451. 	movq	OLD_B,     B

  452. 	movq	OLD_C,     C

  453. 	movq	OLD_LDC,   LDC

  454. #ifdef TRMMKERNEL

  455. 	movsd	OLD_OFFSET, %xmm12

  456. #endif

  457. 	movaps	%xmm3,       %xmm0

  458. 	movsd	OLD_ALPHA_I, %xmm1

  459. #else

  460. 	movq	STACKSIZE +  8(%rsp), LDC

  461. #ifdef TRMMKERNEL

  462. 	movsd	STACKSIZE + 16(%rsp), %xmm12

  463. #endif

  464. 

  465. #endif

  466. 

  467. 	movq	OLD_M, M

  468. 	movq	OLD_N, N

  469. 

  470. 	movlpd	 %xmm0, ALPHA_R

  471. 	movlpd	 %xmm1, ALPHA_I

  472. 

  473. #ifdef TRMMKERNEL

  474. 	movlpd	%xmm12, OFFSET

  475. 	movlpd	%xmm12, KK

  476. #ifndef LEFT

  477. 	negq	KK

  478. #endif

  479. #endif

  480. 

  481. 	subq	$-16 * SIZE, A

  482. 	subq	$-16 * SIZE, B

  483. 

  484. 	salq	$ZBASE_SHIFT, LDC

  485. 

  486. 	movq	N,  J

  487. 	sarq	$1, J

  488. 	jle	.L100

  489. 	ALIGN_4

  490. 

  491. .L01:

  492. #if defined(TRMMKERNEL) && defined(LEFT)

  493. 	movq	OFFSET, %rax

  494. 	movq	%rax, KK

  495. #endif

  496. 

  497. 	movq	C, CO1			# coffset1 = c

  498. 	leaq	(C, LDC, 1), CO2	# coffset2 = c + ldc

  499. 

  500. 	movq	A, AO		# aoffset = a

  501. 

  502. 	movq	K, %rax

  503. 	salq	$ZBASE_SHIFT + 1, %rax

  504. 	leaq	(B, %rax), BB

  505. 

  506. 	movq	M,  I

  507. 	sarq	$1, I		# i = (m >> 2)

  508. 	jle	.L30

  509. 	ALIGN_4

  510. 

  511. .L10:

  512. #if !defined(TRMMKERNEL) || \

  513. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  514. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  515. 	movq	B, BO

  516. #else

  517. 	movq	B, BO

  518. 	movq	KK, %rax

  519. 	leaq	(, %rax, SIZE), %rax

  520. 	leaq	(AO, %rax, 4), AO

  521. 	leaq	(BO, %rax, 4), BO

  522. #endif

  523. 

  524. 	movapd	-16 * SIZE(AO), %xmm0

  525. 	movddup	-16 * SIZE(BO), %xmm1

  526. 	pxor	%xmm8, %xmm8

  527.  	movddup	-15 * SIZE(BO), %xmm3

  528. 	pxor	%xmm9, %xmm9

  529. 	movapd	 -8 * SIZE(AO), %xmm4

  530. 	pxor	%xmm10, %xmm10

  531. 	movddup	 -8 * SIZE(BO), %xmm5

  532. 	pxor	%xmm11, %xmm11

  533. 

  534. 	prefetchw      3 * SIZE(CO1)

  535. 	pxor	%xmm12, %xmm12

  536. 	pxor	%xmm13, %xmm13

  537. 	prefetchw      7 * SIZE(CO2)

  538. 	pxor	%xmm14, %xmm14

  539. 	pxor	%xmm15, %xmm15

  540. 	movapd	%xmm0, %xmm2

  541. 

  542. 	prefetch	 -16 * SIZE(BB)

  543. 

  544. #ifndef TRMMKERNEL

  545. 	movq	K, %rax

  546. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  547. 	movq	K, %rax

  548. 	subq	KK, %rax

  549. 	movq	%rax, KKK

  550. #else

  551. 	movq	KK, %rax

  552. #ifdef LEFT

  553. 	addq	$2, %rax

  554. #else

  555. 	addq	$2, %rax

  556. #endif

  557. 	movq	%rax, KKK

  558. #endif

  559. 	andq	$-8, %rax

  560. 	leaq	(, %rax, SIZE), %rax

  561. 	leaq	(AO, %rax, 4), AO

  562. 	leaq	(BO, %rax, 4), BO

  563. 	negq	%rax

  564. 	NOBRANCH

  565. 	je	.L15

  566. 	ALIGN_4

  567. 

  568. .L12:

  569. 	KERNEL1(16 *  0)

  570. 	KERNEL2(16 *  0)

  571. 	KERNEL3(16 *  0)

  572. 	KERNEL4(16 *  0)

  573. 	KERNEL5(16 *  0)

  574. 	KERNEL6(16 *  0)

  575. 	KERNEL7(16 *  0)

  576. 	KERNEL8(16 *  0)

  577. 	NOBRANCH

  578. 	je	.L15

  579. 	KERNEL1(16 *  0)

  580. 	KERNEL2(16 *  0)

  581. 	KERNEL3(16 *  0)

  582. 	KERNEL4(16 *  0)

  583. 	KERNEL5(16 *  0)

  584. 	KERNEL6(16 *  0)

  585. 	KERNEL7(16 *  0)

  586. 	KERNEL8(16 *  0)

  587. 	NOBRANCH

  588. 	je	.L15

  589. 	KERNEL1(16 *  0)

  590. 	KERNEL2(16 *  0)

  591. 	KERNEL3(16 *  0)

  592. 	KERNEL4(16 *  0)

  593. 	KERNEL5(16 *  0)

  594. 	KERNEL6(16 *  0)

  595. 	KERNEL7(16 *  0)

  596. 	KERNEL8(16 *  0)

  597. 	NOBRANCH

  598. 	je	.L15

  599. 	KERNEL1(16 *  0)

  600. 	KERNEL2(16 *  0)

  601. 	KERNEL3(16 *  0)

  602. 	KERNEL4(16 *  0)

  603. 	KERNEL5(16 *  0)

  604. 	KERNEL6(16 *  0)

  605. 	KERNEL7(16 *  0)

  606. 	KERNEL8(16 *  0)

  607. 	NOBRANCH

  608. 	je	.L15

  609. 	KERNEL1(16 *  0)

  610. 	KERNEL2(16 *  0)

  611. 	KERNEL3(16 *  0)

  612. 	KERNEL4(16 *  0)

  613. 	KERNEL5(16 *  0)

  614. 	KERNEL6(16 *  0)

  615. 	KERNEL7(16 *  0)

  616. 	KERNEL8(16 *  0)

  617. 	NOBRANCH

  618. 	je	.L15

  619. 	KERNEL1(16 *  0)

  620. 	KERNEL2(16 *  0)

  621. 	KERNEL3(16 *  0)

  622. 	KERNEL4(16 *  0)

  623. 	KERNEL5(16 *  0)

  624. 	KERNEL6(16 *  0)

  625. 	KERNEL7(16 *  0)

  626. 	KERNEL8(16 *  0)

  627. 	NOBRANCH

  628. 	je	.L15

  629. 	KERNEL1(16 *  0)

  630. 	KERNEL2(16 *  0)

  631. 	KERNEL3(16 *  0)

  632. 	KERNEL4(16 *  0)

  633. 	KERNEL5(16 *  0)

  634. 	KERNEL6(16 *  0)

  635. 	KERNEL7(16 *  0)

  636. 	KERNEL8(16 *  0)

  637. 	NOBRANCH

  638. 	je	.L15

  639. 	KERNEL1(16 *  0)

  640. 	KERNEL2(16 *  0)

  641. 	KERNEL3(16 *  0)

  642. 	KERNEL4(16 *  0)

  643. 	KERNEL5(16 *  0)

  644. 	KERNEL6(16 *  0)

  645. 	KERNEL7(16 *  0)

  646. 	KERNEL8(16 *  0)

  647. 	jl	.L12

  648. 	ALIGN_4

  649. 

  650. .L15:

  651. #ifndef TRMMKERNEL

  652. 	movq	K, %rax

  653. #else

  654. 	movq	KKK, %rax

  655. #endif

  656. 	testq	$4, %rax

  657. 	je .L16

  658. 	xorq	%rax, %rax

  659. 	ALIGN_4

  660. 

  661. 	KERNEL_SUB1(16 *  0)

  662. 	KERNEL_SUB2(16 *  0)

  663. 	KERNEL_SUB3(16 *  0)

  664. 	KERNEL_SUB4(16 *  0)

  665. 

  666. 	subq	$-16 * SIZE, BO

  667. 	subq	$-16 * SIZE, AO

  668. 	ALIGN_4

  669. 

  670. .L16:

  671. 	movddup	ALPHA_R, %xmm6

  672. 	movddup	ALPHA_I, %xmm7

  673. 

  674. #ifndef TRMMKERNEL

  675. 	movq	K, %rax

  676. #else

  677. 	movq	KKK, %rax

  678. #endif

  679. 	andq	$3, %rax

  680. 	je .L19

  681. 

  682. 	leaq	(, %rax, SIZE), %rax

  683. 	leaq	(AO, %rax, 4), AO

  684. 	leaq	(BO, %rax, 4), BO

  685. 	negq	%rax

  686. 	ALIGN_4

  687. 

  688. .L17:

  689. 	mulpd	%xmm1, %xmm0

  690. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1

  691. 	ADD1	%xmm0, %xmm8

  692. 	movapd	%xmm2, %xmm0

  693. 	ADD1	%xmm1, %xmm12

  694. 	movddup	-14 * SIZE(BO, %rax, 4), %xmm1

  695. 	mulpd	%xmm3, %xmm2

  696. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3

  697. 	ADD2	%xmm2, %xmm9

  698. 	movapd	%xmm0, %xmm2

  699. 	ADD2	%xmm3, %xmm13

  700. 	movddup	-13 * SIZE(BO, %rax, 4), %xmm3

  701. 	mulpd	%xmm1, %xmm0

  702. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1

  703. 	ADD1	%xmm0, %xmm10

  704. 	movapd	-12 * SIZE(AO, %rax, 4), %xmm0

  705. 	ADD1	%xmm1, %xmm14

  706. 	movddup	-12 * SIZE(BO, %rax, 4), %xmm1

  707. 	mulpd	%xmm3, %xmm2

  708. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm3

  709. 	ADD2	%xmm2, %xmm11

  710. 	ADD2	%xmm3, %xmm15

  711.  	movddup	-11 * SIZE(BO, %rax, 4), %xmm3

  712. 	movapd	%xmm0, %xmm2

  713. 

  714. 	addq	$SIZE, %rax

  715. 	jl	.L17

  716. 	ALIGN_4

  717. 

  718. .L19:

  719. 	prefetch	   -8 * SIZE(BB)

  720. 	subq		 $-16 * SIZE, BB

  721. 

  722. #ifndef TRMMKERNEL

  723. 	movupd	        (CO1), %xmm0

  724. 	movupd	2 * SIZE(CO1), %xmm2

  725. 	movupd	        (CO2), %xmm1

  726. 	movupd	2 * SIZE(CO2), %xmm3

  727. #endif

  728. 

  729. 	SHUFPD_1 %xmm9,  %xmm9

  730. 	SHUFPD_1 %xmm11, %xmm11

  731. 	SHUFPD_1 %xmm13, %xmm13

  732. 	SHUFPD_1 %xmm15, %xmm15

  733. 

  734. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  735.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  736. 	addsubpd %xmm9,  %xmm8

  737. 	addsubpd %xmm11, %xmm10

  738. 	addsubpd %xmm13, %xmm12

  739. 	addsubpd %xmm15, %xmm14

  740. 

  741. 	pshufd	$0x4e, %xmm8,  %xmm9

  742. 	pshufd	$0x4e, %xmm10, %xmm11

  743. 	pshufd	$0x4e, %xmm12, %xmm13

  744. 	pshufd	$0x4e, %xmm14, %xmm15

  745. 

  746. #else

  747. 	addsubpd %xmm8,  %xmm9

  748. 	addsubpd %xmm10, %xmm11

  749. 	addsubpd %xmm12, %xmm13

  750. 	addsubpd %xmm14, %xmm15

  751. 

  752. 	movapd	 %xmm9,  %xmm8

  753. 	pshufd	$0x4e, %xmm9,  %xmm9

  754. 	movapd	 %xmm11, %xmm10

  755. 	pshufd	$0x4e, %xmm11, %xmm11

  756. 	movapd	 %xmm13, %xmm12

  757. 	pshufd	$0x4e, %xmm13, %xmm13

  758. 	movapd	 %xmm15, %xmm14

  759. 	pshufd	$0x4e, %xmm15, %xmm15

  760. #endif

  761. 

  762. 	mulpd	%xmm6, %xmm8

  763. 	mulpd	%xmm7, %xmm9

  764. 	mulpd	%xmm6, %xmm10

  765. 	mulpd	%xmm7, %xmm11

  766. 

  767. 	mulpd	%xmm6, %xmm12

  768. 	mulpd	%xmm7, %xmm13

  769. 	mulpd	%xmm6, %xmm14

  770. 	mulpd	%xmm7, %xmm15

  771. 

  772. 	addsubpd %xmm9,  %xmm8

  773. 	addsubpd %xmm11, %xmm10

  774. 	addsubpd %xmm13, %xmm12

  775. 	addsubpd %xmm15, %xmm14

  776. 

  777. #ifndef TRMMKERNEL

  778. 	addpd	%xmm0,  %xmm8

  779. 	addpd	%xmm2,  %xmm12

  780. 	addpd	%xmm1,  %xmm10

  781. 	addpd	%xmm3,  %xmm14

  782. #endif

  783. 

  784. 	movlpd	%xmm8,          (CO1)

  785. 	movhpd	%xmm8,  1 * SIZE(CO1)

  786. 	movlpd	%xmm12, 2 * SIZE(CO1)

  787. 	movhpd	%xmm12, 3 * SIZE(CO1)

  788. 

  789. 	movlpd	%xmm10,         (CO2)

  790. 	movhpd	%xmm10, 1 * SIZE(CO2)

  791. 	movlpd	%xmm14, 2 * SIZE(CO2)

  792. 	movhpd	%xmm14, 3 * SIZE(CO2)

  793. 

  794. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  795.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  796. 	movq	K, %rax

  797. 	subq	KKK, %rax

  798. 	leaq	(,%rax, SIZE), %rax

  799. 	leaq	(AO, %rax, 4), AO

  800. 	leaq	(BO, %rax, 4), BO

  801. #endif

  802. 

  803. #if defined(TRMMKERNEL) && defined(LEFT)

  804. 	addq	$2, KK

  805. #endif

  806. 

  807. 	addq	$4 * SIZE, CO1		# coffset += 4

  808. 	addq	$4 * SIZE, CO2		# coffset += 4

  809. 	decq	I			# i --

  810. 	jg	.L10

  811. 	ALIGN_4

  812. 

  813. .L30:

  814. 	testq	$1, M

  815. 	jle	.L99

  816. 

  817. #if !defined(TRMMKERNEL) || \

  818. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  819. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  820. 	movq	B, BO

  821. #else

  822. 	movq	B, BO

  823. 	movq	KK, %rax

  824. 	leaq	(, %rax, SIZE), %rax

  825. 	leaq	(AO, %rax, 2), AO

  826. 	leaq	(BO, %rax, 4), BO

  827. #endif

  828. 

  829. 	movapd	-16 * SIZE(AO), %xmm0

  830. 	pxor	%xmm8, %xmm8

  831. 	movapd	-12 * SIZE(AO), %xmm2

  832. 	pxor	%xmm9, %xmm9

  833. 	movddup	-16 * SIZE(BO), %xmm1

  834. 	pxor	%xmm10, %xmm10

  835. 	movddup	-15 * SIZE(BO), %xmm5

  836. 	pxor	%xmm11, %xmm11

  837. 	movddup	 -8 * SIZE(BO), %xmm3

  838. 

  839. #ifndef TRMMKERNEL

  840. 	movq	K, %rax

  841. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  842. 	movq	K, %rax

  843. 	subq	KK, %rax

  844. 	movq	%rax, KKK

  845. #else

  846. 	movq	KK, %rax

  847. #ifdef LEFT

  848. 	addq	$1, %rax

  849. #else

  850. 	addq	$2, %rax

  851. #endif

  852. 	movq	%rax, KKK

  853. #endif

  854. 	andq	$-4, %rax

  855. 	leaq	(, %rax, SIZE), %rax

  856. 	leaq	(AO, %rax, 2), AO

  857. 	leaq	(BO, %rax, 4), BO

  858. 	negq	%rax

  859. 	NOBRANCH

  860. 	je	.L46

  861. 	ALIGN_4

  862. 

  863. .L42:

  864. 	mulpd	%xmm0, %xmm1

  865. 	ADD1	%xmm1, %xmm8

  866. 	movddup	-14 * SIZE(BO, %rax, 4), %xmm1

  867. 	mulpd	%xmm0, %xmm5

  868. 	ADD2	%xmm5, %xmm9

  869. 	movddup	-13 * SIZE(BO, %rax, 4), %xmm5

  870. 	mulpd	%xmm0, %xmm1

  871. 	ADD1	%xmm1, %xmm10

  872. 	movddup	-12 * SIZE(BO, %rax, 4), %xmm1

  873. 	mulpd	%xmm0, %xmm5

  874. 	movapd	-14 * SIZE(AO, %rax, 2), %xmm0

  875. 	ADD2	%xmm5, %xmm11

  876. 	movddup	-11 * SIZE(BO, %rax, 4), %xmm5

  877. 	mulpd	%xmm0, %xmm1

  878. 	ADD1	%xmm1, %xmm8

  879. 	movddup	-10 * SIZE(BO, %rax, 4), %xmm1

  880. 	mulpd	%xmm0, %xmm5

  881. 	ADD2	%xmm5, %xmm9

  882. 	movddup	 -9 * SIZE(BO, %rax, 4), %xmm5

  883. 	mulpd	%xmm0, %xmm1

  884. 	ADD1	%xmm1, %xmm10

  885. 	movddup	  (BO, %rax, 4), %xmm1

  886. 	mulpd	%xmm0, %xmm5

  887. 	movapd	 -8 * SIZE(AO, %rax, 2), %xmm0

  888. 	ADD2	%xmm5, %xmm11

  889. 	movddup	 -7 * SIZE(BO, %rax, 4), %xmm5

  890. 	mulpd	%xmm2, %xmm3

  891. 	ADD1	%xmm3, %xmm8

  892. 	movddup	 -6 * SIZE(BO, %rax, 4), %xmm3

  893. 	mulpd	%xmm2, %xmm5

  894. 	ADD2	%xmm5, %xmm9

  895. 	movddup	 -5 * SIZE(BO, %rax, 4), %xmm5

  896. 	mulpd	%xmm2, %xmm3

  897. 	ADD1	%xmm3, %xmm10

  898. 	movddup	 -4 * SIZE(BO, %rax, 4), %xmm3

  899. 	mulpd	%xmm2, %xmm5

  900. 	movapd	-10 * SIZE(AO, %rax, 2), %xmm2

  901. 	ADD2	%xmm5, %xmm11

  902. 	movddup	 -3 * SIZE(BO, %rax, 4), %xmm5

  903. 	mulpd	%xmm2, %xmm3

  904. 	ADD1	%xmm3, %xmm8

  905. 	movddup	 -2 * SIZE(BO, %rax, 4), %xmm3

  906. 	mulpd	%xmm2, %xmm5

  907. 	ADD2	%xmm5, %xmm9

  908. 	movddup	 -1 * SIZE(BO, %rax, 4), %xmm5

  909. 	mulpd	%xmm2, %xmm3

  910. 	ADD1	%xmm3, %xmm10

  911. 	movddup	  8 * SIZE(BO, %rax, 4), %xmm3

  912. 	mulpd	%xmm2, %xmm5

  913. 	movapd	 -4 * SIZE(AO, %rax, 2), %xmm2

  914. 	ADD2	%xmm5, %xmm11

  915. 	movddup	  1 * SIZE(BO, %rax, 4), %xmm5

  916. 

  917. 	addq	$4 * SIZE, %rax

  918. 	BRANCH

  919. 	jl	.L42

  920. 	ALIGN_4

  921. 

  922. .L46:

  923. 	movddup	ALPHA_R, %xmm6

  924. 	movddup	ALPHA_I, %xmm7

  925. 

  926. #ifndef TRMMKERNEL

  927. 	movq	K, %rax

  928. #else

  929. 	movq	KKK, %rax

  930. #endif

  931. 	andq	$3, %rax		# if (k & 1)

  932. 	je .L49

  933. 

  934. 	leaq	(, %rax, SIZE), %rax

  935. 	leaq	(AO, %rax, 2), AO

  936. 	leaq	(BO, %rax, 4), BO

  937. 	negq	%rax

  938. 	ALIGN_4

  939. 

  940. .L47:

  941. 	mulpd	%xmm0, %xmm1

  942. 	ADD1	%xmm1, %xmm8

  943. 	movddup	-14 * SIZE(BO, %rax, 4), %xmm1

  944. 	mulpd	%xmm0, %xmm5

  945. 	ADD2	%xmm5, %xmm9

  946. 	movddup	-13 * SIZE(BO, %rax, 4), %xmm5

  947. 	mulpd	%xmm0, %xmm1

  948. 	ADD1	%xmm1, %xmm10

  949. 	movddup	-12 * SIZE(BO, %rax, 4), %xmm1

  950. 	mulpd	%xmm0, %xmm5

  951. 	movapd	-14 * SIZE(AO, %rax, 2), %xmm0

  952. 	ADD2	%xmm5, %xmm11

  953. 	movddup	-11 * SIZE(BO, %rax, 4), %xmm5

  954. 

  955. 	addq	$SIZE, %rax

  956. 	jl	.L47

  957. 	ALIGN_4

  958. 

  959. .L49:

  960. #ifndef TRMMKERNEL

  961. 	movupd	(CO1), %xmm0

  962. 	movupd	(CO2), %xmm1

  963. #endif

  964. 

  965. 	SHUFPD_1 %xmm9, %xmm9

  966. 	SHUFPD_1 %xmm11, %xmm11

  967. 

  968. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  969.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  970. 	addsubpd %xmm9,  %xmm8

  971. 	addsubpd %xmm11, %xmm10

  972. 

  973. 	pshufd	$0x4e, %xmm8,  %xmm9

  974. 	pshufd	$0x4e, %xmm10, %xmm11

  975. #else

  976. 	addsubpd %xmm8,  %xmm9

  977. 	addsubpd %xmm10, %xmm11

  978. 

  979. 	movapd	 %xmm9,  %xmm8

  980. 	pshufd	$0x4e, %xmm9,  %xmm9

  981. 	movapd	 %xmm11, %xmm10

  982. 	pshufd	$0x4e, %xmm11, %xmm11

  983. #endif

  984. 

  985. 	mulpd	%xmm6, %xmm8

  986. 	mulpd	%xmm7, %xmm9

  987. 	mulpd	%xmm6, %xmm10

  988. 	mulpd	%xmm7, %xmm11

  989. 

  990. 	addsubpd %xmm9, %xmm8

  991. 	addsubpd %xmm11, %xmm10

  992. 

  993. #ifndef TRMMKERNEL

  994. 	addpd	%xmm0,  %xmm8

  995. 	addpd	%xmm1,  %xmm10

  996. #endif

  997. 

  998. 	movlpd	%xmm8,          (CO1)

  999. 	movhpd	%xmm8,  1 * SIZE(CO1)

  1000. 	movlpd	%xmm10,         (CO2)

  1001. 	movhpd	%xmm10, 1 * SIZE(CO2)

  1002. 

  1003. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1004.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1005. 	movq	K, %rax

  1006. 	subq	KKK, %rax

  1007. 	leaq	(,%rax, SIZE), %rax

  1008. 	leaq	(AO, %rax, 2), AO

  1009. 	leaq	(BO, %rax, 4), BO

  1010. #endif

  1011. 

  1012. #if defined(TRMMKERNEL) && defined(LEFT)

  1013. 	addq	$1, KK

  1014. #endif

  1015. 	ALIGN_4

  1016. 

  1017. .L99:

  1018. #if defined(TRMMKERNEL) && !defined(LEFT)

  1019. 	addl	$2, KK

  1020. #endif

  1021. 

  1022. 	movq	BO, B

  1023. 

  1024. 	leaq	(C, LDC, 2), C		# c += 2 * ldc

  1025. 	decq	J			# j --

  1026. 	jg	.L01

  1027. 

  1028. .L100:

  1029. 	testq	$1, N

  1030. 	jle	.L999

  1031. 

  1032. .L101:

  1033. #if defined(TRMMKERNEL) && defined(LEFT)

  1034. 	movq	OFFSET, %rax

  1035. 	movq	%rax, KK

  1036. #endif

  1037. 

  1038. 	movq	C, CO1		# coffset1 = c

  1039. 	movq	A, AO		# aoffset = a

  1040. 

  1041. 	movq	M,  I

  1042. 	sarq	$1, I		# i = (m >> 2)

  1043. 	jle	.L130

  1044. 	ALIGN_4

  1045. 

  1046. .L110:

  1047. #if !defined(TRMMKERNEL) || \

  1048. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1049. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1050. 	movq	B, BO

  1051. #else

  1052. 	movq	B, BO

  1053. 	movq	KK, %rax

  1054. 	leaq	(, %rax, SIZE), %rax

  1055. 	leaq	(AO, %rax, 4), AO

  1056. 	leaq	(BO, %rax, 2), BO

  1057. #endif

  1058. 

  1059. 	movddup	-16 * SIZE(BO), %xmm1

  1060. 	movddup	-15 * SIZE(BO), %xmm5

  1061. 	pxor	%xmm8, %xmm8

  1062. 	movddup	-12 * SIZE(BO), %xmm3

  1063. 	pxor	%xmm9, %xmm9

  1064. 	movapd	-16 * SIZE(AO), %xmm0

  1065. 	pxor	%xmm12, %xmm12

  1066. 	movapd	 -8 * SIZE(AO), %xmm4

  1067. 	pxor	%xmm13, %xmm13

  1068. 	prefetchw      3 * SIZE(CO1)

  1069. 	movapd	%xmm0, %xmm2

  1070. 

  1071. #ifndef TRMMKERNEL

  1072. 	movq	K, %rax

  1073. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1074. 	movq	K, %rax

  1075. 	subq	KK, %rax

  1076. 	movq	%rax, KKK

  1077. #else

  1078. 	movq	KK, %rax

  1079. #ifdef LEFT

  1080. 	addq	$2, %rax

  1081. #else

  1082. 	addq	$1, %rax

  1083. #endif

  1084. 	movq	%rax, KKK

  1085. #endif

  1086. 	andq	$-4, %rax

  1087. 	leaq	(, %rax, SIZE), %rax

  1088. 	leaq	(AO, %rax, 4), AO

  1089. 	leaq	(BO, %rax, 2), BO

  1090. 	negq	%rax

  1091. 	NOBRANCH

  1092. 	je	.L116

  1093. 	ALIGN_4

  1094. 

  1095. .L112:

  1096. 	mulpd	%xmm1, %xmm0

  1097. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1

  1098. 	ADD1	%xmm0, %xmm8

  1099. 	movapd	-12 * SIZE(AO, %rax, 4), %xmm0

  1100. 	ADD1	%xmm1, %xmm12

  1101. 	movddup	-14 * SIZE(BO, %rax, 2), %xmm1

  1102. 	mulpd	%xmm5, %xmm2

  1103. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm5

  1104. 	ADD2	%xmm2, %xmm9

  1105. 	ADD2	%xmm5, %xmm13

  1106. 	movddup	-13 * SIZE(BO, %rax, 2), %xmm5

  1107. 	movapd	%xmm0, %xmm2

  1108. 	mulpd	%xmm1, %xmm0

  1109. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm1

  1110. 	ADD1	%xmm0, %xmm8

  1111. 	movapd	  (AO, %rax, 4), %xmm0

  1112. 	ADD1	%xmm1, %xmm12

  1113. 	movddup	 -8 * SIZE(BO, %rax, 2), %xmm1

  1114. 	mulpd	%xmm5, %xmm2

  1115. 	mulpd	-10 * SIZE(AO, %rax, 4), %xmm5

  1116. 	ADD2	%xmm2, %xmm9

  1117. 	ADD2	%xmm5, %xmm13

  1118. 	movddup	-11 * SIZE(BO, %rax, 2), %xmm5

  1119. 	movapd	%xmm4, %xmm2

  1120. 	mulpd	%xmm3, %xmm4

  1121. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm3

  1122. 	ADD1	%xmm4, %xmm8

  1123. 	movapd	 -4 * SIZE(AO, %rax, 4), %xmm4

  1124. 	ADD1	%xmm3, %xmm12

  1125. 	movddup	-10 * SIZE(BO, %rax, 2), %xmm3

  1126. 	mulpd	%xmm5, %xmm2

  1127. 	mulpd	 -6 * SIZE(AO, %rax, 4), %xmm5

  1128. 	ADD2	%xmm2, %xmm9

  1129. 	ADD2	%xmm5, %xmm13

  1130. 	movddup	 -9 * SIZE(BO, %rax, 2), %xmm5

  1131. 	movapd	%xmm4, %xmm2

  1132. 	mulpd	%xmm3, %xmm4

  1133. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm3

  1134. 	ADD1	%xmm4, %xmm8

  1135. 	movapd	  8 * SIZE(AO, %rax, 4), %xmm4

  1136. 	ADD1	%xmm3, %xmm12

  1137. 	movddup	 -4 * SIZE(BO, %rax, 2), %xmm3

  1138. 	mulpd	%xmm5, %xmm2

  1139. 	mulpd	 -2 * SIZE(AO, %rax, 4), %xmm5

  1140. 	ADD2	%xmm2, %xmm9

  1141. 	ADD2	%xmm5, %xmm13

  1142. 	movddup	 -7 * SIZE(BO, %rax, 2), %xmm5

  1143. 	movapd	%xmm0, %xmm2

  1144. 

  1145. 	addq	$4 * SIZE, %rax

  1146. 	BRANCH

  1147. 	jl	.L112

  1148. 	ALIGN_4

  1149. 

  1150. .L116:

  1151. 	movddup	ALPHA_R, %xmm6

  1152. 	movddup	ALPHA_I, %xmm7

  1153. 

  1154. #ifndef TRMMKERNEL

  1155. 	movq	K, %rax

  1156. #else

  1157. 	movq	KKK, %rax

  1158. #endif

  1159. 	andq	$3, %rax		# if (k & 1)

  1160. 	je .L119

  1161. 

  1162. 	leaq	(, %rax, SIZE), %rax

  1163. 	leaq	(AO, %rax, 4), AO

  1164. 	leaq	(BO, %rax, 2), BO

  1165. 	negq	%rax

  1166. 	ALIGN_4

  1167. 

  1168. .L117:

  1169. 	mulpd	%xmm1, %xmm0

  1170. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm1

  1171. 	ADD1	%xmm0, %xmm8

  1172. 	movapd	-12 * SIZE(AO, %rax, 4), %xmm0

  1173. 	ADD1	%xmm1, %xmm12

  1174. 	movddup	-14 * SIZE(BO, %rax, 2), %xmm1

  1175. 	mulpd	%xmm5, %xmm2

  1176. 	mulpd	-14 * SIZE(AO, %rax, 4), %xmm5

  1177. 	ADD2	%xmm2, %xmm9

  1178. 	ADD2	%xmm5, %xmm13

  1179. 	movddup	-13 * SIZE(BO, %rax, 2), %xmm5

  1180. 	movapd	%xmm0, %xmm2

  1181. 

  1182. 	addq	$SIZE, %rax

  1183. 	jl	.L117

  1184. 	ALIGN_4

  1185. 

  1186. .L119:

  1187. #ifndef TRMMKERNEL

  1188. 	movupd	(CO1), %xmm0

  1189. 	movupd	2 * SIZE(CO1), %xmm2

  1190. #endif

  1191. 

  1192. 	SHUFPD_1 %xmm9, %xmm9

  1193. 	SHUFPD_1 %xmm13, %xmm13

  1194. 

  1195. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1196.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1197. 	addsubpd %xmm9,  %xmm8

  1198. 	addsubpd %xmm13, %xmm12

  1199. 

  1200. 	pshufd	$0x4e, %xmm8,  %xmm9

  1201. 	pshufd	$0x4e, %xmm12, %xmm13

  1202. 

  1203. #else

  1204. 	addsubpd %xmm8,  %xmm9

  1205. 	addsubpd %xmm12, %xmm13

  1206. 

  1207. 	movapd	 %xmm9,  %xmm8

  1208. 	pshufd	$0x4e, %xmm9,  %xmm9

  1209. 	movapd	 %xmm13, %xmm12

  1210. 	pshufd	$0x4e, %xmm13, %xmm13

  1211. #endif

  1212. 

  1213. 	mulpd	%xmm6, %xmm8

  1214. 	mulpd	%xmm7, %xmm9

  1215. 	mulpd	%xmm6, %xmm12

  1216. 	mulpd	%xmm7, %xmm13

  1217. 

  1218. 	addsubpd %xmm9, %xmm8

  1219. 	addsubpd %xmm13, %xmm12

  1220. 

  1221. #ifndef TRMMKERNEL

  1222. 	addpd	%xmm0,  %xmm8

  1223. 	addpd	%xmm2, %xmm12

  1224. #endif

  1225. 

  1226. 	movlpd	%xmm8,          (CO1)

  1227. 	movhpd	%xmm8,  1 * SIZE(CO1)

  1228. 	movlpd	%xmm12, 2 * SIZE(CO1)

  1229. 	movhpd	%xmm12, 3 * SIZE(CO1)

  1230. 

  1231. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1232.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1233. 	movq	K, %rax

  1234. 	subq	KKK, %rax

  1235. 	leaq	(,%rax, SIZE), %rax

  1236. 	leaq	(AO, %rax, 4), AO

  1237. 	leaq	(BO, %rax, 2), BO

  1238. #endif

  1239. 

  1240. #if defined(TRMMKERNEL) && defined(LEFT)

  1241. 	addq	$2, KK

  1242. #endif

  1243. 

  1244. 	addq	$4 * SIZE, CO1		# coffset += 4

  1245. 	decq	I			# i --

  1246. 	jg	.L110

  1247. 	ALIGN_4

  1248. 

  1249. .L130:

  1250. 	testq	$1, M

  1251. 	jle	.L999

  1252. 

  1253. #if !defined(TRMMKERNEL) || \

  1254. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1255. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1256. 	movq	B, BO

  1257. #else

  1258. 	movq	B, BO

  1259. 	movq	KK, %rax

  1260. 	leaq	(, %rax, SIZE), %rax

  1261. 	leaq	(AO, %rax, 2), AO

  1262. 	leaq	(BO, %rax, 2), BO

  1263. #endif

  1264. 

  1265. 	movapd	-16 * SIZE(AO), %xmm0

  1266. 	pxor	%xmm8, %xmm8

  1267. 	movapd	-12 * SIZE(AO), %xmm2

  1268. 	pxor	%xmm9, %xmm9

  1269. 	movddup	-16 * SIZE(BO), %xmm1

  1270. 	pxor	%xmm10, %xmm10

  1271. 	movddup	-15 * SIZE(BO), %xmm3

  1272. 	pxor	%xmm11, %xmm11

  1273. 

  1274. #ifndef TRMMKERNEL

  1275. 	movq	K, %rax

  1276. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1277. 	movq	K, %rax

  1278. 	subq	KK, %rax

  1279. 	movq	%rax, KKK

  1280. #else

  1281. 	movq	KK, %rax

  1282. #ifdef LEFT

  1283. 	addq	$1, %rax

  1284. #else

  1285. 	addq	$1, %rax

  1286. #endif

  1287. 	movq	%rax, KKK

  1288. #endif

  1289. 	andq	$-4, %rax

  1290. 	leaq	(, %rax, SIZE), %rax

  1291. 	leaq	(AO, %rax, 2), AO

  1292. 	leaq	(BO, %rax, 2), BO

  1293. 	negq	%rax

  1294. 	NOBRANCH

  1295. 	je	.L146

  1296. 	ALIGN_4

  1297. 

  1298. .L142:

  1299. 	mulpd	%xmm0, %xmm1

  1300. 	ADD1	%xmm1, %xmm8

  1301. 	movddup	-14 * SIZE(BO, %rax, 2), %xmm1

  1302. 	mulpd	%xmm0, %xmm3

  1303. 	movapd	-14 * SIZE(AO, %rax, 2), %xmm0

  1304. 	ADD2	%xmm3, %xmm9

  1305. 	movddup	-13 * SIZE(BO, %rax, 2), %xmm3

  1306. 	mulpd	%xmm0, %xmm1

  1307. 	ADD1	%xmm1, %xmm10

  1308. 	movddup	-12 * SIZE(BO, %rax, 2), %xmm1

  1309. 	mulpd	%xmm0, %xmm3

  1310. 	movapd	 -8 * SIZE(AO, %rax, 2), %xmm0

  1311. 	ADD2	%xmm3, %xmm11

  1312. 	movddup	-11 * SIZE(BO, %rax, 2), %xmm3

  1313. 	mulpd	%xmm2, %xmm1

  1314. 	ADD1	%xmm1, %xmm8

  1315. 	movddup	-10 * SIZE(BO, %rax, 2), %xmm1

  1316. 	mulpd	%xmm2, %xmm3

  1317. 	movapd	-10 * SIZE(AO, %rax, 2), %xmm2

  1318. 	ADD2	%xmm3, %xmm9

  1319. 	movddup	 -9 * SIZE(BO, %rax, 2), %xmm3

  1320. 	mulpd	%xmm2, %xmm1

  1321. 	ADD1	%xmm1, %xmm10

  1322. 	movddup	 -8 * SIZE(BO, %rax, 2), %xmm1

  1323. 	mulpd	%xmm2, %xmm3

  1324. 	movapd	 -4 * SIZE(AO, %rax, 2), %xmm2

  1325. 	ADD2	%xmm3, %xmm11

  1326. 	movddup	 -7 * SIZE(BO, %rax, 2), %xmm3

  1327. 

  1328. 	addq	$4 * SIZE, %rax

  1329. 	BRANCH

  1330. 	jl	.L142

  1331. 	ALIGN_4

  1332. 

  1333. .L146:

  1334. 	movddup	ALPHA_R, %xmm6

  1335. 	movddup	ALPHA_I, %xmm7

  1336. 

  1337. #ifndef TRMMKERNEL

  1338. 	movq	K, %rax

  1339. #else

  1340. 	movq	KKK, %rax

  1341. #endif

  1342. 	andq	$3, %rax		# if (k & 1)

  1343. 	je .L148

  1344. 

  1345. 	leaq	(, %rax, SIZE), %rax

  1346. 	leaq	(AO, %rax, 2), AO

  1347. 	leaq	(BO, %rax, 2), BO

  1348. 	negq	%rax

  1349. 	ALIGN_4

  1350. 

  1351. .L147:

  1352. 	mulpd	%xmm0, %xmm1

  1353. 	ADD1	%xmm1, %xmm8

  1354. 	movddup	-14 * SIZE(BO, %rax, 2), %xmm1

  1355. 	mulpd	%xmm0, %xmm3

  1356. 	movapd	-14 * SIZE(AO, %rax, 2), %xmm0

  1357. 	ADD2	%xmm3, %xmm9

  1358. 	movddup	-13 * SIZE(BO, %rax, 2), %xmm3

  1359. 

  1360. 	addq	$SIZE, %rax

  1361. 	jl	.L147

  1362. 	ALIGN_4

  1363. 

  1364. .L148:

  1365. #ifndef TRMMKERNEL

  1366. 	movupd	(CO1), %xmm0

  1367. #endif

  1368. 

  1369. 	addpd	%xmm10, %xmm8

  1370. 	addpd	%xmm11, %xmm9

  1371. 

  1372. 	SHUFPD_1 %xmm9, %xmm9

  1373. 

  1374. #if defined(NN) || defined(NT) || defined(TN) || defined(TT) || \

  1375.     defined(NR) || defined(NC) || defined(TR) || defined(TC)

  1376. 	addsubpd %xmm9,  %xmm8

  1377. 	pshufd	$0x4e, %xmm8,  %xmm9

  1378. #else

  1379. 	addsubpd %xmm8,  %xmm9

  1380. 	movapd	 %xmm9,  %xmm8

  1381. 	pshufd	$0x4e, %xmm9,  %xmm9

  1382. #endif

  1383. 

  1384. 	mulpd	%xmm6, %xmm8

  1385. 	mulpd	%xmm7, %xmm9

  1386. 

  1387. 	addsubpd %xmm9, %xmm8

  1388. 

  1389. #ifndef TRMMKERNEL

  1390. 	addpd	%xmm0,  %xmm8

  1391. #endif

  1392. 

  1393. 	movlpd	%xmm8,         (CO1)

  1394. 	movhpd	%xmm8, 1 * SIZE(CO1)

  1395. 	ALIGN_4

  1396. 

  1397. .L999:

  1398. 	movq	   (%rsp), %rbx

  1399. 	movq	  8(%rsp), %rbp

  1400. 	movq	 16(%rsp), %r12

  1401. 	movq	 24(%rsp), %r13

  1402. 	movq	 32(%rsp), %r14

  1403. 	movq	 40(%rsp), %r15

  1404. 

  1405. #ifdef WINDOWS_ABI

  1406. 	movq	 48(%rsp), %rdi

  1407. 	movq	 56(%rsp), %rsi

  1408. 	movups	 64(%rsp), %xmm6

  1409. 	movups	 80(%rsp), %xmm7

  1410. 	movups	 96(%rsp), %xmm8

  1411. 	movups	112(%rsp), %xmm9

  1412. 	movups	128(%rsp), %xmm10

  1413. 	movups	144(%rsp), %xmm11

  1414. 	movups	160(%rsp), %xmm12

  1415. 	movups	176(%rsp), %xmm13

  1416. 	movups	192(%rsp), %xmm14

  1417. 	movups	208(%rsp), %xmm15

  1418. #endif

  1419. 

  1420. 	addq	$STACKSIZE, %rsp

  1421. 	ret

  1422. 

  1423. 	EPILOGUE