OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_tcopy_8_bulldozer.S

668 lines
15 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define VMOVUPS_A1(OFF, ADDR, REGS) vmovups     OFF(ADDR), REGS

  43. #define VMOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS) vmovups        OFF(ADDR, BASE, SCALE), REGS

  44. 

  45. #define A_PRE 256

  46. 

  47. #ifndef WINDOWS_ABI

  48. 

  49. #define N	ARG1	/* rsi */

  50. #define M	ARG2	/* rdi */

  51. #define A	ARG3	/* rdx */

  52. #define LDA	ARG4	/* rcx */

  53. #define B	ARG5	/* r8  */

  54. 

  55. #define AO1	%r9

  56. #define AO2	%r10

  57. #define LDA3	%r11

  58. #define M8	%r12

  59. 

  60. #else

  61. 

  62. #define N	ARG1	/* rdx */

  63. #define M	ARG2	/* rcx */

  64. #define A	ARG3	/* r8  */

  65. #define LDA	ARG4	/* r9  */

  66. #define OLD_B		40 + 56(%rsp)

  67. 

  68. #define B	%r12

  69. 

  70. #define AO1	%rsi

  71. #define AO2	%rdi

  72. #define LDA3	%r10

  73. #define M8	%r11

  74. #endif

  75. 

  76. #define I	%rax

  77. 

  78. #define B0	%rbp

  79. #define	B1	%r13

  80. #define	B2	%r14

  81. #define	B3	%r15

  82. 

  83. 	PROLOGUE

  84. 	PROFCODE

  85. 

  86. #ifdef WINDOWS_ABI

  87. 	pushq	%rdi

  88. 	pushq	%rsi

  89. #endif

  90. 

  91. 	pushq	%r15

  92. 	pushq	%r14

  93. 	pushq	%r13

  94. 	pushq	%r12

  95. 	pushq	%rbp

  96. 

  97. #ifdef WINDOWS_ABI

  98. 	movq	OLD_B,     B

  99. #endif

  100. 

  101. 	subq	$-16 * SIZE, B

  102. 

  103. 	movq	M,    B1

  104. 	movq	M,    B2

  105. 	movq	M,    B3

  106. 

  107. 	andq	$-8,  B1

  108. 	andq	$-4,  B2

  109. 	andq	$-2,  B3

  110. 

  111. 	imulq	N,    B1

  112. 	imulq	N,    B2

  113. 	imulq	N,    B3

  114. 

  115. 	leaq	(B, B1, SIZE), B1

  116. 	leaq	(B, B2, SIZE), B2

  117. 	leaq	(B, B3, SIZE), B3

  118. 

  119. 	leaq	(,LDA, SIZE), LDA

  120. 	leaq	(LDA, LDA, 2), LDA3

  121. 

  122. 	leaq	(, N, SIZE), M8

  123. 

  124. 	cmpq	$8, N

  125. 	jl	.L20

  126. 	ALIGN_4

  127. 

  128. .L11:

  129. 	subq	$8, N

  130. 

  131. 	movq	A, AO1

  132. 	leaq	(A, LDA, 4), AO2

  133. 	leaq	(A, LDA, 8), A

  134. 

  135. 	movq	B, B0

  136. 	addq	$64 * SIZE, B

  137. 

  138. 	movq	M, I

  139. 	sarq	$3, I

  140. 	jle	.L14

  141. 	ALIGN_4

  142. 

  143. .L13:

  144. 

  145. 	prefetchnta	A_PRE(AO1)

  146. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  147. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  148. 	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)

  149. 	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)

  150. 

  151. 	vmovups	%xmm0,  -16 * SIZE(B0)

  152. 	vmovups	%xmm1,  -14 * SIZE(B0)

  153. 	vmovups	%xmm2,  -12 * SIZE(B0)

  154. 	vmovups	%xmm3,  -10 * SIZE(B0)

  155. 

  156. 

  157. 	prefetchnta	A_PRE(AO1, LDA, 1)

  158. 	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm0)

  159. 	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm1)

  160. 	VMOVUPS_A2(4 * SIZE, AO1, LDA, 1, %xmm2)

  161. 	VMOVUPS_A2(6 * SIZE, AO1, LDA, 1, %xmm3)

  162. 

  163. 	vmovups	%xmm0,   -8 * SIZE(B0)

  164. 	vmovups	%xmm1,   -6 * SIZE(B0)

  165. 	vmovups	%xmm2,   -4 * SIZE(B0)

  166. 	vmovups	%xmm3,   -2 * SIZE(B0)

  167. 

  168. 

  169. 	prefetchnta	A_PRE(AO1, LDA, 2)

  170. 	VMOVUPS_A2(0 * SIZE, AO1, LDA, 2, %xmm0)

  171. 	VMOVUPS_A2(2 * SIZE, AO1, LDA, 2, %xmm1)

  172. 	VMOVUPS_A2(4 * SIZE, AO1, LDA, 2, %xmm2)

  173. 	VMOVUPS_A2(6 * SIZE, AO1, LDA, 2, %xmm3)

  174. 

  175. 

  176. 	vmovups	%xmm0,    0 * SIZE(B0)

  177. 	vmovups	%xmm1,    2 * SIZE(B0)

  178. 	vmovups	%xmm2,    4 * SIZE(B0)

  179. 	vmovups	%xmm3,    6 * SIZE(B0)

  180. 

  181. 

  182. 	prefetchnta	A_PRE(AO1, LDA3, 1)

  183. 	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm0)

  184. 	VMOVUPS_A2(2 * SIZE, AO1, LDA3, 1, %xmm1)

  185. 	VMOVUPS_A2(4 * SIZE, AO1, LDA3, 1, %xmm2)

  186. 	VMOVUPS_A2(6 * SIZE, AO1, LDA3, 1, %xmm3)

  187. 

  188. 	vmovups	%xmm0,    8 * SIZE(B0)

  189. 	vmovups	%xmm1,   10 * SIZE(B0)

  190. 	vmovups	%xmm2,   12 * SIZE(B0)

  191. 	vmovups	%xmm3,   14 * SIZE(B0)

  192. 

  193. 	prefetchnta	A_PRE(AO2)

  194. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  195. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)

  196. 	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)

  197. 	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)

  198. 

  199. 	vmovups	%xmm0,   16 * SIZE(B0)

  200. 	vmovups	%xmm1,   18 * SIZE(B0)

  201. 	vmovups	%xmm2,   20 * SIZE(B0)

  202. 	vmovups	%xmm3,   22 * SIZE(B0)

  203. 

  204. 	prefetchnta	A_PRE(AO2, LDA, 1)

  205. 	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm0)

  206. 	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm1)

  207. 	VMOVUPS_A2(4 * SIZE, AO2, LDA, 1, %xmm2)

  208. 	VMOVUPS_A2(6 * SIZE, AO2, LDA, 1, %xmm3)

  209. 

  210. 	vmovups	%xmm0,   24 * SIZE(B0)

  211. 	vmovups	%xmm1,   26 * SIZE(B0)

  212. 	vmovups	%xmm2,   28 * SIZE(B0)

  213. 	vmovups	%xmm3,   30 * SIZE(B0)

  214. 

  215. 	prefetchnta	A_PRE(AO2, LDA, 2)

  216. 	VMOVUPS_A2(0 * SIZE, AO2, LDA, 2, %xmm0)

  217. 	VMOVUPS_A2(2 * SIZE, AO2, LDA, 2, %xmm1)

  218. 	VMOVUPS_A2(4 * SIZE, AO2, LDA, 2, %xmm2)

  219. 	VMOVUPS_A2(6 * SIZE, AO2, LDA, 2, %xmm3)

  220. 

  221. 	vmovups	%xmm0,   32 * SIZE(B0)

  222. 	vmovups	%xmm1,   34 * SIZE(B0)

  223. 	vmovups	%xmm2,   36 * SIZE(B0)

  224. 	vmovups	%xmm3,   38 * SIZE(B0)

  225. 

  226. 	prefetchnta	A_PRE(AO2, LDA3, 1)

  227. 	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm0)

  228. 	VMOVUPS_A2(2 * SIZE, AO2, LDA3, 1, %xmm1)

  229. 	VMOVUPS_A2(4 * SIZE, AO2, LDA3, 1, %xmm2)

  230. 	VMOVUPS_A2(6 * SIZE, AO2, LDA3, 1, %xmm3)

  231. 

  232. 	vmovups	%xmm0,   40 * SIZE(B0)

  233. 	vmovups	%xmm1,   42 * SIZE(B0)

  234. 	vmovups	%xmm2,   44 * SIZE(B0)

  235. 	vmovups	%xmm3,   46 * SIZE(B0)

  236. 

  237. 	addq	$8 * SIZE, AO1

  238. 	addq	$8 * SIZE, AO2

  239. 	leaq	(B0, M8, 8), B0

  240. 

  241. 	decq	I

  242. 	jg	.L13

  243. 	ALIGN_4

  244. 

  245. .L14:

  246. 	testq	$4, M

  247. 	jle	.L16

  248. 

  249. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  250. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  251. 	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm2)

  252. 	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm3)

  253. 

  254. 	vmovups	%xmm0,  -16 * SIZE(B1)

  255. 	vmovups	%xmm1,  -14 * SIZE(B1)

  256. 	vmovups	%xmm2,  -12 * SIZE(B1)

  257. 	vmovups	%xmm3,  -10 * SIZE(B1)

  258. 

  259. 	VMOVUPS_A2(0 * SIZE, AO1, LDA,  2, %xmm0)

  260. 	VMOVUPS_A2(2 * SIZE, AO1, LDA,  2, %xmm1)

  261. 	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm2)

  262. 	VMOVUPS_A2(2 * SIZE, AO1, LDA3, 1, %xmm3)

  263. 

  264. 	vmovups	%xmm0,   -8 * SIZE(B1)

  265. 	vmovups	%xmm1,   -6 * SIZE(B1)

  266. 	vmovups	%xmm2,   -4 * SIZE(B1)

  267. 	vmovups	%xmm3,   -2 * SIZE(B1)

  268. 

  269. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  270. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)

  271. 	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm2)

  272. 	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm3)

  273. 

  274. 	vmovups	%xmm0,    0 * SIZE(B1)

  275. 	vmovups	%xmm1,    2 * SIZE(B1)

  276. 	vmovups	%xmm2,    4 * SIZE(B1)

  277. 	vmovups	%xmm3,    6 * SIZE(B1)

  278. 

  279. 	VMOVUPS_A2(0 * SIZE, AO2, LDA,  2, %xmm0)

  280. 	VMOVUPS_A2(2 * SIZE, AO2, LDA,  2, %xmm1)

  281. 	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm2)

  282. 	VMOVUPS_A2(2 * SIZE, AO2, LDA3, 1, %xmm3)

  283. 

  284. 	vmovups	%xmm0,    8 * SIZE(B1)

  285. 	vmovups	%xmm1,   10 * SIZE(B1)

  286. 	vmovups	%xmm2,   12 * SIZE(B1)

  287. 	vmovups	%xmm3,   14 * SIZE(B1)

  288. 

  289. 	addq	$4 * SIZE, AO1

  290. 	addq	$4 * SIZE, AO2

  291. 	subq	$-32 * SIZE, B1

  292. 	ALIGN_4

  293. 

  294. .L16:

  295. 	testq	$2, M

  296. 	jle	.L18

  297. 

  298. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  299. 	VMOVUPS_A2(0 * SIZE, AO1, LDA,  1, %xmm1)

  300. 	VMOVUPS_A2(0 * SIZE, AO1, LDA,  2, %xmm2)

  301. 	VMOVUPS_A2(0 * SIZE, AO1, LDA3, 1, %xmm3)

  302. 

  303. 	vmovups	%xmm0,  -16 * SIZE(B2)

  304. 	vmovups	%xmm1,  -14 * SIZE(B2)

  305. 	vmovups	%xmm2,  -12 * SIZE(B2)

  306. 	vmovups	%xmm3,  -10 * SIZE(B2)

  307. 

  308. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  309. 	VMOVUPS_A2(0 * SIZE, AO2, LDA,  1, %xmm1)

  310. 	VMOVUPS_A2(0 * SIZE, AO2, LDA,  2, %xmm2)

  311. 	VMOVUPS_A2(0 * SIZE, AO2, LDA3, 1, %xmm3)

  312. 

  313. 	vmovups	%xmm0,   -8 * SIZE(B2)

  314. 	vmovups	%xmm1,   -6 * SIZE(B2)

  315. 	vmovups	%xmm2,   -4 * SIZE(B2)

  316. 	vmovups	%xmm3,   -2 * SIZE(B2)

  317. 

  318. 	addq	$2 * SIZE, AO1

  319. 	addq	$2 * SIZE, AO2

  320. 	subq	$-16 * SIZE, B2

  321. 	ALIGN_4

  322. 

  323. .L18:

  324. 	testq	$1, M

  325. 	jle	.L19

  326. 

  327. 	vmovsd	0 * SIZE(AO1),         %xmm0

  328. 	vmovsd	0 * SIZE(AO1, LDA),    %xmm1

  329. 	vmovsd	0 * SIZE(AO1, LDA, 2), %xmm2

  330. 	vmovsd	0 * SIZE(AO1, LDA3),   %xmm3

  331. 

  332. 	vunpcklpd %xmm1, %xmm0 , %xmm0

  333. 	vunpcklpd %xmm3, %xmm2 , %xmm2

  334. 

  335. 	vmovups	%xmm0,  -16 * SIZE(B3)

  336. 	vmovups	%xmm2,  -14 * SIZE(B3)

  337. 

  338. 	vmovsd	0 * SIZE(AO2),         %xmm0

  339. 	vmovsd	0 * SIZE(AO2, LDA),    %xmm1

  340. 	vmovsd	0 * SIZE(AO2, LDA, 2), %xmm2

  341. 	vmovsd	0 * SIZE(AO2, LDA3),   %xmm3

  342. 

  343. 	vunpcklpd %xmm1, %xmm0 , %xmm0

  344. 	vunpcklpd %xmm3, %xmm2 , %xmm2

  345. 

  346. 	vmovups	%xmm0,  -12 * SIZE(B3)

  347. 	vmovups	%xmm2,  -10 * SIZE(B3)

  348. 

  349. 	subq	$-8 * SIZE, B3

  350. 	ALIGN_4

  351. 

  352. .L19:

  353. 	cmpq	$8, N

  354. 	jge	.L11

  355. 	ALIGN_4

  356. 

  357. .L20:

  358. 	cmpq	$4, N

  359. 	jl	.L30

  360. 

  361. 	subq	$4, N

  362. 

  363. 	movq	A, AO1

  364. 	leaq	(A, LDA, 2), AO2

  365. 	leaq	(A, LDA, 4), A

  366. 

  367. 	movq	B, B0

  368. 	addq	$32 * SIZE, B

  369. 

  370. 	movq	M, I

  371. 	sarq	$3, I

  372. 	jle	.L24

  373. 	ALIGN_4

  374. 

  375. .L23:

  376. 

  377. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  378. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  379. 	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)

  380. 	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)

  381. 

  382. 	vmovups	%xmm0,  -16 * SIZE(B0)

  383. 	vmovups	%xmm1,  -14 * SIZE(B0)

  384. 	vmovups	%xmm2,  -12 * SIZE(B0)

  385. 	vmovups	%xmm3,  -10 * SIZE(B0)

  386. 

  387. 

  388. 	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm0)

  389. 	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm1)

  390. 	VMOVUPS_A2(4 * SIZE, AO1, LDA, 1, %xmm2)

  391. 	VMOVUPS_A2(6 * SIZE, AO1, LDA, 1, %xmm3)

  392. 

  393. 	vmovups	%xmm0,   -8 * SIZE(B0)

  394. 	vmovups	%xmm1,   -6 * SIZE(B0)

  395. 	vmovups	%xmm2,   -4 * SIZE(B0)

  396. 	vmovups	%xmm3,   -2 * SIZE(B0)

  397. 

  398. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  399. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)

  400. 	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)

  401. 	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)

  402. 

  403. 	vmovups	%xmm0,    0 * SIZE(B0)

  404. 	vmovups	%xmm1,    2 * SIZE(B0)

  405. 	vmovups	%xmm2,    4 * SIZE(B0)

  406. 	vmovups	%xmm3,    6 * SIZE(B0)

  407. 

  408. 	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm0)

  409. 	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm1)

  410. 	VMOVUPS_A2(4 * SIZE, AO2, LDA, 1, %xmm2)

  411. 	VMOVUPS_A2(6 * SIZE, AO2, LDA, 1, %xmm3)

  412. 

  413. 	vmovups	%xmm0,    8 * SIZE(B0)

  414. 	vmovups	%xmm1,   10 * SIZE(B0)

  415. 	vmovups	%xmm2,   12 * SIZE(B0)

  416. 	vmovups	%xmm3,   14 * SIZE(B0)

  417. 

  418. 	addq	$8 * SIZE, AO1

  419. 	addq	$8 * SIZE, AO2

  420. 	leaq	(B0, M8, 8), B0

  421. 

  422. 	decq	I

  423. 	jg	.L23

  424. 	ALIGN_4

  425. 

  426. .L24:

  427. 	testq	$4, M

  428. 	jle	.L26

  429. 

  430. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  431. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  432. 	VMOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm2)

  433. 	VMOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm3)

  434. 

  435. 	vmovups	%xmm0,  -16 * SIZE(B1)

  436. 	vmovups	%xmm1,  -14 * SIZE(B1)

  437. 	vmovups	%xmm2,  -12 * SIZE(B1)

  438. 	vmovups	%xmm3,  -10 * SIZE(B1)

  439. 

  440. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  441. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)

  442. 	VMOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm2)

  443. 	VMOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm3)

  444. 

  445. 	vmovups	%xmm0,   -8 * SIZE(B1)

  446. 	vmovups	%xmm1,   -6 * SIZE(B1)

  447. 	vmovups	%xmm2,   -4 * SIZE(B1)

  448. 	vmovups	%xmm3,   -2 * SIZE(B1)

  449. 

  450. 	addq	$4 * SIZE, AO1

  451. 	addq	$4 * SIZE, AO2

  452. 	subq	$-16 * SIZE, B1

  453. 	ALIGN_4

  454. 

  455. .L26:

  456. 	testq	$2, M

  457. 	jle	.L28

  458. 

  459. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  460. 	VMOVUPS_A2(0 * SIZE, AO1, LDA,  1, %xmm1)

  461. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm2)

  462. 	VMOVUPS_A2(0 * SIZE, AO2, LDA,  1, %xmm3)

  463. 

  464. 	vmovups	%xmm0,  -16 * SIZE(B2)

  465. 	vmovups	%xmm1,  -14 * SIZE(B2)

  466. 	vmovups	%xmm2,  -12 * SIZE(B2)

  467. 	vmovups	%xmm3,  -10 * SIZE(B2)

  468. 

  469. 	addq	$2 * SIZE, AO1

  470. 	addq	$2 * SIZE, AO2

  471. 	subq	$-8 * SIZE, B2

  472. 	ALIGN_4

  473. 

  474. .L28:

  475. 	testq	$1, M

  476. 	jle	.L30

  477. 

  478. 	vmovsd	0 * SIZE(AO1),      %xmm0

  479. 	vmovsd	0 * SIZE(AO1, LDA), %xmm1

  480. 	vmovsd	0 * SIZE(AO2),      %xmm2

  481. 	vmovsd	0 * SIZE(AO2, LDA), %xmm3

  482. 

  483. 	vunpcklpd %xmm1, %xmm0, %xmm0

  484. 	vunpcklpd %xmm3, %xmm2, %xmm2

  485. 

  486. 	vmovups	%xmm0,  -16 * SIZE(B3)

  487. 	vmovups	%xmm2,  -14 * SIZE(B3)

  488. 	subq	$-4 * SIZE, B3

  489. 	ALIGN_4

  490. 

  491. .L30:

  492. 	cmpq	$2, N

  493. 	jl	.L40

  494. 

  495. 	subq	$2, N

  496. 

  497. 	movq	A, AO1

  498. 	leaq	(A, LDA), AO2

  499. 	leaq	(A, LDA, 2), A

  500. 

  501. 	movq	B, B0

  502. 	addq	$16 * SIZE, B

  503. 

  504. 	movq	M, I

  505. 	sarq	$3, I

  506. 	jle	.L34

  507. 	ALIGN_4

  508. 

  509. .L33:

  510. 

  511. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  512. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  513. 	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)

  514. 	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)

  515. 

  516. 	vmovups	%xmm0,  -16 * SIZE(B0)

  517. 	vmovups	%xmm1,  -14 * SIZE(B0)

  518. 	vmovups	%xmm2,  -12 * SIZE(B0)

  519. 	vmovups	%xmm3,  -10 * SIZE(B0)

  520. 

  521. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm0)

  522. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm1)

  523. 	VMOVUPS_A1(4 * SIZE, AO2, %xmm2)

  524. 	VMOVUPS_A1(6 * SIZE, AO2, %xmm3)

  525. 

  526. 	vmovups	%xmm0,   -8 * SIZE(B0)

  527. 	vmovups	%xmm1,   -6 * SIZE(B0)

  528. 	vmovups	%xmm2,   -4 * SIZE(B0)

  529. 	vmovups	%xmm3,   -2 * SIZE(B0)

  530. 

  531. 	addq	$8 * SIZE, AO1

  532. 	addq	$8 * SIZE, AO2

  533. 	leaq	(B0, M8, 8), B0

  534. 

  535. 	decq	I

  536. 	jg	.L33

  537. 	ALIGN_4

  538. 

  539. .L34:

  540. 	testq	$4, M

  541. 	jle	.L36

  542. 

  543. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  544. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  545. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm2)

  546. 	VMOVUPS_A1(2 * SIZE, AO2, %xmm3)

  547. 

  548. 	vmovups	%xmm0,  -16 * SIZE(B1)

  549. 	vmovups	%xmm1,  -14 * SIZE(B1)

  550. 	vmovups	%xmm2,  -12 * SIZE(B1)

  551. 	vmovups	%xmm3,  -10 * SIZE(B1)

  552. 

  553. 	addq	$4 * SIZE, AO1

  554. 	addq	$4 * SIZE, AO2

  555. 	subq	$-8 * SIZE, B1

  556. 	ALIGN_4

  557. 

  558. .L36:

  559. 	testq	$2, M

  560. 	jle	.L38

  561. 

  562. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  563. 	VMOVUPS_A1(0 * SIZE, AO2, %xmm1)

  564. 

  565. 	vmovups	%xmm0,  -16 * SIZE(B2)

  566. 	vmovups	%xmm1,  -14 * SIZE(B2)

  567. 

  568. 	addq	$2 * SIZE, AO1

  569. 	addq	$2 * SIZE, AO2

  570. 	subq	$-4 * SIZE, B2

  571. 	ALIGN_4

  572. 

  573. .L38:

  574. 	testq	$1, M

  575. 	jle	.L40

  576. 

  577. 	vmovsd	0 * SIZE(AO1),      %xmm0

  578. 	vmovsd	0 * SIZE(AO2),      %xmm1

  579. 

  580. 	vunpcklpd %xmm1, %xmm0, %xmm0

  581. 

  582. 	vmovups	%xmm0,  -16 * SIZE(B3)

  583. 	subq	$-2 * SIZE, B3

  584. 	ALIGN_4

  585. 

  586. .L40:

  587. 	cmpq	$1, N

  588. 	jl	.L999

  589. 

  590. 	movq	A, AO1

  591. 

  592. 	movq	B, B0

  593. 

  594. 	movq	M, I

  595. 	sarq	$3, I

  596. 	jle	.L44

  597. 	ALIGN_4

  598. 

  599. .L43:

  600. 

  601. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  602. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  603. 	VMOVUPS_A1(4 * SIZE, AO1, %xmm2)

  604. 	VMOVUPS_A1(6 * SIZE, AO1, %xmm3)

  605. 

  606. 	vmovups	%xmm0,  -16 * SIZE(B0)

  607. 	vmovups	%xmm1,  -14 * SIZE(B0)

  608. 	vmovups	%xmm2,  -12 * SIZE(B0)

  609. 	vmovups	%xmm3,  -10 * SIZE(B0)

  610. 

  611. 	addq	$8 * SIZE, AO1

  612. 	leaq	(B0, M8, 8), B0

  613. 

  614. 	decq	I

  615. 	jg	.L43

  616. 	ALIGN_4

  617. 

  618. .L44:

  619. 	testq	$4, M

  620. 	jle	.L45

  621. 

  622. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  623. 	VMOVUPS_A1(2 * SIZE, AO1, %xmm1)

  624. 

  625. 	vmovups	%xmm0, -16 * SIZE(B1)

  626. 	vmovups	%xmm1, -14 * SIZE(B1)

  627. 

  628. 	addq	$4 * SIZE, AO1

  629. 	subq	$-4 * SIZE, B1

  630. 	ALIGN_4

  631. 

  632. .L45:

  633. 	testq	$2, M

  634. 	jle	.L46

  635. 

  636. 	VMOVUPS_A1(0 * SIZE, AO1, %xmm0)

  637. 

  638. 	vmovups	%xmm0,  -16 * SIZE(B2)

  639. 

  640. 	addq	$2 * SIZE, AO1

  641. 	subq	$-2 * SIZE, B2

  642. 	ALIGN_4

  643. 

  644. .L46:

  645. 	testq	$1, M

  646. 	jle	.L999

  647. 

  648. 	vmovsd	0 * SIZE(AO1),      %xmm0

  649. 

  650. 	vmovsd	%xmm0,  -16 * SIZE(B3)

  651. 	jmp	.L999

  652. 	ALIGN_4

  653. 

  654. .L999:

  655. 	popq	%rbp

  656. 	popq	%r12

  657. 	popq	%r13

  658. 	popq	%r14

  659. 	popq	%r15

  660. 

  661. #ifdef WINDOWS_ABI

  662. 	popq	%rsi

  663. 	popq	%rdi

  664. #endif

  665. 	ret

  666. 

  667. 	EPILOGUE