OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/gemm_tcopy_4_opteron.S

477 lines
9.8 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  43. #define RPREFETCHSIZE (12 + 4)

  44. #define WPREFETCHSIZE (12 + 4)

  45. #define MOVNTQ	 MOVQ

  46. #else

  47. #define RPREFETCHSIZE (12 + 4)

  48. #define WPREFETCHSIZE (12 + 4)

  49. #define MOVNTQ	 MOVQ

  50. #endif

  51. 

  52. #ifndef WINDOWS_ABI

  53. 

  54. #define M	ARG1	/* rdi */

  55. #define N	ARG2	/* rsi */

  56. #define A	ARG3	/* rdx */

  57. #define LDA	ARG4	/* rcx */

  58. #define B	ARG5	/* r8  */

  59. 

  60. #define I	%r10

  61. #define J	%rbp

  62. 

  63. #define AO1	%r9

  64. #define AO2	%r15

  65. #define AO3	%r11

  66. #define AO4	%r14

  67. #define BO1	%r13

  68. #define BO2	%r12

  69. #define M8	%rbx

  70. #define BO	%rax

  71. 

  72. #else

  73. 

  74. #define STACKSIZE 256

  75. 

  76. #define M	ARG1	/* rcx */

  77. #define N	ARG2	/* rdx */

  78. #define A	ARG3	/* r8  */

  79. #define LDA	ARG4	/* r9  */

  80. #define OLD_B		40 + 64 + STACKSIZE(%rsp)

  81. 

  82. #define B	%rdi

  83. 

  84. #define I	%r10

  85. #define J	%r11

  86. 

  87. #define AO1	%r12

  88. #define AO2	%r13

  89. #define AO3	%r14

  90. #define AO4	%r15

  91. 

  92. #define BO1	%rsi

  93. #define BO2	%rbx

  94. #define M8	%rbp

  95. #define BO	%rax

  96. 

  97. #endif

  98. 

  99. #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  100. #define RPREFETCH prefetch

  101. #else

  102. #define RPREFETCH prefetch

  103. #endif

  104. 

  105. 	PROLOGUE

  106. 	PROFCODE

  107. 

  108. #ifdef WINDOWS_ABI

  109. 	pushq	%rdi

  110. 	pushq	%rsi

  111. #endif

  112. 	pushq	%r15

  113. 	pushq	%r14

  114. 	pushq	%r13

  115. 	pushq	%r12

  116. 	pushq	%rbp

  117. 	pushq	%rbx

  118. 

  119. #ifdef WINDOWS_ABI

  120. 	subq	$STACKSIZE, %rsp

  121. 

  122. 	movups	%xmm6,    0(%rsp)

  123. 	movups	%xmm7,   16(%rsp)

  124. 	movups	%xmm8,   32(%rsp)

  125. 	movups	%xmm9,   48(%rsp)

  126. 	movups	%xmm10,  64(%rsp)

  127. 	movups	%xmm11,  80(%rsp)

  128. 	movups	%xmm12,  96(%rsp)

  129. 	movups	%xmm13, 112(%rsp)

  130. 	movups	%xmm14, 128(%rsp)

  131. 	movups	%xmm15, 144(%rsp)

  132. 

  133. 	movq	OLD_B,     B

  134. #endif

  135. 

  136. 	movq	N,    %rax

  137. 	movq	N,    %rbx

  138. 	andq	$-4,  %rax

  139. 	andq	$-2,  %rbx

  140. 	imulq	M,    %rax

  141. 	imulq	M,    %rbx

  142. 

  143. 	EMMS

  144. 

  145. 	leaq	(B, %rax, SIZE), BO1

  146. 	leaq	(B, %rbx, SIZE), BO2

  147. 

  148. 	leaq	(,   LDA, SIZE), LDA

  149. 	leaq	(,   M,   SIZE), M8

  150. 	movq	M,  J

  151. 	sarq	$2, J

  152. 	jle	.L20

  153. 	ALIGN_4

  154. 

  155. .L11:

  156. #if 0

  157. 	movq	A, AO1

  158. 	leaq	(A,   LDA, 1), AO2

  159. 	leaq	(A,   LDA, 2), AO3

  160. 	leaq	(AO2, LDA, 2), AO4

  161. 

  162. 	movq	N,  I

  163. 	sarq	$3, I

  164. 	jle	.L13

  165. 	ALIGN_4

  166. 

  167. .L12:

  168. 	MOVQ	0 * SIZE(AO1), %mm0

  169. 	addq	$8 * SIZE, AO1

  170. 	MOVQ	0 * SIZE(AO2), %mm1

  171. 	addq	$8 * SIZE, AO2

  172. 	MOVQ	0 * SIZE(AO3), %mm2

  173. 	addq	$8 * SIZE, AO3

  174. 	MOVQ	0 * SIZE(AO4), %mm3

  175. 	addq	$8 * SIZE, AO4

  176. 

  177. 	decq	I

  178. 	jg	.L12

  179. 	ALIGN_4

  180. 

  181. .L13:

  182. #endif

  183. 

  184. 	movq	A, AO1

  185. 	leaq	(A,   LDA   ), AO2

  186. 	leaq	(A,   LDA, 2), AO3

  187. 	leaq	(AO2, LDA, 2), AO4

  188. 	leaq	(A,   LDA, 4), A

  189. 

  190. 	movq	B, BO

  191. 	addq	$16 * SIZE, B

  192. 

  193. 	movq	N,  I

  194. 	sarq	$2, I

  195. 	jle	.L15

  196. 	ALIGN_4

  197. 

  198. .L14:

  199. 

  200. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO1)

  201. 

  202. 	MOVQ	0 * SIZE(AO1), %mm0

  203. 	MOVNTQ	%mm0,   0 * SIZE(BO)

  204. 	MOVQ	1 * SIZE(AO1), %mm1

  205. 	MOVNTQ	%mm1,   1 * SIZE(BO)

  206. 

  207. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO2)

  208. 

  209. 	MOVQ	2 * SIZE(AO1), %mm2

  210. 	MOVNTQ	%mm2,   2 * SIZE(BO)

  211. 	MOVQ	3 * SIZE(AO1), %mm3

  212. 	MOVNTQ	%mm3,   3 * SIZE(BO)

  213. 

  214. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  215. 	MOVQ	0 * SIZE(AO2), %mm4

  216. 	MOVNTQ	%mm4,   4 * SIZE(BO)

  217. 	MOVQ	1 * SIZE(AO2), %mm5

  218. 	MOVNTQ	%mm5,   5 * SIZE(BO)

  219. 	MOVQ	2 * SIZE(AO2), %mm6

  220. 	MOVNTQ	%mm6,   6 * SIZE(BO)

  221. 	MOVQ	3 * SIZE(AO2), %mm7

  222. 	MOVNTQ	%mm7,   7 * SIZE(BO)

  223. 

  224. 

  225. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO3)

  226. 

  227. 	MOVQ	0 * SIZE(AO3), %mm0

  228. 	MOVNTQ	%mm0,   8 * SIZE(BO)

  229. 	MOVQ	1 * SIZE(AO3), %mm1

  230. 	MOVNTQ	%mm1,   9 * SIZE(BO)

  231. 

  232. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO4)

  233. 

  234. 	MOVQ	2 * SIZE(AO3), %mm2

  235.  	MOVNTQ	%mm2,  10 * SIZE(BO)

  236. 	MOVQ	3 * SIZE(AO3), %mm3

  237.  	MOVNTQ	%mm3,  11 * SIZE(BO)

  238. 

  239. 	prefetchw	(WPREFETCHSIZE +  8) * SIZE(B)

  240. 	MOVQ	0 * SIZE(AO4), %mm4

  241. 	MOVNTQ	%mm4,  12 * SIZE(BO)

  242. 	MOVQ	1 * SIZE(AO4), %mm5

  243. 	MOVNTQ	%mm5,  13 * SIZE(BO)

  244. 	MOVQ	2 * SIZE(AO4), %mm6

  245. 	MOVNTQ	%mm6,  14 * SIZE(BO)

  246. 	MOVQ	3 * SIZE(AO4), %mm7

  247. 	MOVNTQ	%mm7,  15 * SIZE(BO)

  248. 

  249. 	addq	$4 * SIZE, AO1

  250. 	addq	$4 * SIZE, AO2

  251. 	addq	$4 * SIZE, AO3

  252. 	addq	$4 * SIZE, AO4

  253. 

  254. 	leaq	(BO, M8, 4), BO

  255. 	decq	I

  256. 	jg	.L14

  257. 	ALIGN_4

  258. 

  259. .L15:

  260. 	testq	$2, N

  261. 	jle	.L16

  262. 

  263. 	MOVQ	0 * SIZE(AO1), %mm0

  264. 	MOVQ	1 * SIZE(AO1), %mm1

  265. 	MOVQ	0 * SIZE(AO2), %mm2

  266. 	MOVQ	1 * SIZE(AO2), %mm3

  267. 

  268. 	MOVQ	0 * SIZE(AO3), %mm4

  269. 	MOVQ	1 * SIZE(AO3), %mm5

  270. 	MOVQ	0 * SIZE(AO4), %mm6

  271. 	MOVQ	1 * SIZE(AO4), %mm7

  272. 

  273. 	MOVNTQ	%mm0,   0 * SIZE(BO1)

  274. 	MOVNTQ	%mm1,   1 * SIZE(BO1)

  275. 	MOVNTQ	%mm2,   2 * SIZE(BO1)

  276. 	MOVNTQ	%mm3,   3 * SIZE(BO1)

  277. 	MOVNTQ	%mm4,   4 * SIZE(BO1)

  278. 	MOVNTQ	%mm5,   5 * SIZE(BO1)

  279. 	MOVNTQ	%mm6,   6 * SIZE(BO1)

  280. 	MOVNTQ	%mm7,   7 * SIZE(BO1)

  281. 

  282. 	addq	$2 * SIZE, AO1

  283. 	addq	$2 * SIZE, AO2

  284. 	addq	$2 * SIZE, AO3

  285. 	addq	$2 * SIZE, AO4

  286. 	addq	$8 * SIZE, BO1

  287. 	ALIGN_4

  288. 

  289. .L16:

  290. 	testq	$1, N

  291. 	jle	.L19

  292. 

  293. 	MOVQ	0 * SIZE(AO1), %mm0

  294. 	MOVQ	0 * SIZE(AO2), %mm1

  295. 	MOVQ	0 * SIZE(AO3), %mm2

  296. 	MOVQ	0 * SIZE(AO4), %mm3

  297. 

  298. 	MOVNTQ	%mm0,   0 * SIZE(BO2)

  299. 	MOVNTQ	%mm1,   1 * SIZE(BO2)

  300. 	MOVNTQ	%mm2,   2 * SIZE(BO2)

  301. 	MOVNTQ	%mm3,   3 * SIZE(BO2)

  302. 

  303. 	addq	$4 * SIZE, BO2

  304. 	ALIGN_4

  305. 

  306. .L19:

  307. 	decq	J

  308. 	jg	.L11

  309. 	ALIGN_4

  310. 

  311. .L20:

  312. 	testq	$2, M

  313. 	jle	.L30

  314. 	ALIGN_4

  315. 

  316. .L21:

  317. 	movq	A, AO1

  318. 	leaq	(A,   LDA   ), AO2

  319. 	leaq	(A,   LDA, 2), A

  320. 

  321. 	movq	B, BO

  322. 	addq	$8 * SIZE, B

  323. 

  324. 	movq	N,  I

  325. 	sarq	$2, I

  326. 	jle	.L23

  327. 	ALIGN_4

  328. 

  329. .L22:

  330. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO1)

  331. 	MOVQ	0 * SIZE(AO1), %mm0

  332. 	MOVQ	1 * SIZE(AO1), %mm1

  333. 	MOVQ	2 * SIZE(AO1), %mm2

  334. 	MOVQ	3 * SIZE(AO1), %mm3

  335. 

  336. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO2)

  337. 	MOVQ	0 * SIZE(AO2), %mm4

  338. 	MOVQ	1 * SIZE(AO2), %mm5

  339. 	MOVQ	2 * SIZE(AO2), %mm6

  340. 	MOVQ	3 * SIZE(AO2), %mm7

  341. 

  342. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  343. 	MOVNTQ	%mm0,   0 * SIZE(BO)

  344. 	MOVNTQ	%mm1,   1 * SIZE(BO)

  345. 	MOVNTQ	%mm2,   2 * SIZE(BO)

  346. 	MOVNTQ	%mm3,   3 * SIZE(BO)

  347. 	MOVNTQ	%mm4,   4 * SIZE(BO)

  348. 	MOVNTQ	%mm5,   5 * SIZE(BO)

  349. 	MOVNTQ	%mm6,   6 * SIZE(BO)

  350. 	MOVNTQ	%mm7,   7 * SIZE(BO)

  351. 

  352. 	addq	$4 * SIZE, AO1

  353. 	addq	$4 * SIZE, AO2

  354. 	leaq	(BO, M8, 4), BO

  355. 	decq	I

  356. 	jg	.L22

  357. 	ALIGN_4

  358. 

  359. .L23:

  360. 	testq	$2, N

  361. 	jle	.L24

  362. 

  363. 	MOVQ	0 * SIZE(AO1), %mm0

  364. 	MOVQ	1 * SIZE(AO1), %mm1

  365. 	MOVQ	0 * SIZE(AO2), %mm2

  366. 	MOVQ	1 * SIZE(AO2), %mm3

  367. 

  368. 	MOVNTQ	%mm0,   0 * SIZE(BO1)

  369. 	MOVNTQ	%mm1,   1 * SIZE(BO1)

  370. 	MOVNTQ	%mm2,   2 * SIZE(BO1)

  371. 	MOVNTQ	%mm3,   3 * SIZE(BO1)

  372. 

  373. 	addq	$2 * SIZE, AO1

  374. 	addq	$2 * SIZE, AO2

  375. 	addq	$4 * SIZE, BO1

  376. 	ALIGN_4

  377. 

  378. .L24:

  379. 	testq	$1, N

  380. 	jle	.L30

  381. 

  382. 	MOVQ	0 * SIZE(AO1), %mm0

  383. 	MOVQ	0 * SIZE(AO2), %mm1

  384. 

  385. 	MOVNTQ	%mm0,   0 * SIZE(BO2)

  386. 	MOVNTQ	%mm1,   1 * SIZE(BO2)

  387. 

  388. 	addq	$2 * SIZE, BO2

  389. 	ALIGN_4

  390. 

  391. .L30:

  392. 	testq	$1, M

  393. 	jle	.L999

  394. 	ALIGN_4

  395. 

  396. .L31:

  397. 	movq	A, AO1

  398. 	movq	B, BO

  399. 

  400. 	movq	N,  I

  401. 	sarq	$2, I

  402. 	jle	.L33

  403. 	ALIGN_4

  404. 

  405. .L32:

  406. 	MOVQ	0 * SIZE(AO1), %mm0

  407. 	MOVQ	1 * SIZE(AO1), %mm1

  408. 	MOVQ	2 * SIZE(AO1), %mm2

  409. 	MOVQ	3 * SIZE(AO1), %mm3

  410. 

  411. 	MOVNTQ	%mm0,   0 * SIZE(BO)

  412. 	MOVNTQ	%mm1,   1 * SIZE(BO)

  413. 	MOVNTQ	%mm2,   2 * SIZE(BO)

  414. 	MOVNTQ	%mm3,   3 * SIZE(BO)

  415. 

  416. 	addq	$4 * SIZE, AO1

  417. 	leaq	(BO, M8, 4), BO

  418. 	decq	I

  419. 	jg	.L32

  420. 	ALIGN_4

  421. 

  422. .L33:

  423. 	testq	$2, N

  424. 	jle	.L34

  425. 

  426. 	MOVQ	0 * SIZE(AO1), %mm0

  427. 	MOVQ	1 * SIZE(AO1), %mm1

  428. 

  429. 	MOVNTQ	%mm0,   0 * SIZE(BO1)

  430. 	MOVNTQ	%mm1,   1 * SIZE(BO1)

  431. 

  432. 	addq	$2 * SIZE, AO1

  433. 	addq	$2 * SIZE, BO1

  434. 	ALIGN_4

  435. 

  436. .L34:

  437. 	testq	$1, N

  438. 	jle	.L999

  439. 

  440. 	MOVQ	0 * SIZE(AO1), %mm0

  441. 	MOVNTQ	%mm0,   0 * SIZE(BO2)

  442. 

  443. 	addq	$1 * SIZE, BO2

  444. 	ALIGN_4

  445. 

  446. .L999:

  447. 	EMMS

  448. 

  449. #ifdef WINDOWS_ABI

  450. 	movups	  0(%rsp), %xmm6

  451. 	movups	 16(%rsp), %xmm7

  452. 	movups	 32(%rsp), %xmm8

  453. 	movups	 48(%rsp), %xmm9

  454. 	movups	 64(%rsp), %xmm10

  455. 	movups	 80(%rsp), %xmm11

  456. 	movups	 96(%rsp), %xmm12

  457. 	movups	112(%rsp), %xmm13

  458. 	movups	128(%rsp), %xmm14

  459. 	movups	144(%rsp), %xmm15

  460. 

  461. 	addq	$STACKSIZE, %rsp

  462. #endif

  463. 

  464. 	popq	%rbx

  465. 	popq	%rbp

  466. 	popq	%r12

  467. 	popq	%r13

  468. 	popq	%r14

  469. 	popq	%r15

  470. #ifdef WINDOWS_ABI

  471. 	popq	%rsi

  472. 	popq	%rdi

  473. #endif

  474. 	ret

  475. 

  476. 	EPILOGUE