OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfaaa975e6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfaaa975e6'
${ noResults }
OpenBLAS/kernel/x86_64/gemm_ncopy_4_opteron.S

389 lines
8.3 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  43. #define RPREFETCHSIZE (12 + 4)

  44. #define WPREFETCHSIZE (48 + 4)

  45. #define MOVNTQ	 MOVQ

  46. #else

  47. #define RPREFETCHSIZE (12 + 4)

  48. #define WPREFETCHSIZE (24 + 4)

  49. #define MOVNTQ	 MOVQ

  50. #endif

  51. 

  52. #ifndef WINDOWS_ABI

  53. 

  54. #define M	ARG1	/* rdi */

  55. #define N	ARG2	/* rsi */

  56. #define A	ARG3	/* rdx */

  57. #define LDA	ARG4	/* rcx */

  58. #define B	ARG5	/* r8  */

  59. 

  60. #define I	%r9

  61. #else

  62. 

  63. #define STACKSIZE 256

  64. 

  65. #define M	ARG1	/* rcx */

  66. #define N	ARG2	/* rdx */

  67. #define A	ARG3	/* r8  */

  68. #define LDA	ARG4	/* r9  */

  69. #define OLD_B		40 + 32 + STACKSIZE(%rsp)

  70. 

  71. #define B	%r14

  72. #define I	%r15

  73. 

  74. #endif

  75. 

  76. #define J	%r10

  77. #define AO1	%r11

  78. #define AO2	%r12

  79. #define AO3	%r13

  80. #define AO4	%rax

  81. 

  82. #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  83. #define RPREFETCH prefetch

  84. #else

  85. #define RPREFETCH prefetch

  86. #endif

  87. 

  88. 	PROLOGUE

  89. 	PROFCODE

  90. 	

  91. #ifdef WINDOWS_ABI

  92. 	pushq	%r15

  93. 	pushq	%r14

  94. #endif

  95. 	pushq	%r13

  96. 	pushq	%r12

  97. 

  98. #ifdef WINDOWS_ABI

  99. 	subq	$STACKSIZE, %rsp

  100. 

  101. 	movups	%xmm6,    0(%rsp)

  102. 	movups	%xmm7,   16(%rsp)

  103. 	movups	%xmm8,   32(%rsp)

  104. 	movups	%xmm9,   48(%rsp)

  105. 	movups	%xmm10,  64(%rsp)

  106. 	movups	%xmm11,  80(%rsp)

  107. 	movups	%xmm12,  96(%rsp)

  108. 	movups	%xmm13, 112(%rsp)

  109. 	movups	%xmm14, 128(%rsp)

  110. 	movups	%xmm15, 144(%rsp)

  111. 

  112. 	movq	OLD_B,     B

  113. #endif

  114. 

  115. 	EMMS

  116. 

  117. 	leaq	(,LDA, SIZE), LDA		# Scaling

  118. 

  119. 	movq	N,  J

  120. 	sarq	$2, J

  121. 	jle	.L20

  122. 	ALIGN_4

  123. 

  124. .L11:

  125. #if 0

  126. 	movq	A, AO1

  127. 	leaq	(A,   LDA, 1), AO2

  128. 	leaq	(A,   LDA, 2), AO3

  129. 	leaq	(AO2, LDA, 2), AO4

  130. 

  131. 	movq	M,  I

  132. 	sarq	$4, I

  133. 	jle	.L13

  134. 	ALIGN_4

  135. 

  136. .L12:

  137. 	MOVQ	0 * SIZE(AO1), %mm0

  138. 	addq	$8 * SIZE, AO1

  139. 	MOVQ	0 * SIZE(AO2), %mm1

  140. 	addq	$8 * SIZE, AO2

  141. 	MOVQ	0 * SIZE(AO3), %mm2

  142. 	addq	$8 * SIZE, AO3

  143. 	MOVQ	0 * SIZE(AO4), %mm3

  144. 	addq	$8 * SIZE, AO4

  145. 

  146. 	decq	I

  147. 	jg	.L12

  148. 	ALIGN_4

  149. 

  150. .L13:

  151. #endif

  152. 

  153. 	movq	A, AO1

  154. 	leaq	(A,   LDA), AO2

  155. 	leaq	(A,   LDA, 2), AO3

  156. 	leaq	(AO2, LDA, 2), AO4

  157. 	leaq	(A, LDA, 4), A

  158. 

  159. 	movq	M,  I

  160. 	sarq	$2, I

  161. 	jle	.L15

  162. 	ALIGN_4

  163. 

  164. .L14:

  165. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO1)

  166. 

  167. 	MOVQ	0 * SIZE(AO1), %mm0

  168. 	MOVNTQ	%mm0,   0 * SIZE(B)

  169. 	MOVQ	0 * SIZE(AO2), %mm1

  170. 	MOVNTQ	%mm1,   1 * SIZE(B)

  171. 

  172. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO2)

  173. 

  174. 	MOVQ	0 * SIZE(AO3), %mm2

  175. 	MOVNTQ	%mm2,   2 * SIZE(B)

  176. 	MOVQ	0 * SIZE(AO4), %mm3

  177. 	MOVNTQ	%mm3,   3 * SIZE(B)

  178. 

  179. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  180. 	MOVQ	1 * SIZE(AO1), %mm4

  181. 	MOVNTQ	%mm4,   4 * SIZE(B)

  182. 	MOVQ	1 * SIZE(AO2), %mm5

  183. 	MOVNTQ	%mm5,   5 * SIZE(B)

  184. 	MOVQ	1 * SIZE(AO3), %mm6

  185. 	MOVNTQ	%mm6,   6 * SIZE(B)

  186. 	MOVQ	1 * SIZE(AO4), %mm7

  187. 	MOVNTQ	%mm7,   7 * SIZE(B)

  188. 

  189. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO3)

  190. 

  191. 	MOVQ	2 * SIZE(AO1), %mm0

  192. 	MOVNTQ	%mm0,   8 * SIZE(B)

  193. 	MOVQ	2 * SIZE(AO2), %mm1

  194. 	MOVNTQ	%mm1,   9 * SIZE(B)

  195. 

  196. 	RPREFETCH	(RPREFETCHSIZE) * SIZE(AO4)

  197. 

  198. 	MOVQ	2 * SIZE(AO3), %mm2

  199.  	MOVNTQ	%mm2,  10 * SIZE(B)

  200. 	MOVQ	2 * SIZE(AO4), %mm3

  201.  	MOVNTQ	%mm3,  11 * SIZE(B)

  202. 

  203. 	prefetchw	(WPREFETCHSIZE +  8) * SIZE(B)

  204. 	MOVQ	3 * SIZE(AO1), %mm4

  205. 	MOVNTQ	%mm4,  12 * SIZE(B)

  206. 	MOVQ	3 * SIZE(AO2), %mm5

  207. 	MOVNTQ	%mm5,  13 * SIZE(B)

  208. 	MOVQ	3 * SIZE(AO3), %mm6

  209. 	MOVNTQ	%mm6,  14 * SIZE(B)

  210. 	MOVQ	3 * SIZE(AO4), %mm7

  211. 	MOVNTQ	%mm7,  15 * SIZE(B)

  212. 

  213. 	addq	$4 * SIZE, AO1

  214. 	addq	$4 * SIZE, AO2

  215. 	addq	$4 * SIZE, AO3

  216. 	addq	$4 * SIZE, AO4

  217. 

  218. 	subq	$-16 * SIZE, B

  219. 	decq	I

  220. 	jg	.L14

  221. 	ALIGN_4

  222. 

  223. .L15:

  224. 	movq	M,  I

  225. 	andq	$3, I

  226. 	jle	.L17

  227. 	ALIGN_4

  228. 

  229. .L16:

  230. 	MOVQ	0 * SIZE(AO1), %mm0

  231. 	MOVQ	0 * SIZE(AO2), %mm1

  232. 	MOVQ	0 * SIZE(AO3), %mm2

  233. 	MOVQ	0 * SIZE(AO4), %mm3

  234. 

  235. 	MOVNTQ	%mm0,   0 * SIZE(B)

  236. 	MOVNTQ	%mm1,   1 * SIZE(B)

  237. 	MOVNTQ	%mm2,   2 * SIZE(B)

  238. 	MOVNTQ	%mm3,   3 * SIZE(B)

  239. 

  240. 	addq	$SIZE, AO1

  241. 	addq	$SIZE, AO2

  242. 	addq	$SIZE, AO3

  243. 	addq	$SIZE, AO4

  244. 	addq	$4 * SIZE, B

  245. 	decq	I

  246. 	jg	.L16

  247. 	ALIGN_4

  248. 

  249. .L17:

  250. 	decq	J

  251. 	jg	.L11

  252. 	ALIGN_4

  253. 

  254. .L20:

  255. 	testq	$2, N

  256. 	jle	.L30

  257. 

  258. 	movq	A, AO1

  259. 	leaq	(A,   LDA), AO2

  260. 	leaq	(A, LDA, 2), A

  261. 

  262. 	movq	M,  I

  263. 	sarq	$2, I

  264. 	jle	.L24

  265. 	ALIGN_4

  266. 

  267. .L23:

  268. 	prefetch	(RPREFETCHSIZE) * SIZE(AO1)

  269. 	MOVQ	0 * SIZE(AO1), %mm0

  270. 	prefetch	(RPREFETCHSIZE) * SIZE(AO2)

  271. 	MOVQ	0 * SIZE(AO2), %mm1

  272. 	MOVQ	1 * SIZE(AO1), %mm2

  273. 	MOVQ	1 * SIZE(AO2), %mm3

  274. 	MOVQ	2 * SIZE(AO1), %mm4

  275. 	MOVQ	2 * SIZE(AO2), %mm5

  276. 	MOVQ	3 * SIZE(AO1), %mm6

  277. 	MOVQ	3 * SIZE(AO2), %mm7

  278. 

  279. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  280. 

  281. 	MOVNTQ	%mm0,   0 * SIZE(B)

  282. 	MOVNTQ	%mm1,   1 * SIZE(B)

  283. 	MOVNTQ	%mm2,   2 * SIZE(B)

  284. 	MOVNTQ	%mm3,   3 * SIZE(B)

  285. 	MOVNTQ	%mm4,   4 * SIZE(B)

  286. 	MOVNTQ	%mm5,   5 * SIZE(B)

  287. 	MOVNTQ	%mm6,   6 * SIZE(B)

  288. 	MOVNTQ	%mm7,   7 * SIZE(B)

  289. 

  290. 	addq	$4 * SIZE, AO1

  291. 	addq	$4 * SIZE, AO2

  292. 	subq	$-8 * SIZE, B

  293. 	decq	I

  294. 	jg	.L23

  295. 	ALIGN_4

  296. 

  297. .L24:

  298. 	movq	M,  I

  299. 	andq	$3, I

  300. 	jle	.L30

  301. 	ALIGN_4

  302. 

  303. .L25:

  304. 	MOVQ	0 * SIZE(AO1), %mm0

  305. 	MOVQ	0 * SIZE(AO2), %mm1

  306. 

  307. 	MOVNTQ	%mm0,   0 * SIZE(B)

  308. 	MOVNTQ	%mm1,   1 * SIZE(B)

  309. 

  310. 	addq	$SIZE, AO1

  311. 	addq	$SIZE, AO2

  312. 	addq	$2 * SIZE, B

  313. 	decq	I

  314. 	jg	.L25

  315. 	ALIGN_4

  316. 

  317. .L30:

  318. 	testq	$1, N

  319. 	jle	.L999

  320. 

  321. 	movq	A, AO1

  322. 

  323. 	movq	M,  I

  324. 	sarq	$2, I

  325. 	jle	.L34

  326. 	ALIGN_4

  327. 

  328. .L33:

  329. 	MOVQ	0 * SIZE(AO1), %mm0

  330. 	MOVQ	1 * SIZE(AO1), %mm1

  331. 	MOVQ	2 * SIZE(AO1), %mm2

  332. 	MOVQ	3 * SIZE(AO1), %mm3

  333. 

  334. 	MOVNTQ	%mm0,   0 * SIZE(B)

  335. 	MOVNTQ	%mm1,   1 * SIZE(B)

  336. 	MOVNTQ	%mm2,   2 * SIZE(B)

  337. 	MOVNTQ	%mm3,   3 * SIZE(B)

  338. 

  339. 	addq	$4 * SIZE, AO1

  340. 	subq	$-4 * SIZE, B

  341. 	decq	I

  342. 	jg	.L33

  343. 	ALIGN_4

  344. 

  345. .L34:

  346. 	movq	M,  I

  347. 	andq	$3, I

  348. 	jle	.L999

  349. 	ALIGN_4

  350. 

  351. .L35:

  352. 	MOVQ	0 * SIZE(AO1), %mm0

  353. 	addq	$SIZE, AO1

  354. 

  355. 	MOVNTQ	%mm0,   0 * SIZE(B)

  356. 	addq	$1 * SIZE, B

  357. 	decq	I

  358. 	jg	.L35

  359. 	ALIGN_4

  360. 

  361. 

  362. .L999:

  363. 	EMMS

  364. 	

  365. #ifdef WINDOWS_ABI

  366. 	movups	  0(%rsp), %xmm6

  367. 	movups	 16(%rsp), %xmm7

  368. 	movups	 32(%rsp), %xmm8

  369. 	movups	 48(%rsp), %xmm9

  370. 	movups	 64(%rsp), %xmm10

  371. 	movups	 80(%rsp), %xmm11

  372. 	movups	 96(%rsp), %xmm12

  373. 	movups	112(%rsp), %xmm13

  374. 	movups	128(%rsp), %xmm14

  375. 	movups	144(%rsp), %xmm15

  376. 

  377. 	addq	$STACKSIZE, %rsp

  378. #endif

  379. 

  380. 	popq	%r12

  381. 	popq	%r13

  382. #ifdef WINDOWS_ABI

  383. 	popq	%r14

  384. 	popq	%r15

  385. #endif

  386. 	ret

  387. 

  388. 	EPILOGUE