OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: bfaaa975e6
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'bfaaa975e6'
${ noResults }
OpenBLAS/kernel/x86_64/zgemm_ncopy_2.S

360 lines
8.3 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. 

  44. #define M	ARG1	/* rdi */

  45. #define N	ARG2	/* rsi */

  46. #define A	ARG3	/* rdx */

  47. #define LDA	ARG4	/* rcx */

  48. #define B	ARG5	/* r8  */

  49. 

  50. #define I	%r9

  51. #define J	%r10

  52. #define AO1	%r11

  53. #define AO2	%r12

  54. #else

  55. 

  56. #define STACKSIZE 256

  57. 

  58. #define M	ARG1	/* rcx */

  59. #define N	ARG2	/* rdx */

  60. #define A	ARG3	/* r8  */

  61. #define LDA	ARG4	/* r9  */

  62. #define OLD_B		40 + 24 + STACKSIZE(%rsp)

  63. 

  64. #define B	%r10

  65. #define I	%r11

  66. #define J	%r12

  67. #define AO1	%r13

  68. #define AO2	%r14

  69. 

  70. #endif

  71. 

  72. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  73. #define RPREFETCHSIZE 16

  74. #define WPREFETCHSIZE 48

  75. #endif

  76. 

  77. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  78. #define RPREFETCHSIZE 32

  79. #define WPREFETCHSIZE 80

  80. #endif

  81. 

  82. #ifdef OPTERON

  83. #define RPREFETCHSIZE 32

  84. #define WPREFETCHSIZE 48

  85. #define movsd movlpd

  86. #endif

  87. 

  88. #if defined(BARCELONA) || defined(SHANGHAI) || defined(BULLDOZER)

  89. #define RPREFETCHSIZE 32

  90. #define WPREFETCHSIZE 48

  91. #endif

  92. 

  93. 	PROLOGUE

  94. 	PROFCODE

  95. 	

  96. #ifdef WINDOWS_ABI

  97. 	pushq	%r14

  98. 	pushq	%r13

  99. #endif

  100. 	pushq	%r12

  101. 

  102. #ifdef WINDOWS_ABI

  103. 	subq	$STACKSIZE, %rsp

  104. 

  105. 	movups	%xmm6,    0(%rsp)

  106. 	movups	%xmm7,   16(%rsp)

  107. 	movups	%xmm8,   32(%rsp)

  108. 	movups	%xmm9,   48(%rsp)

  109. 	movups	%xmm10,  64(%rsp)

  110. 	movups	%xmm11,  80(%rsp)

  111. 	movups	%xmm12,  96(%rsp)

  112. 	movups	%xmm13, 112(%rsp)

  113. 	movups	%xmm14, 128(%rsp)

  114. 	movups	%xmm15, 144(%rsp)

  115. 

  116. 	movq	OLD_B,     B

  117. #endif

  118. 

  119. 	salq	$ZBASE_SHIFT, LDA

  120. 

  121. 	movq	N,  J

  122. 	sarq	$1, J

  123. 	jle	.L20

  124. 	ALIGN_4

  125. 

  126. .L12:

  127. 	movq	A, AO1

  128. 	leaq	(A,   LDA), AO2

  129. 	leaq	(A, LDA, 2), A

  130. 

  131. 	movq	M,  I

  132. 	sarq	$2, I

  133. 	jle	.L14

  134. 	ALIGN_4

  135. 

  136. .L13:

  137. #ifdef HAVE_3DNOW

  138. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  139. 	prefetchw	(WPREFETCHSIZE +  8) * SIZE(B)

  140. #endif

  141. 

  142. #ifndef DOUBLE

  143. 	movlps	0 * SIZE(AO1), %xmm0

  144. 	movhps	0 * SIZE(AO2), %xmm0

  145. 	movlps	2 * SIZE(AO1), %xmm1

  146. 	movhps	2 * SIZE(AO2), %xmm1

  147. 

  148. 	movlps	4 * SIZE(AO1), %xmm2

  149. 	movhps	4 * SIZE(AO2), %xmm2

  150. 	movlps	6 * SIZE(AO1), %xmm3

  151. 	movhps	6 * SIZE(AO2), %xmm3

  152. 

  153. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  154. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  155. 	prefetcht0	RPREFETCHSIZE * SIZE(AO2)

  156. 

  157. 	prefetcht0	WPREFETCHSIZE * SIZE(B)

  158. #endif

  159. 

  160. 	movaps	%xmm0,   0 * SIZE(B)

  161. 	movaps	%xmm1,   4 * SIZE(B)

  162. 	movaps	%xmm2,   8 * SIZE(B)

  163. 	movaps	%xmm3,  12 * SIZE(B)

  164. 

  165. #else

  166. 	movsd	0 * SIZE(AO1), %xmm0

  167. 	movhpd	1 * SIZE(AO1), %xmm0

  168. 	movsd	0 * SIZE(AO2), %xmm1

  169. 	movhpd	1 * SIZE(AO2), %xmm1

  170. 

  171. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  172. 	prefetcht2	RPREFETCHSIZE * SIZE(AO1)

  173. #endif

  174. 

  175. 	movsd	2 * SIZE(AO1), %xmm2

  176. 	movhpd	3 * SIZE(AO1), %xmm2

  177. 	movsd	2 * SIZE(AO2), %xmm3

  178. 	movhpd	3 * SIZE(AO2), %xmm3

  179. 

  180. 	movsd	4 * SIZE(AO1), %xmm4

  181. 	movhpd	5 * SIZE(AO1), %xmm4

  182. 	movsd	4 * SIZE(AO2), %xmm5

  183. 	movhpd	5 * SIZE(AO2), %xmm5

  184. 

  185. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  186. 	prefetcht2	RPREFETCHSIZE * SIZE(AO2)

  187. #endif

  188. 

  189. 	movsd	6 * SIZE(AO1), %xmm6

  190. 	movhpd	7 * SIZE(AO1), %xmm6

  191. 	movsd	6 * SIZE(AO2), %xmm7

  192. 	movhpd	7 * SIZE(AO2), %xmm7

  193. 

  194. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  195. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  196. 	prefetcht0	RPREFETCHSIZE * SIZE(AO2)

  197. 

  198. 	prefetcht0	WPREFETCHSIZE * SIZE(B)

  199. #endif

  200. 

  201. 	movapd	%xmm0,   0 * SIZE(B)

  202. 	movapd	%xmm1,   2 * SIZE(B)

  203. 	movapd	%xmm2,   4 * SIZE(B)

  204. 	movapd	%xmm3,   6 * SIZE(B)

  205. 

  206. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  207. 	prefetcht2	WPREFETCHSIZE * SIZE(B)

  208. #endif

  209. 

  210. 	movapd	%xmm4,   8 * SIZE(B)

  211.  	movapd	%xmm5,  10 * SIZE(B)

  212. 	movapd	%xmm6,  12 * SIZE(B)

  213. 	movapd	%xmm7,  14 * SIZE(B)

  214. #endif

  215. 

  216. 	addq	$8 * SIZE, AO1

  217. 	addq	$8 * SIZE, AO2

  218. 	subq	$-16 * SIZE, B

  219. 	decq	I

  220. 	jg	.L13

  221. 	ALIGN_4

  222. 

  223. .L14:

  224. 	movq	M,  I

  225. 	andq	$3, I

  226. 	jle	.L16

  227. 	ALIGN_4

  228. 

  229. .L15:

  230. #ifndef DOUBLE

  231. 	movlps	0 * SIZE(AO1), %xmm0

  232. 	movhps	0 * SIZE(AO2), %xmm0

  233. 

  234. 	movaps	%xmm0,   0 * SIZE(B)

  235. #else

  236. 	movsd	0 * SIZE(AO1), %xmm0

  237. 	movhpd	1 * SIZE(AO1), %xmm0

  238. 	movsd	0 * SIZE(AO2), %xmm1

  239. 	movhpd	1 * SIZE(AO2), %xmm1

  240. 

  241. 	movapd	%xmm0,   0 * SIZE(B)

  242. 	movapd	%xmm1,   2 * SIZE(B)

  243. #endif

  244. 

  245. 	addq	$2 * SIZE, AO1

  246. 	addq	$2 * SIZE, AO2

  247. 	addq	$4 * SIZE, B

  248. 	decq	I

  249. 	jg	.L15

  250. 	ALIGN_4

  251. 

  252. .L16:

  253. 	decq	J

  254. 	jg	.L12

  255. 	ALIGN_4

  256. 

  257. .L20:

  258. 	testq	$1, N

  259. 	jle	.L999

  260. 

  261. 	movq	A, AO1

  262. 

  263. 	movq	M,  I

  264. 	sarq	$2, I

  265. 	jle	.L24

  266. 	ALIGN_4

  267. 

  268. .L23:

  269. #ifdef HAVE_3DNOW

  270. 	prefetchw	(WPREFETCHSIZE +  0) * SIZE(B)

  271. 	prefetchw	(WPREFETCHSIZE +  8) * SIZE(B)

  272. #endif

  273. 

  274. #ifndef DOUBLE

  275. 	movlps	0 * SIZE(AO1), %xmm0

  276. 	movhps	2 * SIZE(AO1), %xmm0

  277. 	movlps	4 * SIZE(AO1), %xmm1

  278. 	movhps	6 * SIZE(AO1), %xmm1

  279. 

  280. 	movaps	%xmm0,   0 * SIZE(B)

  281. 	movaps	%xmm1,   4 * SIZE(B)

  282. #else

  283. 	movsd	0 * SIZE(AO1), %xmm0

  284. 	movhpd	1 * SIZE(AO1), %xmm0

  285. 	movsd	2 * SIZE(AO1), %xmm1

  286. 	movhpd	3 * SIZE(AO1), %xmm1

  287. 	movsd	4 * SIZE(AO1), %xmm2

  288. 	movhpd	5 * SIZE(AO1), %xmm2

  289. 	movsd	6 * SIZE(AO1), %xmm3

  290. 	movhpd	7 * SIZE(AO1), %xmm3

  291. 

  292. 	movapd	%xmm0,   0 * SIZE(B)

  293. 	movapd	%xmm1,   2 * SIZE(B)

  294. 	movapd	%xmm2,   4 * SIZE(B)

  295. 	movapd	%xmm3,   6 * SIZE(B)

  296. #endif

  297. 

  298. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  299. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  300. 	prefetcht0	RPREFETCHSIZE * SIZE(AO2)

  301. 

  302. 	prefetcht0	WPREFETCHSIZE * SIZE(B)

  303. #endif

  304. 

  305. 	addq	$8 * SIZE, AO1

  306. 	addq	$8 * SIZE, B

  307. 	decq	I

  308. 	jg	.L23

  309. 	ALIGN_4

  310. 

  311. .L24:

  312. 	movq	M,  I

  313. 	andq	$3, I

  314. 	jle	.L999

  315. 	ALIGN_4

  316. 

  317. .L25:

  318. #ifndef DOUBLE

  319. 	movlps	0 * SIZE(AO1), %xmm0

  320. 

  321. 	movlps	%xmm0,   0 * SIZE(B)

  322. #else

  323. 	movsd	0 * SIZE(AO1), %xmm0

  324. 	movhpd	1 * SIZE(AO1), %xmm0

  325. 

  326. 	movapd	%xmm0,   0 * SIZE(B)

  327. #endif

  328. 

  329. 	addq	$2 * SIZE, AO1

  330. 	addq	$2 * SIZE, B

  331. 	decq	I

  332. 	jg	.L25

  333. 	ALIGN_4

  334. 

  335. 

  336. .L999:

  337. #ifdef WINDOWS_ABI

  338. 	movups	  0(%rsp), %xmm6

  339. 	movups	 16(%rsp), %xmm7

  340. 	movups	 32(%rsp), %xmm8

  341. 	movups	 48(%rsp), %xmm9

  342. 	movups	 64(%rsp), %xmm10

  343. 	movups	 80(%rsp), %xmm11

  344. 	movups	 96(%rsp), %xmm12

  345. 	movups	112(%rsp), %xmm13

  346. 	movups	128(%rsp), %xmm14

  347. 	movups	144(%rsp), %xmm15

  348. 

  349. 	addq	$STACKSIZE, %rsp

  350. #endif

  351. 

  352. 	popq	%r12

  353. #ifdef WINDOWS_ABI

  354. 	popq	%r13

  355. 	popq	%r14

  356. #endif

  357. 	ret

  358. 

  359. 	EPILOGUE