OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 549a9f1dbb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '549a9f1dbb'
${ noResults }
OpenBLAS/kernel/x86_64/zgemm_tcopy_2.S

433 lines
9.5 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define RPREFETCHSIZE 32

  43. #define WPREFETCHSIZE 16

  44. 

  45. #ifndef WINDOWS_ABI

  46. 

  47. #define M	ARG1	/* rdi */

  48. #define N	ARG2	/* rsi */

  49. #define A	ARG3	/* rdx */

  50. #define LDA	ARG4	/* rcx */

  51. #define B	ARG5	/* r8  */

  52. 

  53. #define I	%r9

  54. #define J	%r10

  55. 

  56. #define AO1	%r11

  57. #define AO2	%r12

  58. #define BO1	%r13

  59. #define M8	%r14

  60. #define BO	%rax

  61. 

  62. #else

  63. 

  64. #define STACKSIZE 256

  65. 

  66. #define M	ARG1	/* rcx */

  67. #define N	ARG2	/* rdx */

  68. #define A	ARG3	/* r8  */

  69. #define LDA	ARG4	/* r9  */

  70. #define OLD_B		40 + 48 + STACKSIZE(%rsp)

  71. 

  72. #define B	%r10

  73. 

  74. #define I	%r11

  75. #define J	%r12

  76. 

  77. #define AO1	%r13

  78. #define AO2	%r14

  79. 

  80. #define BO1	%rdi

  81. #define M8	%rsi

  82. #define BO	%rax

  83. 

  84. #endif

  85. 

  86. 	PROLOGUE

  87. 	PROFCODE

  88. 

  89. #ifdef WINDOWS_ABI

  90. 	pushq	%rdi

  91. 	pushq	%rsi

  92. #endif

  93. 	pushq	%r14

  94. 	pushq	%r13

  95. 	pushq	%r12

  96. 	pushq	%r11

  97. 

  98. #ifdef WINDOWS_ABI

  99. 	subq	$STACKSIZE, %rsp

  100. 

  101. 	movups	%xmm6,    0(%rsp)

  102. 	movups	%xmm7,   16(%rsp)

  103. 	movups	%xmm8,   32(%rsp)

  104. 	movups	%xmm9,   48(%rsp)

  105. 	movups	%xmm10,  64(%rsp)

  106. 	movups	%xmm11,  80(%rsp)

  107. 	movups	%xmm12,  96(%rsp)

  108. 	movups	%xmm13, 112(%rsp)

  109. 	movups	%xmm14, 128(%rsp)

  110. 	movups	%xmm15, 144(%rsp)

  111. 

  112. 	movq	OLD_B,     B

  113. #endif

  114. 

  115. 	movq	N,    %rax

  116. 	andq	$-2,  %rax

  117. 	imulq	M,    %rax

  118. 

  119. 	salq	$ZBASE_SHIFT, %rax

  120. 	leaq	(B, %rax, 1), BO1

  121. 

  122. 	salq	$ZBASE_SHIFT, LDA

  123. 

  124. 	leaq	(,   M,   SIZE), M8

  125. 

  126. 	movq	M,  J

  127. 	sarq	$1, J

  128. 	jle	.L20

  129. 	ALIGN_4

  130. 

  131. .L11:

  132. 	movq	A, AO1

  133. 	leaq	(A,   LDA   ), AO2

  134. 	leaq	(A,   LDA, 2), A

  135. 

  136. 	movq	B, BO

  137. 	addq	$8 * SIZE, B

  138. 

  139. 	movq	N,  I

  140. 	sarq	$2, I

  141. 	jle	.L13

  142. 	ALIGN_4

  143. 

  144. .L12:

  145. #ifndef DOUBLE

  146. 	movlps	0 * SIZE(AO1), %xmm0

  147. 	movhps	2 * SIZE(AO1), %xmm0

  148. 	movlps	4 * SIZE(AO1), %xmm1

  149. 	movhps	6 * SIZE(AO1), %xmm1

  150. 

  151. 	movlps	0 * SIZE(AO2), %xmm2

  152. 	movhps	2 * SIZE(AO2), %xmm2

  153. 	movlps	4 * SIZE(AO2), %xmm3

  154. 	movhps	6 * SIZE(AO2), %xmm3

  155. 

  156. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  157. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  158. 	prefetcht0	RPREFETCHSIZE * SIZE(AO2)

  159. 	prefetcht0	WPREFETCHSIZE * SIZE(BO)

  160. #endif

  161. 

  162. #ifdef HAVE_3DNOW

  163. 	prefetchw	(WPREFETCHSIZE + 0) * SIZE(BO)

  164. #endif

  165. 

  166. 	movaps	%xmm0,    0 * SIZE(BO)

  167. 	movaps	%xmm2,    4 * SIZE(BO)

  168. 	leaq	(BO, M8, 4), BO

  169. 	movaps	%xmm1,    0 * SIZE(BO)

  170. 	movaps	%xmm3,    4 * SIZE(BO)

  171. #else

  172. 	movsd	0 * SIZE(AO1), %xmm0

  173. 	movhpd	1 * SIZE(AO1), %xmm0

  174. 	movsd	2 * SIZE(AO1), %xmm1

  175. 	movhpd	3 * SIZE(AO1), %xmm1

  176. 

  177. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  178. 	prefetcht2	RPREFETCHSIZE * SIZE(AO1)

  179. #endif

  180. 

  181. 	movsd	4 * SIZE(AO1), %xmm2

  182. 	movhpd	5 * SIZE(AO1), %xmm2

  183. 	movsd	6 * SIZE(AO1), %xmm3

  184. 	movhpd	7 * SIZE(AO1), %xmm3

  185. 

  186. 	movsd	0 * SIZE(AO2), %xmm4

  187. 	movhpd	1 * SIZE(AO2), %xmm4

  188. 	movsd	2 * SIZE(AO2), %xmm5

  189. 	movhpd	3 * SIZE(AO2), %xmm5

  190. 

  191. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  192. 	prefetcht2	RPREFETCHSIZE * SIZE(AO2)

  193. #endif

  194. 

  195. 	movsd	4 * SIZE(AO2), %xmm6

  196. 	movhpd	5 * SIZE(AO2), %xmm6

  197. 	movsd	6 * SIZE(AO2), %xmm7

  198. 	movhpd	7 * SIZE(AO2), %xmm7

  199. 

  200. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  201. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  202. 	prefetcht0	RPREFETCHSIZE * SIZE(AO2)

  203. 	prefetcht0	WPREFETCHSIZE * SIZE(BO)

  204. #endif

  205. 

  206. #ifdef HAVE_3DNOW

  207. 	prefetchw	(WPREFETCHSIZE + 0) * SIZE(BO)

  208. 	prefetchw	(WPREFETCHSIZE + 8) * SIZE(BO)

  209. #endif

  210. 

  211. 	movapd	%xmm0,    0 * SIZE(BO)

  212. 	movapd	%xmm1,    2 * SIZE(BO)

  213. 	movapd	%xmm4,    4 * SIZE(BO)

  214. 	movapd	%xmm5,    6 * SIZE(BO)

  215. 

  216. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  217. 	prefetcht2	WPREFETCHSIZE * SIZE(BO)

  218. #endif

  219. 	leaq	(BO, M8, 4), BO

  220. 

  221. 	movapd	%xmm2,    0 * SIZE(BO)

  222. 	movapd	%xmm3,    2 * SIZE(BO)

  223. 	movapd	%xmm6,    4 * SIZE(BO)

  224. 	movapd	%xmm7,    6 * SIZE(BO)

  225. #endif

  226. 

  227. 	addq	$8 * SIZE, AO1

  228. 	addq	$8 * SIZE, AO2

  229. 	leaq	(BO, M8, 4), BO

  230. 	decq	I

  231. 	jg	.L12

  232. 	ALIGN_4

  233. 

  234. .L13:

  235. 	testq	$2, N

  236. 	jle	.L14

  237. 

  238. #ifndef DOUBLE

  239. 	movlps	0 * SIZE(AO1), %xmm0

  240. 	movhps	2 * SIZE(AO1), %xmm0

  241. 

  242. 	movlps	0 * SIZE(AO2), %xmm1

  243. 	movhps	2 * SIZE(AO2), %xmm1

  244. 

  245. 	movaps	%xmm0,    0 * SIZE(BO)

  246. 	movaps	%xmm1,    4 * SIZE(BO)

  247. #else

  248. 	movsd	0 * SIZE(AO1), %xmm0

  249. 	movhpd	1 * SIZE(AO1), %xmm0

  250. 	movsd	2 * SIZE(AO1), %xmm1

  251. 	movhpd	3 * SIZE(AO1), %xmm1

  252. 

  253. 	movsd	0 * SIZE(AO2), %xmm2

  254. 	movhpd	1 * SIZE(AO2), %xmm2

  255. 	movsd	2 * SIZE(AO2), %xmm3

  256. 	movhpd	3 * SIZE(AO2), %xmm3

  257. 

  258. 	movapd	%xmm0,    0 * SIZE(BO)

  259. 	movapd	%xmm1,    2 * SIZE(BO)

  260. 	movapd	%xmm2,    4 * SIZE(BO)

  261. 	movapd	%xmm3,    6 * SIZE(BO)

  262. #endif

  263. 

  264. 	addq	$4 * SIZE, AO1

  265. 	addq	$4 * SIZE, AO2

  266. 	leaq	(BO, M8, 4), BO

  267. 	ALIGN_4

  268. 

  269. .L14:

  270. 	testq	$1, N

  271. 	jle	.L19

  272. 

  273. #ifndef DOUBLE

  274. 	movlps	0 * SIZE(AO1), %xmm0

  275. 	movhps	0 * SIZE(AO2), %xmm0

  276. 

  277. 	movaps	%xmm0,    0 * SIZE(BO1)

  278. #else

  279. 	movsd	0 * SIZE(AO1), %xmm0

  280. 	movhpd	1 * SIZE(AO1), %xmm0

  281. 	movsd	0 * SIZE(AO2), %xmm1

  282. 	movhpd	1 * SIZE(AO2), %xmm1

  283. 

  284. 	movapd	%xmm0,    0 * SIZE(BO1)

  285. 	movapd	%xmm1,    2 * SIZE(BO1)

  286. #endif

  287. 

  288. 	addq	$4 * SIZE, BO1

  289. 	ALIGN_4

  290. 

  291. .L19:

  292. 	decq	J

  293. 	jg	.L11

  294. 	ALIGN_4

  295. 

  296. .L20:

  297. 	testq	$1, M

  298. 	jle	.L999

  299. 	ALIGN_4

  300. 

  301. .L21:

  302. 	movq	A, AO1

  303. 

  304. 	movq	B, BO

  305. 

  306. 	movq	N,  I

  307. 	sarq	$2, I

  308. 	jle	.L23

  309. 	ALIGN_4

  310. 

  311. .L22:

  312. #ifndef DOUBLE

  313. 	movlps	0 * SIZE(AO1), %xmm0

  314. 	movhps	2 * SIZE(AO1), %xmm0

  315. 

  316. 	movlps	4 * SIZE(AO1), %xmm1

  317. 	movhps	6 * SIZE(AO1), %xmm1

  318. 

  319. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  320. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  321. 	prefetcht0	WPREFETCHSIZE * SIZE(BO)

  322. #endif

  323. 

  324. #ifdef HAVE_3DNOW

  325. 	prefetchw	(WPREFETCHSIZE + 0) * SIZE(BO)

  326. #endif

  327. 

  328. 	movaps	%xmm0,    0 * SIZE(BO)

  329. 	leaq	(BO, M8, 4), BO

  330. 	movaps	%xmm1,    0 * SIZE(BO)

  331. #else

  332. 	movsd	0 * SIZE(AO1), %xmm0

  333. 	movhpd	1 * SIZE(AO1), %xmm0

  334. 	movsd	2 * SIZE(AO1), %xmm1

  335. 	movhpd	3 * SIZE(AO1), %xmm1

  336. 

  337. 	movsd	4 * SIZE(AO1), %xmm2

  338. 	movhpd	5 * SIZE(AO1), %xmm2

  339. 	movsd	6 * SIZE(AO1), %xmm3

  340. 	movhpd	7 * SIZE(AO1), %xmm3

  341. 

  342. #if defined(PENTIUM4) || defined(GENERIC) || defined(NANO)

  343. 	prefetcht0	RPREFETCHSIZE * SIZE(AO1)

  344. 	prefetcht0	WPREFETCHSIZE * SIZE(BO)

  345. #endif

  346. 

  347. #ifdef HAVE_3DNOW

  348. 	prefetchw	(WPREFETCHSIZE + 0) * SIZE(BO)

  349. #endif

  350. 

  351. 	movapd	%xmm0,    0 * SIZE(BO)

  352. 	movapd	%xmm1,    2 * SIZE(BO)

  353. 

  354. 	leaq	(BO, M8, 4), BO

  355. 

  356. 	movapd	%xmm2,    0 * SIZE(BO)

  357. 	movapd	%xmm3,    2 * SIZE(BO)

  358. #endif

  359. 

  360. 	addq	$8 * SIZE, AO1

  361. 	leaq	(BO, M8, 4), BO

  362. 	decq	I

  363. 	jg	.L22

  364. 	ALIGN_4

  365. 

  366. .L23:

  367. 	testq	$2, N

  368. 	jle	.L24

  369. 

  370. #ifndef DOUBLE

  371. 	movlps	0 * SIZE(AO1), %xmm0

  372. 	movhps	2 * SIZE(AO1), %xmm0

  373. 

  374. 	movaps	%xmm0,    0 * SIZE(BO)

  375. #else

  376. 	movsd	0 * SIZE(AO1), %xmm0

  377. 	movhpd	1 * SIZE(AO1), %xmm0

  378. 	movsd	2 * SIZE(AO1), %xmm1

  379. 	movhpd	3 * SIZE(AO1), %xmm1

  380. 

  381. 	movapd	%xmm0,    0 * SIZE(BO)

  382. 	movapd	%xmm1,    2 * SIZE(BO)

  383. #endif

  384. 

  385. 	addq	$4 * SIZE, AO1

  386. 	leaq	(BO, M8, 4), BO

  387. 	ALIGN_4

  388. 

  389. .L24:

  390. 	testq	$1, N

  391. 	jle	.L999

  392. 

  393. #ifndef DOUBLE

  394. 	movlps	0 * SIZE(AO1), %xmm0

  395. 

  396. 	movlps	%xmm0,    0 * SIZE(BO1)

  397. #else

  398. 	movsd	0 * SIZE(AO1), %xmm0

  399. 	movhpd	1 * SIZE(AO1), %xmm0

  400. 

  401. 	movapd	%xmm0,    0 * SIZE(BO1)

  402. #endif

  403. 	ALIGN_4

  404. 

  405. 

  406. .L999:

  407. #ifdef WINDOWS_ABI

  408. 	movups	  0(%rsp), %xmm6

  409. 	movups	 16(%rsp), %xmm7

  410. 	movups	 32(%rsp), %xmm8

  411. 	movups	 48(%rsp), %xmm9

  412. 	movups	 64(%rsp), %xmm10

  413. 	movups	 80(%rsp), %xmm11

  414. 	movups	 96(%rsp), %xmm12

  415. 	movups	112(%rsp), %xmm13

  416. 	movups	128(%rsp), %xmm14

  417. 	movups	144(%rsp), %xmm15

  418. 

  419. 	addq	$STACKSIZE, %rsp

  420. #endif

  421. 

  422. 	popq	%r11

  423. 	popq	%r12

  424. 	popq	%r13

  425. 	popq	%r14

  426. #ifdef WINDOWS_ABI

  427. 	popq	%rsi

  428. 	popq	%rdi

  429. #endif

  430. 	ret

  431. 

  432. 	EPILOGUE