OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 886cbaf4e4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '886cbaf4e4'
${ noResults }
OpenBLAS/kernel/x86_64/copy.S

367 lines
6.9 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define N	ARG1	/* rdi */

  43. #define X	ARG2	/* rsi */

  44. #define INCX	ARG3	/* rdx */

  45. #define Y	ARG4	/* rcx */

  46. #ifndef WINDOWS_ABI

  47. #define INCY	ARG5	/* r8  */

  48. #define FLAG	ARG6

  49. #else

  50. #define INCY	%r10

  51. #define FLAG	%r11

  52. #endif

  53. 		

  54. #include "l1param.h"

  55. 	

  56. 	PROLOGUE

  57. 	PROFCODE

  58. 	

  59. #ifdef WINDOWS_ABI

  60. 	movq	40(%rsp), INCY

  61. #endif

  62. 

  63. 	EMMS

  64. 	

  65. 	testq	N, N	# if m == 0 goto End

  66. 	jle	.L999

  67. 

  68. 	salq	$BASE_SHIFT, INCX

  69. 	salq	$BASE_SHIFT, INCY

  70. 

  71. 	cmpq	$SIZE, INCX	# if incx != 1

  72. 	jne	.L100

  73. 	cmpq	$SIZE, INCY	# if incy != 1

  74. 	jne	.L100

  75. 

  76. 	movq	N, %rax	# i = m

  77. 	sarq	$3,   %rax

  78. 	jle	.L20

  79. 	ALIGN_2

  80. 

  81. .L11:

  82. #ifdef XDOUBLE

  83. 

  84. #ifdef PREFETCH

  85. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  86. #endif

  87. 

  88. 	movq	  0(X), %mm0

  89. 	movq	  8(X), %mm1

  90. 

  91. 	movq	%mm0,   0(Y)

  92. 	movq	%mm1,   8(Y)

  93. 

  94. 	movq	 16(X), %mm2

  95. 	movq	 24(X), %mm3

  96. 

  97. 	movq	%mm2,  16(Y)

  98. 	movq	%mm3,  24(Y)

  99. 

  100. 	movq	 32(X), %mm4

  101. 	movq	 40(X), %mm5

  102. 

  103. 	movq	%mm4,  32(Y)

  104. 	movq	%mm5,  40(Y)

  105. 

  106. 	movq	 48(X), %mm6

  107. 	movq	 56(X), %mm7

  108. 

  109. 	movq	%mm6,  48(Y)

  110. 	movq	%mm7,  56(Y)

  111. 

  112. #ifdef PREFETCHW

  113. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  114. #endif

  115. 

  116. 	movq	 64(X), %mm0

  117. 	movq	 72(X), %mm1

  118. 

  119. 	movq	%mm0,  64(Y)

  120. 	movq	%mm1,  72(Y)

  121. 

  122. 	movq	 80(X), %mm2

  123. 	movq	 88(X), %mm3

  124. 

  125. 	movq	%mm2,  80(Y)

  126. 	movq	%mm3,  88(Y)

  127. 

  128. 	movq	 96(X), %mm4

  129. 	movq	104(X), %mm5

  130. 

  131. 	movq	%mm4,  96(Y)

  132. 	movq	%mm5, 104(Y)

  133. 

  134. 	movq	112(X), %mm6

  135. 	movq	120(X), %mm7

  136. 

  137. 	movq	%mm6, 112(Y)

  138. 	movq	%mm7, 120(Y)

  139. #elif defined(DOUBLE)

  140. 

  141. 	movq	  0(X), %mm0

  142. 	movq	  8(X), %mm1

  143. 

  144. #ifdef PREFETCH

  145. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  146. #endif

  147. 

  148. 	movq	%mm0,   0(Y)

  149. 	movq	%mm1,   8(Y)

  150. 

  151. 	movq	 16(X), %mm2

  152. 	movq	 24(X), %mm3

  153. 

  154. 	movq	%mm2,  16(Y)

  155. 	movq	%mm3,  24(Y)

  156. 

  157. 	movq	 32(X), %mm4

  158. 	movq	 40(X), %mm5

  159. 

  160. #ifdef PREFETCHW

  161. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  162. #endif

  163. 

  164. 	movq	%mm4,  32(Y)

  165. 	movq	%mm5,  40(Y)

  166. 

  167. 	movq	 48(X), %mm6

  168. 	movq	 56(X), %mm7

  169. 

  170. 	movq	%mm6,  48(Y)

  171. 	movq	%mm7,  56(Y)

  172. #else

  173. 	movq	 0 * SIZE(X), %mm0

  174. 	movq	 2 * SIZE(X), %mm2

  175. 

  176. #ifdef PREFETCH

  177. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  178. #endif

  179. 

  180. 	movq	%mm0,  0 * SIZE(Y)

  181. 	movq	%mm2,  2 * SIZE(Y)

  182. 

  183. 	movq	 4 * SIZE(X), %mm4

  184. 	movq	 6 * SIZE(X), %mm6

  185. 

  186. #ifdef PREFETCHW

  187. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(Y)

  188. #endif

  189. 

  190. 	movq	%mm4,  4 * SIZE(Y)

  191. 	movq	%mm6,  6 * SIZE(Y)

  192. #endif

  193. 

  194. 	addq	$8 * SIZE, X

  195. 	addq	$8 * SIZE, Y

  196. 	decq	%rax

  197. 	jg	.L11

  198. 	ALIGN_2

  199. 

  200. .L20:

  201. 	movq	N, %rax

  202. 	andq	$7,   %rax

  203. 	jle	.L99

  204. 	ALIGN_2

  205. 

  206. .L21:

  207. #ifdef XDOUBLE

  208. 	movq	0(X), %mm0

  209. 	movq	8(X), %mm1

  210. 	movq	%mm0, 0(Y)

  211. 	movq	%mm1, 8(Y)

  212. #else

  213. 	MOVQ	(X), %mm0

  214. 	MOVQ	%mm0, (Y)

  215. #endif

  216. 

  217. 	addq	$SIZE, X

  218. 	addq	$SIZE, Y

  219. 	decq	%rax

  220. 	jg	.L21

  221. 

  222. .L99:

  223. 	xorq	%rax,%rax

  224. 	EMMS

  225. 	ret

  226. 	ALIGN_3

  227. 

  228. .L100:

  229. 	movq	N, %rax

  230. 	sarq	$3,   %rax

  231. 	jle	.L120

  232. 	ALIGN_2

  233. 

  234. .L111:

  235. #ifdef XDOUBLE

  236. 	movq	0(X), %mm0

  237. 	movq	8(X), %mm1

  238. 	addq	INCX, X

  239. 

  240. 	movq	0(X), %mm2

  241. 	movq	8(X), %mm3

  242. 	addq	INCX, X

  243. 

  244. 	movq	0(X), %mm4

  245. 	movq	8(X), %mm5

  246. 	addq	INCX, X

  247. 

  248. 	movq	0(X), %mm6

  249. 	movq	8(X), %mm7

  250. 	addq	INCX, X

  251. 

  252. 	movq	%mm0, 0(Y)

  253. 	movq	%mm1, 8(Y)

  254. 	addq	INCY, Y

  255. 

  256. 	movq	%mm2, 0(Y)

  257. 	movq	%mm3, 8(Y)

  258. 	addq	INCY, Y

  259. 

  260. 	movq	%mm4, 0(Y)

  261. 	movq	%mm5, 8(Y)

  262. 	addq	INCY, Y

  263. 

  264. 	movq	%mm6, 0(Y)

  265. 	movq	%mm7, 8(Y)

  266. 	addq	INCY, Y

  267. 

  268. 	movq	0(X), %mm0

  269. 	movq	8(X), %mm1

  270. 	addq	INCX, X

  271. 

  272. 	movq	0(X), %mm2

  273. 	movq	8(X), %mm3

  274. 	addq	INCX, X

  275. 

  276. 	movq	0(X), %mm4

  277. 	movq	8(X), %mm5

  278. 	addq	INCX, X

  279. 

  280. 	movq	0(X), %mm6

  281. 	movq	8(X), %mm7

  282. 	addq	INCX, X

  283. 

  284. 	movq	%mm0, 0(Y)

  285. 	movq	%mm1, 8(Y)

  286. 	addq	INCY, Y

  287. 

  288. 	movq	%mm2, 0(Y)

  289. 	movq	%mm3, 8(Y)

  290. 	addq	INCY, Y

  291. 

  292. 	movq	%mm4, 0(Y)

  293. 	movq	%mm5, 8(Y)

  294. 	addq	INCY, Y

  295. 

  296. 	movq	%mm6, 0(Y)

  297. 	movq	%mm7, 8(Y)

  298. 	addq	INCY, Y

  299. #else

  300. 	MOVQ	(X), %mm0

  301. 	addq	INCX, X

  302. 	MOVQ	(X), %mm1

  303. 	addq	INCX, X

  304. 	MOVQ	(X), %mm2

  305. 	addq	INCX, X

  306. 	MOVQ	(X), %mm3

  307. 	addq	INCX, X

  308. 	MOVQ	(X), %mm4

  309. 	addq	INCX, X

  310. 	MOVQ	(X), %mm5

  311. 	addq	INCX, X

  312. 	MOVQ	(X), %mm6

  313. 	addq	INCX, X

  314. 	MOVQ	(X), %mm7

  315. 	addq	INCX, X

  316. 

  317. 	MOVQ	%mm0, (Y)

  318. 	addq	INCY, Y

  319. 	MOVQ	%mm1, (Y)

  320. 	addq	INCY, Y

  321. 	MOVQ	%mm2, (Y)

  322. 	addq	INCY, Y

  323. 	MOVQ	%mm3, (Y)

  324. 	addq	INCY, Y

  325. 	MOVQ	%mm4, (Y)

  326. 	addq	INCY, Y

  327. 	MOVQ	%mm5, (Y)

  328. 	addq	INCY, Y

  329. 	MOVQ	%mm6, (Y)

  330. 	addq	INCY, Y

  331. 	MOVQ	%mm7, (Y)

  332. 	addq	INCY, Y

  333. #endif

  334. 

  335. 	decq	%rax

  336. 	jg	.L111

  337. 

  338. .L120:

  339. 	movq	N, %rax

  340. 	andq	$7,   %rax

  341. 	jle	.L999

  342. 	ALIGN_2

  343. 

  344. .L121:

  345. #ifdef XDOUBLE

  346. 	movq	0(X), %mm0

  347. 	movq	8(X), %mm1

  348. 	movq	%mm0, 0(Y)

  349. 	movq	%mm1, 8(Y)

  350. #else

  351. 	MOVQ	(X), %mm0

  352. 	MOVQ	%mm0, (Y)

  353. #endif

  354. 	addq	INCX, X

  355. 	addq	INCY, Y

  356. 

  357. 	decq	%rax

  358. 	jg	.L121

  359. 

  360. .L999:

  361. 	xorq	%rax,%rax

  362. 	EMMS

  363. 	ret

  364. 

  365. 	EPILOGUE

  366.