OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 16012767f4
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '16012767f4'
${ noResults }
OpenBLAS/kernel/x86_64/dcopy_bulldozer.S

292 lines
6.4 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define M	ARG1	/* rdi */

  43. #define X	ARG2	/* rsi */

  44. #define INCX	ARG3	/* rdx */

  45. #define Y	ARG4	/* rcx */

  46. #ifndef WINDOWS_ABI

  47. #define INCY	ARG5	/* r8  */

  48. #else

  49. #define INCY	%r10

  50. #endif

  51. 

  52. #include "l1param.h"

  53. 

  54. #define VLOAD(OFFSET, ADDR, REG)		vmovups	OFFSET(ADDR), REG

  55. #define VSHUFPD_1(REG1 , REG2)			vshufpd $0x01, REG1, REG2, REG2

  56. #define A_PRE 640

  57. #define B_PRE 640

  58. 

  59. 	PROLOGUE

  60. 	PROFCODE

  61. 

  62. #ifdef WINDOWS_ABI

  63. 	movq	40(%rsp), INCY

  64. #endif

  65. 

  66. 	SAVEREGISTERS

  67. 

  68. 	leaq	(, INCX, SIZE), INCX

  69. 	leaq	(, INCY, SIZE), INCY

  70. 

  71. 	cmpq	$SIZE, INCX

  72. 	jne	.L40

  73. 	cmpq	$SIZE, INCY

  74. 	jne	.L40

  75. 

  76. 	testq	$SIZE, X

  77. 	je	.L10

  78. 

  79. 	vmovsd	(X), %xmm0

  80. 	vmovsd	%xmm0, (Y)

  81. 	addq	$1 * SIZE, X

  82. 	addq	$1 * SIZE, Y

  83. 	decq	M

  84. 	jle	.L19

  85. 	ALIGN_4

  86. 

  87. .L10:

  88. 	subq	$-16 * SIZE, X

  89. 	subq	$-16 * SIZE, Y

  90. 

  91. 

  92. 	movq	M,  %rax

  93. 	sarq	$4, %rax

  94. 	jle	.L13

  95. 

  96. 	vmovups	-16 * SIZE(X), %xmm0

  97. 	vmovups	-14 * SIZE(X), %xmm1

  98. 	vmovups	-12 * SIZE(X), %xmm2

  99. 	vmovups	-10 * SIZE(X), %xmm3

  100. 	vmovups	 -8 * SIZE(X), %xmm4

  101. 	vmovups	 -6 * SIZE(X), %xmm5

  102. 	vmovups	 -4 * SIZE(X), %xmm6

  103. 	vmovups	 -2 * SIZE(X), %xmm7

  104. 

  105. 	decq	%rax

  106. 	jle .L12

  107. 	ALIGN_4

  108. 

  109. .L11:

  110. 

  111. 	prefetchnta	A_PRE(X)

  112. 	nop

  113. 	vmovups	%xmm0, -16 * SIZE(Y)

  114. 	vmovups	%xmm1, -14 * SIZE(Y)

  115. 	prefetchnta	B_PRE(Y)

  116. 	nop

  117. 	vmovups	%xmm2, -12 * SIZE(Y)

  118. 	vmovups	%xmm3, -10 * SIZE(Y)

  119. 

  120. 	VLOAD( 0 * SIZE, X, %xmm0)

  121. 	VLOAD( 2 * SIZE, X, %xmm1)

  122. 	VLOAD( 4 * SIZE, X, %xmm2)

  123. 	VLOAD( 6 * SIZE, X, %xmm3)

  124. 

  125. 	prefetchnta	A_PRE+64(X)

  126. 	nop

  127. 	vmovups	%xmm4, -8 * SIZE(Y)

  128. 	vmovups	%xmm5, -6 * SIZE(Y)

  129. 	prefetchnta	B_PRE+64(Y)

  130. 	nop

  131. 	vmovups	%xmm6, -4 * SIZE(Y)

  132. 	vmovups	%xmm7, -2 * SIZE(Y)

  133. 

  134. 	VLOAD( 8 * SIZE, X, %xmm4)

  135. 	VLOAD(10 * SIZE, X, %xmm5)

  136. 	subq	$-16 * SIZE, Y

  137. 	VLOAD(12 * SIZE, X, %xmm6)

  138. 	VLOAD(14 * SIZE, X, %xmm7)

  139. 

  140. 	subq	$-16 * SIZE, X

  141. 	decq	%rax

  142. 	jg	.L11

  143. 	ALIGN_3

  144. 

  145. .L12:

  146. 	vmovups	%xmm0, -16 * SIZE(Y)

  147. 	vmovups	%xmm1, -14 * SIZE(Y)

  148. 	vmovups	%xmm2, -12 * SIZE(Y)

  149. 	vmovups	%xmm3, -10 * SIZE(Y)

  150. 	vmovups	%xmm4,  -8 * SIZE(Y)

  151. 	vmovups	%xmm5,  -6 * SIZE(Y)

  152. 	vmovups	%xmm6,  -4 * SIZE(Y)

  153. 	vmovups	%xmm7,  -2 * SIZE(Y)

  154. 

  155. 	subq	$-16 * SIZE, Y

  156. 	subq	$-16 * SIZE, X

  157. 	ALIGN_3

  158. 

  159. .L13:

  160. 	testq	$8, M

  161. 	jle	.L14

  162. 	ALIGN_3

  163. 

  164. 	vmovups	-16 * SIZE(X), %xmm0

  165. 	vmovups	-14 * SIZE(X), %xmm1

  166. 	vmovups	-12 * SIZE(X), %xmm2

  167. 	vmovups	-10 * SIZE(X), %xmm3

  168. 

  169. 	vmovups	%xmm0, -16 * SIZE(Y)

  170. 	vmovups	%xmm1, -14 * SIZE(Y)

  171. 	vmovups	%xmm2, -12 * SIZE(Y)

  172. 	vmovups	%xmm3, -10 * SIZE(Y)

  173. 

  174. 	addq	$8 * SIZE, X

  175. 	addq	$8 * SIZE, Y

  176. 	ALIGN_3

  177. 

  178. .L14:

  179. 	testq	$4, M

  180. 	jle	.L15

  181. 	ALIGN_3

  182. 

  183. 	vmovups	-16 * SIZE(X), %xmm0

  184. 	vmovups	-14 * SIZE(X), %xmm1

  185. 

  186. 	vmovups	%xmm0, -16 * SIZE(Y)

  187. 	vmovups	%xmm1, -14 * SIZE(Y)

  188. 

  189. 	addq	$4 * SIZE, X

  190. 	addq	$4 * SIZE, Y

  191. 	ALIGN_3

  192. 

  193. .L15:

  194. 	testq	$2, M

  195. 	jle	.L16

  196. 	ALIGN_3

  197. 

  198. 	vmovups	-16 * SIZE(X), %xmm0

  199. 	vmovups	%xmm0, -16 * SIZE(Y)

  200. 

  201. 	addq	$2 * SIZE, X

  202. 	addq	$2 * SIZE, Y

  203. 	ALIGN_3

  204. 

  205. .L16:

  206. 	testq	$1, M

  207. 	jle	.L19

  208. 	ALIGN_3

  209. 

  210. 	vmovsd	-16 * SIZE(X), %xmm0

  211. 	vmovsd	%xmm0, 	-16 * SIZE(Y)

  212. 	ALIGN_3

  213. 

  214. .L19:

  215. 	xorq	%rax,%rax

  216. 

  217. 	RESTOREREGISTERS

  218. 

  219. 	ret

  220. 	ALIGN_3

  221. 

  222. 

  223. 

  224. .L40:

  225. 	movq	M,  %rax

  226. 	sarq	$3, %rax

  227. 	jle	.L45

  228. 	ALIGN_3

  229. 

  230. .L41:

  231. 	vmovsd	(X), %xmm0

  232. 	addq	INCX, X

  233. 	vmovsd	(X), %xmm4

  234. 	addq	INCX, X

  235. 	vmovsd	(X), %xmm1

  236. 	addq	INCX, X

  237. 	vmovsd	(X), %xmm5

  238. 	addq	INCX, X

  239. 	vmovsd	(X), %xmm2

  240. 	addq	INCX, X

  241. 	vmovsd	(X), %xmm6

  242. 	addq	INCX, X

  243. 	vmovsd	(X), %xmm3

  244. 	addq	INCX, X

  245. 	vmovsd	(X), %xmm7

  246. 	addq	INCX, X

  247. 

  248. 	vmovsd	%xmm0, (Y)

  249. 	addq	INCY, Y

  250. 	vmovsd	%xmm4, (Y)

  251. 	addq	INCY, Y

  252. 	vmovsd	%xmm1, (Y)

  253. 	addq	INCY, Y

  254. 	vmovsd	%xmm5, (Y)

  255. 	addq	INCY, Y

  256. 	vmovsd	%xmm2, (Y)

  257. 	addq	INCY, Y

  258. 	vmovsd	%xmm6, (Y)

  259. 	addq	INCY, Y

  260. 	vmovsd	%xmm3, (Y)

  261. 	addq	INCY, Y

  262. 	vmovsd	%xmm7, (Y)

  263. 	addq	INCY, Y

  264. 

  265. 	decq	%rax

  266. 	jg	.L41

  267. 	ALIGN_3

  268. 

  269. .L45:

  270. 	movq	M,  %rax

  271. 	andq	$7, %rax

  272. 	jle	.L47

  273. 	ALIGN_3

  274. 

  275. .L46:

  276. 	vmovsd	(X), %xmm0

  277. 	addq	INCX, X

  278. 	vmovsd	%xmm0, (Y)

  279. 	addq	INCY, Y

  280. 	decq	%rax

  281. 	jg	.L46

  282. 	ALIGN_3

  283. 

  284. .L47:

  285. 	xorq	%rax, %rax

  286. 

  287. 	RESTOREREGISTERS

  288. 

  289. 	ret

  290. 

  291. 	EPILOGUE