OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 66d9185ebe
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '66d9185ebe'
${ noResults }
OpenBLAS/kernel/x86_64/scal_atom.S

453 lines
8.8 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifndef WINDOWS_ABI

  43. #define M	ARG1

  44. #define X	ARG4

  45. #define INCX	ARG5

  46. #else

  47. #define M	ARG1

  48. #define X	ARG2

  49. #define INCX	ARG3

  50. #endif

  51. 

  52. #define XX	%r10

  53. #define I	%rax

  54. 

  55. #include "l1param.h"

  56. 

  57. 	PROLOGUE

  58. 	PROFCODE

  59. 

  60. #ifdef WINDOWS_ABI

  61. 	movq	40(%rsp), X

  62. 	movq	48(%rsp), INCX

  63. 	movq    64(%rsp), %r9

  64. 	movaps	%xmm3, %xmm0

  65. #else

  66. 	movq	24(%rsp), %r9

  67. #endif

  68. 

  69. 	SAVEREGISTERS

  70. 

  71. 	testq	M, M

  72. 	jle	.L999

  73. 

  74. 	pxor	%xmm1, %xmm1

  75. 	lea	(, INCX, SIZE), INCX

  76. 	comisd	%xmm0, %xmm1

  77. 	jne	.L100

  78. 	jp	.L100

  79. 

  80. 	cmpq	$1, %r9

  81. 	je	.L100

  82. 

  83. /* Alpha == ZERO */

  84. 	cmpq	$SIZE, INCX

  85. 	jne	.L50

  86. 

  87. 	movq	M,  I

  88. 	sarq	$3, I

  89. 	jle	.L12

  90. 	ALIGN_4

  91. 

  92. .L11:

  93. 	movsd	%xmm1,  0 * SIZE(X)

  94. 	movsd	%xmm1,  1 * SIZE(X)

  95. 	movsd	%xmm1,  2 * SIZE(X)

  96. 	movsd	%xmm1,  3 * SIZE(X)

  97. 

  98. 	movsd	%xmm1,  4 * SIZE(X)

  99. 	movsd	%xmm1,  5 * SIZE(X)

  100. 	movsd	%xmm1,  6 * SIZE(X)

  101. 	movsd	%xmm1,  7 * SIZE(X)

  102. 

  103. 	addq	$8 * SIZE, X

  104. 	decq	I

  105. 	jg	.L11

  106. 	ALIGN_4

  107. 

  108. .L12:

  109. 	testq  $4, M

  110. 	je    .L14

  111. 

  112. 	movsd	%xmm1,  0 * SIZE(X)

  113. 	movsd	%xmm1,  1 * SIZE(X)

  114. 	movsd	%xmm1,  2 * SIZE(X)

  115. 	movsd	%xmm1,  3 * SIZE(X)

  116. 

  117. 	addq	$4 * SIZE, X

  118. 	ALIGN_3

  119. 

  120. .L14:

  121. 	testq  $2, M

  122. 	je    .L15

  123. 

  124. 	movsd	%xmm1,  0 * SIZE(X)

  125. 	movsd	%xmm1,  1 * SIZE(X)

  126. 

  127. 	addq	$2 * SIZE, X

  128. 	ALIGN_3

  129. 

  130. .L15:

  131. 	testq  $1, M

  132. 	je    .L999

  133. 

  134. 	movsd	%xmm1,  0 * SIZE(X)

  135. 	jmp	.L999

  136. 	ALIGN_4

  137. 

  138. .L50:

  139. 	movq	M,  I

  140. 	sarq	$3, I

  141. 	jle	.L52

  142. 	ALIGN_4

  143. 

  144. .L51:

  145. 	movsd	%xmm1, 0 * SIZE(X)

  146. 	addq	INCX, X

  147. 	movsd	%xmm1, 0 * SIZE(X)

  148. 	addq	INCX, X

  149. 	movsd	%xmm1, 0 * SIZE(X)

  150. 	addq	INCX, X

  151. 	movsd	%xmm1, 0 * SIZE(X)

  152. 	addq	INCX, X

  153. 	movsd	%xmm1, 0 * SIZE(X)

  154. 	addq	INCX, X

  155. 	movsd	%xmm1, 0 * SIZE(X)

  156. 	addq	INCX, X

  157. 	movsd	%xmm1, 0 * SIZE(X)

  158. 	addq	INCX, X

  159. 	movsd	%xmm1, 0 * SIZE(X)

  160. 	addq	INCX, X

  161. 

  162. 	decq	I

  163. 	jg	.L51

  164. 	ALIGN_4

  165. 

  166. .L52:

  167. 	testq	$7, M

  168. 	je	.L999

  169. 

  170. 	testq	$4, M

  171. 	je	.L53

  172. 

  173. 	movsd	%xmm1, 0 * SIZE(X)

  174. 	addq	INCX, X

  175. 	movsd	%xmm1, 0 * SIZE(X)

  176. 	addq	INCX, X

  177. 	movsd	%xmm1, 0 * SIZE(X)

  178. 	addq	INCX, X

  179. 	movsd	%xmm1, 0 * SIZE(X)

  180. 	addq	INCX, X

  181. 	ALIGN_3

  182. 

  183. .L53:

  184. 	testq	$2, M

  185. 	je	.L54

  186. 

  187. 	movsd	%xmm1, 0 * SIZE(X)

  188. 	addq	INCX, X

  189. 	movsd	%xmm1, 0 * SIZE(X)

  190. 	addq	INCX, X

  191. 	ALIGN_3

  192. 

  193. .L54:

  194. 	testq	$1, M

  195. 	je	.L999

  196. 

  197. 	movsd	%xmm1, 0 * SIZE(X)

  198. 	jmp	.L999

  199. 	ALIGN_4

  200. 

  201. /* Alpha != ZERO */

  202. 

  203. .L100:

  204. 	cmpq	$SIZE, INCX

  205. 	jne	.L150

  206. 

  207. 	unpcklpd %xmm0, %xmm0

  208. 

  209. 	movq	M,  I

  210. 	sarq	$3, I

  211. 	jle	.L113

  212. 

  213. 	movsd	 0 * SIZE(X), %xmm1

  214. 	movsd	 1 * SIZE(X), %xmm2

  215. 	movsd	 2 * SIZE(X), %xmm3

  216. 	movsd	 3 * SIZE(X), %xmm4

  217. 	movsd	 4 * SIZE(X), %xmm5

  218. 	movsd	 5 * SIZE(X), %xmm6

  219. 	movsd	 6 * SIZE(X), %xmm7

  220. 	movsd	 7 * SIZE(X), %xmm8

  221. 

  222. 	mulsd	%xmm0, %xmm1

  223. 	mulsd	%xmm0, %xmm2

  224. 	mulsd	%xmm0, %xmm3

  225. 	mulsd	%xmm0, %xmm4

  226. 

  227. 	decq	I

  228. 	jle	.L112

  229. 	ALIGN_4

  230. 

  231. .L111:

  232. #ifdef PREFETCHW

  233. 	PREFETCHW (PREFETCHSIZE +  0) - PREOFFSET(X)

  234. #endif

  235. 

  236. 	movsd	%xmm1,  0 * SIZE(X)

  237. 	movsd	%xmm2,  1 * SIZE(X)

  238. 	movsd	%xmm3,  2 * SIZE(X)

  239. 	movsd	%xmm4,  3 * SIZE(X)

  240. 

  241. 	movsd	 8 * SIZE(X), %xmm1

  242. 	mulsd	%xmm0, %xmm5

  243. 	movsd	 9 * SIZE(X), %xmm2

  244. 	mulsd	%xmm0, %xmm6

  245. 	movsd	10 * SIZE(X), %xmm3

  246. 	mulsd	%xmm0, %xmm7

  247. 	movsd	11 * SIZE(X), %xmm4

  248. 	mulsd	%xmm0, %xmm8

  249. 

  250. 	movsd	%xmm5,  4 * SIZE(X)

  251. 	movsd	%xmm6,  5 * SIZE(X)

  252. 	movsd	%xmm7,  6 * SIZE(X)

  253. 	movsd	%xmm8,  7 * SIZE(X)

  254. 

  255. 	movsd	12 * SIZE(X), %xmm5

  256. 	mulsd	%xmm0, %xmm1

  257. 	movsd	13 * SIZE(X), %xmm6

  258. 	mulsd	%xmm0, %xmm2

  259. 	movsd	14 * SIZE(X), %xmm7

  260. 	mulsd	%xmm0, %xmm3

  261. 	movsd	15 * SIZE(X), %xmm8

  262. 	mulsd	%xmm0, %xmm4

  263. 

  264. 	addq	$8 * SIZE, X

  265. 	decq	I

  266. 	jg	.L111

  267. 	ALIGN_4

  268. 

  269. .L112:

  270. 	movsd	%xmm1,  0 * SIZE(X)

  271. 	mulsd	%xmm0, %xmm5

  272. 	movsd	%xmm2,  1 * SIZE(X)

  273. 	mulsd	%xmm0, %xmm6

  274. 	movsd	%xmm3,  2 * SIZE(X)

  275. 	mulsd	%xmm0, %xmm7

  276. 	movsd	%xmm4,  3 * SIZE(X)

  277. 	mulsd	%xmm0, %xmm8

  278. 

  279. 	movsd	%xmm5,  4 * SIZE(X)

  280. 	movsd	%xmm6,  5 * SIZE(X)

  281. 	movsd	%xmm7,  6 * SIZE(X)

  282. 	movsd	%xmm8,  7 * SIZE(X)

  283. 	addq	$8 * SIZE, X

  284. 	ALIGN_3

  285. 

  286. .L113:

  287. 	testq	$4, M

  288. 	je	.L115

  289. 

  290. 	movsd	 0 * SIZE(X), %xmm1

  291. 	movsd	 1 * SIZE(X), %xmm2

  292. 	movsd	 2 * SIZE(X), %xmm3

  293. 	movsd	 3 * SIZE(X), %xmm4

  294. 

  295. 	mulsd	%xmm0, %xmm1

  296. 	mulsd	%xmm0, %xmm2

  297. 	mulsd	%xmm0, %xmm3

  298. 	mulsd	%xmm0, %xmm4

  299. 

  300. 	movsd	%xmm1,  0 * SIZE(X)

  301. 	movsd	%xmm2,  1 * SIZE(X)

  302. 	movsd	%xmm3,  2 * SIZE(X)

  303. 	movsd	%xmm4,  3 * SIZE(X)

  304. 

  305. 	addq	$4 * SIZE, X

  306. 	ALIGN_3

  307. 

  308. .L115:

  309. 	testq	$2, M

  310. 	je	.L116

  311. 

  312. 	movsd	 0 * SIZE(X), %xmm1

  313. 	movsd	 1 * SIZE(X), %xmm2

  314. 

  315. 	mulsd	%xmm0, %xmm1

  316. 	mulsd	%xmm0, %xmm2

  317. 

  318. 	movsd	%xmm1,  0 * SIZE(X)

  319. 	movsd	%xmm2,  1 * SIZE(X)

  320. 

  321. 	addq	$2 * SIZE, X

  322. 	ALIGN_3

  323. 

  324. .L116:

  325. 	testq	$1, M

  326. 	je	.L999

  327. 

  328. 	movsd	 0 * SIZE(X), %xmm1

  329. 	mulsd	%xmm0, %xmm1

  330. 	movsd	%xmm1,  0 * SIZE(X)

  331. 	jmp	.L999

  332. 	ALIGN_3

  333. 

  334. /* incx != 1 */

  335. 

  336. .L150:

  337. 	movq	X, XX

  338. 	movq	M,  I		# rcx = n

  339. 	sarq	$3, I		# (n >> 3)

  340. 	jle	.L152

  341. 	ALIGN_4

  342. 

  343. .L151:

  344. 	movsd	0 * SIZE(X), %xmm1

  345. 	addq	INCX, X

  346. 	movsd	0 * SIZE(X), %xmm2

  347. 	addq	INCX, X

  348. 	mulsd	%xmm0, %xmm1

  349. 	movsd	0 * SIZE(X), %xmm3

  350. 	addq	INCX, X

  351. 	mulsd	%xmm0, %xmm2

  352. 	movsd	0 * SIZE(X), %xmm4

  353. 	addq	INCX, X

  354. 	mulsd	%xmm0, %xmm3

  355. 	movsd	0 * SIZE(X), %xmm5

  356. 	addq	INCX, X

  357. 	mulsd	%xmm0, %xmm4

  358. 	movsd	0 * SIZE(X), %xmm6

  359. 	addq	INCX, X

  360. 	mulsd	%xmm0, %xmm5

  361. 	movsd	0 * SIZE(X), %xmm7

  362. 	addq	INCX, X

  363. 	mulsd	%xmm0, %xmm6

  364. 	movsd	0 * SIZE(X), %xmm8

  365. 	addq	INCX, X

  366. 	mulsd	%xmm0, %xmm7

  367. 

  368. 	movsd	%xmm1, 0 * SIZE(XX)

  369. 	addq	INCX, XX

  370. 	mulsd	%xmm0, %xmm8

  371. 	movsd	%xmm2, 0 * SIZE(XX)

  372. 	addq	INCX, XX

  373. 	movsd	%xmm3, 0 * SIZE(XX)

  374. 	addq	INCX, XX

  375. 	movsd	%xmm4, 0 * SIZE(XX)

  376. 	addq	INCX, XX

  377. 	movsd	%xmm5, 0 * SIZE(XX)

  378. 	addq	INCX, XX

  379. 	movsd	%xmm6, 0 * SIZE(XX)

  380. 	addq	INCX, XX

  381. 	movsd	%xmm7, 0 * SIZE(XX)

  382. 	addq	INCX, XX

  383. 	movsd	%xmm8, 0 * SIZE(XX)

  384. 	addq	INCX, XX

  385. 	decq	I

  386. 	jg	.L151

  387. 	ALIGN_4

  388. 

  389. .L152:

  390. 	testq	$7, M

  391. 	je	.L999

  392. 

  393. 	testq	$4, M

  394. 	je	.L153

  395. 

  396. 	movsd	 0 * SIZE(X), %xmm1

  397. 	addq	INCX, X

  398. 	movsd	 0 * SIZE(X), %xmm2

  399. 	addq	INCX, X

  400. 	mulsd	%xmm0, %xmm1

  401. 	movsd	 0 * SIZE(X), %xmm3

  402. 	addq	INCX, X

  403. 	mulsd	%xmm0, %xmm2

  404. 	movsd	 0 * SIZE(X), %xmm4

  405. 	addq	INCX, X

  406. 	mulsd	%xmm0, %xmm3

  407. 

  408. 	movsd	%xmm1,  0 * SIZE(XX)

  409. 	addq	INCX, XX

  410. 	mulsd	%xmm0, %xmm4

  411. 	movsd	%xmm2,  0 * SIZE(XX)

  412. 	addq	INCX, XX

  413. 	movsd	%xmm3,  0 * SIZE(XX)

  414. 	addq	INCX, XX

  415. 	movsd	%xmm4,  0 * SIZE(XX)

  416. 	addq	INCX, XX

  417. 	ALIGN_3

  418. 

  419. .L153:

  420. 	testq	$2, M

  421. 	je	.L154

  422. 

  423. 	movsd	 0 * SIZE(X), %xmm1

  424. 	addq	INCX, X

  425. 	mulsd	%xmm0, %xmm1

  426. 	movsd	 0 * SIZE(X), %xmm2

  427. 	addq	INCX, X

  428. 	mulsd	%xmm0, %xmm2

  429. 

  430. 	movsd	%xmm1,  0 * SIZE(XX)

  431. 	addq	INCX, XX

  432. 	movsd	%xmm2,  0 * SIZE(XX)

  433. 	addq	INCX, XX

  434. 	ALIGN_3

  435. 

  436. .L154:

  437. 	testq	$1, M

  438. 	je	.L999

  439. 

  440. 	movsd	 0 * SIZE(X), %xmm1

  441. 	mulsd	%xmm0, %xmm1

  442. 	movsd	%xmm1,  0 * SIZE(X)

  443. 	ALIGN_4

  444. 

  445. .L999:

  446. 	xorq	%rax, %rax

  447. 

  448. 	RESTOREREGISTERS

  449. 

  450. 	ret

  451. 

  452. 	EPILOGUE