OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 3d6d026fe1
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '3d6d026fe1'
${ noResults }
OpenBLAS/kernel/x86/zgemv_t_atom.S

446 lines
9.3 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef ATOM

  43. #define PREFETCH	prefetchnta

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCHSIZE	(8 * 6)

  46. #endif

  47. 

  48. #define STACKSIZE	16

  49. 

  50. #define M		 4 + STACKSIZE(%esp)

  51. #define N		 8 + STACKSIZE(%esp)

  52. #define ALPHA_R		16 + STACKSIZE(%esp)

  53. #define ALPHA_I		24 + STACKSIZE(%esp)

  54. #define A		32 + STACKSIZE(%esp)

  55. #define STACK_LDA	36 + STACKSIZE(%esp)

  56. #define STACK_X		40 + STACKSIZE(%esp)

  57. #define STACK_INCX	44 + STACKSIZE(%esp)

  58. #define Y		48 + STACKSIZE(%esp)

  59. #define STACK_INCY	52 + STACKSIZE(%esp)

  60. #define BUFFER		56 + STACKSIZE(%esp)

  61. 

  62. #define I	%eax

  63. #define J	%ebx

  64. 

  65. #define INCX	J

  66. #define INCY	%ecx

  67. 

  68. #define A1	%esi

  69. #define X	%edx

  70. #define Y1	%edi

  71. #define LDA	%ebp

  72. 

  73. #if !defined(CONJ) && !defined(XCONJ)

  74. #define ADD1	   addsd

  75. #define ADD2	   addsd

  76. #define ADD3	   subsd

  77. #define ADD4	   addsd

  78. #endif

  79. 

  80. #if  defined(CONJ) && !defined(XCONJ)

  81. #define ADD1	   addsd

  82. #define ADD2	   addsd

  83. #define ADD3	   addsd

  84. #define ADD4	   subsd

  85. #endif

  86. 

  87. #if !defined(CONJ) &&  defined(XCONJ)

  88. #define ADD1	   addsd

  89. #define ADD2	   subsd

  90. #define ADD3	   addsd

  91. #define ADD4	   addsd

  92. #endif

  93. 

  94. #if  defined(CONJ) &&  defined(XCONJ)

  95. #define ADD1	   addsd

  96. #define ADD2	   subsd

  97. #define ADD3	   subsd

  98. #define ADD4	   subsd

  99. #endif

  100. 

  101. 	PROLOGUE

  102. 

  103. 	pushl	%ebp

  104. 	pushl	%edi

  105. 	pushl	%esi

  106. 	pushl	%ebx

  107. 

  108. 	PROFCODE

  109. 

  110. 	movl	STACK_LDA,  LDA

  111. 	movl	STACK_X,    X

  112. 	movl	STACK_INCX, INCX

  113. 	movl	STACK_INCY, INCY

  114. 

  115. 	sall	$ZBASE_SHIFT, INCX

  116. 	sall	$ZBASE_SHIFT, INCY

  117. 	sall	$ZBASE_SHIFT, LDA

  118. 

  119. 	subl	$-16 * SIZE, A

  120. 

  121. 	cmpl	$0, N

  122. 	jle	.L999

  123. 	cmpl	$0, M

  124. 	jle	.L999

  125. 

  126. 	movl	BUFFER, Y1

  127. 

  128. 	movl	M,  I

  129. 	sarl	$2, I

  130. 	jle	.L05

  131. 	ALIGN_4

  132. 

  133. .L02:

  134. 	movsd	0 * SIZE(X), %xmm0

  135. 	movhpd	1 * SIZE(X), %xmm0

  136. 	addl	INCX, X

  137. 

  138. 	movsd	0 * SIZE(X), %xmm1

  139. 	movhpd	1 * SIZE(X), %xmm1

  140. 	addl	INCX, X

  141. 

  142. 	movsd	0 * SIZE(X), %xmm2

  143. 	movhpd	1 * SIZE(X), %xmm2

  144. 	addl	INCX, X

  145. 

  146. 	movsd	0 * SIZE(X), %xmm3

  147. 	movhpd	1 * SIZE(X), %xmm3

  148. 	addl	INCX, X

  149. 

  150. 	movapd	%xmm0, 0 * SIZE(Y1)

  151. 	movapd	%xmm1, 2 * SIZE(Y1)

  152. 	movapd	%xmm2, 4 * SIZE(Y1)

  153. 	movapd	%xmm3, 6 * SIZE(Y1)

  154. 

  155. 	addl	$8 * SIZE, Y1

  156. 	decl	I

  157. 	jg	.L02

  158. 	ALIGN_4

  159. 

  160. .L05:

  161. 	movl	M,  I

  162. 	andl	$3, I

  163. 	jle	.L10

  164. 	ALIGN_2

  165. 

  166. .L06:

  167. 	movsd	0 * SIZE(X), %xmm0

  168. 	movhpd	1 * SIZE(X), %xmm0

  169. 	addl	INCX, X

  170. 

  171. 	movapd	%xmm0, 0 * SIZE(Y1)

  172. 	addl	$2 * SIZE, Y1

  173. 	decl	I

  174. 	jg	.L06

  175. 	ALIGN_4

  176. 

  177. .L10:

  178. 	movl	Y, Y1

  179. 

  180. 	movl	N,  J

  181. 	ALIGN_3

  182. 

  183. .L11:

  184. 	movl	BUFFER, X

  185. 	addl	$16 * SIZE, X

  186. 

  187. 	movl	A, A1

  188. 	addl	LDA, A

  189. 

  190. 	xorps	%xmm0, %xmm0

  191. 	xorps	%xmm1, %xmm1

  192. 

  193. 	movsd	-16 * SIZE(X), %xmm2

  194. 	movsd	-15 * SIZE(X), %xmm3

  195. 

  196. 	movl	M,   I

  197. 	sarl	$2,  I

  198. 	jle	.L15

  199. 

  200. 	movsd	-16 * SIZE(A1), %xmm4

  201. 	movsd	-15 * SIZE(A1), %xmm5

  202. 

  203. 	movapd	%xmm4, %xmm6

  204. 	mulsd	%xmm2, %xmm4

  205. 	mulsd	%xmm3, %xmm6

  206. 

  207. 	decl	I

  208. 	jle	.L13

  209. 	ALIGN_4

  210. 

  211. .L12:

  212. #ifdef PREFETCH

  213. 	PREFETCH PREFETCHSIZE * SIZE(A1)

  214. #endif

  215. 

  216. 	movapd	%xmm5, %xmm7

  217. 	mulsd	%xmm3, %xmm5

  218. 	movsd	-13 * SIZE(X), %xmm3

  219. 	ADD1	%xmm4, %xmm0

  220. 	movsd	-14 * SIZE(A1), %xmm4

  221. 	mulsd	%xmm2, %xmm7

  222. 	movsd	-14 * SIZE(X), %xmm2

  223. 	ADD2	%xmm6, %xmm1

  224. 

  225. 	movapd	%xmm4, %xmm6

  226. 	mulsd	%xmm2, %xmm4

  227. 	ADD3	%xmm5, %xmm0

  228. 	movsd	-13 * SIZE(A1), %xmm5

  229. 	mulsd	%xmm3, %xmm6

  230. 	ADD4	%xmm7, %xmm1

  231. 

  232. 	movapd	%xmm5, %xmm7

  233. 	mulsd	%xmm3, %xmm5

  234. 	movsd	-11 * SIZE(X), %xmm3

  235. 	ADD1	%xmm4, %xmm0

  236. 	movsd	-12 * SIZE(A1), %xmm4

  237. 	mulsd	%xmm2, %xmm7

  238. 	movsd	-12 * SIZE(X), %xmm2

  239. 	ADD2	%xmm6, %xmm1

  240. 

  241. 	movapd	%xmm4, %xmm6

  242. 	mulsd	%xmm2, %xmm4

  243. 	ADD3	%xmm5, %xmm0

  244. 	movsd	-11 * SIZE(A1), %xmm5

  245. 	mulsd	%xmm3, %xmm6

  246. 	ADD4	%xmm7, %xmm1

  247. 

  248. 	movapd	%xmm5, %xmm7

  249. 	mulsd	%xmm3, %xmm5

  250. 	movsd	 -9 * SIZE(X), %xmm3

  251. 	ADD1	%xmm4, %xmm0

  252. 	movsd	-10 * SIZE(A1), %xmm4

  253. 	mulsd	%xmm2, %xmm7

  254. 	movsd	-10 * SIZE(X), %xmm2

  255. 	ADD2	%xmm6, %xmm1

  256. 

  257. 	movapd	%xmm4, %xmm6

  258. 	mulsd	%xmm2, %xmm4

  259. 	ADD3	%xmm5, %xmm0

  260. 	movsd	 -9 * SIZE(A1), %xmm5

  261. 	mulsd	%xmm3, %xmm6

  262. 	ADD4	%xmm7, %xmm1

  263. 

  264. 	movapd	%xmm5, %xmm7

  265. 	mulsd	%xmm3, %xmm5

  266. 	movsd	 -7 * SIZE(X), %xmm3

  267. 	ADD1	%xmm4, %xmm0

  268. 	movsd	 -8 * SIZE(A1), %xmm4

  269. 	mulsd	%xmm2, %xmm7

  270. 	movsd	 -8 * SIZE(X), %xmm2

  271. 	ADD2	%xmm6, %xmm1

  272. 

  273. 	movapd	%xmm4, %xmm6

  274. 	mulsd	%xmm2, %xmm4

  275. 	ADD3	%xmm5, %xmm0

  276. 	movsd	 -7 * SIZE(A1), %xmm5

  277. 	mulsd	%xmm3, %xmm6

  278. 	ADD4	%xmm7, %xmm1

  279. 

  280. 	addl	$8 * SIZE, A1

  281. 	addl	$8 * SIZE, X

  282. 

  283. 	decl	I

  284. 	jg	.L12

  285. 	ALIGN_4

  286. 

  287. .L13:

  288. 	movapd	%xmm5, %xmm7

  289. 	mulsd	%xmm3, %xmm5

  290. 	movsd	-13 * SIZE(X), %xmm3

  291. 	ADD1	%xmm4, %xmm0

  292. 	movsd	-14 * SIZE(A1), %xmm4

  293. 	mulsd	%xmm2, %xmm7

  294. 	movsd	-14 * SIZE(X), %xmm2

  295. 	ADD2	%xmm6, %xmm1

  296. 

  297. 	movapd	%xmm4, %xmm6

  298. 	mulsd	%xmm2, %xmm4

  299. 	ADD3	%xmm5, %xmm0

  300. 	movsd	-13 * SIZE(A1), %xmm5

  301. 	mulsd	%xmm3, %xmm6

  302. 	ADD4	%xmm7, %xmm1

  303. 

  304. 	movapd	%xmm5, %xmm7

  305. 	mulsd	%xmm3, %xmm5

  306. 	movsd	-11 * SIZE(X), %xmm3

  307. 	ADD1	%xmm4, %xmm0

  308. 	movsd	-12 * SIZE(A1), %xmm4

  309. 	mulsd	%xmm2, %xmm7

  310. 	movsd	-12 * SIZE(X), %xmm2

  311. 	ADD2	%xmm6, %xmm1

  312. 

  313. 	movapd	%xmm4, %xmm6

  314. 	mulsd	%xmm2, %xmm4

  315. 	ADD3	%xmm5, %xmm0

  316. 	movsd	-11 * SIZE(A1), %xmm5

  317. 	mulsd	%xmm3, %xmm6

  318. 	ADD4	%xmm7, %xmm1

  319. 

  320. 	movapd	%xmm5, %xmm7

  321. 	mulsd	%xmm3, %xmm5

  322. 	movsd	 -9 * SIZE(X), %xmm3

  323. 	ADD1	%xmm4, %xmm0

  324. 	movsd	-10 * SIZE(A1), %xmm4

  325. 	mulsd	%xmm2, %xmm7

  326. 	movsd	-10 * SIZE(X), %xmm2

  327. 	ADD2	%xmm6, %xmm1

  328. 

  329. 	movapd	%xmm4, %xmm6

  330. 	mulsd	%xmm2, %xmm4

  331. 	ADD3	%xmm5, %xmm0

  332. 	movsd	 -9 * SIZE(A1), %xmm5

  333. 	mulsd	%xmm3, %xmm6

  334. 	ADD4	%xmm7, %xmm1

  335. 

  336. 	movapd	%xmm5, %xmm7

  337. 	mulsd	%xmm3, %xmm5

  338. 	movsd	 -7 * SIZE(X), %xmm3

  339. 	ADD1	%xmm4, %xmm0

  340. 	mulsd	%xmm2, %xmm7

  341. 	movsd	 -8 * SIZE(X), %xmm2

  342. 	ADD2	%xmm6, %xmm1

  343. 

  344. 	ADD3	%xmm5, %xmm0

  345. 	ADD4	%xmm7, %xmm1

  346. 

  347. 	addl	$8 * SIZE, A1

  348. 	addl	$8 * SIZE, X

  349. 	ALIGN_4

  350. 

  351. .L15:

  352. 	testl	$2, M

  353. 	jle	.L17

  354. 

  355. 	movsd	-16 * SIZE(A1), %xmm4

  356. 	movsd	-15 * SIZE(A1), %xmm5

  357. 

  358. 	movapd	%xmm4, %xmm6

  359. 	mulsd	%xmm2, %xmm4

  360. 	mulsd	%xmm3, %xmm6

  361. 

  362. 	movapd	%xmm5, %xmm7

  363. 	mulsd	%xmm3, %xmm5

  364. 	movsd	-13 * SIZE(X), %xmm3

  365. 	ADD1	%xmm4, %xmm0

  366. 	movsd	-14 * SIZE(A1), %xmm4

  367. 	mulsd	%xmm2, %xmm7

  368. 	movsd	-14 * SIZE(X), %xmm2

  369. 	ADD2	%xmm6, %xmm1

  370. 

  371. 	movapd	%xmm4, %xmm6

  372. 	mulsd	%xmm2, %xmm4

  373. 	ADD3	%xmm5, %xmm0

  374. 	movsd	-13 * SIZE(A1), %xmm5

  375. 	mulsd	%xmm3, %xmm6

  376. 	ADD4	%xmm7, %xmm1

  377. 

  378. 	movapd	%xmm5, %xmm7

  379. 	mulsd	%xmm3, %xmm5

  380. 	movsd	-11 * SIZE(X), %xmm3

  381. 	ADD1	%xmm4, %xmm0

  382. 	mulsd	%xmm2, %xmm7

  383. 	movsd	-12 * SIZE(X), %xmm2

  384. 	ADD2	%xmm6, %xmm1

  385. 

  386. 	ADD3	%xmm5, %xmm0

  387. 	ADD4	%xmm7, %xmm1

  388. 

  389. 	addl	$4 * SIZE, A1

  390. 	ALIGN_4

  391. 

  392. .L17:

  393. 	testl	$1, M

  394. 	jle	.L18

  395. 

  396. 	movsd	-16 * SIZE(A1), %xmm4

  397. 	movsd	-15 * SIZE(A1), %xmm5

  398. 

  399. 	movapd	%xmm4, %xmm6

  400. 	mulsd	%xmm2, %xmm4

  401. 	mulsd	%xmm3, %xmm6

  402. 

  403. 	movapd	%xmm5, %xmm7

  404. 	mulsd	%xmm3, %xmm5

  405. 	ADD1	%xmm4, %xmm0

  406. 	mulsd	%xmm2, %xmm7

  407. 	ADD2	%xmm6, %xmm1

  408. 

  409. 	ADD3	%xmm5, %xmm0

  410. 	ADD4	%xmm7, %xmm1

  411. 	ALIGN_4

  412. 

  413. .L18:

  414. 	movsd	 0 * SIZE(Y1), %xmm4

  415. 	movapd	%xmm0, %xmm2

  416. 	mulsd	ALPHA_R, %xmm0

  417. 	movsd	 1 * SIZE(Y1), %xmm5

  418. 	movapd	%xmm1, %xmm3

  419. 	mulsd	ALPHA_R, %xmm1

  420. 

  421. 	mulsd	ALPHA_I, %xmm2

  422. 	mulsd	ALPHA_I, %xmm3

  423. 

  424. 	addsd	%xmm2, %xmm1

  425. 	subsd	%xmm3, %xmm0

  426. 	addsd	%xmm4, %xmm0

  427. 	addsd	%xmm5, %xmm1

  428. 

  429. 	movlpd	%xmm0,  0 * SIZE(Y1)

  430. 	movlpd	%xmm1,  1 * SIZE(Y1)

  431. 

  432. 	addl	INCY, Y1

  433. 

  434. 	decl	J

  435. 	jg	.L11

  436. 	ALIGN_4

  437. 

  438. .L999:

  439. 	popl	%ebx

  440. 	popl	%esi

  441. 	popl	%edi

  442. 	popl	%ebp

  443. 	ret

  444. 

  445. 	EPILOGUE