OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86/zgemv_n_sse2.S

512 lines
11 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #ifdef PENTIUM4

  43. #define PREFETCH	prefetcht0

  44. #define PREFETCHW	prefetcht0

  45. #define PREFETCHSIZE	(8 * 2)

  46. #endif

  47. 

  48. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON) || defined(NEHALEM) || defined(SANDYBRIDGE)

  49. #define PREFETCH	prefetcht0

  50. #define PREFETCHW	prefetcht0

  51. #define PREFETCHSIZE	(8 * 7)

  52. #endif

  53. 

  54. #ifdef OPTERON

  55. #define PREFETCH	prefetchnta

  56. #define PREFETCHW	prefetchw

  57. #define PREFETCHSIZE	(8 * 3)

  58. #define movsd		movlps

  59. #endif

  60. 

  61. #if defined(BARCELONA) || defined(BULLDOZER)

  62. #define PREFETCH	prefetchnta

  63. #define PREFETCHW	prefetchw

  64. #define PREFETCHSIZE	(8 * 5)

  65. #endif

  66. 

  67. #ifdef ATOM

  68. #define PREFETCH	prefetchnta

  69. #define PREFETCHW	prefetcht0

  70. #define PREFETCHSIZE	(8 * 6)

  71. #endif

  72. 

  73. #ifdef NANO

  74. #define PREFETCH	prefetcht0

  75. #define PREFETCHSIZE	(8 * 4)

  76. #endif

  77. 

  78. #define STACKSIZE	16

  79. #define ARGS	16

  80. 

  81. #define M		 4 + STACKSIZE+ARGS(%esp)

  82. #define N		 8 + STACKSIZE+ARGS(%esp)

  83. #define ALPHA_R		16 + STACKSIZE+ARGS(%esp)

  84. #define ALPHA_I		24 + STACKSIZE+ARGS(%esp)

  85. #define A		32 + STACKSIZE+ARGS(%esp)

  86. #define STACK_LDA	36 + STACKSIZE+ARGS(%esp)

  87. #define STACK_X		40 + STACKSIZE+ARGS(%esp)

  88. #define STACK_INCX	44 + STACKSIZE+ARGS(%esp)

  89. #define Y		48 + STACKSIZE+ARGS(%esp)

  90. #define STACK_INCY	52 + STACKSIZE+ARGS(%esp)

  91. #define BUFFER		56 + STACKSIZE+ARGS(%esp)

  92. #define MMM	0 + ARGS(%esp)

  93. #define YY	4 + ARGS(%esp)

  94. #define AA	8 + ARGS(%esp)

  95. 

  96. 

  97. #define I	%eax

  98. #define J	%ebx

  99. 

  100. #define INCX	%ecx

  101. #define INCY	J

  102. 

  103. #define A1	%esi

  104. #define X	%edx

  105. #define Y1	%edi

  106. #define LDA	%ebp

  107. 

  108. #undef SUBPD

  109. 

  110. #if (!defined(CONJ) && !defined(XCONJ)) || (defined(CONJ) && defined(XCONJ))

  111. #define SUBPD	   subpd

  112. #else

  113. #define SUBPD	   addpd

  114. #endif

  115. 

  116. 	PROLOGUE

  117. 

  118. 	subl	$ARGS,%esp

  119. 	pushl	%ebp

  120. 	pushl	%edi

  121. 	pushl	%esi

  122. 	pushl	%ebx

  123. 

  124. 	PROFCODE

  125. 

  126. 	movl	Y,J

  127. 	movl	J,YY

  128. 	movl	A,J

  129. 	movl	J,AA

  130. 	movl	M,J

  131. 	movl	J,MMM

  132. .L0t:

  133. 	xorl	J,J

  134. 	addl	$1,J

  135. 	sall	$18,J

  136. 	subl	J,MMM

  137. 	movl	J,M

  138. 	jge	.L00t

  139. 	ALIGN_3

  140. 

  141. 	movl	MMM,%eax

  142. 	addl	J,%eax

  143. 	jle	.L999x

  144. 	movl	%eax,M

  145. 

  146. .L00t:

  147. 	movl	AA,%eax

  148. 	movl	%eax,A

  149. 

  150. 	movl	YY,J

  151. 	movl	J,Y

  152. 

  153. 	movl	STACK_LDA,  LDA

  154. 	movl	STACK_X,    X

  155. 	movl	STACK_INCX, INCX

  156. 

  157. 	sall	$ZBASE_SHIFT, INCX

  158. 	sall	$ZBASE_SHIFT, LDA

  159. 

  160. 	subl	$-16 * SIZE, A

  161. 

  162. 	cmpl	$0, N

  163. 	jle	.L999

  164. 	cmpl	$0, M

  165. 	jle	.L999

  166. 

  167. 	movl	BUFFER, Y1

  168. 

  169. 	movl	N,  J

  170. 

  171. 	pxor	%xmm7, %xmm7

  172. 

  173. 	movl	M,  %eax

  174. 	addl	$8, %eax

  175. 	sarl	$3, %eax

  176. 	ALIGN_3

  177. 

  178. .L01:

  179. 	movapd	%xmm7,  0 * SIZE(Y1)

  180. 	movapd	%xmm7,  2 * SIZE(Y1)

  181. 	movapd	%xmm7,  4 * SIZE(Y1)

  182. 	movapd	%xmm7,  6 * SIZE(Y1)

  183. 	movapd	%xmm7,  8 * SIZE(Y1)

  184. 	movapd	%xmm7, 10 * SIZE(Y1)

  185. 	movapd	%xmm7, 12 * SIZE(Y1)

  186. 	movapd	%xmm7, 14 * SIZE(Y1)

  187. 	subl	$-16 * SIZE, Y1

  188. 	decl	%eax

  189. 	jg	.L01

  190. 	ALIGN_3

  191. 

  192. .L10:

  193. 	movl	BUFFER, Y1

  194. 	addl	$16 * SIZE, Y1

  195. 

  196. 	movl	A,  A1

  197. 	addl	LDA, A

  198. 

  199. 	movsd	0 * SIZE(X), %xmm6

  200. 	movhpd	1 * SIZE(X), %xmm6

  201. 	addl	INCX, X

  202. 

  203. 	pcmpeqb	%xmm5, %xmm5

  204. 	psllq	$63,    %xmm5

  205. 	shufps	$0xc0, %xmm5, %xmm5

  206. 

  207. 	pshufd	$0x4e, %xmm6, %xmm7

  208. 

  209. #ifdef HAVE_SSE3

  210. 	movddup	ALPHA_R, %xmm3

  211. 	movddup	ALPHA_I, %xmm4

  212. #else

  213. 	movsd	ALPHA_R, %xmm3

  214. 	movsd	ALPHA_I, %xmm4

  215. 

  216. 	unpcklpd %xmm3, %xmm3

  217. 	unpcklpd %xmm4, %xmm4

  218. #endif

  219. 

  220. 	xorpd	 %xmm5, %xmm7

  221. 

  222. 	mulpd	 %xmm3, %xmm6

  223. 	mulpd	 %xmm4, %xmm7

  224. 

  225. #ifndef XCONJ

  226. 	subpd	 %xmm7, %xmm6

  227. #else

  228. 	addpd	 %xmm7, %xmm6

  229. #endif

  230. 

  231. 	pshufd	 $0xee, %xmm6, %xmm7

  232. 	pshufd	 $0x44, %xmm6, %xmm6

  233. 

  234. #ifndef CONJ

  235. 	xorpd	 %xmm5, %xmm7

  236. #else

  237. 	xorpd	 %xmm5, %xmm6

  238. #endif

  239. 

  240. 	movapd	 -16 * SIZE(Y1), %xmm0

  241. 	movapd	 -14 * SIZE(Y1), %xmm1

  242. 	ALIGN_3

  243. 

  244. 	movl	M,   I

  245. 	sarl	$2,  I

  246. 	jle	.L15

  247. 

  248. 	movsd	 -16 * SIZE(A1), %xmm2

  249. 	movhpd	 -15 * SIZE(A1), %xmm2

  250. 	movsd	 -14 * SIZE(A1), %xmm4

  251. 	movhpd	 -13 * SIZE(A1), %xmm4

  252. 

  253. 	decl	 I

  254. 	jle	 .L14

  255. 	ALIGN_3

  256. 

  257. .L13:

  258. #ifdef PREFETCH

  259. 	PREFETCH	(PREFETCHSIZE + 0) * SIZE(A1)

  260. #endif

  261. 

  262. 	pshufd	 $0x4e, %xmm2,  %xmm3

  263. 	mulpd	 %xmm6, %xmm2

  264. 	addpd	 %xmm2, %xmm0

  265. 	movsd	 -12 * SIZE(A1), %xmm2

  266. 	movhpd	 -11 * SIZE(A1), %xmm2

  267. 	pshufd	 $0x4e, %xmm4,  %xmm5

  268. 	mulpd	 %xmm6, %xmm4

  269. 	addpd	 %xmm4, %xmm1

  270. 	movsd	 -10 * SIZE(A1), %xmm4

  271. 	movhpd	  -9 * SIZE(A1), %xmm4

  272. 

  273. 	mulpd	 %xmm7, %xmm3

  274. 	SUBPD	 %xmm3, %xmm0

  275. 	movapd	 %xmm0, -16 * SIZE(Y1)

  276. 	movapd	 -12 * SIZE(Y1), %xmm0

  277. 	mulpd	 %xmm7, %xmm5

  278. 	SUBPD	 %xmm5, %xmm1

  279. 	movapd	 %xmm1, -14 * SIZE(Y1)

  280. 	movapd	 -10 * SIZE(Y1), %xmm1

  281. 

  282. 	pshufd	 $0x4e, %xmm2,  %xmm3

  283. 	mulpd	 %xmm6, %xmm2

  284. 	addpd	 %xmm2, %xmm0

  285. 	movsd	  -8 * SIZE(A1), %xmm2

  286. 	movhpd	  -7 * SIZE(A1), %xmm2

  287. 	pshufd	 $0x4e, %xmm4,  %xmm5

  288. 	mulpd	 %xmm6, %xmm4

  289. 	addpd	 %xmm4, %xmm1

  290. 	movsd	  -6 * SIZE(A1), %xmm4

  291. 	movhpd	  -5 * SIZE(A1), %xmm4

  292. 

  293. 	mulpd	 %xmm7, %xmm3

  294. 	SUBPD	 %xmm3, %xmm0

  295. 	movapd	 %xmm0, -12 * SIZE(Y1)

  296. 	movapd	  -8 * SIZE(Y1), %xmm0

  297. 	mulpd	 %xmm7, %xmm5

  298. 	SUBPD	 %xmm5, %xmm1

  299. 	movapd	 %xmm1, -10 * SIZE(Y1)

  300. 	movapd	  -6 * SIZE(Y1), %xmm1

  301. 

  302. 	subl	 $-8 * SIZE, A1

  303. 	subl	 $-8 * SIZE, Y1

  304. 

  305. 	subl	 $1, I

  306. 	BRANCH

  307. 	jg	.L13

  308. 	ALIGN_3

  309. 

  310. .L14:

  311. 	pshufd	 $0x4e, %xmm2,  %xmm3

  312. 	mulpd	 %xmm6, %xmm2

  313. 	addpd	 %xmm2, %xmm0

  314. 	movsd	 -12 * SIZE(A1), %xmm2

  315. 	movhpd	 -11 * SIZE(A1), %xmm2

  316. 	pshufd	 $0x4e, %xmm4,  %xmm5

  317. 	mulpd	 %xmm6, %xmm4

  318. 	addpd	 %xmm4, %xmm1

  319. 	movsd	 -10 * SIZE(A1), %xmm4

  320. 	movhpd	  -9 * SIZE(A1), %xmm4

  321. 

  322. 	mulpd	 %xmm7, %xmm3

  323. 	SUBPD	 %xmm3, %xmm0

  324. 	movapd	 %xmm0, -16 * SIZE(Y1)

  325. 	movapd	 -12 * SIZE(Y1), %xmm0

  326. 	mulpd	 %xmm7, %xmm5

  327. 	SUBPD	 %xmm5, %xmm1

  328. 	movapd	 %xmm1, -14 * SIZE(Y1)

  329. 	movapd	 -10 * SIZE(Y1), %xmm1

  330. 

  331. 	pshufd	 $0x4e, %xmm2,  %xmm3

  332. 	mulpd	 %xmm6, %xmm2

  333. 	addpd	 %xmm2, %xmm0

  334. 	pshufd	 $0x4e, %xmm4,  %xmm5

  335. 	mulpd	 %xmm6, %xmm4

  336. 	addpd	 %xmm4, %xmm1

  337. 

  338. 	mulpd	 %xmm7, %xmm3

  339. 	SUBPD	 %xmm3, %xmm0

  340. 	movapd	 %xmm0, -12 * SIZE(Y1)

  341. 	movapd	  -8 * SIZE(Y1), %xmm0

  342. 	mulpd	 %xmm7, %xmm5

  343. 	SUBPD	 %xmm5, %xmm1

  344. 	movapd	 %xmm1, -10 * SIZE(Y1)

  345. 	movapd	  -6 * SIZE(Y1), %xmm1

  346. 

  347. 	subl	 $-8 * SIZE, A1

  348. 	subl	 $-8 * SIZE, Y1

  349. 	ALIGN_3

  350. 

  351. .L15:

  352. 	testl	$2, M

  353. 	je	.L17

  354. 

  355. 	movsd	 -16 * SIZE(A1), %xmm2

  356. 	movhpd	 -15 * SIZE(A1), %xmm2

  357. 	movsd	 -14 * SIZE(A1), %xmm4

  358. 	movhpd	 -13 * SIZE(A1), %xmm4

  359. 

  360. 	pshufd	 $0x4e, %xmm2,  %xmm3

  361. 	mulpd	 %xmm6, %xmm2

  362. 	addpd	 %xmm2, %xmm0

  363. 	pshufd	 $0x4e, %xmm4,  %xmm5

  364. 	mulpd	 %xmm6, %xmm4

  365. 	addpd	 %xmm4, %xmm1

  366. 

  367. 	mulpd	 %xmm7, %xmm3

  368. 	SUBPD	 %xmm3, %xmm0

  369. 	movapd	 %xmm0, -16 * SIZE(Y1)

  370. 	mulpd	 %xmm7, %xmm5

  371. 	SUBPD	 %xmm5, %xmm1

  372. 	movapd	 %xmm1, -14 * SIZE(Y1)

  373. 

  374. 	movapd	 -12 * SIZE(Y1), %xmm0

  375. 

  376. 	addl	 $4 * SIZE, A1

  377. 	addl	 $4 * SIZE, Y1

  378. 	ALIGN_3

  379. 

  380. .L17:

  381. 	testl	$1, M

  382. 	je	.L19

  383. 

  384. 	movsd	 -16 * SIZE(A1), %xmm2

  385. 	movhpd	 -15 * SIZE(A1), %xmm2

  386. 

  387. 	pshufd	 $0x4e, %xmm2, %xmm3

  388. 	mulpd	 %xmm6, %xmm2

  389. 	addpd	 %xmm2, %xmm0

  390. 	mulpd	 %xmm7, %xmm3

  391. 	SUBPD	 %xmm3, %xmm0

  392. 

  393. 	movapd	 %xmm0, -16 * SIZE(Y1)

  394. 	ALIGN_3

  395. 

  396. .L19:

  397. 	decl	J

  398. 	jg	.L10

  399. 	ALIGN_4

  400. 

  401. .L990:

  402. 	movl	Y,   Y1

  403. 	movl	BUFFER, X

  404. 

  405. 	movl	STACK_INCY, INCY

  406. 	sall	$ZBASE_SHIFT, INCY

  407. 

  408. 	movl	M,   %eax

  409. 	sarl	$2,  %eax

  410. 	jle	.L994

  411. 	ALIGN_3

  412. 

  413. .L992:

  414. 	movsd	0 * SIZE(Y1), %xmm0

  415. 	movhpd	1 * SIZE(Y1), %xmm0

  416. 

  417. 	addpd	0 * SIZE(X), %xmm0

  418. 

  419. 	movlpd	%xmm0, 0 * SIZE(Y1)

  420. 	movhpd	%xmm0, 1 * SIZE(Y1)

  421. 	addl	INCY, Y1

  422. 

  423. 	movsd	0 * SIZE(Y1), %xmm0

  424. 	movhpd	1 * SIZE(Y1), %xmm0

  425. 

  426. 	addpd	2 * SIZE(X), %xmm0

  427. 

  428. 	movlpd	%xmm0, 0 * SIZE(Y1)

  429. 	movhpd	%xmm0, 1 * SIZE(Y1)

  430. 	addl	INCY, Y1

  431. 

  432. 	movsd	0 * SIZE(Y1), %xmm0

  433. 	movhpd	1 * SIZE(Y1), %xmm0

  434. 

  435. 	addpd	4 * SIZE(X), %xmm0

  436. 

  437. 	movlpd	%xmm0, 0 * SIZE(Y1)

  438. 	movhpd	%xmm0, 1 * SIZE(Y1)

  439. 	addl	INCY, Y1

  440. 

  441. 	movsd	0 * SIZE(Y1), %xmm0

  442. 	movhpd	1 * SIZE(Y1), %xmm0

  443. 

  444. 	addpd	6 * SIZE(X), %xmm0

  445. 

  446. 	movlpd	%xmm0, 0 * SIZE(Y1)

  447. 	movhpd	%xmm0, 1 * SIZE(Y1)

  448. 	addl	INCY, Y1

  449. 

  450. 	addl	$8 * SIZE, X

  451. 	decl	%eax

  452. 	jg	.L992

  453. 	ALIGN_3

  454. 

  455. .L994:

  456. 	testl	$2, M

  457. 	jle	.L996

  458. 

  459. 	movsd	0 * SIZE(Y1), %xmm0

  460. 	movhpd	1 * SIZE(Y1), %xmm0

  461. 

  462. 	addpd	0 * SIZE(X), %xmm0

  463. 

  464. 	movlpd	%xmm0, 0 * SIZE(Y1)

  465. 	movhpd	%xmm0, 1 * SIZE(Y1)

  466. 	addl	INCY, Y1

  467. 

  468. 	movsd	0 * SIZE(Y1), %xmm0

  469. 	movhpd	1 * SIZE(Y1), %xmm0

  470. 

  471. 	addpd	2 * SIZE(X), %xmm0

  472. 

  473. 	movlpd	%xmm0, 0 * SIZE(Y1)

  474. 	movhpd	%xmm0, 1 * SIZE(Y1)

  475. 	addl	INCY, Y1

  476. 

  477. 	addl	$4 * SIZE, X

  478. 	ALIGN_3

  479. 

  480. .L996:

  481. 	testl	$1, M

  482. 	jle	.L999

  483. 

  484. 	movsd	0 * SIZE(Y1), %xmm0

  485. 	movhpd	1 * SIZE(Y1), %xmm0

  486. 

  487. 	addpd	0 * SIZE(X), %xmm0

  488. 

  489. 	movlpd	%xmm0, 0 * SIZE(Y1)

  490. 	movhpd	%xmm0, 1 * SIZE(Y1)

  491. 	ALIGN_3

  492. 

  493. .L999:

  494. 	movl	M,%eax

  495. 	sall	$ZBASE_SHIFT,%eax

  496. 	addl	%eax,AA

  497. 	movl	STACK_INCY,INCY

  498. 	imull	INCY,%eax

  499. 	addl	%eax,YY

  500. 	jmp		.L0t

  501. 	ALIGN_3

  502. 

  503. .L999x:

  504. 	popl	%ebx

  505. 	popl	%esi

  506. 	popl	%edi

  507. 	popl	%ebp

  508. 	addl	$ARGS,%esp

  509. 	ret

  510. 

  511. 	EPILOGUE