OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/qgemv_t.S

467 lines
9.0 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. #include "l2param.h"

  42. 

  43. #define STACKSIZE	80

  44. #define P		4096

  45. 

  46. #define ALPHA		 8 + STACKSIZE(%rsp)

  47. #define OLD_INCX	24 + STACKSIZE(%rsp)

  48. #define OLD_Y		32 + STACKSIZE(%rsp)

  49. #define OLD_INCY	40 + STACKSIZE(%rsp)

  50. #define BUFFER		48 + STACKSIZE(%rsp)

  51. 

  52. #define NLDA	        56            (%rsp)

  53. #define IS		64	      (%rsp)

  54. #define XP		72	      (%rsp)

  55. 

  56. #define M	  %rdi

  57. #define N	  %rsi

  58. #define A	  %rcx

  59. #define LDA	  %r8

  60. #define X	  %r9

  61. #define INCX	  %rdx

  62. #define Y	  %rbp

  63. #define INCY	  %r10

  64. 

  65. #define TEMP	%rax

  66. #define I	%rax

  67. #define J	%r11

  68. #define A1	%r12

  69. #define A2	%r15

  70. #define X1	%r13

  71. #define Y1	%r14

  72. #define MIN_M	%rbx

  73. 

  74. 

  75. 	PROLOGUE

  76. 	PROFCODE

  77. 

  78. 	subq	$STACKSIZE, %rsp

  79. 	movq	%rbx,  0(%rsp)

  80. 	movq	%rbp,  8(%rsp)

  81. 	movq	%r12, 16(%rsp)

  82. 	movq	%r13, 24(%rsp)

  83. 	movq	%r14, 32(%rsp)

  84. 	movq	%r15, 40(%rsp)

  85. 

  86. 	movq	OLD_INCX,   INCX

  87. 	movq	OLD_Y,      Y

  88. 	movq	OLD_INCY,   INCY

  89. 

  90. 	FLD	ALPHA

  91. 

  92. 	salq	$BASE_SHIFT, INCX

  93. 	salq	$BASE_SHIFT, INCY

  94. 

  95. 	movq	$0, IS

  96. 

  97. 	test	M, M

  98. 	jle	.L79			# goto END

  99. 	test	N, N

  100. 	jle	.L79			# goto END

  101. 

  102. 	movq	N,    %rax

  103. 	imulq	LDA,  %rax

  104. 	movq	$P,   NLDA

  105. 	subq	%rax, NLDA

  106. 	salq	$BASE_SHIFT, NLDA

  107. 

  108. 	salq	$BASE_SHIFT, LDA

  109. 	ALIGN_2

  110. 

  111. .L32:

  112. 	movq	$P,   %rax

  113. 	movq	M,    MIN_M

  114. 	subq	IS  , MIN_M

  115. 	cmpq	%rax, MIN_M

  116. 	cmovg	%rax, MIN_M

  117. 

  118. 	movq	IS, X1

  119. 	salq	$BASE_SHIFT, X1

  120. 	leaq	(X,X1, 1), X1

  121. 

  122. 	movq	X1, XP

  123. 

  124. 	cmpq	$SIZE, INCX

  125. 	je	.L34

  126. 

  127. 	movq	BUFFER, X1

  128. 	movq	X1, XP

  129. 

  130. 	movq	MIN_M, I

  131. 	sarq	$2, I

  132. 	jle	.L35

  133. 	ALIGN_3

  134. 

  135. .L36:

  136. 	FLD	(X)

  137. 	addq	INCX, X

  138. 	FST	0 * SIZE(X1)

  139. 

  140. 	FLD	(X)

  141. 	addq	INCX, X

  142. 	FST	1 * SIZE(X1)

  143. 

  144. 	FLD	(X)

  145. 	addq	INCX, X

  146. 	FST	2 * SIZE(X1)

  147. 

  148. 	FLD	(X)

  149. 	addq	INCX, X

  150. 	FST	3 * SIZE(X1)

  151. 

  152. 	addq	$4 * SIZE, X1

  153. 	decq	I

  154. 	jg	.L36

  155. 	ALIGN_3

  156. 

  157. .L35:

  158. 	movq	MIN_M, I

  159. 	andq	$3,I

  160. 	jle	.L34

  161. 	ALIGN_2

  162. 

  163. .L42:

  164. 	FLD	(X)

  165. 	addq	INCX,  X

  166. 	FST	(X1)

  167. 	addq	$SIZE, X1

  168. 	decq	I

  169. 	jg	.L42

  170. 	ALIGN_3

  171. 

  172. /* Main Routine */

  173. 

  174. .L34:

  175. 	movq	Y, Y1			# coffset = y

  176. 

  177. 	movq	N,  J

  178. 	sarq	$2, J

  179. 	jle	.L47

  180. 	ALIGN_3

  181. 

  182. .L48:

  183. 	movq	A,  A1

  184. 	leaq	(A, LDA), A2

  185. 	leaq	(A, LDA, 4), A

  186. 

  187. 	fldz

  188. 	fldz

  189. 	fldz

  190. 	fldz

  191. 

  192. 	movq	XP, X1

  193. 	FLD	(X1)

  194. 

  195. 	movq	MIN_M, I

  196. 	sarq	$2,I

  197. 	jle	.L51

  198. 	ALIGN_3

  199. 

  200. .L80:

  201. 	FLD	0 * SIZE(A1)		# at   = *(a_offset  + 0 * lda)

  202. 	fmul	%st(1),%st		# at1 *= bt1

  203. 

  204. 	faddp	%st,%st(2)		# ct1 += at1

  205. 	FLD	0 * SIZE(A2)		# at1  = *(a_offset2 + 0 * lda)

  206. 

  207. 	fmul	%st(1),%st		# at1 *= bt1

  208. 	faddp	%st,%st(3)		# ct2 += at1

  209. 

  210. 	FLD	0 * SIZE(A1, LDA, 2)	# at   = *(a_offset  + 2 * lda)

  211. 	fmul	%st(1),%st

  212. 

  213. 	faddp	%st,%st(4)

  214. 	FLD	0 * SIZE(A2, LDA, 2)	# at1  = *(a_offset2 + 2 * lda)

  215. 	fmulp	%st, %st(1)

  216. 

  217. 	faddp	%st,%st(4)

  218. 	FLD	1 * SIZE(X1)

  219. 	FLD	1 * SIZE(A1)		# at   = *(a_offset  + 0 * lda)

  220. 

  221. 	fmul	%st(1),%st		# at1 *= bt1

  222. 	faddp	%st,%st(2)		# ct1 += at1

  223. 	FLD	1 * SIZE(A2)		# at1  = *(a_offset2 + 0 * lda)

  224. 

  225. 	fmul	%st(1),%st		# at1 *= bt1

  226. 	faddp	%st,%st(3)		# ct2 += at1

  227. 	FLD	1 * SIZE(A1, LDA, 2)	# at   = *(a_offset  + 2 * lda)

  228. 

  229. 	fmul	%st(1),%st

  230. 	faddp	%st,%st(4)

  231. 	FLD	1 * SIZE(A2, LDA, 2)	# at1  = *(a_offset2 + 2 * lda)

  232. 

  233. 	fmulp	%st, %st(1)

  234. 	faddp	%st,%st(4)

  235. 	FLD	2 * SIZE(X1)

  236. 

  237. 	FLD	2 * SIZE(A1)		# at   = *(a_offset  + 0 * lda)

  238. 	fmul	%st(1),%st		# at1 *= bt1

  239. 	faddp	%st,%st(2)		# ct1 += at1

  240. 

  241. 	FLD	2 * SIZE(A2)		# at1  = *(a_offset2 + 0 * lda)

  242. 	fmul	%st(1),%st		# at1 *= bt1

  243. 	faddp	%st,%st(3)		# ct2 += at1

  244. 

  245. 	FLD	2 * SIZE(A1, LDA, 2)	# at   = *(a_offset  + 2 * lda)

  246. 	fmul	%st(1),%st

  247. 	faddp	%st,%st(4)

  248. 

  249. 	FLD	2 * SIZE(A2, LDA, 2)	# at1  = *(a_offset2 + 2 * lda)

  250. 	fmulp	%st, %st(1)

  251. 	faddp	%st,%st(4)

  252. 

  253. 	FLD	3 * SIZE(X1)

  254. 	FLD	3 * SIZE(A1)		# at   = *(a_offset  + 0 * lda)

  255. 	fmul	%st(1),%st		# at1 *= bt1

  256. 

  257. 	faddp	%st,%st(2)		# ct1 += at1

  258. 	FLD	3 * SIZE(A2)		# at1  = *(a_offset2 + 0 * lda)

  259. 	fmul	%st(1),%st		# at1 *= bt1

  260. 

  261. 	faddp	%st,%st(3)		# ct2 += at1

  262. 	FLD	3 * SIZE(A1, LDA, 2)	# at   = *(a_offset  + 2 * lda)

  263. 	fmul	%st(1),%st

  264. 

  265. 	faddp	%st,%st(4)

  266. 	FLD	3 * SIZE(A2, LDA, 2)	# at1  = *(a_offset2 + 2 * lda)

  267. 	fmulp	%st, %st(1)

  268. 

  269. 	addq	$4 * SIZE, A1

  270. 	faddp	%st,%st(4)

  271. 	addq	$4 * SIZE, A2

  272. 

  273. 	FLD	4 * SIZE(X1)

  274. 	addq	$4 * SIZE, X1

  275. 

  276. 	decq	I

  277. 	jg	.L80

  278. 	ALIGN_3

  279. 

  280. .L51:

  281. 	movq	MIN_M, I

  282. 	andq	$3,    I

  283. 	je	.L81

  284. 	ALIGN_3

  285. 

  286. .L52:

  287. 	FLD	(A1)			# at   = *(a_offset  + 0 * lda)

  288. 	fmul	%st(1),%st		# at1 *= bt1

  289. 	faddp	%st,%st(2)		# ct1 += at1

  290. 

  291. 	FLD	(A2)			# at1  = *(a_offset2 + 0 * lda)

  292. 	fmul	%st(1),%st		# at1 *= bt1

  293. 	faddp	%st,%st(3)		# ct2 += at1

  294. 

  295. 	FLD	(A1, LDA, 2)		# at   = *(a_offset  + 2 * lda)

  296. 	fmul	%st(1),%st

  297. 	faddp	%st,%st(4)

  298. 

  299. 	FLD	(A2, LDA, 2)		# at1  = *(a_offset2 + 2 * lda)

  300. 	fmulp	%st, %st(1)

  301. 	faddp	%st,%st(4)

  302. 	FLD	1 * SIZE(X1)

  303. 

  304. 	addq	$SIZE, A1

  305. 	addq	$SIZE, A2

  306. 	addq	$SIZE, X1

  307. 	decq	I

  308. 	jg	.L52

  309. 	ALIGN_3

  310. 

  311. .L81:

  312. 	ffreep	%st(0)

  313. 

  314. 	fxch	%st(4)

  315. 	fmul	%st, %st(4)

  316. 	fmul	%st, %st(1)

  317. 	fmul	%st, %st(2)

  318. 	fmul	%st, %st(3)

  319. 	fxch	%st(4)

  320. 

  321. 	FLD	(Y1)

  322. 	faddp	%st, %st(1)

  323. 	FST	(Y1)

  324. 	addq	INCY, Y1

  325. 

  326. 	FLD	(Y1)

  327. 	faddp	%st, %st(1)

  328. 	FST	(Y1)

  329. 	addq	INCY, Y1

  330. 

  331. 	FLD	(Y1)

  332. 	faddp	%st, %st(1)

  333. 	FST	(Y1)

  334. 	addq	INCY, Y1

  335. 

  336. 	FLD	(Y1)

  337. 	faddp	%st, %st(1)

  338. 	FST	(Y1)

  339. 	addq	INCY, Y1

  340. 

  341. 	decq	J

  342. 	jg	.L48

  343. 	ALIGN_3

  344. 

  345. .L47:

  346. 	movq	N,  J

  347. 	andq	$3, J

  348. 	jle	.L60

  349. 	ALIGN_2

  350. 

  351. .L61:

  352. 	movq	A, A1			# a_offset = a

  353. 	fldz				# ct1 = ZERO

  354. 	fldz				# ct1 = ZERO

  355. 

  356. 	addq	LDA, A

  357. 	fldz				# ct1 = ZERO

  358. 	fldz				# ct1 = ZERO

  359. 

  360. 	movq	XP, X1

  361. 

  362. 	movq	MIN_M, I

  363. 	sarq	$3,I

  364. 	jle	.L64

  365. 	ALIGN_3

  366. 

  367. .L65:

  368. 	FLD	0 * SIZE(X1)

  369. 	FLD	0 * SIZE(A1)

  370. 	fmulp	%st, %st(1)

  371. 	faddp	%st,%st(1)

  372. 

  373. 	FLD	1 * SIZE(X1)

  374. 	FLD	1 * SIZE(A1)

  375. 	fmulp	%st, %st(1)

  376. 	faddp	%st,%st(2)

  377. 

  378. 	FLD	2 * SIZE(X1)

  379. 	FLD	2 * SIZE(A1)

  380. 	fmulp	%st, %st(1)

  381. 	faddp	%st,%st(3)

  382. 

  383. 	FLD	3 * SIZE(X1)

  384. 	FLD	3 * SIZE(A1)

  385. 	fmulp	%st, %st(1)

  386. 	faddp	%st,%st(4)

  387. 

  388. 	FLD	4 * SIZE(X1)

  389. 	FLD	4 * SIZE(A1)

  390. 	fmulp	%st, %st(1)

  391. 	faddp	%st,%st(1)

  392. 

  393. 	FLD	5 * SIZE(X1)

  394. 	FLD	5 * SIZE(A1)

  395. 	fmulp	%st, %st(1)

  396. 	faddp	%st,%st(2)

  397. 

  398. 	FLD	6 * SIZE(X1)

  399. 	FLD	6 * SIZE(A1)

  400. 	fmulp	%st, %st(1)

  401. 	faddp	%st,%st(3)

  402. 

  403. 	FLD	7 * SIZE(X1)

  404. 	FLD	7 * SIZE(A1)

  405. 	fmulp	%st, %st(1)

  406. 	faddp	%st,%st(4)

  407. 

  408. 	addq	$8 * SIZE, X1

  409. 	addq	$8 * SIZE, A1

  410. 

  411. 	decq	I

  412. 	jg	.L65

  413. 	ALIGN_3

  414. 

  415. .L64:

  416. 	movq	MIN_M, I

  417. 	andq	$7, I

  418. 	jle	.L70

  419. 	ALIGN_3

  420. 

  421. .L71:

  422. 	FLD	(X1)

  423. 	FLD	(A1)

  424. 	fmulp	%st, %st(1)

  425. 	faddp	%st,%st(1)

  426. 

  427. 	addq	$SIZE, X1

  428. 	addq	$SIZE, A1

  429. 	decq	I

  430. 	jg	.L71

  431. 	ALIGN_3

  432. 

  433. .L70:

  434. 	faddp	%st, %st(1)

  435. 	faddp	%st, %st(1)

  436. 	faddp	%st, %st(1)

  437. 

  438. 	fmul	%st(1),%st

  439. 	FLD	(Y1)

  440. 	faddp	%st, %st(1)

  441. 	FST	(Y1)

  442. 	addq	INCY, Y1

  443. 	decq	J

  444. 	jg	.L61

  445. 	ALIGN_3

  446. 

  447. .L60:

  448. 	addq	NLDA, A

  449. 

  450. 	addq	$P, IS

  451. 	cmpq	M,  IS

  452. 	jl	.L32

  453. 	ALIGN_3

  454. 

  455. .L79:

  456. 	EMMS

  457. 

  458. 	movq	  0(%rsp), %rbx

  459. 	movq	  8(%rsp), %rbp

  460. 	movq	 16(%rsp), %r12

  461. 	movq	 24(%rsp), %r13

  462. 	movq	 32(%rsp), %r14

  463. 	movq	 40(%rsp), %r15

  464. 	addq	$STACKSIZE, %rsp

  465. 	ret

  466. 	EPILOGUE