OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_tcopy_4.S

523 lines
11 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(PENTIUM4) || defined(GENERIC)

  43. #define PREFETCHSIZE	16

  44. #define PREFETCH      prefetcht0

  45. #define PREFETCHW     prefetcht0

  46. #endif

  47. 

  48. #ifdef NEHALEM

  49. #define PREFETCHSIZE	12

  50. #define PREFETCH      prefetcht0

  51. #define MOVUPS_A	movups

  52. #endif

  53. 

  54. #ifdef SANDYBRIDGE

  55. #define PREFETCHSIZE	12

  56. #define PREFETCH      prefetcht0

  57. #define MOVUPS_A	movups

  58. #endif

  59. 

  60. #if defined(CORE2) || defined(PENRYN) || defined(DUNNINGTON)

  61. #define PREFETCHSIZE	16

  62. #define PREFETCH      prefetcht0

  63. #define PREFETCHW     prefetcht0

  64. #endif

  65. 

  66. #ifdef OPTERON

  67. #define PREFETCHSIZE	16

  68. #define PREFETCH      prefetch

  69. #define PREFETCHW     prefetchw

  70. #endif

  71. 

  72. #ifdef MOVUPS_A

  73. #define MOVUPS_A1(OFF, ADDR, REGS)		MOVUPS_A	OFF(ADDR), REGS

  74. #define MOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS)	MOVUPS_A	OFF(ADDR, BASE, SCALE), REGS

  75. #else

  76. #define MOVUPS_A1(OFF, ADDR, REGS)		movsd	OFF(ADDR), REGS; movhps	OFF + 8(ADDR), REGS

  77. #define MOVUPS_A2(OFF, ADDR, BASE, SCALE, REGS)	movsd	OFF(ADDR, BASE, SCALE), REGS; movhps	OFF + 8(ADDR, BASE, SCALE), REGS

  78. #endif

  79. 

  80. #ifndef WINDOWS_ABI

  81. 

  82. #define N	ARG1	/* rsi */

  83. #define M	ARG2	/* rdi */

  84. #define A	ARG3	/* rdx */

  85. #define LDA	ARG4	/* rcx */

  86. #define B	ARG5	/* r8  */

  87. 

  88. #define AO1	%r9

  89. #define AO2	%r10

  90. #define LDA3	%r11

  91. #define M8	%r12

  92. 

  93. #else

  94. 

  95. #define STACKSIZE 256

  96. 

  97. #define N	ARG1	/* rdx */

  98. #define M	ARG2	/* rcx */

  99. #define A	ARG3	/* r8  */

  100. #define LDA	ARG4	/* r9  */

  101. #define OLD_B		64 + 32 + STACKSIZE(%rsp)

  102. 

  103. #define B	%r12

  104. 

  105. #define AO1	%rsi

  106. #define AO2	%rdi

  107. #define LDA3	%r10

  108. #define M8	%r11

  109. #endif

  110. 

  111. #define I	%rax

  112. 

  113. #define B0	%rbp

  114. #define	B2	%r14

  115. #define	B3	%r15

  116. 

  117. 	PROLOGUE

  118. 	PROFCODE

  119. 

  120. #ifdef WINDOWS_ABI

  121. 	pushq	%rdi

  122. 	pushq	%rsi

  123. #endif

  124. 

  125. 	pushq	%r15

  126. 	pushq	%r14

  127. 	pushq	%r13

  128. 	pushq	%r12

  129. 	pushq	%rbp

  130. 

  131. #ifdef WINDOWS_ABI

  132. 	movq	OLD_B,     B

  133. #endif

  134. 

  135. 	subq	$-16 * SIZE, B

  136. 

  137. 	movq	M,    B2

  138. 	movq	M,    B3

  139. 

  140. 	andq	$-4,  B2

  141. 	andq	$-2,  B3

  142. 

  143. 	imulq	N,    B2

  144. 	imulq	N,    B3

  145. 

  146. 	leaq	(B, B2, SIZE), B2

  147. 	leaq	(B, B3, SIZE), B3

  148. 

  149. 	leaq	(,LDA, SIZE), LDA

  150. 	leaq	(LDA, LDA, 2), LDA3

  151. 

  152. 	leaq	(, N, SIZE), M8

  153. 

  154. 	cmpq	$4, N

  155. 	jl	.L30

  156. 	ALIGN_4

  157. 

  158. .L21:

  159. 	subq	$4, N

  160. 

  161. 	movq	A, AO1

  162. 	leaq	(A, LDA, 2), AO2

  163. 	leaq	(A, LDA, 4), A

  164. 

  165. 	movq	B, B0

  166. 	addq	$16 * SIZE, B

  167. 

  168. 	movq	M, I

  169. 	sarq	$3, I

  170. 	jle	.L24

  171. 	ALIGN_4

  172. 

  173. .L23:

  174. #ifdef PREFETCH

  175. 	PREFETCH	PREFETCHSIZE * SIZE(AO1)

  176. #endif

  177. 

  178. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  179. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  180. 	MOVUPS_A1(4 * SIZE, AO1, %xmm2)

  181. 	MOVUPS_A1(6 * SIZE, AO1, %xmm3)

  182. 

  183. #ifdef PREFETCHW

  184. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  185. #endif

  186. 

  187. 	movaps	%xmm0,  -16 * SIZE(B0)

  188. 	movaps	%xmm1,  -14 * SIZE(B0)

  189. 	movaps	%xmm2,  -16 * SIZE(B0, M8, 4)

  190. 	movaps	%xmm3,  -14 * SIZE(B0, M8, 4)

  191. 

  192. #ifdef PREFETCH

  193. 	PREFETCH	PREFETCHSIZE * SIZE(AO1, LDA)

  194. #endif

  195. 

  196. 	MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm0)

  197. 	MOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm1)

  198. 	MOVUPS_A2(4 * SIZE, AO1, LDA, 1, %xmm2)

  199. 	MOVUPS_A2(6 * SIZE, AO1, LDA, 1, %xmm3)

  200. 

  201. #ifdef PREFETCHW

  202. 	PREFETCHW	(PREFETCHSIZE * 4 +  8) * SIZE(B)

  203. #endif

  204. 

  205. 	movaps	%xmm0,  -12 * SIZE(B0)

  206. 	movaps	%xmm1,  -10 * SIZE(B0)

  207. 	movaps	%xmm2,  -12 * SIZE(B0, M8, 4)

  208. 	movaps	%xmm3,  -10 * SIZE(B0, M8, 4)

  209. 

  210. #ifdef PREFETCH

  211. 	PREFETCH	PREFETCHSIZE * SIZE(AO2)

  212. #endif

  213. 

  214. 	MOVUPS_A1(0 * SIZE, AO2, %xmm0)

  215. 	MOVUPS_A1(2 * SIZE, AO2, %xmm1)

  216. 	MOVUPS_A1(4 * SIZE, AO2, %xmm2)

  217. 	MOVUPS_A1(6 * SIZE, AO2, %xmm3)

  218. 

  219. #ifdef PREFETCHW

  220. 	PREFETCHW	(PREFETCHSIZE * 4 + 16) * SIZE(B)

  221. #endif

  222. 

  223. 	movaps	%xmm0,   -8 * SIZE(B0)

  224. 	movaps	%xmm1,   -6 * SIZE(B0)

  225. 	movaps	%xmm2,   -8 * SIZE(B0, M8, 4)

  226. 	movaps	%xmm3,   -6 * SIZE(B0, M8, 4)

  227. 

  228. #ifdef PREFETCH

  229. 	PREFETCH	PREFETCHSIZE * SIZE(AO2, LDA)

  230. #endif

  231. 

  232. 	MOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm0)

  233. 	MOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm1)

  234. 	MOVUPS_A2(4 * SIZE, AO2, LDA, 1, %xmm2)

  235. 	MOVUPS_A2(6 * SIZE, AO2, LDA, 1, %xmm3)

  236. 

  237. #ifdef PREFETCHW

  238. 	PREFETCHW	(PREFETCHSIZE * 4 + 24) * SIZE(B)

  239. #endif

  240. 

  241. 	movaps	%xmm0,   -4 * SIZE(B0)

  242. 	movaps	%xmm1,   -2 * SIZE(B0)

  243. 	movaps	%xmm2,   -4 * SIZE(B0, M8, 4)

  244. 	movaps	%xmm3,   -2 * SIZE(B0, M8, 4)

  245. 

  246. 	addq	$8 * SIZE, AO1

  247. 	addq	$8 * SIZE, AO2

  248. 	leaq	(B0, M8, 8), B0

  249. 

  250. 	decq	I

  251. 	jg	.L23

  252. 	ALIGN_4

  253. 

  254. .L24:

  255. 	testq	$4, M

  256. 	jle	.L26

  257. 

  258. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  259. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  260. 	MOVUPS_A2(0 * SIZE, AO1, LDA, 1, %xmm2)

  261. 	MOVUPS_A2(2 * SIZE, AO1, LDA, 1, %xmm3)

  262. 

  263. 	movaps	%xmm0,  -16 * SIZE(B0)

  264. 	movaps	%xmm1,  -14 * SIZE(B0)

  265. 	movaps	%xmm2,  -12 * SIZE(B0)

  266. 	movaps	%xmm3,  -10 * SIZE(B0)

  267. 

  268. 	MOVUPS_A1(0 * SIZE, AO2, %xmm0)

  269. 	MOVUPS_A1(2 * SIZE, AO2, %xmm1)

  270. 	MOVUPS_A2(0 * SIZE, AO2, LDA, 1, %xmm2)

  271. 	MOVUPS_A2(2 * SIZE, AO2, LDA, 1, %xmm3)

  272. 

  273. 	movaps	%xmm0,   -8 * SIZE(B0)

  274. 	movaps	%xmm1,   -6 * SIZE(B0)

  275. 	movaps	%xmm2,   -4 * SIZE(B0)

  276. 	movaps	%xmm3,   -2 * SIZE(B0)

  277. 

  278. 	addq	$4 * SIZE, AO1

  279. 	addq	$4 * SIZE, AO2

  280. 	leaq	(B0, M8, 4), B0

  281. 	ALIGN_4

  282. 

  283. .L26:

  284. 	testq	$2, M

  285. 	jle	.L28

  286. 

  287. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  288. 	MOVUPS_A2(0 * SIZE, AO1, LDA,  1, %xmm1)

  289. 	MOVUPS_A1(0 * SIZE, AO2, %xmm2)

  290. 	MOVUPS_A2(0 * SIZE, AO2, LDA,  1, %xmm3)

  291. 

  292. 	movaps	%xmm0,  -16 * SIZE(B2)

  293. 	movaps	%xmm1,  -14 * SIZE(B2)

  294. 	movaps	%xmm2,  -12 * SIZE(B2)

  295. 	movaps	%xmm3,  -10 * SIZE(B2)

  296. 

  297. 	addq	$2 * SIZE, AO1

  298. 	addq	$2 * SIZE, AO2

  299. 	subq	$-8 * SIZE, B2

  300. 	ALIGN_4

  301. 

  302. .L28:

  303. 	testq	$1, M

  304. 	jle	.L29

  305. 

  306. 	movsd	0 * SIZE(AO1),      %xmm0

  307. 	movsd	0 * SIZE(AO1, LDA), %xmm1

  308. 	movsd	0 * SIZE(AO2),      %xmm2

  309. 	movsd	0 * SIZE(AO2, LDA), %xmm3

  310. 

  311. 	unpcklpd %xmm1, %xmm0

  312. 	unpcklpd %xmm3, %xmm2

  313. 

  314. 	movaps	%xmm0,  -16 * SIZE(B3)

  315. 	movaps	%xmm2,  -14 * SIZE(B3)

  316. 	subq	$-4 * SIZE, B3

  317. 	ALIGN_4

  318. 

  319. .L29:

  320. 	cmpq	$4, N

  321. 	jge	.L21

  322. 	ALIGN_4

  323. 

  324. .L30:

  325. 	cmpq	$2, N

  326. 	jl	.L40

  327. 

  328. 	subq	$2, N

  329. 

  330. 	movq	A, AO1

  331. 	leaq	(A, LDA), AO2

  332. 	leaq	(A, LDA, 2), A

  333. 

  334. 	movq	B, B0

  335. 	addq	$8 * SIZE, B

  336. 

  337. 	movq	M, I

  338. 	sarq	$3, I

  339. 	jle	.L34

  340. 	ALIGN_4

  341. 

  342. .L33:

  343. #ifdef PREFETCH

  344. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO1)

  345. #endif

  346. 

  347. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  348. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  349. 	MOVUPS_A1(4 * SIZE, AO1, %xmm2)

  350. 	MOVUPS_A1(6 * SIZE, AO1, %xmm3)

  351. 

  352. #ifdef PREFETCHW

  353. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  354. #endif

  355. 

  356. 	movaps	%xmm0,  -16 * SIZE(B0)

  357. 	movaps	%xmm1,  -14 * SIZE(B0)

  358. 	movaps	%xmm2,  -16 * SIZE(B0, M8, 4)

  359. 	movaps	%xmm3,  -14 * SIZE(B0, M8, 4)

  360. 

  361. #ifdef PREFETCH

  362. 	PREFETCH	PREFETCHSIZE * 2 * SIZE(AO2)

  363. #endif

  364. 

  365. 	MOVUPS_A1(0 * SIZE, AO2, %xmm0)

  366. 	MOVUPS_A1(2 * SIZE, AO2, %xmm1)

  367. 	MOVUPS_A1(4 * SIZE, AO2, %xmm2)

  368. 	MOVUPS_A1(6 * SIZE, AO2, %xmm3)

  369. 

  370. #ifdef PREFETCHW

  371. 	PREFETCHW	(PREFETCHSIZE * 4 +  8) * SIZE(B)

  372. #endif

  373. 

  374. 	movaps	%xmm0,  -12 * SIZE(B0)

  375. 	movaps	%xmm1,  -10 * SIZE(B0)

  376. 	movaps	%xmm2,  -12 * SIZE(B0, M8, 4)

  377. 	movaps	%xmm3,  -10 * SIZE(B0, M8, 4)

  378. 

  379. 	addq	$8 * SIZE, AO1

  380. 	addq	$8 * SIZE, AO2

  381. 	leaq	(B0, M8, 8), B0

  382. 

  383. 	decq	I

  384. 	jg	.L33

  385. 	ALIGN_4

  386. 

  387. .L34:

  388. 	testq	$4, M

  389. 	jle	.L36

  390. 

  391. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  392. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  393. 	MOVUPS_A1(0 * SIZE, AO2, %xmm2)

  394. 	MOVUPS_A1(2 * SIZE, AO2, %xmm3)

  395. 

  396. 	movaps	%xmm0,  -16 * SIZE(B0)

  397. 	movaps	%xmm1,  -14 * SIZE(B0)

  398. 	movaps	%xmm2,  -12 * SIZE(B0)

  399. 	movaps	%xmm3,  -10 * SIZE(B0)

  400. 

  401. 	addq	$4 * SIZE, AO1

  402. 	addq	$4 * SIZE, AO2

  403. 	leaq	(B0, M8, 4), B0

  404. 	ALIGN_4

  405. 

  406. .L36:

  407. 	testq	$2, M

  408. 	jle	.L38

  409. 

  410. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  411. 	MOVUPS_A1(0 * SIZE, AO2, %xmm1)

  412. 

  413. 	movaps	%xmm0,  -16 * SIZE(B2)

  414. 	movaps	%xmm1,  -14 * SIZE(B2)

  415. 

  416. 	addq	$2 * SIZE, AO1

  417. 	addq	$2 * SIZE, AO2

  418. 	subq	$-4 * SIZE, B2

  419. 	ALIGN_4

  420. 

  421. .L38:

  422. 	testq	$1, M

  423. 	jle	.L40

  424. 

  425. 	movsd	0 * SIZE(AO1),      %xmm0

  426. 	movsd	0 * SIZE(AO2),      %xmm1

  427. 

  428. 	unpcklpd %xmm1, %xmm0

  429. 

  430. 	movaps	%xmm0,  -16 * SIZE(B3)

  431. 	subq	$-2 * SIZE, B3

  432. 	ALIGN_4

  433. 

  434. .L40:

  435. 	cmpq	$1, N

  436. 	jl	.L999

  437. 

  438. 	movq	A, AO1

  439. 

  440. 	movq	B, B0

  441. 

  442. 	movq	M, I

  443. 	sarq	$3, I

  444. 	jle	.L44

  445. 	ALIGN_4

  446. 

  447. .L43:

  448. #ifdef PREFETCH

  449. 	PREFETCH	PREFETCHSIZE * 4 * SIZE(AO1)

  450. #endif

  451. 

  452. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  453. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  454. 	MOVUPS_A1(4 * SIZE, AO1, %xmm2)

  455. 	MOVUPS_A1(6 * SIZE, AO1, %xmm3)

  456. 

  457. #ifdef PREFETCHW

  458. 	PREFETCHW	(PREFETCHSIZE * 4 +  0) * SIZE(B)

  459. #endif

  460. 

  461. 	movaps	%xmm0,  -16 * SIZE(B0)

  462. 	movaps	%xmm1,  -14 * SIZE(B0)

  463. 	movaps	%xmm2,  -16 * SIZE(B0, M8, 4)

  464. 	movaps	%xmm3,  -14 * SIZE(B0, M8, 4)

  465. 

  466. 	addq	$8 * SIZE, AO1

  467. 	leaq	(B0, M8, 8), B0

  468. 

  469. 	decq	I

  470. 	jg	.L43

  471. 	ALIGN_4

  472. 

  473. .L44:

  474. 	testq	$4, M

  475. 	jle	.L45

  476. 

  477. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  478. 	MOVUPS_A1(2 * SIZE, AO1, %xmm1)

  479. 

  480. 	movaps	%xmm0, -16 * SIZE(B0)

  481. 	movaps	%xmm1, -14 * SIZE(B0)

  482. 

  483. 	addq	$4 * SIZE, AO1

  484. 	leaq	(B0, M8, 4), B0

  485. 	ALIGN_4

  486. 

  487. .L45:

  488. 	testq	$2, M

  489. 	jle	.L46

  490. 

  491. 	MOVUPS_A1(0 * SIZE, AO1, %xmm0)

  492. 

  493. 	movaps	%xmm0,  -16 * SIZE(B2)

  494. 

  495. 	addq	$2 * SIZE, AO1

  496. 	subq	$-2 * SIZE, B2

  497. 	ALIGN_4

  498. 

  499. .L46:

  500. 	testq	$1, M

  501. 	jle	.L999

  502. 

  503. 	movsd	0 * SIZE(AO1),      %xmm0

  504. 

  505. 	movlpd	%xmm0,  -16 * SIZE(B3)

  506. 	jmp	.L999

  507. 	ALIGN_4

  508. 

  509. .L999:

  510. 	popq	%rbp

  511. 	popq	%r12

  512. 	popq	%r13

  513. 	popq	%r14

  514. 	popq	%r15

  515. 

  516. #ifdef WINDOWS_ABI

  517. 	popq	%rsi

  518. 	popq	%rdi

  519. #endif

  520. 	ret

  521. 

  522. 	EPILOGUE