OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/izamax_sse.S

555 lines
10 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define M	ARG1	/* rdi */

  43. #define X	ARG2	/* rsi */

  44. #define INCX	ARG3	/* rdx */

  45. 

  46. #define RET	%rax

  47. #define I	ARG4

  48. #define XX	%r10

  49. #define	MM	%r11

  50. 

  51. #ifdef USE_MIN

  52. #define maxps	minps

  53. #define maxss	minss

  54. #endif

  55. 

  56. #include "l1param.h"

  57. 

  58. 	PROLOGUE

  59. 	PROFCODE

  60. 

  61. 	SAVEREGISTERS

  62. 

  63. 	pxor	%xmm0, %xmm0

  64. 	xor	RET, RET

  65. 	testq	M, M

  66. 	jle	.L999

  67. 	testq	INCX, INCX

  68. 	jle	.L999

  69. 

  70. 	salq	$ZBASE_SHIFT, INCX

  71. 	movq	M, MM

  72. 	movq	X, XX

  73. 

  74. 	pcmpeqb	%xmm15, %xmm15

  75. 	psrld	$1, %xmm15

  76. 

  77. 	movss	0 * SIZE(X), %xmm0

  78. 	movss	1 * SIZE(X), %xmm1

  79. 	addq	INCX, X

  80. 	decq	M

  81. 	andps	%xmm15, %xmm0

  82. 	andps	%xmm15, %xmm1

  83. 	addps	%xmm1,  %xmm0

  84. 	shufps	$0, %xmm0, %xmm0

  85. 	movaps	%xmm0, %xmm1

  86. 	cmpq	$2 * SIZE, INCX

  87. 	jne	.L70

  88. 

  89. .L30:

  90. 	movq	M,  I

  91. 	sarq	$3, I

  92. 	jle	.L35

  93. 	ALIGN_4

  94. 

  95. .L31:

  96. #ifdef PREFETCH

  97. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  98. #endif

  99. 

  100. 	movsd	0 * SIZE(X), %xmm4

  101. 	movhps	2 * SIZE(X), %xmm4

  102. 	movsd	4 * SIZE(X), %xmm5

  103. 	movhps	6 * SIZE(X), %xmm5

  104. 

  105. 	movaps	%xmm4, %xmm6

  106. 

  107. 	shufps	$0x88, %xmm5, %xmm4

  108. 	shufps	$0xdd, %xmm5, %xmm6

  109. 

  110. 	andps	%xmm15, %xmm4

  111. 	andps	%xmm15, %xmm6

  112. 	addps	%xmm6,  %xmm4

  113. 	maxps	%xmm4,  %xmm0

  114. 

  115. 	movsd	 8 * SIZE(X), %xmm7

  116. 	movhps	10 * SIZE(X), %xmm7

  117. 	movsd	12 * SIZE(X), %xmm8

  118. 	movhps	14 * SIZE(X), %xmm8

  119. 	movaps	%xmm7, %xmm9

  120. 

  121. 	shufps	$0x88, %xmm8, %xmm7

  122. 	shufps	$0xdd, %xmm8, %xmm9

  123. 

  124. 	andps	%xmm15, %xmm7

  125. 	andps	%xmm15, %xmm9

  126. 	addps	%xmm9,  %xmm7

  127. 	maxps	%xmm7,  %xmm0

  128. 

  129. 	addq	$16 * SIZE, X

  130. 	decq	I

  131. 	jg	.L31

  132. 	ALIGN_4

  133. 

  134. .L35:

  135. 	andq	$7,  M

  136. 	jle	.L40

  137. 

  138. 	testq	$4, M

  139. 	je	.L36

  140. 

  141. 	movsd	0 * SIZE(X), %xmm4

  142. 	movhps	2 * SIZE(X), %xmm4

  143. 	movsd	4 * SIZE(X), %xmm5

  144. 	movhps	6 * SIZE(X), %xmm5

  145. 	movaps	%xmm4, %xmm6

  146. 

  147. 	shufps	$0x88, %xmm5, %xmm4

  148. 	shufps	$0xdd, %xmm5, %xmm6

  149. 

  150. 	andps	%xmm15, %xmm4

  151. 	andps	%xmm15, %xmm6

  152. 	addps	%xmm6,  %xmm4

  153. 	maxps	%xmm4,  %xmm0

  154. 

  155. 	addq	$8 * SIZE, X

  156. 	ALIGN_3

  157. 

  158. .L36:

  159. 	testq	$2, M

  160. 	je	.L37

  161. 

  162. 	movss	0 * SIZE(X), %xmm4

  163. 	movss	1 * SIZE(X), %xmm5

  164. 	movss	2 * SIZE(X), %xmm6

  165. 	movss	3 * SIZE(X), %xmm7

  166. 	andps	%xmm15, %xmm4

  167. 	andps	%xmm15, %xmm5

  168. 	andps	%xmm15, %xmm6

  169. 	andps	%xmm15, %xmm7

  170. 	addps	%xmm5,  %xmm4

  171. 	addps	%xmm7,  %xmm6

  172. 	maxss	%xmm4,  %xmm0

  173. 	maxss	%xmm6,  %xmm1

  174. 	addq	$4 * SIZE, X

  175. 	ALIGN_3

  176. 

  177. .L37:

  178. 	testq	$1, M

  179. 	je	.L40

  180. 

  181. 	movss	0 * SIZE(X), %xmm4

  182. 	movss	1 * SIZE(X), %xmm5

  183. 	andps	%xmm15, %xmm4

  184. 	andps	%xmm15, %xmm5

  185. 	addps	%xmm5,  %xmm4

  186. 	maxss	%xmm4,  %xmm0

  187. 	ALIGN_4

  188. 

  189. .L40:

  190. 	movq	XX, X

  191. 	movq	MM, M

  192. 

  193. 	maxps	%xmm1, %xmm0

  194. 	movaps	%xmm0, %xmm1

  195. 	movhlps %xmm0, %xmm0

  196. 	maxps	%xmm1, %xmm0

  197. 	movaps	%xmm0, %xmm1

  198. 	shufps	$1, %xmm0, %xmm0

  199. 	maxss	%xmm1, %xmm0

  200. 	shufps	$0, %xmm0, %xmm0

  201. 

  202. 	movq	M,  I

  203. 	sarq	$2, I

  204. 	jle	.L45

  205. 	ALIGN_4

  206. 

  207. .L41:

  208. #ifdef PREFETCH

  209. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  210. #endif

  211. 

  212. 	movsd	0 * SIZE(X), %xmm1

  213. 	movhps	2 * SIZE(X), %xmm1

  214. 	movsd	4 * SIZE(X), %xmm2

  215. 	movhps	6 * SIZE(X), %xmm2

  216. 

  217. 	movaps	%xmm1, %xmm3

  218. 

  219. 	shufps	$0x88, %xmm2, %xmm1

  220. 	shufps	$0xdd, %xmm2, %xmm3

  221. 

  222. 	andps	%xmm15, %xmm1

  223. 	andps	%xmm15, %xmm3

  224. 	addps	%xmm3,  %xmm1

  225. 

  226. 	cmpeqps	%xmm0, %xmm1

  227. #ifndef C_SUN

  228. 	movmskps %xmm1, %r11

  229. #else

  230. 	.long	0xd9500f4c

  231. #endif

  232. 	testq	 $15, %r11

  233. 	jne	 .L43

  234. 

  235. 	addq	$8 * SIZE, X

  236. 	addq	$4, RET

  237. 	decq	I

  238. 	jg	.L41

  239. 	jmp	.L45

  240. 	ALIGN_4

  241. 

  242. .L43:

  243. 	movss	0 * SIZE(X), %xmm1

  244. 	movss	1 * SIZE(X), %xmm2

  245. 	movss	2 * SIZE(X), %xmm3

  246. 	movss	3 * SIZE(X), %xmm4

  247. 	movss	4 * SIZE(X), %xmm5

  248. 	movss	5 * SIZE(X), %xmm6

  249. 	movss	6 * SIZE(X), %xmm7

  250. 	movss	7 * SIZE(X), %xmm8

  251. 	addq	$8 * SIZE, X

  252. 

  253. 	andps	%xmm15, %xmm1

  254. 	andps	%xmm15, %xmm2

  255. 	andps	%xmm15, %xmm3

  256. 	andps	%xmm15, %xmm4

  257. 	andps	%xmm15, %xmm5

  258. 	andps	%xmm15, %xmm6

  259. 	andps	%xmm15, %xmm7

  260. 	andps	%xmm15, %xmm8

  261. 

  262. 	addps	%xmm2,  %xmm1

  263. 	addps	%xmm4,  %xmm3

  264. 	addps	%xmm6,  %xmm5

  265. 	addps	%xmm8,  %xmm7

  266. 

  267. 	incq	RET

  268. 	comiss	%xmm0, %xmm1

  269. 	je	.L999

  270. 	incq	RET

  271. 	comiss	%xmm0, %xmm3

  272. 	je	.L999

  273. 	incq	RET

  274. 	comiss	%xmm0, %xmm5

  275. 	je	.L999

  276. 	incq	RET

  277. 	comiss	%xmm0, %xmm7

  278. 	je	.L999

  279. 	ALIGN_3

  280. 

  281. .L45:

  282. 	testq	$2, M

  283. 	je	.L47

  284. 

  285. 	movss	0 * SIZE(X), %xmm1

  286. 	movss	1 * SIZE(X), %xmm2

  287. 	movss	2 * SIZE(X), %xmm3

  288. 	movss	3 * SIZE(X), %xmm4

  289. 	addq	$4 * SIZE, X

  290. 

  291. 	andps	%xmm15, %xmm1

  292. 	andps	%xmm15, %xmm2

  293. 	andps	%xmm15, %xmm3

  294. 	andps	%xmm15, %xmm4

  295. 	addps	%xmm2,  %xmm1

  296. 	addps	%xmm4,  %xmm3

  297. 

  298. 	incq	RET

  299. 	comiss	%xmm0, %xmm1

  300. 	je	.L999

  301. 	incq	RET

  302. 	comiss	%xmm0, %xmm3

  303. 	je	.L999

  304. 	ALIGN_3

  305. 

  306. .L47:

  307. 	incq	RET

  308. 	jmp	.L999

  309. 	ALIGN_3

  310. 

  311. .L70:

  312. 	movq	M,  I

  313. 	sarq	$3, I

  314. 	jle	.L75

  315. 	ALIGN_4

  316. 

  317. .L71:

  318. #ifdef PREFETCH

  319. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  320. #endif

  321. 

  322. 	movsd	0 * SIZE(X), %xmm4

  323. 	addq	INCX, X

  324. 	movhps	0 * SIZE(X), %xmm4

  325. 	addq	INCX, X

  326. 	movsd	0 * SIZE(X), %xmm5

  327. 	addq	INCX, X

  328. 	movhps	0 * SIZE(X), %xmm5

  329. 	addq	INCX, X

  330. 

  331. 	movaps	%xmm4, %xmm6

  332. 

  333. 	shufps	$0x88, %xmm5, %xmm4

  334. 	shufps	$0xdd, %xmm5, %xmm6

  335. 

  336. 	andps	%xmm15, %xmm4

  337. 	andps	%xmm15, %xmm6

  338. 	addps	%xmm6,  %xmm4

  339. 	maxps	%xmm4,  %xmm0

  340. 

  341. 	movsd	 0 * SIZE(X), %xmm7

  342. 	addq	INCX, X

  343. 	movhps	 0 * SIZE(X), %xmm7

  344. 	addq	INCX, X

  345. 	movsd	 0 * SIZE(X), %xmm8

  346. 	addq	INCX, X

  347. 	movhps	 0 * SIZE(X), %xmm8

  348. 	addq	INCX, X

  349. 	movaps	%xmm7, %xmm9

  350. 

  351. 	shufps	$0x88, %xmm8, %xmm7

  352. 	shufps	$0xdd, %xmm8, %xmm9

  353. 

  354. 	andps	%xmm15, %xmm7

  355. 	andps	%xmm15, %xmm9

  356. 	addps	%xmm9,  %xmm7

  357. 	maxps	%xmm7,  %xmm0

  358. 

  359. 	decq	I

  360. 	jg	.L71

  361. 	ALIGN_4

  362. 

  363. .L75:

  364. 	andq	$7,  M

  365. 	jle	.L80

  366. 

  367. 	testq	$4, M

  368. 	je	.L76

  369. 

  370. 	movsd	0 * SIZE(X), %xmm4

  371. 	addq	INCX, X

  372. 	movhps	0 * SIZE(X), %xmm4

  373. 	addq	INCX, X

  374. 	movsd	0 * SIZE(X), %xmm5

  375. 	addq	INCX, X

  376. 	movhps	0 * SIZE(X), %xmm5

  377. 	addq	INCX, X

  378. 	movaps	%xmm4, %xmm6

  379. 

  380. 	shufps	$0x88, %xmm5, %xmm4

  381. 	shufps	$0xdd, %xmm5, %xmm6

  382. 

  383. 	andps	%xmm15, %xmm4

  384. 	andps	%xmm15, %xmm6

  385. 	addps	%xmm6,  %xmm4

  386. 	maxps	%xmm4,  %xmm0

  387. 	ALIGN_3

  388. 

  389. .L76:

  390. 	testq	$2, M

  391. 	je	.L77

  392. 

  393. 	movss	0 * SIZE(X), %xmm4

  394. 	movss	1 * SIZE(X), %xmm5

  395. 	addq	INCX, X

  396. 	movss	0 * SIZE(X), %xmm6

  397. 	movss	1 * SIZE(X), %xmm7

  398. 	addq	INCX, X

  399. 	andps	%xmm15, %xmm4

  400. 	andps	%xmm15, %xmm5

  401. 	andps	%xmm15, %xmm6

  402. 	andps	%xmm15, %xmm7

  403. 	addps	%xmm5,  %xmm4

  404. 	addps	%xmm7,  %xmm6

  405. 	maxss	%xmm4,  %xmm0

  406. 	maxss	%xmm6,  %xmm1

  407. 	ALIGN_3

  408. 

  409. .L77:

  410. 	testq	$1, M

  411. 	je	.L80

  412. 

  413. 	movss	0 * SIZE(X), %xmm4

  414. 	movss	1 * SIZE(X), %xmm5

  415. 	andps	%xmm15, %xmm4

  416. 	andps	%xmm15, %xmm5

  417. 	addps	%xmm5,  %xmm4

  418. 	maxss	%xmm4,  %xmm0

  419. 	ALIGN_4

  420. 

  421. .L80:

  422. 	movq	XX, X

  423. 	movq	MM, M

  424. 

  425. 	maxps	%xmm1, %xmm0

  426. 	movaps	%xmm0, %xmm1

  427. 	movhlps %xmm0, %xmm0

  428. 	maxps	%xmm1, %xmm0

  429. 	movaps	%xmm0, %xmm1

  430. 	shufps	$1, %xmm0, %xmm0

  431. 	maxss	%xmm1, %xmm0

  432. 	shufps	$0, %xmm0, %xmm0

  433. 

  434. 	movq	M,  I

  435. 	sarq	$2, I

  436. 	jle	.L85

  437. 	ALIGN_4

  438. 

  439. .L81:

  440. #ifdef PREFETCH

  441. 	PREFETCH (PREFETCHSIZE +  0) - PREOFFSET(X)

  442. #endif

  443. 

  444. 	movsd	0 * SIZE(X), %xmm1

  445. 	addq	INCX, X

  446. 	movhps	0 * SIZE(X), %xmm1

  447. 	addq	INCX, X

  448. 	movsd	0 * SIZE(X), %xmm2

  449. 	addq	INCX, X

  450. 	movhps	0 * SIZE(X), %xmm2

  451. 	addq	INCX, X

  452. 

  453. 	movaps	%xmm1, %xmm3

  454. 

  455. 	shufps	$0x88, %xmm2, %xmm1

  456. 	shufps	$0xdd, %xmm2, %xmm3

  457. 

  458. 	andps	%xmm15, %xmm1

  459. 	andps	%xmm15, %xmm3

  460. 	addps	%xmm3,  %xmm1

  461. 

  462. 	cmpeqps	%xmm0, %xmm1

  463. #ifndef C_SUN

  464. 	movmskps %xmm1, %r11

  465. #else

  466. 	.long	0xd9500f4c

  467. #endif

  468. 	testq	 $15, %r11

  469. 	jne	 .L83

  470. 

  471. 	addq	$4, RET

  472. 	decq	I

  473. 	jg	.L81

  474. 	jmp	.L85

  475. 	ALIGN_4

  476. 

  477. .L83:

  478. 	subq	INCX, X

  479. 	movss	0 * SIZE(X), %xmm7

  480. 	movss	1 * SIZE(X), %xmm8

  481. 	subq	INCX, X

  482. 	movss	0 * SIZE(X), %xmm5

  483. 	movss	1 * SIZE(X), %xmm6

  484. 	subq	INCX, X

  485. 	movss	0 * SIZE(X), %xmm3

  486. 	movss	1 * SIZE(X), %xmm4

  487. 	subq	INCX, X

  488. 	movss	0 * SIZE(X), %xmm1

  489. 	movss	1 * SIZE(X), %xmm2

  490. 

  491. 	andps	%xmm15, %xmm1

  492. 	andps	%xmm15, %xmm2

  493. 	andps	%xmm15, %xmm3

  494. 	andps	%xmm15, %xmm4

  495. 	andps	%xmm15, %xmm5

  496. 	andps	%xmm15, %xmm6

  497. 	andps	%xmm15, %xmm7

  498. 	andps	%xmm15, %xmm8

  499. 

  500. 	addps	%xmm2,  %xmm1

  501. 	addps	%xmm4,  %xmm3

  502. 	addps	%xmm6,  %xmm5

  503. 	addps	%xmm8,  %xmm7

  504. 

  505. 	incq	RET

  506. 	comiss	%xmm0, %xmm1

  507. 	je	.L999

  508. 	incq	RET

  509. 	comiss	%xmm0, %xmm3

  510. 	je	.L999

  511. 	incq	RET

  512. 	comiss	%xmm0, %xmm5

  513. 	je	.L999

  514. 	incq	RET

  515. 	comiss	%xmm0, %xmm7

  516. 	je	.L999

  517. 	ALIGN_3

  518. 

  519. .L85:

  520. 	testq	$2, M

  521. 	je	.L87

  522. 

  523. 	movss	0 * SIZE(X), %xmm1

  524. 	movss	1 * SIZE(X), %xmm2

  525. 	addq	INCX, X

  526. 	movss	0 * SIZE(X), %xmm3

  527. 	movss	1 * SIZE(X), %xmm4

  528. 	addq	INCX, X

  529. 

  530. 	andps	%xmm15, %xmm1

  531. 	andps	%xmm15, %xmm2

  532. 	andps	%xmm15, %xmm3

  533. 	andps	%xmm15, %xmm4

  534. 	addps	%xmm2,  %xmm1

  535. 	addps	%xmm4,  %xmm3

  536. 

  537. 	incq	RET

  538. 	comiss	%xmm0, %xmm1

  539. 	je	.L999

  540. 	incq	RET

  541. 	comiss	%xmm0, %xmm3

  542. 	je	.L999

  543. 	ALIGN_3

  544. 

  545. .L87:

  546. 	incq	RET

  547. 	ALIGN_4

  548. 

  549. .L999:

  550. 	RESTOREREGISTERS

  551. 

  552. 	ret

  553. 

  554. 	EPILOGUE