OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/power/zdot_hummer.S

530 lines
10 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #if defined(F_INTERFACE) && defined(F_INTERFACE_F2C)

  43. #define RESULT	r3

  44. #define N	r4

  45. #define X	r5

  46. #define INCX	r6

  47. #define Y	r7

  48. #define INCY	r8

  49. #else

  50. #define N	r3

  51. #define X	r4

  52. #define INCX	r5

  53. #define Y	r6

  54. #define INCY	r7

  55. #endif

  56. 

  57. #define INCX2	r9

  58. #define INCY2	r10

  59. 

  60. #define C1	f1

  61. #define C2	f2

  62. #define C3	f0

  63. #define C4	f3

  64. #define C5	f4

  65. #define C6	f5

  66. #define C7	f6

  67. #define C8	f7

  68. 

  69. #define A1	f8

  70. #define A2	f9

  71. #define A3	f10

  72. #define A4	f11

  73. #define A5	f12

  74. #define A6	f13

  75. #define A7	f14

  76. #define A8	f15

  77. 

  78. #define B1	f16

  79. #define B2	f17

  80. #define B3	f18

  81. #define B4	f19

  82. #define B5	f20

  83. #define B6	f21

  84. #define B7	f22

  85. #define B8	f23

  86. 

  87. #ifndef CONJ

  88. #define FXCXNPMA fxcxnpma

  89. #else

  90. #define FXCXNPMA fxcxnsma

  91. #endif

  92. 

  93. 	PROLOGUE

  94. 	PROFCODE

  95. 

  96. 	li	r10, -16

  97. 

  98. 	stfpdux	f14, SP, r10

  99. 	stfpdux	f15, SP, r10

  100. 

  101. 	stfpdux	f16, SP, r10

  102. 	stfpdux	f17, SP, r10

  103. 	stfpdux	f18, SP, r10

  104. 	stfpdux	f19, SP, r10

  105. 

  106. 	stfpdux	f20, SP, r10

  107. 	stfpdux	f21, SP, r10

  108. 	stfpdux	f22, SP, r10

  109. 	stfpdux	f23, SP, r10

  110. 

  111. 	li	r10,   0

  112. 	stwu	r10,   -4(SP)

  113. 	stwu	r10,   -4(SP)

  114. 	stwu	r10,   -4(SP)

  115. 	stwu	r10,   -4(SP)

  116. 

  117. #ifdef F_INTERFACE

  118. 	LDINT	N,    0(N)

  119. 	LDINT	INCX, 0(INCX)

  120. 	LDINT	INCY, 0(INCY)

  121. #endif

  122. 

  123. 	lfpdx	C1, SP, r10		# Zero clear

  124. 

  125. 	slwi	INCX,  INCX, BASE_SHIFT

  126. 	add	INCX2, INCX, INCX

  127. 	fpmr	C2, C1

  128. 

  129. 	slwi	INCY,  INCY, BASE_SHIFT

  130. 	fpmr	C3, C1

  131. 	add	INCY2, INCY, INCY

  132. 	fpmr	C4, C1

  133. 

  134. 	fpmr	C5, C1

  135. 	fpmr	C6, C1

  136. 	fpmr	C7, C1

  137. 	fpmr	C8, C1

  138. 

  139. 	cmpwi	cr0, N, 0

  140. 	ble	LL(99)

  141. 

  142. #ifdef F_INTERFACE

  143. 	cmpwi	cr0, INCX, 0

  144. 	bge+	LL(05)

  145. 

  146. 	subi	r0, N, 1

  147. 	mullw	r0, r0, INCX2

  148. 	sub	X, X, r0

  149. 	.align 4

  150. 

  151. LL(05):

  152. 	cmpwi	cr0, INCY, 0

  153. 	bge+	LL(06)

  154. 

  155. 	subi	r0, N, 1

  156. 	mullw	r0, r0, INCY2

  157. 	sub	Y, Y, r0

  158. 	.align 4

  159. 

  160. LL(06):

  161. #endif

  162. 

  163. 	andi.	r0, X, 2 * SIZE - 1

  164. 	bne	LL(100)

  165. 	andi.	r0, Y, 2 * SIZE - 1

  166. 	bne	LL(100)

  167. 

  168. /* X is aligned, Y is aligned */

  169. LL(10):

  170. 	sub	X, X, INCX2

  171. 	sub	Y, Y, INCY2

  172. 

  173. 	srawi.	r0, N, 3

  174. 	mtspr	CTR,  r0

  175. 	beq-	LL(15)

  176. 

  177. 	LFPDUX	A1,    X, INCX2

  178. 	LFPDUX	B1,    Y, INCY2

  179. 	LFPDUX	A2,    X, INCX2

  180. 	LFPDUX	B2,    Y, INCY2

  181. 

  182. 	LFPDUX	A3,    X, INCX2

  183. 	LFPDUX	B3,    Y, INCY2

  184. 	LFPDUX	A4,    X, INCX2

  185. 	LFPDUX	B4,    Y, INCY2

  186. 

  187. 	LFPDUX	A5,    X, INCX2

  188. 	LFPDUX	B5,    Y, INCY2

  189. 	LFPDUX	A6,    X, INCX2

  190. 	LFPDUX	B6,    Y, INCY2

  191. 

  192. 	LFPDUX	A7,    X, INCX2

  193. 	LFPDUX	B7,    Y, INCY2

  194. 	LFPDUX	A8,    X, INCX2

  195. 	bdz	LL(14)

  196. 	.align 4

  197. 

  198. LL(13):

  199. 	fxcpmadd C1, A1, B1, C1

  200. 	LFPDUX	B8,    Y, INCY2

  201. 	FXCXNPMA C2, A1, B1, C2

  202. 	LFPDUX	A1,    X, INCX2

  203. 	fxcpmadd C3, A2, B2, C3

  204. 	LFPDUX	B1,    Y, INCY2

  205. 	FXCXNPMA C4, A2, B2, C4

  206. 	LFPDUX	A2,    X, INCX2

  207. 

  208. 	fxcpmadd C5, A3, B3, C5

  209. 	LFPDUX	B2,    Y, INCY2

  210. 	FXCXNPMA C6, A3, B3, C6

  211. 	LFPDUX	A3,    X, INCX2

  212. 	fxcpmadd C7, A4, B4, C7

  213. 	LFPDUX	B3,    Y, INCY2

  214. 	FXCXNPMA C8, A4, B4, C8

  215. 	LFPDUX	A4,    X, INCX2

  216. 

  217. 	fxcpmadd C1, A5, B5, C1

  218. 	LFPDUX	B4,    Y, INCY2

  219. 	FXCXNPMA C2, A5, B5, C2

  220. 	LFPDUX	A5,    X, INCX2

  221. 	fxcpmadd C3, A6, B6, C3

  222. 	LFPDUX	B5,    Y, INCY2

  223. 	FXCXNPMA C4, A6, B6, C4

  224. 	LFPDUX	A6,    X, INCX2

  225. 

  226. 	fxcpmadd C5, A7, B7, C5

  227. 	LFPDUX	B6,    Y, INCY2

  228. 	FXCXNPMA C6, A7, B7, C6

  229. 	LFPDUX	A7,    X, INCX2

  230. 	fxcpmadd C7, A8, B8, C7

  231. 	LFPDUX	B7,    Y, INCY2

  232. 	FXCXNPMA C8, A8, B8, C8

  233. 	LFPDUX	A8,    X, INCX2

  234. 	bdnz	LL(13)

  235. 	.align 4

  236. 

  237. LL(14):

  238. 	LFPDUX	B8,    Y, INCY2

  239. 	fxcpmadd C1, A1, B1, C1

  240. 	FXCXNPMA C2, A1, B1, C2

  241. 	fxcpmadd C3, A2, B2, C3

  242. 	FXCXNPMA C4, A2, B2, C4

  243. 

  244. 	fxcpmadd C5, A3, B3, C5

  245. 	FXCXNPMA C6, A3, B3, C6

  246. 	fxcpmadd C7, A4, B4, C7

  247. 	FXCXNPMA C8, A4, B4, C8

  248. 

  249. 	fxcpmadd C1, A5, B5, C1

  250. 	FXCXNPMA C2, A5, B5, C2

  251. 	fxcpmadd C3, A6, B6, C3

  252. 	FXCXNPMA C4, A6, B6, C4

  253. 

  254. 	fxcpmadd C5, A7, B7, C5

  255. 	FXCXNPMA C6, A7, B7, C6

  256. 	fxcpmadd C7, A8, B8, C7

  257. 	FXCXNPMA C8, A8, B8, C8

  258. 	.align 4

  259. 

  260. LL(15):

  261. 	andi.	r0,  N, 7

  262. 	beq	LL(99)

  263. 

  264. 	andi.	r0,  N, 4

  265. 	beq	LL(16)

  266. 

  267. 	LFPDUX	A1,    X, INCX2

  268. 	LFPDUX	B1,    Y, INCY2

  269. 	LFPDUX	A2,    X, INCX2

  270. 	LFPDUX	B2,    Y, INCY2

  271. 	LFPDUX	A3,    X, INCX2

  272. 	LFPDUX	B3,    Y, INCY2

  273. 	LFPDUX	A4,    X, INCX2

  274. 	LFPDUX	B4,    Y, INCY2

  275. 

  276. 	fxcpmadd C1, A1, B1, C1

  277. 	FXCXNPMA C2, A1, B1, C2

  278. 	fxcpmadd C3, A2, B2, C3

  279. 	FXCXNPMA C4, A2, B2, C4

  280. 

  281. 	fxcpmadd C5, A3, B3, C5

  282. 	FXCXNPMA C6, A3, B3, C6

  283. 	fxcpmadd C7, A4, B4, C7

  284. 	FXCXNPMA C8, A4, B4, C8

  285. 	.align 4

  286. 

  287. LL(16):

  288. 	andi.	r0,  N, 2

  289. 	beq	LL(17)

  290. 

  291. 	LFPDUX	A1,    X, INCX2

  292. 	LFPDUX	B1,    Y, INCY2

  293. 	LFPDUX	A2,    X, INCX2

  294. 	LFPDUX	B2,    Y, INCY2

  295. 

  296. 	fxcpmadd C1, A1, B1, C1

  297. 	FXCXNPMA C2, A1, B1, C2

  298. 	fxcpmadd C3, A2, B2, C3

  299. 	FXCXNPMA C4, A2, B2, C4

  300. 	.align 4

  301. 

  302. LL(17):

  303. 	andi.	r0,  N, 1

  304. 	beq	LL(99)

  305. 

  306. 	LFPDUX	A1,    X, INCX2

  307. 	LFPDUX	B1,    Y, INCY2

  308. 

  309. 	fxcpmadd C1, A1, B1, C1

  310. 	FXCXNPMA C2, A1, B1, C2

  311. 	.align 4

  312. 

  313. LL(99):

  314. 	li	r10, 16

  315. 

  316. 	fpadd	C1, C1, C5

  317. 	lfpdux	f23, SP, r10

  318. 	fpadd	C2, C2, C6

  319. 	lfpdux	f22, SP, r10

  320. 	fpadd	C3, C3, C7

  321. 	lfpdux	f21, SP, r10

  322. 	fpadd	C4, C4, C8

  323. 	lfpdux	f20, SP, r10

  324. 

  325. 	lfpdux	f19, SP, r10

  326. 	lfpdux	f18, SP, r10

  327. 	fpadd	C1, C1, C3

  328. 	lfpdux	f17, SP, r10

  329. 	fpadd	C2, C2, C4

  330. 	lfpdux	f16, SP, r10

  331. 

  332. 	fpadd	C1, C1, C2

  333. 	lfpdux	f15, SP, r10

  334. 	lfpdux	f14, SP, r10

  335. 	fsmtp	C2, C1

  336. 

  337. #if defined(F_INTERFACE) && defined(F_INTERFACE_F2C)

  338. 	STFD	C1,    0 * SIZE(RESULT)

  339. 	STFD	C2,    1 * SIZE(RESULT)

  340. #endif

  341. 	addi	SP, SP,  16

  342. 	blr

  343. 	.align 4

  344. 

  345. 

  346. /* X is aligned, Y is NOT aligned */

  347. 

  348. LL(100):

  349. 	subi	INCX2, INCX2, SIZE

  350. 	subi	INCY2, INCY2, SIZE

  351. 

  352. 	li	INCX, SIZE

  353. 	li	INCY, SIZE

  354. 

  355. 	sub	X, X, INCX2

  356. 	sub	Y, Y, INCY2

  357. 

  358. 	srawi.	r0, N, 2

  359. 	mtspr	CTR,  r0

  360. 	beq-	LL(105)

  361. 

  362. 	LFDUX	A1,    X, INCX2

  363. 	LFDUX	B1,    Y, INCY2

  364. 	LFDUX	A2,    X, INCX

  365. 	LFDUX	B2,    Y, INCY

  366. 

  367. 	LFDUX	A3,    X, INCX2

  368. 	LFDUX	B3,    Y, INCY2

  369. 	LFDUX	A4,    X, INCX

  370. 	LFDUX	B4,    Y, INCY

  371. 

  372. 	LFDUX	A5,    X, INCX2

  373. 	LFDUX	B5,    Y, INCY2

  374. 	LFDUX	A6,    X, INCX

  375. 	LFDUX	B6,    Y, INCY

  376. 

  377. 	LFDUX	A7,    X, INCX2

  378. 	LFDUX	B7,    Y, INCY2

  379. 	LFDUX	A8,    X, INCX

  380. 	bdz	LL(104)

  381. 	.align 4

  382. 

  383. LL(103):

  384. 	fmadd	C1, A1, B1, C1

  385. 	LFDUX	B8,    Y, INCY

  386. 	fmadd	C2, A1, B2, C2

  387. 	LFDUX	A1,    X, INCX2

  388. 

  389. 	fmadd	C3, A2, B1, C3

  390. 	LFDUX	B1,    Y, INCY2

  391. 	fmadd	C4, A2, B2, C4

  392. 	LFDUX	A2,    X, INCX

  393. 

  394. 	fmadd	C5, A3, B3, C5

  395. 	LFDUX	B2,    Y, INCY

  396. 	fmadd	C6, A3, B4, C6

  397. 	LFDUX	A3,    X, INCX2

  398. 

  399. 	fmadd	C7, A4, B3, C7

  400. 	LFDUX	B3,    Y, INCY2

  401. 	fmadd	C8, A4, B4, C8

  402. 	LFDUX	A4,    X, INCX

  403. 

  404. 	fmadd	C1, A5, B5, C1

  405. 	LFDUX	B4,    Y, INCY

  406. 	fmadd	C2, A5, B6, C2

  407. 	LFDUX	A5,    X, INCX2

  408. 

  409. 	fmadd	C3, A6, B5, C3

  410. 	LFDUX	B5,    Y, INCY2

  411. 	fmadd	C4, A6, B6, C4

  412. 	LFDUX	A6,    X, INCX

  413. 

  414. 	fmadd	C5, A7, B7, C5

  415. 	LFDUX	B6,    Y, INCY

  416. 	fmadd	C6, A7, B8, C6

  417. 	LFDUX	A7,    X, INCX2

  418. 

  419. 	fmadd	C7, A8, B7, C7

  420. 	LFDUX	B7,    Y, INCY2

  421. 	fmadd	C8, A8, B8, C8

  422. 	LFDUX	A8,    X, INCX

  423. 

  424. 	bdnz	LL(103)

  425. 	.align 4

  426. 

  427. LL(104):

  428. 	LFDUX	B8,    Y, INCY

  429. 	fmadd	C1, A1, B1, C1

  430. 	fmadd	C2, A1, B2, C2

  431. 	fmadd	C3, A2, B1, C3

  432. 	fmadd	C4, A2, B2, C4

  433. 

  434. 	fmadd	C5, A3, B3, C5

  435. 	fmadd	C6, A3, B4, C6

  436. 	fmadd	C7, A4, B3, C7

  437. 	fmadd	C8, A4, B4, C8

  438. 

  439. 	fmadd	C1, A5, B5, C1

  440. 	fmadd	C2, A5, B6, C2

  441. 	fmadd	C3, A6, B5, C3

  442. 	fmadd	C4, A6, B6, C4

  443. 

  444. 	fmadd	C5, A7, B7, C5

  445. 	fmadd	C6, A7, B8, C6

  446. 	fmadd	C7, A8, B7, C7

  447. 	fmadd	C8, A8, B8, C8

  448. 	.align 4

  449. 

  450. LL(105):

  451. 	andi.	r0,  N, 3

  452. 	beq	LL(999)

  453. 

  454. 	andi.	r0,  N, 2

  455. 	beq	LL(107)

  456. 

  457. 	LFDUX	A1,    X, INCX2

  458. 	LFDUX	B1,    Y, INCY2

  459. 	LFDUX	A2,    X, INCX

  460. 	LFDUX	B2,    Y, INCY

  461. 

  462. 	LFDUX	A3,    X, INCX2

  463. 	LFDUX	B3,    Y, INCY2

  464. 	LFDUX	A4,    X, INCX

  465. 	LFDUX	B4,    Y, INCY

  466. 

  467. 	fmadd	C1, A1, B1, C1

  468. 	fmadd	C2, A1, B2, C2

  469. 	fmadd	C3, A2, B1, C3

  470. 	fmadd	C4, A2, B2, C4

  471. 

  472. 	fmadd	C5, A3, B3, C5

  473. 	fmadd	C6, A3, B4, C6

  474. 	fmadd	C7, A4, B3, C7

  475. 	fmadd	C8, A4, B4, C8

  476. 	.align 4

  477. 

  478. LL(107):

  479. 	andi.	r0,  N, 1

  480. 	beq	LL(999)

  481. 

  482. 	LFDUX	A1,    X, INCX2

  483. 	LFDUX	B1,    Y, INCY2

  484. 

  485. 	LFDUX	A2,    X, INCX

  486. 	LFDUX	B2,    Y, INCY

  487. 

  488. 	fmadd	C1, A1, B1, C1

  489. 	fmadd	C2, A1, B2, C2

  490. 	fmadd	C3, A2, B1, C3

  491. 	fmadd	C4, A2, B2, C4

  492. 	.align 4

  493. 

  494. LL(999):

  495. 	li	r10, 16

  496. 

  497. 	fadd	C1, C1, C5

  498. 	lfpdux	f23, SP, r10

  499. 	fadd	C2, C2, C6

  500. 	lfpdux	f22, SP, r10

  501. 	fadd	C3, C3, C7

  502. 	lfpdux	f21, SP, r10

  503. 	fadd	C4, C4, C8

  504. 	lfpdux	f20, SP, r10

  505. 

  506. 	lfpdux	f19, SP, r10

  507. 	lfpdux	f18, SP, r10

  508. 	lfpdux	f17, SP, r10

  509. 	lfpdux	f16, SP, r10

  510. 

  511. #ifndef CONJ

  512. 	FSUB	C1,  C1,  C4

  513. 	FADD	C2,  C2,  C3

  514. #else

  515. 	FADD	C1,  C1,  C4

  516. 	FSUB	C2,  C2,  C3

  517. #endif

  518. 

  519. 	lfpdux	f15, SP, r10

  520. 	lfpdux	f14, SP, r10

  521. 

  522. #if defined(F_INTERFACE) && defined(F_INTERFACE_F2C)

  523. 	STFD	C1,    0 * SIZE(RESULT)

  524. 	STFD	C2,    1 * SIZE(RESULT)

  525. #endif

  526. 	addi	SP, SP,  16

  527. 	blr

  528. 

  529. 	EPILOGUE