OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cd8ac192a9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cd8ac192a9'
${ noResults }
OpenBLAS/kernel/arm64/zgemm_kernel_sve_v1x4.S

834 lines
19 kB
Raw Blame History 

  1. /*******************************************************************************

  2. Copyright (c) 2015, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *******************************************************************************/

  27. 

  28. #define ASSEMBLER

  29. #include "common.h"

  30. 

  31. /*                   X0          X1          X2          s0        X3        x4       x5           x6 */

  32. /*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha0,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc */

  33. 

  34. #define origM		x0

  35. #define origN		x1

  36. #define origK		x2

  37. #define origPA		x3

  38. #define origPB		x4

  39. #define pC		x5

  40. #define LDC		x6

  41. #define temp		x7

  42. #define counterL	x8

  43. #define counterI	x9

  44. #define counterJ	x10

  45. #define pB		x11

  46. #define pCRow0		x12

  47. #define pCRow1		x13

  48. #define pCRow2		x14

  49. #define pCRow3		x15

  50. #define pA		x16

  51. #define lanes		x17

  52. 

  53. #define alphaR		x19

  54. #define alphaI		x20

  55. 

  56. #define alphaz_R	z6.d

  57. #define alphaz_I	z7.d

  58. #define alpha0_R	d6

  59. #define alpha0_I	d7

  60. 

  61. 

  62. #define A_PRE_SIZE	2560

  63. #define B_PRE_SIZE	448

  64. #define C_PRE_SIZE	128

  65. 

  66. #if defined(NN) || defined(NT) || defined(TN) || defined(TT)

  67. #define OP_rr		fmla

  68. #define OP_ii		fmls

  69. #define OP_ri		fmla

  70. #define OP_ir		fmla

  71. #elif defined(NR) || defined(NC) || defined(TR) || defined(TC)

  72. #define OP_rr		fmla

  73. #define OP_ii		fmla

  74. #define OP_ri		fmls

  75. #define OP_ir		fmla

  76. #elif defined(RN) || defined(RT) || defined(CN) || defined(CT)

  77. #define OP_rr		fmla

  78. #define OP_ii		fmla

  79. #define OP_ri		fmla

  80. #define OP_ir		fmls

  81. #elif defined(RR) || defined(RC) || defined(CR) || defined(CC)

  82. #define OP_rr		fmla

  83. #define OP_ii		fmls

  84. #define OP_ri		fmls

  85. #define OP_ir		fmls

  86. #endif

  87. 

  88. // 00 origM

  89. // 01 origN

  90. // 02 origK

  91. // 03 origPA

  92. // 04 origPB

  93. // 05 pC

  94. // 06 origLDC -> LDC

  95. // 07 offset -> temp

  96. // 08 counterL

  97. // 09 counterI

  98. // 10 counterJ

  99. // 11 pB

  100. // 12 pCRow0

  101. // 13 pCRow1

  102. // 14 pCRow2

  103. // 15 pCRow3

  104. // 16 pA

  105. // 17 alpha_save_R

  106. // 18 must save alpha_save_I

  107. // 19 must save

  108. // 20 must save

  109. // 21 must save

  110. // 22 must save

  111. // 23 must save

  112. // 24 must save

  113. // 25 must save

  114. // 26 must save

  115. // 27 must save

  116. // 28 must save

  117. // 29 frame

  118. // 30 link

  119. // 31 sp

  120. 

  121. //v00 ALPHA_R -> pA00_R, pA01_R

  122. //v01 ALPHA_I -> pA00_I, pA01_I

  123. //v02 pA02_R, pA03_R

  124. //v03 pA02_I, pA03_I

  125. //v04 pA10_R, pA11_R

  126. //v05 pA10_I, pA11_I

  127. //v06 pA12_R, pA13_R

  128. //v07 pA12_I, pA13_I

  129. //v08 must save pB00_R, pB01_R

  130. //v09 must save pB00_I, pB01_I

  131. //v10 must save pB02_R, pB03_R OR ALPHA0_R

  132. //v11 must save pB02_I, pB03_I OR ALPHA0_I

  133. //v12 must save pB10_R, pB11_R

  134. //v13 must save pB10_I, pB11_I

  135. //v14 must save pB12_R, pB13_R OR ALPHA1_R

  136. //v15 must save pB12_I, pB13_I OR ALPHA1_R

  137. //v16 pC0R

  138. //v17 pC0I

  139. //v18 pC1R

  140. //v19 pC1I

  141. //v20 pC2R

  142. //v21 pC2I

  143. //v22 pC3R

  144. //v23 pC3I

  145. //v24 pC3R

  146. //v25 pC3I

  147. //v26 pC22_R, pC23_R

  148. //v27 pC22_I, pC23_I

  149. //v28 pC30_R, pC31_R

  150. //v29 pC30_I, pC31_I

  151. //v30 pC32_R, pC33_R

  152. //v31 pC32_I, pC33_I

  153. 

  154. /*******************************************************************************

  155. * Macro definitions

  156. *******************************************************************************/

  157. 

  158. .macro INITv1x4

  159. 	dup		z16.d, #0

  160. 	dup		z17.d, #0

  161. 	dup		z18.d, #0

  162. 	dup		z19.d, #0

  163. 	dup		z20.d, #0

  164. 	dup		z21.d, #0

  165. 	dup		z22.d, #0

  166. 	dup		z23.d, #0

  167. .endm

  168. 

  169. .macro KERNELv1x4_I

  170. 	ld2d	{z0.d, z1.d}, p1/z, [pA]

  171. 	add	pA, pA, lanes, lsl #4    // pA += lanes*2*8

  172. 	ld2d	{z2.d, z3.d}, p1/z, [pA] // next one

  173. 	add	pA, pA, lanes, lsl #4    // pA += lanes*2*8

  174. 

  175.     ld1rd  z8.d, p0/z,  [pB]

  176.     ld1rd  z9.d, p0/z,  [pB, 8]

  177.     ld1rd  z10.d, p0/z, [pB, 16]

  178.     ld1rd  z11.d, p0/z, [pB, 24]

  179.     ld1rd  z12.d, p0/z, [pB, 32]

  180.     ld1rd  z13.d, p0/z, [pB, 40]

  181.     ld1rd  z14.d, p0/z, [pB, 48]

  182.     ld1rd  z15.d, p0/z, [pB, 56]

  183. 

  184.     add pB, pB, 64

  185. 

  186. 	fmla	z16.d, p1/m, z0.d, z8.d

  187. 	OP_ir	z17.d, p1/m, z1.d, z8.d

  188.     ld1rd  z8.d, p0/z,  [pB]

  189. #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \

  190.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  191. 	#eor	z17.16b, z17.16b, z17.16b

  192. 	fmls	z17.d, p1/m, z0.d, z9.d

  193. #else

  194. 	fmla	z17.d, p1/m, z0.d, z9.d

  195. #endif

  196. 	OP_ii	z16.d, p1/m, z1.d, z9.d

  197.     ld1rd  z9.d, p0/z,  [pB, 8]

  198. 

  199. 

  200. 	fmla	z18.d, p1/m, z0.d, z10.d

  201. 	OP_ir	z19.d, p1/m, z1.d, z10.d

  202.     ld1rd  z10.d, p0/z,  [pB, 16]

  203. 	OP_ii	z18.d, p1/m, z1.d, z11.d

  204. #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \

  205.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  206. 	#eor	z19.16b, z21.16b, z21.16b

  207. 	fmls	z19.d, p1/m, z0.d, z11.d

  208. #else

  209. 	fmla	z19.d, p1/m, z0.d, z11.d

  210. #endif

  211.     ld1rd  z11.d, p0/z,  [pB, 24]

  212. 

  213. 

  214. 	fmla	z20.d, p1/m, z0.d, z12.d

  215. 	OP_ir	z21.d, p1/m, z1.d, z12.d

  216.     ld1rd  z12.d, p0/z,  [pB, 32]

  217. #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \

  218.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  219. 	#eor	z21.16b, z23.16b, z23.16b

  220. 	fmls	z21.d, p1/m, z0.d, z13.d

  221. #else

  222. 	fmla	z21.d, p1/m, z0.d, z13.d

  223. #endif

  224. 	OP_ii	z20.d, p1/m, z1.d, z13.d

  225.     ld1rd  z13.d, p0/z,  [pB, 40]

  226. 

  227. 

  228. 	fmla	z22.d, p1/m, z0.d, z14.d

  229. 	OP_ir	z23.d, p1/m, z1.d, z14.d

  230.     ld1rd  z14.d, p0/z,  [pB, 48]

  231. #if defined(NR) || defined(NC) || defined(TR) || defined(TC) || \

  232.     defined(RR) || defined(RC) || defined(CR) || defined(CC)

  233. 	#eor	z23.16b, z19.16b, z19.16b

  234. 	fmls	z23.d, p1/m, z0.d, z15.d

  235. #else

  236. 	fmla	z23.d, p1/m, z0.d, z15.d

  237. #endif

  238. 	OP_ii	z22.d, p1/m, z1.d, z15.d

  239.     ld1rd  z15.d, p0/z,  [pB, 56]

  240. 

  241.     add pB, pB, 64

  242. .endm

  243. 

  244. .macro KERNELv1x4_M1

  245. 	ld2d	{z2.d, z3.d}, p1/z, [pA]

  246. 	add	pA, pA, lanes, lsl #4	// pA = pA + lanes * 2 * 8

  247. 

  248. 	OP_rr	z16.d, p1/m, z0.d, z8.d

  249. 	OP_ir	z17.d, p1/m, z1.d, z8.d

  250.     ld1rd  z8.d, p0/z,  [pB]

  251. 	OP_ii	z16.d, p1/m, z1.d, z9.d

  252. 	OP_ri	z17.d, p1/m, z0.d, z9.d

  253.     ld1rd  z9.d, p0/z,  [pB, 8]

  254. 

  255. 	OP_rr	z18.d, p1/m, z0.d, z10.d

  256. 	OP_ir	z19.d, p1/m, z1.d, z10.d

  257.     ld1rd  z10.d, p0/z,  [pB, 16]

  258. 	OP_ii	z18.d, p1/m, z1.d, z11.d

  259. 	OP_ri	z19.d, p1/m, z0.d, z11.d

  260.     ld1rd  z11.d, p0/z,  [pB, 24]

  261. 

  262. 	OP_rr	z20.d, p1/m, z0.d, z12.d

  263. 	OP_ir	z21.d, p1/m, z1.d, z12.d

  264.     ld1rd  z12.d, p0/z,  [pB, 32]

  265. 	OP_ii	z20.d, p1/m, z1.d, z13.d

  266. 	OP_ri	z21.d, p1/m, z0.d, z13.d

  267.     ld1rd  z13.d, p0/z,  [pB, 40]

  268. 

  269. 	OP_rr	z22.d, p1/m, z0.d, z14.d

  270. 	OP_ir	z23.d, p1/m, z1.d, z14.d

  271.     ld1rd  z14.d, p0/z,  [pB, 48]

  272. 	OP_ii	z22.d, p1/m, z1.d, z15.d

  273. 	OP_ri	z23.d, p1/m, z0.d, z15.d

  274.     ld1rd  z15.d, p0/z,  [pB, 56]

  275. 

  276.     add pB, pB, 64

  277. .endm

  278. 

  279. .macro KERNELv1x4_M2

  280. 	ld2d	{z0.d, z1.d}, p1/z, [pA]

  281. 	add	pA, pA, lanes, lsl #4	// pA = pA + lanes *2 * 8

  282. 

  283. 	OP_rr	z16.d, p1/m, z2.d, z8.d

  284. 	OP_ir	z17.d, p1/m, z3.d, z8.d

  285.     ld1rd  z8.d, p0/z,  [pB]

  286. 	OP_ii	z16.d, p1/m, z3.d, z9.d

  287. 	OP_ri	z17.d, p1/m, z2.d, z9.d

  288.     ld1rd  z9.d, p0/z,  [pB, 8]

  289. 

  290. 	OP_rr	z18.d, p1/m, z2.d, z10.d

  291. 	OP_ir	z19.d, p1/m, z3.d, z10.d

  292.     ld1rd  z10.d, p0/z,  [pB, 16]

  293. 	OP_ii	z18.d, p1/m, z3.d, z11.d

  294. 	OP_ri	z19.d, p1/m, z2.d, z11.d

  295.     ld1rd  z11.d, p0/z,  [pB, 24]

  296. 

  297. 	OP_rr	z20.d, p1/m, z2.d, z12.d

  298. 	OP_ir	z21.d, p1/m, z3.d, z12.d

  299.     ld1rd  z12.d, p0/z,  [pB, 32]

  300. 	OP_ii	z20.d, p1/m, z3.d, z13.d

  301. 	OP_ri	z21.d, p1/m, z2.d, z13.d

  302.     ld1rd  z13.d, p0/z,  [pB, 40]

  303. 

  304. 	OP_rr	z22.d, p1/m, z2.d, z14.d

  305. 	OP_ir	z23.d, p1/m, z3.d, z14.d

  306.     ld1rd  z14.d, p0/z,  [pB, 48]

  307. 	OP_ii	z22.d, p1/m, z3.d, z15.d

  308. 	OP_ri	z23.d, p1/m, z2.d, z15.d

  309.     ld1rd  z15.d, p0/z,  [pB, 56]

  310. 

  311.     add pB, pB, 64

  312. .endm

  313. 

  314. .macro KERNELv1x4_E

  315. 	OP_rr	z16.d, p1/m, z2.d, z8.d

  316. 	OP_ir	z17.d, p1/m, z3.d, z8.d

  317. 	OP_ii	z16.d, p1/m, z3.d, z9.d

  318. 	OP_ri	z17.d, p1/m, z2.d, z9.d

  319. 

  320. 	OP_rr	z18.d, p1/m, z2.d, z10.d

  321. 	OP_ir	z19.d, p1/m, z3.d, z10.d

  322. 	OP_ii	z18.d, p1/m, z3.d, z11.d

  323. 	OP_ri	z19.d, p1/m, z2.d, z11.d

  324. 

  325. 	OP_rr	z20.d, p1/m, z2.d, z12.d

  326. 	OP_ir	z21.d, p1/m, z3.d, z12.d

  327. 	OP_ii	z20.d, p1/m, z3.d, z13.d

  328. 	OP_ri	z21.d, p1/m, z2.d, z13.d

  329. 

  330. 	OP_rr	z22.d, p1/m, z2.d, z14.d

  331. 	OP_ir	z23.d, p1/m, z3.d, z14.d

  332. 	OP_ii	z22.d, p1/m, z3.d, z15.d

  333. 	OP_ri	z23.d, p1/m, z2.d, z15.d

  334. .endm

  335. 

  336. .macro KERNELv1x4_SUB

  337. 	ld2d	{z0.d, z1.d}, p1/z, [pA]

  338. 	add	pA, pA, lanes, lsl #4	// pA = pA + lanes* 2  * 8

  339. 

  340.     ld1rd  z8.d, p0/z,  [pB]

  341.     ld1rd  z9.d, p0/z,  [pB, 8]

  342.     ld1rd  z10.d, p0/z,  [pB, 16]

  343.     ld1rd  z11.d, p0/z,  [pB, 24]

  344. 

  345. 	OP_rr	z16.d, p1/m, z0.d, z8.d

  346. 	OP_ir	z17.d, p1/m, z1.d, z8.d

  347. 	OP_ii	z16.d, p1/m, z1.d, z9.d

  348. 	OP_ri	z17.d, p1/m, z0.d, z9.d

  349. 

  350.     ld1rd  z12.d, p0/z,  [pB, 32]

  351.     ld1rd  z13.d, p0/z,  [pB, 40]

  352.     ld1rd  z14.d, p0/z,  [pB, 48]

  353.     ld1rd  z15.d, p0/z,  [pB, 56]

  354. 

  355. 	OP_rr	z18.d, p1/m, z0.d, z10.d

  356. 	OP_ir	z19.d, p1/m, z1.d, z10.d

  357. 	OP_ii	z18.d, p1/m, z1.d, z11.d

  358. 	OP_ri	z19.d, p1/m, z0.d, z11.d

  359. 

  360.     add pB, pB, 64

  361. 

  362. 	OP_rr	z20.d, p1/m, z0.d, z12.d

  363. 	OP_ir	z21.d, p1/m, z1.d, z12.d

  364. 	OP_ii	z20.d, p1/m, z1.d, z13.d

  365. 	OP_ri	z21.d, p1/m, z0.d, z13.d

  366. 

  367. 	OP_rr	z22.d, p1/m, z0.d, z14.d

  368. 	OP_ir	z23.d, p1/m, z1.d, z14.d

  369. 	OP_ii	z22.d, p1/m, z1.d, z15.d

  370. 	OP_ri	z23.d, p1/m, z0.d, z15.d

  371. .endm

  372. 

  373. .macro SAVEv1x4

  374. 	ld2d	{z24.d, z25.d}, p1/z, [pCRow0]

  375. 	fmla	z24.d, p1/m, z16.d, alphaz_R

  376. 	fmls	z24.d, p1/m, z17.d, alphaz_I

  377. 	fmla	z25.d, p1/m, z16.d, alphaz_I

  378. 	fmla	z25.d, p1/m, z17.d, alphaz_R

  379. 	st2d 	{z24.d, z25.d}, p1, [pCRow0]

  380. 

  381. 	add	pCRow0, pCRow0, lanes, lsl #4

  382. 

  383. 	ld2d	{z26.d, z27.d}, p1/z, [pCRow1]

  384. 	fmla	z26.d, p1/m, z18.d, alphaz_R

  385. 	fmls	z26.d, p1/m, z19.d, alphaz_I

  386. 	fmla	z27.d, p1/m, z18.d, alphaz_I

  387. 	fmla	z27.d, p1/m, z19.d, alphaz_R

  388. 	st2d 	{z26.d, z27.d}, p1, [pCRow1]

  389. 

  390. 	add	pCRow1, pCRow1, lanes, lsl #4

  391. 

  392. 	ld2d	{z28.d, z29.d}, p1/z, [pCRow2]

  393. 	fmla	z28.d, p1/m, z20.d, alphaz_R

  394. 	fmls	z28.d, p1/m, z21.d, alphaz_I

  395. 	fmla	z29.d, p1/m, z20.d, alphaz_I

  396. 	fmla	z29.d, p1/m, z21.d, alphaz_R

  397. 	st2d 	{z28.d, z29.d}, p1, [pCRow2]

  398. 

  399. 	add	pCRow2, pCRow2, lanes, lsl #4

  400. 

  401. 	ld2d	{z30.d, z31.d}, p1/z, [pCRow3]

  402. 	fmla	z30.d, p1/m, z22.d, alphaz_R

  403. 	fmls	z30.d, p1/m, z23.d, alphaz_I

  404. 	fmla	z31.d, p1/m, z22.d, alphaz_I

  405. 	fmla	z31.d, p1/m, z23.d, alphaz_R

  406. 	st2d 	{z30.d, z31.d}, p1, [pCRow3]

  407. 

  408. 	add	pCRow3, pCRow3, lanes, lsl #4	// pC = pC + lanes  * 2 *8

  409. 

  410. .endm

  411. 

  412. /******************************************************************************/

  413. 

  414. 

  415. .macro INITv1x2

  416. 	dup		z16.d, #0

  417. 	dup		z17.d, #0

  418. 	dup		z18.d, #0

  419. 	dup		z19.d, #0

  420. .endm

  421. 

  422. .macro KERNELv1x2_SUB

  423. 	ld2d	{z0.d, z1.d}, p1/z, [pA]

  424. 	add	pA, pA, lanes, lsl #4	// pA = pA + lanes* 2  * 8

  425. 

  426.     ld1rd  z8.d, p0/z,  [pB]

  427.     ld1rd  z9.d, p0/z,  [pB, 8]

  428.     ld1rd  z10.d, p0/z,  [pB, 16]

  429.     ld1rd  z11.d, p0/z,  [pB, 24]

  430. 

  431. 	OP_rr	z16.d, p1/m, z0.d, z8.d

  432. 	OP_ir	z17.d, p1/m, z1.d, z8.d

  433. 	OP_ii	z16.d, p1/m, z1.d, z9.d

  434. 	OP_ri	z17.d, p1/m, z0.d, z9.d

  435. 

  436. 	OP_rr	z18.d, p1/m, z0.d, z10.d

  437. 	OP_ir	z19.d, p1/m, z1.d, z10.d

  438. 	OP_ii	z18.d, p1/m, z1.d, z11.d

  439. 	OP_ri	z19.d, p1/m, z0.d, z11.d

  440. 

  441.     add pB, pB, 32

  442. .endm

  443. 

  444. .macro SAVEv1x2

  445. 	ld2d	{z24.d, z25.d}, p1/z, [pCRow0]

  446. 	fmla	z24.d, p1/m, z16.d, alphaz_R

  447. 	fmls	z24.d, p1/m, z17.d, alphaz_I

  448. 	fmla	z25.d, p1/m, z16.d, alphaz_I

  449. 	fmla	z25.d, p1/m, z17.d, alphaz_R

  450. 	st2d 	{z24.d, z25.d}, p1, [pCRow0]

  451. 

  452. 	add	pCRow0, pCRow0, lanes, lsl #4

  453. 

  454. 	ld2d	{z26.d, z27.d}, p1/z, [pCRow1]

  455. 	fmla	z26.d, p1/m, z18.d, alphaz_R

  456. 	fmls	z26.d, p1/m, z19.d, alphaz_I

  457. 	fmla	z27.d, p1/m, z18.d, alphaz_I

  458. 	fmla	z27.d, p1/m, z19.d, alphaz_R

  459. 	st2d 	{z26.d, z27.d}, p1, [pCRow1]

  460. 

  461. 	add	pCRow1, pCRow1, lanes, lsl #4

  462. .endm

  463. 

  464. /******************************************************************************/

  465. 

  466. 

  467. .macro INITv1x1

  468. 	dup		z16.d, #0

  469. 	dup		z17.d, #0

  470. .endm

  471. 

  472. 

  473. .macro KERNELv1x1_SUB

  474. 	ld2d	{z0.d, z1.d}, p1/z, [pA]

  475. 	add	pA, pA, lanes, lsl #4	// pA = pA + lanes* 2  * 8

  476. 

  477.     ld1rd  z8.d, p0/z,  [pB]

  478.     ld1rd  z9.d, p0/z,  [pB, 8]

  479. 

  480.     add pB, pB, 16

  481. 

  482. 	OP_rr	z16.d, p1/m, z0.d, z8.d

  483. 	OP_ir	z17.d, p1/m, z1.d, z8.d

  484. 	OP_ii	z16.d, p1/m, z1.d, z9.d

  485. 	OP_ri	z17.d, p1/m, z0.d, z9.d

  486. .endm

  487. 

  488. .macro SAVEv1x1

  489. 	ld2d	{z24.d, z25.d}, p1/z, [pCRow0]

  490. 	fmla	z24.d, p1/m, z16.d, alphaz_R

  491. 	fmls	z24.d, p1/m, z17.d, alphaz_I

  492. 	fmla	z25.d, p1/m, z16.d, alphaz_I

  493. 	fmla	z25.d, p1/m, z17.d, alphaz_R

  494. 	st2d 	{z24.d, z25.d}, p1, [pCRow0]

  495. 

  496. 	add	pCRow0, pCRow0, lanes, lsl #4	// pC = pC + lanes  * 2 *8

  497. 

  498. .endm

  499. 

  500. /******************************************************************************/

  501. 

  502. /*******************************************************************************

  503. * End of macro definitions

  504. *******************************************************************************/

  505. 

  506. 	PROLOGUE

  507. 

  508. 	.align 5

  509. 	add	sp, sp, #-(11 * 16)

  510. 	stp	d8, d9, [sp, #(0 * 16)]

  511. 	stp	d10, d11, [sp, #(1 * 16)]

  512. 	stp	d12, d13, [sp, #(2 * 16)]

  513. 	stp	d14, d15, [sp, #(3 * 16)]

  514. 	stp	d16, d17, [sp, #(4 * 16)]

  515. 	stp	x18, x19, [sp, #(5 * 16)]

  516. 	stp	x20, x21, [sp, #(6 * 16)]

  517. 	stp	x22, x23, [sp, #(7 * 16)]

  518. 	stp	x24, x25, [sp, #(8 * 16)]

  519. 	stp	x26, x27, [sp, #(9 * 16)]

  520. 	str	x28, [sp, #(10 * 16)]

  521. 

  522. 	fmov	alphaR, d0

  523. 	dup	    alphaz_R, alphaR

  524. 	fmov	alphaI, d1

  525. 	dup	    alphaz_I, alphaI

  526. 

  527. 	lsl	LDC, LDC, #4			// ldc = ldc * 2 * 8

  528.     ptrue p0.d                  // create true predicate 

  529. 

  530. 	mov	pB, origPB

  531. 

  532. // Loop over N

  533. 	mov	counterJ, origN

  534. 	asr 	counterJ, counterJ, #2		// J = J / 4

  535. 	cmp 	counterJ, #0

  536. 	ble	.Lzgemm_kernel_L2_BEGIN

  537. 

  538. /******************************************************************************/

  539. .Lzgemm_kernel_L4_BEGIN:

  540. 	mov	pCRow0, pC

  541. 	add	pCRow1, pCRow0, LDC

  542. 	add	pCRow2, pCRow1, LDC

  543. 	add	pCRow3, pCRow2, LDC

  544. 

  545. 	add	pC, pCRow3, LDC

  546. 

  547. 	mov	pA, origPA			// pA = start of A array

  548. 

  549. .Lzgemm_kernel_L4_Mv1_BEGIN:

  550. 

  551. /* Loop over M is done in an SVE fashion. This has the benefit of the last M%SVE_LEN iterations being done in a single sweep */

  552.     mov counterI, #0

  553.     whilelt p1.d, counterI, origM   

  554.     cntp lanes, p0, p1.d                        // lanes contain number of active SVE lanes in M dimension

  555. 

  556. 	.align 5

  557. .Lzgemm_kernel_L4_Mv1_20:

  558. 

  559. 	mov	pB, origPB

  560.     INITv1x4                     // fill with zeros

  561. 

  562. 	asr 	counterL , origK, #3

  563. 	cmp	counterL , #2

  564. 	blt	.Lzgemm_kernel_L4_Mv1_32

  565. 

  566. 	KERNELv1x4_I

  567. 	KERNELv1x4_M2

  568. 	KERNELv1x4_M1

  569. 	KERNELv1x4_M2

  570. 	KERNELv1x4_M1

  571. 	KERNELv1x4_M2

  572. 	KERNELv1x4_M1

  573. 	KERNELv1x4_M2

  574. 

  575. 	subs	counterL, counterL, #2		// subtract 2

  576. 	ble	.Lzgemm_kernel_L4_Mv1_22a

  577. 

  578. 	.align 5

  579. .Lzgemm_kernel_L4_Mv1_22:

  580. 

  581. 	KERNELv1x4_M1

  582. 	KERNELv1x4_M2

  583. 	KERNELv1x4_M1

  584. 	KERNELv1x4_M2

  585. 	KERNELv1x4_M1

  586. 	KERNELv1x4_M2

  587. 	KERNELv1x4_M1

  588. 	KERNELv1x4_M2

  589. 

  590. 	subs	counterL, counterL, #1

  591. 	bgt	.Lzgemm_kernel_L4_Mv1_22

  592. 

  593. 	.align 5

  594. .Lzgemm_kernel_L4_Mv1_22a:

  595. 

  596. 	KERNELv1x4_M1

  597. 	KERNELv1x4_M2

  598. 	KERNELv1x4_M1

  599. 	KERNELv1x4_M2

  600. 	KERNELv1x4_M1

  601. 	KERNELv1x4_M2

  602. 	KERNELv1x4_M1

  603. 	KERNELv1x4_E

  604. 

  605. 	b	 .Lzgemm_kernel_L4_Mv1_44

  606. 

  607. 	.align 5

  608. .Lzgemm_kernel_L4_Mv1_32:

  609. 

  610. 	tst	counterL, #1

  611. 	ble	.Lzgemm_kernel_L4_Mv1_40

  612. 

  613. 	KERNELv1x4_I

  614. 	KERNELv1x4_M2

  615. 	KERNELv1x4_M1

  616. 	KERNELv1x4_M2

  617. 	KERNELv1x4_M1

  618. 	KERNELv1x4_M2

  619. 	KERNELv1x4_M1

  620. 	KERNELv1x4_E

  621. 

  622. 	b	.Lzgemm_kernel_L4_Mv1_44

  623. 

  624. 

  625. .Lzgemm_kernel_L4_Mv1_40:

  626. 

  627. 	INITv1x4

  628. 

  629. .Lzgemm_kernel_L4_Mv1_44:

  630. 

  631. 	ands	counterL , origK, #7

  632. 	ble	.Lzgemm_kernel_L4_Mv1_100

  633. 

  634. 	.align 5

  635. .Lzgemm_kernel_L4_Mv1_46:

  636. 	KERNELv1x4_SUB

  637. 

  638. 	subs	counterL, counterL, #1

  639. 	bne	.Lzgemm_kernel_L4_Mv1_46

  640. 

  641. .Lzgemm_kernel_L4_Mv1_100:

  642. 	SAVEv1x4

  643. 

  644. .Lzgemm_kernel_L4_Mv1_END:

  645. 

  646.     incd    counterI

  647.     whilelt p1.d, counterI, origM             //SVE instruction

  648.     cntp lanes, p0, p1.d                        // lanes contain number of active SVE lanes in M dimension

  649.     b.any   .Lzgemm_kernel_L4_Mv1_20   

  650. 

  651. 

  652. 

  653. .Lzgemm_kernel_L4_END:

  654. 

  655. 	lsl	temp, origK, #6

  656. 	add	origPB, origPB, temp		// B = B + K * 4 * 8 * 2

  657. 

  658. 	subs	counterJ, counterJ , #1		// j--

  659. 	bgt	.Lzgemm_kernel_L4_BEGIN

  660. 

  661. 

  662. /******************************************************************************/

  663. 

  664. .Lzgemm_kernel_L2_BEGIN:   // less than 2 left in N direction

  665. 

  666. 	mov	counterJ , origN

  667. 	tst	counterJ , #3

  668. 	ble	.Lzgemm_kernel_L999

  669. 

  670. 	tst	counterJ , #2

  671. 	ble	.Lzgemm_kernel_L1_BEGIN

  672. 

  673. 	mov	pCRow0, pC			// pCRow0 = pC

  674. 	add	pCRow1, pCRow0, LDC

  675. 

  676. 	add	pC,pC,LDC, lsl #1

  677. 

  678. 	mov	pA, origPA			// pA = A

  679. 

  680. 

  681. 

  682. .Lzgemm_kernel_L2_Mv1_BEGIN:

  683. 

  684.     mov counterI, #0

  685.     whilelt p1.d, counterI, origM               //SVE instruction

  686.     cntp lanes, p0, p1.d

  687. 

  688. 

  689. .Lzgemm_kernel_L2_Mv1_20:

  690. 

  691. 	INITv1x2

  692. 

  693. 	mov	pB, origPB

  694. 	asr	counterL , origK, #3		// counterL = counterL / 8

  695. 	cmp	counterL,#0

  696. 	ble	.Lzgemm_kernel_L2_Mv1_40

  697. 	.align 5

  698. 

  699. .Lzgemm_kernel_L2_Mv1_22:

  700. 	KERNELv1x2_SUB

  701. 	KERNELv1x2_SUB

  702. 	KERNELv1x2_SUB

  703. 	KERNELv1x2_SUB

  704. 

  705. 	KERNELv1x2_SUB

  706. 	KERNELv1x2_SUB

  707. 	KERNELv1x2_SUB

  708. 	KERNELv1x2_SUB

  709. 

  710. 	subs	counterL, counterL, #1

  711. 	bgt	.Lzgemm_kernel_L2_Mv1_22

  712. 

  713. 

  714. .Lzgemm_kernel_L2_Mv1_40:

  715. 

  716. 	ands	counterL , origK, #7		// counterL = counterL % 8

  717. 	ble	.Lzgemm_kernel_L2_Mv1_100

  718. 

  719. .Lzgemm_kernel_L2_Mv1_42:

  720. 

  721. 	KERNELv1x2_SUB

  722. 

  723. 	subs	counterL, counterL, #1

  724. 	bgt	.Lzgemm_kernel_L2_Mv1_42

  725. 

  726. .Lzgemm_kernel_L2_Mv1_100:

  727. 

  728. 	SAVEv1x2

  729. 

  730. .Lzgemm_kernel_L2_Mv1_END:

  731. 

  732. 

  733.     incd    counterI

  734.     whilelt p1.d, counterI, origM             //SVE instruction

  735.     cntp lanes, p0, p1.d

  736.     b.any   .Lzgemm_kernel_L2_Mv1_20   

  737. 

  738. 

  739. .Lzgemm_kernel_L2_END:

  740. 	lsl	temp, origK, #5

  741. 	add	origPB, origPB, temp // B = B + K * 2 * 8 * 2

  742. 

  743. /******************************************************************************/

  744. 

  745. .Lzgemm_kernel_L1_BEGIN:

  746. 

  747. 	mov	counterJ , origN

  748. 	tst	counterJ , #1

  749. 	ble	.Lzgemm_kernel_L999 // done

  750. 

  751. 

  752. 	mov	pCRow0, pC			// pCRow0 = C

  753. 	add	pC , pC , LDC			// Update pC to point to next

  754. 

  755. 	mov	pA, origPA			// pA = A

  756. 

  757. .Lzgemm_kernel_L1_Mv1_BEGIN:

  758. 

  759.     mov counterI, #0

  760.     whilelt p1.d, counterI, origM               //SVE instruction

  761.     cntp lanes, p0, p1.d

  762. 

  763. 

  764. .Lzgemm_kernel_L1_Mv1_20:

  765. 

  766. 	INITv1x1

  767. 

  768. 	mov	pB, origPB

  769. 	asr	counterL , origK, #3		// counterL = counterL / 8

  770. 	cmp	counterL , #0

  771. 	ble	.Lzgemm_kernel_L1_Mv1_40

  772. 	.align 5

  773. 

  774. .Lzgemm_kernel_L1_Mv1_22:

  775. 	KERNELv1x1_SUB

  776. 	KERNELv1x1_SUB

  777. 	KERNELv1x1_SUB

  778. 	KERNELv1x1_SUB

  779. 

  780. 	KERNELv1x1_SUB

  781. 	KERNELv1x1_SUB

  782. 	KERNELv1x1_SUB

  783. 	KERNELv1x1_SUB

  784. 

  785. 	subs	counterL, counterL, #1

  786. 	bgt	.Lzgemm_kernel_L1_Mv1_22

  787. 

  788. 

  789. .Lzgemm_kernel_L1_Mv1_40:

  790. 

  791. 	ands	counterL , origK, #7		// counterL = counterL % 8

  792. 	ble	.Lzgemm_kernel_L1_Mv1_100

  793. 

  794. .Lzgemm_kernel_L1_Mv1_42:

  795. 

  796. 	KERNELv1x1_SUB

  797. 

  798. 	subs	counterL, counterL, #1

  799. 	bgt	.Lzgemm_kernel_L1_Mv1_42

  800. 

  801. .Lzgemm_kernel_L1_Mv1_100:

  802. 

  803. 	SAVEv1x1

  804. 

  805. .Lzgemm_kernel_L1_Mv1_END:

  806. 

  807.     incd    counterI

  808.     whilelt p1.d, counterI, origM             //SVE instruction

  809.     cntp lanes, p0, p1.d

  810.     b.any   .Lzgemm_kernel_L1_Mv1_20   

  811. 

  812. .Lzgemm_kernel_L1_END:

  813. 

  814. /******************************************************************************/

  815. 

  816. .Lzgemm_kernel_L999:

  817. 	mov	x0, #0				// set return value

  818. 	ldp	d8, d9, [sp, #(0 * 16)]

  819. 	ldp	d10, d11, [sp, #(1 * 16)]

  820. 	ldp	d12, d13, [sp, #(2 * 16)]

  821. 	ldp	d14, d15, [sp, #(3 * 16)]

  822. 	ldp	d16, d17, [sp, #(4 * 16)]

  823. 	ldp	x18, x19, [sp, #(5 * 16)]

  824. 	ldp	x20, x21, [sp, #(6 * 16)]

  825. 	ldp	x22, x23, [sp, #(7 * 16)]

  826. 	ldp	x24, x25, [sp, #(8 * 16)]

  827. 	ldp	x26, x27, [sp, #(9 * 16)]

  828. 	ldr	x28, [sp, #(10 * 16)]

  829. 	add	sp, sp, #(11*16)

  830. 	ret

  831. 

  832. 	EPILOGUE