OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 39c90f9859
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '39c90f9859'
${ noResults }
OpenBLAS/kernel/arm64/sgemm_kernel_8x8.S

2306 lines
43 kB
Raw Blame History 

  1. /*******************************************************************************

  2. Copyright (c) 2015, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *******************************************************************************/

  27. 

  28. #define ASSEMBLER

  29. #include "common.h"

  30. 

  31. /*                   X0          X1          X2          s0        X3        x4       x5           x6  */

  32. /*int CNAME(BLASLONG bm,BLASLONG bn,BLASLONG bk,FLOAT alpha,FLOAT* ba,FLOAT* bb,FLOAT* C,BLASLONG ldc) */

  33. 

  34. #define origM		x0

  35. #define origN		x1

  36. #define origK		x2

  37. #define origPA		x3

  38. #define origPB		x4

  39. #define pC		x5

  40. #define LDC		x6

  41. #define offset		x7

  42. #define counterL	x8

  43. #define counterI	x9

  44. #define counterJ	x10

  45. #define pB		x11

  46. #define pCRow0		x12

  47. #define pCRow1		x13

  48. #define pCRow2		x14

  49. #define pA		x15

  50. #define temp		x16

  51. 

  52. #define alpha0		s10

  53. #define alphaV0		v10.s[0]

  54. #define alpha1		s11

  55. #define alphaV1		v11.s[0]

  56. #define alpha2		s14

  57. #define alphaV2		v14.s[0]

  58. #define alpha3		s15

  59. #define alphaV3		v15.s[0]

  60. 

  61. // 00 origM

  62. // 01 origN

  63. // 02 origK

  64. // 03 origPA

  65. // 04 origPB

  66. // 05 pC

  67. // 06 origLDC -> LDC

  68. // 07 offset

  69. // 08 counterL

  70. // 09 counterI

  71. // 10 counterJ

  72. // 11 pB

  73. // 12 pCRow0

  74. // 13 pCRow1

  75. // 14 pCRow2

  76. // 15 pA

  77. // 16 temp

  78. // 17

  79. // 18 must save

  80. // 19 must save

  81. // 20 must save

  82. // 21 must save

  83. // 22 must save

  84. // 23 must save

  85. // 24 must save

  86. // 25 must save

  87. // 26 must save

  88. // 27 must save

  89. // 28 must save

  90. // 29 frame

  91. // 30 link

  92. // 31 sp

  93. 

  94. //v00 ALPHA -> pA0_0, pA0_1, pA0_2, pA0_3

  95. //v01 pA0_4, pA0_5, pA0_6, pA0_7

  96. //v02 pA1_0, pA1_1, pA1_2, pA1_3

  97. //v03 pA1_4, pA1_5, pA1_6, pA1_7

  98. //v04 pB0_0, pB0_1, pB0_2, pB0_3

  99. //v05 pB0_4, pB0_5, pB0_6, pB0_7

  100. //v06 pB1_0, pB1_1, pB1_2, pB1_3

  101. //v07 pB1_4, pB1_5, pB1_6, pB1_7

  102. //v08 must save

  103. //v09 must save

  104. //v10 must save ALPHA0

  105. //v11 must save ALPHA1

  106. //v12 must save

  107. //v13 must save

  108. //v14 must save ALPHA2

  109. //v15 must save ALPHA3

  110. //v16 must save C00, C01, C02, C03

  111. //v17 must save C04, C05, C06, C07

  112. //v18 C08, C09, C10, C11

  113. //v19 C12, C13, C14, C15

  114. //v20 C16, C17, C18, C19

  115. //v21 C20, C21, C22, C23

  116. //v22 C24, C25, C26, C27

  117. //v23 C28, C29, C30, C31

  118. //v24 C32, C33, C34, C35

  119. //v25 C36, C37, C38, C39

  120. //v26 C40, C41, C42, C43

  121. //v27 C44, C45, C46, C47

  122. //v28 C48, C49, C50, C51

  123. //v29 C52, C53, C54, C55

  124. //v30 C56, C57, C58, C59

  125. //v31 C60, C61, C62, C63

  126. 

  127. /*******************************************************************************

  128. * Macro definitions

  129. *******************************************************************************/

  130. 

  131. .macro INIT8x8

  132. 	fmov		s16, wzr

  133. 	fmov		s17, wzr

  134. 	fmov		s18, s16

  135. 	fmov		s19, s17

  136. 	fmov		s20, wzr

  137. 	fmov		s21, s16

  138. 	fmov		s22, s17

  139. 	fmov		s23, s18

  140. 	fmov		s24, wzr

  141. 	fmov		s25, s16

  142. 	fmov		s26, s17

  143. 	fmov		s27, s18

  144. 	fmov		s28, wzr

  145. 	fmov		s29, s16

  146. 	fmov		s30, s17

  147. 	fmov		s31, s18

  148. .endm

  149. 

  150. .macro KERNEL8x8_I

  151. 	ld1	{v4.4s}, [pB]

  152. 	add	pB, pB, #16

  153. 	ld1	{v5.4s}, [pB]

  154. 	add	pB, pB, #16

  155. 	ld1	{v0.4s}, [pA]

  156. 	add	pA, pA, #16

  157. 	ld1	{v1.4s}, [pA]

  158. 	add	pA, pA, #16

  159. 

  160. 	fmul	v16.4s, v0.4s, v4.s[0]

  161. 	fmul	v17.4s, v1.4s, v4.s[0]

  162. 	fmul	v18.4s, v0.4s, v4.s[1]

  163. 	fmul	v19.4s, v1.4s, v4.s[1]

  164. 	fmul	v20.4s, v0.4s, v4.s[2]

  165. 	fmul	v21.4s, v1.4s, v4.s[2]

  166. 	fmul	v22.4s, v0.4s, v4.s[3]

  167. 	fmul	v23.4s, v1.4s, v4.s[3]

  168. 	fmul	v24.4s, v0.4s, v5.s[0]

  169. 	fmul	v25.4s, v1.4s, v5.s[0]

  170. 	fmul	v26.4s, v0.4s, v5.s[1]

  171. 	fmul	v27.4s, v1.4s, v5.s[1]

  172. 	fmul	v28.4s, v0.4s, v5.s[2]

  173. 	fmul	v29.4s, v1.4s, v5.s[2]

  174. 	fmul	v30.4s, v0.4s, v5.s[3]

  175. 	fmul	v31.4s, v1.4s, v5.s[3]

  176. 

  177. 	ld1	{v6.4s}, [pB]

  178. 	add	pB, pB, #16

  179. 	ld1	{v7.4s}, [pB]

  180. 	add	pB, pB, #16

  181. 	ld1	{v2.4s}, [pA]

  182. 	add	pA, pA, #16

  183. 	ld1	{v3.4s}, [pA]

  184. 	add	pA, pA, #16

  185. .endm

  186. 

  187. .macro KERNEL8x8_M1

  188. 	fmla	v16.4s, v0.4s, v4.s[0]

  189. 	fmla	v17.4s, v1.4s, v4.s[0]

  190. 	fmla	v18.4s, v0.4s, v4.s[1]

  191. 	fmla	v19.4s, v1.4s, v4.s[1]

  192. 	fmla	v20.4s, v0.4s, v4.s[2]

  193. 	fmla	v21.4s, v1.4s, v4.s[2]

  194. 	fmla	v22.4s, v0.4s, v4.s[3]

  195. 	fmla	v23.4s, v1.4s, v4.s[3]

  196. 	fmla	v24.4s, v0.4s, v5.s[0]

  197. 	fmla	v25.4s, v1.4s, v5.s[0]

  198. 	fmla	v26.4s, v0.4s, v5.s[1]

  199. 	fmla	v27.4s, v1.4s, v5.s[1]

  200. 	fmla	v28.4s, v0.4s, v5.s[2]

  201. 	fmla	v29.4s, v1.4s, v5.s[2]

  202. 	fmla	v30.4s, v0.4s, v5.s[3]

  203. 	fmla	v31.4s, v1.4s, v5.s[3]

  204. 

  205. 	ld1	{v6.4s}, [pB]

  206. 	add	pB, pB, #16

  207. 	ld1	{v7.4s}, [pB]

  208. 	add	pB, pB, #16

  209. 	ld1	{v2.4s}, [pA]

  210. 	add	pA, pA, #16

  211. 	ld1	{v3.4s}, [pA]

  212. 	add	pA, pA, #16

  213. .endm

  214. 

  215. .macro KERNEL8x8_M2

  216. 	fmla	v16.4s, v2.4s, v6.s[0]

  217. 	fmla	v17.4s, v3.4s, v6.s[0]

  218. 	fmla	v18.4s, v2.4s, v6.s[1]

  219. 	fmla	v19.4s, v3.4s, v6.s[1]

  220. 	fmla	v20.4s, v2.4s, v6.s[2]

  221. 	fmla	v21.4s, v3.4s, v6.s[2]

  222. 	fmla	v22.4s, v2.4s, v6.s[3]

  223. 	fmla	v23.4s, v3.4s, v6.s[3]

  224. 	fmla	v24.4s, v2.4s, v7.s[0]

  225. 	fmla	v25.4s, v3.4s, v7.s[0]

  226. 	fmla	v26.4s, v2.4s, v7.s[1]

  227. 	fmla	v27.4s, v3.4s, v7.s[1]

  228. 	fmla	v28.4s, v2.4s, v7.s[2]

  229. 	fmla	v29.4s, v3.4s, v7.s[2]

  230. 	fmla	v30.4s, v2.4s, v7.s[3]

  231. 	fmla	v31.4s, v3.4s, v7.s[3]

  232. 

  233. 	ld1	{v4.4s}, [pB]

  234. 	add	pB, pB, #16

  235. 	ld1	{v5.4s}, [pB]

  236. 	add	pB, pB, #16

  237. 	ld1	{v0.4s}, [pA]

  238. 	add	pA, pA, #16

  239. 	ld1	{v1.4s}, [pA]

  240. 	add	pA, pA, #16

  241. .endm

  242. 

  243. .macro KERNEL8x8_E

  244. 	fmla	v16.4s, v2.4s, v6.s[0]

  245. 	fmla	v17.4s, v3.4s, v6.s[0]

  246. 	fmla	v18.4s, v2.4s, v6.s[1]

  247. 	fmla	v19.4s, v3.4s, v6.s[1]

  248. 	fmla	v20.4s, v2.4s, v6.s[2]

  249. 	fmla	v21.4s, v3.4s, v6.s[2]

  250. 	fmla	v22.4s, v2.4s, v6.s[3]

  251. 	fmla	v23.4s, v3.4s, v6.s[3]

  252. 	fmla	v24.4s, v2.4s, v7.s[0]

  253. 	fmla	v25.4s, v3.4s, v7.s[0]

  254. 	fmla	v26.4s, v2.4s, v7.s[1]

  255. 	fmla	v27.4s, v3.4s, v7.s[1]

  256. 	fmla	v28.4s, v2.4s, v7.s[2]

  257. 	fmla	v29.4s, v3.4s, v7.s[2]

  258. 	fmla	v30.4s, v2.4s, v7.s[3]

  259. 	fmla	v31.4s, v3.4s, v7.s[3]

  260. .endm

  261. 

  262. .macro KERNEL8x8_SUB

  263. 	ld1	{v4.4s}, [pB]

  264. 	add	pB, pB, #16

  265. 	ld1	{v5.4s}, [pB]

  266. 	add	pB, pB, #16

  267. 	ld1	{v0.4s}, [pA]

  268. 	add	pA, pA, #16

  269. 	ld1	{v1.4s}, [pA]

  270. 	add	pA, pA, #16

  271. 

  272. 	fmla	v16.4s, v0.4s, v4.s[0]

  273. 	fmla	v17.4s, v1.4s, v4.s[0]

  274. 	fmla	v18.4s, v0.4s, v4.s[1]

  275. 	fmla	v19.4s, v1.4s, v4.s[1]

  276. 	fmla	v20.4s, v0.4s, v4.s[2]

  277. 	fmla	v21.4s, v1.4s, v4.s[2]

  278. 	fmla	v22.4s, v0.4s, v4.s[3]

  279. 	fmla	v23.4s, v1.4s, v4.s[3]

  280. 	fmla	v24.4s, v0.4s, v5.s[0]

  281. 	fmla	v25.4s, v1.4s, v5.s[0]

  282. 	fmla	v26.4s, v0.4s, v5.s[1]

  283. 	fmla	v27.4s, v1.4s, v5.s[1]

  284. 	fmla	v28.4s, v0.4s, v5.s[2]

  285. 	fmla	v29.4s, v1.4s, v5.s[2]

  286. 	fmla	v30.4s, v0.4s, v5.s[3]

  287. 	fmla	v31.4s, v1.4s, v5.s[3]

  288. .endm

  289. 

  290. .macro SAVE8x8

  291. 	add	pCRow1, pCRow0, LDC

  292. 

  293. 	ld1	{v0.4s, v1.4s}, [pCRow0]

  294. 	fmla	v0.4s, v16.4s, alphaV0

  295. 	fmla	v1.4s, v17.4s, alphaV1

  296. 	st1 	{v0.4s, v1.4s}, [pCRow0]

  297. 

  298. 	add	pCRow2, pCRow1, LDC

  299. 

  300. 	ld1	{v2.4s, v3.4s}, [pCRow1]

  301. 	fmla	v2.4s, v18.4s, alphaV2

  302. 	fmla	v3.4s, v19.4s, alphaV3

  303. 	st1 	{v2.4s, v3.4s}, [pCRow1]

  304. 

  305. 	add	pCRow1, pCRow2, LDC

  306. 

  307. 	ld1	{v4.4s, v5.4s}, [pCRow2]

  308. 	fmla	v4.4s, v20.4s, alphaV0

  309. 	fmla	v5.4s, v21.4s, alphaV1

  310. 	st1 	{v4.4s, v5.4s}, [pCRow2]

  311. 

  312. 	add	pCRow2, pCRow1, LDC

  313. 

  314. 	ld1	{v6.4s, v7.4s}, [pCRow1]

  315. 	fmla	v6.4s, v22.4s, alphaV2

  316. 	fmla	v7.4s, v23.4s, alphaV3

  317. 	st1 	{v6.4s, v7.4s}, [pCRow1]

  318. 

  319. 	add	pCRow1, pCRow2, LDC

  320. 

  321. 	ld1	{v0.4s, v1.4s}, [pCRow2]

  322. 	fmla	v0.4s, v24.4s, alphaV0

  323. 	fmla	v1.4s, v25.4s, alphaV1

  324. 	st1 	{v0.4s, v1.4s}, [pCRow2]

  325. 

  326. 	add	pCRow2, pCRow1, LDC

  327. 

  328. 	ld1	{v2.4s, v3.4s}, [pCRow1]

  329. 	fmla	v2.4s, v26.4s, alphaV2

  330. 	fmla	v3.4s, v27.4s, alphaV3

  331. 	st1 	{v2.4s, v3.4s}, [pCRow1]

  332. 

  333. 	add	pCRow1, pCRow2, LDC

  334. 

  335. 	ld1	{v4.4s, v5.4s}, [pCRow2]

  336. 	fmla	v4.4s, v28.4s, alphaV0

  337. 	fmla	v5.4s, v29.4s, alphaV1

  338. 	st1 	{v4.4s, v5.4s}, [pCRow2]

  339. 

  340. 	ld1	{v6.4s, v7.4s}, [pCRow1]

  341. 	fmla	v6.4s, v30.4s, alphaV2

  342. 	fmla	v7.4s, v31.4s, alphaV3

  343. 	st1 	{v6.4s, v7.4s}, [pCRow1]

  344. 

  345. 	add	pCRow0, pCRow0, #32

  346. .endm

  347. 

  348. /******************************************************************************/

  349. 

  350. 

  351. .macro INIT4x8

  352. 	fmov		s16, wzr

  353. 	fmov		s18, wzr

  354. 	fmov		s20, wzr

  355. 	fmov		s22, s16

  356. 	fmov		s24, wzr

  357. 	fmov		s26, s16

  358. 	fmov		s28, s18

  359. 	fmov		s30, s20

  360. .endm

  361. 

  362. .macro KERNEL4x8_I

  363. 	ld1	{v4.4s}, [pB]

  364. 	add	pB, pB, #16

  365. 	ld1	{v5.4s}, [pB]

  366. 	add	pB, pB, #16

  367. 	ld1	{v0.4s}, [pA]

  368. 	add	pA, pA, #16

  369. 

  370. 	fmul	v16.4s, v0.4s, v4.s[0]

  371. 	fmul	v18.4s, v0.4s, v4.s[1]

  372. 	fmul	v20.4s, v0.4s, v4.s[2]

  373. 	fmul	v22.4s, v0.4s, v4.s[3]

  374. 	fmul	v24.4s, v0.4s, v5.s[0]

  375. 	fmul	v26.4s, v0.4s, v5.s[1]

  376. 	fmul	v28.4s, v0.4s, v5.s[2]

  377. 	fmul	v30.4s, v0.4s, v5.s[3]

  378. 

  379. 	ld1	{v6.4s}, [pB]

  380. 	add	pB, pB, #16

  381. 	ld1	{v7.4s}, [pB]

  382. 	add	pB, pB, #16

  383. 	ld1	{v2.4s}, [pA]

  384. 	add	pA, pA, #16

  385. .endm

  386. 

  387. .macro KERNEL4x8_M1

  388. 	fmla	v16.4s, v0.4s, v4.s[0]

  389. 	fmla	v18.4s, v0.4s, v4.s[1]

  390. 	fmla	v20.4s, v0.4s, v4.s[2]

  391. 	fmla	v22.4s, v0.4s, v4.s[3]

  392. 	fmla	v24.4s, v0.4s, v5.s[0]

  393. 	fmla	v26.4s, v0.4s, v5.s[1]

  394. 	fmla	v28.4s, v0.4s, v5.s[2]

  395. 	fmla	v30.4s, v0.4s, v5.s[3]

  396. 

  397. 	ld1	{v6.4s}, [pB]

  398. 	add	pB, pB, #16

  399. 	ld1	{v7.4s}, [pB]

  400. 	add	pB, pB, #16

  401. 	ld1	{v2.4s}, [pA]

  402. 	add	pA, pA, #16

  403. .endm

  404. 

  405. .macro KERNEL4x8_M2

  406. 	fmla	v16.4s, v2.4s, v6.s[0]

  407. 	fmla	v18.4s, v2.4s, v6.s[1]

  408. 	fmla	v20.4s, v2.4s, v6.s[2]

  409. 	fmla	v22.4s, v2.4s, v6.s[3]

  410. 	fmla	v24.4s, v2.4s, v7.s[0]

  411. 	fmla	v26.4s, v2.4s, v7.s[1]

  412. 	fmla	v28.4s, v2.4s, v7.s[2]

  413. 	fmla	v30.4s, v2.4s, v7.s[3]

  414. 

  415. 	ld1	{v4.4s}, [pB]

  416. 	add	pB, pB, #16

  417. 	ld1	{v5.4s}, [pB]

  418. 	add	pB, pB, #16

  419. 	ld1	{v0.4s}, [pA]

  420. 	add	pA, pA, #16

  421. .endm

  422. 

  423. .macro KERNEL4x8_E

  424. 	fmla	v16.4s, v2.4s, v6.s[0]

  425. 	fmla	v18.4s, v2.4s, v6.s[1]

  426. 	fmla	v20.4s, v2.4s, v6.s[2]

  427. 	fmla	v22.4s, v2.4s, v6.s[3]

  428. 	fmla	v24.4s, v2.4s, v7.s[0]

  429. 	fmla	v26.4s, v2.4s, v7.s[1]

  430. 	fmla	v28.4s, v2.4s, v7.s[2]

  431. 	fmla	v30.4s, v2.4s, v7.s[3]

  432. .endm

  433. 

  434. .macro KERNEL4x8_SUB

  435. 	ld1	{v4.4s}, [pB]

  436. 	add	pB, pB, #16

  437. 	ld1	{v5.4s}, [pB]

  438. 	add	pB, pB, #16

  439. 	ld1	{v0.4s}, [pA]

  440. 	add	pA, pA, #16

  441. 

  442. 	fmla	v16.4s, v0.4s, v4.s[0]

  443. 	fmla	v18.4s, v0.4s, v4.s[1]

  444. 	fmla	v20.4s, v0.4s, v4.s[2]

  445. 	fmla	v22.4s, v0.4s, v4.s[3]

  446. 	fmla	v24.4s, v0.4s, v5.s[0]

  447. 	fmla	v26.4s, v0.4s, v5.s[1]

  448. 	fmla	v28.4s, v0.4s, v5.s[2]

  449. 	fmla	v30.4s, v0.4s, v5.s[3]

  450. .endm

  451. 

  452. .macro SAVE4x8

  453. 	add	pCRow1, pCRow0, LDC

  454. 

  455. 	ld1	{v0.4s}, [pCRow0]

  456. 	fmla	v0.4s, v16.4s, alphaV0

  457. 	st1 	{v0.4s}, [pCRow0]

  458. 

  459. 	add	pCRow2, pCRow1, LDC

  460. 

  461. 	ld1	{v2.4s}, [pCRow1]

  462. 	fmla	v2.4s, v18.4s, alphaV2

  463. 	st1 	{v2.4s}, [pCRow1]

  464. 

  465. 	add	pCRow1, pCRow2, LDC

  466. 

  467. 	ld1	{v4.4s}, [pCRow2]

  468. 	fmla	v4.4s, v20.4s, alphaV0

  469. 	st1 	{v4.4s}, [pCRow2]

  470. 

  471. 	add	pCRow2, pCRow1, LDC

  472. 

  473. 	ld1	{v6.4s}, [pCRow1]

  474. 	fmla	v6.4s, v22.4s, alphaV2

  475. 	st1 	{v6.4s}, [pCRow1]

  476. 

  477. 	add	pCRow1, pCRow2, LDC

  478. 

  479. 	ld1	{v0.4s}, [pCRow2]

  480. 	fmla	v0.4s, v24.4s, alphaV0

  481. 	st1 	{v0.4s}, [pCRow2]

  482. 

  483. 	add	pCRow2, pCRow1, LDC

  484. 

  485. 	ld1	{v2.4s}, [pCRow1]

  486. 	fmla	v2.4s, v26.4s, alphaV2

  487. 	st1 	{v2.4s}, [pCRow1]

  488. 

  489. 	add	pCRow1, pCRow2, LDC

  490. 

  491. 	ld1	{v4.4s}, [pCRow2]

  492. 	fmla	v4.4s, v28.4s, alphaV0

  493. 	st1 	{v4.4s}, [pCRow2]

  494. 

  495. 	ld1	{v6.4s}, [pCRow1]

  496. 	fmla	v6.4s, v30.4s, alphaV2

  497. 	st1 	{v6.4s}, [pCRow1]

  498. 

  499. 	add	pCRow0, pCRow0, #16

  500. .endm

  501. 

  502. /******************************************************************************/

  503. 

  504. .macro INIT2x8

  505. 	fmov		s16, wzr

  506. 	fmov		s18, wzr

  507. 	fmov		s20, wzr

  508. 	fmov		s22, s16

  509. 	fmov		s24, wzr

  510. 	fmov		s26, s16

  511. 	fmov		s28, s18

  512. 	fmov		s30, s20

  513. .endm

  514. 

  515. .macro KERNEL2x8_SUB

  516. 	ld1	{v4.4s}, [pB]

  517. 	add	pB, pB, #16

  518. 	ld1	{v5.4s}, [pB]

  519. 	add	pB, pB, #16

  520. 	ld1	{v0.2s}, [pA]

  521. 	add	pA, pA, #8

  522. 

  523. 	fmla	v16.2s, v0.2s, v4.s[0]

  524. 	fmla	v18.2s, v0.2s, v4.s[1]

  525. 	fmla	v20.2s, v0.2s, v4.s[2]

  526. 	fmla	v22.2s, v0.2s, v4.s[3]

  527. 	fmla	v24.2s, v0.2s, v5.s[0]

  528. 	fmla	v26.2s, v0.2s, v5.s[1]

  529. 	fmla	v28.2s, v0.2s, v5.s[2]

  530. 	fmla	v30.2s, v0.2s, v5.s[3]

  531. .endm

  532. 

  533. .macro SAVE2x8

  534. 	add	pCRow1, pCRow0, LDC

  535. 

  536. 	ld1	{v0.2s}, [pCRow0]

  537. 	fmla	v0.2s, v16.2s, alphaV0

  538. 	st1 	{v0.2s}, [pCRow0]

  539. 

  540. 	add	pCRow2, pCRow1, LDC

  541. 

  542. 	ld1	{v2.2s}, [pCRow1]

  543. 	fmla	v2.2s, v18.2s, alphaV2

  544. 	st1 	{v2.2s}, [pCRow1]

  545. 

  546. 	add	pCRow1, pCRow2, LDC

  547. 

  548. 	ld1	{v4.2s}, [pCRow2]

  549. 	fmla	v4.2s, v20.2s, alphaV0

  550. 	st1 	{v4.2s}, [pCRow2]

  551. 

  552. 	add	pCRow2, pCRow1, LDC

  553. 

  554. 	ld1	{v6.2s}, [pCRow1]

  555. 	fmla	v6.2s, v22.2s, alphaV2

  556. 	st1 	{v6.2s}, [pCRow1]

  557. 

  558. 	add	pCRow1, pCRow2, LDC

  559. 

  560. 	ld1	{v0.2s}, [pCRow2]

  561. 	fmla	v0.2s, v24.2s, alphaV0

  562. 	st1 	{v0.2s}, [pCRow2]

  563. 

  564. 	add	pCRow2, pCRow1, LDC

  565. 

  566. 	ld1	{v2.2s}, [pCRow1]

  567. 	fmla	v2.2s, v26.2s, alphaV2

  568. 	st1 	{v2.2s}, [pCRow1]

  569. 

  570. 	add	pCRow1, pCRow2, LDC

  571. 

  572. 	ld1	{v4.2s}, [pCRow2]

  573. 	fmla	v4.2s, v28.2s, alphaV0

  574. 	st1 	{v4.2s}, [pCRow2]

  575. 

  576. 	ld1	{v6.2s}, [pCRow1]

  577. 	fmla	v6.2s, v30.2s, alphaV2

  578. 	st1 	{v6.2s}, [pCRow1]

  579. 

  580. 	add	pCRow0, pCRow0, #8

  581. .endm

  582. 

  583. /******************************************************************************/

  584. 

  585. .macro INIT1x8

  586. 	fmov		s16, wzr

  587. 	fmov		s18, wzr

  588. 	fmov		s20, wzr

  589. 	fmov		s22, s16

  590. 	fmov		s24, wzr

  591. 	fmov		s26, s16

  592. 	fmov		s28, s18

  593. 	fmov		s30, s20

  594. .endm

  595. 

  596. .macro KERNEL1x8_SUB

  597. 	ld1	{v4.4s}, [pB]

  598. 	add	pB, pB, #16

  599. 	ld1	{v5.4s}, [pB]

  600. 	add	pB, pB, #16

  601. 	ldr	s0, [pA]

  602. 	add	pA, pA, #4

  603. 

  604. 	fmla	s16, s0, v4.s[0]

  605. 	fmla	s18, s0, v4.s[1]

  606. 	fmla	s20, s0, v4.s[2]

  607. 	fmla	s22, s0, v4.s[3]

  608. 	fmla	s24, s0, v5.s[0]

  609. 	fmla	s26, s0, v5.s[1]

  610. 	fmla	s28, s0, v5.s[2]

  611. 	fmla	s30, s0, v5.s[3]

  612. .endm

  613. 

  614. .macro SAVE1x8

  615. 	add	pCRow1, pCRow0, LDC

  616. 

  617. 	ldr	s0, [pCRow0]

  618. 	fmla	s0, s16, alphaV0

  619. 	str 	s0, [pCRow0]

  620. 

  621. 	add	pCRow2, pCRow1, LDC

  622. 

  623. 	ldr	s2, [pCRow1]

  624. 	fmla	s2, s18, alphaV2

  625. 	str 	s2, [pCRow1]

  626. 

  627. 	add	pCRow1, pCRow2, LDC

  628. 

  629. 	ldr	s4, [pCRow2]

  630. 	fmla	s4, s20, alphaV0

  631. 	str 	s4, [pCRow2]

  632. 

  633. 	add	pCRow2, pCRow1, LDC

  634. 

  635. 	ldr	s6, [pCRow1]

  636. 	fmla	s6, s22, alphaV2

  637. 	str 	s6, [pCRow1]

  638. 

  639. 	add	pCRow1, pCRow2, LDC

  640. 

  641. 	ldr	s0, [pCRow2]

  642. 	fmla	s0, s24, alphaV0

  643. 	str 	s0, [pCRow2]

  644. 

  645. 	add	pCRow2, pCRow1, LDC

  646. 

  647. 	ldr	s2, [pCRow1]

  648. 	fmla	s2, s26, alphaV2

  649. 	str 	s2, [pCRow1]

  650. 

  651. 	add	pCRow1, pCRow2, LDC

  652. 

  653. 	ldr	s4, [pCRow2]

  654. 	fmla	s4, s28, alphaV0

  655. 	str 	s4, [pCRow2]

  656. 

  657. 	ldr	s6, [pCRow1]

  658. 	fmla	s6, s30, alphaV2

  659. 	str 	s6, [pCRow1]

  660. 

  661. 	add	pCRow0, pCRow0, #4

  662. .endm

  663. 

  664. /******************************************************************************/

  665. 

  666. .macro INIT8x4

  667. 	fmov		s16, wzr

  668. 	fmov		s17, wzr

  669. 	fmov		s20, wzr

  670. 	fmov		s21, s16

  671. 	fmov		s24, wzr

  672. 	fmov		s25, s16

  673. 	fmov		s28, wzr

  674. 	fmov		s29, s16

  675. .endm

  676. 

  677. .macro KERNEL8x4_I

  678. 	ld1	{v8.2s, v9.2s}, [pB]

  679. 	add	pB, pB, #16

  680. 	ld1	{v0.4s}, [pA]

  681. 	add	pA, pA, #16

  682. 	ld1	{v1.4s}, [pA]

  683. 	add	pA, pA, #16

  684. 

  685. 	fmul	v16.4s, v0.4s, v8.s[0]

  686. 	fmul	v17.4s, v1.4s, v8.s[0]

  687. 	fmul	v20.4s, v0.4s, v8.s[1]

  688. 	fmul	v21.4s, v1.4s, v8.s[1]

  689. 	fmul	v24.4s, v0.4s, v9.s[0]

  690. 	fmul	v25.4s, v1.4s, v9.s[0]

  691. 	fmul	v28.4s, v0.4s, v9.s[1]

  692. 	fmul	v29.4s, v1.4s, v9.s[1]

  693. 

  694. 	ld1	{v12.2s, v13.2s}, [pB]

  695. 	add	pB, pB, #16

  696. 	ld1	{v4.4s}, [pA]

  697. 	add	pA, pA, #16

  698. 	ld1	{v5.4s}, [pA]

  699. 	add	pA, pA, #16

  700. .endm

  701. 

  702. .macro KERNEL8x4_M1

  703. 	fmla	v16.4s, v0.4s, v8.s[0]

  704. 	fmla	v17.4s, v1.4s, v8.s[0]

  705. 	fmla	v20.4s, v0.4s, v8.s[1]

  706. 	fmla	v21.4s, v1.4s, v8.s[1]

  707. 	fmla	v24.4s, v0.4s, v9.s[0]

  708. 	fmla	v25.4s, v1.4s, v9.s[0]

  709. 	fmla	v28.4s, v0.4s, v9.s[1]

  710. 	fmla	v29.4s, v1.4s, v9.s[1]

  711. 

  712. 	ld1	{v12.2s, v13.2s}, [pB]

  713. 	add	pB, pB, #16

  714. 	ld1	{v4.4s}, [pA]

  715. 	add	pA, pA, #16

  716. 	ld1	{v5.4s}, [pA]

  717. 	add	pA, pA, #16

  718. .endm

  719. 

  720. .macro KERNEL8x4_M2

  721. 	fmla	v16.4s, v4.4s, v12.s[0]

  722. 	fmla	v17.4s, v5.4s, v12.s[0]

  723. 	fmla	v20.4s, v4.4s, v12.s[1]

  724. 	fmla	v21.4s, v5.4s, v12.s[1]

  725. 	fmla	v24.4s, v4.4s, v13.s[0]

  726. 	fmla	v25.4s, v5.4s, v13.s[0]

  727. 	fmla	v28.4s, v4.4s, v13.s[1]

  728. 	fmla	v29.4s, v5.4s, v13.s[1]

  729. 

  730. 	ld1	{v8.2s, v9.2s}, [pB]

  731. 	add	pB, pB, #16

  732. 	ld1	{v0.4s}, [pA]

  733. 	add	pA, pA, #16

  734. 	ld1	{v1.4s}, [pA]

  735. 	add	pA, pA, #16

  736. .endm

  737. 

  738. .macro KERNEL8x4_E

  739. 	fmla	v16.4s, v4.4s, v12.s[0]

  740. 	fmla	v17.4s, v5.4s, v12.s[0]

  741. 	fmla	v20.4s, v4.4s, v12.s[1]

  742. 	fmla	v21.4s, v5.4s, v12.s[1]

  743. 	fmla	v24.4s, v4.4s, v13.s[0]

  744. 	fmla	v25.4s, v5.4s, v13.s[0]

  745. 	fmla	v28.4s, v4.4s, v13.s[1]

  746. 	fmla	v29.4s, v5.4s, v13.s[1]

  747. .endm

  748. 

  749. .macro KERNEL8x4_SUB

  750. 	ld1	{v8.2s, v9.2s}, [pB]

  751. 	add	pB, pB, #16

  752. 	ld1	{v0.4s}, [pA]

  753. 	add	pA, pA, #16

  754. 	ld1	{v1.4s}, [pA]

  755. 	add	pA, pA, #16

  756. 

  757. 	fmla	v16.4s, v0.4s, v8.s[0]

  758. 	fmla	v17.4s, v1.4s, v8.s[0]

  759. 	fmla	v20.4s, v0.4s, v8.s[1]

  760. 	fmla	v21.4s, v1.4s, v8.s[1]

  761. 	fmla	v24.4s, v0.4s, v9.s[0]

  762. 	fmla	v25.4s, v1.4s, v9.s[0]

  763. 	fmla	v28.4s, v0.4s, v9.s[1]

  764. 	fmla	v29.4s, v1.4s, v9.s[1]

  765. .endm

  766. 

  767. .macro SAVE8x4

  768. 	add	pCRow1, pCRow0, LDC

  769. 

  770. 	ld1	{v0.4s, v1.4s}, [pCRow0]

  771. 	fmla	v0.4s, v16.4s, alphaV0

  772. 	fmla	v1.4s, v17.4s, alphaV1

  773. 	st1 	{v0.4s, v1.4s}, [pCRow0]

  774. 

  775. 	add	pCRow2, pCRow1, LDC

  776. 

  777. 	ld1	{v4.4s, v5.4s}, [pCRow1]

  778. 	fmla	v4.4s, v20.4s, alphaV0

  779. 	fmla	v5.4s, v21.4s, alphaV1

  780. 	st1 	{v4.4s, v5.4s}, [pCRow1]

  781. 

  782. 	add	pCRow1, pCRow2, LDC

  783. 

  784. 	ld1	{v0.4s, v1.4s}, [pCRow2]

  785. 	fmla	v0.4s, v24.4s, alphaV0

  786. 	fmla	v1.4s, v25.4s, alphaV1

  787. 	st1 	{v0.4s, v1.4s}, [pCRow2]

  788. 

  789. 	ld1	{v4.4s, v5.4s}, [pCRow1]

  790. 	fmla	v4.4s, v28.4s, alphaV0

  791. 	fmla	v5.4s, v29.4s, alphaV1

  792. 	st1 	{v4.4s, v5.4s}, [pCRow1]

  793. 

  794. 	add	pCRow0, pCRow0, #32

  795. .endm

  796. 

  797. /******************************************************************************/

  798. 

  799. 

  800. .macro INIT4x4

  801. 	fmov		s16, wzr

  802. 	fmov		s17, s16

  803. 	fmov		s20, s17

  804. 	fmov		s21, s16

  805. 	fmov		s24, s17

  806. 	fmov		s25, s16

  807. 	fmov		s28, s17

  808. 	fmov		s29, s16

  809. .endm

  810. 

  811. .macro KERNEL4x4_I

  812. 	ld1	{v8.2s, v9.2s}, [pB]

  813. 	add	pB, pB, #16

  814. 	ld1	{v0.2s, v1.2s}, [pA]

  815. 	add	pA, pA, #16

  816. 

  817. 	fmul	v16.2s, v0.2s, v8.s[0]

  818. 	fmul	v29.2s, v1.2s, v9.s[1]

  819. 

  820. 	fmul	v20.2s, v0.2s, v8.s[1]

  821. 	fmul	v25.2s, v1.2s, v9.s[0]

  822. 

  823. 	fmul	v24.2s, v0.2s, v9.s[0]

  824. 	fmul	v21.2s, v1.2s, v8.s[1]

  825. 

  826. 	fmul	v28.2s, v0.2s, v9.s[1]

  827. 	fmul	v17.2s, v1.2s, v8.s[0]

  828. 

  829. 	ld1	{v12.2s, v13.2s}, [pB]

  830. 	add	pB, pB, #16

  831. 	ld1	{v4.2s, v5.2s}, [pA]

  832. 	add	pA, pA, #16

  833. .endm

  834. 

  835. .macro KERNEL4x4_M1

  836. 	fmla	v16.2s, v0.2s, v8.s[0]

  837. 	fmla	v29.2s, v1.2s, v9.s[1]

  838. 

  839. 	ld1	{v12.2s, v13.2s}, [pB]		// For next round

  840. 	add	pB, pB, #16

  841. 

  842. 	fmla	v20.2s, v0.2s, v8.s[1]

  843. 	fmla	v25.2s, v1.2s, v9.s[0]

  844. 

  845. 	ld1	{v4.2s, v5.2s}, [pA]		// For next round

  846. 	add	pA, pA, #16

  847. 

  848. 	fmla	v24.2s, v0.2s, v9.s[0]

  849. 	fmla	v21.2s, v1.2s, v8.s[1]

  850. 

  851. 	prfm	PLDL1KEEP, [pB, #512]

  852. 

  853. 	fmla	v28.2s, v0.2s, v9.s[1]

  854. 	fmla	v17.2s, v1.2s, v8.s[0]

  855. .endm

  856. 

  857. .macro KERNEL4x4_M2

  858. 	fmla	v16.2s, v4.2s, v12.s[0]

  859. 	fmla	v29.2s, v5.2s, v13.s[1]

  860. 

  861. 	ld1	{v8.2s, v9.2s}, [pB]		// For next round

  862. 	add	pB, pB, #16

  863. 

  864. 	fmla	v20.2s, v4.2s, v12.s[1]

  865. 	fmla	v25.2s, v5.2s, v13.s[0]

  866. 

  867. 	ld1	{v0.2s, v1.2s}, [pA]		// For next round

  868. 	add	pA, pA, #16

  869. 

  870. 	fmla	v24.2s, v4.2s, v13.s[0]

  871. 	fmla	v21.2s, v5.2s, v12.s[1]

  872. 

  873. 	prfm	PLDL1KEEP, [pA, #512]

  874. 

  875. 	fmla	v28.2s, v4.2s, v13.s[1]

  876. 	fmla	v17.2s, v5.2s, v12.s[0]

  877. .endm

  878. 

  879. .macro KERNEL4x4_E

  880. 	fmla	v16.2s, v4.2s, v12.s[0]

  881. 	fmla	v29.2s, v5.2s, v13.s[1]

  882. 

  883. 	fmla	v20.2s, v4.2s, v12.s[1]

  884. 	fmla	v25.2s, v5.2s, v13.s[0]

  885. 

  886. 	fmla	v24.2s, v4.2s, v13.s[0]

  887. 	fmla	v21.2s, v5.2s, v12.s[1]

  888. 

  889. 	fmla	v28.2s, v4.2s, v13.s[1]

  890. 	fmla	v17.2s, v5.2s, v12.s[0]

  891. .endm

  892. 

  893. .macro KERNEL4x4_SUB

  894. 	ld1	{v8.2s, v9.2s}, [pB]

  895. 	add	pB, pB, #16

  896. 	ld1	{v0.2s, v1.2s}, [pA]

  897. 	add	pA, pA, #16

  898. 

  899. 	fmla	v16.2s, v0.2s, v8.s[0]

  900. 	fmla	v29.2s, v1.2s, v9.s[1]

  901. 

  902. 	fmla	v20.2s, v0.2s, v8.s[1]

  903. 	fmla	v25.2s, v1.2s, v9.s[0]

  904. 

  905. 	fmla	v24.2s, v0.2s, v9.s[0]

  906. 	fmla	v21.2s, v1.2s, v8.s[1]

  907. 

  908. 	fmla	v28.2s, v0.2s, v9.s[1]

  909. 	fmla	v17.2s, v1.2s, v8.s[0]

  910. .endm

  911. 

  912. .macro SAVE4x4

  913. 	ld1 	{v8.2s, v9.2s}, [pCRow0]

  914. 	fmla	v8.2s, v16.2s, alphaV0

  915. 	fmla	v9.2s, v17.2s, alphaV1

  916. 	st1 	{v8.2s, v9.2s}, [pCRow0]

  917. 

  918. 	add	pCRow1, pCRow0, LDC

  919. 	ld1 	{v12.2s, v13.2s}, [pCRow1]

  920. 	fmla	v12.2s, v20.2s, alphaV2

  921. 	fmla	v13.2s, v21.2s, alphaV3

  922. 	st1 	{v12.2s, v13.2s}, [pCRow1]

  923. 

  924. 	add	pCRow2, pCRow1, LDC

  925. 	ld1 	{v8.2s, v9.2s}, [pCRow2]

  926. 	fmla	v8.2s, v24.2s, alphaV0

  927. 	fmla	v9.2s, v25.2s, alphaV1

  928. 	st1 	{v8.2s, v9.2s}, [pCRow2]

  929. 

  930. 	add	pCRow1, pCRow2, LDC

  931. 	ld1 	{v12.2s, v13.2s}, [pCRow1]

  932. 	fmla	v12.2s, v28.2s, alphaV2

  933. 	fmla	v13.2s, v29.2s, alphaV3

  934. 	st1 	{v12.2s, v13.2s}, [pCRow1]

  935. 

  936. 	add	pCRow0, pCRow0, #16

  937. .endm

  938. 

  939. /******************************************************************************/

  940. 

  941. .macro INIT2x4

  942. 	fmov		s16, wzr

  943. 	fmov		s20, s16

  944. 	fmov		s24, s20

  945. 	fmov		s28, s16

  946. .endm

  947. 

  948. .macro KERNEL2x4_SUB

  949. 	ld1	{v8.2s, v9.2s}, [pB]

  950. 	add	pB, pB, #16

  951. 	ld1	{v0.2s}, [pA]

  952. 	add	pA, pA, #8

  953. 

  954. 	fmla	v16.2s, v0.2s, v8.s[0]

  955. 	fmla	v20.2s, v0.2s, v8.s[1]

  956. 	fmla	v24.2s, v0.2s, v9.s[0]

  957. 	fmla	v28.2s, v0.2s, v9.s[1]

  958. .endm

  959. 

  960. .macro SAVE2x4

  961. 	ld1	{v8.2s}, [pCRow0]

  962. 	fmla	v8.2s, v16.2s, alphaV0

  963. 	st1	{v8.2s}, [pCRow0]

  964. 

  965. 	add	pCRow1, pCRow0, LDC

  966. 	ld1	{v12.2s}, [pCRow1]

  967. 	fmla	v12.2s, v20.2s, alphaV1

  968. 	st1	{v12.2s}, [pCRow1]

  969. 

  970. 	add	pCRow2, pCRow1, LDC

  971. 	ld1	{v8.2s}, [pCRow2]

  972. 	fmla	v8.2s, v24.2s, alphaV2

  973. 	st1	{v8.2s}, [pCRow2]

  974. 

  975. 	add	pCRow1, pCRow2, LDC

  976. 	ld1	{v12.2s}, [pCRow1]

  977. 	fmla	v12.2s, v28.2s, alphaV3

  978. 	st1	{v12.2s}, [pCRow1]

  979. 

  980. 	add	pCRow0, pCRow0, #8

  981. .endm

  982. 

  983. /******************************************************************************/

  984. 

  985. .macro INIT1x4

  986. 	fmov		s16, wzr

  987. 	fmov		s20, s16

  988. .endm

  989. 

  990. .macro KERNEL1x4_SUB

  991. 	ldr	s0, [pA]

  992. 	add	pA, pA, #4

  993. 

  994. 	ld1	{v8.2s, v9.2s}, [pB]

  995. 	add	pB, pB, #16

  996. 

  997. 	fmla	v16.2s, v8.2s, v0.s[0]

  998. 	fmla	v20.2s, v9.2s, v0.s[0]

  999. .endm

  1000. 

  1001. .macro SAVE1x4

  1002. 	add	pCRow1, pCRow0, LDC

  1003. 	ld1	{v8.s}[0], [pCRow0]

  1004. 	ld1	{v8.s}[1], [pCRow1]

  1005. 	fmla	v8.2s, v16.2s, alphaV0

  1006. 	st1	{v8.s}[0], [pCRow0]

  1007. 	st1	{v8.s}[1], [pCRow1]

  1008. 

  1009. 	add	pCRow2, pCRow1, LDC

  1010. 	add	pCRow1, pCRow2, LDC

  1011. 	ld1	{v12.s}[0], [pCRow2]

  1012. 	ld1	{v12.s}[1], [pCRow1]

  1013. 	fmla	v12.2s, v20.2s, alphaV1

  1014. 	st1	{v12.s}[0], [pCRow2]

  1015. 	st1	{v12.s}[1], [pCRow1]

  1016. 

  1017. 	add	pCRow0, pCRow0, #4

  1018. .endm

  1019. 

  1020. /******************************************************************************/

  1021. 

  1022. .macro INIT8x2

  1023. 	fmov	s16, wzr

  1024. 	fmov	s17, s16

  1025. 	fmov	s20, s17

  1026. 	fmov	s21, s16

  1027. .endm

  1028. 

  1029. .macro KERNEL8x2_SUB

  1030. 	ld1	{v8.2s}, [pB]

  1031. 	add	pB, pB, #8

  1032. 	ld1	{v0.4s}, [pA]

  1033. 	add	pA, pA, #16

  1034. 	ld1	{v1.4s}, [pA]

  1035. 	add	pA, pA, #16

  1036. 

  1037. 	fmla	v16.4s, v0.4s, v8.s[0]

  1038. 	fmla	v17.4s, v1.4s, v8.s[0]

  1039. 

  1040. 	fmla	v20.4s, v0.4s, v8.s[1]

  1041. 	fmla	v21.4s, v1.4s, v8.s[1]

  1042. .endm

  1043. 

  1044. .macro SAVE8x2

  1045. 	add	pCRow1, pCRow0, LDC

  1046. 

  1047. 	ld1	{v0.4s, v1.4s}, [pCRow0]

  1048. 	fmla	v0.4s, v16.4s, alphaV0

  1049. 	fmla	v1.4s, v17.4s, alphaV1

  1050. 	st1 	{v0.4s, v1.4s}, [pCRow0]

  1051. 

  1052. 	add	pCRow2, pCRow1, LDC

  1053. 

  1054. 	ld1	{v4.4s, v5.4s}, [pCRow1]

  1055. 	fmla	v4.4s, v20.4s, alphaV0

  1056. 	fmla	v5.4s, v21.4s, alphaV1

  1057. 	st1 	{v4.4s, v5.4s}, [pCRow1]

  1058. 

  1059. 	add	pCRow0, pCRow0, #32

  1060. .endm

  1061. 

  1062. /******************************************************************************/

  1063. 

  1064. .macro INIT4x2

  1065. 	fmov	s16, wzr

  1066. 	fmov	s17, s16

  1067. 	fmov	s20, s17

  1068. 	fmov	s21, s16

  1069. .endm

  1070. 

  1071. .macro KERNEL4x2_SUB

  1072. 	ld1	{v8.2s}, [pB]

  1073. 	add	pB, pB, #8

  1074. 	ld1	{v0.2s, v1.2s}, [pA]

  1075. 	add	pA, pA, #16

  1076. 

  1077. 	fmla	v16.2s, v0.2s, v8.s[0]

  1078. 	fmla	v17.2s, v1.2s, v8.s[0]

  1079. 	fmla	v20.2s, v0.2s, v8.s[1]

  1080. 	fmla	v21.2s, v1.2s, v8.s[1]

  1081. .endm

  1082. 

  1083. .macro SAVE4x2

  1084. 	ld1	{v8.2s, v9.2s}, [pCRow0]

  1085. 	fmla	v8.2s, v16.2s, alphaV0

  1086. 	fmla	v9.2s, v17.2s, alphaV1

  1087. 	st1	{v8.2s, v9.2s}, [pCRow0]

  1088. 

  1089. 	add	pCRow1, pCRow0, LDC

  1090. 	ld1	{v12.2s, v13.2s}, [pCRow1]

  1091. 	fmla	v12.2s, v20.2s, alphaV2

  1092. 	fmla	v13.2s, v21.2s, alphaV3

  1093. 	st1	{v12.2s, v13.2s}, [pCRow1]

  1094. 

  1095. 	add	pCRow0, pCRow0, #16

  1096. .endm

  1097. 

  1098. /******************************************************************************/

  1099. 

  1100. .macro INIT2x2

  1101. 	fmov		s16, wzr

  1102. 	fmov		s20, s16

  1103. .endm

  1104. 

  1105. .macro KERNEL2x2_SUB

  1106. 	ld1	{v8.2s}, [pB]

  1107. 	add	pB, pB, #8

  1108. 

  1109. 	ld1	{v0.2s}, [pA]

  1110. 	add	pA, pA, #8

  1111. 

  1112. 	fmla	v16.2s, v0.2s, v8.s[0]

  1113. 	fmla	v20.2s, v0.2s, v8.s[1]

  1114. .endm

  1115. 

  1116. .macro SAVE2x2

  1117. 	ld1	{v8.2s}, [pCRow0]

  1118. 	fmla	v8.2s, v16.2s, alphaV0

  1119. 	st1	{v8.2s}, [pCRow0]

  1120. 

  1121. 	add	pCRow1 , pCRow0, LDC

  1122. 	ld1	{v12.2s}, [pCRow1]

  1123. 	fmla	v12.2s, v20.2s, alphaV1

  1124. 	st1	{v12.2s}, [pCRow1]

  1125. 

  1126. 	add	pCRow0, pCRow0, #8

  1127. .endm

  1128. 

  1129. /******************************************************************************/

  1130. 

  1131. .macro INIT1x2

  1132. 	fmov		s16, wzr

  1133. .endm

  1134. 

  1135. .macro KERNEL1x2_SUB

  1136. 	ld1	{v8.2s} , [pB]

  1137. 	add	pB , pB, #8

  1138. 

  1139. 	ldr	s0 , [pA]

  1140. 	add	pA, pA, #4

  1141. 

  1142. 	fmla	v16.2s, v8.2s, v0.s[0]

  1143. .endm

  1144. 

  1145. .macro SAVE1x2

  1146. 	add	pCRow1 , pCRow0, LDC

  1147. 	ld1	{v8.s}[0], [pCRow0]

  1148. 	ld1	{v8.s}[1], [pCRow1]

  1149. 	fmla	v8.2s, v16.2s, alphaV0

  1150. 	st1	{v8.s}[0], [pCRow0]

  1151. 	st1	{v8.s}[1], [pCRow1]

  1152. 

  1153. 	add	pCRow0, pCRow0, #4

  1154. .endm

  1155. 

  1156. /******************************************************************************/

  1157. 

  1158. .macro INIT8x1

  1159. 	fmov	s16, wzr

  1160. 	fmov	s17, wzr

  1161. .endm

  1162. 

  1163. .macro KERNEL8x1_SUB

  1164. 	ldr	s8, [pB]

  1165. 	add	pB , pB, #4

  1166. 

  1167. 	ld1	{v0.4s}, [pA]

  1168. 	add	pA, pA, #16

  1169. 	ld1	{v1.4s}, [pA]

  1170. 	add	pA, pA, #16

  1171. 

  1172. 	fmla	v16.4s, v0.4s, v8.s[0]

  1173. 	fmla	v17.4s, v1.4s, v8.s[0]

  1174. .endm

  1175. 

  1176. .macro SAVE8x1

  1177. 	ld1	{v0.4s, v1.4s}, [pCRow0]

  1178. 	fmla	v0.4s, v16.4s, alphaV0

  1179. 	fmla	v1.4s, v17.4s, alphaV1

  1180. 	st1 	{v0.4s, v1.4s}, [pCRow0]

  1181. 

  1182. 	add	pCRow0, pCRow0, #32

  1183. .endm

  1184. 

  1185. /******************************************************************************/

  1186. 

  1187. .macro INIT4x1

  1188. 	fmov	s16, wzr

  1189. 	fmov	s17, s16

  1190. .endm

  1191. 

  1192. .macro KERNEL4x1_SUB

  1193. 	ldr	s8, [pB]

  1194. 	add	pB , pB, #4

  1195. 

  1196. 	ld1	{v0.2s, v1.2s}, [pA]

  1197. 	add	pA , pA, #16

  1198. 

  1199. 	fmla	v16.2s, v0.2s, v8.s[0]

  1200. 	fmla	v17.2s, v1.2s, v8.s[0]

  1201. .endm

  1202. 

  1203. .macro SAVE4x1

  1204. 	ld1	{v8.2s, v9.2s}, [pCRow0]

  1205. 	fmla	v8.2s, v16.2s, alphaV0

  1206. 	fmla	v9.2s, v17.2s, alphaV1

  1207. 	st1	{v8.2s, v9.2s}, [pCRow0]

  1208. 

  1209. 	add	pCRow0, pCRow0, #16

  1210. .endm

  1211. 

  1212. /******************************************************************************/

  1213. 

  1214. .macro INIT2x1

  1215. 	fmov		s16, wzr

  1216. .endm

  1217. 

  1218. .macro KERNEL2x1_SUB

  1219. 	ldr	s8, [pB]

  1220. 	add	pB , pB, #4

  1221. 

  1222. 	ld1	{v0.2s}, [pA]

  1223. 	add	pA , pA, #8

  1224. 

  1225. 	fmla	v16.2s, v0.2s, v8.s[0]

  1226. .endm

  1227. 

  1228. .macro SAVE2x1

  1229. 	ld1	{v8.2s}, [pCRow0]

  1230. 	fmla	v8.2s, v16.2s, alphaV0

  1231. 	st1	{v8.2s}, [pCRow0]

  1232. 

  1233. 	add	pCRow0, pCRow0, #8

  1234. .endm

  1235. 

  1236. /******************************************************************************/

  1237. 

  1238. .macro INIT1x1

  1239. 	fmov	s16, wzr

  1240. .endm

  1241. 

  1242. .macro KERNEL1x1_SUB

  1243. 	ldr	s8, [pB]

  1244. 	add	pB , pB, #4

  1245. 

  1246. 	ldr	s0, [pA]

  1247. 	add	pA , pA, #4

  1248. 

  1249. 	fmadd 	s16, s0, s8, s16  

  1250. .endm

  1251. 

  1252. .macro SAVE1x1

  1253. 	ldr 	s8, [pCRow0]

  1254. 	fmla	s8, s16, alphaV0

  1255. 	str 	s8, [pCRow0]

  1256. 

  1257. 	add	pCRow0, pCRow0, #4

  1258. .endm

  1259. 

  1260. /*******************************************************************************

  1261. * End of macro definitions

  1262. *******************************************************************************/

  1263. 

  1264. 	PROLOGUE

  1265. 

  1266. .Lsgemm_kernel_begin:

  1267. 

  1268. 	.align 5

  1269. 	add	sp, sp, #-(11 * 16)

  1270. 	stp	d8, d9, [sp, #(0 * 16)]

  1271. 	stp	d10, d11, [sp, #(1 * 16)]

  1272. 	stp	d12, d13, [sp, #(2 * 16)]

  1273. 	stp	d14, d15, [sp, #(3 * 16)]

  1274. 	stp	d16, d17, [sp, #(4 * 16)]

  1275. 	stp	x18, x19, [sp, #(5 * 16)]

  1276. 	stp	x20, x21, [sp, #(6 * 16)]

  1277. 	stp	x22, x23, [sp, #(7 * 16)]

  1278. 	stp	x24, x25, [sp, #(8 * 16)]

  1279. 	stp	x26, x27, [sp, #(9 * 16)]

  1280. 	str	x28, [sp, #(10 * 16)]

  1281. 

  1282. 	fmov	alpha0, s0

  1283. 	fmov	alpha1, s0

  1284. 	fmov	alpha2, s0

  1285. 	fmov	alpha3, s0

  1286. 

  1287. 	lsl	LDC, LDC, #2			// ldc = ldc * 4

  1288. 

  1289. 	mov	pB, origPB

  1290. 

  1291. 	mov	counterJ, origN

  1292. 	asr 	counterJ, counterJ, #3		// J = J / 8

  1293. 	cmp 	counterJ, #0

  1294. 	ble	.Lsgemm_kernel_L4_BEGIN

  1295. 

  1296. /******************************************************************************/

  1297. /******************************************************************************/

  1298. 

  1299. .Lsgemm_kernel_L8_BEGIN:

  1300. 	mov	pCRow0, pC			// pCRow0 = C

  1301. 	add	pC, pC, LDC, lsl #3

  1302. 

  1303. 	mov	pA, origPA			// pA = start of A array

  1304. 

  1305. /******************************************************************************/

  1306. 

  1307. .Lsgemm_kernel_L8_M8_BEGIN:

  1308. 

  1309. 	mov	counterI, origM

  1310. 	asr 	counterI, counterI, #3		// counterI = counterI / 8

  1311. 	cmp 	counterI, #0

  1312. 	ble	.Lsgemm_kernel_L8_M4_BEGIN

  1313. 

  1314. .Lsgemm_kernel_L8_M8_20:

  1315. 

  1316. 	mov	pB, origPB

  1317. 

  1318. 	asr 	counterL , origK, #1		// L = K / 2

  1319. 	cmp	counterL , #2			// is there at least 4 to do?

  1320. 	blt	.Lsgemm_kernel_L8_M8_32

  1321. 

  1322. 	KERNEL8x8_I				// do one in the K

  1323. 	KERNEL8x8_M2				// do another in the K

  1324. 

  1325. 	subs	counterL, counterL, #2

  1326. 	ble	.Lsgemm_kernel_L8_M8_22a

  1327. 	.align 5

  1328. 

  1329. .Lsgemm_kernel_L8_M8_22:

  1330. 

  1331. 	KERNEL8x8_M1

  1332. 	KERNEL8x8_M2

  1333. 

  1334. 	subs	counterL, counterL, #1

  1335. 	bgt	.Lsgemm_kernel_L8_M8_22

  1336. 

  1337. .Lsgemm_kernel_L8_M8_22a:

  1338. 

  1339. 	KERNEL8x8_M1

  1340. 	KERNEL8x8_E

  1341. 

  1342. 	b	 .Lsgemm_kernel_L8_M8_44

  1343. 

  1344. .Lsgemm_kernel_L8_M8_32:

  1345. 

  1346. 	tst	counterL, #1

  1347. 	ble	.Lsgemm_kernel_L8_M8_40

  1348. 

  1349. 	KERNEL8x8_I

  1350. 	KERNEL8x8_E

  1351. 

  1352. 	b	.Lsgemm_kernel_L8_M8_44

  1353. 

  1354. .Lsgemm_kernel_L8_M8_40:

  1355. 

  1356. 	INIT8x8

  1357. 

  1358. .Lsgemm_kernel_L8_M8_44:

  1359. 

  1360. 	ands	counterL , origK, #1

  1361. 	ble	.Lsgemm_kernel_L8_M8_100

  1362. 

  1363. .Lsgemm_kernel_L8_M8_46:

  1364. 

  1365. 	KERNEL8x8_SUB

  1366. 

  1367. .Lsgemm_kernel_L8_M8_100:

  1368. 

  1369. 	SAVE8x8

  1370. 

  1371. .Lsgemm_kernel_L8_M8_END:

  1372. 	subs	counterI, counterI, #1

  1373. 	bne	.Lsgemm_kernel_L8_M8_20

  1374. 

  1375. /******************************************************************************/

  1376. 

  1377. .Lsgemm_kernel_L8_M4_BEGIN:

  1378. 

  1379. 	mov	counterI, origM

  1380. 	tst	counterI , #7

  1381. 	ble	.Lsgemm_kernel_L8_END

  1382. 

  1383. 	tst	counterI, #4

  1384. 	ble	.Lsgemm_kernel_L8_M2_BEGIN

  1385. 

  1386. .Lsgemm_kernel_L8_M4_20:

  1387. 

  1388. 	mov	pB, origPB

  1389. 

  1390. 	asr 	counterL , origK, #1		// L = K / 2

  1391. 	cmp	counterL , #2			// is there at least 4 to do?

  1392. 	blt	.Lsgemm_kernel_L8_M4_32

  1393. 

  1394. 	KERNEL4x8_I				// do one in the K

  1395. 	KERNEL4x8_M2				// do another in the K

  1396. 

  1397. 	subs	counterL, counterL, #2

  1398. 	ble	.Lsgemm_kernel_L8_M4_22a

  1399. 	.align 5

  1400. 

  1401. .Lsgemm_kernel_L8_M4_22:

  1402. 

  1403. 	KERNEL4x8_M1

  1404. 	KERNEL4x8_M2

  1405. 

  1406. 	subs	counterL, counterL, #1

  1407. 	bgt	.Lsgemm_kernel_L8_M4_22

  1408. 

  1409. .Lsgemm_kernel_L8_M4_22a:

  1410. 

  1411. 	KERNEL4x8_M1

  1412. 	KERNEL4x8_E

  1413. 

  1414. 	b	 .Lsgemm_kernel_L8_M4_44

  1415. 

  1416. .Lsgemm_kernel_L8_M4_32:

  1417. 

  1418. 	tst	counterL, #1

  1419. 	ble	.Lsgemm_kernel_L8_M4_40

  1420. 

  1421. 	KERNEL4x8_I

  1422. 	KERNEL4x8_E

  1423. 

  1424. 	b	.Lsgemm_kernel_L8_M4_44

  1425. 

  1426. .Lsgemm_kernel_L8_M4_40:

  1427. 

  1428. 	INIT4x8

  1429. 

  1430. .Lsgemm_kernel_L8_M4_44:

  1431. 

  1432. 	ands	counterL , origK, #1

  1433. 	ble	.Lsgemm_kernel_L8_M4_100

  1434. 

  1435. .Lsgemm_kernel_L8_M4_46:

  1436. 

  1437. 	KERNEL4x8_SUB

  1438. 

  1439. .Lsgemm_kernel_L8_M4_100:

  1440. 

  1441. 	SAVE4x8

  1442. 

  1443. .Lsgemm_kernel_L8_M4_END:

  1444. 

  1445. /******************************************************************************/

  1446. 

  1447. .Lsgemm_kernel_L8_M2_BEGIN:

  1448. 

  1449. 	mov	counterI, origM

  1450. 	tst	counterI , #3

  1451. 	ble	.Lsgemm_kernel_L8_END

  1452. 

  1453. 	tst	counterI, #2			// counterI = counterI / 2

  1454. 	ble	.Lsgemm_kernel_L8_M1_BEGIN

  1455. 

  1456. .Lsgemm_kernel_L8_M2_20:

  1457. 

  1458. 	INIT2x8

  1459. 

  1460. 	mov	pB, origPB

  1461. 

  1462. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  1463. 	cmp	counterL , #0

  1464. 	ble	.Lsgemm_kernel_L8_M2_40

  1465. 

  1466. .Lsgemm_kernel_L8_M2_22:

  1467. 

  1468. 	KERNEL2x8_SUB

  1469. 	KERNEL2x8_SUB

  1470. 	KERNEL2x8_SUB

  1471. 	KERNEL2x8_SUB

  1472. 

  1473. 	KERNEL2x8_SUB

  1474. 	KERNEL2x8_SUB

  1475. 	KERNEL2x8_SUB

  1476. 	KERNEL2x8_SUB

  1477. 

  1478. 	subs	counterL, counterL, #1

  1479. 	bgt	.Lsgemm_kernel_L8_M2_22

  1480. 

  1481. 

  1482. .Lsgemm_kernel_L8_M2_40:

  1483. 

  1484. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1485. 	ble	.Lsgemm_kernel_L8_M2_100

  1486. 

  1487. .Lsgemm_kernel_L8_M2_42:

  1488. 

  1489. 	KERNEL2x8_SUB

  1490. 

  1491. 	subs	counterL, counterL, #1

  1492. 	bgt	.Lsgemm_kernel_L8_M2_42

  1493. 

  1494. .Lsgemm_kernel_L8_M2_100:

  1495. 

  1496. 	SAVE2x8

  1497. 

  1498. .Lsgemm_kernel_L8_M2_END:

  1499. 

  1500. /******************************************************************************/

  1501. 

  1502. .Lsgemm_kernel_L8_M1_BEGIN:

  1503. 

  1504. 	tst	counterI, #1			// counterI = counterI % 2

  1505. 	ble	.Lsgemm_kernel_L8_END

  1506. 

  1507. .Lsgemm_kernel_L8_M1_20:

  1508. 

  1509. 	INIT1x8

  1510. 

  1511. 	mov	pB, origPB

  1512. 

  1513. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  1514. 	cmp	counterL , #0

  1515. 	ble	.Lsgemm_kernel_L8_M1_40

  1516. 

  1517. .Lsgemm_kernel_L8_M1_22:

  1518. 	KERNEL1x8_SUB

  1519. 	KERNEL1x8_SUB

  1520. 	KERNEL1x8_SUB

  1521. 	KERNEL1x8_SUB

  1522. 

  1523. 	KERNEL1x8_SUB

  1524. 	KERNEL1x8_SUB

  1525. 	KERNEL1x8_SUB

  1526. 	KERNEL1x8_SUB

  1527. 

  1528. 	subs	counterL, counterL, #1

  1529. 	bgt	.Lsgemm_kernel_L8_M1_22

  1530. 

  1531. 

  1532. .Lsgemm_kernel_L8_M1_40:

  1533. 

  1534. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1535. 	ble	.Lsgemm_kernel_L8_M1_100

  1536. 

  1537. .Lsgemm_kernel_L8_M1_42:

  1538. 

  1539. 	KERNEL1x8_SUB

  1540. 

  1541. 	subs	counterL, counterL, #1

  1542. 	bgt	.Lsgemm_kernel_L8_M1_42

  1543. 

  1544. .Lsgemm_kernel_L8_M1_100:

  1545. 

  1546. 	SAVE1x8

  1547. 

  1548. .Lsgemm_kernel_L8_END:

  1549. 	lsl	temp, origK, #5			// B = B + K * 4 * 8

  1550. 	add	origPB, origPB, temp

  1551. 

  1552. 	subs	counterJ, counterJ , #1		// j--

  1553. 	bgt	.Lsgemm_kernel_L8_BEGIN

  1554. 

  1555. /******************************************************************************/

  1556. /******************************************************************************/

  1557. 

  1558. .Lsgemm_kernel_L4_BEGIN:

  1559. 

  1560. 	mov	counterJ , origN

  1561. 	tst	counterJ , #7

  1562. 	ble	.Lsgemm_kernel_L999

  1563. 

  1564. 	tst	counterJ , #4

  1565. 	ble	.Lsgemm_kernel_L2_BEGIN

  1566. 

  1567. 	mov	pCRow0, pC			// pCRow0 = pC

  1568. 

  1569. 	add	pC,pC,LDC, lsl #2

  1570. 

  1571. 	mov	pA, origPA			// pA = A

  1572. 

  1573. /******************************************************************************/

  1574. 

  1575. .Lsgemm_kernel_L4_M8_BEGIN:

  1576. 

  1577. 	mov	counterI, origM

  1578. 	asr 	counterI, counterI, #3		// counterI = counterI / 8

  1579. 	cmp 	counterI, #0

  1580. 	ble	.Lsgemm_kernel_L4_M4_BEGIN

  1581. 

  1582. .Lsgemm_kernel_L4_M8_20:

  1583. 

  1584. 	mov	pB, origPB

  1585. 

  1586. 	asr 	counterL , origK, #1		// L = K / 2

  1587. 	cmp	counterL , #2			// is there at least 4 to do?

  1588. 	blt	.Lsgemm_kernel_L4_M8_32

  1589. 

  1590. 	KERNEL8x4_I				// do one in the K

  1591. 	KERNEL8x4_M2				// do another in the K

  1592. 

  1593. 	subs	counterL, counterL, #2

  1594. 	ble	.Lsgemm_kernel_L4_M8_22a

  1595. 	.align 5

  1596. 

  1597. .Lsgemm_kernel_L4_M8_22:

  1598. 

  1599. 	KERNEL8x4_M1

  1600. 	KERNEL8x4_M2

  1601. 

  1602. 	subs	counterL, counterL, #1

  1603. 	bgt	.Lsgemm_kernel_L4_M8_22

  1604. 

  1605. .Lsgemm_kernel_L4_M8_22a:

  1606. 

  1607. 	KERNEL8x4_M1

  1608. 	KERNEL8x4_E

  1609. 

  1610. 	b	 .Lsgemm_kernel_L4_M8_44

  1611. 

  1612. .Lsgemm_kernel_L4_M8_32:

  1613. 

  1614. 	tst	counterL, #1

  1615. 	ble	.Lsgemm_kernel_L4_M8_40

  1616. 

  1617. 	KERNEL8x4_I

  1618. 	KERNEL8x4_E

  1619. 

  1620. 	b	.Lsgemm_kernel_L4_M8_44

  1621. 

  1622. .Lsgemm_kernel_L4_M8_40:

  1623. 

  1624. 	INIT8x4

  1625. 

  1626. .Lsgemm_kernel_L4_M8_44:

  1627. 

  1628. 	ands	counterL , origK, #1

  1629. 	ble	.Lsgemm_kernel_L4_M8_100

  1630. 

  1631. .Lsgemm_kernel_L4_M8_46:

  1632. 

  1633. 	KERNEL8x4_SUB

  1634. 

  1635. .Lsgemm_kernel_L4_M8_100:

  1636. 

  1637. 	SAVE8x4

  1638. 

  1639. .Lsgemm_kernel_L4_M8_END:

  1640. 	subs	counterI, counterI, #1

  1641. 	bne	.Lsgemm_kernel_L4_M8_20

  1642. 

  1643. /******************************************************************************/

  1644. 

  1645. .Lsgemm_kernel_L4_M4_BEGIN:

  1646. 

  1647. 	mov	counterI, origM

  1648. 	tst	counterI , #7

  1649. 	ble	.Lsgemm_kernel_L4_END

  1650. 

  1651. 	tst	counterI, #4

  1652. 	ble	.Lsgemm_kernel_L4_M2_BEGIN

  1653. 

  1654. .Lsgemm_kernel_L4_M4_20:

  1655. 

  1656. 	mov	pB, origPB

  1657. 

  1658. 	asr 	counterL , origK, #1		// L = K / 2

  1659. 	cmp	counterL , #2			// is there at least 4 to do?

  1660. 	blt	.Lsgemm_kernel_L4_M4_32

  1661. 

  1662. 	KERNEL4x4_I				// do one in the K

  1663. 	KERNEL4x4_M2				// do another in the K

  1664. 

  1665. 	subs	counterL, counterL, #2

  1666. 	ble	.Lsgemm_kernel_L4_M4_22a

  1667. 	.align 5

  1668. 

  1669. .Lsgemm_kernel_L4_M4_22:

  1670. 

  1671. 	KERNEL4x4_M1

  1672. 	KERNEL4x4_M2

  1673. 

  1674. 	subs	counterL, counterL, #1

  1675. 	bgt	.Lsgemm_kernel_L4_M4_22

  1676. 

  1677. .Lsgemm_kernel_L4_M4_22a:

  1678. 

  1679. 	KERNEL4x4_M1

  1680. 	KERNEL4x4_E

  1681. 

  1682. 	b	 .Lsgemm_kernel_L4_M4_44

  1683. 

  1684. .Lsgemm_kernel_L4_M4_32:

  1685. 

  1686. 	tst	counterL, #1

  1687. 	ble	.Lsgemm_kernel_L4_M4_40

  1688. 

  1689. 	KERNEL4x4_I

  1690. 	KERNEL4x4_E

  1691. 

  1692. 	b	.Lsgemm_kernel_L4_M4_44

  1693. 

  1694. .Lsgemm_kernel_L4_M4_40:

  1695. 

  1696. 	INIT4x4

  1697. 

  1698. .Lsgemm_kernel_L4_M4_44:

  1699. 

  1700. 	ands	counterL , origK, #1

  1701. 	ble	.Lsgemm_kernel_L4_M4_100

  1702. 

  1703. .Lsgemm_kernel_L4_M4_46:

  1704. 

  1705. 	KERNEL4x4_SUB

  1706. 

  1707. .Lsgemm_kernel_L4_M4_100:

  1708. 

  1709. 	SAVE4x4

  1710. 

  1711. .Lsgemm_kernel_L4_M4_END:

  1712. 

  1713. /******************************************************************************/

  1714. 

  1715. .Lsgemm_kernel_L4_M2_BEGIN:

  1716. 

  1717. 	mov	counterI, origM

  1718. 	tst	counterI , #3

  1719. 	ble	.Lsgemm_kernel_L4_END

  1720. 

  1721. 	tst	counterI, #2			// counterI = counterI / 2

  1722. 	ble	.Lsgemm_kernel_L4_M1_BEGIN

  1723. 

  1724. .Lsgemm_kernel_L4_M2_20:

  1725. 

  1726. 	INIT2x4

  1727. 

  1728. 	mov	pB, origPB

  1729. 

  1730. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  1731. 	cmp	counterL , #0

  1732. 	ble	.Lsgemm_kernel_L4_M2_40

  1733. 

  1734. .Lsgemm_kernel_L4_M2_22:

  1735. 

  1736. 	KERNEL2x4_SUB

  1737. 	KERNEL2x4_SUB

  1738. 	KERNEL2x4_SUB

  1739. 	KERNEL2x4_SUB

  1740. 

  1741. 	KERNEL2x4_SUB

  1742. 	KERNEL2x4_SUB

  1743. 	KERNEL2x4_SUB

  1744. 	KERNEL2x4_SUB

  1745. 

  1746. 	subs	counterL, counterL, #1

  1747. 	bgt	.Lsgemm_kernel_L4_M2_22

  1748. 

  1749. 

  1750. .Lsgemm_kernel_L4_M2_40:

  1751. 

  1752. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1753. 	ble	.Lsgemm_kernel_L4_M2_100

  1754. 

  1755. .Lsgemm_kernel_L4_M2_42:

  1756. 

  1757. 	KERNEL2x4_SUB

  1758. 

  1759. 	subs	counterL, counterL, #1

  1760. 	bgt	.Lsgemm_kernel_L4_M2_42

  1761. 

  1762. .Lsgemm_kernel_L4_M2_100:

  1763. 

  1764. 	SAVE2x4

  1765. 

  1766. .Lsgemm_kernel_L4_M2_END:

  1767. 

  1768. /******************************************************************************/

  1769. 

  1770. .Lsgemm_kernel_L4_M1_BEGIN:

  1771. 

  1772. 	tst	counterI, #1			// counterI = counterI % 2

  1773. 	ble	.Lsgemm_kernel_L4_END

  1774. 

  1775. .Lsgemm_kernel_L4_M1_20:

  1776. 

  1777. 	INIT1x4

  1778. 

  1779. 	mov	pB, origPB

  1780. 

  1781. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  1782. 	cmp	counterL , #0

  1783. 	ble	.Lsgemm_kernel_L4_M1_40

  1784. 

  1785. .Lsgemm_kernel_L4_M1_22:

  1786. 	KERNEL1x4_SUB

  1787. 	KERNEL1x4_SUB

  1788. 	KERNEL1x4_SUB

  1789. 	KERNEL1x4_SUB

  1790. 

  1791. 	KERNEL1x4_SUB

  1792. 	KERNEL1x4_SUB

  1793. 	KERNEL1x4_SUB

  1794. 	KERNEL1x4_SUB

  1795. 

  1796. 	subs	counterL, counterL, #1

  1797. 	bgt	.Lsgemm_kernel_L4_M1_22

  1798. 

  1799. 

  1800. .Lsgemm_kernel_L4_M1_40:

  1801. 

  1802. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1803. 	ble	.Lsgemm_kernel_L4_M1_100

  1804. 

  1805. .Lsgemm_kernel_L4_M1_42:

  1806. 

  1807. 	KERNEL1x4_SUB

  1808. 

  1809. 	subs	counterL, counterL, #1

  1810. 	bgt	.Lsgemm_kernel_L4_M1_42

  1811. 

  1812. .Lsgemm_kernel_L4_M1_100:

  1813. 

  1814. 	SAVE1x4

  1815. 

  1816. .Lsgemm_kernel_L4_END:

  1817. 	add	origPB, origPB, origK, lsl #4	// B = B + K * 4 * 4

  1818. 

  1819. /******************************************************************************/

  1820. /******************************************************************************/

  1821. 

  1822. .Lsgemm_kernel_L2_BEGIN:   // less than 2 left in N direction

  1823. 

  1824. 	mov	counterJ , origN

  1825. 	tst	counterJ , #3

  1826. 	ble	.Lsgemm_kernel_L999

  1827. 

  1828. 	tst	counterJ , #2

  1829. 	ble	.Lsgemm_kernel_L1_BEGIN

  1830. 

  1831. 	mov	pCRow0, pC			// pCRow0 = pC

  1832. 

  1833. 	add	pC,pC,LDC, lsl #1

  1834. 

  1835. 	mov	pA, origPA			// pA = A

  1836. 

  1837. /******************************************************************************/

  1838. 

  1839. .Lsgemm_kernel_L2_M8_BEGIN:

  1840. 

  1841. 	mov	counterI, origM

  1842. 	asr 	counterI, counterI, #3		// counterI = counterI / 8

  1843. 	cmp	counterI,#0

  1844. 	ble	.Lsgemm_kernel_L2_M4_BEGIN

  1845. 

  1846. .Lsgemm_kernel_L2_M8_20:

  1847. 

  1848. 	INIT8x2

  1849. 

  1850. 	mov	pB, origPB

  1851. 

  1852. 	asr	counterL , origK, #3		// counterL = counterL / 8

  1853. 	cmp	counterL,#0

  1854. 	ble	.Lsgemm_kernel_L2_M8_40

  1855. 	.align 5

  1856. 

  1857. .Lsgemm_kernel_L2_M8_22:

  1858. 	KERNEL8x2_SUB

  1859. 	KERNEL8x2_SUB

  1860. 	KERNEL8x2_SUB

  1861. 	KERNEL8x2_SUB

  1862. 

  1863. 	KERNEL8x2_SUB

  1864. 	KERNEL8x2_SUB

  1865. 	KERNEL8x2_SUB

  1866. 	KERNEL8x2_SUB

  1867. 

  1868. 	subs	counterL, counterL, #1

  1869. 	bgt	.Lsgemm_kernel_L2_M8_22

  1870. 

  1871. 

  1872. .Lsgemm_kernel_L2_M8_40:

  1873. 

  1874. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1875. 	ble	.Lsgemm_kernel_L2_M8_100

  1876. 

  1877. .Lsgemm_kernel_L2_M8_42:

  1878. 

  1879. 	KERNEL8x2_SUB

  1880. 

  1881. 	subs	counterL, counterL, #1

  1882. 	bgt	.Lsgemm_kernel_L2_M8_42

  1883. 

  1884. .Lsgemm_kernel_L2_M8_100:

  1885. 

  1886. 	SAVE8x2

  1887. 

  1888. .Lsgemm_kernel_L2_M8_END:

  1889. 

  1890. 	subs	counterI, counterI, #1

  1891. 	bgt	.Lsgemm_kernel_L2_M8_20

  1892. 

  1893. /******************************************************************************/

  1894. 

  1895. .Lsgemm_kernel_L2_M4_BEGIN:

  1896. 

  1897. 	mov	counterI, origM

  1898. 	tst	counterI , #7

  1899. 	ble	.Lsgemm_kernel_L2_END

  1900. 

  1901. 	tst	counterI, #4

  1902. 	ble	.Lsgemm_kernel_L2_M2_BEGIN

  1903. 

  1904. .Lsgemm_kernel_L2_M4_20:

  1905. 

  1906. 	INIT4x2

  1907. 

  1908. 	mov	pB, origPB

  1909. 

  1910. 	asr	counterL , origK, #3		// counterL = counterL / 8

  1911. 	cmp	counterL,#0

  1912. 	ble	.Lsgemm_kernel_L2_M4_40

  1913. 	.align 5

  1914. 

  1915. .Lsgemm_kernel_L2_M4_22:

  1916. 	KERNEL4x2_SUB

  1917. 	KERNEL4x2_SUB

  1918. 	KERNEL4x2_SUB

  1919. 	KERNEL4x2_SUB

  1920. 

  1921. 	KERNEL4x2_SUB

  1922. 	KERNEL4x2_SUB

  1923. 	KERNEL4x2_SUB

  1924. 	KERNEL4x2_SUB

  1925. 

  1926. 	subs	counterL, counterL, #1

  1927. 	bgt	.Lsgemm_kernel_L2_M4_22

  1928. 

  1929. 

  1930. .Lsgemm_kernel_L2_M4_40:

  1931. 

  1932. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1933. 	ble	.Lsgemm_kernel_L2_M4_100

  1934. 

  1935. .Lsgemm_kernel_L2_M4_42:

  1936. 

  1937. 	KERNEL4x2_SUB

  1938. 

  1939. 	subs	counterL, counterL, #1

  1940. 	bgt	.Lsgemm_kernel_L2_M4_42

  1941. 

  1942. .Lsgemm_kernel_L2_M4_100:

  1943. 

  1944. 	SAVE4x2

  1945. 

  1946. .Lsgemm_kernel_L2_M4_END:

  1947. 

  1948. /******************************************************************************/

  1949. 

  1950. .Lsgemm_kernel_L2_M2_BEGIN:

  1951. 

  1952. 	mov	counterI, origM

  1953. 	tst	counterI , #3

  1954. 	ble	.Lsgemm_kernel_L2_END

  1955. 

  1956. 	tst	counterI, #2			// counterI = counterI / 2

  1957. 	ble	.Lsgemm_kernel_L2_M1_BEGIN

  1958. 

  1959. .Lsgemm_kernel_L2_M2_20:

  1960. 

  1961. 	INIT2x2

  1962. 

  1963. 	mov	pB, origPB

  1964. 

  1965. 	asr	counterL , origK, #3		// counterL = counterL / 8

  1966.         cmp	counterL,#0

  1967. 	ble	.Lsgemm_kernel_L2_M2_40

  1968. 

  1969. .Lsgemm_kernel_L2_M2_22:

  1970. 

  1971. 	KERNEL2x2_SUB

  1972. 	KERNEL2x2_SUB

  1973. 	KERNEL2x2_SUB

  1974. 	KERNEL2x2_SUB

  1975. 

  1976. 	KERNEL2x2_SUB

  1977. 	KERNEL2x2_SUB

  1978. 	KERNEL2x2_SUB

  1979. 	KERNEL2x2_SUB

  1980. 

  1981. 	subs	counterL, counterL, #1

  1982. 	bgt	.Lsgemm_kernel_L2_M2_22

  1983. 

  1984. 

  1985. .Lsgemm_kernel_L2_M2_40:

  1986. 

  1987. 	ands	counterL , origK, #7		// counterL = counterL % 8

  1988. 	ble	.Lsgemm_kernel_L2_M2_100

  1989. 

  1990. .Lsgemm_kernel_L2_M2_42:

  1991. 

  1992. 	KERNEL2x2_SUB

  1993. 

  1994. 	subs	counterL, counterL, #1

  1995. 	bgt	.Lsgemm_kernel_L2_M2_42

  1996. 

  1997. .Lsgemm_kernel_L2_M2_100:

  1998. 

  1999. 	SAVE2x2

  2000. 

  2001. .Lsgemm_kernel_L2_M2_END:

  2002. 

  2003. /******************************************************************************/

  2004. 

  2005. .Lsgemm_kernel_L2_M1_BEGIN:

  2006. 

  2007. 	tst	counterI, #1			// counterI = counterI % 2

  2008. 	ble	.Lsgemm_kernel_L2_END

  2009. 

  2010. .Lsgemm_kernel_L2_M1_20:

  2011. 

  2012. 	INIT1x2

  2013. 

  2014. 	mov	pB, origPB

  2015. 

  2016. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  2017.         cmp     counterL, #0

  2018. 	ble	.Lsgemm_kernel_L2_M1_40

  2019. 

  2020. .Lsgemm_kernel_L2_M1_22:

  2021. 	KERNEL1x2_SUB

  2022. 	KERNEL1x2_SUB

  2023. 	KERNEL1x2_SUB

  2024. 	KERNEL1x2_SUB

  2025. 

  2026. 	KERNEL1x2_SUB

  2027. 	KERNEL1x2_SUB

  2028. 	KERNEL1x2_SUB

  2029. 	KERNEL1x2_SUB

  2030. 

  2031. 	subs	counterL, counterL, #1

  2032. 	bgt	.Lsgemm_kernel_L2_M1_22

  2033. 

  2034. 

  2035. .Lsgemm_kernel_L2_M1_40:

  2036. 

  2037. 	ands	counterL , origK, #7		// counterL = counterL % 8

  2038. 	ble	.Lsgemm_kernel_L2_M1_100

  2039. 

  2040. .Lsgemm_kernel_L2_M1_42:

  2041. 

  2042. 	KERNEL1x2_SUB

  2043. 

  2044. 	subs	counterL, counterL, #1

  2045. 	bgt	.Lsgemm_kernel_L2_M1_42

  2046. 

  2047. .Lsgemm_kernel_L2_M1_100:

  2048. 

  2049. 	SAVE1x2

  2050. 

  2051. .Lsgemm_kernel_L2_END:

  2052. 

  2053. 	add	origPB, origPB, origK, lsl #3	// B = B + K * 2 * 4

  2054. 

  2055. /******************************************************************************/

  2056. /******************************************************************************/

  2057. 

  2058. .Lsgemm_kernel_L1_BEGIN:

  2059. 

  2060. 	mov	counterJ , origN

  2061. 	tst	counterJ , #1

  2062. 	ble	.Lsgemm_kernel_L999 // done

  2063. 

  2064. 

  2065. 	mov	pCRow0, pC			// pCRow0 = C

  2066. 	add	pC , pC , LDC			// Update pC to point to next

  2067. 

  2068. 	mov	pA, origPA			// pA = A

  2069. 

  2070. /******************************************************************************/

  2071. 

  2072. .Lsgemm_kernel_L1_M8_BEGIN:

  2073. 

  2074. 	mov	counterI, origM

  2075. 	asr 	counterI, counterI, #3

  2076. 	cmp	counterI, #0

  2077. 	ble	.Lsgemm_kernel_L1_M4_BEGIN

  2078. 

  2079. .Lsgemm_kernel_L1_M8_20:

  2080. 

  2081. 	INIT8x1

  2082. 

  2083. 	mov	pB, origPB

  2084. 

  2085. 	asr	counterL , origK, #3		// counterL = counterL / 8

  2086. 	cmp	counterL , #0

  2087. 	ble	.Lsgemm_kernel_L1_M8_40

  2088. 	.align 5

  2089. 

  2090. .Lsgemm_kernel_L1_M8_22:

  2091. 	KERNEL8x1_SUB

  2092. 	KERNEL8x1_SUB

  2093. 	KERNEL8x1_SUB

  2094. 	KERNEL8x1_SUB

  2095. 

  2096. 	KERNEL8x1_SUB

  2097. 	KERNEL8x1_SUB

  2098. 	KERNEL8x1_SUB

  2099. 	KERNEL8x1_SUB

  2100. 

  2101. 	subs	counterL, counterL, #1

  2102. 	bgt	.Lsgemm_kernel_L1_M8_22

  2103. 

  2104. 

  2105. .Lsgemm_kernel_L1_M8_40:

  2106. 

  2107. 	ands	counterL , origK, #7		// counterL = counterL % 8

  2108. 	ble	.Lsgemm_kernel_L1_M8_100

  2109. 

  2110. .Lsgemm_kernel_L1_M8_42:

  2111. 

  2112. 	KERNEL8x1_SUB

  2113. 

  2114. 	subs	counterL, counterL, #1

  2115. 	bgt	.Lsgemm_kernel_L1_M8_42

  2116. 

  2117. .Lsgemm_kernel_L1_M8_100:

  2118. 

  2119. 	SAVE8x1

  2120. 

  2121. .Lsgemm_kernel_L1_M8_END:

  2122. 

  2123. 	subs	counterI, counterI, #1

  2124. 	bgt	.Lsgemm_kernel_L1_M8_20

  2125. 

  2126. /******************************************************************************/

  2127. 

  2128. .Lsgemm_kernel_L1_M4_BEGIN:

  2129. 

  2130. 	mov	counterI, origM

  2131. 	tst	counterI , #7

  2132. 	ble	.Lsgemm_kernel_L1_END

  2133. 

  2134. 	tst	counterI, #4

  2135. 	ble	.Lsgemm_kernel_L1_M2_BEGIN

  2136. 

  2137. .Lsgemm_kernel_L1_M4_20:

  2138. 

  2139. 	INIT4x1

  2140. 

  2141. 	mov	pB, origPB

  2142. 

  2143. 	asr	counterL , origK, #3		// counterL = counterL / 8

  2144. 	cmp	counterL , #0

  2145. 	ble	.Lsgemm_kernel_L1_M4_40

  2146. 	.align 5

  2147. 

  2148. .Lsgemm_kernel_L1_M4_22:

  2149. 	KERNEL4x1_SUB

  2150. 	KERNEL4x1_SUB

  2151. 	KERNEL4x1_SUB

  2152. 	KERNEL4x1_SUB

  2153. 

  2154. 	KERNEL4x1_SUB

  2155. 	KERNEL4x1_SUB

  2156. 	KERNEL4x1_SUB

  2157. 	KERNEL4x1_SUB

  2158. 

  2159. 	subs	counterL, counterL, #1

  2160. 	bgt	.Lsgemm_kernel_L1_M4_22

  2161. 

  2162. 

  2163. .Lsgemm_kernel_L1_M4_40:

  2164. 

  2165. 	ands	counterL , origK, #7		// counterL = counterL % 8

  2166. 	ble	.Lsgemm_kernel_L1_M4_100

  2167. 

  2168. .Lsgemm_kernel_L1_M4_42:

  2169. 

  2170. 	KERNEL4x1_SUB

  2171. 

  2172. 	subs	counterL, counterL, #1

  2173. 	bgt	.Lsgemm_kernel_L1_M4_42

  2174. 

  2175. .Lsgemm_kernel_L1_M4_100:

  2176. 

  2177. 	SAVE4x1

  2178. 

  2179. .Lsgemm_kernel_L1_M4_END:

  2180. 

  2181. /******************************************************************************/

  2182. 

  2183. .Lsgemm_kernel_L1_M2_BEGIN:

  2184. 

  2185. 	mov	counterI, origM

  2186. 	tst	counterI , #3

  2187. 	ble	.Lsgemm_kernel_L1_END

  2188. 

  2189. 	tst	counterI, #2			// counterI = counterI / 2

  2190. 	ble	.Lsgemm_kernel_L1_M1_BEGIN

  2191. 

  2192. .Lsgemm_kernel_L1_M2_20:

  2193. 

  2194. 	INIT2x1

  2195. 

  2196. 	mov	pB, origPB

  2197. 

  2198. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  2199. 	cmp	counterL , #0

  2200. 	ble	.Lsgemm_kernel_L1_M2_40

  2201. 

  2202. .Lsgemm_kernel_L1_M2_22:

  2203. 

  2204. 	KERNEL2x1_SUB

  2205. 	KERNEL2x1_SUB

  2206. 	KERNEL2x1_SUB

  2207. 	KERNEL2x1_SUB

  2208. 

  2209. 	KERNEL2x1_SUB

  2210. 	KERNEL2x1_SUB

  2211. 	KERNEL2x1_SUB

  2212. 	KERNEL2x1_SUB

  2213. 

  2214. 	subs	counterL, counterL, #1

  2215. 	bgt	.Lsgemm_kernel_L1_M2_22

  2216. 

  2217. 

  2218. .Lsgemm_kernel_L1_M2_40:

  2219. 

  2220. 	ands	counterL , origK, #7		// counterL = counterL % 8

  2221. 	ble	.Lsgemm_kernel_L1_M2_100

  2222. 

  2223. .Lsgemm_kernel_L1_M2_42:

  2224. 

  2225. 	KERNEL2x1_SUB

  2226. 

  2227. 	subs	counterL, counterL, #1

  2228. 	bgt	.Lsgemm_kernel_L1_M2_42

  2229. 

  2230. .Lsgemm_kernel_L1_M2_100:

  2231. 

  2232. 	SAVE2x1

  2233. 

  2234. .Lsgemm_kernel_L1_M2_END:

  2235. 

  2236. /******************************************************************************/

  2237. 

  2238. .Lsgemm_kernel_L1_M1_BEGIN:

  2239. 

  2240. 	tst	counterI, #1			// counterI = counterI % 2

  2241. 	ble	.Lsgemm_kernel_L1_END

  2242. 

  2243. .Lsgemm_kernel_L1_M1_20:

  2244. 

  2245. 	INIT1x1

  2246. 

  2247. 	mov	pB, origPB

  2248. 

  2249. 	asr 	counterL , origK, #3		// counterL = counterL / 8

  2250. 	cmp	counterL , #0

  2251. 	ble	.Lsgemm_kernel_L1_M1_40

  2252. 

  2253. .Lsgemm_kernel_L1_M1_22:

  2254. 	KERNEL1x1_SUB

  2255. 	KERNEL1x1_SUB

  2256. 	KERNEL1x1_SUB

  2257. 	KERNEL1x1_SUB

  2258. 

  2259. 	KERNEL1x1_SUB

  2260. 	KERNEL1x1_SUB

  2261. 	KERNEL1x1_SUB

  2262. 	KERNEL1x1_SUB

  2263. 

  2264. 	subs	counterL, counterL, #1

  2265. 	bgt	.Lsgemm_kernel_L1_M1_22

  2266. 

  2267. 

  2268. .Lsgemm_kernel_L1_M1_40:

  2269. 

  2270. 	ands	counterL , origK, #7		// counterL = counterL % 8

  2271. 	ble	.Lsgemm_kernel_L1_M1_100

  2272. 

  2273. .Lsgemm_kernel_L1_M1_42:

  2274. 

  2275. 	KERNEL1x1_SUB

  2276. 

  2277. 	subs	counterL, counterL, #1

  2278. 	bgt	.Lsgemm_kernel_L1_M1_42

  2279. 

  2280. .Lsgemm_kernel_L1_M1_100:

  2281. 

  2282. 	SAVE1x1

  2283. 

  2284. .Lsgemm_kernel_L1_END:

  2285. 

  2286. /******************************************************************************/

  2287. 

  2288. .Lsgemm_kernel_L999:

  2289. 	mov	x0, #0				// set return value

  2290. 	ldp	d8, d9, [sp, #(0 * 16)]

  2291. 	ldp	d10, d11, [sp, #(1 * 16)]

  2292. 	ldp	d12, d13, [sp, #(2 * 16)]

  2293. 	ldp	d14, d15, [sp, #(3 * 16)]

  2294. 	ldp	d16, d17, [sp, #(4 * 16)]

  2295. 	ldp	x18, x19, [sp, #(5 * 16)]

  2296. 	ldp	x20, x21, [sp, #(6 * 16)]

  2297. 	ldp	x22, x23, [sp, #(7 * 16)]

  2298. 	ldp	x24, x25, [sp, #(8 * 16)]

  2299. 	ldp	x26, x27, [sp, #(9 * 16)]

  2300. 	ldr	x28, [sp, #(10 * 16)]

  2301. 	add	sp, sp, #(11*16)

  2302. 	ret

  2303. 

  2304. 	EPILOGUE