OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cd8ac192a9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cd8ac192a9'
${ noResults }
OpenBLAS/kernel/x86_64/sgemm_kernel_16x4_skylakex.S

6812 lines
134 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2013, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. /*********************************************************************

  29. * 2014/07/28 Saar

  30. *        BLASTEST               : OK

  31. *        CTEST                  : OK

  32. *        TEST                   : OK

  33. *

  34. * 2013/10/28 Saar

  35. * Parameter:

  36. *	SGEMM_DEFAULT_UNROLL_N	4

  37. *	SGEMM_DEFAULT_UNROLL_M	16

  38. *	SGEMM_DEFAULT_P		768

  39. *	SGEMM_DEFAULT_Q		384

  40. *	A_PR1			512

  41. *	B_PR1			512

  42. *	

  43. * 

  44. * 2014/07/28 Saar

  45. * Performance at 9216x9216x9216:

  46. *       1 thread:      102 GFLOPS       (SANDYBRIDGE:  59)      (MKL:   83)

  47. *       2 threads:     195 GFLOPS       (SANDYBRIDGE: 116)      (MKL:  155)

  48. *       3 threads:     281 GFLOPS       (SANDYBRIDGE: 165)      (MKL:  230)

  49. *       4 threads:     366 GFLOPS       (SANDYBRIDGE: 223)      (MKL:  267)

  50. *

  51. *********************************************************************/

  52. 

  53. #define ASSEMBLER

  54. #include "common.h"

  55.  

  56. #define OLD_M	%rdi

  57. #define OLD_N	%rsi

  58. #define M	%r13

  59. #define J	%r14

  60. #define OLD_K	%rdx

  61. 

  62. #define A	%rcx

  63. #define B	%r8

  64. #define C	%r9

  65. #define LDC	%r10

  66. 	

  67. #define I	%r11

  68. #define AO	%rdi

  69. #define BO	%rsi

  70. #define	CO1	%r15

  71. #define K	%r12

  72. #define BI	%rbp

  73. #define BO2	%rbp

  74. #define	SP	%rbx

  75. 

  76. #define BO1	%rdi

  77. #define	CO2	%rdx

  78. 

  79. #ifndef WINDOWS_ABI

  80. 

  81. #define STACKSIZE 96

  82. 

  83. #else

  84. 

  85. #define STACKSIZE 256

  86. 

  87. #define OLD_A		40 + STACKSIZE(%rsp)

  88. #define OLD_B		48 + STACKSIZE(%rsp)

  89. #define OLD_C		56 + STACKSIZE(%rsp)

  90. #define OLD_LDC		64 + STACKSIZE(%rsp)

  91. #define OLD_OFFSET	72 + STACKSIZE(%rsp)

  92. 

  93. #endif

  94. 

  95. #if defined(OS_WINDOWS)

  96. #define L_BUFFER_SIZE 8192

  97. #else

  98. #define L_BUFFER_SIZE 12288

  99. #endif

  100. 

  101. #define Ndiv6	 24(%rsp)

  102. #define Nmod6	 32(%rsp)

  103. #define N	 40(%rsp)

  104. #define ALPHA	 48(%rsp)

  105. #define OFFSET	 56(%rsp)

  106. #define KK	 64(%rsp)

  107. #define KKK	 72(%rsp)

  108. #define BUFFER1	           128(%rsp)

  109. 

  110. #if defined(OS_WINDOWS)

  111. #if   L_BUFFER_SIZE > 16384

  112. #define STACK_TOUCH \

  113.         movl    $0,  4096 * 4(%rsp);\

  114.         movl    $0,  4096 * 3(%rsp);\

  115.         movl    $0,  4096 * 2(%rsp);\

  116.         movl    $0,  4096 * 1(%rsp);

  117. #elif L_BUFFER_SIZE > 12288

  118. #define STACK_TOUCH \

  119.         movl    $0,  4096 * 3(%rsp);\

  120.         movl    $0,  4096 * 2(%rsp);\

  121.         movl    $0,  4096 * 1(%rsp);

  122. #elif L_BUFFER_SIZE > 8192

  123. #define STACK_TOUCH \

  124.         movl    $0,  4096 * 2(%rsp);\

  125.         movl    $0,  4096 * 1(%rsp);

  126. #elif L_BUFFER_SIZE > 4096

  127. #define STACK_TOUCH \

  128.         movl    $0,  4096 * 1(%rsp);

  129. #else

  130. #define STACK_TOUCH

  131. #endif

  132. #else

  133. #define STACK_TOUCH

  134. #endif

  135. 

  136. #if defined(BULLDOZER)

  137. 

  138. #define VFMADD231PS_( y0,y1,y2 ) vfmaddps y0,y1,y2,y0

  139. 

  140. #define VFMADD231SS_( x0,x1,x2 ) vfmaddss x0,x1,x2,x0

  141. 

  142. #else

  143. 

  144. #define VFMADD231PS_( y0,y1,y2 ) vfmadd231ps y1,y2,y0

  145. 

  146. #define VFMADD231SS_( x0,x1,x2 ) vfmadd231ss x1,x2,x0

  147. 

  148. #endif

  149. 

  150. 

  151. #define	A_PR1	512

  152. #define	B_PR1	512

  153. 

  154. /*******************************************************************************************

  155. * 6 lines of N

  156. *******************************************************************************************/

  157. 

  158. .macro KERNEL16x6_SUB

  159. 	vmovups 	-16 * SIZE(AO), %zmm0

  160. 	vbroadcastss	 -4 * SIZE(BO), %zmm2

  161. 	vbroadcastss	 -3 * SIZE(BO), %zmm3

  162. #	prefetcht0	A_PR1(AO)

  163. 

  164. 	VFMADD231PS_(  	%zmm4,%zmm2,%zmm0 )

  165. 	VFMADD231PS_(  	%zmm6,%zmm3,%zmm0 )

  166. 

  167. 	vbroadcastss	 -2 * SIZE(BO), %zmm2

  168. 	vbroadcastss	 -1 * SIZE(BO), %zmm3

  169. 	VFMADD231PS_(  	%zmm8,%zmm2,%zmm0  )

  170. 	VFMADD231PS_(  	%zmm10,%zmm3,%zmm0 )

  171. 

  172. 	vbroadcastss	  0 * SIZE(BO), %zmm2

  173. 	vbroadcastss	  1 * SIZE(BO), %zmm3

  174. 	VFMADD231PS_(  	%zmm12,%zmm2,%zmm0  )

  175. 	VFMADD231PS_(  	%zmm14,%zmm3,%zmm0 )

  176. 

  177. 	addq	$ 6*SIZE, BO 

  178. 	addq	$ 16*SIZE, AO 

  179. 	decq	%rax 

  180. .endm

  181. 

  182. .macro KERNEL16x6_SUB4

  183. 	vmovups 	-16 * SIZE(AO), %zmm0

  184. 	vbroadcastss	 -4 * SIZE(BO), %zmm2

  185. 	vbroadcastss	 -3 * SIZE(BO), %zmm3

  186. 

  187. 	VFMADD231PS_(  	%zmm4,%zmm2,%zmm0 )

  188. 	VFMADD231PS_(  	%zmm6,%zmm3,%zmm0 )

  189. 

  190. 	vbroadcastss	 -2 * SIZE(BO), %zmm7

  191. 	vbroadcastss	 -1 * SIZE(BO), %zmm9

  192. 	VFMADD231PS_(  	%zmm8,%zmm7,%zmm0  )

  193. 	VFMADD231PS_(  	%zmm10,%zmm9,%zmm0 )

  194. 

  195. 	vbroadcastss	  0 * SIZE(BO), %zmm11

  196. 	vbroadcastss	  1 * SIZE(BO), %zmm13

  197. 	VFMADD231PS_(  	%zmm12,%zmm11,%zmm0  )

  198. 	VFMADD231PS_(  	%zmm14,%zmm13,%zmm0 )

  199. 

  200. 	addq	$ 6*SIZE, BO 

  201. 	addq	$ 16*SIZE, AO 

  202. 	decq	%rax 

  203. 	vmovups 	-16 * SIZE(AO), %zmm0

  204. 	vbroadcastss	 -4 * SIZE(BO), %zmm16

  205. 	vbroadcastss	 -3 * SIZE(BO), %zmm17

  206. 

  207. 	VFMADD231PS_(  	%zmm4,%zmm16,%zmm0 )

  208. 	VFMADD231PS_(  	%zmm6,%zmm17,%zmm0 )

  209. 

  210. 	vbroadcastss	 -2 * SIZE(BO), %zmm18

  211. 	vbroadcastss	 -1 * SIZE(BO), %zmm19

  212. 	VFMADD231PS_(  	%zmm8,%zmm18,%zmm0  )

  213. 	VFMADD231PS_(  	%zmm10,%zmm19,%zmm0 )

  214. 

  215. 	vbroadcastss	  0 * SIZE(BO), %zmm20

  216. 	vbroadcastss	  1 * SIZE(BO), %zmm21

  217. 	VFMADD231PS_(  	%zmm12,%zmm20,%zmm0  )

  218. 	VFMADD231PS_(  	%zmm14,%zmm21,%zmm0 )

  219. 

  220. 	addq	$ 6*SIZE, BO 

  221. 	addq	$ 16*SIZE, AO 

  222. 	decq	%rax 

  223. 

  224. 	vmovups 	-16 * SIZE(AO), %zmm0

  225. 	vbroadcastss	 -4 * SIZE(BO), %zmm22

  226. 	vbroadcastss	 -3 * SIZE(BO), %zmm23

  227. 

  228. 	VFMADD231PS_(  	%zmm4,%zmm22,%zmm0 )

  229. 	VFMADD231PS_(  	%zmm6,%zmm23,%zmm0 )

  230. 

  231. 	vbroadcastss	 -2 * SIZE(BO), %zmm24

  232. 	vbroadcastss	 -1 * SIZE(BO), %zmm25

  233. 	VFMADD231PS_(  	%zmm8,%zmm24,%zmm0  )

  234. 	VFMADD231PS_(  	%zmm10,%zmm25,%zmm0 )

  235. 

  236. 	vbroadcastss	  0 * SIZE(BO), %zmm26

  237. 	vbroadcastss	  1 * SIZE(BO), %zmm27

  238. 	VFMADD231PS_(  	%zmm12,%zmm26,%zmm0  )

  239. 	VFMADD231PS_(  	%zmm14,%zmm27,%zmm0 )

  240. 

  241. 	addq	$ 6*SIZE, BO 

  242. 	addq	$ 16*SIZE, AO 

  243. 	decq	%rax 

  244. 	vmovups 	-16 * SIZE(AO), %zmm0

  245. 	vbroadcastss	 -4 * SIZE(BO), %zmm28

  246. 	vbroadcastss	 -3 * SIZE(BO), %zmm29

  247. 

  248. 	VFMADD231PS_(  	%zmm4,%zmm28,%zmm0 )

  249. 	VFMADD231PS_(  	%zmm6,%zmm29,%zmm0 )

  250. 

  251. 	vbroadcastss	 -2 * SIZE(BO), %zmm30

  252. 	vbroadcastss	 -1 * SIZE(BO), %zmm31

  253. 	VFMADD231PS_(  	%zmm8,%zmm30,%zmm0  )

  254. 	VFMADD231PS_(  	%zmm10,%zmm31,%zmm0 )

  255. 

  256. 	vbroadcastss	  0 * SIZE(BO), %zmm1

  257. 	vbroadcastss	  1 * SIZE(BO), %zmm5

  258. 	VFMADD231PS_(  	%zmm12,%zmm1,%zmm0  )

  259. 	VFMADD231PS_(  	%zmm14,%zmm5,%zmm0 )

  260. 

  261. 	addq	$ 6*SIZE, BO 

  262. 	addq	$ 16*SIZE, AO 

  263. 	decq	%rax 

  264. .endm

  265. 

  266. .macro SAVE16x6

  267. 

  268. 	vbroadcastss	ALPHA, %zmm0

  269. 

  270. 	vmulps	%zmm0 , %zmm4 , %zmm4

  271. 	vmulps	%zmm0 , %zmm6 , %zmm6

  272. 	vmulps	%zmm0 , %zmm8 , %zmm8

  273. 	vmulps	%zmm0 , %zmm10, %zmm10

  274. 	vmulps	%zmm0 , %zmm12, %zmm12

  275. 	vmulps	%zmm0 , %zmm14, %zmm14

  276. 

  277. 

  278. #if !defined(TRMMKERNEL)

  279. 

  280. 	vaddps 	        (CO1), %zmm4,%zmm4

  281. 

  282. 	vaddps 	        (CO1, LDC), %zmm6,%zmm6

  283. 

  284. 	vaddps 	        (CO1, LDC,2), %zmm8,%zmm8

  285. 

  286. 	vaddps 	        (CO2), %zmm10,%zmm10

  287. 

  288. 	vaddps 	        (CO2, LDC), %zmm12,%zmm12

  289. 

  290. 	vaddps 	        (CO2, LDC,2), %zmm14,%zmm14

  291. 

  292. #endif

  293. 

  294. 	vmovups	%zmm4 ,  	(CO1)

  295. 

  296. 	vmovups	%zmm6 ,  	(CO1, LDC)

  297. 

  298. 	vmovups	%zmm8 ,  	(CO1, LDC,2)

  299. 

  300. 	vmovups	%zmm10,  	(CO2)

  301. 

  302. 	vmovups	%zmm12,  	(CO2, LDC)

  303. 

  304. 	vmovups	%zmm14,  	(CO2, LDC,2)

  305. 

  306. .endm

  307. 

  308. 

  309. 

  310. 

  311. /*******************************************************************************************/

  312. 

  313. .macro KERNEL8x6_SUB

  314. 	vmovups 	-16 * SIZE(AO), %ymm0

  315. 	vbroadcastss	 -4 * SIZE(BO), %ymm2

  316. 	vbroadcastss	 -3 * SIZE(BO), %ymm3

  317. 

  318. 	VFMADD231PS_(  	%ymm4,%ymm2,%ymm0 )

  319. 	VFMADD231PS_(  	%ymm6,%ymm3,%ymm0 )

  320. 

  321. 	vbroadcastss	 -2 * SIZE(BO), %ymm2

  322. 	vbroadcastss	 -1 * SIZE(BO), %ymm3

  323. 	VFMADD231PS_(  	%ymm8,%ymm2,%ymm0  )

  324. 	VFMADD231PS_(  	%ymm10,%ymm3,%ymm0 )

  325. 

  326. 	vbroadcastss	  0 * SIZE(BO), %ymm2

  327. 	vbroadcastss	  1 * SIZE(BO), %ymm3

  328. 	VFMADD231PS_(  	%ymm12,%ymm2,%ymm0  )

  329. 	VFMADD231PS_(  	%ymm14,%ymm3,%ymm0 )

  330. 

  331. 	addq	$ 6*SIZE, BO 

  332. 	addq	$ 8*SIZE, AO 

  333. 	decq	%rax 

  334. .endm

  335. 

  336. .macro SAVE8x6

  337. 

  338. 	vbroadcastss	ALPHA, %ymm0

  339. 

  340. 	vmulps	%ymm0 , %ymm4 , %ymm4

  341. 	vmulps	%ymm0 , %ymm6 , %ymm6

  342. 	vmulps	%ymm0 , %ymm8 , %ymm8

  343. 	vmulps	%ymm0 , %ymm10, %ymm10

  344. 	vmulps	%ymm0 , %ymm12, %ymm12

  345. 	vmulps	%ymm0 , %ymm14, %ymm14

  346. 

  347. 

  348. #if !defined(TRMMKERNEL)

  349. 

  350. 	vaddps 	        (CO1), %ymm4,%ymm4

  351. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  352. 	vaddps 	        (CO1, LDC,2), %ymm8,%ymm8

  353. 	vaddps 	        (CO2), %ymm10,%ymm10

  354. 	vaddps 	        (CO2, LDC), %ymm12,%ymm12

  355. 	vaddps 	        (CO2, LDC,2), %ymm14,%ymm14

  356. 

  357. #endif

  358. 

  359. 	vmovups	%ymm4 ,  	(CO1)

  360. 	vmovups	%ymm6 ,  	(CO1, LDC)

  361. 	vmovups	%ymm8 ,  	(CO1, LDC,2)

  362. 	vmovups	%ymm10,  	(CO2)

  363. 	vmovups	%ymm12,  	(CO2, LDC)

  364. 	vmovups	%ymm14,  	(CO2, LDC,2)

  365. 

  366. .endm

  367. 

  368. 

  369. 

  370. /*******************************************************************************************/

  371. 

  372. .macro KERNEL4x6_SUB

  373. 	vmovups 	-16 * SIZE(AO), %xmm0

  374. 	vbroadcastss	 -4 * SIZE(BO), %xmm2

  375. 	vbroadcastss	 -3 * SIZE(BO), %xmm3

  376. 

  377. 	VFMADD231PS_(  	%xmm4,%xmm2,%xmm0 )

  378. 	VFMADD231PS_(  	%xmm6,%xmm3,%xmm0 )

  379. 

  380. 	vbroadcastss	 -2 * SIZE(BO), %xmm2

  381. 	vbroadcastss	 -1 * SIZE(BO), %xmm3

  382. 	VFMADD231PS_(  	%xmm8,%xmm2,%xmm0  )

  383. 	VFMADD231PS_(  	%xmm10,%xmm3,%xmm0 )

  384. 

  385. 	vbroadcastss	  0 * SIZE(BO), %xmm2

  386. 	vbroadcastss	  1 * SIZE(BO), %xmm3

  387. 	VFMADD231PS_(  	%xmm12,%xmm2,%xmm0  )

  388. 	VFMADD231PS_(  	%xmm14,%xmm3,%xmm0 )

  389. 

  390. 	addq	$ 6*SIZE, BO 

  391. 	addq	$ 4*SIZE, AO 

  392. 	decq	%rax 

  393. .endm

  394. 

  395. .macro SAVE4x6

  396. 

  397. 	vbroadcastss	ALPHA, %xmm0

  398. 

  399. 	vmulps	%xmm0 , %xmm4 , %xmm4

  400. 	vmulps	%xmm0 , %xmm6 , %xmm6

  401. 	vmulps	%xmm0 , %xmm8 , %xmm8

  402. 	vmulps	%xmm0 , %xmm10, %xmm10

  403. 	vmulps	%xmm0 , %xmm12, %xmm12

  404. 	vmulps	%xmm0 , %xmm14, %xmm14

  405. 

  406. 

  407. #if !defined(TRMMKERNEL)

  408. 

  409. 	vaddps 	        (CO1), %xmm4,%xmm4

  410. 	vaddps 	        (CO1, LDC), %xmm6,%xmm6

  411. 	vaddps 	        (CO1, LDC,2), %xmm8,%xmm8

  412. 	vaddps 	        (CO2), %xmm10,%xmm10

  413. 	vaddps 	        (CO2, LDC), %xmm12,%xmm12

  414. 	vaddps 	        (CO2, LDC,2), %xmm14,%xmm14

  415. 

  416. #endif

  417. 

  418. 	vmovups	%xmm4 ,  	(CO1)

  419. 	vmovups	%xmm6 ,  	(CO1, LDC)

  420. 	vmovups	%xmm8 ,  	(CO1, LDC,2)

  421. 	vmovups	%xmm10,  	(CO2)

  422. 	vmovups	%xmm12,  	(CO2, LDC)

  423. 	vmovups	%xmm14,  	(CO2, LDC,2)

  424. 

  425. .endm

  426. 

  427. 

  428. /*******************************************************************************************/

  429. 

  430. .macro KERNEL2x6_SUB

  431. 	vmovss 	-16 * SIZE(AO), %xmm0

  432. 	vmovss 	-15 * SIZE(AO), %xmm1

  433. 	vmovss	 -4 * SIZE(BO), %xmm2

  434. 	vmovss	 -3 * SIZE(BO), %xmm3

  435. 

  436. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  437. 	VFMADD231SS_(  	%xmm5,%xmm2,%xmm1 )

  438. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  439. 	VFMADD231SS_(  	%xmm7,%xmm3,%xmm1 )

  440. 

  441. 	vmovss	 -2 * SIZE(BO), %xmm2

  442. 	vmovss	 -1 * SIZE(BO), %xmm3

  443. 	VFMADD231SS_(  	%xmm8,%xmm2,%xmm0  )

  444. 	VFMADD231SS_(  	%xmm9,%xmm2,%xmm1  )

  445. 	VFMADD231SS_(  	%xmm10,%xmm3,%xmm0 )

  446. 	VFMADD231SS_(  	%xmm11,%xmm3,%xmm1 )

  447. 

  448. 	vmovss	  0 * SIZE(BO), %xmm2

  449. 	vmovss	  1 * SIZE(BO), %xmm3

  450. 	VFMADD231SS_(  	%xmm12,%xmm2,%xmm0  )

  451. 	VFMADD231SS_(  	%xmm13,%xmm2,%xmm1  )

  452. 	VFMADD231SS_(  	%xmm14,%xmm3,%xmm0 )

  453. 	VFMADD231SS_(  	%xmm15,%xmm3,%xmm1 )

  454. 

  455. 	addq	$ 6*SIZE, BO 

  456. 	addq	$ 2*SIZE, AO 

  457. 	decq	%rax 

  458. .endm

  459. 

  460. .macro SAVE2x6

  461. 

  462. 	vmovss	ALPHA, %xmm0

  463. 

  464. 	vmulss	%xmm0 , %xmm4 , %xmm4

  465. 	vmulss	%xmm0 , %xmm5 , %xmm5

  466. 	vmulss	%xmm0 , %xmm6 , %xmm6

  467. 	vmulss	%xmm0 , %xmm7 , %xmm7

  468. 	vmulss	%xmm0 , %xmm8 , %xmm8

  469. 	vmulss	%xmm0 , %xmm9 , %xmm9

  470. 	vmulss	%xmm0 , %xmm10, %xmm10

  471. 	vmulss	%xmm0 , %xmm11, %xmm11

  472. 	vmulss	%xmm0 , %xmm12, %xmm12

  473. 	vmulss	%xmm0 , %xmm13, %xmm13

  474. 	vmulss	%xmm0 , %xmm14, %xmm14

  475. 	vmulss	%xmm0 , %xmm15, %xmm15

  476. 

  477. 

  478. #if !defined(TRMMKERNEL)

  479. 

  480. 	vaddss 	        (CO1), %xmm4,%xmm4

  481. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  482. 

  483. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  484. 	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7

  485. 

  486. 	vaddss 	        (CO1, LDC,2), %xmm8,%xmm8

  487. 	vaddss  1 * SIZE(CO1, LDC,2), %xmm9,%xmm9

  488. 

  489. 	vaddss 	        (CO2), %xmm10,%xmm10

  490. 	vaddss  1 * SIZE(CO2), %xmm11,%xmm11

  491. 

  492. 	vaddss 	        (CO2, LDC), %xmm12,%xmm12

  493. 	vaddss  1 * SIZE(CO2, LDC), %xmm13,%xmm13

  494. 

  495. 	vaddss 	        (CO2, LDC,2), %xmm14,%xmm14

  496. 	vaddss  1 * SIZE(CO2, LDC,2), %xmm15,%xmm15

  497. 

  498. #endif

  499. 

  500. 	vmovss	%xmm4 ,  	(CO1)

  501. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  502. 

  503. 	vmovss	%xmm6 ,  	(CO1, LDC)

  504. 	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)

  505. 

  506. 	vmovss	%xmm8 ,  	(CO1, LDC,2)

  507. 	vmovss	%xmm9 , 1 * SIZE(CO1, LDC,2)

  508. 

  509. 	vmovss	%xmm10,  	(CO2)

  510. 	vmovss	%xmm11, 1 * SIZE(CO2)

  511. 

  512. 	vmovss	%xmm12,  	(CO2, LDC)

  513. 	vmovss	%xmm13, 1 * SIZE(CO2, LDC)

  514. 

  515. 	vmovss	%xmm14,  	(CO2, LDC,2)

  516. 	vmovss	%xmm15, 1 * SIZE(CO2, LDC,2)

  517. 

  518. .endm

  519. 

  520. 

  521. /*******************************************************************************************/

  522. 

  523. .macro KERNEL1x6_SUB

  524. 	vmovss 	-16 * SIZE(AO), %xmm0

  525. 	vmovss	 -4 * SIZE(BO), %xmm2

  526. 	vmovss	 -3 * SIZE(BO), %xmm3

  527. 

  528. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  529. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  530. 

  531. 	vmovss	 -2 * SIZE(BO), %xmm2

  532. 	vmovss	 -1 * SIZE(BO), %xmm3

  533. 	VFMADD231SS_(  	%xmm8,%xmm2,%xmm0  )

  534. 	VFMADD231SS_(  	%xmm10,%xmm3,%xmm0 )

  535. 

  536. 	vmovss	  0 * SIZE(BO), %xmm2

  537. 	vmovss	  1 * SIZE(BO), %xmm3

  538. 	VFMADD231SS_(  	%xmm12,%xmm2,%xmm0  )

  539. 	VFMADD231SS_(  	%xmm14,%xmm3,%xmm0 )

  540. 

  541. 	addq	$ 6*SIZE, BO 

  542. 	addq	$ 1*SIZE, AO 

  543. 	decq	%rax 

  544. .endm

  545. 

  546. .macro SAVE1x6

  547. 

  548. 	vmovss	ALPHA, %xmm0

  549. 

  550. 	vmulss	%xmm0 , %xmm4 , %xmm4

  551. 	vmulss	%xmm0 , %xmm6 , %xmm6

  552. 	vmulss	%xmm0 , %xmm8 , %xmm8

  553. 	vmulss	%xmm0 , %xmm10, %xmm10

  554. 	vmulss	%xmm0 , %xmm12, %xmm12

  555. 	vmulss	%xmm0 , %xmm14, %xmm14

  556. 

  557. #if !defined(TRMMKERNEL)

  558. 

  559. 	vaddss 	        (CO1), %xmm4,%xmm4

  560. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  561. 	vaddss 	        (CO1, LDC,2), %xmm8,%xmm8

  562. 	vaddss 	        (CO2), %xmm10,%xmm10

  563. 	vaddss 	        (CO2, LDC), %xmm12,%xmm12

  564. 	vaddss 	        (CO2, LDC,2), %xmm14,%xmm14

  565. 

  566. #endif

  567. 

  568. 	vmovss	%xmm4 ,  	(CO1)

  569. 	vmovss	%xmm6 ,  	(CO1, LDC)

  570. 	vmovss	%xmm8 ,  	(CO1, LDC,2)

  571. 	vmovss	%xmm10,  	(CO2)

  572. 	vmovss	%xmm12,  	(CO2, LDC)

  573. 	vmovss	%xmm14,  	(CO2, LDC,2)

  574. 

  575. .endm

  576. 

  577. 

  578. /*******************************************************************************************/

  579. 

  580. 

  581. /*******************************************************************************************

  582. * 4 lines of N

  583. *******************************************************************************************/

  584. 

  585. .macro KERNEL16x4_SUB

  586. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %zmm0

  587. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %zmm2

  588. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %zmm3

  589. 	VFMADD231PS_(  	%zmm4,%zmm2,%zmm0 )

  590. 	VFMADD231PS_(  	%zmm6,%zmm3,%zmm0 )

  591. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %zmm2

  592. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %zmm3

  593. 	VFMADD231PS_(  	%zmm8,%zmm2,%zmm0  )

  594. 	VFMADD231PS_(  	%zmm10,%zmm3,%zmm0 )

  595. 	addq	$ 4 , BI	

  596. 	addq	$ 16, %rax 

  597. .endm

  598. 

  599. .macro SAVE16x4

  600. 

  601. 	vbroadcastss	ALPHA, %zmm0

  602. 

  603. 	vmulps	%zmm0 , %zmm4 , %zmm4

  604. 	vmulps	%zmm0 , %zmm6 , %zmm6

  605. 	vmulps	%zmm0 , %zmm8 , %zmm8

  606. 	vmulps	%zmm0 , %zmm10, %zmm10

  607. 

  608. 

  609. #if !defined(TRMMKERNEL)

  610. 

  611. 	vaddps 	        (CO1), %zmm4,%zmm4

  612. 

  613. 	vaddps 	        (CO1, LDC), %zmm6,%zmm6

  614. 

  615. 	vaddps 	        (CO2), %zmm8,%zmm8

  616. 

  617. 	vaddps 	        (CO2, LDC), %zmm10,%zmm10

  618. 

  619. #endif

  620. 

  621. 	vmovups	%zmm4 ,  	(CO1)

  622. 

  623. 	vmovups	%zmm6 ,  	(CO1, LDC)

  624. 

  625. 	vmovups	%zmm8 ,  	(CO2)

  626. 

  627. 	vmovups	%zmm10,  	(CO2, LDC)

  628. 

  629. 	prefetcht0	64(CO1)

  630. 	prefetcht0	64(CO1, LDC)

  631. 	prefetcht0	64(CO2)

  632. 	prefetcht0	64(CO2, LDC)

  633. 

  634. .endm

  635. 

  636. 

  637. 

  638. /*******************************************************************************************/

  639. 

  640. .macro KERNEL8x4_SUB

  641. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  642. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  643. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  644. 	VFMADD231PS_(  	%ymm4,%ymm2,%ymm0 )

  645. 	VFMADD231PS_(  	%ymm6,%ymm3,%ymm0 )

  646. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2

  647. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3

  648. 	VFMADD231PS_(  	%ymm8,%ymm2,%ymm0  )

  649. 	VFMADD231PS_(  	%ymm10,%ymm3,%ymm0 )

  650. 	addq	$ 4 , BI	

  651. 	addq	$ 8 , %rax 

  652. .endm

  653. 

  654. .macro SAVE8x4

  655. 

  656. 	vbroadcastss	ALPHA, %ymm0

  657. 

  658. 	vmulps	%ymm0 , %ymm4 , %ymm4

  659. 	vmulps	%ymm0 , %ymm6 , %ymm6

  660. 	vmulps	%ymm0 , %ymm8 , %ymm8

  661. 	vmulps	%ymm0 , %ymm10, %ymm10

  662. 

  663. 

  664. #if !defined(TRMMKERNEL)

  665. 

  666. 	vaddps 	        (CO1), %ymm4,%ymm4

  667. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  668. 	vaddps 	        (CO2), %ymm8,%ymm8

  669. 	vaddps 	        (CO2, LDC), %ymm10,%ymm10

  670. 

  671. #endif

  672. 

  673. 	vmovups	%ymm4 ,  	(CO1)

  674. 	vmovups	%ymm6 ,  	(CO1, LDC)

  675. 	vmovups	%ymm8 ,  	(CO2)

  676. 	vmovups	%ymm10,  	(CO2, LDC)

  677. 

  678. .endm

  679. 

  680. 

  681. 

  682. /*******************************************************************************************/

  683. 

  684. .macro KERNEL4x4_SUB

  685. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  686. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  687. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  688. 	VFMADD231PS_(  	%xmm4,%xmm2,%xmm0 )

  689. 	VFMADD231PS_(  	%xmm6,%xmm3,%xmm0 )

  690. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  691. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  692. 	VFMADD231PS_(  	%xmm8,%xmm2,%xmm0  )

  693. 	VFMADD231PS_(  	%xmm10,%xmm3,%xmm0 )

  694. 	addq	$ 4 , BI	

  695. 	addq	$ 4 , %rax 

  696. .endm

  697. 

  698. .macro SAVE4x4

  699. 

  700. 	vbroadcastss	ALPHA, %xmm0

  701. 

  702. 	vmulps	%xmm0 , %xmm4 , %xmm4

  703. 	vmulps	%xmm0 , %xmm6 , %xmm6

  704. 	vmulps	%xmm0 , %xmm8 , %xmm8

  705. 	vmulps	%xmm0 , %xmm10, %xmm10

  706. 

  707. 

  708. #if !defined(TRMMKERNEL)

  709. 

  710. 	vaddps 	        (CO1), %xmm4,%xmm4

  711. 	vaddps 	        (CO1, LDC), %xmm6,%xmm6

  712. 	vaddps 	        (CO2), %xmm8,%xmm8

  713. 	vaddps 	        (CO2, LDC), %xmm10,%xmm10

  714. 

  715. #endif

  716. 

  717. 	vmovups	%xmm4 ,  	(CO1)

  718. 	vmovups	%xmm6 ,  	(CO1, LDC)

  719. 	vmovups	%xmm8 ,  	(CO2)

  720. 	vmovups	%xmm10,  	(CO2, LDC)

  721. 

  722. .endm

  723. 

  724. 

  725. /*******************************************************************************************/

  726. 

  727. .macro KERNEL2x4_SUB

  728. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  729. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  730. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  731. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  732. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  733. 	VFMADD231SS_(  	%xmm5,%xmm2,%xmm1 )

  734. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  735. 	VFMADD231SS_(  	%xmm7,%xmm3,%xmm1 )

  736. 	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  737. 	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  738. 	VFMADD231SS_(  	%xmm8,%xmm2,%xmm0  )

  739. 	VFMADD231SS_(  	%xmm9,%xmm2,%xmm1  )

  740. 	VFMADD231SS_(  	%xmm10,%xmm3,%xmm0 )

  741. 	VFMADD231SS_(  	%xmm11,%xmm3,%xmm1 )

  742. 	addq	$ 4 , BI	

  743. 	addq	$ 2, %rax 

  744. .endm

  745. 

  746. .macro SAVE2x4

  747. 

  748. 	vmovss	ALPHA, %xmm0

  749. 

  750. 	vmulss	%xmm0 , %xmm4 , %xmm4

  751. 	vmulss	%xmm0 , %xmm5 , %xmm5

  752. 	vmulss	%xmm0 , %xmm6 , %xmm6

  753. 	vmulss	%xmm0 , %xmm7 , %xmm7

  754. 	vmulss	%xmm0 , %xmm8 , %xmm8

  755. 	vmulss	%xmm0 , %xmm9 , %xmm9

  756. 	vmulss	%xmm0 , %xmm10, %xmm10

  757. 	vmulss	%xmm0 , %xmm11, %xmm11

  758. 

  759. 

  760. #if !defined(TRMMKERNEL)

  761. 

  762. 	vaddss 	        (CO1), %xmm4,%xmm4

  763. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  764. 

  765. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  766. 	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7

  767. 

  768. 	vaddss 	        (CO2), %xmm8,%xmm8

  769. 	vaddss  1 * SIZE(CO2), %xmm9,%xmm9

  770. 

  771. 	vaddss 	        (CO2, LDC), %xmm10,%xmm10

  772. 	vaddss  1 * SIZE(CO2, LDC), %xmm11,%xmm11

  773. 

  774. #endif

  775. 

  776. 	vmovss	%xmm4 ,  	(CO1)

  777. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  778. 

  779. 	vmovss	%xmm6 ,  	(CO1, LDC)

  780. 	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)

  781. 

  782. 	vmovss	%xmm8 ,  	(CO2)

  783. 	vmovss	%xmm9 , 1 * SIZE(CO2)

  784. 

  785. 	vmovss	%xmm10,  	(CO2, LDC)

  786. 	vmovss	%xmm11, 1 * SIZE(CO2, LDC)

  787. 

  788. .endm

  789. 

  790. 

  791. /*******************************************************************************************/

  792. 

  793. .macro KERNEL1x4_SUB

  794. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  795. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  796. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  797. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  798. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  799. 	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  800. 	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  801. 	VFMADD231SS_(  	%xmm8,%xmm2,%xmm0  )

  802. 	VFMADD231SS_(  	%xmm10,%xmm3,%xmm0 )

  803. 	addq	$ 4 , BI	

  804. 	addq	$ 1, %rax 

  805. .endm

  806. 

  807. .macro SAVE1x4

  808. 

  809. 	vmovss	ALPHA, %xmm0

  810. 

  811. 	vmulss	%xmm0 , %xmm4 , %xmm4

  812. 	vmulss	%xmm0 , %xmm6 , %xmm6

  813. 	vmulss	%xmm0 , %xmm8 , %xmm8

  814. 	vmulss	%xmm0 , %xmm10, %xmm10

  815. 

  816. 

  817. #if !defined(TRMMKERNEL)

  818. 

  819. 	vaddss 	        (CO1), %xmm4,%xmm4

  820. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  821. 	vaddss 	        (CO2), %xmm8,%xmm8

  822. 	vaddss 	        (CO2, LDC), %xmm10,%xmm10

  823. 

  824. #endif

  825. 

  826. 	vmovss	%xmm4 ,  	(CO1)

  827. 	vmovss	%xmm6 ,  	(CO1, LDC)

  828. 	vmovss	%xmm8 ,  	(CO2)

  829. 	vmovss	%xmm10,  	(CO2, LDC)

  830. 

  831. .endm

  832. 

  833. 

  834. /*******************************************************************************************/

  835. 

  836. /*******************************************************************************************

  837. * 2 lines of N

  838. *******************************************************************************************/

  839. 

  840. .macro KERNEL16x2_SUB

  841. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %zmm0

  842. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %zmm2

  843. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %zmm3

  844. 	VFMADD231PS_(  	%zmm4,%zmm2,%zmm0 )

  845. 	VFMADD231PS_(  	%zmm6,%zmm3,%zmm0 )

  846. 	addq	$ 2 , BI	

  847. 	addq	$ 16, %rax 

  848. .endm

  849. 

  850. .macro SAVE16x2

  851. 

  852. 	vbroadcastss	ALPHA, %zmm0

  853. 

  854. 	vmulps	%zmm0 , %zmm4 , %zmm4

  855. 	vmulps	%zmm0 , %zmm6 , %zmm6

  856. 

  857. 

  858. #if !defined(TRMMKERNEL)

  859. 

  860. 	vaddps 	        (CO1), %zmm4,%zmm4

  861. 

  862. 	vaddps 	        (CO1, LDC), %zmm6,%zmm6

  863. 

  864. #endif

  865. 

  866. 	vmovups	%zmm4 ,  	(CO1)

  867. 

  868. 	vmovups	%zmm6 ,  	(CO1, LDC)

  869. 

  870. .endm

  871. 

  872. 

  873. 

  874. /*******************************************************************************************/

  875. 

  876. .macro KERNEL8x2_SUB

  877. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  878. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  879. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  880. 	VFMADD231PS_(  	%ymm4,%ymm2,%ymm0 )

  881. 	VFMADD231PS_(  	%ymm6,%ymm3,%ymm0 )

  882. 	addq	$ 2 , BI	

  883. 	addq	$ 8 , %rax 

  884. .endm

  885. 

  886. .macro SAVE8x2

  887. 

  888. 	vbroadcastss	ALPHA, %ymm0

  889. 

  890. 	vmulps	%ymm0 , %ymm4 , %ymm4

  891. 	vmulps	%ymm0 , %ymm6 , %ymm6

  892. 

  893. #if !defined(TRMMKERNEL)

  894. 

  895. 	vaddps 	        (CO1), %ymm4,%ymm4

  896. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  897. 

  898. #endif

  899. 

  900. 	vmovups	%ymm4 ,  	(CO1)

  901. 	vmovups	%ymm6 ,  	(CO1, LDC)

  902. 

  903. .endm

  904. 

  905. 

  906. 

  907. /*******************************************************************************************/

  908. 

  909. .macro KERNEL4x2_SUB

  910. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  911. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  912. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  913. 	VFMADD231PS_(  	%xmm4,%xmm2,%xmm0 )

  914. 	VFMADD231PS_(  	%xmm6,%xmm3,%xmm0 )

  915. 	addq	$ 2 , BI	

  916. 	addq	$ 4 , %rax 

  917. .endm

  918. 

  919. .macro SAVE4x2

  920. 

  921. 	vbroadcastss	ALPHA, %xmm0

  922. 

  923. 	vmulps	%xmm0 , %xmm4 , %xmm4

  924. 	vmulps	%xmm0 , %xmm6 , %xmm6

  925. 

  926. #if !defined(TRMMKERNEL)

  927. 

  928. 	vaddps 	        (CO1), %xmm4,%xmm4

  929. 	vaddps 	        (CO1, LDC), %xmm6,%xmm6

  930. 

  931. #endif

  932. 

  933. 	vmovups	%xmm4 ,  	(CO1)

  934. 	vmovups	%xmm6 ,  	(CO1, LDC)

  935. 

  936. .endm

  937. 

  938. 

  939. /*******************************************************************************************/

  940. 

  941. .macro KERNEL2x2_SUB

  942. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  943. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  944. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  945. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  946. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  947. 	VFMADD231SS_(  	%xmm5,%xmm2,%xmm1 )

  948. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  949. 	VFMADD231SS_(  	%xmm7,%xmm3,%xmm1 )

  950. 	addq	$ 2 , BI	

  951. 	addq	$ 2, %rax 

  952. .endm

  953. 

  954. .macro SAVE2x2

  955. 

  956. 	vmovss	ALPHA, %xmm0

  957. 

  958. 	vmulss	%xmm0 , %xmm4 , %xmm4

  959. 	vmulss	%xmm0 , %xmm5 , %xmm5

  960. 	vmulss	%xmm0 , %xmm6 , %xmm6

  961. 	vmulss	%xmm0 , %xmm7 , %xmm7

  962. 

  963. 

  964. #if !defined(TRMMKERNEL)

  965. 

  966. 	vaddss 	        (CO1), %xmm4,%xmm4

  967. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  968. 

  969. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  970. 	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7

  971. 

  972. #endif

  973. 

  974. 	vmovss	%xmm4 ,  	(CO1)

  975. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  976. 

  977. 	vmovss	%xmm6 ,  	(CO1, LDC)

  978. 	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)

  979. 

  980. .endm

  981. 

  982. 

  983. /*******************************************************************************************/

  984. 

  985. .macro KERNEL1x2_SUB

  986. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  987. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  988. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  989. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  990. 	VFMADD231SS_(  	%xmm6,%xmm3,%xmm0 )

  991. 	addq	$ 2 , BI	

  992. 	addq	$ 1, %rax 

  993. .endm

  994. 

  995. .macro SAVE1x2

  996. 

  997. 	vmovss	ALPHA, %xmm0

  998. 

  999. 	vmulss	%xmm0 , %xmm4 , %xmm4

  1000. 	vmulss	%xmm0 , %xmm6 , %xmm6

  1001. 

  1002. #if !defined(TRMMKERNEL)

  1003. 

  1004. 	vaddss 	        (CO1), %xmm4,%xmm4

  1005. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  1006. 

  1007. #endif

  1008. 

  1009. 	vmovss	%xmm4 ,  	(CO1)

  1010. 	vmovss	%xmm6 ,  	(CO1, LDC)

  1011. 

  1012. .endm

  1013. 

  1014. 

  1015. /*******************************************************************************************/

  1016. 

  1017. /*******************************************************************************************

  1018. * 1 line of N

  1019. *******************************************************************************************/

  1020. 

  1021. .macro KERNEL16x1_SUB

  1022. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %zmm0

  1023. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %zmm2

  1024. 	VFMADD231PS_(  	%zmm4,%zmm2,%zmm0 )

  1025. 	addq	$ 1 , BI	

  1026. 	addq	$ 16, %rax 

  1027. .endm

  1028. 

  1029. .macro SAVE16x1

  1030. 

  1031. 	vbroadcastss	ALPHA, %zmm0

  1032. 

  1033. 	vmulps	%zmm0 , %zmm4 , %zmm4

  1034. 

  1035. #if !defined(TRMMKERNEL)

  1036. 

  1037. 	vaddps 	        (CO1), %zmm4,%zmm4

  1038. 

  1039. #endif

  1040. 

  1041. 	vmovups	%zmm4 ,  	(CO1)

  1042. 

  1043. .endm

  1044. 

  1045. 

  1046. /*******************************************************************************************/

  1047. 

  1048. .macro KERNEL8x1_SUB

  1049. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  1050. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  1051. 	VFMADD231PS_(  	%ymm4,%ymm2,%ymm0 )

  1052. 	addq	$ 1 , BI	

  1053. 	addq	$ 8 , %rax 

  1054. .endm

  1055. 

  1056. .macro SAVE8x1

  1057. 

  1058. 	vbroadcastss	ALPHA, %ymm0

  1059. 

  1060. 	vmulps	%ymm0 , %ymm4 , %ymm4

  1061. 

  1062. #if !defined(TRMMKERNEL)

  1063. 

  1064. 	vaddps 	        (CO1), %ymm4,%ymm4

  1065. 

  1066. #endif

  1067. 

  1068. 	vmovups	%ymm4 ,  	(CO1)

  1069. 

  1070. .endm

  1071. 

  1072. 

  1073. 

  1074. /*******************************************************************************************/

  1075. 

  1076. .macro KERNEL4x1_SUB

  1077. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  1078. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  1079. 	VFMADD231PS_(  	%xmm4,%xmm2,%xmm0 )

  1080. 	addq	$ 1 , BI	

  1081. 	addq	$ 4 , %rax 

  1082. .endm

  1083. 

  1084. .macro SAVE4x1

  1085. 

  1086. 	vbroadcastss	ALPHA, %xmm0

  1087. 

  1088. 	vmulps	%xmm0 , %xmm4 , %xmm4

  1089. 

  1090. #if !defined(TRMMKERNEL)

  1091. 

  1092. 	vaddps 	        (CO1), %xmm4,%xmm4

  1093. 

  1094. #endif

  1095. 

  1096. 	vmovups	%xmm4 ,  	(CO1)

  1097. 

  1098. .endm

  1099. 

  1100. 

  1101. /*******************************************************************************************/

  1102. 

  1103. .macro KERNEL2x1_SUB

  1104. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  1105. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  1106. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  1107. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  1108. 	VFMADD231SS_(  	%xmm5,%xmm2,%xmm1 )

  1109. 	addq	$ 1 , BI	

  1110. 	addq	$ 2 , %rax 

  1111. .endm

  1112. 

  1113. .macro SAVE2x1

  1114. 

  1115. 	vmovss	ALPHA, %xmm0

  1116. 

  1117. 	vmulss	%xmm0 , %xmm4 , %xmm4

  1118. 	vmulss	%xmm0 , %xmm5 , %xmm5

  1119. 

  1120. #if !defined(TRMMKERNEL)

  1121. 

  1122. 	vaddss 	        (CO1), %xmm4,%xmm4

  1123. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  1124. 

  1125. #endif

  1126. 

  1127. 	vmovss	%xmm4 ,  	(CO1)

  1128. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  1129. 

  1130. .endm

  1131. 

  1132. 

  1133. /*******************************************************************************************/

  1134. 

  1135. .macro KERNEL1x1_SUB

  1136. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  1137. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  1138. 	VFMADD231SS_(  	%xmm4,%xmm2,%xmm0 )

  1139. 	addq	$ 1 , BI	

  1140. 	addq	$ 1 , %rax 

  1141. .endm

  1142. 

  1143. .macro SAVE1x1

  1144. 

  1145. 	vmovss	ALPHA, %xmm0

  1146. 

  1147. 	vmulss	%xmm0 , %xmm4 , %xmm4

  1148. 

  1149. #if !defined(TRMMKERNEL)

  1150. 

  1151. 	vaddss 	        (CO1), %xmm4,%xmm4

  1152. 

  1153. #endif

  1154. 

  1155. 	vmovss	%xmm4 ,  	(CO1)

  1156. 

  1157. .endm

  1158. 

  1159. 

  1160. /*******************************************************************************************/

  1161. 

  1162. #if !defined(TRMMKERNEL)

  1163. 

  1164. /*************************************************************************************

  1165. * GEMM Kernel

  1166. *************************************************************************************/

  1167. 

  1168. 

  1169. 	PROLOGUE

  1170. 	PROFCODE

  1171. 	

  1172. 	subq	$STACKSIZE, %rsp

  1173. 	movq	%rbx,   (%rsp)

  1174. 	movq	%rbp,  8(%rsp)

  1175. 	movq	%r12, 16(%rsp)

  1176. 	movq	%r13, 24(%rsp)

  1177. 	movq	%r14, 32(%rsp)

  1178. 	movq	%r15, 40(%rsp)

  1179. 

  1180. 	vzeroupper

  1181. 

  1182. #ifdef WINDOWS_ABI

  1183. 	movq	%rdi,    48(%rsp)

  1184. 	movq	%rsi,    56(%rsp)

  1185. 	movups	%xmm6,   64(%rsp)

  1186. 	movups	%xmm7,   80(%rsp)

  1187. 	movups	%xmm8,   96(%rsp)

  1188. 	movups	%xmm9,  112(%rsp)

  1189. 	movups	%xmm10, 128(%rsp)

  1190. 	movups	%xmm11, 144(%rsp)

  1191. 	movups	%xmm12, 160(%rsp)

  1192. 	movups	%xmm13, 176(%rsp)

  1193. 	movups	%xmm14, 192(%rsp)

  1194. 	movups	%xmm15, 208(%rsp)

  1195. 

  1196. 	movq	ARG1,      OLD_M

  1197. 	movq	ARG2,      OLD_N

  1198. 	movq	ARG3,      OLD_K

  1199. 	movq	OLD_A,     A

  1200. 	movq	OLD_B,     B

  1201. 	movq	OLD_C,     C

  1202. 	movq	OLD_LDC,   LDC

  1203. #ifdef TRMMKERNEL

  1204. 	vmovsd	OLD_OFFSET, %xmm12

  1205. #endif

  1206. 	vmovaps	%xmm3, %xmm0

  1207. 

  1208. #else

  1209. 	movq	STACKSIZE +  8(%rsp), LDC

  1210. #ifdef TRMMKERNEL

  1211. 	movsd	STACKSIZE + 16(%rsp), %xmm12

  1212. #endif

  1213. 

  1214. #endif

  1215. 

  1216. 	movq    %rsp, SP      # save old stack

  1217.         subq    $128 + L_BUFFER_SIZE, %rsp

  1218.         andq    $-4096, %rsp    # align stack

  1219. 

  1220.         STACK_TOUCH

  1221. 

  1222. 	cmpq	$0, OLD_M

  1223. 	je	.L999

  1224. 

  1225. 	cmpq	$0, OLD_N

  1226. 	je	.L999

  1227. 

  1228. 	cmpq	$0, OLD_K

  1229. 	je	.L999

  1230. 

  1231. 	movq	OLD_M, M

  1232. 	movq	OLD_N, N

  1233. 	movq	OLD_K, K

  1234. 

  1235. 	vmovss	 %xmm0, ALPHA

  1236. 

  1237. 	salq	$BASE_SHIFT, LDC

  1238. 

  1239. 	movq    N, %rax

  1240.         xorq    %rdx, %rdx

  1241.         movq    $12,  %rdi

  1242.         divq    %rdi                    //    N / 12

  1243.         movq    %rax, Ndiv6             //    N / 12

  1244.         movq    %rdx, Nmod6             //    N % 12

  1245. 

  1246. 	movq	Ndiv6,  J

  1247. 	cmpq	$0, J

  1248. 	je	.L4_00

  1249. 	ALIGN_4

  1250. 

  1251. 

  1252. /*******************************************************************************************/

  1253. 

  1254. .L6_01:

  1255. 	// copy to sub buffer

  1256. 	movq	B, BO1

  1257. 	leaq    BUFFER1, BO		// first buffer to BO

  1258. 	movq	K, %rax

  1259. 	salq	$2, %rax		// 4 values of B

  1260.         leaq    (B, %rax,4), BO2

  1261.         movq    BO2, B                  // next offset of B

  1262.         movq    K, %rax

  1263. 

  1264. 	ALIGN_4

  1265. 

  1266. 

  1267. .L6_02c:

  1268. 

  1269. 	vmovups	(BO1), %xmm0

  1270. 	vmovsd	(BO2), %xmm1

  1271. 	vmovups	%xmm0, (BO)

  1272. 	vmovsd	%xmm1, 4*SIZE(BO)

  1273. 	addq	$ 4*SIZE,BO1

  1274. 	addq	$ 4*SIZE,BO2

  1275. 	addq	$ 6*SIZE,BO

  1276. 	decq	%rax

  1277. 	jnz	.L6_02c

  1278. 

  1279. 

  1280. .L6_10:

  1281. 	movq	 C, CO1

  1282. 	leaq	(C,   LDC, 2), CO2	

  1283. 	leaq	(CO2, LDC, 1), CO2		// co2 = c + 3 * ldc

  1284. 	leaq	(C,   LDC, 4), C	

  1285. 	leaq	(C,   LDC, 2), C		// c = c + 6 * ldc

  1286. 

  1287. 	movq	A, AO		 	// aoffset = a

  1288. 	addq	$ 16 * SIZE, AO

  1289. 

  1290. 	movq	M,  I

  1291. 	sarq	$4, I			// i = (m >> 4)

  1292. 	je	.L6_20

  1293. 

  1294. 	ALIGN_4

  1295. 

  1296. .L6_11:

  1297.         leaq    BUFFER1, BO             // first buffer to BO

  1298.         addq    $4 * SIZE, BO

  1299. 

  1300. 	vzeroall

  1301. 

  1302.         movq    K, %rax

  1303. 

  1304. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  1305. 	je	.L6_16

  1306. 

  1307. 	ALIGN_4

  1308. 

  1309. .L6_12:

  1310. 

  1311. 	KERNEL16x6_SUB4

  1312. 

  1313. 	KERNEL16x6_SUB4

  1314. 

  1315. 	je	.L6_16

  1316. 

  1317. 	KERNEL16x6_SUB4

  1318. 

  1319. 	KERNEL16x6_SUB4

  1320. 

  1321. 	je	.L6_16

  1322. 

  1323. 	jmp	.L6_12

  1324. 	ALIGN_4

  1325. 

  1326. .L6_16:

  1327.         movq    K, %rax

  1328. 

  1329. 	andq	$7, %rax		# if (k & 1)

  1330. 	je .L6_19

  1331. 

  1332. 	ALIGN_4

  1333. 

  1334. .L6_17:

  1335. 

  1336. 	KERNEL16x6_SUB

  1337. 

  1338. 	jnz	.L6_17

  1339. 	ALIGN_4

  1340. 

  1341. 

  1342. .L6_19:

  1343. 

  1344. 	SAVE16x6

  1345. 

  1346. 	addq	$16 * SIZE, CO1		# coffset += 16

  1347. 	addq	$16 * SIZE, CO2		# coffset += 16

  1348. 	decq	I			# i --

  1349. 	jg	.L6_11

  1350. 	ALIGN_4	

  1351. 

  1352. /**************************************************************************

  1353. * Rest of M 

  1354. ***************************************************************************/

  1355. .L6_20:

  1356. 	// Test rest of M

  1357. 

  1358. 	testq	$15, M

  1359. 	jz	.L6_60		// to next 6 lines of N

  1360. 

  1361. 	testq	$8, M		

  1362. 	jz	.L6_21pre

  1363. 	ALIGN_4

  1364. 

  1365. /**************************************************************************/

  1366. 

  1367. .L6_20_1:

  1368.         leaq    BUFFER1, BO             // first buffer to BO

  1369.         addq    $4 * SIZE, BO

  1370. 

  1371. 	vzeroall

  1372. 

  1373.         movq    K, %rax

  1374. 

  1375. 	andq	$-8, %rax

  1376. 	je	.L6_20_6

  1377. 

  1378. 	ALIGN_4

  1379. 

  1380. .L6_20_2:

  1381. 

  1382. 	prefetcht0	A_PR1(AO)

  1383. 	KERNEL8x6_SUB

  1384. 	KERNEL8x6_SUB

  1385. 	prefetcht0	A_PR1(AO)

  1386. 	KERNEL8x6_SUB

  1387. 	KERNEL8x6_SUB

  1388. 

  1389. 	prefetcht0	A_PR1(AO)

  1390. 	KERNEL8x6_SUB

  1391. 	KERNEL8x6_SUB

  1392. 	prefetcht0	A_PR1(AO)

  1393. 	KERNEL8x6_SUB

  1394. 	KERNEL8x6_SUB

  1395. 

  1396. 	je	.L6_20_6

  1397. 

  1398. 	prefetcht0	A_PR1(AO)

  1399. 	KERNEL8x6_SUB

  1400. 	KERNEL8x6_SUB

  1401. 	prefetcht0	A_PR1(AO)

  1402. 	KERNEL8x6_SUB

  1403. 	KERNEL8x6_SUB

  1404. 

  1405. 	prefetcht0	A_PR1(AO)

  1406. 	KERNEL8x6_SUB

  1407. 	KERNEL8x6_SUB

  1408. 	prefetcht0	A_PR1(AO)

  1409. 	KERNEL8x6_SUB

  1410. 	KERNEL8x6_SUB

  1411. 

  1412. 	je	.L6_20_6

  1413. 

  1414. 	jmp	.L6_20_2

  1415. 	ALIGN_4

  1416. 

  1417. .L6_20_6:

  1418.         movq    K, %rax

  1419. 

  1420. 	andq	$7, %rax		# if (k & 1)

  1421. 	je .L6_20_9

  1422. 

  1423. 	ALIGN_4

  1424. 

  1425. .L6_20_7:

  1426. 

  1427. 	KERNEL8x6_SUB

  1428. 

  1429. 	jnz	.L6_20_7

  1430. 	ALIGN_4

  1431. 

  1432. 

  1433. .L6_20_9:

  1434. 

  1435. 	SAVE8x6

  1436. 

  1437. 	addq	$8 * SIZE, CO1		# coffset += 8

  1438. 	addq	$8 * SIZE, CO2		# coffset += 8

  1439. 	ALIGN_4

  1440. 	

  1441. 

  1442. 

  1443. /**************************************************************************/

  1444. 

  1445. .L6_21pre:

  1446. 

  1447. 	testq	$4, M		

  1448. 	jz	.L6_30

  1449. 	ALIGN_4

  1450. 

  1451. .L6_21:

  1452.         leaq    BUFFER1, BO             // first buffer to BO

  1453.         addq    $4 * SIZE, BO

  1454. 

  1455. 	vzeroall

  1456. 

  1457.         movq    K, %rax

  1458. 

  1459. 	andq	$-8, %rax

  1460. 	je	.L6_26

  1461. 

  1462. 	ALIGN_4

  1463. 

  1464. .L6_22:

  1465. 

  1466. 	prefetcht0	A_PR1(AO)

  1467. 	KERNEL4x6_SUB

  1468. 	KERNEL4x6_SUB

  1469. 	KERNEL4x6_SUB

  1470. 	KERNEL4x6_SUB

  1471. 

  1472. 	prefetcht0	A_PR1(AO)

  1473. 	KERNEL4x6_SUB

  1474. 	KERNEL4x6_SUB

  1475. 	KERNEL4x6_SUB

  1476. 	KERNEL4x6_SUB

  1477. 

  1478. 	je	.L6_26

  1479. 

  1480. 	prefetcht0	A_PR1(AO)

  1481. 	KERNEL4x6_SUB

  1482. 	KERNEL4x6_SUB

  1483. 	KERNEL4x6_SUB

  1484. 	KERNEL4x6_SUB

  1485. 

  1486. 	prefetcht0	A_PR1(AO)

  1487. 	KERNEL4x6_SUB

  1488. 	KERNEL4x6_SUB

  1489. 	KERNEL4x6_SUB

  1490. 	KERNEL4x6_SUB

  1491. 

  1492. 	je	.L6_26

  1493. 

  1494. 	jmp	.L6_22

  1495. 	ALIGN_4

  1496. 

  1497. .L6_26:

  1498.         movq    K, %rax

  1499. 

  1500. 	andq	$7, %rax		# if (k & 1)

  1501. 	je .L6_29

  1502. 

  1503. 	ALIGN_4

  1504. 

  1505. .L6_27:

  1506. 

  1507. 	KERNEL4x6_SUB

  1508. 

  1509. 	jnz	.L6_27

  1510. 	ALIGN_4

  1511. 

  1512. 

  1513. .L6_29:

  1514. 

  1515. 	SAVE4x6

  1516. 

  1517. 	addq	$4 * SIZE, CO1		# coffset += 4

  1518. 	addq	$4 * SIZE, CO2		# coffset += 4

  1519. 	ALIGN_4

  1520. 	

  1521. 

  1522. .L6_30:

  1523. 	testq	$2, M		

  1524. 	jz	.L6_40

  1525. 

  1526. 	ALIGN_4

  1527. 

  1528. .L6_31:

  1529.         leaq    BUFFER1, BO             // first buffer to BO

  1530.         addq    $4 * SIZE, BO

  1531. 

  1532. 	vzeroall

  1533. 

  1534.         movq    K, %rax

  1535. 

  1536. 	andq	$-8, %rax

  1537. 	je	.L6_36

  1538. 

  1539. 	ALIGN_4

  1540. 

  1541. .L6_32:

  1542. 

  1543. 	prefetcht0	A_PR1(AO)

  1544. 	KERNEL2x6_SUB

  1545. 	KERNEL2x6_SUB

  1546. 	KERNEL2x6_SUB

  1547. 	KERNEL2x6_SUB

  1548. 

  1549. 	KERNEL2x6_SUB

  1550. 	KERNEL2x6_SUB

  1551. 	KERNEL2x6_SUB

  1552. 	KERNEL2x6_SUB

  1553. 

  1554. 	je	.L6_36

  1555. 

  1556. 	prefetcht0	A_PR1(AO)

  1557. 	KERNEL2x6_SUB

  1558. 	KERNEL2x6_SUB

  1559. 	KERNEL2x6_SUB

  1560. 	KERNEL2x6_SUB

  1561. 

  1562. 	KERNEL2x6_SUB

  1563. 	KERNEL2x6_SUB

  1564. 	KERNEL2x6_SUB

  1565. 	KERNEL2x6_SUB

  1566. 

  1567. 	je	.L6_36

  1568. 

  1569. 	jmp	.L6_32

  1570. 	ALIGN_4

  1571. 

  1572. .L6_36:

  1573.         movq    K, %rax

  1574. 

  1575. 	andq	$7, %rax		# if (k & 1)

  1576. 	je .L6_39

  1577. 

  1578. 	ALIGN_4

  1579. 

  1580. .L6_37:

  1581. 

  1582. 	KERNEL2x6_SUB

  1583. 

  1584. 	jnz	.L6_37

  1585. 	ALIGN_4

  1586. 

  1587. 

  1588. .L6_39:

  1589. 

  1590. 	SAVE2x6

  1591. 

  1592. 	addq	$2 * SIZE, CO1		# coffset += 2

  1593. 	addq	$2 * SIZE, CO2		# coffset += 2

  1594. 	ALIGN_4

  1595. 

  1596. .L6_40:

  1597. 	testq	$1, M		

  1598. 	jz	.L6_60		// to next 4 lines of N

  1599. 

  1600. 	ALIGN_4

  1601. 

  1602. .L6_41:

  1603.         leaq    BUFFER1, BO             // first buffer to BO

  1604.         addq    $4 * SIZE, BO

  1605. 

  1606. 	vzeroall

  1607. 

  1608.         movq    K, %rax

  1609. 

  1610. 	andq	$-8, %rax

  1611. 	je	.L6_46

  1612. 

  1613. 	ALIGN_4

  1614. 

  1615. .L6_42:

  1616. 

  1617. 	prefetcht0	A_PR1(AO)

  1618. 	KERNEL1x6_SUB

  1619. 	KERNEL1x6_SUB

  1620. 	KERNEL1x6_SUB

  1621. 	KERNEL1x6_SUB

  1622. 

  1623. 	KERNEL1x6_SUB

  1624. 	KERNEL1x6_SUB

  1625. 	KERNEL1x6_SUB

  1626. 	KERNEL1x6_SUB

  1627. 

  1628. 	je	.L6_46

  1629. 

  1630. 	KERNEL1x6_SUB

  1631. 	KERNEL1x6_SUB

  1632. 	KERNEL1x6_SUB

  1633. 	KERNEL1x6_SUB

  1634. 

  1635. 	KERNEL1x6_SUB

  1636. 	KERNEL1x6_SUB

  1637. 	KERNEL1x6_SUB

  1638. 	KERNEL1x6_SUB

  1639. 

  1640. 	je	.L6_46

  1641. 

  1642. 	jmp	.L6_42

  1643. 	ALIGN_4

  1644. 

  1645. .L6_46:

  1646.         movq    K, %rax

  1647. 

  1648. 	andq	$7, %rax		# if (k & 1)

  1649. 	je .L6_49

  1650. 

  1651. 	ALIGN_4

  1652. 

  1653. .L6_47:

  1654. 

  1655. 	KERNEL1x6_SUB

  1656. 

  1657. 	jnz	.L6_47

  1658. 	ALIGN_4

  1659. 

  1660. 

  1661. .L6_49:

  1662. 

  1663. 	SAVE1x6

  1664. 

  1665. 	addq	$1 * SIZE, CO1		# coffset += 1

  1666. 	addq	$1 * SIZE, CO2		# coffset += 1

  1667. 	ALIGN_4

  1668. 	

  1669. 

  1670. 

  1671. 

  1672. 	

  1673. .L6_60:

  1674. 

  1675. 

  1676. /*******************************************************************************************/

  1677. 

  1678. 

  1679. .L7_01:

  1680. 	// copy to sub buffer

  1681. 	movq	B, BO1

  1682. 	leaq    BUFFER1, BO		// first buffer to BO

  1683. 	movq	K, %rax

  1684. 	salq	$2, %rax		// 4 values of B

  1685.         leaq    (B, %rax,4), BO2

  1686.         movq    K, %rax

  1687. 

  1688. 	ALIGN_4

  1689. 

  1690. 

  1691. .L7_02c:

  1692. 

  1693. 	vmovsd	2*SIZE(BO1), %xmm0

  1694. 	vmovups	      (BO2), %xmm1

  1695. 	vmovsd	%xmm0, (BO)

  1696. 	vmovups	%xmm1, 2*SIZE(BO)

  1697. 	addq	$ 4*SIZE,BO1

  1698. 	addq	$ 4*SIZE,BO2

  1699. 	addq	$ 6*SIZE,BO

  1700. 	decq	%rax

  1701. 	jnz	.L7_02c

  1702. 

  1703.         movq    BO2, B                  // next offset of B

  1704. 

  1705. .L7_10:

  1706. 	movq	 C, CO1

  1707. 	leaq	(C,   LDC, 2), CO2	

  1708. 	leaq	(CO2, LDC, 1), CO2		// co2 = c + 3 * ldc

  1709. 	leaq	(C,   LDC, 4), C	

  1710. 	leaq	(C,   LDC, 2), C		// c = c + 6 * ldc

  1711. 

  1712. 	movq	A, AO		 	// aoffset = a

  1713. 	addq	$ 16 * SIZE, AO

  1714. 

  1715. 	movq	M,  I

  1716. 	sarq	$4, I			// i = (m >> 4)

  1717. 	je	.L7_20

  1718. 

  1719. 	ALIGN_4

  1720. 

  1721. .L7_11:

  1722.         leaq    BUFFER1, BO             // first buffer to BO

  1723.         addq    $4 * SIZE, BO

  1724. 

  1725. 	vzeroall

  1726. 

  1727.         movq    K, %rax

  1728. 

  1729. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  1730. 	je	.L7_16

  1731. 

  1732. 	ALIGN_4

  1733. 

  1734. .L7_12:

  1735. 

  1736. 	KERNEL16x6_SUB4

  1737. 

  1738. 	KERNEL16x6_SUB4

  1739. 

  1740. 	je	.L7_16

  1741. 

  1742. 	KERNEL16x6_SUB4

  1743. 

  1744. 	KERNEL16x6_SUB4

  1745. 

  1746. 	je	.L7_16

  1747. 

  1748. 	jmp	.L7_12

  1749. 	ALIGN_4

  1750. 

  1751. .L7_16:

  1752.         movq    K, %rax

  1753. 

  1754. 	andq	$7, %rax		# if (k & 1)

  1755. 	je .L7_19

  1756. 

  1757. 	ALIGN_4

  1758. 

  1759. .L7_17:

  1760. 

  1761. 	KERNEL16x6_SUB

  1762. 

  1763. 	jnz	.L7_17

  1764. 	ALIGN_4

  1765. 

  1766. 

  1767. .L7_19:

  1768. 

  1769. 	SAVE16x6

  1770. 

  1771. 	addq	$16 * SIZE, CO1		# coffset += 16

  1772. 	addq	$16 * SIZE, CO2		# coffset += 16

  1773. 	decq	I			# i --

  1774. 	jg	.L7_11

  1775. 	ALIGN_4	

  1776. 

  1777. /**************************************************************************

  1778. * Rest of M 

  1779. ***************************************************************************/

  1780. .L7_20:

  1781. 	// Test rest of M

  1782. 

  1783. 	testq	$15, M

  1784. 	jz	.L7_60		// to next 6 lines of N

  1785. 

  1786. 	testq	$8, M		

  1787. 	jz	.L7_21pre

  1788. 	ALIGN_4

  1789. 

  1790. /**************************************************************************/

  1791. 

  1792. .L7_20_1:

  1793.         leaq    BUFFER1, BO             // first buffer to BO

  1794.         addq    $4 * SIZE, BO

  1795. 

  1796. 	vzeroall

  1797. 

  1798.         movq    K, %rax

  1799. 

  1800. 	andq	$-8, %rax

  1801. 	je	.L7_20_6

  1802. 

  1803. 	ALIGN_4

  1804. 

  1805. .L7_20_2:

  1806. 

  1807. 	prefetcht0	A_PR1(AO)

  1808. 	KERNEL8x6_SUB

  1809. 	KERNEL8x6_SUB

  1810. 	prefetcht0	A_PR1(AO)

  1811. 	KERNEL8x6_SUB

  1812. 	KERNEL8x6_SUB

  1813. 

  1814. 	prefetcht0	A_PR1(AO)

  1815. 	KERNEL8x6_SUB

  1816. 	KERNEL8x6_SUB

  1817. 	prefetcht0	A_PR1(AO)

  1818. 	KERNEL8x6_SUB

  1819. 	KERNEL8x6_SUB

  1820. 

  1821. 	je	.L7_20_6

  1822. 

  1823. 	prefetcht0	A_PR1(AO)

  1824. 	KERNEL8x6_SUB

  1825. 	KERNEL8x6_SUB

  1826. 	prefetcht0	A_PR1(AO)

  1827. 	KERNEL8x6_SUB

  1828. 	KERNEL8x6_SUB

  1829. 

  1830. 	prefetcht0	A_PR1(AO)

  1831. 	KERNEL8x6_SUB

  1832. 	KERNEL8x6_SUB

  1833. 	prefetcht0	A_PR1(AO)

  1834. 	KERNEL8x6_SUB

  1835. 	KERNEL8x6_SUB

  1836. 

  1837. 	je	.L7_20_6

  1838. 

  1839. 	jmp	.L7_20_2

  1840. 	ALIGN_4

  1841. 

  1842. .L7_20_6:

  1843.         movq    K, %rax

  1844. 

  1845. 	andq	$7, %rax		# if (k & 1)

  1846. 	je .L7_20_9

  1847. 

  1848. 	ALIGN_4

  1849. 

  1850. .L7_20_7:

  1851. 

  1852. 	KERNEL8x6_SUB

  1853. 

  1854. 	jnz	.L7_20_7

  1855. 	ALIGN_4

  1856. 

  1857. 

  1858. .L7_20_9:

  1859. 

  1860. 	SAVE8x6

  1861. 

  1862. 	addq	$8 * SIZE, CO1		# coffset += 8

  1863. 	addq	$8 * SIZE, CO2		# coffset += 8

  1864. 	ALIGN_4

  1865. 	

  1866. 

  1867. 

  1868. /**************************************************************************/

  1869. 

  1870. .L7_21pre:

  1871. 

  1872. 	testq	$4, M		

  1873. 	jz	.L7_30

  1874. 	ALIGN_4

  1875. 

  1876. .L7_21:

  1877.         leaq    BUFFER1, BO             // first buffer to BO

  1878.         addq    $4 * SIZE, BO

  1879. 

  1880. 	vzeroall

  1881. 

  1882.         movq    K, %rax

  1883. 

  1884. 	andq	$-8, %rax

  1885. 	je	.L7_26

  1886. 

  1887. 	ALIGN_4

  1888. 

  1889. .L7_22:

  1890. 

  1891. 	prefetcht0	A_PR1(AO)

  1892. 	KERNEL4x6_SUB

  1893. 	KERNEL4x6_SUB

  1894. 	KERNEL4x6_SUB

  1895. 	KERNEL4x6_SUB

  1896. 

  1897. 	prefetcht0	A_PR1(AO)

  1898. 	KERNEL4x6_SUB

  1899. 	KERNEL4x6_SUB

  1900. 	KERNEL4x6_SUB

  1901. 	KERNEL4x6_SUB

  1902. 

  1903. 	je	.L7_26

  1904. 

  1905. 	prefetcht0	A_PR1(AO)

  1906. 	KERNEL4x6_SUB

  1907. 	KERNEL4x6_SUB

  1908. 	KERNEL4x6_SUB

  1909. 	KERNEL4x6_SUB

  1910. 

  1911. 	prefetcht0	A_PR1(AO)

  1912. 	KERNEL4x6_SUB

  1913. 	KERNEL4x6_SUB

  1914. 	KERNEL4x6_SUB

  1915. 	KERNEL4x6_SUB

  1916. 

  1917. 	je	.L7_26

  1918. 

  1919. 	jmp	.L7_22

  1920. 	ALIGN_4

  1921. 

  1922. .L7_26:

  1923.         movq    K, %rax

  1924. 

  1925. 	andq	$7, %rax		# if (k & 1)

  1926. 	je .L7_29

  1927. 

  1928. 	ALIGN_4

  1929. 

  1930. .L7_27:

  1931. 

  1932. 	KERNEL4x6_SUB

  1933. 

  1934. 	jnz	.L7_27

  1935. 	ALIGN_4

  1936. 

  1937. 

  1938. .L7_29:

  1939. 

  1940. 	SAVE4x6

  1941. 

  1942. 	addq	$4 * SIZE, CO1		# coffset += 4

  1943. 	addq	$4 * SIZE, CO2		# coffset += 4

  1944. 	ALIGN_4

  1945. 	

  1946. 

  1947. .L7_30:

  1948. 	testq	$2, M		

  1949. 	jz	.L7_40

  1950. 

  1951. 	ALIGN_4

  1952. 

  1953. .L7_31:

  1954.         leaq    BUFFER1, BO             // first buffer to BO

  1955.         addq    $4 * SIZE, BO

  1956. 

  1957. 	vzeroall

  1958. 

  1959.         movq    K, %rax

  1960. 

  1961. 	andq	$-8, %rax

  1962. 	je	.L7_36

  1963. 

  1964. 	ALIGN_4

  1965. 

  1966. .L7_32:

  1967. 

  1968. 	prefetcht0	A_PR1(AO)

  1969. 	KERNEL2x6_SUB

  1970. 	KERNEL2x6_SUB

  1971. 	KERNEL2x6_SUB

  1972. 	KERNEL2x6_SUB

  1973. 

  1974. 	KERNEL2x6_SUB

  1975. 	KERNEL2x6_SUB

  1976. 	KERNEL2x6_SUB

  1977. 	KERNEL2x6_SUB

  1978. 

  1979. 	je	.L7_36

  1980. 

  1981. 	prefetcht0	A_PR1(AO)

  1982. 	KERNEL2x6_SUB

  1983. 	KERNEL2x6_SUB

  1984. 	KERNEL2x6_SUB

  1985. 	KERNEL2x6_SUB

  1986. 

  1987. 	KERNEL2x6_SUB

  1988. 	KERNEL2x6_SUB

  1989. 	KERNEL2x6_SUB

  1990. 	KERNEL2x6_SUB

  1991. 

  1992. 	je	.L7_36

  1993. 

  1994. 	jmp	.L7_32

  1995. 	ALIGN_4

  1996. 

  1997. .L7_36:

  1998.         movq    K, %rax

  1999. 

  2000. 	andq	$7, %rax		# if (k & 1)

  2001. 	je .L7_39

  2002. 

  2003. 	ALIGN_4

  2004. 

  2005. .L7_37:

  2006. 

  2007. 	KERNEL2x6_SUB

  2008. 

  2009. 	jnz	.L7_37

  2010. 	ALIGN_4

  2011. 

  2012. 

  2013. .L7_39:

  2014. 

  2015. 	SAVE2x6

  2016. 

  2017. 	addq	$2 * SIZE, CO1		# coffset += 2

  2018. 	addq	$2 * SIZE, CO2		# coffset += 2

  2019. 	ALIGN_4

  2020. 

  2021. .L7_40:

  2022. 	testq	$1, M		

  2023. 	jz	.L7_60		// to next 4 lines of N

  2024. 

  2025. 	ALIGN_4

  2026. 

  2027. .L7_41:

  2028.         leaq    BUFFER1, BO             // first buffer to BO

  2029.         addq    $4 * SIZE, BO

  2030. 

  2031. 	vzeroall

  2032. 

  2033.         movq    K, %rax

  2034. 

  2035. 	andq	$-8, %rax

  2036. 	je	.L7_46

  2037. 

  2038. 	ALIGN_4

  2039. 

  2040. .L7_42:

  2041. 

  2042. 	prefetcht0	A_PR1(AO)

  2043. 	KERNEL1x6_SUB

  2044. 	KERNEL1x6_SUB

  2045. 	KERNEL1x6_SUB

  2046. 	KERNEL1x6_SUB

  2047. 

  2048. 	KERNEL1x6_SUB

  2049. 	KERNEL1x6_SUB

  2050. 	KERNEL1x6_SUB

  2051. 	KERNEL1x6_SUB

  2052. 

  2053. 	je	.L7_46

  2054. 

  2055. 	KERNEL1x6_SUB

  2056. 	KERNEL1x6_SUB

  2057. 	KERNEL1x6_SUB

  2058. 	KERNEL1x6_SUB

  2059. 

  2060. 	KERNEL1x6_SUB

  2061. 	KERNEL1x6_SUB

  2062. 	KERNEL1x6_SUB

  2063. 	KERNEL1x6_SUB

  2064. 

  2065. 	je	.L7_46

  2066. 

  2067. 	jmp	.L7_42

  2068. 	ALIGN_4

  2069. 

  2070. .L7_46:

  2071.         movq    K, %rax

  2072. 

  2073. 	andq	$7, %rax		# if (k & 1)

  2074. 	je .L7_49

  2075. 

  2076. 	ALIGN_4

  2077. 

  2078. .L7_47:

  2079. 

  2080. 	KERNEL1x6_SUB

  2081. 

  2082. 	jnz	.L7_47

  2083. 	ALIGN_4

  2084. 

  2085. 

  2086. .L7_49:

  2087. 

  2088. 	SAVE1x6

  2089. 

  2090. 	addq	$1 * SIZE, CO1		# coffset += 1

  2091. 	addq	$1 * SIZE, CO2		# coffset += 1

  2092. 	ALIGN_4

  2093. 	

  2094. 

  2095. 

  2096. 

  2097. 	

  2098. .L7_60:

  2099. 

  2100. 	decq	J			// j --

  2101. 	jg	.L6_01			// next 12 lines of N

  2102. 

  2103. 

  2104. 

  2105. 

  2106. /*******************************************************************************************/

  2107. .L4_00:

  2108. 

  2109.  	movq    Nmod6,  J

  2110.         sarq    $2, J           // j = j / 4

  2111.         cmpq    $ 0, J

  2112.         je      .L2_00

  2113.         ALIGN_4

  2114. 

  2115. 

  2116. .L4_01:

  2117. 	// copy to sub buffer

  2118. 	movq	B, BO1

  2119. 	leaq    BUFFER1, BO		// first buffer to BO

  2120. 	movq	K, %rax

  2121. 	sarq	$2, %rax		// K / 4

  2122. 	jz	.L4_01b

  2123. 	ALIGN_4

  2124. 

  2125. 

  2126. .L4_01a:

  2127.         prefetcht0 512(BO1)

  2128.         prefetchw  512(BO)

  2129. 

  2130. 	vmovups	       (BO1), %xmm0

  2131. 	vmovups	 4*SIZE(BO1), %xmm1

  2132. 	vmovups	 8*SIZE(BO1), %xmm2

  2133. 	vmovups	12*SIZE(BO1), %xmm3

  2134. 

  2135. 	vmovups	%xmm0,       (BO)

  2136. 	vmovups	%xmm1, 4*SIZE(BO)

  2137. 	vmovups	%xmm2, 8*SIZE(BO)

  2138. 	vmovups	%xmm3,12*SIZE(BO)

  2139. 

  2140. 	addq	$ 16*SIZE,BO1

  2141. 	addq	$ 16*SIZE,BO

  2142. 	decq	%rax

  2143. 	jnz	.L4_01a

  2144. 

  2145. 

  2146. .L4_01b:

  2147. 

  2148.         movq    K, %rax

  2149.         andq    $3, %rax                // K % 4

  2150.         jz      .L4_02d

  2151.         ALIGN_4

  2152. 

  2153. .L4_02c:

  2154. 

  2155. 	vmovups	(BO1), %xmm0

  2156. 	vmovups	%xmm0, (BO)

  2157. 	addq	$ 4*SIZE,BO1

  2158. 	addq	$ 4*SIZE,BO

  2159. 	decq	%rax

  2160. 	jnz	.L4_02c

  2161. 

  2162. .L4_02d:

  2163. 

  2164. 	movq	BO1, B			// next offset of B

  2165. 

  2166. .L4_10:

  2167. 	movq	 C, CO1

  2168. 	leaq	(C, LDC, 2), CO2	

  2169. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  2170. 

  2171. #if defined(TRMMKERNEL) && defined(LEFT)

  2172.         movq    OFFSET, %rax

  2173.         movq    %rax, KK

  2174. #endif

  2175. 	

  2176. 	movq	A, AO		 	// aoffset = a

  2177. 	addq	$ 16 * SIZE, AO

  2178. 

  2179. 	movq	M,  I

  2180. 	sarq	$4, I			// i = (m >> 4)

  2181. 	je	.L4_20

  2182. 

  2183. 	ALIGN_4

  2184. 

  2185. .L4_11:

  2186. #if !defined(TRMMKERNEL) || \

  2187.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2188.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2189.         leaq    BUFFER1, BO             // first buffer to BO

  2190.         addq    $4 * SIZE, BO

  2191. #else

  2192.         movq    KK, %rax

  2193.         leaq    BUFFER1, BO             	// first buffer to BO

  2194.         addq    $4 * SIZE, BO

  2195.         movq    %rax, BI                        //  Index for BO

  2196. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2197.         leaq    (BO, BI, SIZE), BO

  2198.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  2199.         leaq    (AO, %rax, SIZE), AO

  2200. #endif

  2201. 

  2202. 

  2203. 	vzeroall

  2204. 

  2205. #ifndef TRMMKERNEL

  2206.         movq    K, %rax

  2207. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2208.         movq    K, %rax

  2209.         subq    KK, %rax

  2210.         movq    %rax, KKK

  2211. #else

  2212.         movq    KK, %rax

  2213. #ifdef LEFT

  2214.         addq    $16, %rax	// number of values in AO

  2215. #else

  2216.         addq    $4, %rax	// number of values in BO

  2217. #endif

  2218.         movq    %rax, KKK

  2219. #endif

  2220. 

  2221. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  2222. 	je	.L4_16

  2223. 	movq    %rax, BI                        //  Index for BO

  2224.         leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values

  2225. 

  2226. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  2227. 	leaq	(AO, %rax, SIZE), AO

  2228. 	leaq	(BO, BI, SIZE), BO

  2229. 	negq	BI

  2230. 	negq	%rax

  2231. 	ALIGN_4

  2232. 

  2233. .L4_12:

  2234. 

  2235. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2236. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  2237. 	KERNEL16x4_SUB

  2238. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2239. 	KERNEL16x4_SUB

  2240. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2241. 	KERNEL16x4_SUB

  2242. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2243. 	KERNEL16x4_SUB

  2244. 

  2245. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2246. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  2247. 	KERNEL16x4_SUB

  2248. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2249. 	KERNEL16x4_SUB

  2250. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2251. 	KERNEL16x4_SUB

  2252. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2253. 	KERNEL16x4_SUB

  2254. 

  2255. 	je	.L4_16

  2256. 

  2257. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2258. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  2259. 	KERNEL16x4_SUB

  2260. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2261. 	KERNEL16x4_SUB

  2262. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2263. 	KERNEL16x4_SUB

  2264. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2265. 	KERNEL16x4_SUB

  2266. 

  2267. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2268. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  2269. 	KERNEL16x4_SUB

  2270. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2271. 	KERNEL16x4_SUB

  2272. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2273. 	KERNEL16x4_SUB

  2274. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  2275. 	KERNEL16x4_SUB

  2276. 

  2277. 	je	.L4_16

  2278. 

  2279. 	jmp	.L4_12

  2280. 	ALIGN_4

  2281. 

  2282. .L4_16:

  2283. #ifndef TRMMKERNEL

  2284.         movq    K, %rax

  2285. #else

  2286.         movq    KKK, %rax

  2287. #endif

  2288. 

  2289. 	andq	$7, %rax		# if (k & 1)

  2290. 	je .L4_19

  2291. 

  2292. 	movq    %rax, BI                        //  Index for BO

  2293.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2294. 

  2295. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  2296. 	leaq	(AO, %rax, SIZE), AO

  2297. 	leaq	(BO, BI, SIZE), BO

  2298. 	negq	BI

  2299. 	negq	%rax

  2300. 	ALIGN_4

  2301. 

  2302. .L4_17:

  2303. 

  2304. 	KERNEL16x4_SUB

  2305. 

  2306. 	jl	.L4_17

  2307. 	ALIGN_4

  2308. 

  2309. 

  2310. .L4_19:

  2311. 

  2312. 	SAVE16x4

  2313. 

  2314. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2315.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2316.         movq    K, %rax

  2317.         subq    KKK, %rax

  2318.         movq    %rax, BI                        //  Index for BO

  2319. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2320.         leaq    (BO, BI, SIZE), BO

  2321.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  2322.         leaq    (AO, %rax, SIZE), AO

  2323. #endif

  2324. 

  2325. 

  2326. #if defined(TRMMKERNEL) && defined(LEFT)

  2327.         addq    $16, KK				

  2328. #endif

  2329. 

  2330. 	addq	$16 * SIZE, CO1		# coffset += 16

  2331. 	addq	$16 * SIZE, CO2		# coffset += 16

  2332. 	decq	I			# i --

  2333. 	jg	.L4_11

  2334. 	ALIGN_4	

  2335. 

  2336. /**************************************************************************

  2337. * Rest of M 

  2338. ***************************************************************************/

  2339. .L4_20:

  2340. 	// Test rest of M

  2341. 

  2342. 	testq	$15, M

  2343. 	jz	.L4_60		// to next 3 lines of N

  2344. 

  2345. 	testq	$8, M		

  2346. 	jz	.L4_21pre

  2347. 	ALIGN_4

  2348. 

  2349. /**************************************************************************/

  2350. 

  2351. .L4_20_1:

  2352. #if !defined(TRMMKERNEL) || \

  2353.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2354.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2355.         leaq    BUFFER1, BO             // first buffer to BO

  2356.         addq    $4 * SIZE, BO

  2357. #else

  2358.         movq    KK, %rax

  2359.         leaq    BUFFER1, BO             // first buffer to BO

  2360.         addq    $4 * SIZE, BO

  2361.         movq    %rax, BI                        //  Index for BO

  2362. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2363.         leaq    (BO, BI, SIZE), BO

  2364.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  2365.         leaq    (AO, %rax, SIZE), AO

  2366. #endif

  2367. 

  2368. 

  2369. 	vzeroall

  2370. 

  2371. #ifndef TRMMKERNEL

  2372.         movq    K, %rax

  2373. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2374.         movq    K, %rax

  2375.         subq    KK, %rax

  2376.         movq    %rax, KKK

  2377. #else

  2378.         movq    KK, %rax

  2379. #ifdef LEFT

  2380.         addq    $8, %rax        // number of values in A

  2381. #else

  2382.         addq    $4, %rax        // number of values in BO

  2383. #endif

  2384.         movq    %rax, KKK

  2385. #endif

  2386. 

  2387. 

  2388. 	andq	$-8, %rax

  2389. 	je	.L4_20_6

  2390. 	movq    %rax, BI                        //  Index for BO

  2391.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2392. 

  2393. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  2394. 	leaq	(AO, %rax, SIZE), AO

  2395. 	leaq	(BO, BI, SIZE), BO

  2396. 	negq	BI

  2397. 	negq	%rax

  2398. 	ALIGN_4

  2399. 

  2400. .L4_20_2:

  2401. 

  2402. 	KERNEL8x4_SUB

  2403. 	KERNEL8x4_SUB

  2404. 	KERNEL8x4_SUB

  2405. 	KERNEL8x4_SUB

  2406. 

  2407. 	KERNEL8x4_SUB

  2408. 	KERNEL8x4_SUB

  2409. 	KERNEL8x4_SUB

  2410. 	KERNEL8x4_SUB

  2411. 

  2412. 	je	.L4_20_6

  2413. 

  2414. 	KERNEL8x4_SUB

  2415. 	KERNEL8x4_SUB

  2416. 	KERNEL8x4_SUB

  2417. 	KERNEL8x4_SUB

  2418. 

  2419. 	KERNEL8x4_SUB

  2420. 	KERNEL8x4_SUB

  2421. 	KERNEL8x4_SUB

  2422. 	KERNEL8x4_SUB

  2423. 

  2424. 	je	.L4_20_6

  2425. 

  2426. 	jmp	.L4_20_2

  2427. 	ALIGN_4

  2428. 

  2429. .L4_20_6:

  2430. #ifndef TRMMKERNEL

  2431.         movq    K, %rax

  2432. #else

  2433.         movq    KKK, %rax

  2434. #endif

  2435. 

  2436. 	andq	$7, %rax		# if (k & 1)

  2437. 	je .L4_20_9

  2438. 

  2439. 	movq    %rax, BI                        //  Index for BO

  2440.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2441. 

  2442. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  2443. 	leaq	(AO, %rax, SIZE), AO

  2444. 	leaq	(BO, BI, SIZE), BO

  2445. 	negq	BI

  2446. 	negq	%rax

  2447. 	ALIGN_4

  2448. 

  2449. .L4_20_7:

  2450. 

  2451. 	KERNEL8x4_SUB

  2452. 

  2453. 	jl	.L4_20_7

  2454. 	ALIGN_4

  2455. 

  2456. 

  2457. .L4_20_9:

  2458. 

  2459. 	SAVE8x4

  2460. 

  2461. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2462.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2463.         movq    K, %rax 

  2464.         subq    KKK, %rax

  2465.         movq    %rax, BI                        //  Index for BO

  2466. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2467.         leaq    (BO, BI, SIZE), BO         

  2468.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  2469.         leaq    (AO, %rax, SIZE), AO

  2470. #endif  

  2471. 

  2472. 

  2473. #if defined(TRMMKERNEL) && defined(LEFT)

  2474.         addq    $8, KK

  2475. #endif

  2476. 

  2477. 	addq	$8 * SIZE, CO1		# coffset += 8

  2478. 	addq	$8 * SIZE, CO2		# coffset += 8

  2479. 	ALIGN_4

  2480. 	

  2481. 

  2482. 

  2483. /**************************************************************************/

  2484. 

  2485. .L4_21pre:

  2486. 

  2487. 	testq	$4, M		

  2488. 	jz	.L4_30

  2489. 	ALIGN_4

  2490. 

  2491. .L4_21:

  2492. #if !defined(TRMMKERNEL) || \

  2493.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2494.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2495.         leaq    BUFFER1, BO             // first buffer to BO

  2496.         addq    $4 * SIZE, BO

  2497. #else

  2498.         movq    KK, %rax

  2499.         leaq    BUFFER1, BO             // first buffer to BO

  2500.         addq    $4 * SIZE, BO

  2501.         movq    %rax, BI                        //  Index for BO

  2502. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2503.         leaq    (BO, BI, SIZE), BO

  2504.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2505.         leaq    (AO, %rax, SIZE), AO

  2506. #endif

  2507. 

  2508. 

  2509. 	vzeroall

  2510. 

  2511. #ifndef TRMMKERNEL

  2512.         movq    K, %rax

  2513. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2514.         movq    K, %rax

  2515.         subq    KK, %rax

  2516.         movq    %rax, KKK

  2517. #else

  2518.         movq    KK, %rax

  2519. #ifdef LEFT

  2520.         addq    $4, %rax        // number of values in A

  2521. #else

  2522.         addq    $4, %rax        // number of values in BO

  2523. #endif

  2524.         movq    %rax, KKK

  2525. #endif

  2526. 

  2527. 

  2528. 	andq	$-8, %rax

  2529. 	je	.L4_26

  2530. 	movq    %rax, BI                        //  Index for BO

  2531.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2532. 

  2533. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2534. 	leaq	(AO, %rax, SIZE), AO

  2535. 	leaq	(BO, BI, SIZE), BO

  2536. 	negq	BI

  2537. 	negq	%rax

  2538. 	ALIGN_4

  2539. 

  2540. .L4_22:

  2541. 

  2542. 	KERNEL4x4_SUB

  2543. 	KERNEL4x4_SUB

  2544. 	KERNEL4x4_SUB

  2545. 	KERNEL4x4_SUB

  2546. 

  2547. 	KERNEL4x4_SUB

  2548. 	KERNEL4x4_SUB

  2549. 	KERNEL4x4_SUB

  2550. 	KERNEL4x4_SUB

  2551. 

  2552. 	je	.L4_26

  2553. 

  2554. 	KERNEL4x4_SUB

  2555. 	KERNEL4x4_SUB

  2556. 	KERNEL4x4_SUB

  2557. 	KERNEL4x4_SUB

  2558. 

  2559. 	KERNEL4x4_SUB

  2560. 	KERNEL4x4_SUB

  2561. 	KERNEL4x4_SUB

  2562. 	KERNEL4x4_SUB

  2563. 

  2564. 	je	.L4_26

  2565. 

  2566. 	jmp	.L4_22

  2567. 	ALIGN_4

  2568. 

  2569. .L4_26:

  2570. #ifndef TRMMKERNEL

  2571.         movq    K, %rax

  2572. #else

  2573.         movq    KKK, %rax

  2574. #endif

  2575. 

  2576. 	andq	$7, %rax		# if (k & 1)

  2577. 	je .L4_29

  2578. 

  2579. 	movq    %rax, BI                        //  Index for BO

  2580.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2581. 

  2582. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2583. 	leaq	(AO, %rax, SIZE), AO

  2584. 	leaq	(BO, BI, SIZE), BO

  2585. 	negq	BI

  2586. 	negq	%rax

  2587. 	ALIGN_4

  2588. 

  2589. .L4_27:

  2590. 

  2591. 	KERNEL4x4_SUB

  2592. 

  2593. 	jl	.L4_27

  2594. 	ALIGN_4

  2595. 

  2596. 

  2597. .L4_29:

  2598. 

  2599. 	SAVE4x4

  2600. 

  2601. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2602.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2603.         movq    K, %rax 

  2604.         subq    KKK, %rax

  2605.         movq    %rax, BI                        //  Index for BO

  2606. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2607.         leaq    (BO, BI, SIZE), BO         

  2608.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2609.         leaq    (AO, %rax, SIZE), AO

  2610. #endif  

  2611. 

  2612. 

  2613. #if defined(TRMMKERNEL) && defined(LEFT)

  2614.         addq    $4, KK

  2615. #endif

  2616. 

  2617. 	addq	$4 * SIZE, CO1		# coffset += 4

  2618. 	addq	$4 * SIZE, CO2		# coffset += 4

  2619. 	ALIGN_4

  2620. 	

  2621. 

  2622. .L4_30:

  2623. 	testq	$2, M		

  2624. 	jz	.L4_40

  2625. 

  2626. 	ALIGN_4

  2627. 

  2628. .L4_31:

  2629. #if !defined(TRMMKERNEL) || \

  2630.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2631.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2632.         leaq    BUFFER1, BO             // first buffer to BO

  2633.         addq    $4 * SIZE, BO

  2634. #else

  2635.         movq    KK, %rax

  2636.         leaq    BUFFER1, BO             // first buffer to BO

  2637.         addq    $4 * SIZE, BO

  2638.         movq    %rax, BI                        //  Index for BO

  2639. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2640.         leaq    (BO, BI, SIZE), BO

  2641.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2642.         leaq    (AO, %rax, SIZE), AO

  2643. #endif

  2644. 

  2645. 

  2646. 	vzeroall

  2647. 

  2648. #ifndef TRMMKERNEL

  2649.         movq    K, %rax

  2650. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2651.         movq    K, %rax

  2652.         subq    KK, %rax

  2653.         movq    %rax, KKK

  2654. #else

  2655.         movq    KK, %rax

  2656. #ifdef LEFT

  2657.         addq    $2, %rax        // number of values in AO

  2658. #else

  2659.         addq    $4, %rax        // number of values in BO

  2660. #endif

  2661.         movq    %rax, KKK

  2662. #endif

  2663. 

  2664. 

  2665. 	andq	$-8, %rax

  2666. 	je	.L4_36

  2667. 	movq    %rax, BI                        //  Index for BO

  2668.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2669. 

  2670. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2671. 	leaq	(AO, %rax, SIZE), AO

  2672. 	leaq	(BO, BI, SIZE), BO

  2673. 	negq	BI

  2674. 	negq	%rax

  2675. 	ALIGN_4

  2676. 

  2677. .L4_32:

  2678. 

  2679. 	KERNEL2x4_SUB

  2680. 	KERNEL2x4_SUB

  2681. 	KERNEL2x4_SUB

  2682. 	KERNEL2x4_SUB

  2683. 

  2684. 	KERNEL2x4_SUB

  2685. 	KERNEL2x4_SUB

  2686. 	KERNEL2x4_SUB

  2687. 	KERNEL2x4_SUB

  2688. 

  2689. 	je	.L4_36

  2690. 

  2691. 	KERNEL2x4_SUB

  2692. 	KERNEL2x4_SUB

  2693. 	KERNEL2x4_SUB

  2694. 	KERNEL2x4_SUB

  2695. 

  2696. 	KERNEL2x4_SUB

  2697. 	KERNEL2x4_SUB

  2698. 	KERNEL2x4_SUB

  2699. 	KERNEL2x4_SUB

  2700. 

  2701. 	je	.L4_36

  2702. 

  2703. 	jmp	.L4_32

  2704. 	ALIGN_4

  2705. 

  2706. .L4_36:

  2707. #ifndef TRMMKERNEL

  2708.         movq    K, %rax

  2709. #else

  2710.         movq    KKK, %rax

  2711. #endif

  2712. 

  2713. 	andq	$7, %rax		# if (k & 1)

  2714. 	je .L4_39

  2715. 

  2716. 	movq    %rax, BI                        //  Index for BO

  2717. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2718. 	

  2719. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2720. 	leaq	(AO, %rax, SIZE), AO

  2721. 	leaq	(BO, BI, SIZE), BO

  2722. 	negq	BI

  2723. 	negq	%rax

  2724. 	ALIGN_4

  2725. 

  2726. .L4_37:

  2727. 

  2728. 	KERNEL2x4_SUB

  2729. 

  2730. 	jl	.L4_37

  2731. 	ALIGN_4

  2732. 

  2733. 

  2734. .L4_39:

  2735. 

  2736. 	SAVE2x4

  2737. 

  2738. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2739.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2740.         movq    K, %rax 

  2741.         subq    KKK, %rax

  2742.         movq    %rax, BI                        //  Index for BO

  2743. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2744.         leaq    (BO, BI, SIZE), BO         

  2745.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2746.         leaq    (AO, %rax, SIZE), AO

  2747. #endif  

  2748. 

  2749. 

  2750. #if defined(TRMMKERNEL) && defined(LEFT)

  2751.         addq    $2, KK

  2752. #endif

  2753. 

  2754. 	addq	$2 * SIZE, CO1		# coffset += 2

  2755. 	addq	$2 * SIZE, CO2		# coffset += 2

  2756. 	ALIGN_4

  2757. 

  2758. .L4_40:

  2759. 	testq	$1, M		

  2760. 	jz	.L4_60		// to next 4 lines of N

  2761. 

  2762. 	ALIGN_4

  2763. 

  2764. .L4_41:

  2765. #if !defined(TRMMKERNEL) || \

  2766.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2767.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2768.         leaq    BUFFER1, BO             // first buffer to BO

  2769.         addq    $4 * SIZE, BO

  2770. #else

  2771.         movq    KK, %rax

  2772.         leaq    BUFFER1, BO             // first buffer to BO

  2773.         addq    $4 * SIZE, BO

  2774.         movq    %rax, BI                        //  Index for BO

  2775. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2776.         leaq    (BO, BI, SIZE), BO

  2777.         leaq    (AO, %rax, SIZE), AO

  2778. #endif

  2779. 

  2780. 

  2781. 	vzeroall

  2782. 

  2783. #ifndef TRMMKERNEL

  2784.         movq    K, %rax

  2785. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2786.         movq    K, %rax

  2787.         subq    KK, %rax

  2788.         movq    %rax, KKK

  2789. #else

  2790.         movq    KK, %rax

  2791. #ifdef LEFT

  2792.         addq    $1, %rax        // number of values in AO

  2793. #else

  2794.         addq    $4, %rax        // number of values in BO

  2795. #endif

  2796.         movq    %rax, KKK

  2797. #endif

  2798. 

  2799. 	andq	$-8, %rax

  2800. 	je	.L4_46

  2801. 	movq    %rax, BI                        //  Index for BO

  2802.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2803. 

  2804. 	leaq	(AO, %rax, SIZE), AO

  2805. 	leaq	(BO, BI, SIZE), BO

  2806. 	negq	BI

  2807. 	negq	%rax

  2808. 	ALIGN_4

  2809. 

  2810. .L4_42:

  2811. 

  2812. 	KERNEL1x4_SUB

  2813. 	KERNEL1x4_SUB

  2814. 	KERNEL1x4_SUB

  2815. 	KERNEL1x4_SUB

  2816. 

  2817. 	KERNEL1x4_SUB

  2818. 	KERNEL1x4_SUB

  2819. 	KERNEL1x4_SUB

  2820. 	KERNEL1x4_SUB

  2821. 

  2822. 	je	.L4_46

  2823. 

  2824. 	KERNEL1x4_SUB

  2825. 	KERNEL1x4_SUB

  2826. 	KERNEL1x4_SUB

  2827. 	KERNEL1x4_SUB

  2828. 

  2829. 	KERNEL1x4_SUB

  2830. 	KERNEL1x4_SUB

  2831. 	KERNEL1x4_SUB

  2832. 	KERNEL1x4_SUB

  2833. 

  2834. 	je	.L4_46

  2835. 

  2836. 	jmp	.L4_42

  2837. 	ALIGN_4

  2838. 

  2839. .L4_46:

  2840. #ifndef TRMMKERNEL

  2841.         movq    K, %rax

  2842. #else

  2843.         movq    KKK, %rax

  2844. #endif

  2845. 

  2846. 	andq	$7, %rax		# if (k & 1)

  2847. 	je .L4_49

  2848. 

  2849. 	movq    %rax, BI                        //  Index for BO

  2850.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  2851. 

  2852. 	leaq	(AO, %rax, SIZE), AO

  2853. 	leaq	(BO, BI, SIZE), BO

  2854. 	negq	BI

  2855. 	negq	%rax

  2856. 	ALIGN_4

  2857. 

  2858. .L4_47:

  2859. 

  2860. 	KERNEL1x4_SUB

  2861. 

  2862. 	jl	.L4_47

  2863. 	ALIGN_4

  2864. 

  2865. 

  2866. .L4_49:

  2867. 

  2868. 	SAVE1x4

  2869. 

  2870. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2871.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2872.         movq    K, %rax 

  2873.         subq    KKK, %rax

  2874.         movq    %rax, BI                        //  Index for BO

  2875. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  2876.         leaq    (BO, BI, SIZE), BO         

  2877.         leaq    (AO, %rax, SIZE), AO

  2878. #endif  

  2879. 

  2880. 

  2881. #if defined(TRMMKERNEL) && defined(LEFT)

  2882.         addq    $1, KK

  2883. #endif

  2884. 

  2885. 	addq	$1 * SIZE, CO1		# coffset += 1

  2886. 	addq	$1 * SIZE, CO2		# coffset += 1

  2887. 	ALIGN_4

  2888. 	

  2889. 

  2890. 

  2891. 

  2892. 	

  2893. .L4_60:

  2894. #if defined(TRMMKERNEL) && !defined(LEFT)

  2895.         addq    $4, KK

  2896. #endif

  2897. 

  2898. 	decq	J			// j --

  2899. 	jg	.L4_01			// next 4 lines of N

  2900. 

  2901. 

  2902. 

  2903. /*******************************************************************************************/

  2904. .L2_00:

  2905. 

  2906. 	movq	Nmod6, J		

  2907. 	andq	$3, J			// j % 4

  2908. 	je	.L999

  2909. 

  2910. 	movq	Nmod6, J		

  2911. 	andq	$2, J			// j % 4

  2912. 	je	.L1_0

  2913. 

  2914. .L2_01:

  2915. 

  2916. 	// copy to sub buffer

  2917. 	movq	B, BO1

  2918. 	leaq    BUFFER1, BO		// first buffer to BO

  2919. 	movq	K, %rax

  2920. 	sarq	$2, %rax		// K / 4

  2921. 	jz	.L2_01b

  2922. 	ALIGN_4

  2923. 

  2924. .L2_01a:

  2925. 

  2926. 	vmovsd	      (BO1), %xmm0

  2927. 	vmovsd	2*SIZE(BO1), %xmm1

  2928. 	vmovsd	4*SIZE(BO1), %xmm2

  2929. 	vmovsd	6*SIZE(BO1), %xmm3

  2930. 

  2931. 	vmovsd	%xmm0,       (BO)

  2932. 	vmovsd	%xmm1, 2*SIZE(BO)

  2933. 	vmovsd	%xmm2, 4*SIZE(BO)

  2934. 	vmovsd	%xmm3, 6*SIZE(BO)

  2935. 

  2936. 	addq	$8*SIZE,BO1

  2937. 	addq	$8*SIZE,BO

  2938. 	decq	%rax

  2939. 	jnz	.L2_01a

  2940. 

  2941. 

  2942. .L2_01b:

  2943. 

  2944.         movq    K, %rax

  2945.         andq    $3, %rax                // K % 4

  2946.         jz      .L2_02d

  2947.         ALIGN_4

  2948. 

  2949. .L2_02c:

  2950. 

  2951. 	vmovsd 	(BO1), %xmm0

  2952. 	vmovsd 	%xmm0, (BO)

  2953. 	addq	$2*SIZE,BO1

  2954. 	addq	$2*SIZE,BO

  2955. 	decq	%rax

  2956. 	jnz	.L2_02c

  2957. 

  2958. .L2_02d:

  2959. 

  2960. 	movq	BO1, B			// next offset of B

  2961. 

  2962. .L2_10:

  2963. 	movq	C, CO1

  2964. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  2965. 

  2966. #if defined(TRMMKERNEL) && defined(LEFT)

  2967.         movq    OFFSET, %rax

  2968.         movq    %rax, KK

  2969. #endif

  2970. 	

  2971. 	movq	A, AO		 	// aoffset = a

  2972. 	addq	$16 * SIZE, AO

  2973. 

  2974. 	movq	M,  I

  2975. 	sarq	$4, I			// i = (m >> 4)

  2976. 	je	.L2_20

  2977. 

  2978. 	ALIGN_4

  2979. 

  2980. .L2_11:

  2981. #if !defined(TRMMKERNEL) || \

  2982.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2983.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2984.         leaq    BUFFER1, BO             // first buffer to BO

  2985.         addq    $4 * SIZE, BO

  2986. #else

  2987.         movq    KK, %rax

  2988.         leaq    BUFFER1, BO             // first buffer to BO

  2989.         addq    $4 * SIZE, BO

  2990.         movq    %rax, BI                        //  Index for BO

  2991.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2992.         leaq    (BO, BI, SIZE), BO

  2993.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  2994.         leaq    (AO, %rax, SIZE), AO

  2995. #endif

  2996. 

  2997. 

  2998. 	vzeroall

  2999. 

  3000. #ifndef TRMMKERNEL

  3001.         movq    K, %rax

  3002. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3003.         movq    K, %rax

  3004.         subq    KK, %rax

  3005.         movq    %rax, KKK

  3006. #else

  3007.         movq    KK, %rax

  3008. #ifdef LEFT

  3009.         addq    $16, %rax	// number of values in AO

  3010. #else

  3011.         addq    $2, %rax	// number of values in BO

  3012. #endif

  3013.         movq    %rax, KKK

  3014. #endif

  3015. 

  3016. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  3017. 	je	.L2_16

  3018. 	movq    %rax, BI                        //  Index for BO

  3019.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3020. 

  3021. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  3022. 	leaq	(AO, %rax, SIZE), AO

  3023. 	leaq	(BO, BI, SIZE), BO

  3024. 	negq	BI

  3025. 	negq	%rax

  3026. 	ALIGN_4

  3027. 

  3028. .L2_12:

  3029. 

  3030. 	KERNEL16x2_SUB

  3031. 	KERNEL16x2_SUB

  3032. 	KERNEL16x2_SUB

  3033. 	KERNEL16x2_SUB

  3034. 

  3035. 	KERNEL16x2_SUB

  3036. 	KERNEL16x2_SUB

  3037. 	KERNEL16x2_SUB

  3038. 	KERNEL16x2_SUB

  3039. 

  3040. 	je	.L2_16

  3041. 

  3042. 	KERNEL16x2_SUB

  3043. 	KERNEL16x2_SUB

  3044. 	KERNEL16x2_SUB

  3045. 	KERNEL16x2_SUB

  3046. 

  3047. 	KERNEL16x2_SUB

  3048. 	KERNEL16x2_SUB

  3049. 	KERNEL16x2_SUB

  3050. 	KERNEL16x2_SUB

  3051. 

  3052. 	je	.L2_16

  3053. 

  3054. 	jmp	.L2_12

  3055. 	ALIGN_4

  3056. 

  3057. .L2_16:

  3058. #ifndef TRMMKERNEL

  3059.         movq    K, %rax

  3060. #else

  3061.         movq    KKK, %rax

  3062. #endif

  3063. 

  3064. 	andq	$7, %rax		# if (k & 1)

  3065. 	je .L2_19

  3066. 

  3067. 	movq    %rax, BI                        //  Index for BO

  3068.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3069. 

  3070. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  3071. 	leaq	(AO, %rax, SIZE), AO

  3072. 	leaq	(BO, BI, SIZE), BO

  3073. 	negq	BI

  3074. 	negq	%rax

  3075. 	ALIGN_4

  3076. 

  3077. .L2_17:

  3078. 

  3079. 	KERNEL16x2_SUB

  3080. 

  3081. 	jl	.L2_17

  3082. 	ALIGN_4

  3083. 

  3084. 

  3085. .L2_19:

  3086. 

  3087. 	SAVE16x2

  3088. 

  3089. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3090.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3091.         movq    K, %rax

  3092.         subq    KKK, %rax

  3093.         movq    %rax, BI                        //  Index for BO

  3094.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3095.         leaq    (BO, BI, SIZE), BO

  3096.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  3097.         leaq    (AO, %rax, SIZE), AO

  3098. #endif

  3099. 

  3100. 

  3101. #if defined(TRMMKERNEL) && defined(LEFT)

  3102.         addq    $16, KK				

  3103. #endif

  3104. 

  3105. 	addq	$16 * SIZE, CO1		# coffset += 16

  3106. 	decq	I			# i --

  3107. 	jg	.L2_11

  3108. 	ALIGN_4	

  3109. 

  3110. /**************************************************************************

  3111. * Rest of M 

  3112. ***************************************************************************/

  3113. .L2_20:

  3114. 	// Test rest of M

  3115. 

  3116. 	testq	$15, M

  3117. 	jz	.L2_60		// to next 2 lines of N

  3118. 

  3119. 	testq	$8, M		

  3120. 	jz	.L2_21pre

  3121. 	ALIGN_4

  3122. 

  3123. /**************************************************************************/

  3124. 

  3125. .L2_20_1:

  3126. #if !defined(TRMMKERNEL) || \

  3127.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3128.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3129.         leaq    BUFFER1, BO             // first buffer to BO

  3130.         addq    $4 * SIZE, BO

  3131. #else

  3132.         movq    KK, %rax

  3133.         leaq    BUFFER1, BO             // first buffer to BO

  3134.         addq    $4 * SIZE, BO

  3135.         movq    %rax, BI                        //  Index for BO

  3136.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3137.         leaq    (BO, BI, SIZE), BO

  3138.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  3139.         leaq    (AO, %rax, SIZE), AO

  3140. #endif

  3141. 

  3142. 

  3143. 	vzeroall

  3144. 

  3145. #ifndef TRMMKERNEL

  3146.         movq    K, %rax

  3147. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3148.         movq    K, %rax

  3149.         subq    KK, %rax

  3150.         movq    %rax, KKK

  3151. #else

  3152.         movq    KK, %rax

  3153. #ifdef LEFT

  3154.         addq    $8, %rax        // number of values in A

  3155. #else

  3156.         addq    $2, %rax        // number of values in BO

  3157. #endif

  3158.         movq    %rax, KKK

  3159. #endif

  3160. 

  3161. 

  3162. 	andq	$-8, %rax

  3163. 	je	.L2_20_6

  3164. 	movq    %rax, BI                        //  Index for BO

  3165.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3166. 

  3167. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  3168. 	leaq	(AO, %rax, SIZE), AO

  3169. 	leaq	(BO, BI, SIZE), BO

  3170. 	negq	BI

  3171. 	negq	%rax

  3172. 	ALIGN_4

  3173. 

  3174. .L2_20_2:

  3175. 

  3176. 

  3177. 	KERNEL8x2_SUB

  3178. 	KERNEL8x2_SUB

  3179. 	KERNEL8x2_SUB

  3180. 	KERNEL8x2_SUB

  3181. 

  3182. 	KERNEL8x2_SUB

  3183. 	KERNEL8x2_SUB

  3184. 	KERNEL8x2_SUB

  3185. 	KERNEL8x2_SUB

  3186. 

  3187. 	je	.L2_20_6

  3188. 

  3189. 	KERNEL8x2_SUB

  3190. 	KERNEL8x2_SUB

  3191. 	KERNEL8x2_SUB

  3192. 	KERNEL8x2_SUB

  3193. 

  3194. 	KERNEL8x2_SUB

  3195. 	KERNEL8x2_SUB

  3196. 	KERNEL8x2_SUB

  3197. 	KERNEL8x2_SUB

  3198. 

  3199. 	je	.L2_20_6

  3200. 

  3201. 	jmp	.L2_20_2

  3202. 	ALIGN_4

  3203. 

  3204. .L2_20_6:

  3205. #ifndef TRMMKERNEL

  3206.         movq    K, %rax

  3207. #else

  3208.         movq    KKK, %rax

  3209. #endif

  3210. 

  3211. 	andq	$7, %rax		# if (k & 1)

  3212. 	je .L2_20_9

  3213. 

  3214. 	movq    %rax, BI                        //  Index for BO

  3215.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3216. 

  3217. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  3218. 	leaq	(AO, %rax, SIZE), AO

  3219. 	leaq	(BO, BI, SIZE), BO

  3220. 	negq	BI

  3221. 	negq	%rax

  3222. 	ALIGN_4

  3223. 

  3224. .L2_20_7:

  3225. 

  3226. 	KERNEL8x2_SUB

  3227. 

  3228. 	jl	.L2_20_7

  3229. 	ALIGN_4

  3230. 

  3231. 

  3232. .L2_20_9:

  3233. 

  3234. 	SAVE8x2

  3235. 

  3236. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3237.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3238.         movq    K, %rax 

  3239.         subq    KKK, %rax

  3240.         movq    %rax, BI                        //  Index for BO

  3241.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3242.         leaq    (BO, BI, SIZE), BO         

  3243.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  3244.         leaq    (AO, %rax, SIZE), AO

  3245. #endif  

  3246. 

  3247. 

  3248. #if defined(TRMMKERNEL) && defined(LEFT)

  3249.         addq    $8, KK

  3250. #endif

  3251. 

  3252. 	addq	$8 * SIZE, CO1		# coffset += 8

  3253. 	ALIGN_4

  3254. 	

  3255. 

  3256. 

  3257. /**************************************************************************/

  3258. 

  3259. .L2_21pre:

  3260. 

  3261. 	testq	$4, M		

  3262. 	jz	.L2_30

  3263. 	ALIGN_4

  3264. 

  3265. .L2_21:

  3266. #if !defined(TRMMKERNEL) || \

  3267.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3268.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3269.         leaq    BUFFER1, BO             // first buffer to BO

  3270.         addq    $4 * SIZE, BO

  3271. #else

  3272.         movq    KK, %rax

  3273.         leaq    BUFFER1, BO             // first buffer to BO

  3274.         addq    $4 * SIZE, BO

  3275.         movq    %rax, BI                        //  Index for BO

  3276.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3277.         leaq    (BO, BI, SIZE), BO

  3278.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  3279.         leaq    (AO, %rax, SIZE), AO

  3280. #endif

  3281. 

  3282. 

  3283. 	vzeroall

  3284. 

  3285. #ifndef TRMMKERNEL

  3286.         movq    K, %rax

  3287. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3288.         movq    K, %rax

  3289.         subq    KK, %rax

  3290.         movq    %rax, KKK

  3291. #else

  3292.         movq    KK, %rax

  3293. #ifdef LEFT

  3294.         addq    $4, %rax        // number of values in A

  3295. #else

  3296.         addq    $2, %rax        // number of values in BO

  3297. #endif

  3298.         movq    %rax, KKK

  3299. #endif

  3300. 

  3301. 

  3302. 	andq	$-8, %rax

  3303. 	je	.L2_26

  3304. 	movq    %rax, BI                        //  Index for BO

  3305.         leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values

  3306. 

  3307. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  3308. 	leaq	(AO, %rax, SIZE), AO

  3309. 	leaq	(BO, BI, SIZE), BO

  3310. 	negq	BI

  3311. 	negq	%rax

  3312. 	ALIGN_4

  3313. 

  3314. .L2_22:

  3315. 

  3316. 

  3317. 	KERNEL4x2_SUB

  3318. 	KERNEL4x2_SUB

  3319. 	KERNEL4x2_SUB

  3320. 	KERNEL4x2_SUB

  3321. 

  3322. 	KERNEL4x2_SUB

  3323. 	KERNEL4x2_SUB

  3324. 	KERNEL4x2_SUB

  3325. 	KERNEL4x2_SUB

  3326. 

  3327. 	je	.L2_26

  3328. 

  3329. 	KERNEL4x2_SUB

  3330. 	KERNEL4x2_SUB

  3331. 	KERNEL4x2_SUB

  3332. 	KERNEL4x2_SUB

  3333. 

  3334. 	KERNEL4x2_SUB

  3335. 	KERNEL4x2_SUB

  3336. 	KERNEL4x2_SUB

  3337. 	KERNEL4x2_SUB

  3338. 

  3339. 	je	.L2_26

  3340. 

  3341. 	jmp	.L2_22

  3342. 	ALIGN_4

  3343. 

  3344. .L2_26:

  3345. #ifndef TRMMKERNEL

  3346.         movq    K, %rax

  3347. #else

  3348.         movq    KKK, %rax

  3349. #endif

  3350. 

  3351. 	andq	$7, %rax		# if (k & 1)

  3352. 	je .L2_29

  3353. 

  3354. 	movq    %rax, BI                        //  Index for BO

  3355.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3356. 

  3357. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  3358. 	leaq	(AO, %rax, SIZE), AO

  3359. 	leaq	(BO, BI, SIZE), BO

  3360. 	negq	BI

  3361. 	negq	%rax

  3362. 	ALIGN_4

  3363. 

  3364. .L2_27:

  3365. 

  3366. 	KERNEL4x2_SUB

  3367. 

  3368. 	jl	.L2_27

  3369. 	ALIGN_4

  3370. 

  3371. 

  3372. .L2_29:

  3373. 

  3374. 	SAVE4x2

  3375. 

  3376. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3377.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3378.         movq    K, %rax 

  3379.         subq    KKK, %rax

  3380.         movq    %rax, BI                        //  Index for BO

  3381.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3382.         leaq    (BO, BI, SIZE), BO         

  3383.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  3384.         leaq    (AO, %rax, SIZE), AO

  3385. #endif  

  3386. 

  3387. 

  3388. #if defined(TRMMKERNEL) && defined(LEFT)

  3389.         addq    $4, KK

  3390. #endif

  3391. 

  3392. 	addq	$4 * SIZE, CO1		# coffset += 4

  3393. 	ALIGN_4

  3394. 	

  3395. 

  3396. .L2_30:

  3397. 	testq	$2, M		

  3398. 	jz	.L2_40

  3399. 

  3400. 	ALIGN_4

  3401. 

  3402. .L2_31:

  3403. #if !defined(TRMMKERNEL) || \

  3404.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3405.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3406.         leaq    BUFFER1, BO             // first buffer to BO

  3407.         addq    $4 * SIZE, BO

  3408. #else

  3409.         movq    KK, %rax

  3410.         leaq    BUFFER1, BO             // first buffer to BO

  3411.         addq    $4 * SIZE, BO

  3412.         movq    %rax, BI                        //  Index for BO

  3413.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3414.         leaq    (BO, BI, SIZE), BO

  3415.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  3416.         leaq    (AO, %rax, SIZE), AO

  3417. #endif

  3418. 

  3419. 

  3420. 	vzeroall

  3421. 

  3422. #ifndef TRMMKERNEL

  3423.         movq    K, %rax

  3424. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3425.         movq    K, %rax

  3426.         subq    KK, %rax

  3427.         movq    %rax, KKK

  3428. #else

  3429.         movq    KK, %rax

  3430. #ifdef LEFT

  3431.         addq    $2, %rax        // number of values in AO

  3432. #else

  3433.         addq    $2, %rax        // number of values in BO

  3434. #endif

  3435.         movq    %rax, KKK

  3436. #endif

  3437. 

  3438. 

  3439. 	andq	$-8, %rax

  3440. 	je	.L2_36

  3441. 	movq    %rax, BI                        //  Index for BO

  3442.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3443. 

  3444. 	salq	$1, %rax			// rax = rax *2 ; number of values

  3445. 	leaq	(AO, %rax, SIZE), AO

  3446. 	leaq	(BO, BI, SIZE), BO

  3447. 	negq	BI

  3448. 	negq	%rax

  3449. 	ALIGN_4

  3450. 

  3451. .L2_32:

  3452. 

  3453. 	KERNEL2x2_SUB

  3454. 	KERNEL2x2_SUB

  3455. 	KERNEL2x2_SUB

  3456. 	KERNEL2x2_SUB

  3457. 

  3458. 	KERNEL2x2_SUB

  3459. 	KERNEL2x2_SUB

  3460. 	KERNEL2x2_SUB

  3461. 	KERNEL2x2_SUB

  3462. 

  3463. 	je	.L2_36

  3464. 

  3465. 	KERNEL2x2_SUB

  3466. 	KERNEL2x2_SUB

  3467. 	KERNEL2x2_SUB

  3468. 	KERNEL2x2_SUB

  3469. 

  3470. 	KERNEL2x2_SUB

  3471. 	KERNEL2x2_SUB

  3472. 	KERNEL2x2_SUB

  3473. 	KERNEL2x2_SUB

  3474. 

  3475. 	je	.L2_36

  3476. 

  3477. 	jmp	.L2_32

  3478. 	ALIGN_4

  3479. 

  3480. .L2_36:

  3481. #ifndef TRMMKERNEL

  3482.         movq    K, %rax

  3483. #else

  3484.         movq    KKK, %rax

  3485. #endif

  3486. 

  3487. 	andq	$7, %rax		# if (k & 1)

  3488. 	je .L2_39

  3489. 

  3490. 	movq    %rax, BI                        //  Index for BO

  3491.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3492. 	

  3493. 	salq	$1, %rax			// rax = rax *2 ; number of values

  3494. 	leaq	(AO, %rax, SIZE), AO

  3495. 	leaq	(BO, BI, SIZE), BO

  3496. 	negq	BI

  3497. 	negq	%rax

  3498. 	ALIGN_4

  3499. 

  3500. .L2_37:

  3501. 

  3502. 	KERNEL2x2_SUB

  3503. 

  3504. 	jl	.L2_37

  3505. 	ALIGN_4

  3506. 

  3507. 

  3508. .L2_39:

  3509. 

  3510. 	SAVE2x2

  3511. 

  3512. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3513.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3514.         movq    K, %rax 

  3515.         subq    KKK, %rax

  3516.         movq    %rax, BI                        //  Index for BO

  3517.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3518.         leaq    (BO, BI, SIZE), BO         

  3519.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  3520.         leaq    (AO, %rax, SIZE), AO

  3521. #endif  

  3522. 

  3523. 

  3524. #if defined(TRMMKERNEL) && defined(LEFT)

  3525.         addq    $2, KK

  3526. #endif

  3527. 

  3528. 	addq	$2 * SIZE, CO1		# coffset += 2

  3529. 	ALIGN_4

  3530. 

  3531. .L2_40:

  3532. 	testq	$1, M		

  3533. 	jz	.L2_60		// to next 2 lines of N

  3534. 

  3535. 	ALIGN_4

  3536. 

  3537. .L2_41:

  3538. #if !defined(TRMMKERNEL) || \

  3539.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3540.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3541.         leaq    BUFFER1, BO             // first buffer to BO

  3542.         addq    $4 * SIZE, BO

  3543. #else

  3544.         movq    KK, %rax

  3545.         leaq    BUFFER1, BO             // first buffer to BO

  3546.         addq    $4 * SIZE, BO

  3547.         movq    %rax, BI                        //  Index for BO

  3548.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3549.         leaq    (BO, BI, SIZE), BO

  3550.         leaq    (AO, %rax, SIZE), AO

  3551. #endif

  3552. 

  3553. 

  3554. 	vzeroall

  3555. 

  3556. #ifndef TRMMKERNEL

  3557.         movq    K, %rax

  3558. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3559.         movq    K, %rax

  3560.         subq    KK, %rax

  3561.         movq    %rax, KKK

  3562. #else

  3563.         movq    KK, %rax

  3564. #ifdef LEFT

  3565.         addq    $1, %rax        // number of values in AO

  3566. #else

  3567.         addq    $2, %rax        // number of values in BO

  3568. #endif

  3569.         movq    %rax, KKK

  3570. #endif

  3571. 

  3572. 	andq	$-8, %rax

  3573. 	je	.L2_46

  3574. 	movq    %rax, BI                        //  Index for BO

  3575.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3576. 

  3577. 	leaq	(AO, %rax, SIZE), AO

  3578. 	leaq	(BO, BI, SIZE), BO

  3579. 	negq	BI

  3580. 	negq	%rax

  3581. 	ALIGN_4

  3582. 

  3583. .L2_42:

  3584. 

  3585. 	KERNEL1x2_SUB

  3586. 	KERNEL1x2_SUB

  3587. 	KERNEL1x2_SUB

  3588. 	KERNEL1x2_SUB

  3589. 

  3590. 	KERNEL1x2_SUB

  3591. 	KERNEL1x2_SUB

  3592. 	KERNEL1x2_SUB

  3593. 	KERNEL1x2_SUB

  3594. 

  3595. 	je	.L2_46

  3596. 

  3597. 	KERNEL1x2_SUB

  3598. 	KERNEL1x2_SUB

  3599. 	KERNEL1x2_SUB

  3600. 	KERNEL1x2_SUB

  3601. 

  3602. 	KERNEL1x2_SUB

  3603. 	KERNEL1x2_SUB

  3604. 	KERNEL1x2_SUB

  3605. 	KERNEL1x2_SUB

  3606. 

  3607. 	je	.L2_46

  3608. 

  3609. 	jmp	.L2_42

  3610. 	ALIGN_4

  3611. 

  3612. .L2_46:

  3613. #ifndef TRMMKERNEL

  3614.         movq    K, %rax

  3615. #else

  3616.         movq    KKK, %rax

  3617. #endif

  3618. 

  3619. 	andq	$7, %rax		# if (k & 1)

  3620. 	je .L2_49

  3621. 

  3622. 	movq    %rax, BI                        //  Index for BO

  3623.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3624. 

  3625. 	leaq	(AO, %rax, SIZE), AO

  3626. 	leaq	(BO, BI, SIZE), BO

  3627. 	negq	BI

  3628. 	negq	%rax

  3629. 	ALIGN_4

  3630. 

  3631. .L2_47:

  3632. 

  3633. 	KERNEL1x2_SUB

  3634. 

  3635. 	jl	.L2_47

  3636. 	ALIGN_4

  3637. 

  3638. 

  3639. .L2_49:

  3640. 

  3641. 	SAVE1x2

  3642. 

  3643. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3644.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3645.         movq    K, %rax 

  3646.         subq    KKK, %rax

  3647.         movq    %rax, BI                        //  Index for BO

  3648.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  3649.         leaq    (BO, BI, SIZE), BO         

  3650.         leaq    (AO, %rax, SIZE), AO

  3651. #endif  

  3652. 

  3653. 

  3654. #if defined(TRMMKERNEL) && defined(LEFT)

  3655.         addq    $1, KK

  3656. #endif

  3657. 

  3658. 	addq	$1 * SIZE, CO1		# coffset += 1

  3659. 	ALIGN_4

  3660. 	

  3661. 

  3662. 

  3663. 

  3664. 	

  3665. .L2_60:

  3666. #if defined(TRMMKERNEL) && !defined(LEFT)

  3667.         addq    $2, KK

  3668. #endif

  3669. 

  3670. 

  3671. 

  3672. 

  3673. .L1_0:

  3674. 

  3675. /************************************************************************************************

  3676. * Loop for Nmod6 % 2 > 0

  3677. *************************************************************************************************/

  3678. 

  3679. 	movq	Nmod6, J		

  3680. 	andq	$1, J			// j % 2

  3681. 	je	.L999

  3682. 	ALIGN_4

  3683. 

  3684. .L1_01:

  3685. 	// copy to sub buffer

  3686. 	movq	B, BO1

  3687. 	leaq    BUFFER1, BO		// first buffer to BO

  3688. 	movq	K, %rax

  3689. 	ALIGN_4

  3690. 

  3691. .L1_02b:

  3692. 

  3693. 	vmovss	(BO1), %xmm0

  3694. 	vmovss	%xmm0,       (BO)

  3695. 	addq	$1*SIZE,BO1

  3696. 	addq	$1*SIZE,BO

  3697. 	decq	%rax

  3698. 	jnz	.L1_02b

  3699. 

  3700. .L1_02c:

  3701. 

  3702. 	movq	BO1, B			// next offset of B

  3703. 

  3704. .L1_10:

  3705. 	movq	C, CO1

  3706. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  3707. 

  3708. #if defined(TRMMKERNEL) && defined(LEFT)

  3709.         movq    OFFSET, %rax

  3710.         movq    %rax, KK

  3711. #endif

  3712. 	

  3713. 	movq	A, AO		 	// aoffset = a

  3714. 	addq	$16 * SIZE, AO

  3715. 

  3716. 	movq	M,  I

  3717. 	sarq	$4, I			// i = (m >> 4)

  3718. 	je	.L1_20

  3719. 

  3720. 	ALIGN_4

  3721. 

  3722. .L1_11:

  3723. #if !defined(TRMMKERNEL) || \

  3724.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3725.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3726.         leaq    BUFFER1, BO             // first buffer to BO

  3727.         addq    $4 * SIZE, BO

  3728. #else

  3729.         movq    KK, %rax

  3730.         leaq    BUFFER1, BO             // first buffer to BO

  3731.         addq    $4 * SIZE, BO

  3732.         movq    %rax, BI                        //  Index for BO

  3733.         leaq    (BO, BI, SIZE), BO

  3734.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  3735.         leaq    (AO, %rax, SIZE), AO

  3736. #endif

  3737. 

  3738. 

  3739. 	vzeroall

  3740. 

  3741. #ifndef TRMMKERNEL

  3742.         movq    K, %rax

  3743. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3744.         movq    K, %rax

  3745.         subq    KK, %rax

  3746.         movq    %rax, KKK

  3747. #else

  3748.         movq    KK, %rax

  3749. #ifdef LEFT

  3750.         addq    $16, %rax	// number of values in AO

  3751. #else

  3752.         addq    $1, %rax	// number of values in BO

  3753. #endif

  3754.         movq    %rax, KKK

  3755. #endif

  3756. 

  3757. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  3758. 	je	.L1_16

  3759. 	movq    %rax, BI                        //  Index for BO

  3760. 

  3761. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  3762. 	leaq	(AO, %rax, SIZE), AO

  3763. 	leaq	(BO, BI, SIZE), BO

  3764. 	negq	BI

  3765. 	negq	%rax

  3766. 	ALIGN_4

  3767. 

  3768. .L1_12:

  3769. 

  3770. 	KERNEL16x1_SUB

  3771. 	KERNEL16x1_SUB

  3772. 	KERNEL16x1_SUB

  3773. 	KERNEL16x1_SUB

  3774. 

  3775. 	KERNEL16x1_SUB

  3776. 	KERNEL16x1_SUB

  3777. 	KERNEL16x1_SUB

  3778. 	KERNEL16x1_SUB

  3779. 

  3780. 	je	.L1_16

  3781. 

  3782. 	KERNEL16x1_SUB

  3783. 	KERNEL16x1_SUB

  3784. 	KERNEL16x1_SUB

  3785. 	KERNEL16x1_SUB

  3786. 

  3787. 	KERNEL16x1_SUB

  3788. 	KERNEL16x1_SUB

  3789. 	KERNEL16x1_SUB

  3790. 	KERNEL16x1_SUB

  3791. 

  3792. 	je	.L1_16

  3793. 

  3794. 	jmp	.L1_12

  3795. 	ALIGN_4

  3796. 

  3797. .L1_16:

  3798. #ifndef TRMMKERNEL

  3799.         movq    K, %rax

  3800. #else

  3801.         movq    KKK, %rax

  3802. #endif

  3803. 

  3804. 	andq	$7, %rax		# if (k & 1)

  3805. 	je .L1_19

  3806. 

  3807. 	movq    %rax, BI                        //  Index for BO

  3808. 

  3809. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  3810. 	leaq	(AO, %rax, SIZE), AO

  3811. 	leaq	(BO, BI, SIZE), BO

  3812. 	negq	BI

  3813. 	negq	%rax

  3814. 	ALIGN_4

  3815. 

  3816. .L1_17:

  3817. 

  3818. 	KERNEL16x1_SUB

  3819. 

  3820. 	jl	.L1_17

  3821. 	ALIGN_4

  3822. 

  3823. 

  3824. .L1_19:

  3825. 

  3826. 	SAVE16x1

  3827. 

  3828. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3829.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3830.         movq    K, %rax

  3831.         subq    KKK, %rax

  3832.         movq    %rax, BI                        //  Index for BO

  3833.         leaq    (BO, BI, SIZE), BO

  3834.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  3835.         leaq    (AO, %rax, SIZE), AO

  3836. #endif

  3837. 

  3838. 

  3839. #if defined(TRMMKERNEL) && defined(LEFT)

  3840.         addq    $16, KK				

  3841. #endif

  3842. 

  3843. 	addq	$16 * SIZE, CO1		# coffset += 16

  3844. 	decq	I			# i --

  3845. 	jg	.L1_11

  3846. 	ALIGN_4	

  3847. 

  3848. /**************************************************************************

  3849. * Rest of M 

  3850. ***************************************************************************/

  3851. .L1_20:

  3852. 	// Test rest of M

  3853. 

  3854. 	testq	$15, M

  3855. 	jz	.L999

  3856. 

  3857. 	testq	$8, M		

  3858. 	jz	.L1_21pre

  3859. 	ALIGN_4

  3860. 

  3861. /**************************************************************************/

  3862. 

  3863. .L1_20_1:

  3864. #if !defined(TRMMKERNEL) || \

  3865.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3866.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3867.         leaq    BUFFER1, BO             // first buffer to BO

  3868.         addq    $4 * SIZE, BO

  3869. #else

  3870.         movq    KK, %rax

  3871.         leaq    BUFFER1, BO             // first buffer to BO

  3872.         addq    $4 * SIZE, BO

  3873.         movq    %rax, BI                        //  Index for BO

  3874.         leaq    (BO, BI, SIZE), BO

  3875.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  3876.         leaq    (AO, %rax, SIZE), AO

  3877. #endif

  3878. 

  3879. 

  3880. 	vzeroall

  3881. 

  3882. #ifndef TRMMKERNEL

  3883.         movq    K, %rax

  3884. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3885.         movq    K, %rax

  3886.         subq    KK, %rax

  3887.         movq    %rax, KKK

  3888. #else

  3889.         movq    KK, %rax

  3890. #ifdef LEFT

  3891.         addq    $8, %rax        // number of values in A

  3892. #else

  3893.         addq    $1, %rax        // number of values in BO

  3894. #endif

  3895.         movq    %rax, KKK

  3896. #endif

  3897. 

  3898. 

  3899. 	andq	$-8, %rax

  3900. 	je	.L1_20_6

  3901. 	movq    %rax, BI                        //  Index for BO

  3902. 

  3903. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  3904. 	leaq	(AO, %rax, SIZE), AO

  3905. 	leaq	(BO, BI, SIZE), BO

  3906. 	negq	BI

  3907. 	negq	%rax

  3908. 	ALIGN_4

  3909. 

  3910. .L1_20_2:

  3911. 

  3912. 	KERNEL8x1_SUB

  3913. 	KERNEL8x1_SUB

  3914. 	KERNEL8x1_SUB

  3915. 	KERNEL8x1_SUB

  3916. 

  3917. 	KERNEL8x1_SUB

  3918. 	KERNEL8x1_SUB

  3919. 	KERNEL8x1_SUB

  3920. 	KERNEL8x1_SUB

  3921. 

  3922. 	je	.L1_20_6

  3923. 

  3924. 	KERNEL8x1_SUB

  3925. 	KERNEL8x1_SUB

  3926. 	KERNEL8x1_SUB

  3927. 	KERNEL8x1_SUB

  3928. 

  3929. 	KERNEL8x1_SUB

  3930. 	KERNEL8x1_SUB

  3931. 	KERNEL8x1_SUB

  3932. 	KERNEL8x1_SUB

  3933. 

  3934. 	je	.L1_20_6

  3935. 

  3936. 	jmp	.L1_20_2

  3937. 	ALIGN_4

  3938. 

  3939. .L1_20_6:

  3940. #ifndef TRMMKERNEL

  3941.         movq    K, %rax

  3942. #else

  3943.         movq    KKK, %rax

  3944. #endif

  3945. 

  3946. 	andq	$7, %rax		# if (k & 1)

  3947. 	je .L1_20_9

  3948. 

  3949. 	movq    %rax, BI                        //  Index for BO

  3950. 

  3951. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  3952. 	leaq	(AO, %rax, SIZE), AO

  3953. 	leaq	(BO, BI, SIZE), BO

  3954. 	negq	BI

  3955. 	negq	%rax

  3956. 	ALIGN_4

  3957. 

  3958. .L1_20_7:

  3959. 

  3960. 	KERNEL8x1_SUB

  3961. 

  3962. 	jl	.L1_20_7

  3963. 	ALIGN_4

  3964. 

  3965. 

  3966. .L1_20_9:

  3967. 

  3968. 	SAVE8x1

  3969. 

  3970. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3971.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3972.         movq    K, %rax 

  3973.         subq    KKK, %rax

  3974.         movq    %rax, BI                        //  Index for BO

  3975.         leaq    (BO, BI, SIZE), BO         

  3976.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  3977.         leaq    (AO, %rax, SIZE), AO

  3978. #endif  

  3979. 

  3980. 

  3981. #if defined(TRMMKERNEL) && defined(LEFT)

  3982.         addq    $8, KK

  3983. #endif

  3984. 

  3985. 	addq	$8 * SIZE, CO1		# coffset += 8

  3986. 	ALIGN_4

  3987. 	

  3988. 

  3989. 

  3990. /**************************************************************************/

  3991. 

  3992. .L1_21pre:

  3993. 

  3994. 	testq	$4, M		

  3995. 	jz	.L1_30

  3996. 	ALIGN_4

  3997. 

  3998. .L1_21:

  3999. #if !defined(TRMMKERNEL) || \

  4000.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4001.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4002.         leaq    BUFFER1, BO             // first buffer to BO

  4003.         addq    $4 * SIZE, BO

  4004. #else

  4005.         movq    KK, %rax

  4006.         leaq    BUFFER1, BO             // first buffer to BO

  4007.         addq    $4 * SIZE, BO

  4008.         movq    %rax, BI                        //  Index for BO

  4009.         leaq    (BO, BI, SIZE), BO

  4010.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  4011.         leaq    (AO, %rax, SIZE), AO

  4012. #endif

  4013. 

  4014. 

  4015. 	vzeroall

  4016. 

  4017. #ifndef TRMMKERNEL

  4018.         movq    K, %rax

  4019. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4020.         movq    K, %rax

  4021.         subq    KK, %rax

  4022.         movq    %rax, KKK

  4023. #else

  4024.         movq    KK, %rax

  4025. #ifdef LEFT

  4026.         addq    $4, %rax        // number of values in A

  4027. #else

  4028.         addq    $1, %rax        // number of values in BO

  4029. #endif

  4030.         movq    %rax, KKK

  4031. #endif

  4032. 

  4033. 

  4034. 	andq	$-8, %rax

  4035. 	je	.L1_26

  4036. 	movq    %rax, BI                        //  Index for BO

  4037. 

  4038. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  4039. 	leaq	(AO, %rax, SIZE), AO

  4040. 	leaq	(BO, BI, SIZE), BO

  4041. 	negq	BI

  4042. 	negq	%rax

  4043. 	ALIGN_4

  4044. 

  4045. .L1_22:

  4046. 

  4047. 	KERNEL4x1_SUB

  4048. 	KERNEL4x1_SUB

  4049. 	KERNEL4x1_SUB

  4050. 	KERNEL4x1_SUB

  4051. 

  4052. 	KERNEL4x1_SUB

  4053. 	KERNEL4x1_SUB

  4054. 	KERNEL4x1_SUB

  4055. 	KERNEL4x1_SUB

  4056. 

  4057. 	je	.L1_26

  4058. 

  4059. 	KERNEL4x1_SUB

  4060. 	KERNEL4x1_SUB

  4061. 	KERNEL4x1_SUB

  4062. 	KERNEL4x1_SUB

  4063. 

  4064. 	KERNEL4x1_SUB

  4065. 	KERNEL4x1_SUB

  4066. 	KERNEL4x1_SUB

  4067. 	KERNEL4x1_SUB

  4068. 

  4069. 	je	.L1_26

  4070. 

  4071. 	jmp	.L1_22

  4072. 	ALIGN_4

  4073. 

  4074. .L1_26:

  4075. #ifndef TRMMKERNEL

  4076.         movq    K, %rax

  4077. #else

  4078.         movq    KKK, %rax

  4079. #endif

  4080. 

  4081. 	andq	$7, %rax		# if (k & 1)

  4082. 	je .L1_29

  4083. 

  4084. 	movq    %rax, BI                        //  Index for BO

  4085. 

  4086. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  4087. 	leaq	(AO, %rax, SIZE), AO

  4088. 	leaq	(BO, BI, SIZE), BO

  4089. 	negq	BI

  4090. 	negq	%rax

  4091. 	ALIGN_4

  4092. 

  4093. .L1_27:

  4094. 

  4095. 	KERNEL4x1_SUB

  4096. 

  4097. 	jl	.L1_27

  4098. 	ALIGN_4

  4099. 

  4100. 

  4101. .L1_29:

  4102. 

  4103. 	SAVE4x1

  4104. 

  4105. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4106.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4107.         movq    K, %rax 

  4108.         subq    KKK, %rax

  4109.         movq    %rax, BI                        //  Index for BO

  4110.         leaq    (BO, BI, SIZE), BO         

  4111.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  4112.         leaq    (AO, %rax, SIZE), AO

  4113. #endif  

  4114. 

  4115. 

  4116. #if defined(TRMMKERNEL) && defined(LEFT)

  4117.         addq    $4, KK

  4118. #endif

  4119. 

  4120. 	addq	$4 * SIZE, CO1		# coffset += 4

  4121. 	ALIGN_4

  4122. 	

  4123. 

  4124. .L1_30:

  4125. 	testq	$2, M		

  4126. 	jz	.L1_40

  4127. 

  4128. 	ALIGN_4

  4129. 

  4130. .L1_31:

  4131. #if !defined(TRMMKERNEL) || \

  4132.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4133.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4134.         leaq    BUFFER1, BO             // first buffer to BO

  4135.         addq    $4 * SIZE, BO

  4136. #else

  4137.         movq    KK, %rax

  4138.         leaq    BUFFER1, BO             // first buffer to BO

  4139.         addq    $4 * SIZE, BO

  4140.         movq    %rax, BI                        //  Index for BO

  4141.         leaq    (BO, BI, SIZE), BO

  4142.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  4143.         leaq    (AO, %rax, SIZE), AO

  4144. #endif

  4145. 

  4146. 

  4147. 	vzeroall

  4148. 

  4149. #ifndef TRMMKERNEL

  4150.         movq    K, %rax

  4151. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4152.         movq    K, %rax

  4153.         subq    KK, %rax

  4154.         movq    %rax, KKK

  4155. #else

  4156.         movq    KK, %rax

  4157. #ifdef LEFT

  4158.         addq    $2, %rax        // number of values in AO

  4159. #else

  4160.         addq    $1, %rax        // number of values in BO

  4161. #endif

  4162.         movq    %rax, KKK

  4163. #endif

  4164. 

  4165. 

  4166. 	andq	$-8, %rax

  4167. 	je	.L1_36

  4168. 	movq    %rax, BI                        //  Index for BO

  4169. 

  4170. 	salq	$1, %rax			// rax = rax *2 ; number of values

  4171. 	leaq	(AO, %rax, SIZE), AO

  4172. 	leaq	(BO, BI, SIZE), BO

  4173. 	negq	BI

  4174. 	negq	%rax

  4175. 	ALIGN_4

  4176. 

  4177. .L1_32:

  4178. 

  4179. 	KERNEL2x1_SUB

  4180. 	KERNEL2x1_SUB

  4181. 	KERNEL2x1_SUB

  4182. 	KERNEL2x1_SUB

  4183. 

  4184. 	KERNEL2x1_SUB

  4185. 	KERNEL2x1_SUB

  4186. 	KERNEL2x1_SUB

  4187. 	KERNEL2x1_SUB

  4188. 

  4189. 	je	.L1_36

  4190. 

  4191. 	KERNEL2x1_SUB

  4192. 	KERNEL2x1_SUB

  4193. 	KERNEL2x1_SUB

  4194. 	KERNEL2x1_SUB

  4195. 

  4196. 	KERNEL2x1_SUB

  4197. 	KERNEL2x1_SUB

  4198. 	KERNEL2x1_SUB

  4199. 	KERNEL2x1_SUB

  4200. 

  4201. 	je	.L1_36

  4202. 

  4203. 	jmp	.L1_32

  4204. 	ALIGN_4

  4205. 

  4206. .L1_36:

  4207. #ifndef TRMMKERNEL

  4208.         movq    K, %rax

  4209. #else

  4210.         movq    KKK, %rax

  4211. #endif

  4212. 

  4213. 	andq	$7, %rax		# if (k & 1)

  4214. 	je .L1_39

  4215. 

  4216. 	movq    %rax, BI                        //  Index for BO

  4217. 	

  4218. 	salq	$1, %rax			// rax = rax *2 ; number of values

  4219. 	leaq	(AO, %rax, SIZE), AO

  4220. 	leaq	(BO, BI, SIZE), BO

  4221. 	negq	BI

  4222. 	negq	%rax

  4223. 	ALIGN_4

  4224. 

  4225. .L1_37:

  4226. 

  4227. 	KERNEL2x1_SUB

  4228. 

  4229. 	jl	.L1_37

  4230. 	ALIGN_4

  4231. 

  4232. 

  4233. .L1_39:

  4234. 

  4235. 	SAVE2x1

  4236. 

  4237. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4238.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4239.         movq    K, %rax 

  4240.         subq    KKK, %rax

  4241.         movq    %rax, BI                        //  Index for BO

  4242.         leaq    (BO, BI, SIZE), BO         

  4243.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  4244.         leaq    (AO, %rax, SIZE), AO

  4245. #endif  

  4246. 

  4247. 

  4248. #if defined(TRMMKERNEL) && defined(LEFT)

  4249.         addq    $2, KK

  4250. #endif

  4251. 

  4252. 	addq	$2 * SIZE, CO1		# coffset += 2

  4253. 	ALIGN_4

  4254. 

  4255. .L1_40:

  4256. 	testq	$1, M		

  4257. 	jz	.L999

  4258. 

  4259. 	ALIGN_4

  4260. 

  4261. .L1_41:

  4262. #if !defined(TRMMKERNEL) || \

  4263.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4264.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4265.         leaq    BUFFER1, BO             // first buffer to BO

  4266.         addq    $4 * SIZE, BO

  4267. #else

  4268.         movq    KK, %rax

  4269.         leaq    BUFFER1, BO             // first buffer to BO

  4270.         addq    $4 * SIZE, BO

  4271.         movq    %rax, BI                        //  Index for BO

  4272.         leaq    (BO, BI, SIZE), BO

  4273.         leaq    (AO, %rax, SIZE), AO

  4274. #endif

  4275. 

  4276. 

  4277. 	vzeroall

  4278. 

  4279. #ifndef TRMMKERNEL

  4280.         movq    K, %rax

  4281. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4282.         movq    K, %rax

  4283.         subq    KK, %rax

  4284.         movq    %rax, KKK

  4285. #else

  4286.         movq    KK, %rax

  4287. #ifdef LEFT

  4288.         addq    $1, %rax        // number of values in AO

  4289. #else

  4290.         addq    $1, %rax        // number of values in BO

  4291. #endif

  4292.         movq    %rax, KKK

  4293. #endif

  4294. 

  4295. 	andq	$-8, %rax

  4296. 	je	.L1_46

  4297. 	movq    %rax, BI                        //  Index for BO

  4298. 

  4299. 	leaq	(AO, %rax, SIZE), AO

  4300. 	leaq	(BO, BI, SIZE), BO

  4301. 	negq	BI

  4302. 	negq	%rax

  4303. 	ALIGN_4

  4304. 

  4305. .L1_42:

  4306. 

  4307. 	KERNEL1x1_SUB

  4308. 	KERNEL1x1_SUB

  4309. 	KERNEL1x1_SUB

  4310. 	KERNEL1x1_SUB

  4311. 

  4312. 	KERNEL1x1_SUB

  4313. 	KERNEL1x1_SUB

  4314. 	KERNEL1x1_SUB

  4315. 	KERNEL1x1_SUB

  4316. 

  4317. 	je	.L1_46

  4318. 

  4319. 	KERNEL1x1_SUB

  4320. 	KERNEL1x1_SUB

  4321. 	KERNEL1x1_SUB

  4322. 	KERNEL1x1_SUB

  4323. 

  4324. 	KERNEL1x1_SUB

  4325. 	KERNEL1x1_SUB

  4326. 	KERNEL1x1_SUB

  4327. 	KERNEL1x1_SUB

  4328. 

  4329. 	je	.L1_46

  4330. 

  4331. 	jmp	.L1_42

  4332. 	ALIGN_4

  4333. 

  4334. .L1_46:

  4335. #ifndef TRMMKERNEL

  4336.         movq    K, %rax

  4337. #else

  4338.         movq    KKK, %rax

  4339. #endif

  4340. 

  4341. 	andq	$7, %rax		# if (k & 1)

  4342. 	je .L1_49

  4343. 

  4344. 	movq    %rax, BI                        //  Index for BO

  4345. 

  4346. 	leaq	(AO, %rax, SIZE), AO

  4347. 	leaq	(BO, BI, SIZE), BO

  4348. 	negq	BI

  4349. 	negq	%rax

  4350. 	ALIGN_4

  4351. 

  4352. .L1_47:

  4353. 

  4354. 	KERNEL1x1_SUB

  4355. 

  4356. 	jl	.L1_47

  4357. 	ALIGN_4

  4358. 

  4359. 

  4360. .L1_49:

  4361. 

  4362. 	SAVE1x1

  4363. 

  4364. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4365.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4366.         movq    K, %rax 

  4367.         subq    KKK, %rax

  4368.         movq    %rax, BI                        //  Index for BO

  4369.         leaq    (BO, BI, SIZE), BO         

  4370.         leaq    (AO, %rax, SIZE), AO

  4371. #endif  

  4372. 

  4373. 

  4374. #if defined(TRMMKERNEL) && defined(LEFT)

  4375.         addq    $1, KK

  4376. #endif

  4377. 

  4378. 	addq	$1 * SIZE, CO1		# coffset += 1

  4379. 	ALIGN_4

  4380. 	

  4381. 

  4382. .L999:

  4383. 	movq   		SP, %rsp

  4384. 	movq	   (%rsp), %rbx

  4385. 	movq	  8(%rsp), %rbp

  4386. 	movq	 16(%rsp), %r12

  4387. 	movq	 24(%rsp), %r13

  4388. 	movq	 32(%rsp), %r14

  4389. 	movq	 40(%rsp), %r15

  4390. 

  4391. #ifdef WINDOWS_ABI

  4392. 	movq	 48(%rsp), %rdi

  4393. 	movq	 56(%rsp), %rsi

  4394. 	movups	 64(%rsp), %xmm6

  4395. 	movups	 80(%rsp), %xmm7

  4396. 	movups	 96(%rsp), %xmm8

  4397. 	movups	112(%rsp), %xmm9

  4398. 	movups	128(%rsp), %xmm10

  4399. 	movups	144(%rsp), %xmm11

  4400. 	movups	160(%rsp), %xmm12

  4401. 	movups	176(%rsp), %xmm13

  4402. 	movups	192(%rsp), %xmm14

  4403. 	movups	208(%rsp), %xmm15

  4404. #endif

  4405. 

  4406. 	addq	$STACKSIZE, %rsp

  4407. 	ret

  4408. 

  4409. 	EPILOGUE

  4410. 

  4411. 

  4412. 

  4413. #else

  4414. 

  4415. /*************************************************************************************

  4416. * TRMM Kernel

  4417. *************************************************************************************/

  4418. 

  4419. 

  4420. 	PROLOGUE

  4421. 	PROFCODE

  4422. 	

  4423. 	subq	$STACKSIZE, %rsp

  4424. 	movq	%rbx,   (%rsp)

  4425. 	movq	%rbp,  8(%rsp)

  4426. 	movq	%r12, 16(%rsp)

  4427. 	movq	%r13, 24(%rsp)

  4428. 	movq	%r14, 32(%rsp)

  4429. 	movq	%r15, 40(%rsp)

  4430. 

  4431. 	vzeroupper

  4432. 

  4433. #ifdef WINDOWS_ABI

  4434. 	movq	%rdi,    48(%rsp)

  4435. 	movq	%rsi,    56(%rsp)

  4436. 	movups	%xmm6,   64(%rsp)

  4437. 	movups	%xmm7,   80(%rsp)

  4438. 	movups	%xmm8,   96(%rsp)

  4439. 	movups	%xmm9,  112(%rsp)

  4440. 	movups	%xmm10, 128(%rsp)

  4441. 	movups	%xmm11, 144(%rsp)

  4442. 	movups	%xmm12, 160(%rsp)

  4443. 	movups	%xmm13, 176(%rsp)

  4444. 	movups	%xmm14, 192(%rsp)

  4445. 	movups	%xmm15, 208(%rsp)

  4446. 

  4447. 	movq	ARG1,      OLD_M

  4448. 	movq	ARG2,      OLD_N

  4449. 	movq	ARG3,      OLD_K

  4450. 	movq	OLD_A,     A

  4451. 	movq	OLD_B,     B

  4452. 	movq	OLD_C,     C

  4453. 	movq	OLD_LDC,   LDC

  4454. #ifdef TRMMKERNEL

  4455. 	vmovsd	OLD_OFFSET, %xmm12

  4456. #endif

  4457. 	vmovaps	%xmm3, %xmm0

  4458. 

  4459. #else

  4460. 	movq	STACKSIZE +  8(%rsp), LDC

  4461. #ifdef TRMMKERNEL

  4462. 	movsd	STACKSIZE + 16(%rsp), %xmm12

  4463. #endif

  4464. 

  4465. #endif

  4466. 

  4467. 	movq    %rsp, SP      # save old stack

  4468.         subq    $128 + L_BUFFER_SIZE, %rsp

  4469.         andq    $-4096, %rsp    # align stack

  4470. 

  4471.         STACK_TOUCH

  4472. 

  4473. 	cmpq	$0, OLD_M

  4474. 	je	.L999

  4475. 

  4476. 	cmpq	$0, OLD_N

  4477. 	je	.L999

  4478. 

  4479. 	cmpq	$0, OLD_K

  4480. 	je	.L999

  4481. 

  4482. 	movq	OLD_M, M

  4483. 	movq	OLD_N, N

  4484. 	movq	OLD_K, K

  4485. 

  4486. 	vmovss	 %xmm0, ALPHA

  4487. 

  4488. 	salq	$BASE_SHIFT, LDC

  4489. 

  4490. 	movq    N, %rax

  4491.         xorq    %rdx, %rdx

  4492.         movq    $4,  %rdi

  4493.         divq    %rdi                    //    N / 4

  4494.         movq    %rax, Ndiv6             //    N / 4

  4495.         movq    %rdx, Nmod6             //    N % 4

  4496. 

  4497. 	

  4498. 

  4499. #ifdef TRMMKERNEL

  4500. 	vmovsd	%xmm12, OFFSET

  4501. 	vmovsd	%xmm12, KK

  4502. #ifndef LEFT

  4503. 	negq	KK

  4504. #endif	

  4505. #endif

  4506. 

  4507. 	movq	Ndiv6,  J

  4508. 	cmpq	$0, J

  4509. 	je	.L2_0

  4510. 	ALIGN_4

  4511. 

  4512. /*******************************************************************************************/

  4513. 

  4514. .L4_01:

  4515. 	// copy to sub buffer

  4516. 	movq	B, BO1

  4517. 	leaq    BUFFER1, BO		// first buffer to BO

  4518. 	movq	K, %rax

  4519. 	sarq	$2, %rax		// K / 4

  4520. 	jz	.L4_01b

  4521. 	ALIGN_4

  4522. 

  4523. 

  4524. .L4_01a:

  4525.         prefetcht0 512(BO1)

  4526.         prefetchw  512(BO)

  4527. 

  4528. 	vmovups	       (BO1), %xmm0

  4529. 	vmovups	 4*SIZE(BO1), %xmm1

  4530. 	vmovups	 8*SIZE(BO1), %xmm2

  4531. 	vmovups	12*SIZE(BO1), %xmm3

  4532. 

  4533. 	vmovups	%xmm0,       (BO)

  4534. 	vmovups	%xmm1, 4*SIZE(BO)

  4535. 	vmovups	%xmm2, 8*SIZE(BO)

  4536. 	vmovups	%xmm3,12*SIZE(BO)

  4537. 

  4538. 	addq	$ 16*SIZE,BO1

  4539. 	addq	$ 16*SIZE,BO

  4540. 	decq	%rax

  4541. 	jnz	.L4_01a

  4542. 

  4543. 

  4544. .L4_01b:

  4545. 

  4546.         movq    K, %rax

  4547.         andq    $3, %rax                // K % 4

  4548.         jz      .L4_02d

  4549.         ALIGN_4

  4550. 

  4551. .L4_02c:

  4552. 

  4553. 	vmovups	(BO1), %xmm0

  4554. 	vmovups	%xmm0, (BO)

  4555. 	addq	$ 4*SIZE,BO1

  4556. 	addq	$ 4*SIZE,BO

  4557. 	decq	%rax

  4558. 	jnz	.L4_02c

  4559. 

  4560. .L4_02d:

  4561. 

  4562. 	movq	BO1, B			// next offset of B

  4563. 

  4564. .L4_10:

  4565. 	movq	 C, CO1

  4566. 	leaq	(C, LDC, 2), CO2	

  4567. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  4568. 

  4569. #if defined(TRMMKERNEL) && defined(LEFT)

  4570.         movq    OFFSET, %rax

  4571.         movq    %rax, KK

  4572. #endif

  4573. 	

  4574. 	movq	A, AO		 	// aoffset = a

  4575. 	addq	$ 16 * SIZE, AO

  4576. 

  4577. 	movq	M,  I

  4578. 	sarq	$4, I			// i = (m >> 4)

  4579. 	je	.L4_20

  4580. 

  4581. 	ALIGN_4

  4582. 

  4583. .L4_11:

  4584. #if !defined(TRMMKERNEL) || \

  4585.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4586.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4587.         leaq    BUFFER1, BO             // first buffer to BO

  4588.         addq    $4 * SIZE, BO

  4589. #else

  4590.         movq    KK, %rax

  4591.         leaq    BUFFER1, BO             	// first buffer to BO

  4592.         addq    $4 * SIZE, BO

  4593.         movq    %rax, BI                        //  Index for BO

  4594. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  4595.         leaq    (BO, BI, SIZE), BO

  4596.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  4597.         leaq    (AO, %rax, SIZE), AO

  4598. #endif

  4599. 

  4600. 

  4601. 	vzeroall

  4602. 

  4603. #ifndef TRMMKERNEL

  4604.         movq    K, %rax

  4605. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4606.         movq    K, %rax

  4607.         subq    KK, %rax

  4608.         movq    %rax, KKK

  4609. #else

  4610.         movq    KK, %rax

  4611. #ifdef LEFT

  4612.         addq    $16, %rax	// number of values in AO

  4613. #else

  4614.         addq    $4, %rax	// number of values in BO

  4615. #endif

  4616.         movq    %rax, KKK

  4617. #endif

  4618. 

  4619. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  4620. 	je	.L4_16

  4621. 	movq    %rax, BI                        //  Index for BO

  4622.         leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values

  4623. 

  4624. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  4625. 	leaq	(AO, %rax, SIZE), AO

  4626. 	leaq	(BO, BI, SIZE), BO

  4627. 	negq	BI

  4628. 	negq	%rax

  4629. 	ALIGN_4

  4630. 

  4631. .L4_12:

  4632. 

  4633. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4634. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  4635. 	KERNEL16x4_SUB

  4636. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4637. 	KERNEL16x4_SUB

  4638. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4639. 	KERNEL16x4_SUB

  4640. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4641. 	KERNEL16x4_SUB

  4642. 

  4643. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4644. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  4645. 	KERNEL16x4_SUB

  4646. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4647. 	KERNEL16x4_SUB

  4648. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4649. 	KERNEL16x4_SUB

  4650. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4651. 	KERNEL16x4_SUB

  4652. 

  4653. 	je	.L4_16

  4654. 

  4655. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4656. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  4657. 	KERNEL16x4_SUB

  4658. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4659. 	KERNEL16x4_SUB

  4660. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4661. 	KERNEL16x4_SUB

  4662. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4663. 	KERNEL16x4_SUB

  4664. 

  4665. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4666. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  4667. 	KERNEL16x4_SUB

  4668. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4669. 	KERNEL16x4_SUB

  4670. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4671. 	KERNEL16x4_SUB

  4672. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  4673. 	KERNEL16x4_SUB

  4674. 

  4675. 	je	.L4_16

  4676. 

  4677. 	jmp	.L4_12

  4678. 	ALIGN_4

  4679. 

  4680. .L4_16:

  4681. #ifndef TRMMKERNEL

  4682.         movq    K, %rax

  4683. #else

  4684.         movq    KKK, %rax

  4685. #endif

  4686. 

  4687. 	andq	$7, %rax		# if (k & 1)

  4688. 	je .L4_19

  4689. 

  4690. 	movq    %rax, BI                        //  Index for BO

  4691.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  4692. 

  4693. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  4694. 	leaq	(AO, %rax, SIZE), AO

  4695. 	leaq	(BO, BI, SIZE), BO

  4696. 	negq	BI

  4697. 	negq	%rax

  4698. 	ALIGN_4

  4699. 

  4700. .L4_17:

  4701. 

  4702. 	KERNEL16x4_SUB

  4703. 

  4704. 	jl	.L4_17

  4705. 	ALIGN_4

  4706. 

  4707. 

  4708. .L4_19:

  4709. 

  4710. 	SAVE16x4

  4711. 

  4712. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4713.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4714.         movq    K, %rax

  4715.         subq    KKK, %rax

  4716.         movq    %rax, BI                        //  Index for BO

  4717. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  4718.         leaq    (BO, BI, SIZE), BO

  4719.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  4720.         leaq    (AO, %rax, SIZE), AO

  4721. #endif

  4722. 

  4723. 

  4724. #if defined(TRMMKERNEL) && defined(LEFT)

  4725.         addq    $16, KK				

  4726. #endif

  4727. 

  4728. 	addq	$16 * SIZE, CO1		# coffset += 16

  4729. 	addq	$16 * SIZE, CO2		# coffset += 16

  4730. 	decq	I			# i --

  4731. 	jg	.L4_11

  4732. 	ALIGN_4	

  4733. 

  4734. /**************************************************************************

  4735. * Rest of M 

  4736. ***************************************************************************/

  4737. .L4_20:

  4738. 	// Test rest of M

  4739. 

  4740. 	testq	$15, M

  4741. 	jz	.L4_60		// to next 3 lines of N

  4742. 

  4743. 	testq	$8, M		

  4744. 	jz	.L4_21pre

  4745. 	ALIGN_4

  4746. 

  4747. /**************************************************************************/

  4748. 

  4749. .L4_20_1:

  4750. #if !defined(TRMMKERNEL) || \

  4751.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4752.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4753.         leaq    BUFFER1, BO             // first buffer to BO

  4754.         addq    $4 * SIZE, BO

  4755. #else

  4756.         movq    KK, %rax

  4757.         leaq    BUFFER1, BO             // first buffer to BO

  4758.         addq    $4 * SIZE, BO

  4759.         movq    %rax, BI                        //  Index for BO

  4760. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  4761.         leaq    (BO, BI, SIZE), BO

  4762.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  4763.         leaq    (AO, %rax, SIZE), AO

  4764. #endif

  4765. 

  4766. 

  4767. 	vzeroall

  4768. 

  4769. #ifndef TRMMKERNEL

  4770.         movq    K, %rax

  4771. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4772.         movq    K, %rax

  4773.         subq    KK, %rax

  4774.         movq    %rax, KKK

  4775. #else

  4776.         movq    KK, %rax

  4777. #ifdef LEFT

  4778.         addq    $8, %rax        // number of values in A

  4779. #else

  4780.         addq    $4, %rax        // number of values in BO

  4781. #endif

  4782.         movq    %rax, KKK

  4783. #endif

  4784. 

  4785. 

  4786. 	andq	$-8, %rax

  4787. 	je	.L4_20_6

  4788. 	movq    %rax, BI                        //  Index for BO

  4789.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  4790. 

  4791. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  4792. 	leaq	(AO, %rax, SIZE), AO

  4793. 	leaq	(BO, BI, SIZE), BO

  4794. 	negq	BI

  4795. 	negq	%rax

  4796. 	ALIGN_4

  4797. 

  4798. .L4_20_2:

  4799. 

  4800. 	KERNEL8x4_SUB

  4801. 	KERNEL8x4_SUB

  4802. 	KERNEL8x4_SUB

  4803. 	KERNEL8x4_SUB

  4804. 

  4805. 	KERNEL8x4_SUB

  4806. 	KERNEL8x4_SUB

  4807. 	KERNEL8x4_SUB

  4808. 	KERNEL8x4_SUB

  4809. 

  4810. 	je	.L4_20_6

  4811. 

  4812. 	KERNEL8x4_SUB

  4813. 	KERNEL8x4_SUB

  4814. 	KERNEL8x4_SUB

  4815. 	KERNEL8x4_SUB

  4816. 

  4817. 	KERNEL8x4_SUB

  4818. 	KERNEL8x4_SUB

  4819. 	KERNEL8x4_SUB

  4820. 	KERNEL8x4_SUB

  4821. 

  4822. 	je	.L4_20_6

  4823. 

  4824. 	jmp	.L4_20_2

  4825. 	ALIGN_4

  4826. 

  4827. .L4_20_6:

  4828. #ifndef TRMMKERNEL

  4829.         movq    K, %rax

  4830. #else

  4831.         movq    KKK, %rax

  4832. #endif

  4833. 

  4834. 	andq	$7, %rax		# if (k & 1)

  4835. 	je .L4_20_9

  4836. 

  4837. 	movq    %rax, BI                        //  Index for BO

  4838.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  4839. 

  4840. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  4841. 	leaq	(AO, %rax, SIZE), AO

  4842. 	leaq	(BO, BI, SIZE), BO

  4843. 	negq	BI

  4844. 	negq	%rax

  4845. 	ALIGN_4

  4846. 

  4847. .L4_20_7:

  4848. 

  4849. 	KERNEL8x4_SUB

  4850. 

  4851. 	jl	.L4_20_7

  4852. 	ALIGN_4

  4853. 

  4854. 

  4855. .L4_20_9:

  4856. 

  4857. 	SAVE8x4

  4858. 

  4859. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4860.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4861.         movq    K, %rax 

  4862.         subq    KKK, %rax

  4863.         movq    %rax, BI                        //  Index for BO

  4864. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  4865.         leaq    (BO, BI, SIZE), BO         

  4866.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  4867.         leaq    (AO, %rax, SIZE), AO

  4868. #endif  

  4869. 

  4870. 

  4871. #if defined(TRMMKERNEL) && defined(LEFT)

  4872.         addq    $8, KK

  4873. #endif

  4874. 

  4875. 	addq	$8 * SIZE, CO1		# coffset += 8

  4876. 	addq	$8 * SIZE, CO2		# coffset += 8

  4877. 	ALIGN_4

  4878. 	

  4879. 

  4880. 

  4881. /**************************************************************************/

  4882. 

  4883. .L4_21pre:

  4884. 

  4885. 	testq	$4, M		

  4886. 	jz	.L4_30

  4887. 	ALIGN_4

  4888. 

  4889. .L4_21:

  4890. #if !defined(TRMMKERNEL) || \

  4891.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  4892.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  4893.         leaq    BUFFER1, BO             // first buffer to BO

  4894.         addq    $4 * SIZE, BO

  4895. #else

  4896.         movq    KK, %rax

  4897.         leaq    BUFFER1, BO             // first buffer to BO

  4898.         addq    $4 * SIZE, BO

  4899.         movq    %rax, BI                        //  Index for BO

  4900. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  4901.         leaq    (BO, BI, SIZE), BO

  4902.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  4903.         leaq    (AO, %rax, SIZE), AO

  4904. #endif

  4905. 

  4906. 

  4907. 	vzeroall

  4908. 

  4909. #ifndef TRMMKERNEL

  4910.         movq    K, %rax

  4911. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  4912.         movq    K, %rax

  4913.         subq    KK, %rax

  4914.         movq    %rax, KKK

  4915. #else

  4916.         movq    KK, %rax

  4917. #ifdef LEFT

  4918.         addq    $4, %rax        // number of values in A

  4919. #else

  4920.         addq    $4, %rax        // number of values in BO

  4921. #endif

  4922.         movq    %rax, KKK

  4923. #endif

  4924. 

  4925. 

  4926. 	andq	$-8, %rax

  4927. 	je	.L4_26

  4928. 	movq    %rax, BI                        //  Index for BO

  4929.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  4930. 

  4931. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  4932. 	leaq	(AO, %rax, SIZE), AO

  4933. 	leaq	(BO, BI, SIZE), BO

  4934. 	negq	BI

  4935. 	negq	%rax

  4936. 	ALIGN_4

  4937. 

  4938. .L4_22:

  4939. 

  4940. 	KERNEL4x4_SUB

  4941. 	KERNEL4x4_SUB

  4942. 	KERNEL4x4_SUB

  4943. 	KERNEL4x4_SUB

  4944. 

  4945. 	KERNEL4x4_SUB

  4946. 	KERNEL4x4_SUB

  4947. 	KERNEL4x4_SUB

  4948. 	KERNEL4x4_SUB

  4949. 

  4950. 	je	.L4_26

  4951. 

  4952. 	KERNEL4x4_SUB

  4953. 	KERNEL4x4_SUB

  4954. 	KERNEL4x4_SUB

  4955. 	KERNEL4x4_SUB

  4956. 

  4957. 	KERNEL4x4_SUB

  4958. 	KERNEL4x4_SUB

  4959. 	KERNEL4x4_SUB

  4960. 	KERNEL4x4_SUB

  4961. 

  4962. 	je	.L4_26

  4963. 

  4964. 	jmp	.L4_22

  4965. 	ALIGN_4

  4966. 

  4967. .L4_26:

  4968. #ifndef TRMMKERNEL

  4969.         movq    K, %rax

  4970. #else

  4971.         movq    KKK, %rax

  4972. #endif

  4973. 

  4974. 	andq	$7, %rax		# if (k & 1)

  4975. 	je .L4_29

  4976. 

  4977. 	movq    %rax, BI                        //  Index for BO

  4978.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  4979. 

  4980. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  4981. 	leaq	(AO, %rax, SIZE), AO

  4982. 	leaq	(BO, BI, SIZE), BO

  4983. 	negq	BI

  4984. 	negq	%rax

  4985. 	ALIGN_4

  4986. 

  4987. .L4_27:

  4988. 

  4989. 	KERNEL4x4_SUB

  4990. 

  4991. 	jl	.L4_27

  4992. 	ALIGN_4

  4993. 

  4994. 

  4995. .L4_29:

  4996. 

  4997. 	SAVE4x4

  4998. 

  4999. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5000.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5001.         movq    K, %rax 

  5002.         subq    KKK, %rax

  5003.         movq    %rax, BI                        //  Index for BO

  5004. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5005.         leaq    (BO, BI, SIZE), BO         

  5006.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  5007.         leaq    (AO, %rax, SIZE), AO

  5008. #endif  

  5009. 

  5010. 

  5011. #if defined(TRMMKERNEL) && defined(LEFT)

  5012.         addq    $4, KK

  5013. #endif

  5014. 

  5015. 	addq	$4 * SIZE, CO1		# coffset += 4

  5016. 	addq	$4 * SIZE, CO2		# coffset += 4

  5017. 	ALIGN_4

  5018. 	

  5019. 

  5020. .L4_30:

  5021. 	testq	$2, M		

  5022. 	jz	.L4_40

  5023. 

  5024. 	ALIGN_4

  5025. 

  5026. .L4_31:

  5027. #if !defined(TRMMKERNEL) || \

  5028.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5029.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5030.         leaq    BUFFER1, BO             // first buffer to BO

  5031.         addq    $4 * SIZE, BO

  5032. #else

  5033.         movq    KK, %rax

  5034.         leaq    BUFFER1, BO             // first buffer to BO

  5035.         addq    $4 * SIZE, BO

  5036.         movq    %rax, BI                        //  Index for BO

  5037. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5038.         leaq    (BO, BI, SIZE), BO

  5039.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  5040.         leaq    (AO, %rax, SIZE), AO

  5041. #endif

  5042. 

  5043. 

  5044. 	vzeroall

  5045. 

  5046. #ifndef TRMMKERNEL

  5047.         movq    K, %rax

  5048. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5049.         movq    K, %rax

  5050.         subq    KK, %rax

  5051.         movq    %rax, KKK

  5052. #else

  5053.         movq    KK, %rax

  5054. #ifdef LEFT

  5055.         addq    $2, %rax        // number of values in AO

  5056. #else

  5057.         addq    $4, %rax        // number of values in BO

  5058. #endif

  5059.         movq    %rax, KKK

  5060. #endif

  5061. 

  5062. 

  5063. 	andq	$-8, %rax

  5064. 	je	.L4_36

  5065. 	movq    %rax, BI                        //  Index for BO

  5066.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  5067. 

  5068. 	salq	$1, %rax			// rax = rax *2 ; number of values

  5069. 	leaq	(AO, %rax, SIZE), AO

  5070. 	leaq	(BO, BI, SIZE), BO

  5071. 	negq	BI

  5072. 	negq	%rax

  5073. 	ALIGN_4

  5074. 

  5075. .L4_32:

  5076. 

  5077. 	KERNEL2x4_SUB

  5078. 	KERNEL2x4_SUB

  5079. 	KERNEL2x4_SUB

  5080. 	KERNEL2x4_SUB

  5081. 

  5082. 	KERNEL2x4_SUB

  5083. 	KERNEL2x4_SUB

  5084. 	KERNEL2x4_SUB

  5085. 	KERNEL2x4_SUB

  5086. 

  5087. 	je	.L4_36

  5088. 

  5089. 	KERNEL2x4_SUB

  5090. 	KERNEL2x4_SUB

  5091. 	KERNEL2x4_SUB

  5092. 	KERNEL2x4_SUB

  5093. 

  5094. 	KERNEL2x4_SUB

  5095. 	KERNEL2x4_SUB

  5096. 	KERNEL2x4_SUB

  5097. 	KERNEL2x4_SUB

  5098. 

  5099. 	je	.L4_36

  5100. 

  5101. 	jmp	.L4_32

  5102. 	ALIGN_4

  5103. 

  5104. .L4_36:

  5105. #ifndef TRMMKERNEL

  5106.         movq    K, %rax

  5107. #else

  5108.         movq    KKK, %rax

  5109. #endif

  5110. 

  5111. 	andq	$7, %rax		# if (k & 1)

  5112. 	je .L4_39

  5113. 

  5114. 	movq    %rax, BI                        //  Index for BO

  5115. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5116. 	

  5117. 	salq	$1, %rax			// rax = rax *2 ; number of values

  5118. 	leaq	(AO, %rax, SIZE), AO

  5119. 	leaq	(BO, BI, SIZE), BO

  5120. 	negq	BI

  5121. 	negq	%rax

  5122. 	ALIGN_4

  5123. 

  5124. .L4_37:

  5125. 

  5126. 	KERNEL2x4_SUB

  5127. 

  5128. 	jl	.L4_37

  5129. 	ALIGN_4

  5130. 

  5131. 

  5132. .L4_39:

  5133. 

  5134. 	SAVE2x4

  5135. 

  5136. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5137.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5138.         movq    K, %rax 

  5139.         subq    KKK, %rax

  5140.         movq    %rax, BI                        //  Index for BO

  5141. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5142.         leaq    (BO, BI, SIZE), BO         

  5143.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  5144.         leaq    (AO, %rax, SIZE), AO

  5145. #endif  

  5146. 

  5147. 

  5148. #if defined(TRMMKERNEL) && defined(LEFT)

  5149.         addq    $2, KK

  5150. #endif

  5151. 

  5152. 	addq	$2 * SIZE, CO1		# coffset += 2

  5153. 	addq	$2 * SIZE, CO2		# coffset += 2

  5154. 	ALIGN_4

  5155. 

  5156. .L4_40:

  5157. 	testq	$1, M		

  5158. 	jz	.L4_60		// to next 4 lines of N

  5159. 

  5160. 	ALIGN_4

  5161. 

  5162. .L4_41:

  5163. #if !defined(TRMMKERNEL) || \

  5164.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5165.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5166.         leaq    BUFFER1, BO             // first buffer to BO

  5167.         addq    $4 * SIZE, BO

  5168. #else

  5169.         movq    KK, %rax

  5170.         leaq    BUFFER1, BO             // first buffer to BO

  5171.         addq    $4 * SIZE, BO

  5172.         movq    %rax, BI                        //  Index for BO

  5173. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5174.         leaq    (BO, BI, SIZE), BO

  5175.         leaq    (AO, %rax, SIZE), AO

  5176. #endif

  5177. 

  5178. 

  5179. 	vzeroall

  5180. 

  5181. #ifndef TRMMKERNEL

  5182.         movq    K, %rax

  5183. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5184.         movq    K, %rax

  5185.         subq    KK, %rax

  5186.         movq    %rax, KKK

  5187. #else

  5188.         movq    KK, %rax

  5189. #ifdef LEFT

  5190.         addq    $1, %rax        // number of values in AO

  5191. #else

  5192.         addq    $4, %rax        // number of values in BO

  5193. #endif

  5194.         movq    %rax, KKK

  5195. #endif

  5196. 

  5197. 	andq	$-8, %rax

  5198. 	je	.L4_46

  5199. 	movq    %rax, BI                        //  Index for BO

  5200.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  5201. 

  5202. 	leaq	(AO, %rax, SIZE), AO

  5203. 	leaq	(BO, BI, SIZE), BO

  5204. 	negq	BI

  5205. 	negq	%rax

  5206. 	ALIGN_4

  5207. 

  5208. .L4_42:

  5209. 

  5210. 	KERNEL1x4_SUB

  5211. 	KERNEL1x4_SUB

  5212. 	KERNEL1x4_SUB

  5213. 	KERNEL1x4_SUB

  5214. 

  5215. 	KERNEL1x4_SUB

  5216. 	KERNEL1x4_SUB

  5217. 	KERNEL1x4_SUB

  5218. 	KERNEL1x4_SUB

  5219. 

  5220. 	je	.L4_46

  5221. 

  5222. 	KERNEL1x4_SUB

  5223. 	KERNEL1x4_SUB

  5224. 	KERNEL1x4_SUB

  5225. 	KERNEL1x4_SUB

  5226. 

  5227. 	KERNEL1x4_SUB

  5228. 	KERNEL1x4_SUB

  5229. 	KERNEL1x4_SUB

  5230. 	KERNEL1x4_SUB

  5231. 

  5232. 	je	.L4_46

  5233. 

  5234. 	jmp	.L4_42

  5235. 	ALIGN_4

  5236. 

  5237. .L4_46:

  5238. #ifndef TRMMKERNEL

  5239.         movq    K, %rax

  5240. #else

  5241.         movq    KKK, %rax

  5242. #endif

  5243. 

  5244. 	andq	$7, %rax		# if (k & 1)

  5245. 	je .L4_49

  5246. 

  5247. 	movq    %rax, BI                        //  Index for BO

  5248.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  5249. 

  5250. 	leaq	(AO, %rax, SIZE), AO

  5251. 	leaq	(BO, BI, SIZE), BO

  5252. 	negq	BI

  5253. 	negq	%rax

  5254. 	ALIGN_4

  5255. 

  5256. .L4_47:

  5257. 

  5258. 	KERNEL1x4_SUB

  5259. 

  5260. 	jl	.L4_47

  5261. 	ALIGN_4

  5262. 

  5263. 

  5264. .L4_49:

  5265. 

  5266. 	SAVE1x4

  5267. 

  5268. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5269.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5270.         movq    K, %rax 

  5271.         subq    KKK, %rax

  5272.         movq    %rax, BI                        //  Index for BO

  5273. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  5274.         leaq    (BO, BI, SIZE), BO         

  5275.         leaq    (AO, %rax, SIZE), AO

  5276. #endif  

  5277. 

  5278. 

  5279. #if defined(TRMMKERNEL) && defined(LEFT)

  5280.         addq    $1, KK

  5281. #endif

  5282. 

  5283. 	addq	$1 * SIZE, CO1		# coffset += 1

  5284. 	addq	$1 * SIZE, CO2		# coffset += 1

  5285. 	ALIGN_4

  5286. 	

  5287. 

  5288. 

  5289. 

  5290. 	

  5291. .L4_60:

  5292. #if defined(TRMMKERNEL) && !defined(LEFT)

  5293.         addq    $4, KK

  5294. #endif

  5295. 

  5296. 	decq	J			// j --

  5297. 	jg	.L4_01			// next 4 lines of N

  5298. 

  5299. 

  5300. 

  5301. /*******************************************************************************************/

  5302. .L2_0:

  5303. 

  5304. 	movq	Nmod6, J		

  5305. 	andq	$3, J			// j % 4

  5306. 	je	.L999

  5307. 

  5308. 	movq	Nmod6, J		

  5309. 	andq	$2, J			// j % 4

  5310. 	je	.L1_0

  5311. 

  5312. .L2_01:

  5313. 

  5314. 	// copy to sub buffer

  5315. 	movq	B, BO1

  5316. 	leaq    BUFFER1, BO		// first buffer to BO

  5317. 	movq	K, %rax

  5318. 	sarq	$2, %rax		// K / 4

  5319. 	jz	.L2_01b

  5320. 	ALIGN_4

  5321. 

  5322. .L2_01a:

  5323. 

  5324. 	vmovsd	      (BO1), %xmm0

  5325. 	vmovsd	2*SIZE(BO1), %xmm1

  5326. 	vmovsd	4*SIZE(BO1), %xmm2

  5327. 	vmovsd	6*SIZE(BO1), %xmm3

  5328. 

  5329. 	vmovsd	%xmm0,       (BO)

  5330. 	vmovsd	%xmm1, 2*SIZE(BO)

  5331. 	vmovsd	%xmm2, 4*SIZE(BO)

  5332. 	vmovsd	%xmm3, 6*SIZE(BO)

  5333. 

  5334. 	addq	$8*SIZE,BO1

  5335. 	addq	$8*SIZE,BO

  5336. 	decq	%rax

  5337. 	jnz	.L2_01a

  5338. 

  5339. 

  5340. .L2_01b:

  5341. 

  5342.         movq    K, %rax

  5343.         andq    $3, %rax                // K % 4

  5344.         jz      .L2_02d

  5345.         ALIGN_4

  5346. 

  5347. .L2_02c:

  5348. 

  5349. 	vmovsd 	(BO1), %xmm0

  5350. 	vmovsd 	%xmm0, (BO)

  5351. 	addq	$2*SIZE,BO1

  5352. 	addq	$2*SIZE,BO

  5353. 	decq	%rax

  5354. 	jnz	.L2_02c

  5355. 

  5356. .L2_02d:

  5357. 

  5358. 	movq	BO1, B			// next offset of B

  5359. 

  5360. .L2_10:

  5361. 	movq	C, CO1

  5362. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  5363. 

  5364. #if defined(TRMMKERNEL) && defined(LEFT)

  5365.         movq    OFFSET, %rax

  5366.         movq    %rax, KK

  5367. #endif

  5368. 	

  5369. 	movq	A, AO		 	// aoffset = a

  5370. 	addq	$16 * SIZE, AO

  5371. 

  5372. 	movq	M,  I

  5373. 	sarq	$4, I			// i = (m >> 4)

  5374. 	je	.L2_20

  5375. 

  5376. 	ALIGN_4

  5377. 

  5378. .L2_11:

  5379. #if !defined(TRMMKERNEL) || \

  5380.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5381.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5382.         leaq    BUFFER1, BO             // first buffer to BO

  5383.         addq    $4 * SIZE, BO

  5384. #else

  5385.         movq    KK, %rax

  5386.         leaq    BUFFER1, BO             // first buffer to BO

  5387.         addq    $4 * SIZE, BO

  5388.         movq    %rax, BI                        //  Index for BO

  5389.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5390.         leaq    (BO, BI, SIZE), BO

  5391.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  5392.         leaq    (AO, %rax, SIZE), AO

  5393. #endif

  5394. 

  5395. 

  5396. 	vzeroall

  5397. 

  5398. #ifndef TRMMKERNEL

  5399.         movq    K, %rax

  5400. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5401.         movq    K, %rax

  5402.         subq    KK, %rax

  5403.         movq    %rax, KKK

  5404. #else

  5405.         movq    KK, %rax

  5406. #ifdef LEFT

  5407.         addq    $16, %rax	// number of values in AO

  5408. #else

  5409.         addq    $2, %rax	// number of values in BO

  5410. #endif

  5411.         movq    %rax, KKK

  5412. #endif

  5413. 

  5414. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  5415. 	je	.L2_16

  5416. 	movq    %rax, BI                        //  Index for BO

  5417.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5418. 

  5419. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  5420. 	leaq	(AO, %rax, SIZE), AO

  5421. 	leaq	(BO, BI, SIZE), BO

  5422. 	negq	BI

  5423. 	negq	%rax

  5424. 	ALIGN_4

  5425. 

  5426. .L2_12:

  5427. 

  5428. 	KERNEL16x2_SUB

  5429. 	KERNEL16x2_SUB

  5430. 	KERNEL16x2_SUB

  5431. 	KERNEL16x2_SUB

  5432. 

  5433. 	KERNEL16x2_SUB

  5434. 	KERNEL16x2_SUB

  5435. 	KERNEL16x2_SUB

  5436. 	KERNEL16x2_SUB

  5437. 

  5438. 	je	.L2_16

  5439. 

  5440. 	KERNEL16x2_SUB

  5441. 	KERNEL16x2_SUB

  5442. 	KERNEL16x2_SUB

  5443. 	KERNEL16x2_SUB

  5444. 

  5445. 	KERNEL16x2_SUB

  5446. 	KERNEL16x2_SUB

  5447. 	KERNEL16x2_SUB

  5448. 	KERNEL16x2_SUB

  5449. 

  5450. 	je	.L2_16

  5451. 

  5452. 	jmp	.L2_12

  5453. 	ALIGN_4

  5454. 

  5455. .L2_16:

  5456. #ifndef TRMMKERNEL

  5457.         movq    K, %rax

  5458. #else

  5459.         movq    KKK, %rax

  5460. #endif

  5461. 

  5462. 	andq	$7, %rax		# if (k & 1)

  5463. 	je .L2_19

  5464. 

  5465. 	movq    %rax, BI                        //  Index for BO

  5466.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5467. 

  5468. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  5469. 	leaq	(AO, %rax, SIZE), AO

  5470. 	leaq	(BO, BI, SIZE), BO

  5471. 	negq	BI

  5472. 	negq	%rax

  5473. 	ALIGN_4

  5474. 

  5475. .L2_17:

  5476. 

  5477. 	KERNEL16x2_SUB

  5478. 

  5479. 	jl	.L2_17

  5480. 	ALIGN_4

  5481. 

  5482. 

  5483. .L2_19:

  5484. 

  5485. 	SAVE16x2

  5486. 

  5487. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5488.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5489.         movq    K, %rax

  5490.         subq    KKK, %rax

  5491.         movq    %rax, BI                        //  Index for BO

  5492.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5493.         leaq    (BO, BI, SIZE), BO

  5494.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  5495.         leaq    (AO, %rax, SIZE), AO

  5496. #endif

  5497. 

  5498. 

  5499. #if defined(TRMMKERNEL) && defined(LEFT)

  5500.         addq    $16, KK				

  5501. #endif

  5502. 

  5503. 	addq	$16 * SIZE, CO1		# coffset += 16

  5504. 	decq	I			# i --

  5505. 	jg	.L2_11

  5506. 	ALIGN_4	

  5507. 

  5508. /**************************************************************************

  5509. * Rest of M 

  5510. ***************************************************************************/

  5511. .L2_20:

  5512. 	// Test rest of M

  5513. 

  5514. 	testq	$15, M

  5515. 	jz	.L2_60		// to next 2 lines of N

  5516. 

  5517. 	testq	$8, M		

  5518. 	jz	.L2_21pre

  5519. 	ALIGN_4

  5520. 

  5521. /**************************************************************************/

  5522. 

  5523. .L2_20_1:

  5524. #if !defined(TRMMKERNEL) || \

  5525.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5526.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5527.         leaq    BUFFER1, BO             // first buffer to BO

  5528.         addq    $4 * SIZE, BO

  5529. #else

  5530.         movq    KK, %rax

  5531.         leaq    BUFFER1, BO             // first buffer to BO

  5532.         addq    $4 * SIZE, BO

  5533.         movq    %rax, BI                        //  Index for BO

  5534.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5535.         leaq    (BO, BI, SIZE), BO

  5536.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  5537.         leaq    (AO, %rax, SIZE), AO

  5538. #endif

  5539. 

  5540. 

  5541. 	vzeroall

  5542. 

  5543. #ifndef TRMMKERNEL

  5544.         movq    K, %rax

  5545. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5546.         movq    K, %rax

  5547.         subq    KK, %rax

  5548.         movq    %rax, KKK

  5549. #else

  5550.         movq    KK, %rax

  5551. #ifdef LEFT

  5552.         addq    $8, %rax        // number of values in A

  5553. #else

  5554.         addq    $2, %rax        // number of values in BO

  5555. #endif

  5556.         movq    %rax, KKK

  5557. #endif

  5558. 

  5559. 

  5560. 	andq	$-8, %rax

  5561. 	je	.L2_20_6

  5562. 	movq    %rax, BI                        //  Index for BO

  5563.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5564. 

  5565. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  5566. 	leaq	(AO, %rax, SIZE), AO

  5567. 	leaq	(BO, BI, SIZE), BO

  5568. 	negq	BI

  5569. 	negq	%rax

  5570. 	ALIGN_4

  5571. 

  5572. .L2_20_2:

  5573. 

  5574. 

  5575. 	KERNEL8x2_SUB

  5576. 	KERNEL8x2_SUB

  5577. 	KERNEL8x2_SUB

  5578. 	KERNEL8x2_SUB

  5579. 

  5580. 	KERNEL8x2_SUB

  5581. 	KERNEL8x2_SUB

  5582. 	KERNEL8x2_SUB

  5583. 	KERNEL8x2_SUB

  5584. 

  5585. 	je	.L2_20_6

  5586. 

  5587. 	KERNEL8x2_SUB

  5588. 	KERNEL8x2_SUB

  5589. 	KERNEL8x2_SUB

  5590. 	KERNEL8x2_SUB

  5591. 

  5592. 	KERNEL8x2_SUB

  5593. 	KERNEL8x2_SUB

  5594. 	KERNEL8x2_SUB

  5595. 	KERNEL8x2_SUB

  5596. 

  5597. 	je	.L2_20_6

  5598. 

  5599. 	jmp	.L2_20_2

  5600. 	ALIGN_4

  5601. 

  5602. .L2_20_6:

  5603. #ifndef TRMMKERNEL

  5604.         movq    K, %rax

  5605. #else

  5606.         movq    KKK, %rax

  5607. #endif

  5608. 

  5609. 	andq	$7, %rax		# if (k & 1)

  5610. 	je .L2_20_9

  5611. 

  5612. 	movq    %rax, BI                        //  Index for BO

  5613.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5614. 

  5615. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  5616. 	leaq	(AO, %rax, SIZE), AO

  5617. 	leaq	(BO, BI, SIZE), BO

  5618. 	negq	BI

  5619. 	negq	%rax

  5620. 	ALIGN_4

  5621. 

  5622. .L2_20_7:

  5623. 

  5624. 	KERNEL8x2_SUB

  5625. 

  5626. 	jl	.L2_20_7

  5627. 	ALIGN_4

  5628. 

  5629. 

  5630. .L2_20_9:

  5631. 

  5632. 	SAVE8x2

  5633. 

  5634. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5635.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5636.         movq    K, %rax 

  5637.         subq    KKK, %rax

  5638.         movq    %rax, BI                        //  Index for BO

  5639.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5640.         leaq    (BO, BI, SIZE), BO         

  5641.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  5642.         leaq    (AO, %rax, SIZE), AO

  5643. #endif  

  5644. 

  5645. 

  5646. #if defined(TRMMKERNEL) && defined(LEFT)

  5647.         addq    $8, KK

  5648. #endif

  5649. 

  5650. 	addq	$8 * SIZE, CO1		# coffset += 8

  5651. 	ALIGN_4

  5652. 	

  5653. 

  5654. 

  5655. /**************************************************************************/

  5656. 

  5657. .L2_21pre:

  5658. 

  5659. 	testq	$4, M		

  5660. 	jz	.L2_30

  5661. 	ALIGN_4

  5662. 

  5663. .L2_21:

  5664. #if !defined(TRMMKERNEL) || \

  5665.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5666.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5667.         leaq    BUFFER1, BO             // first buffer to BO

  5668.         addq    $4 * SIZE, BO

  5669. #else

  5670.         movq    KK, %rax

  5671.         leaq    BUFFER1, BO             // first buffer to BO

  5672.         addq    $4 * SIZE, BO

  5673.         movq    %rax, BI                        //  Index for BO

  5674.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5675.         leaq    (BO, BI, SIZE), BO

  5676.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  5677.         leaq    (AO, %rax, SIZE), AO

  5678. #endif

  5679. 

  5680. 

  5681. 	vzeroall

  5682. 

  5683. #ifndef TRMMKERNEL

  5684.         movq    K, %rax

  5685. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5686.         movq    K, %rax

  5687.         subq    KK, %rax

  5688.         movq    %rax, KKK

  5689. #else

  5690.         movq    KK, %rax

  5691. #ifdef LEFT

  5692.         addq    $4, %rax        // number of values in A

  5693. #else

  5694.         addq    $2, %rax        // number of values in BO

  5695. #endif

  5696.         movq    %rax, KKK

  5697. #endif

  5698. 

  5699. 

  5700. 	andq	$-8, %rax

  5701. 	je	.L2_26

  5702. 	movq    %rax, BI                        //  Index for BO

  5703.         leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values

  5704. 

  5705. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  5706. 	leaq	(AO, %rax, SIZE), AO

  5707. 	leaq	(BO, BI, SIZE), BO

  5708. 	negq	BI

  5709. 	negq	%rax

  5710. 	ALIGN_4

  5711. 

  5712. .L2_22:

  5713. 

  5714. 

  5715. 	KERNEL4x2_SUB

  5716. 	KERNEL4x2_SUB

  5717. 	KERNEL4x2_SUB

  5718. 	KERNEL4x2_SUB

  5719. 

  5720. 	KERNEL4x2_SUB

  5721. 	KERNEL4x2_SUB

  5722. 	KERNEL4x2_SUB

  5723. 	KERNEL4x2_SUB

  5724. 

  5725. 	je	.L2_26

  5726. 

  5727. 	KERNEL4x2_SUB

  5728. 	KERNEL4x2_SUB

  5729. 	KERNEL4x2_SUB

  5730. 	KERNEL4x2_SUB

  5731. 

  5732. 	KERNEL4x2_SUB

  5733. 	KERNEL4x2_SUB

  5734. 	KERNEL4x2_SUB

  5735. 	KERNEL4x2_SUB

  5736. 

  5737. 	je	.L2_26

  5738. 

  5739. 	jmp	.L2_22

  5740. 	ALIGN_4

  5741. 

  5742. .L2_26:

  5743. #ifndef TRMMKERNEL

  5744.         movq    K, %rax

  5745. #else

  5746.         movq    KKK, %rax

  5747. #endif

  5748. 

  5749. 	andq	$7, %rax		# if (k & 1)

  5750. 	je .L2_29

  5751. 

  5752. 	movq    %rax, BI                        //  Index for BO

  5753.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5754. 

  5755. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  5756. 	leaq	(AO, %rax, SIZE), AO

  5757. 	leaq	(BO, BI, SIZE), BO

  5758. 	negq	BI

  5759. 	negq	%rax

  5760. 	ALIGN_4

  5761. 

  5762. .L2_27:

  5763. 

  5764. 	KERNEL4x2_SUB

  5765. 

  5766. 	jl	.L2_27

  5767. 	ALIGN_4

  5768. 

  5769. 

  5770. .L2_29:

  5771. 

  5772. 	SAVE4x2

  5773. 

  5774. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5775.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5776.         movq    K, %rax 

  5777.         subq    KKK, %rax

  5778.         movq    %rax, BI                        //  Index for BO

  5779.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5780.         leaq    (BO, BI, SIZE), BO         

  5781.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  5782.         leaq    (AO, %rax, SIZE), AO

  5783. #endif  

  5784. 

  5785. 

  5786. #if defined(TRMMKERNEL) && defined(LEFT)

  5787.         addq    $4, KK

  5788. #endif

  5789. 

  5790. 	addq	$4 * SIZE, CO1		# coffset += 4

  5791. 	ALIGN_4

  5792. 	

  5793. 

  5794. .L2_30:

  5795. 	testq	$2, M		

  5796. 	jz	.L2_40

  5797. 

  5798. 	ALIGN_4

  5799. 

  5800. .L2_31:

  5801. #if !defined(TRMMKERNEL) || \

  5802.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5803.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5804.         leaq    BUFFER1, BO             // first buffer to BO

  5805.         addq    $4 * SIZE, BO

  5806. #else

  5807.         movq    KK, %rax

  5808.         leaq    BUFFER1, BO             // first buffer to BO

  5809.         addq    $4 * SIZE, BO

  5810.         movq    %rax, BI                        //  Index for BO

  5811.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5812.         leaq    (BO, BI, SIZE), BO

  5813.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  5814.         leaq    (AO, %rax, SIZE), AO

  5815. #endif

  5816. 

  5817. 

  5818. 	vzeroall

  5819. 

  5820. #ifndef TRMMKERNEL

  5821.         movq    K, %rax

  5822. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5823.         movq    K, %rax

  5824.         subq    KK, %rax

  5825.         movq    %rax, KKK

  5826. #else

  5827.         movq    KK, %rax

  5828. #ifdef LEFT

  5829.         addq    $2, %rax        // number of values in AO

  5830. #else

  5831.         addq    $2, %rax        // number of values in BO

  5832. #endif

  5833.         movq    %rax, KKK

  5834. #endif

  5835. 

  5836. 

  5837. 	andq	$-8, %rax

  5838. 	je	.L2_36

  5839. 	movq    %rax, BI                        //  Index for BO

  5840.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5841. 

  5842. 	salq	$1, %rax			// rax = rax *2 ; number of values

  5843. 	leaq	(AO, %rax, SIZE), AO

  5844. 	leaq	(BO, BI, SIZE), BO

  5845. 	negq	BI

  5846. 	negq	%rax

  5847. 	ALIGN_4

  5848. 

  5849. .L2_32:

  5850. 

  5851. 	KERNEL2x2_SUB

  5852. 	KERNEL2x2_SUB

  5853. 	KERNEL2x2_SUB

  5854. 	KERNEL2x2_SUB

  5855. 

  5856. 	KERNEL2x2_SUB

  5857. 	KERNEL2x2_SUB

  5858. 	KERNEL2x2_SUB

  5859. 	KERNEL2x2_SUB

  5860. 

  5861. 	je	.L2_36

  5862. 

  5863. 	KERNEL2x2_SUB

  5864. 	KERNEL2x2_SUB

  5865. 	KERNEL2x2_SUB

  5866. 	KERNEL2x2_SUB

  5867. 

  5868. 	KERNEL2x2_SUB

  5869. 	KERNEL2x2_SUB

  5870. 	KERNEL2x2_SUB

  5871. 	KERNEL2x2_SUB

  5872. 

  5873. 	je	.L2_36

  5874. 

  5875. 	jmp	.L2_32

  5876. 	ALIGN_4

  5877. 

  5878. .L2_36:

  5879. #ifndef TRMMKERNEL

  5880.         movq    K, %rax

  5881. #else

  5882.         movq    KKK, %rax

  5883. #endif

  5884. 

  5885. 	andq	$7, %rax		# if (k & 1)

  5886. 	je .L2_39

  5887. 

  5888. 	movq    %rax, BI                        //  Index for BO

  5889.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5890. 	

  5891. 	salq	$1, %rax			// rax = rax *2 ; number of values

  5892. 	leaq	(AO, %rax, SIZE), AO

  5893. 	leaq	(BO, BI, SIZE), BO

  5894. 	negq	BI

  5895. 	negq	%rax

  5896. 	ALIGN_4

  5897. 

  5898. .L2_37:

  5899. 

  5900. 	KERNEL2x2_SUB

  5901. 

  5902. 	jl	.L2_37

  5903. 	ALIGN_4

  5904. 

  5905. 

  5906. .L2_39:

  5907. 

  5908. 	SAVE2x2

  5909. 

  5910. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5911.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5912.         movq    K, %rax 

  5913.         subq    KKK, %rax

  5914.         movq    %rax, BI                        //  Index for BO

  5915.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5916.         leaq    (BO, BI, SIZE), BO         

  5917.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  5918.         leaq    (AO, %rax, SIZE), AO

  5919. #endif  

  5920. 

  5921. 

  5922. #if defined(TRMMKERNEL) && defined(LEFT)

  5923.         addq    $2, KK

  5924. #endif

  5925. 

  5926. 	addq	$2 * SIZE, CO1		# coffset += 2

  5927. 	ALIGN_4

  5928. 

  5929. .L2_40:

  5930. 	testq	$1, M		

  5931. 	jz	.L2_60		// to next 2 lines of N

  5932. 

  5933. 	ALIGN_4

  5934. 

  5935. .L2_41:

  5936. #if !defined(TRMMKERNEL) || \

  5937.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  5938.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  5939.         leaq    BUFFER1, BO             // first buffer to BO

  5940.         addq    $4 * SIZE, BO

  5941. #else

  5942.         movq    KK, %rax

  5943.         leaq    BUFFER1, BO             // first buffer to BO

  5944.         addq    $4 * SIZE, BO

  5945.         movq    %rax, BI                        //  Index for BO

  5946.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5947.         leaq    (BO, BI, SIZE), BO

  5948.         leaq    (AO, %rax, SIZE), AO

  5949. #endif

  5950. 

  5951. 

  5952. 	vzeroall

  5953. 

  5954. #ifndef TRMMKERNEL

  5955.         movq    K, %rax

  5956. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5957.         movq    K, %rax

  5958.         subq    KK, %rax

  5959.         movq    %rax, KKK

  5960. #else

  5961.         movq    KK, %rax

  5962. #ifdef LEFT

  5963.         addq    $1, %rax        // number of values in AO

  5964. #else

  5965.         addq    $2, %rax        // number of values in BO

  5966. #endif

  5967.         movq    %rax, KKK

  5968. #endif

  5969. 

  5970. 	andq	$-8, %rax

  5971. 	je	.L2_46

  5972. 	movq    %rax, BI                        //  Index for BO

  5973.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  5974. 

  5975. 	leaq	(AO, %rax, SIZE), AO

  5976. 	leaq	(BO, BI, SIZE), BO

  5977. 	negq	BI

  5978. 	negq	%rax

  5979. 	ALIGN_4

  5980. 

  5981. .L2_42:

  5982. 

  5983. 	KERNEL1x2_SUB

  5984. 	KERNEL1x2_SUB

  5985. 	KERNEL1x2_SUB

  5986. 	KERNEL1x2_SUB

  5987. 

  5988. 	KERNEL1x2_SUB

  5989. 	KERNEL1x2_SUB

  5990. 	KERNEL1x2_SUB

  5991. 	KERNEL1x2_SUB

  5992. 

  5993. 	je	.L2_46

  5994. 

  5995. 	KERNEL1x2_SUB

  5996. 	KERNEL1x2_SUB

  5997. 	KERNEL1x2_SUB

  5998. 	KERNEL1x2_SUB

  5999. 

  6000. 	KERNEL1x2_SUB

  6001. 	KERNEL1x2_SUB

  6002. 	KERNEL1x2_SUB

  6003. 	KERNEL1x2_SUB

  6004. 

  6005. 	je	.L2_46

  6006. 

  6007. 	jmp	.L2_42

  6008. 	ALIGN_4

  6009. 

  6010. .L2_46:

  6011. #ifndef TRMMKERNEL

  6012.         movq    K, %rax

  6013. #else

  6014.         movq    KKK, %rax

  6015. #endif

  6016. 

  6017. 	andq	$7, %rax		# if (k & 1)

  6018. 	je .L2_49

  6019. 

  6020. 	movq    %rax, BI                        //  Index for BO

  6021.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  6022. 

  6023. 	leaq	(AO, %rax, SIZE), AO

  6024. 	leaq	(BO, BI, SIZE), BO

  6025. 	negq	BI

  6026. 	negq	%rax

  6027. 	ALIGN_4

  6028. 

  6029. .L2_47:

  6030. 

  6031. 	KERNEL1x2_SUB

  6032. 

  6033. 	jl	.L2_47

  6034. 	ALIGN_4

  6035. 

  6036. 

  6037. .L2_49:

  6038. 

  6039. 	SAVE1x2

  6040. 

  6041. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6042.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6043.         movq    K, %rax 

  6044.         subq    KKK, %rax

  6045.         movq    %rax, BI                        //  Index for BO

  6046.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  6047.         leaq    (BO, BI, SIZE), BO         

  6048.         leaq    (AO, %rax, SIZE), AO

  6049. #endif  

  6050. 

  6051. 

  6052. #if defined(TRMMKERNEL) && defined(LEFT)

  6053.         addq    $1, KK

  6054. #endif

  6055. 

  6056. 	addq	$1 * SIZE, CO1		# coffset += 1

  6057. 	ALIGN_4

  6058. 	

  6059. 

  6060. 

  6061. 

  6062. 	

  6063. .L2_60:

  6064. #if defined(TRMMKERNEL) && !defined(LEFT)

  6065.         addq    $2, KK

  6066. #endif

  6067. 

  6068. 

  6069. 

  6070. 

  6071. .L1_0:

  6072. 

  6073. /************************************************************************************************

  6074. * Loop for Nmod6 % 2 > 0

  6075. *************************************************************************************************/

  6076. 

  6077. 	movq	Nmod6, J		

  6078. 	andq	$1, J			// j % 2

  6079. 	je	.L999

  6080. 	ALIGN_4

  6081. 

  6082. .L1_01:

  6083. 	// copy to sub buffer

  6084. 	movq	B, BO1

  6085. 	leaq    BUFFER1, BO		// first buffer to BO

  6086. 	movq	K, %rax

  6087. 	ALIGN_4

  6088. 

  6089. .L1_02b:

  6090. 

  6091. 	vmovss	(BO1), %xmm0

  6092. 	vmovss	%xmm0,       (BO)

  6093. 	addq	$1*SIZE,BO1

  6094. 	addq	$1*SIZE,BO

  6095. 	decq	%rax

  6096. 	jnz	.L1_02b

  6097. 

  6098. .L1_02c:

  6099. 

  6100. 	movq	BO1, B			// next offset of B

  6101. 

  6102. .L1_10:

  6103. 	movq	C, CO1

  6104. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  6105. 

  6106. #if defined(TRMMKERNEL) && defined(LEFT)

  6107.         movq    OFFSET, %rax

  6108.         movq    %rax, KK

  6109. #endif

  6110. 	

  6111. 	movq	A, AO		 	// aoffset = a

  6112. 	addq	$16 * SIZE, AO

  6113. 

  6114. 	movq	M,  I

  6115. 	sarq	$4, I			// i = (m >> 4)

  6116. 	je	.L1_20

  6117. 

  6118. 	ALIGN_4

  6119. 

  6120. .L1_11:

  6121. #if !defined(TRMMKERNEL) || \

  6122.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6123.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6124.         leaq    BUFFER1, BO             // first buffer to BO

  6125.         addq    $4 * SIZE, BO

  6126. #else

  6127.         movq    KK, %rax

  6128.         leaq    BUFFER1, BO             // first buffer to BO

  6129.         addq    $4 * SIZE, BO

  6130.         movq    %rax, BI                        //  Index for BO

  6131.         leaq    (BO, BI, SIZE), BO

  6132.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  6133.         leaq    (AO, %rax, SIZE), AO

  6134. #endif

  6135. 

  6136. 

  6137. 	vzeroall

  6138. 

  6139. #ifndef TRMMKERNEL

  6140.         movq    K, %rax

  6141. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  6142.         movq    K, %rax

  6143.         subq    KK, %rax

  6144.         movq    %rax, KKK

  6145. #else

  6146.         movq    KK, %rax

  6147. #ifdef LEFT

  6148.         addq    $16, %rax	// number of values in AO

  6149. #else

  6150.         addq    $1, %rax	// number of values in BO

  6151. #endif

  6152.         movq    %rax, KKK

  6153. #endif

  6154. 

  6155. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  6156. 	je	.L1_16

  6157. 	movq    %rax, BI                        //  Index for BO

  6158. 

  6159. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  6160. 	leaq	(AO, %rax, SIZE), AO

  6161. 	leaq	(BO, BI, SIZE), BO

  6162. 	negq	BI

  6163. 	negq	%rax

  6164. 	ALIGN_4

  6165. 

  6166. .L1_12:

  6167. 

  6168. 	KERNEL16x1_SUB

  6169. 	KERNEL16x1_SUB

  6170. 	KERNEL16x1_SUB

  6171. 	KERNEL16x1_SUB

  6172. 

  6173. 	KERNEL16x1_SUB

  6174. 	KERNEL16x1_SUB

  6175. 	KERNEL16x1_SUB

  6176. 	KERNEL16x1_SUB

  6177. 

  6178. 	je	.L1_16

  6179. 

  6180. 	KERNEL16x1_SUB

  6181. 	KERNEL16x1_SUB

  6182. 	KERNEL16x1_SUB

  6183. 	KERNEL16x1_SUB

  6184. 

  6185. 	KERNEL16x1_SUB

  6186. 	KERNEL16x1_SUB

  6187. 	KERNEL16x1_SUB

  6188. 	KERNEL16x1_SUB

  6189. 

  6190. 	je	.L1_16

  6191. 

  6192. 	jmp	.L1_12

  6193. 	ALIGN_4

  6194. 

  6195. .L1_16:

  6196. #ifndef TRMMKERNEL

  6197.         movq    K, %rax

  6198. #else

  6199.         movq    KKK, %rax

  6200. #endif

  6201. 

  6202. 	andq	$7, %rax		# if (k & 1)

  6203. 	je .L1_19

  6204. 

  6205. 	movq    %rax, BI                        //  Index for BO

  6206. 

  6207. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  6208. 	leaq	(AO, %rax, SIZE), AO

  6209. 	leaq	(BO, BI, SIZE), BO

  6210. 	negq	BI

  6211. 	negq	%rax

  6212. 	ALIGN_4

  6213. 

  6214. .L1_17:

  6215. 

  6216. 	KERNEL16x1_SUB

  6217. 

  6218. 	jl	.L1_17

  6219. 	ALIGN_4

  6220. 

  6221. 

  6222. .L1_19:

  6223. 

  6224. 	SAVE16x1

  6225. 

  6226. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6227.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6228.         movq    K, %rax

  6229.         subq    KKK, %rax

  6230.         movq    %rax, BI                        //  Index for BO

  6231.         leaq    (BO, BI, SIZE), BO

  6232.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  6233.         leaq    (AO, %rax, SIZE), AO

  6234. #endif

  6235. 

  6236. 

  6237. #if defined(TRMMKERNEL) && defined(LEFT)

  6238.         addq    $16, KK				

  6239. #endif

  6240. 

  6241. 	addq	$16 * SIZE, CO1		# coffset += 16

  6242. 	decq	I			# i --

  6243. 	jg	.L1_11

  6244. 	ALIGN_4	

  6245. 

  6246. /**************************************************************************

  6247. * Rest of M 

  6248. ***************************************************************************/

  6249. .L1_20:

  6250. 	// Test rest of M

  6251. 

  6252. 	testq	$15, M

  6253. 	jz	.L999

  6254. 

  6255. 	testq	$8, M		

  6256. 	jz	.L1_21pre

  6257. 	ALIGN_4

  6258. 

  6259. /**************************************************************************/

  6260. 

  6261. .L1_20_1:

  6262. #if !defined(TRMMKERNEL) || \

  6263.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6264.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6265.         leaq    BUFFER1, BO             // first buffer to BO

  6266.         addq    $4 * SIZE, BO

  6267. #else

  6268.         movq    KK, %rax

  6269.         leaq    BUFFER1, BO             // first buffer to BO

  6270.         addq    $4 * SIZE, BO

  6271.         movq    %rax, BI                        //  Index for BO

  6272.         leaq    (BO, BI, SIZE), BO

  6273.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  6274.         leaq    (AO, %rax, SIZE), AO

  6275. #endif

  6276. 

  6277. 

  6278. 	vzeroall

  6279. 

  6280. #ifndef TRMMKERNEL

  6281.         movq    K, %rax

  6282. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  6283.         movq    K, %rax

  6284.         subq    KK, %rax

  6285.         movq    %rax, KKK

  6286. #else

  6287.         movq    KK, %rax

  6288. #ifdef LEFT

  6289.         addq    $8, %rax        // number of values in A

  6290. #else

  6291.         addq    $1, %rax        // number of values in BO

  6292. #endif

  6293.         movq    %rax, KKK

  6294. #endif

  6295. 

  6296. 

  6297. 	andq	$-8, %rax

  6298. 	je	.L1_20_6

  6299. 	movq    %rax, BI                        //  Index for BO

  6300. 

  6301. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  6302. 	leaq	(AO, %rax, SIZE), AO

  6303. 	leaq	(BO, BI, SIZE), BO

  6304. 	negq	BI

  6305. 	negq	%rax

  6306. 	ALIGN_4

  6307. 

  6308. .L1_20_2:

  6309. 

  6310. 	KERNEL8x1_SUB

  6311. 	KERNEL8x1_SUB

  6312. 	KERNEL8x1_SUB

  6313. 	KERNEL8x1_SUB

  6314. 

  6315. 	KERNEL8x1_SUB

  6316. 	KERNEL8x1_SUB

  6317. 	KERNEL8x1_SUB

  6318. 	KERNEL8x1_SUB

  6319. 

  6320. 	je	.L1_20_6

  6321. 

  6322. 	KERNEL8x1_SUB

  6323. 	KERNEL8x1_SUB

  6324. 	KERNEL8x1_SUB

  6325. 	KERNEL8x1_SUB

  6326. 

  6327. 	KERNEL8x1_SUB

  6328. 	KERNEL8x1_SUB

  6329. 	KERNEL8x1_SUB

  6330. 	KERNEL8x1_SUB

  6331. 

  6332. 	je	.L1_20_6

  6333. 

  6334. 	jmp	.L1_20_2

  6335. 	ALIGN_4

  6336. 

  6337. .L1_20_6:

  6338. #ifndef TRMMKERNEL

  6339.         movq    K, %rax

  6340. #else

  6341.         movq    KKK, %rax

  6342. #endif

  6343. 

  6344. 	andq	$7, %rax		# if (k & 1)

  6345. 	je .L1_20_9

  6346. 

  6347. 	movq    %rax, BI                        //  Index for BO

  6348. 

  6349. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  6350. 	leaq	(AO, %rax, SIZE), AO

  6351. 	leaq	(BO, BI, SIZE), BO

  6352. 	negq	BI

  6353. 	negq	%rax

  6354. 	ALIGN_4

  6355. 

  6356. .L1_20_7:

  6357. 

  6358. 	KERNEL8x1_SUB

  6359. 

  6360. 	jl	.L1_20_7

  6361. 	ALIGN_4

  6362. 

  6363. 

  6364. .L1_20_9:

  6365. 

  6366. 	SAVE8x1

  6367. 

  6368. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6369.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6370.         movq    K, %rax 

  6371.         subq    KKK, %rax

  6372.         movq    %rax, BI                        //  Index for BO

  6373.         leaq    (BO, BI, SIZE), BO         

  6374.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  6375.         leaq    (AO, %rax, SIZE), AO

  6376. #endif  

  6377. 

  6378. 

  6379. #if defined(TRMMKERNEL) && defined(LEFT)

  6380.         addq    $8, KK

  6381. #endif

  6382. 

  6383. 	addq	$8 * SIZE, CO1		# coffset += 8

  6384. 	ALIGN_4

  6385. 	

  6386. 

  6387. 

  6388. /**************************************************************************/

  6389. 

  6390. .L1_21pre:

  6391. 

  6392. 	testq	$4, M		

  6393. 	jz	.L1_30

  6394. 	ALIGN_4

  6395. 

  6396. .L1_21:

  6397. #if !defined(TRMMKERNEL) || \

  6398.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6399.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6400.         leaq    BUFFER1, BO             // first buffer to BO

  6401.         addq    $4 * SIZE, BO

  6402. #else

  6403.         movq    KK, %rax

  6404.         leaq    BUFFER1, BO             // first buffer to BO

  6405.         addq    $4 * SIZE, BO

  6406.         movq    %rax, BI                        //  Index for BO

  6407.         leaq    (BO, BI, SIZE), BO

  6408.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  6409.         leaq    (AO, %rax, SIZE), AO

  6410. #endif

  6411. 

  6412. 

  6413. 	vzeroall

  6414. 

  6415. #ifndef TRMMKERNEL

  6416.         movq    K, %rax

  6417. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  6418.         movq    K, %rax

  6419.         subq    KK, %rax

  6420.         movq    %rax, KKK

  6421. #else

  6422.         movq    KK, %rax

  6423. #ifdef LEFT

  6424.         addq    $4, %rax        // number of values in A

  6425. #else

  6426.         addq    $1, %rax        // number of values in BO

  6427. #endif

  6428.         movq    %rax, KKK

  6429. #endif

  6430. 

  6431. 

  6432. 	andq	$-8, %rax

  6433. 	je	.L1_26

  6434. 	movq    %rax, BI                        //  Index for BO

  6435. 

  6436. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  6437. 	leaq	(AO, %rax, SIZE), AO

  6438. 	leaq	(BO, BI, SIZE), BO

  6439. 	negq	BI

  6440. 	negq	%rax

  6441. 	ALIGN_4

  6442. 

  6443. .L1_22:

  6444. 

  6445. 	KERNEL4x1_SUB

  6446. 	KERNEL4x1_SUB

  6447. 	KERNEL4x1_SUB

  6448. 	KERNEL4x1_SUB

  6449. 

  6450. 	KERNEL4x1_SUB

  6451. 	KERNEL4x1_SUB

  6452. 	KERNEL4x1_SUB

  6453. 	KERNEL4x1_SUB

  6454. 

  6455. 	je	.L1_26

  6456. 

  6457. 	KERNEL4x1_SUB

  6458. 	KERNEL4x1_SUB

  6459. 	KERNEL4x1_SUB

  6460. 	KERNEL4x1_SUB

  6461. 

  6462. 	KERNEL4x1_SUB

  6463. 	KERNEL4x1_SUB

  6464. 	KERNEL4x1_SUB

  6465. 	KERNEL4x1_SUB

  6466. 

  6467. 	je	.L1_26

  6468. 

  6469. 	jmp	.L1_22

  6470. 	ALIGN_4

  6471. 

  6472. .L1_26:

  6473. #ifndef TRMMKERNEL

  6474.         movq    K, %rax

  6475. #else

  6476.         movq    KKK, %rax

  6477. #endif

  6478. 

  6479. 	andq	$7, %rax		# if (k & 1)

  6480. 	je .L1_29

  6481. 

  6482. 	movq    %rax, BI                        //  Index for BO

  6483. 

  6484. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  6485. 	leaq	(AO, %rax, SIZE), AO

  6486. 	leaq	(BO, BI, SIZE), BO

  6487. 	negq	BI

  6488. 	negq	%rax

  6489. 	ALIGN_4

  6490. 

  6491. .L1_27:

  6492. 

  6493. 	KERNEL4x1_SUB

  6494. 

  6495. 	jl	.L1_27

  6496. 	ALIGN_4

  6497. 

  6498. 

  6499. .L1_29:

  6500. 

  6501. 	SAVE4x1

  6502. 

  6503. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6504.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6505.         movq    K, %rax 

  6506.         subq    KKK, %rax

  6507.         movq    %rax, BI                        //  Index for BO

  6508.         leaq    (BO, BI, SIZE), BO         

  6509.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  6510.         leaq    (AO, %rax, SIZE), AO

  6511. #endif  

  6512. 

  6513. 

  6514. #if defined(TRMMKERNEL) && defined(LEFT)

  6515.         addq    $4, KK

  6516. #endif

  6517. 

  6518. 	addq	$4 * SIZE, CO1		# coffset += 4

  6519. 	ALIGN_4

  6520. 	

  6521. 

  6522. .L1_30:

  6523. 	testq	$2, M		

  6524. 	jz	.L1_40

  6525. 

  6526. 	ALIGN_4

  6527. 

  6528. .L1_31:

  6529. #if !defined(TRMMKERNEL) || \

  6530.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6531.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6532.         leaq    BUFFER1, BO             // first buffer to BO

  6533.         addq    $4 * SIZE, BO

  6534. #else

  6535.         movq    KK, %rax

  6536.         leaq    BUFFER1, BO             // first buffer to BO

  6537.         addq    $4 * SIZE, BO

  6538.         movq    %rax, BI                        //  Index for BO

  6539.         leaq    (BO, BI, SIZE), BO

  6540.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  6541.         leaq    (AO, %rax, SIZE), AO

  6542. #endif

  6543. 

  6544. 

  6545. 	vzeroall

  6546. 

  6547. #ifndef TRMMKERNEL

  6548.         movq    K, %rax

  6549. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  6550.         movq    K, %rax

  6551.         subq    KK, %rax

  6552.         movq    %rax, KKK

  6553. #else

  6554.         movq    KK, %rax

  6555. #ifdef LEFT

  6556.         addq    $2, %rax        // number of values in AO

  6557. #else

  6558.         addq    $1, %rax        // number of values in BO

  6559. #endif

  6560.         movq    %rax, KKK

  6561. #endif

  6562. 

  6563. 

  6564. 	andq	$-8, %rax

  6565. 	je	.L1_36

  6566. 	movq    %rax, BI                        //  Index for BO

  6567. 

  6568. 	salq	$1, %rax			// rax = rax *2 ; number of values

  6569. 	leaq	(AO, %rax, SIZE), AO

  6570. 	leaq	(BO, BI, SIZE), BO

  6571. 	negq	BI

  6572. 	negq	%rax

  6573. 	ALIGN_4

  6574. 

  6575. .L1_32:

  6576. 

  6577. 	KERNEL2x1_SUB

  6578. 	KERNEL2x1_SUB

  6579. 	KERNEL2x1_SUB

  6580. 	KERNEL2x1_SUB

  6581. 

  6582. 	KERNEL2x1_SUB

  6583. 	KERNEL2x1_SUB

  6584. 	KERNEL2x1_SUB

  6585. 	KERNEL2x1_SUB

  6586. 

  6587. 	je	.L1_36

  6588. 

  6589. 	KERNEL2x1_SUB

  6590. 	KERNEL2x1_SUB

  6591. 	KERNEL2x1_SUB

  6592. 	KERNEL2x1_SUB

  6593. 

  6594. 	KERNEL2x1_SUB

  6595. 	KERNEL2x1_SUB

  6596. 	KERNEL2x1_SUB

  6597. 	KERNEL2x1_SUB

  6598. 

  6599. 	je	.L1_36

  6600. 

  6601. 	jmp	.L1_32

  6602. 	ALIGN_4

  6603. 

  6604. .L1_36:

  6605. #ifndef TRMMKERNEL

  6606.         movq    K, %rax

  6607. #else

  6608.         movq    KKK, %rax

  6609. #endif

  6610. 

  6611. 	andq	$7, %rax		# if (k & 1)

  6612. 	je .L1_39

  6613. 

  6614. 	movq    %rax, BI                        //  Index for BO

  6615. 	

  6616. 	salq	$1, %rax			// rax = rax *2 ; number of values

  6617. 	leaq	(AO, %rax, SIZE), AO

  6618. 	leaq	(BO, BI, SIZE), BO

  6619. 	negq	BI

  6620. 	negq	%rax

  6621. 	ALIGN_4

  6622. 

  6623. .L1_37:

  6624. 

  6625. 	KERNEL2x1_SUB

  6626. 

  6627. 	jl	.L1_37

  6628. 	ALIGN_4

  6629. 

  6630. 

  6631. .L1_39:

  6632. 

  6633. 	SAVE2x1

  6634. 

  6635. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6636.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6637.         movq    K, %rax 

  6638.         subq    KKK, %rax

  6639.         movq    %rax, BI                        //  Index for BO

  6640.         leaq    (BO, BI, SIZE), BO         

  6641.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  6642.         leaq    (AO, %rax, SIZE), AO

  6643. #endif  

  6644. 

  6645. 

  6646. #if defined(TRMMKERNEL) && defined(LEFT)

  6647.         addq    $2, KK

  6648. #endif

  6649. 

  6650. 	addq	$2 * SIZE, CO1		# coffset += 2

  6651. 	ALIGN_4

  6652. 

  6653. .L1_40:

  6654. 	testq	$1, M		

  6655. 	jz	.L999

  6656. 

  6657. 	ALIGN_4

  6658. 

  6659. .L1_41:

  6660. #if !defined(TRMMKERNEL) || \

  6661.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6662.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6663.         leaq    BUFFER1, BO             // first buffer to BO

  6664.         addq    $4 * SIZE, BO

  6665. #else

  6666.         movq    KK, %rax

  6667.         leaq    BUFFER1, BO             // first buffer to BO

  6668.         addq    $4 * SIZE, BO

  6669.         movq    %rax, BI                        //  Index for BO

  6670.         leaq    (BO, BI, SIZE), BO

  6671.         leaq    (AO, %rax, SIZE), AO

  6672. #endif

  6673. 

  6674. 

  6675. 	vzeroall

  6676. 

  6677. #ifndef TRMMKERNEL

  6678.         movq    K, %rax

  6679. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  6680.         movq    K, %rax

  6681.         subq    KK, %rax

  6682.         movq    %rax, KKK

  6683. #else

  6684.         movq    KK, %rax

  6685. #ifdef LEFT

  6686.         addq    $1, %rax        // number of values in AO

  6687. #else

  6688.         addq    $1, %rax        // number of values in BO

  6689. #endif

  6690.         movq    %rax, KKK

  6691. #endif

  6692. 

  6693. 	andq	$-8, %rax

  6694. 	je	.L1_46

  6695. 	movq    %rax, BI                        //  Index for BO

  6696. 

  6697. 	leaq	(AO, %rax, SIZE), AO

  6698. 	leaq	(BO, BI, SIZE), BO

  6699. 	negq	BI

  6700. 	negq	%rax

  6701. 	ALIGN_4

  6702. 

  6703. .L1_42:

  6704. 

  6705. 	KERNEL1x1_SUB

  6706. 	KERNEL1x1_SUB

  6707. 	KERNEL1x1_SUB

  6708. 	KERNEL1x1_SUB

  6709. 

  6710. 	KERNEL1x1_SUB

  6711. 	KERNEL1x1_SUB

  6712. 	KERNEL1x1_SUB

  6713. 	KERNEL1x1_SUB

  6714. 

  6715. 	je	.L1_46

  6716. 

  6717. 	KERNEL1x1_SUB

  6718. 	KERNEL1x1_SUB

  6719. 	KERNEL1x1_SUB

  6720. 	KERNEL1x1_SUB

  6721. 

  6722. 	KERNEL1x1_SUB

  6723. 	KERNEL1x1_SUB

  6724. 	KERNEL1x1_SUB

  6725. 	KERNEL1x1_SUB

  6726. 

  6727. 	je	.L1_46

  6728. 

  6729. 	jmp	.L1_42

  6730. 	ALIGN_4

  6731. 

  6732. .L1_46:

  6733. #ifndef TRMMKERNEL

  6734.         movq    K, %rax

  6735. #else

  6736.         movq    KKK, %rax

  6737. #endif

  6738. 

  6739. 	andq	$7, %rax		# if (k & 1)

  6740. 	je .L1_49

  6741. 

  6742. 	movq    %rax, BI                        //  Index for BO

  6743. 

  6744. 	leaq	(AO, %rax, SIZE), AO

  6745. 	leaq	(BO, BI, SIZE), BO

  6746. 	negq	BI

  6747. 	negq	%rax

  6748. 	ALIGN_4

  6749. 

  6750. .L1_47:

  6751. 

  6752. 	KERNEL1x1_SUB

  6753. 

  6754. 	jl	.L1_47

  6755. 	ALIGN_4

  6756. 

  6757. 

  6758. .L1_49:

  6759. 

  6760. 	SAVE1x1

  6761. 

  6762. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  6763.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  6764.         movq    K, %rax 

  6765.         subq    KKK, %rax

  6766.         movq    %rax, BI                        //  Index for BO

  6767.         leaq    (BO, BI, SIZE), BO         

  6768.         leaq    (AO, %rax, SIZE), AO

  6769. #endif  

  6770. 

  6771. 

  6772. #if defined(TRMMKERNEL) && defined(LEFT)

  6773.         addq    $1, KK

  6774. #endif

  6775. 

  6776. 	addq	$1 * SIZE, CO1		# coffset += 1

  6777. 	ALIGN_4

  6778. 	

  6779. 

  6780. .L999:

  6781. 	movq   		SP, %rsp

  6782. 	movq	   (%rsp), %rbx

  6783. 	movq	  8(%rsp), %rbp

  6784. 	movq	 16(%rsp), %r12

  6785. 	movq	 24(%rsp), %r13

  6786. 	movq	 32(%rsp), %r14

  6787. 	movq	 40(%rsp), %r15

  6788. 

  6789. #ifdef WINDOWS_ABI

  6790. 	movq	 48(%rsp), %rdi

  6791. 	movq	 56(%rsp), %rsi

  6792. 	movups	 64(%rsp), %xmm6

  6793. 	movups	 80(%rsp), %xmm7

  6794. 	movups	 96(%rsp), %xmm8

  6795. 	movups	112(%rsp), %xmm9

  6796. 	movups	128(%rsp), %xmm10

  6797. 	movups	144(%rsp), %xmm11

  6798. 	movups	160(%rsp), %xmm12

  6799. 	movups	176(%rsp), %xmm13

  6800. 	movups	192(%rsp), %xmm14

  6801. 	movups	208(%rsp), %xmm15

  6802. #endif

  6803. 

  6804. 	addq	$STACKSIZE, %rsp

  6805. 	ret

  6806. 

  6807. 	EPILOGUE

  6808. 

  6809. 

  6810. #endif