OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/sgemm_kernel_16x4_sandy.S

3168 lines
70 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2013, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. #define ASSEMBLER

  29. #include "common.h"

  30.  

  31. #define OLD_M	%rdi

  32. #define OLD_N	%rsi

  33. #define M	%r13

  34. #define J	%r14

  35. #define OLD_K	%rdx

  36. 

  37. #define A	%rcx

  38. #define B	%r8

  39. #define C	%r9

  40. #define LDC	%r10

  41. 	

  42. #define I	%r11

  43. #define AO	%rdi

  44. #define BO	%rsi

  45. #define	CO1	%r15

  46. #define K	%r12

  47. #define BI	%rbp

  48. #define	SP	%rbx

  49. 

  50. #define BO1	%rdi

  51. #define	CO2	%rdx

  52. 

  53. #ifndef WINDOWS_ABI

  54. 

  55. #define STACKSIZE 96

  56. 

  57. #else

  58. 

  59. #define STACKSIZE 256

  60. 

  61. #define OLD_A		40 + STACKSIZE(%rsp)

  62. #define OLD_B		48 + STACKSIZE(%rsp)

  63. #define OLD_C		56 + STACKSIZE(%rsp)

  64. #define OLD_LDC		64 + STACKSIZE(%rsp)

  65. #define OLD_OFFSET	72 + STACKSIZE(%rsp)

  66. 

  67. #endif

  68. 

  69. #define L_BUFFER_SIZE 8192

  70. 

  71. #define Ndiv6	 24(%rsp)

  72. #define Nmod6	 32(%rsp)

  73. #define N	 40(%rsp)

  74. #define ALPHA	 48(%rsp)

  75. #define OFFSET	 56(%rsp)

  76. #define KK	 64(%rsp)

  77. #define KKK	 72(%rsp)

  78. #define BUFFER1	           128(%rsp)

  79. 

  80. #if defined(OS_WINDOWS)

  81. #if   L_BUFFER_SIZE > 16384

  82. #define STACK_TOUCH \

  83.         movl    $0,  4096 * 4(%rsp);\

  84.         movl    $0,  4096 * 3(%rsp);\

  85.         movl    $0,  4096 * 2(%rsp);\

  86.         movl    $0,  4096 * 1(%rsp);

  87. #elif L_BUFFER_SIZE > 12288

  88. #define STACK_TOUCH \

  89.         movl    $0,  4096 * 3(%rsp);\

  90.         movl    $0,  4096 * 2(%rsp);\

  91.         movl    $0,  4096 * 1(%rsp);

  92. #elif L_BUFFER_SIZE > 8192

  93. #define STACK_TOUCH \

  94.         movl    $0,  4096 * 2(%rsp);\

  95.         movl    $0,  4096 * 1(%rsp);

  96. #elif L_BUFFER_SIZE > 4096

  97. #define STACK_TOUCH \

  98.         movl    $0,  4096 * 1(%rsp);

  99. #else

  100. #define STACK_TOUCH

  101. #endif

  102. #else

  103. #define STACK_TOUCH

  104. #endif

  105. 

  106. #define	A_PR1	512

  107. #define	B_PR1	512

  108. 

  109. /*******************************************************************************************

  110. * 4 lines of N

  111. *******************************************************************************************/

  112. 

  113. .macro KERNEL16x4_SUB

  114. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  115. 	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1

  116. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  117. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  118. 	vmulps		%ymm2 , %ymm0 , %ymm12

  119. 	vmulps		%ymm2 , %ymm1 , %ymm13

  120. 	vmulps		%ymm3 , %ymm0 , %ymm14

  121. 	vmulps		%ymm3 , %ymm1 , %ymm15

  122. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  123. 	vaddps 	        %ymm13, %ymm5 , %ymm5

  124. 	vaddps 	        %ymm14, %ymm6 , %ymm6

  125. 	vaddps 	        %ymm15, %ymm7 , %ymm7

  126. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2

  127. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3

  128. 	vmulps		%ymm2 , %ymm0 , %ymm12

  129. 	vmulps		%ymm2 , %ymm1 , %ymm13

  130. 	vmulps		%ymm3 , %ymm0 , %ymm14

  131. 	vmulps		%ymm3 , %ymm1 , %ymm15

  132. 	vaddps 	        %ymm12, %ymm8 , %ymm8

  133. 	vaddps 	        %ymm13, %ymm9 , %ymm9

  134. 	vaddps 	        %ymm14, %ymm10, %ymm10

  135. 	vaddps 	        %ymm15, %ymm11, %ymm11

  136. 	addq	$ 4 , BI	

  137. 	addq	$ 16, %rax 

  138. .endm

  139. 

  140. .macro SAVE16x4

  141. 

  142. 	vbroadcastss	ALPHA, %ymm0

  143. 

  144. 	vmulps	%ymm0 , %ymm4 , %ymm4

  145. 	vmulps	%ymm0 , %ymm5 , %ymm5

  146. 	vmulps	%ymm0 , %ymm6 , %ymm6

  147. 	vmulps	%ymm0 , %ymm7 , %ymm7

  148. 	vmulps	%ymm0 , %ymm8 , %ymm8

  149. 	vmulps	%ymm0 , %ymm9 , %ymm9

  150. 	vmulps	%ymm0 , %ymm10, %ymm10

  151. 	vmulps	%ymm0 , %ymm11, %ymm11

  152. 

  153. 

  154. #if !defined(TRMMKERNEL)

  155. 

  156. 	vaddps 	        (CO1), %ymm4,%ymm4

  157. 	vaddps  8 * SIZE(CO1), %ymm5,%ymm5

  158. 

  159. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  160. 	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7

  161. 

  162. 	vaddps 	        (CO2), %ymm8,%ymm8

  163. 	vaddps  8 * SIZE(CO2), %ymm9,%ymm9

  164. 

  165. 	vaddps 	        (CO2, LDC), %ymm10,%ymm10

  166. 	vaddps  8 * SIZE(CO2, LDC), %ymm11,%ymm11

  167. 

  168. #endif

  169. 

  170. 	vmovups	%ymm4 ,  	(CO1)

  171. 	vmovups	%ymm5 , 8 * SIZE(CO1)

  172. 

  173. 	vmovups	%ymm6 ,  	(CO1, LDC)

  174. 	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)

  175. 

  176. 	vmovups	%ymm8 ,  	(CO2)

  177. 	vmovups	%ymm9 , 8 * SIZE(CO2)

  178. 

  179. 	vmovups	%ymm10,  	(CO2, LDC)

  180. 	vmovups	%ymm11, 8 * SIZE(CO2, LDC)

  181. 

  182. 	prefetcht0	64(CO1)

  183. 	prefetcht0	64(CO1, LDC)

  184. 	prefetcht0	64(CO2)

  185. 	prefetcht0	64(CO2, LDC)

  186. 

  187. .endm

  188. 

  189. 

  190. 

  191. /*******************************************************************************************/

  192. 

  193. .macro KERNEL8x4_SUB

  194. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  195. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  196. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  197. 	vmulps		%ymm2 , %ymm0 , %ymm12

  198. 	vmulps		%ymm3 , %ymm0 , %ymm14

  199. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  200. 	vaddps 	        %ymm14, %ymm6 , %ymm6

  201. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %ymm2

  202. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %ymm3

  203. 	vmulps		%ymm2 , %ymm0 , %ymm12

  204. 	vmulps		%ymm3 , %ymm0 , %ymm14

  205. 	vaddps 	        %ymm12, %ymm8 , %ymm8

  206. 	vaddps 	        %ymm14, %ymm10, %ymm10

  207. 	addq	$ 4 , BI	

  208. 	addq	$ 8 , %rax 

  209. .endm

  210. 

  211. .macro SAVE8x4

  212. 

  213. 	vbroadcastss	ALPHA, %ymm0

  214. 

  215. 	vmulps	%ymm0 , %ymm4 , %ymm4

  216. 	vmulps	%ymm0 , %ymm6 , %ymm6

  217. 	vmulps	%ymm0 , %ymm8 , %ymm8

  218. 	vmulps	%ymm0 , %ymm10, %ymm10

  219. 

  220. 

  221. #if !defined(TRMMKERNEL)

  222. 

  223. 	vaddps 	        (CO1), %ymm4,%ymm4

  224. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  225. 	vaddps 	        (CO2), %ymm8,%ymm8

  226. 	vaddps 	        (CO2, LDC), %ymm10,%ymm10

  227. 

  228. #endif

  229. 

  230. 	vmovups	%ymm4 ,  	(CO1)

  231. 	vmovups	%ymm6 ,  	(CO1, LDC)

  232. 	vmovups	%ymm8 ,  	(CO2)

  233. 	vmovups	%ymm10,  	(CO2, LDC)

  234. 

  235. .endm

  236. 

  237. 

  238. 

  239. /*******************************************************************************************/

  240. 

  241. .macro KERNEL4x4_SUB

  242. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  243. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  244. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  245. 	vmulps		%xmm2 , %xmm0 , %xmm12

  246. 	vmulps		%xmm3 , %xmm0 , %xmm14

  247. 	vaddps 	        %xmm12, %xmm4 , %xmm4

  248. 	vaddps 	        %xmm14, %xmm6 , %xmm6

  249. 	vbroadcastss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  250. 	vbroadcastss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  251. 	vmulps		%xmm2 , %xmm0 , %xmm12

  252. 	vmulps		%xmm3 , %xmm0 , %xmm14

  253. 	vaddps 	        %xmm12, %xmm8 , %xmm8

  254. 	vaddps 	        %xmm14, %xmm10, %xmm10

  255. 	addq	$ 4 , BI	

  256. 	addq	$ 4 , %rax 

  257. .endm

  258. 

  259. .macro SAVE4x4

  260. 

  261. 	vbroadcastss	ALPHA, %xmm0

  262. 

  263. 	vmulps	%xmm0 , %xmm4 , %xmm4

  264. 	vmulps	%xmm0 , %xmm6 , %xmm6

  265. 	vmulps	%xmm0 , %xmm8 , %xmm8

  266. 	vmulps	%xmm0 , %xmm10, %xmm10

  267. 

  268. 

  269. #if !defined(TRMMKERNEL)

  270. 

  271. 	vaddps 	        (CO1), %xmm4,%xmm4

  272. 	vaddps 	        (CO1, LDC), %xmm6,%xmm6

  273. 	vaddps 	        (CO2), %xmm8,%xmm8

  274. 	vaddps 	        (CO2, LDC), %xmm10,%xmm10

  275. 

  276. #endif

  277. 

  278. 	vmovups	%xmm4 ,  	(CO1)

  279. 	vmovups	%xmm6 ,  	(CO1, LDC)

  280. 	vmovups	%xmm8 ,  	(CO2)

  281. 	vmovups	%xmm10,  	(CO2, LDC)

  282. 

  283. .endm

  284. 

  285. 

  286. /*******************************************************************************************/

  287. 

  288. .macro KERNEL2x4_SUB

  289. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  290. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  291. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  292. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  293. 	vmulss		%xmm2 , %xmm0 , %xmm12

  294. 	vmulss		%xmm2 , %xmm1 , %xmm13

  295. 	vmulss		%xmm3 , %xmm0 , %xmm14

  296. 	vmulss		%xmm3 , %xmm1 , %xmm15

  297. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  298. 	vaddss 	        %xmm13, %xmm5 , %xmm5

  299. 	vaddss 	        %xmm14, %xmm6 , %xmm6

  300. 	vaddss 	        %xmm15, %xmm7 , %xmm7

  301. 	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  302. 	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  303. 	vmulss		%xmm2 , %xmm0 , %xmm12

  304. 	vmulss		%xmm2 , %xmm1 , %xmm13

  305. 	vmulss		%xmm3 , %xmm0 , %xmm14

  306. 	vmulss		%xmm3 , %xmm1 , %xmm15

  307. 	vaddss 	        %xmm12, %xmm8 , %xmm8

  308. 	vaddss 	        %xmm13, %xmm9 , %xmm9

  309. 	vaddss 	        %xmm14, %xmm10, %xmm10

  310. 	vaddss 	        %xmm15, %xmm11, %xmm11

  311. 	addq	$ 4 , BI	

  312. 	addq	$ 2, %rax 

  313. .endm

  314. 

  315. .macro SAVE2x4

  316. 

  317. 	vmovss	ALPHA, %xmm0

  318. 

  319. 	vmulss	%xmm0 , %xmm4 , %xmm4

  320. 	vmulss	%xmm0 , %xmm5 , %xmm5

  321. 	vmulss	%xmm0 , %xmm6 , %xmm6

  322. 	vmulss	%xmm0 , %xmm7 , %xmm7

  323. 	vmulss	%xmm0 , %xmm8 , %xmm8

  324. 	vmulss	%xmm0 , %xmm9 , %xmm9

  325. 	vmulss	%xmm0 , %xmm10, %xmm10

  326. 	vmulss	%xmm0 , %xmm11, %xmm11

  327. 

  328. 

  329. #if !defined(TRMMKERNEL)

  330. 

  331. 	vaddss 	        (CO1), %xmm4,%xmm4

  332. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  333. 

  334. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  335. 	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7

  336. 

  337. 	vaddss 	        (CO2), %xmm8,%xmm8

  338. 	vaddss  1 * SIZE(CO2), %xmm9,%xmm9

  339. 

  340. 	vaddss 	        (CO2, LDC), %xmm10,%xmm10

  341. 	vaddss  1 * SIZE(CO2, LDC), %xmm11,%xmm11

  342. 

  343. #endif

  344. 

  345. 	vmovss	%xmm4 ,  	(CO1)

  346. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  347. 

  348. 	vmovss	%xmm6 ,  	(CO1, LDC)

  349. 	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)

  350. 

  351. 	vmovss	%xmm8 ,  	(CO2)

  352. 	vmovss	%xmm9 , 1 * SIZE(CO2)

  353. 

  354. 	vmovss	%xmm10,  	(CO2, LDC)

  355. 	vmovss	%xmm11, 1 * SIZE(CO2, LDC)

  356. 

  357. .endm

  358. 

  359. 

  360. /*******************************************************************************************/

  361. 

  362. .macro KERNEL1x4_SUB

  363. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  364. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  365. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  366. 	vmulss		%xmm2 , %xmm0 , %xmm12

  367. 	vmulss		%xmm3 , %xmm0 , %xmm14

  368. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  369. 	vaddss 	        %xmm14, %xmm6 , %xmm6

  370. 	vmovss	 -2 * SIZE(BO, BI, SIZE), %xmm2

  371. 	vmovss	 -1 * SIZE(BO, BI, SIZE), %xmm3

  372. 	vmulss		%xmm2 , %xmm0 , %xmm12

  373. 	vmulss		%xmm3 , %xmm0 , %xmm14

  374. 	vaddss 	        %xmm12, %xmm8 , %xmm8

  375. 	vaddss 	        %xmm14, %xmm10, %xmm10

  376. 	addq	$ 4 , BI	

  377. 	addq	$ 1, %rax 

  378. .endm

  379. 

  380. .macro SAVE1x4

  381. 

  382. 	vmovss	ALPHA, %xmm0

  383. 

  384. 	vmulss	%xmm0 , %xmm4 , %xmm4

  385. 	vmulss	%xmm0 , %xmm6 , %xmm6

  386. 	vmulss	%xmm0 , %xmm8 , %xmm8

  387. 	vmulss	%xmm0 , %xmm10, %xmm10

  388. 

  389. 

  390. #if !defined(TRMMKERNEL)

  391. 

  392. 	vaddss 	        (CO1), %xmm4,%xmm4

  393. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  394. 	vaddss 	        (CO2), %xmm8,%xmm8

  395. 	vaddss 	        (CO2, LDC), %xmm10,%xmm10

  396. 

  397. #endif

  398. 

  399. 	vmovss	%xmm4 ,  	(CO1)

  400. 	vmovss	%xmm6 ,  	(CO1, LDC)

  401. 	vmovss	%xmm8 ,  	(CO2)

  402. 	vmovss	%xmm10,  	(CO2, LDC)

  403. 

  404. .endm

  405. 

  406. 

  407. /*******************************************************************************************/

  408. 

  409. /*******************************************************************************************

  410. * 2 lines of N

  411. *******************************************************************************************/

  412. 

  413. .macro KERNEL16x2_SUB

  414. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  415. 	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1

  416. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  417. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  418. 	vmulps		%ymm2 , %ymm0 , %ymm12

  419. 	vmulps		%ymm2 , %ymm1 , %ymm13

  420. 	vmulps		%ymm3 , %ymm0 , %ymm14

  421. 	vmulps		%ymm3 , %ymm1 , %ymm15

  422. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  423. 	vaddps 	        %ymm13, %ymm5 , %ymm5

  424. 	vaddps 	        %ymm14, %ymm6 , %ymm6

  425. 	vaddps 	        %ymm15, %ymm7 , %ymm7

  426. 	addq	$ 2 , BI	

  427. 	addq	$ 16, %rax 

  428. .endm

  429. 

  430. .macro SAVE16x2

  431. 

  432. 	vbroadcastss	ALPHA, %ymm0

  433. 

  434. 	vmulps	%ymm0 , %ymm4 , %ymm4

  435. 	vmulps	%ymm0 , %ymm5 , %ymm5

  436. 	vmulps	%ymm0 , %ymm6 , %ymm6

  437. 	vmulps	%ymm0 , %ymm7 , %ymm7

  438. 

  439. 

  440. #if !defined(TRMMKERNEL)

  441. 

  442. 	vaddps 	        (CO1), %ymm4,%ymm4

  443. 	vaddps  8 * SIZE(CO1), %ymm5,%ymm5

  444. 

  445. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  446. 	vaddps  8 * SIZE(CO1, LDC), %ymm7,%ymm7

  447. 

  448. #endif

  449. 

  450. 	vmovups	%ymm4 ,  	(CO1)

  451. 	vmovups	%ymm5 , 8 * SIZE(CO1)

  452. 

  453. 	vmovups	%ymm6 ,  	(CO1, LDC)

  454. 	vmovups	%ymm7 , 8 * SIZE(CO1, LDC)

  455. 

  456. .endm

  457. 

  458. 

  459. 

  460. /*******************************************************************************************/

  461. 

  462. .macro KERNEL8x2_SUB

  463. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  464. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  465. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %ymm3

  466. 	vmulps		%ymm2 , %ymm0 , %ymm12

  467. 	vmulps		%ymm3 , %ymm0 , %ymm14

  468. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  469. 	vaddps 	        %ymm14, %ymm6 , %ymm6

  470. 	addq	$ 2 , BI	

  471. 	addq	$ 8 , %rax 

  472. .endm

  473. 

  474. .macro SAVE8x2

  475. 

  476. 	vbroadcastss	ALPHA, %ymm0

  477. 

  478. 	vmulps	%ymm0 , %ymm4 , %ymm4

  479. 	vmulps	%ymm0 , %ymm6 , %ymm6

  480. 

  481. #if !defined(TRMMKERNEL)

  482. 

  483. 	vaddps 	        (CO1), %ymm4,%ymm4

  484. 	vaddps 	        (CO1, LDC), %ymm6,%ymm6

  485. 

  486. #endif

  487. 

  488. 	vmovups	%ymm4 ,  	(CO1)

  489. 	vmovups	%ymm6 ,  	(CO1, LDC)

  490. 

  491. .endm

  492. 

  493. 

  494. 

  495. /*******************************************************************************************/

  496. 

  497. .macro KERNEL4x2_SUB

  498. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  499. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  500. 	vbroadcastss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  501. 	vmulps		%xmm2 , %xmm0 , %xmm12

  502. 	vmulps		%xmm3 , %xmm0 , %xmm14

  503. 	vaddps 	        %xmm12, %xmm4 , %xmm4

  504. 	vaddps 	        %xmm14, %xmm6 , %xmm6

  505. 	addq	$ 2 , BI	

  506. 	addq	$ 4 , %rax 

  507. .endm

  508. 

  509. .macro SAVE4x2

  510. 

  511. 	vbroadcastss	ALPHA, %xmm0

  512. 

  513. 	vmulps	%xmm0 , %xmm4 , %xmm4

  514. 	vmulps	%xmm0 , %xmm6 , %xmm6

  515. 

  516. #if !defined(TRMMKERNEL)

  517. 

  518. 	vaddps 	        (CO1), %xmm4,%xmm4

  519. 	vaddps 	        (CO1, LDC), %xmm6,%xmm6

  520. 

  521. #endif

  522. 

  523. 	vmovups	%xmm4 ,  	(CO1)

  524. 	vmovups	%xmm6 ,  	(CO1, LDC)

  525. 

  526. .endm

  527. 

  528. 

  529. /*******************************************************************************************/

  530. 

  531. .macro KERNEL2x2_SUB

  532. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  533. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  534. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  535. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  536. 	vmulss		%xmm2 , %xmm0 , %xmm12

  537. 	vmulss		%xmm2 , %xmm1 , %xmm13

  538. 	vmulss		%xmm3 , %xmm0 , %xmm14

  539. 	vmulss		%xmm3 , %xmm1 , %xmm15

  540. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  541. 	vaddss 	        %xmm13, %xmm5 , %xmm5

  542. 	vaddss 	        %xmm14, %xmm6 , %xmm6

  543. 	vaddss 	        %xmm15, %xmm7 , %xmm7

  544. 	addq	$ 2 , BI	

  545. 	addq	$ 2, %rax 

  546. .endm

  547. 

  548. .macro SAVE2x2

  549. 

  550. 	vmovss	ALPHA, %xmm0

  551. 

  552. 	vmulss	%xmm0 , %xmm4 , %xmm4

  553. 	vmulss	%xmm0 , %xmm5 , %xmm5

  554. 	vmulss	%xmm0 , %xmm6 , %xmm6

  555. 	vmulss	%xmm0 , %xmm7 , %xmm7

  556. 

  557. 

  558. #if !defined(TRMMKERNEL)

  559. 

  560. 	vaddss 	        (CO1), %xmm4,%xmm4

  561. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  562. 

  563. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  564. 	vaddss  1 * SIZE(CO1, LDC), %xmm7,%xmm7

  565. 

  566. #endif

  567. 

  568. 	vmovss	%xmm4 ,  	(CO1)

  569. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  570. 

  571. 	vmovss	%xmm6 ,  	(CO1, LDC)

  572. 	vmovss	%xmm7 , 1 * SIZE(CO1, LDC)

  573. 

  574. .endm

  575. 

  576. 

  577. /*******************************************************************************************/

  578. 

  579. .macro KERNEL1x2_SUB

  580. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  581. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  582. 	vmovss	 -3 * SIZE(BO, BI, SIZE), %xmm3

  583. 	vmulss		%xmm2 , %xmm0 , %xmm12

  584. 	vmulss		%xmm3 , %xmm0 , %xmm14

  585. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  586. 	vaddss 	        %xmm14, %xmm6 , %xmm6

  587. 	addq	$ 2 , BI	

  588. 	addq	$ 1, %rax 

  589. .endm

  590. 

  591. .macro SAVE1x2

  592. 

  593. 	vmovss	ALPHA, %xmm0

  594. 

  595. 	vmulss	%xmm0 , %xmm4 , %xmm4

  596. 	vmulss	%xmm0 , %xmm6 , %xmm6

  597. 

  598. #if !defined(TRMMKERNEL)

  599. 

  600. 	vaddss 	        (CO1), %xmm4,%xmm4

  601. 	vaddss 	        (CO1, LDC), %xmm6,%xmm6

  602. 

  603. #endif

  604. 

  605. 	vmovss	%xmm4 ,  	(CO1)

  606. 	vmovss	%xmm6 ,  	(CO1, LDC)

  607. 

  608. .endm

  609. 

  610. 

  611. /*******************************************************************************************/

  612. 

  613. /*******************************************************************************************

  614. * 1 line of N

  615. *******************************************************************************************/

  616. 

  617. .macro KERNEL16x1_SUB

  618. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  619. 	vmovups 	 -8 * SIZE(AO, %rax, SIZE), %ymm1

  620. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  621. 	vmulps		%ymm2 , %ymm0 , %ymm12

  622. 	vmulps		%ymm2 , %ymm1 , %ymm13

  623. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  624. 	vaddps 	        %ymm13, %ymm5 , %ymm5

  625. 	addq	$ 1 , BI	

  626. 	addq	$ 16, %rax 

  627. .endm

  628. 

  629. .macro SAVE16x1

  630. 

  631. 	vbroadcastss	ALPHA, %ymm0

  632. 

  633. 	vmulps	%ymm0 , %ymm4 , %ymm4

  634. 	vmulps	%ymm0 , %ymm5 , %ymm5

  635. 

  636. #if !defined(TRMMKERNEL)

  637. 

  638. 	vaddps 	        (CO1), %ymm4,%ymm4

  639. 	vaddps  8 * SIZE(CO1), %ymm5,%ymm5

  640. 

  641. #endif

  642. 

  643. 	vmovups	%ymm4 ,  	(CO1)

  644. 	vmovups	%ymm5 , 8 * SIZE(CO1)

  645. 

  646. .endm

  647. 

  648. 

  649. /*******************************************************************************************/

  650. 

  651. .macro KERNEL8x1_SUB

  652. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %ymm0

  653. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %ymm2

  654. 	vmulps		%ymm2 , %ymm0 , %ymm12

  655. 	vaddps 	        %ymm12, %ymm4 , %ymm4

  656. 	addq	$ 1 , BI	

  657. 	addq	$ 8 , %rax 

  658. .endm

  659. 

  660. .macro SAVE8x1

  661. 

  662. 	vbroadcastss	ALPHA, %ymm0

  663. 

  664. 	vmulps	%ymm0 , %ymm4 , %ymm4

  665. 

  666. #if !defined(TRMMKERNEL)

  667. 

  668. 	vaddps 	        (CO1), %ymm4,%ymm4

  669. 

  670. #endif

  671. 

  672. 	vmovups	%ymm4 ,  	(CO1)

  673. 

  674. .endm

  675. 

  676. 

  677. 

  678. /*******************************************************************************************/

  679. 

  680. .macro KERNEL4x1_SUB

  681. 	vmovups 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  682. 	vbroadcastss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  683. 	vmulps		%xmm2 , %xmm0 , %xmm12

  684. 	vaddps 	        %xmm12, %xmm4 , %xmm4

  685. 	addq	$ 1 , BI	

  686. 	addq	$ 4 , %rax 

  687. .endm

  688. 

  689. .macro SAVE4x1

  690. 

  691. 	vbroadcastss	ALPHA, %xmm0

  692. 

  693. 	vmulps	%xmm0 , %xmm4 , %xmm4

  694. 

  695. #if !defined(TRMMKERNEL)

  696. 

  697. 	vaddps 	        (CO1), %xmm4,%xmm4

  698. 

  699. #endif

  700. 

  701. 	vmovups	%xmm4 ,  	(CO1)

  702. 

  703. .endm

  704. 

  705. 

  706. /*******************************************************************************************/

  707. 

  708. .macro KERNEL2x1_SUB

  709. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  710. 	vmovss 	-15 * SIZE(AO, %rax, SIZE), %xmm1

  711. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  712. 	vmulss		%xmm2 , %xmm0 , %xmm12

  713. 	vmulss		%xmm2 , %xmm1 , %xmm13

  714. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  715. 	vaddss 	        %xmm13, %xmm5 , %xmm5

  716. 	addq	$ 1 , BI	

  717. 	addq	$ 2 , %rax 

  718. .endm

  719. 

  720. .macro SAVE2x1

  721. 

  722. 	vmovss	ALPHA, %xmm0

  723. 

  724. 	vmulss	%xmm0 , %xmm4 , %xmm4

  725. 	vmulss	%xmm0 , %xmm5 , %xmm5

  726. 

  727. #if !defined(TRMMKERNEL)

  728. 

  729. 	vaddss 	        (CO1), %xmm4,%xmm4

  730. 	vaddss  1 * SIZE(CO1), %xmm5,%xmm5

  731. 

  732. #endif

  733. 

  734. 	vmovss	%xmm4 ,  	(CO1)

  735. 	vmovss	%xmm5 , 1 * SIZE(CO1)

  736. 

  737. .endm

  738. 

  739. 

  740. /*******************************************************************************************/

  741. 

  742. .macro KERNEL1x1_SUB

  743. 	vmovss 	-16 * SIZE(AO, %rax, SIZE), %xmm0

  744. 	vmovss	 -4 * SIZE(BO, BI, SIZE), %xmm2

  745. 	vmulss		%xmm2 , %xmm0 , %xmm12

  746. 	vaddss 	        %xmm12, %xmm4 , %xmm4

  747. 	addq	$ 1 , BI	

  748. 	addq	$ 1 , %rax 

  749. .endm

  750. 

  751. .macro SAVE1x1

  752. 

  753. 	vmovss	ALPHA, %xmm0

  754. 

  755. 	vmulss	%xmm0 , %xmm4 , %xmm4

  756. 

  757. #if !defined(TRMMKERNEL)

  758. 

  759. 	vaddss 	        (CO1), %xmm4,%xmm4

  760. 

  761. #endif

  762. 

  763. 	vmovss	%xmm4 ,  	(CO1)

  764. 

  765. .endm

  766. 

  767. 

  768. /*******************************************************************************************/

  769. 

  770. /*************************************************************************************

  771. * TRMM Kernel

  772. *************************************************************************************/

  773. 

  774. 

  775. 	PROLOGUE

  776. 	PROFCODE

  777. 	

  778. 	subq	$STACKSIZE, %rsp

  779. 	movq	%rbx,   (%rsp)

  780. 	movq	%rbp,  8(%rsp)

  781. 	movq	%r12, 16(%rsp)

  782. 	movq	%r13, 24(%rsp)

  783. 	movq	%r14, 32(%rsp)

  784. 	movq	%r15, 40(%rsp)

  785. 

  786. 	vzeroupper

  787. 

  788. #ifdef WINDOWS_ABI

  789. 	movq	%rdi,    48(%rsp)

  790. 	movq	%rsi,    56(%rsp)

  791. 	movups	%xmm6,   64(%rsp)

  792. 	movups	%xmm7,   80(%rsp)

  793. 	movups	%xmm8,   96(%rsp)

  794. 	movups	%xmm9,  112(%rsp)

  795. 	movups	%xmm10, 128(%rsp)

  796. 	movups	%xmm11, 144(%rsp)

  797. 	movups	%xmm12, 160(%rsp)

  798. 	movups	%xmm13, 176(%rsp)

  799. 	movups	%xmm14, 192(%rsp)

  800. 	movups	%xmm15, 208(%rsp)

  801. 

  802. 	movq	ARG1,      OLD_M

  803. 	movq	ARG2,      OLD_N

  804. 	movq	ARG3,      OLD_K

  805. 	movq	OLD_A,     A

  806. 	movq	OLD_B,     B

  807. 	movq	OLD_C,     C

  808. 	movq	OLD_LDC,   LDC

  809. #ifdef TRMMKERNEL

  810. 	vmovsd	OLD_OFFSET, %xmm12

  811. #endif

  812. 	vmovaps	%xmm3, %xmm0

  813. 

  814. #else

  815. 	movq	STACKSIZE +  8(%rsp), LDC

  816. #ifdef TRMMKERNEL

  817. 	movsd	STACKSIZE + 16(%rsp), %xmm12

  818. #endif

  819. 

  820. #endif

  821. 

  822. 	movq    %rsp, SP      # save old stack

  823.         subq    $128 + L_BUFFER_SIZE, %rsp

  824.         andq    $-4096, %rsp    # align stack

  825. 

  826.         STACK_TOUCH

  827. 

  828. 	cmpq	$0, OLD_M

  829. 	je	.L999

  830. 

  831. 	cmpq	$0, OLD_N

  832. 	je	.L999

  833. 

  834. 	cmpq	$0, OLD_K

  835. 	je	.L999

  836. 

  837. 	movq	OLD_M, M

  838. 	movq	OLD_N, N

  839. 	movq	OLD_K, K

  840. 

  841. 	vmovss	 %xmm0, ALPHA

  842. 

  843. 	salq	$BASE_SHIFT, LDC

  844. 

  845. 	movq    N, %rax

  846.         xorq    %rdx, %rdx

  847.         movq    $4,  %rdi

  848.         divq    %rdi                    //    N / 4

  849.         movq    %rax, Ndiv6             //    N / 4

  850.         movq    %rdx, Nmod6             //    N % 4

  851. 

  852. 	

  853. 

  854. #ifdef TRMMKERNEL

  855. 	vmovsd	%xmm12, OFFSET

  856. 	vmovsd	%xmm12, KK

  857. #ifndef LEFT

  858. 	negq	KK

  859. #endif	

  860. #endif

  861. 

  862. 	movq	Ndiv6,  J

  863. 	cmpq	$0, J

  864. 	je	.L2_0

  865. 	ALIGN_4

  866. 

  867. /*******************************************************************************************/

  868. 

  869. .L4_01:

  870. 	// copy to sub buffer

  871. 	movq	B, BO1

  872. 	leaq    BUFFER1, BO		// first buffer to BO

  873. 	movq	K, %rax

  874. 	sarq	$2, %rax		// K / 4

  875. 	jz	.L4_01b

  876. 	ALIGN_4

  877. 

  878. 

  879. .L4_01a:

  880.         prefetcht0 512(BO1)

  881.         prefetchw  512(BO)

  882. 

  883. 	vmovups	       (BO1), %xmm0

  884. 	vmovups	 4*SIZE(BO1), %xmm1

  885. 	vmovups	 8*SIZE(BO1), %xmm2

  886. 	vmovups	12*SIZE(BO1), %xmm3

  887. 

  888. 	vmovups	%xmm0,       (BO)

  889. 	vmovups	%xmm1, 4*SIZE(BO)

  890. 	vmovups	%xmm2, 8*SIZE(BO)

  891. 	vmovups	%xmm3,12*SIZE(BO)

  892. 

  893. 	addq	$ 16*SIZE,BO1

  894. 	addq	$ 16*SIZE,BO

  895. 	decq	%rax

  896. 	jnz	.L4_01a

  897. 

  898. 

  899. .L4_01b:

  900. 

  901.         movq    K, %rax

  902.         andq    $3, %rax                // K % 4

  903.         jz      .L4_02d

  904.         ALIGN_4

  905. 

  906. .L4_02c:

  907. 

  908. 	vmovups	(BO1), %xmm0

  909. 	vmovups	%xmm0, (BO)

  910. 	addq	$ 4*SIZE,BO1

  911. 	addq	$ 4*SIZE,BO

  912. 	decq	%rax

  913. 	jnz	.L4_02c

  914. 

  915. .L4_02d:

  916. 

  917. 	movq	BO1, B			// next offset of B

  918. 

  919. .L4_10:

  920. 	movq	 C, CO1

  921. 	leaq	(C, LDC, 2), CO2	

  922. 	leaq	(C, LDC, 4), C		// c += 4 * ldc

  923. 

  924. #if defined(TRMMKERNEL) && defined(LEFT)

  925.         movq    OFFSET, %rax

  926.         movq    %rax, KK

  927. #endif

  928. 	

  929. 	movq	A, AO		 	// aoffset = a

  930. 	addq	$ 16 * SIZE, AO

  931. 

  932. 	movq	M,  I

  933. 	sarq	$4, I			// i = (m >> 4)

  934. 	je	.L4_20

  935. 

  936. 	ALIGN_4

  937. 

  938. .L4_11:

  939. #if !defined(TRMMKERNEL) || \

  940.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  941.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  942.         leaq    BUFFER1, BO             // first buffer to BO

  943.         addq    $4 * SIZE, BO

  944. #else

  945.         movq    KK, %rax

  946.         leaq    BUFFER1, BO             	// first buffer to BO

  947.         addq    $4 * SIZE, BO

  948.         movq    %rax, BI                        //  Index for BO

  949. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  950.         leaq    (BO, BI, SIZE), BO

  951.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  952.         leaq    (AO, %rax, SIZE), AO

  953. #endif

  954. 

  955. 

  956. 	vzeroall

  957. 

  958. #ifndef TRMMKERNEL

  959.         movq    K, %rax

  960. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  961.         movq    K, %rax

  962.         subq    KK, %rax

  963.         movq    %rax, KKK

  964. #else

  965.         movq    KK, %rax

  966. #ifdef LEFT

  967.         addq    $16, %rax	// number of values in AO

  968. #else

  969.         addq    $4, %rax	// number of values in BO

  970. #endif

  971.         movq    %rax, KKK

  972. #endif

  973. 

  974. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  975. 	je	.L4_16

  976. 	movq    %rax, BI                        //  Index for BO

  977.         leaq    (,BI,4) , BI                   	//  BI = BI * 4 ; number of values

  978. 

  979. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  980. 	leaq	(AO, %rax, SIZE), AO

  981. 	leaq	(BO, BI, SIZE), BO

  982. 	negq	BI

  983. 	negq	%rax

  984. 	ALIGN_4

  985. 

  986. .L4_12:

  987. 

  988. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  989. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  990. 	KERNEL16x4_SUB

  991. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  992. 	KERNEL16x4_SUB

  993. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  994. 	KERNEL16x4_SUB

  995. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  996. 	KERNEL16x4_SUB

  997. 

  998. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  999. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  1000. 	KERNEL16x4_SUB

  1001. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1002. 	KERNEL16x4_SUB

  1003. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1004. 	KERNEL16x4_SUB

  1005. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1006. 	KERNEL16x4_SUB

  1007. 

  1008. 	je	.L4_16

  1009. 

  1010. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1011. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  1012. 	KERNEL16x4_SUB

  1013. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1014. 	KERNEL16x4_SUB

  1015. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1016. 	KERNEL16x4_SUB

  1017. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1018. 	KERNEL16x4_SUB

  1019. 

  1020. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1021. 	prefetcht0	B_PR1(BO, BI  , SIZE)

  1022. 	KERNEL16x4_SUB

  1023. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1024. 	KERNEL16x4_SUB

  1025. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1026. 	KERNEL16x4_SUB

  1027. 	prefetcht0	A_PR1(AO, %rax, SIZE)

  1028. 	KERNEL16x4_SUB

  1029. 

  1030. 	je	.L4_16

  1031. 

  1032. 	jmp	.L4_12

  1033. 	ALIGN_4

  1034. 

  1035. .L4_16:

  1036. #ifndef TRMMKERNEL

  1037.         movq    K, %rax

  1038. #else

  1039.         movq    KKK, %rax

  1040. #endif

  1041. 

  1042. 	andq	$7, %rax		# if (k & 1)

  1043. 	je .L4_19

  1044. 

  1045. 	movq    %rax, BI                        //  Index for BO

  1046.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1047. 

  1048. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  1049. 	leaq	(AO, %rax, SIZE), AO

  1050. 	leaq	(BO, BI, SIZE), BO

  1051. 	negq	BI

  1052. 	negq	%rax

  1053. 	ALIGN_4

  1054. 

  1055. .L4_17:

  1056. 

  1057. 	KERNEL16x4_SUB

  1058. 

  1059. 	jl	.L4_17

  1060. 	ALIGN_4

  1061. 

  1062. 

  1063. .L4_19:

  1064. 

  1065. 	SAVE16x4

  1066. 

  1067. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1068.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1069.         movq    K, %rax

  1070.         subq    KKK, %rax

  1071.         movq    %rax, BI                        //  Index for BO

  1072. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1073.         leaq    (BO, BI, SIZE), BO

  1074.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  1075.         leaq    (AO, %rax, SIZE), AO

  1076. #endif

  1077. 

  1078. 

  1079. #if defined(TRMMKERNEL) && defined(LEFT)

  1080.         addq    $16, KK				

  1081. #endif

  1082. 

  1083. 	addq	$16 * SIZE, CO1		# coffset += 16

  1084. 	addq	$16 * SIZE, CO2		# coffset += 16

  1085. 	decq	I			# i --

  1086. 	jg	.L4_11

  1087. 	ALIGN_4	

  1088. 

  1089. /**************************************************************************

  1090. * Rest of M 

  1091. ***************************************************************************/

  1092. .L4_20:

  1093. 	// Test rest of M

  1094. 

  1095. 	testq	$15, M

  1096. 	jz	.L4_60		// to next 3 lines of N

  1097. 

  1098. 	testq	$8, M		

  1099. 	jz	.L4_21pre

  1100. 	ALIGN_4

  1101. 

  1102. /**************************************************************************/

  1103. 

  1104. .L4_20_1:

  1105. #if !defined(TRMMKERNEL) || \

  1106.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1107.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1108.         leaq    BUFFER1, BO             // first buffer to BO

  1109.         addq    $4 * SIZE, BO

  1110. #else

  1111.         movq    KK, %rax

  1112.         leaq    BUFFER1, BO             // first buffer to BO

  1113.         addq    $4 * SIZE, BO

  1114.         movq    %rax, BI                        //  Index for BO

  1115. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1116.         leaq    (BO, BI, SIZE), BO

  1117.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  1118.         leaq    (AO, %rax, SIZE), AO

  1119. #endif

  1120. 

  1121. 

  1122. 	vzeroall

  1123. 

  1124. #ifndef TRMMKERNEL

  1125.         movq    K, %rax

  1126. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1127.         movq    K, %rax

  1128.         subq    KK, %rax

  1129.         movq    %rax, KKK

  1130. #else

  1131.         movq    KK, %rax

  1132. #ifdef LEFT

  1133.         addq    $8, %rax        // number of values in A

  1134. #else

  1135.         addq    $4, %rax        // number of values in BO

  1136. #endif

  1137.         movq    %rax, KKK

  1138. #endif

  1139. 

  1140. 

  1141. 	andq	$-8, %rax

  1142. 	je	.L4_20_6

  1143. 	movq    %rax, BI                        //  Index for BO

  1144.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1145. 

  1146. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  1147. 	leaq	(AO, %rax, SIZE), AO

  1148. 	leaq	(BO, BI, SIZE), BO

  1149. 	negq	BI

  1150. 	negq	%rax

  1151. 	ALIGN_4

  1152. 

  1153. .L4_20_2:

  1154. 

  1155. 	KERNEL8x4_SUB

  1156. 	KERNEL8x4_SUB

  1157. 	KERNEL8x4_SUB

  1158. 	KERNEL8x4_SUB

  1159. 

  1160. 	KERNEL8x4_SUB

  1161. 	KERNEL8x4_SUB

  1162. 	KERNEL8x4_SUB

  1163. 	KERNEL8x4_SUB

  1164. 

  1165. 	je	.L4_20_6

  1166. 

  1167. 	KERNEL8x4_SUB

  1168. 	KERNEL8x4_SUB

  1169. 	KERNEL8x4_SUB

  1170. 	KERNEL8x4_SUB

  1171. 

  1172. 	KERNEL8x4_SUB

  1173. 	KERNEL8x4_SUB

  1174. 	KERNEL8x4_SUB

  1175. 	KERNEL8x4_SUB

  1176. 

  1177. 	je	.L4_20_6

  1178. 

  1179. 	jmp	.L4_20_2

  1180. 	ALIGN_4

  1181. 

  1182. .L4_20_6:

  1183. #ifndef TRMMKERNEL

  1184.         movq    K, %rax

  1185. #else

  1186.         movq    KKK, %rax

  1187. #endif

  1188. 

  1189. 	andq	$7, %rax		# if (k & 1)

  1190. 	je .L4_20_9

  1191. 

  1192. 	movq    %rax, BI                        //  Index for BO

  1193.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1194. 

  1195. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  1196. 	leaq	(AO, %rax, SIZE), AO

  1197. 	leaq	(BO, BI, SIZE), BO

  1198. 	negq	BI

  1199. 	negq	%rax

  1200. 	ALIGN_4

  1201. 

  1202. .L4_20_7:

  1203. 

  1204. 	KERNEL8x4_SUB

  1205. 

  1206. 	jl	.L4_20_7

  1207. 	ALIGN_4

  1208. 

  1209. 

  1210. .L4_20_9:

  1211. 

  1212. 	SAVE8x4

  1213. 

  1214. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1215.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1216.         movq    K, %rax 

  1217.         subq    KKK, %rax

  1218.         movq    %rax, BI                        //  Index for BO

  1219. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1220.         leaq    (BO, BI, SIZE), BO         

  1221.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  1222.         leaq    (AO, %rax, SIZE), AO

  1223. #endif  

  1224. 

  1225. 

  1226. #if defined(TRMMKERNEL) && defined(LEFT)

  1227.         addq    $8, KK

  1228. #endif

  1229. 

  1230. 	addq	$8 * SIZE, CO1		# coffset += 8

  1231. 	addq	$8 * SIZE, CO2		# coffset += 8

  1232. 	ALIGN_4

  1233. 	

  1234. 

  1235. 

  1236. /**************************************************************************/

  1237. 

  1238. .L4_21pre:

  1239. 

  1240. 	testq	$4, M		

  1241. 	jz	.L4_30

  1242. 	ALIGN_4

  1243. 

  1244. .L4_21:

  1245. #if !defined(TRMMKERNEL) || \

  1246.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1247.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1248.         leaq    BUFFER1, BO             // first buffer to BO

  1249.         addq    $4 * SIZE, BO

  1250. #else

  1251.         movq    KK, %rax

  1252.         leaq    BUFFER1, BO             // first buffer to BO

  1253.         addq    $4 * SIZE, BO

  1254.         movq    %rax, BI                        //  Index for BO

  1255. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1256.         leaq    (BO, BI, SIZE), BO

  1257.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  1258.         leaq    (AO, %rax, SIZE), AO

  1259. #endif

  1260. 

  1261. 

  1262. 	vzeroall

  1263. 

  1264. #ifndef TRMMKERNEL

  1265.         movq    K, %rax

  1266. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1267.         movq    K, %rax

  1268.         subq    KK, %rax

  1269.         movq    %rax, KKK

  1270. #else

  1271.         movq    KK, %rax

  1272. #ifdef LEFT

  1273.         addq    $4, %rax        // number of values in A

  1274. #else

  1275.         addq    $4, %rax        // number of values in BO

  1276. #endif

  1277.         movq    %rax, KKK

  1278. #endif

  1279. 

  1280. 

  1281. 	andq	$-8, %rax

  1282. 	je	.L4_26

  1283. 	movq    %rax, BI                        //  Index for BO

  1284.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1285. 

  1286. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  1287. 	leaq	(AO, %rax, SIZE), AO

  1288. 	leaq	(BO, BI, SIZE), BO

  1289. 	negq	BI

  1290. 	negq	%rax

  1291. 	ALIGN_4

  1292. 

  1293. .L4_22:

  1294. 

  1295. 	KERNEL4x4_SUB

  1296. 	KERNEL4x4_SUB

  1297. 	KERNEL4x4_SUB

  1298. 	KERNEL4x4_SUB

  1299. 

  1300. 	KERNEL4x4_SUB

  1301. 	KERNEL4x4_SUB

  1302. 	KERNEL4x4_SUB

  1303. 	KERNEL4x4_SUB

  1304. 

  1305. 	je	.L4_26

  1306. 

  1307. 	KERNEL4x4_SUB

  1308. 	KERNEL4x4_SUB

  1309. 	KERNEL4x4_SUB

  1310. 	KERNEL4x4_SUB

  1311. 

  1312. 	KERNEL4x4_SUB

  1313. 	KERNEL4x4_SUB

  1314. 	KERNEL4x4_SUB

  1315. 	KERNEL4x4_SUB

  1316. 

  1317. 	je	.L4_26

  1318. 

  1319. 	jmp	.L4_22

  1320. 	ALIGN_4

  1321. 

  1322. .L4_26:

  1323. #ifndef TRMMKERNEL

  1324.         movq    K, %rax

  1325. #else

  1326.         movq    KKK, %rax

  1327. #endif

  1328. 

  1329. 	andq	$7, %rax		# if (k & 1)

  1330. 	je .L4_29

  1331. 

  1332. 	movq    %rax, BI                        //  Index for BO

  1333.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1334. 

  1335. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  1336. 	leaq	(AO, %rax, SIZE), AO

  1337. 	leaq	(BO, BI, SIZE), BO

  1338. 	negq	BI

  1339. 	negq	%rax

  1340. 	ALIGN_4

  1341. 

  1342. .L4_27:

  1343. 

  1344. 	KERNEL4x4_SUB

  1345. 

  1346. 	jl	.L4_27

  1347. 	ALIGN_4

  1348. 

  1349. 

  1350. .L4_29:

  1351. 

  1352. 	SAVE4x4

  1353. 

  1354. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1355.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1356.         movq    K, %rax 

  1357.         subq    KKK, %rax

  1358.         movq    %rax, BI                        //  Index for BO

  1359. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1360.         leaq    (BO, BI, SIZE), BO         

  1361.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  1362.         leaq    (AO, %rax, SIZE), AO

  1363. #endif  

  1364. 

  1365. 

  1366. #if defined(TRMMKERNEL) && defined(LEFT)

  1367.         addq    $4, KK

  1368. #endif

  1369. 

  1370. 	addq	$4 * SIZE, CO1		# coffset += 4

  1371. 	addq	$4 * SIZE, CO2		# coffset += 4

  1372. 	ALIGN_4

  1373. 	

  1374. 

  1375. .L4_30:

  1376. 	testq	$2, M		

  1377. 	jz	.L4_40

  1378. 

  1379. 	ALIGN_4

  1380. 

  1381. .L4_31:

  1382. #if !defined(TRMMKERNEL) || \

  1383.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1384.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1385.         leaq    BUFFER1, BO             // first buffer to BO

  1386.         addq    $4 * SIZE, BO

  1387. #else

  1388.         movq    KK, %rax

  1389.         leaq    BUFFER1, BO             // first buffer to BO

  1390.         addq    $4 * SIZE, BO

  1391.         movq    %rax, BI                        //  Index for BO

  1392. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1393.         leaq    (BO, BI, SIZE), BO

  1394.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  1395.         leaq    (AO, %rax, SIZE), AO

  1396. #endif

  1397. 

  1398. 

  1399. 	vzeroall

  1400. 

  1401. #ifndef TRMMKERNEL

  1402.         movq    K, %rax

  1403. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1404.         movq    K, %rax

  1405.         subq    KK, %rax

  1406.         movq    %rax, KKK

  1407. #else

  1408.         movq    KK, %rax

  1409. #ifdef LEFT

  1410.         addq    $2, %rax        // number of values in AO

  1411. #else

  1412.         addq    $4, %rax        // number of values in BO

  1413. #endif

  1414.         movq    %rax, KKK

  1415. #endif

  1416. 

  1417. 

  1418. 	andq	$-8, %rax

  1419. 	je	.L4_36

  1420. 	movq    %rax, BI                        //  Index for BO

  1421.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1422. 

  1423. 	salq	$1, %rax			// rax = rax *2 ; number of values

  1424. 	leaq	(AO, %rax, SIZE), AO

  1425. 	leaq	(BO, BI, SIZE), BO

  1426. 	negq	BI

  1427. 	negq	%rax

  1428. 	ALIGN_4

  1429. 

  1430. .L4_32:

  1431. 

  1432. 	KERNEL2x4_SUB

  1433. 	KERNEL2x4_SUB

  1434. 	KERNEL2x4_SUB

  1435. 	KERNEL2x4_SUB

  1436. 

  1437. 	KERNEL2x4_SUB

  1438. 	KERNEL2x4_SUB

  1439. 	KERNEL2x4_SUB

  1440. 	KERNEL2x4_SUB

  1441. 

  1442. 	je	.L4_36

  1443. 

  1444. 	KERNEL2x4_SUB

  1445. 	KERNEL2x4_SUB

  1446. 	KERNEL2x4_SUB

  1447. 	KERNEL2x4_SUB

  1448. 

  1449. 	KERNEL2x4_SUB

  1450. 	KERNEL2x4_SUB

  1451. 	KERNEL2x4_SUB

  1452. 	KERNEL2x4_SUB

  1453. 

  1454. 	je	.L4_36

  1455. 

  1456. 	jmp	.L4_32

  1457. 	ALIGN_4

  1458. 

  1459. .L4_36:

  1460. #ifndef TRMMKERNEL

  1461.         movq    K, %rax

  1462. #else

  1463.         movq    KKK, %rax

  1464. #endif

  1465. 

  1466. 	andq	$7, %rax		# if (k & 1)

  1467. 	je .L4_39

  1468. 

  1469. 	movq    %rax, BI                        //  Index for BO

  1470. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1471. 	

  1472. 	salq	$1, %rax			// rax = rax *2 ; number of values

  1473. 	leaq	(AO, %rax, SIZE), AO

  1474. 	leaq	(BO, BI, SIZE), BO

  1475. 	negq	BI

  1476. 	negq	%rax

  1477. 	ALIGN_4

  1478. 

  1479. .L4_37:

  1480. 

  1481. 	KERNEL2x4_SUB

  1482. 

  1483. 	jl	.L4_37

  1484. 	ALIGN_4

  1485. 

  1486. 

  1487. .L4_39:

  1488. 

  1489. 	SAVE2x4

  1490. 

  1491. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1492.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1493.         movq    K, %rax 

  1494.         subq    KKK, %rax

  1495.         movq    %rax, BI                        //  Index for BO

  1496. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1497.         leaq    (BO, BI, SIZE), BO         

  1498.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  1499.         leaq    (AO, %rax, SIZE), AO

  1500. #endif  

  1501. 

  1502. 

  1503. #if defined(TRMMKERNEL) && defined(LEFT)

  1504.         addq    $2, KK

  1505. #endif

  1506. 

  1507. 	addq	$2 * SIZE, CO1		# coffset += 2

  1508. 	addq	$2 * SIZE, CO2		# coffset += 2

  1509. 	ALIGN_4

  1510. 

  1511. .L4_40:

  1512. 	testq	$1, M		

  1513. 	jz	.L4_60		// to next 4 lines of N

  1514. 

  1515. 	ALIGN_4

  1516. 

  1517. .L4_41:

  1518. #if !defined(TRMMKERNEL) || \

  1519.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1520.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1521.         leaq    BUFFER1, BO             // first buffer to BO

  1522.         addq    $4 * SIZE, BO

  1523. #else

  1524.         movq    KK, %rax

  1525.         leaq    BUFFER1, BO             // first buffer to BO

  1526.         addq    $4 * SIZE, BO

  1527.         movq    %rax, BI                        //  Index for BO

  1528. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1529.         leaq    (BO, BI, SIZE), BO

  1530.         leaq    (AO, %rax, SIZE), AO

  1531. #endif

  1532. 

  1533. 

  1534. 	vzeroall

  1535. 

  1536. #ifndef TRMMKERNEL

  1537.         movq    K, %rax

  1538. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1539.         movq    K, %rax

  1540.         subq    KK, %rax

  1541.         movq    %rax, KKK

  1542. #else

  1543.         movq    KK, %rax

  1544. #ifdef LEFT

  1545.         addq    $1, %rax        // number of values in AO

  1546. #else

  1547.         addq    $4, %rax        // number of values in BO

  1548. #endif

  1549.         movq    %rax, KKK

  1550. #endif

  1551. 

  1552. 	andq	$-8, %rax

  1553. 	je	.L4_46

  1554. 	movq    %rax, BI                        //  Index for BO

  1555.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1556. 

  1557. 	leaq	(AO, %rax, SIZE), AO

  1558. 	leaq	(BO, BI, SIZE), BO

  1559. 	negq	BI

  1560. 	negq	%rax

  1561. 	ALIGN_4

  1562. 

  1563. .L4_42:

  1564. 

  1565. 	KERNEL1x4_SUB

  1566. 	KERNEL1x4_SUB

  1567. 	KERNEL1x4_SUB

  1568. 	KERNEL1x4_SUB

  1569. 

  1570. 	KERNEL1x4_SUB

  1571. 	KERNEL1x4_SUB

  1572. 	KERNEL1x4_SUB

  1573. 	KERNEL1x4_SUB

  1574. 

  1575. 	je	.L4_46

  1576. 

  1577. 	KERNEL1x4_SUB

  1578. 	KERNEL1x4_SUB

  1579. 	KERNEL1x4_SUB

  1580. 	KERNEL1x4_SUB

  1581. 

  1582. 	KERNEL1x4_SUB

  1583. 	KERNEL1x4_SUB

  1584. 	KERNEL1x4_SUB

  1585. 	KERNEL1x4_SUB

  1586. 

  1587. 	je	.L4_46

  1588. 

  1589. 	jmp	.L4_42

  1590. 	ALIGN_4

  1591. 

  1592. .L4_46:

  1593. #ifndef TRMMKERNEL

  1594.         movq    K, %rax

  1595. #else

  1596.         movq    KKK, %rax

  1597. #endif

  1598. 

  1599. 	andq	$7, %rax		# if (k & 1)

  1600. 	je .L4_49

  1601. 

  1602. 	movq    %rax, BI                        //  Index for BO

  1603.         leaq    (,BI,4), BI                   	//  BI = BI * 4 ; number of values

  1604. 

  1605. 	leaq	(AO, %rax, SIZE), AO

  1606. 	leaq	(BO, BI, SIZE), BO

  1607. 	negq	BI

  1608. 	negq	%rax

  1609. 	ALIGN_4

  1610. 

  1611. .L4_47:

  1612. 

  1613. 	KERNEL1x4_SUB

  1614. 

  1615. 	jl	.L4_47

  1616. 	ALIGN_4

  1617. 

  1618. 

  1619. .L4_49:

  1620. 

  1621. 	SAVE1x4

  1622. 

  1623. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1624.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1625.         movq    K, %rax 

  1626.         subq    KKK, %rax

  1627.         movq    %rax, BI                        //  Index for BO

  1628. 	leaq	(,BI, 4), BI			// BI = BI * 4 ; number of values

  1629.         leaq    (BO, BI, SIZE), BO         

  1630.         leaq    (AO, %rax, SIZE), AO

  1631. #endif  

  1632. 

  1633. 

  1634. #if defined(TRMMKERNEL) && defined(LEFT)

  1635.         addq    $1, KK

  1636. #endif

  1637. 

  1638. 	addq	$1 * SIZE, CO1		# coffset += 1

  1639. 	addq	$1 * SIZE, CO2		# coffset += 1

  1640. 	ALIGN_4

  1641. 	

  1642. 

  1643. 

  1644. 

  1645. 	

  1646. .L4_60:

  1647. #if defined(TRMMKERNEL) && !defined(LEFT)

  1648.         addq    $4, KK

  1649. #endif

  1650. 

  1651. 	decq	J			// j --

  1652. 	jg	.L4_01			// next 4 lines of N

  1653. 

  1654. 

  1655. 

  1656. /*******************************************************************************************/

  1657. .L2_0:

  1658. 

  1659. 	movq	Nmod6, J		

  1660. 	andq	$3, J			// j % 4

  1661. 	je	.L999

  1662. 

  1663. 	movq	Nmod6, J		

  1664. 	andq	$2, J			// j % 4

  1665. 	je	.L1_0

  1666. 

  1667. .L2_01:

  1668. 

  1669. 	// copy to sub buffer

  1670. 	movq	B, BO1

  1671. 	leaq    BUFFER1, BO		// first buffer to BO

  1672. 	movq	K, %rax

  1673. 	sarq	$2, %rax		// K / 4

  1674. 	jz	.L2_01b

  1675. 	ALIGN_4

  1676. 

  1677. .L2_01a:

  1678. 

  1679. 	vmovsd	      (BO1), %xmm0

  1680. 	vmovsd	2*SIZE(BO1), %xmm1

  1681. 	vmovsd	4*SIZE(BO1), %xmm2

  1682. 	vmovsd	6*SIZE(BO1), %xmm3

  1683. 

  1684. 	vmovsd	%xmm0,       (BO)

  1685. 	vmovsd	%xmm1, 2*SIZE(BO)

  1686. 	vmovsd	%xmm2, 4*SIZE(BO)

  1687. 	vmovsd	%xmm3, 6*SIZE(BO)

  1688. 

  1689. 	addq	$8*SIZE,BO1

  1690. 	addq	$8*SIZE,BO

  1691. 	decq	%rax

  1692. 	jnz	.L2_01a

  1693. 

  1694. 

  1695. .L2_01b:

  1696. 

  1697.         movq    K, %rax

  1698.         andq    $3, %rax                // K % 4

  1699.         jz      .L2_02d

  1700.         ALIGN_4

  1701. 

  1702. .L2_02c:

  1703. 

  1704. 	vmovsd 	(BO1), %xmm0

  1705. 	vmovsd 	%xmm0, (BO)

  1706. 	addq	$2*SIZE,BO1

  1707. 	addq	$2*SIZE,BO

  1708. 	decq	%rax

  1709. 	jnz	.L2_02c

  1710. 

  1711. .L2_02d:

  1712. 

  1713. 	movq	BO1, B			// next offset of B

  1714. 

  1715. .L2_10:

  1716. 	movq	C, CO1

  1717. 	leaq	(C, LDC, 2), C		// c += 2 * ldc

  1718. 

  1719. #if defined(TRMMKERNEL) && defined(LEFT)

  1720.         movq    OFFSET, %rax

  1721.         movq    %rax, KK

  1722. #endif

  1723. 	

  1724. 	movq	A, AO		 	// aoffset = a

  1725. 	addq	$16 * SIZE, AO

  1726. 

  1727. 	movq	M,  I

  1728. 	sarq	$4, I			// i = (m >> 4)

  1729. 	je	.L2_20

  1730. 

  1731. 	ALIGN_4

  1732. 

  1733. .L2_11:

  1734. #if !defined(TRMMKERNEL) || \

  1735.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1736.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1737.         leaq    BUFFER1, BO             // first buffer to BO

  1738.         addq    $4 * SIZE, BO

  1739. #else

  1740.         movq    KK, %rax

  1741.         leaq    BUFFER1, BO             // first buffer to BO

  1742.         addq    $4 * SIZE, BO

  1743.         movq    %rax, BI                        //  Index for BO

  1744.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1745.         leaq    (BO, BI, SIZE), BO

  1746.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  1747.         leaq    (AO, %rax, SIZE), AO

  1748. #endif

  1749. 

  1750. 

  1751. 	vzeroall

  1752. 

  1753. #ifndef TRMMKERNEL

  1754.         movq    K, %rax

  1755. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1756.         movq    K, %rax

  1757.         subq    KK, %rax

  1758.         movq    %rax, KKK

  1759. #else

  1760.         movq    KK, %rax

  1761. #ifdef LEFT

  1762.         addq    $16, %rax	// number of values in AO

  1763. #else

  1764.         addq    $2, %rax	// number of values in BO

  1765. #endif

  1766.         movq    %rax, KKK

  1767. #endif

  1768. 

  1769. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  1770. 	je	.L2_16

  1771. 	movq    %rax, BI                        //  Index for BO

  1772.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1773. 

  1774. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  1775. 	leaq	(AO, %rax, SIZE), AO

  1776. 	leaq	(BO, BI, SIZE), BO

  1777. 	negq	BI

  1778. 	negq	%rax

  1779. 	ALIGN_4

  1780. 

  1781. .L2_12:

  1782. 

  1783. 	KERNEL16x2_SUB

  1784. 	KERNEL16x2_SUB

  1785. 	KERNEL16x2_SUB

  1786. 	KERNEL16x2_SUB

  1787. 

  1788. 	KERNEL16x2_SUB

  1789. 	KERNEL16x2_SUB

  1790. 	KERNEL16x2_SUB

  1791. 	KERNEL16x2_SUB

  1792. 

  1793. 	je	.L2_16

  1794. 

  1795. 	KERNEL16x2_SUB

  1796. 	KERNEL16x2_SUB

  1797. 	KERNEL16x2_SUB

  1798. 	KERNEL16x2_SUB

  1799. 

  1800. 	KERNEL16x2_SUB

  1801. 	KERNEL16x2_SUB

  1802. 	KERNEL16x2_SUB

  1803. 	KERNEL16x2_SUB

  1804. 

  1805. 	je	.L2_16

  1806. 

  1807. 	jmp	.L2_12

  1808. 	ALIGN_4

  1809. 

  1810. .L2_16:

  1811. #ifndef TRMMKERNEL

  1812.         movq    K, %rax

  1813. #else

  1814.         movq    KKK, %rax

  1815. #endif

  1816. 

  1817. 	andq	$7, %rax		# if (k & 1)

  1818. 	je .L2_19

  1819. 

  1820. 	movq    %rax, BI                        //  Index for BO

  1821.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1822. 

  1823. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  1824. 	leaq	(AO, %rax, SIZE), AO

  1825. 	leaq	(BO, BI, SIZE), BO

  1826. 	negq	BI

  1827. 	negq	%rax

  1828. 	ALIGN_4

  1829. 

  1830. .L2_17:

  1831. 

  1832. 	KERNEL16x2_SUB

  1833. 

  1834. 	jl	.L2_17

  1835. 	ALIGN_4

  1836. 

  1837. 

  1838. .L2_19:

  1839. 

  1840. 	SAVE16x2

  1841. 

  1842. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1843.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1844.         movq    K, %rax

  1845.         subq    KKK, %rax

  1846.         movq    %rax, BI                        //  Index for BO

  1847.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1848.         leaq    (BO, BI, SIZE), BO

  1849.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  1850.         leaq    (AO, %rax, SIZE), AO

  1851. #endif

  1852. 

  1853. 

  1854. #if defined(TRMMKERNEL) && defined(LEFT)

  1855.         addq    $16, KK				

  1856. #endif

  1857. 

  1858. 	addq	$16 * SIZE, CO1		# coffset += 16

  1859. 	decq	I			# i --

  1860. 	jg	.L2_11

  1861. 	ALIGN_4	

  1862. 

  1863. /**************************************************************************

  1864. * Rest of M 

  1865. ***************************************************************************/

  1866. .L2_20:

  1867. 	// Test rest of M

  1868. 

  1869. 	testq	$15, M

  1870. 	jz	.L2_60		// to next 2 lines of N

  1871. 

  1872. 	testq	$8, M		

  1873. 	jz	.L2_21pre

  1874. 	ALIGN_4

  1875. 

  1876. /**************************************************************************/

  1877. 

  1878. .L2_20_1:

  1879. #if !defined(TRMMKERNEL) || \

  1880.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1881.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1882.         leaq    BUFFER1, BO             // first buffer to BO

  1883.         addq    $4 * SIZE, BO

  1884. #else

  1885.         movq    KK, %rax

  1886.         leaq    BUFFER1, BO             // first buffer to BO

  1887.         addq    $4 * SIZE, BO

  1888.         movq    %rax, BI                        //  Index for BO

  1889.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1890.         leaq    (BO, BI, SIZE), BO

  1891.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  1892.         leaq    (AO, %rax, SIZE), AO

  1893. #endif

  1894. 

  1895. 

  1896. 	vzeroall

  1897. 

  1898. #ifndef TRMMKERNEL

  1899.         movq    K, %rax

  1900. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  1901.         movq    K, %rax

  1902.         subq    KK, %rax

  1903.         movq    %rax, KKK

  1904. #else

  1905.         movq    KK, %rax

  1906. #ifdef LEFT

  1907.         addq    $8, %rax        // number of values in A

  1908. #else

  1909.         addq    $2, %rax        // number of values in BO

  1910. #endif

  1911.         movq    %rax, KKK

  1912. #endif

  1913. 

  1914. 

  1915. 	andq	$-8, %rax

  1916. 	je	.L2_20_6

  1917. 	movq    %rax, BI                        //  Index for BO

  1918.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1919. 

  1920. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  1921. 	leaq	(AO, %rax, SIZE), AO

  1922. 	leaq	(BO, BI, SIZE), BO

  1923. 	negq	BI

  1924. 	negq	%rax

  1925. 	ALIGN_4

  1926. 

  1927. .L2_20_2:

  1928. 

  1929. 

  1930. 	KERNEL8x2_SUB

  1931. 	KERNEL8x2_SUB

  1932. 	KERNEL8x2_SUB

  1933. 	KERNEL8x2_SUB

  1934. 

  1935. 	KERNEL8x2_SUB

  1936. 	KERNEL8x2_SUB

  1937. 	KERNEL8x2_SUB

  1938. 	KERNEL8x2_SUB

  1939. 

  1940. 	je	.L2_20_6

  1941. 

  1942. 	KERNEL8x2_SUB

  1943. 	KERNEL8x2_SUB

  1944. 	KERNEL8x2_SUB

  1945. 	KERNEL8x2_SUB

  1946. 

  1947. 	KERNEL8x2_SUB

  1948. 	KERNEL8x2_SUB

  1949. 	KERNEL8x2_SUB

  1950. 	KERNEL8x2_SUB

  1951. 

  1952. 	je	.L2_20_6

  1953. 

  1954. 	jmp	.L2_20_2

  1955. 	ALIGN_4

  1956. 

  1957. .L2_20_6:

  1958. #ifndef TRMMKERNEL

  1959.         movq    K, %rax

  1960. #else

  1961.         movq    KKK, %rax

  1962. #endif

  1963. 

  1964. 	andq	$7, %rax		# if (k & 1)

  1965. 	je .L2_20_9

  1966. 

  1967. 	movq    %rax, BI                        //  Index for BO

  1968.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1969. 

  1970. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  1971. 	leaq	(AO, %rax, SIZE), AO

  1972. 	leaq	(BO, BI, SIZE), BO

  1973. 	negq	BI

  1974. 	negq	%rax

  1975. 	ALIGN_4

  1976. 

  1977. .L2_20_7:

  1978. 

  1979. 	KERNEL8x2_SUB

  1980. 

  1981. 	jl	.L2_20_7

  1982. 	ALIGN_4

  1983. 

  1984. 

  1985. .L2_20_9:

  1986. 

  1987. 	SAVE8x2

  1988. 

  1989. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  1990.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  1991.         movq    K, %rax 

  1992.         subq    KKK, %rax

  1993.         movq    %rax, BI                        //  Index for BO

  1994.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  1995.         leaq    (BO, BI, SIZE), BO         

  1996.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  1997.         leaq    (AO, %rax, SIZE), AO

  1998. #endif  

  1999. 

  2000. 

  2001. #if defined(TRMMKERNEL) && defined(LEFT)

  2002.         addq    $8, KK

  2003. #endif

  2004. 

  2005. 	addq	$8 * SIZE, CO1		# coffset += 8

  2006. 	ALIGN_4

  2007. 	

  2008. 

  2009. 

  2010. /**************************************************************************/

  2011. 

  2012. .L2_21pre:

  2013. 

  2014. 	testq	$4, M		

  2015. 	jz	.L2_30

  2016. 	ALIGN_4

  2017. 

  2018. .L2_21:

  2019. #if !defined(TRMMKERNEL) || \

  2020.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2021.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2022.         leaq    BUFFER1, BO             // first buffer to BO

  2023.         addq    $4 * SIZE, BO

  2024. #else

  2025.         movq    KK, %rax

  2026.         leaq    BUFFER1, BO             // first buffer to BO

  2027.         addq    $4 * SIZE, BO

  2028.         movq    %rax, BI                        //  Index for BO

  2029.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2030.         leaq    (BO, BI, SIZE), BO

  2031.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2032.         leaq    (AO, %rax, SIZE), AO

  2033. #endif

  2034. 

  2035. 

  2036. 	vzeroall

  2037. 

  2038. #ifndef TRMMKERNEL

  2039.         movq    K, %rax

  2040. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2041.         movq    K, %rax

  2042.         subq    KK, %rax

  2043.         movq    %rax, KKK

  2044. #else

  2045.         movq    KK, %rax

  2046. #ifdef LEFT

  2047.         addq    $4, %rax        // number of values in A

  2048. #else

  2049.         addq    $2, %rax        // number of values in BO

  2050. #endif

  2051.         movq    %rax, KKK

  2052. #endif

  2053. 

  2054. 

  2055. 	andq	$-8, %rax

  2056. 	je	.L2_26

  2057. 	movq    %rax, BI                        //  Index for BO

  2058.         leaq    (BI,BI,1), BI                   //  BI = BI * 1 ; number of values

  2059. 

  2060. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2061. 	leaq	(AO, %rax, SIZE), AO

  2062. 	leaq	(BO, BI, SIZE), BO

  2063. 	negq	BI

  2064. 	negq	%rax

  2065. 	ALIGN_4

  2066. 

  2067. .L2_22:

  2068. 

  2069. 

  2070. 	KERNEL4x2_SUB

  2071. 	KERNEL4x2_SUB

  2072. 	KERNEL4x2_SUB

  2073. 	KERNEL4x2_SUB

  2074. 

  2075. 	KERNEL4x2_SUB

  2076. 	KERNEL4x2_SUB

  2077. 	KERNEL4x2_SUB

  2078. 	KERNEL4x2_SUB

  2079. 

  2080. 	je	.L2_26

  2081. 

  2082. 	KERNEL4x2_SUB

  2083. 	KERNEL4x2_SUB

  2084. 	KERNEL4x2_SUB

  2085. 	KERNEL4x2_SUB

  2086. 

  2087. 	KERNEL4x2_SUB

  2088. 	KERNEL4x2_SUB

  2089. 	KERNEL4x2_SUB

  2090. 	KERNEL4x2_SUB

  2091. 

  2092. 	je	.L2_26

  2093. 

  2094. 	jmp	.L2_22

  2095. 	ALIGN_4

  2096. 

  2097. .L2_26:

  2098. #ifndef TRMMKERNEL

  2099.         movq    K, %rax

  2100. #else

  2101.         movq    KKK, %rax

  2102. #endif

  2103. 

  2104. 	andq	$7, %rax		# if (k & 1)

  2105. 	je .L2_29

  2106. 

  2107. 	movq    %rax, BI                        //  Index for BO

  2108.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2109. 

  2110. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2111. 	leaq	(AO, %rax, SIZE), AO

  2112. 	leaq	(BO, BI, SIZE), BO

  2113. 	negq	BI

  2114. 	negq	%rax

  2115. 	ALIGN_4

  2116. 

  2117. .L2_27:

  2118. 

  2119. 	KERNEL4x2_SUB

  2120. 

  2121. 	jl	.L2_27

  2122. 	ALIGN_4

  2123. 

  2124. 

  2125. .L2_29:

  2126. 

  2127. 	SAVE4x2

  2128. 

  2129. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2130.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2131.         movq    K, %rax 

  2132.         subq    KKK, %rax

  2133.         movq    %rax, BI                        //  Index for BO

  2134.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2135.         leaq    (BO, BI, SIZE), BO         

  2136.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2137.         leaq    (AO, %rax, SIZE), AO

  2138. #endif  

  2139. 

  2140. 

  2141. #if defined(TRMMKERNEL) && defined(LEFT)

  2142.         addq    $4, KK

  2143. #endif

  2144. 

  2145. 	addq	$4 * SIZE, CO1		# coffset += 4

  2146. 	ALIGN_4

  2147. 	

  2148. 

  2149. .L2_30:

  2150. 	testq	$2, M		

  2151. 	jz	.L2_40

  2152. 

  2153. 	ALIGN_4

  2154. 

  2155. .L2_31:

  2156. #if !defined(TRMMKERNEL) || \

  2157.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2158.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2159.         leaq    BUFFER1, BO             // first buffer to BO

  2160.         addq    $4 * SIZE, BO

  2161. #else

  2162.         movq    KK, %rax

  2163.         leaq    BUFFER1, BO             // first buffer to BO

  2164.         addq    $4 * SIZE, BO

  2165.         movq    %rax, BI                        //  Index for BO

  2166.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2167.         leaq    (BO, BI, SIZE), BO

  2168.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2169.         leaq    (AO, %rax, SIZE), AO

  2170. #endif

  2171. 

  2172. 

  2173. 	vzeroall

  2174. 

  2175. #ifndef TRMMKERNEL

  2176.         movq    K, %rax

  2177. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2178.         movq    K, %rax

  2179.         subq    KK, %rax

  2180.         movq    %rax, KKK

  2181. #else

  2182.         movq    KK, %rax

  2183. #ifdef LEFT

  2184.         addq    $2, %rax        // number of values in AO

  2185. #else

  2186.         addq    $2, %rax        // number of values in BO

  2187. #endif

  2188.         movq    %rax, KKK

  2189. #endif

  2190. 

  2191. 

  2192. 	andq	$-8, %rax

  2193. 	je	.L2_36

  2194. 	movq    %rax, BI                        //  Index for BO

  2195.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2196. 

  2197. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2198. 	leaq	(AO, %rax, SIZE), AO

  2199. 	leaq	(BO, BI, SIZE), BO

  2200. 	negq	BI

  2201. 	negq	%rax

  2202. 	ALIGN_4

  2203. 

  2204. .L2_32:

  2205. 

  2206. 	KERNEL2x2_SUB

  2207. 	KERNEL2x2_SUB

  2208. 	KERNEL2x2_SUB

  2209. 	KERNEL2x2_SUB

  2210. 

  2211. 	KERNEL2x2_SUB

  2212. 	KERNEL2x2_SUB

  2213. 	KERNEL2x2_SUB

  2214. 	KERNEL2x2_SUB

  2215. 

  2216. 	je	.L2_36

  2217. 

  2218. 	KERNEL2x2_SUB

  2219. 	KERNEL2x2_SUB

  2220. 	KERNEL2x2_SUB

  2221. 	KERNEL2x2_SUB

  2222. 

  2223. 	KERNEL2x2_SUB

  2224. 	KERNEL2x2_SUB

  2225. 	KERNEL2x2_SUB

  2226. 	KERNEL2x2_SUB

  2227. 

  2228. 	je	.L2_36

  2229. 

  2230. 	jmp	.L2_32

  2231. 	ALIGN_4

  2232. 

  2233. .L2_36:

  2234. #ifndef TRMMKERNEL

  2235.         movq    K, %rax

  2236. #else

  2237.         movq    KKK, %rax

  2238. #endif

  2239. 

  2240. 	andq	$7, %rax		# if (k & 1)

  2241. 	je .L2_39

  2242. 

  2243. 	movq    %rax, BI                        //  Index for BO

  2244.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2245. 	

  2246. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2247. 	leaq	(AO, %rax, SIZE), AO

  2248. 	leaq	(BO, BI, SIZE), BO

  2249. 	negq	BI

  2250. 	negq	%rax

  2251. 	ALIGN_4

  2252. 

  2253. .L2_37:

  2254. 

  2255. 	KERNEL2x2_SUB

  2256. 

  2257. 	jl	.L2_37

  2258. 	ALIGN_4

  2259. 

  2260. 

  2261. .L2_39:

  2262. 

  2263. 	SAVE2x2

  2264. 

  2265. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2266.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2267.         movq    K, %rax 

  2268.         subq    KKK, %rax

  2269.         movq    %rax, BI                        //  Index for BO

  2270.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2271.         leaq    (BO, BI, SIZE), BO         

  2272.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2273.         leaq    (AO, %rax, SIZE), AO

  2274. #endif  

  2275. 

  2276. 

  2277. #if defined(TRMMKERNEL) && defined(LEFT)

  2278.         addq    $2, KK

  2279. #endif

  2280. 

  2281. 	addq	$2 * SIZE, CO1		# coffset += 2

  2282. 	ALIGN_4

  2283. 

  2284. .L2_40:

  2285. 	testq	$1, M		

  2286. 	jz	.L2_60		// to next 2 lines of N

  2287. 

  2288. 	ALIGN_4

  2289. 

  2290. .L2_41:

  2291. #if !defined(TRMMKERNEL) || \

  2292.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2293.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2294.         leaq    BUFFER1, BO             // first buffer to BO

  2295.         addq    $4 * SIZE, BO

  2296. #else

  2297.         movq    KK, %rax

  2298.         leaq    BUFFER1, BO             // first buffer to BO

  2299.         addq    $4 * SIZE, BO

  2300.         movq    %rax, BI                        //  Index for BO

  2301.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2302.         leaq    (BO, BI, SIZE), BO

  2303.         leaq    (AO, %rax, SIZE), AO

  2304. #endif

  2305. 

  2306. 

  2307. 	vzeroall

  2308. 

  2309. #ifndef TRMMKERNEL

  2310.         movq    K, %rax

  2311. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2312.         movq    K, %rax

  2313.         subq    KK, %rax

  2314.         movq    %rax, KKK

  2315. #else

  2316.         movq    KK, %rax

  2317. #ifdef LEFT

  2318.         addq    $1, %rax        // number of values in AO

  2319. #else

  2320.         addq    $2, %rax        // number of values in BO

  2321. #endif

  2322.         movq    %rax, KKK

  2323. #endif

  2324. 

  2325. 	andq	$-8, %rax

  2326. 	je	.L2_46

  2327. 	movq    %rax, BI                        //  Index for BO

  2328.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2329. 

  2330. 	leaq	(AO, %rax, SIZE), AO

  2331. 	leaq	(BO, BI, SIZE), BO

  2332. 	negq	BI

  2333. 	negq	%rax

  2334. 	ALIGN_4

  2335. 

  2336. .L2_42:

  2337. 

  2338. 	KERNEL1x2_SUB

  2339. 	KERNEL1x2_SUB

  2340. 	KERNEL1x2_SUB

  2341. 	KERNEL1x2_SUB

  2342. 

  2343. 	KERNEL1x2_SUB

  2344. 	KERNEL1x2_SUB

  2345. 	KERNEL1x2_SUB

  2346. 	KERNEL1x2_SUB

  2347. 

  2348. 	je	.L2_46

  2349. 

  2350. 	KERNEL1x2_SUB

  2351. 	KERNEL1x2_SUB

  2352. 	KERNEL1x2_SUB

  2353. 	KERNEL1x2_SUB

  2354. 

  2355. 	KERNEL1x2_SUB

  2356. 	KERNEL1x2_SUB

  2357. 	KERNEL1x2_SUB

  2358. 	KERNEL1x2_SUB

  2359. 

  2360. 	je	.L2_46

  2361. 

  2362. 	jmp	.L2_42

  2363. 	ALIGN_4

  2364. 

  2365. .L2_46:

  2366. #ifndef TRMMKERNEL

  2367.         movq    K, %rax

  2368. #else

  2369.         movq    KKK, %rax

  2370. #endif

  2371. 

  2372. 	andq	$7, %rax		# if (k & 1)

  2373. 	je .L2_49

  2374. 

  2375. 	movq    %rax, BI                        //  Index for BO

  2376.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2377. 

  2378. 	leaq	(AO, %rax, SIZE), AO

  2379. 	leaq	(BO, BI, SIZE), BO

  2380. 	negq	BI

  2381. 	negq	%rax

  2382. 	ALIGN_4

  2383. 

  2384. .L2_47:

  2385. 

  2386. 	KERNEL1x2_SUB

  2387. 

  2388. 	jl	.L2_47

  2389. 	ALIGN_4

  2390. 

  2391. 

  2392. .L2_49:

  2393. 

  2394. 	SAVE1x2

  2395. 

  2396. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2397.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2398.         movq    K, %rax 

  2399.         subq    KKK, %rax

  2400.         movq    %rax, BI                        //  Index for BO

  2401.         leaq    (BI,BI,1), BI                   //  BI = BI * 2 ; number of values

  2402.         leaq    (BO, BI, SIZE), BO         

  2403.         leaq    (AO, %rax, SIZE), AO

  2404. #endif  

  2405. 

  2406. 

  2407. #if defined(TRMMKERNEL) && defined(LEFT)

  2408.         addq    $1, KK

  2409. #endif

  2410. 

  2411. 	addq	$1 * SIZE, CO1		# coffset += 1

  2412. 	ALIGN_4

  2413. 	

  2414. 

  2415. 

  2416. 

  2417. 	

  2418. .L2_60:

  2419. #if defined(TRMMKERNEL) && !defined(LEFT)

  2420.         addq    $2, KK

  2421. #endif

  2422. 

  2423. 

  2424. 

  2425. 

  2426. .L1_0:

  2427. 

  2428. /************************************************************************************************

  2429. * Loop for Nmod6 % 2 > 0

  2430. *************************************************************************************************/

  2431. 

  2432. 	movq	Nmod6, J		

  2433. 	andq	$1, J			// j % 2

  2434. 	je	.L999

  2435. 	ALIGN_4

  2436. 

  2437. .L1_01:

  2438. 	// copy to sub buffer

  2439. 	movq	B, BO1

  2440. 	leaq    BUFFER1, BO		// first buffer to BO

  2441. 	movq	K, %rax

  2442. 	ALIGN_4

  2443. 

  2444. .L1_02b:

  2445. 

  2446. 	vmovss	(BO1), %xmm0

  2447. 	vmovss	%xmm0,       (BO)

  2448. 	addq	$1*SIZE,BO1

  2449. 	addq	$1*SIZE,BO

  2450. 	decq	%rax

  2451. 	jnz	.L1_02b

  2452. 

  2453. .L1_02c:

  2454. 

  2455. 	movq	BO1, B			// next offset of B

  2456. 

  2457. .L1_10:

  2458. 	movq	C, CO1

  2459. 	leaq	(C, LDC, 1), C		// c += 1 * ldc

  2460. 

  2461. #if defined(TRMMKERNEL) && defined(LEFT)

  2462.         movq    OFFSET, %rax

  2463.         movq    %rax, KK

  2464. #endif

  2465. 	

  2466. 	movq	A, AO		 	// aoffset = a

  2467. 	addq	$16 * SIZE, AO

  2468. 

  2469. 	movq	M,  I

  2470. 	sarq	$4, I			// i = (m >> 4)

  2471. 	je	.L1_20

  2472. 

  2473. 	ALIGN_4

  2474. 

  2475. .L1_11:

  2476. #if !defined(TRMMKERNEL) || \

  2477.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2478.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2479.         leaq    BUFFER1, BO             // first buffer to BO

  2480.         addq    $4 * SIZE, BO

  2481. #else

  2482.         movq    KK, %rax

  2483.         leaq    BUFFER1, BO             // first buffer to BO

  2484.         addq    $4 * SIZE, BO

  2485.         movq    %rax, BI                        //  Index for BO

  2486.         leaq    (BO, BI, SIZE), BO

  2487.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  2488.         leaq    (AO, %rax, SIZE), AO

  2489. #endif

  2490. 

  2491. 

  2492. 	vzeroall

  2493. 

  2494. #ifndef TRMMKERNEL

  2495.         movq    K, %rax

  2496. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2497.         movq    K, %rax

  2498.         subq    KK, %rax

  2499.         movq    %rax, KKK

  2500. #else

  2501.         movq    KK, %rax

  2502. #ifdef LEFT

  2503.         addq    $16, %rax	// number of values in AO

  2504. #else

  2505.         addq    $1, %rax	// number of values in BO

  2506. #endif

  2507.         movq    %rax, KKK

  2508. #endif

  2509. 

  2510. 	andq	$-8, %rax			//  K = K - ( K % 8 )

  2511. 	je	.L1_16

  2512. 	movq    %rax, BI                        //  Index for BO

  2513. 

  2514. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  2515. 	leaq	(AO, %rax, SIZE), AO

  2516. 	leaq	(BO, BI, SIZE), BO

  2517. 	negq	BI

  2518. 	negq	%rax

  2519. 	ALIGN_4

  2520. 

  2521. .L1_12:

  2522. 

  2523. 	KERNEL16x1_SUB

  2524. 	KERNEL16x1_SUB

  2525. 	KERNEL16x1_SUB

  2526. 	KERNEL16x1_SUB

  2527. 

  2528. 	KERNEL16x1_SUB

  2529. 	KERNEL16x1_SUB

  2530. 	KERNEL16x1_SUB

  2531. 	KERNEL16x1_SUB

  2532. 

  2533. 	je	.L1_16

  2534. 

  2535. 	KERNEL16x1_SUB

  2536. 	KERNEL16x1_SUB

  2537. 	KERNEL16x1_SUB

  2538. 	KERNEL16x1_SUB

  2539. 

  2540. 	KERNEL16x1_SUB

  2541. 	KERNEL16x1_SUB

  2542. 	KERNEL16x1_SUB

  2543. 	KERNEL16x1_SUB

  2544. 

  2545. 	je	.L1_16

  2546. 

  2547. 	jmp	.L1_12

  2548. 	ALIGN_4

  2549. 

  2550. .L1_16:

  2551. #ifndef TRMMKERNEL

  2552.         movq    K, %rax

  2553. #else

  2554.         movq    KKK, %rax

  2555. #endif

  2556. 

  2557. 	andq	$7, %rax		# if (k & 1)

  2558. 	je .L1_19

  2559. 

  2560. 	movq    %rax, BI                        //  Index for BO

  2561. 

  2562. 	salq	$4, %rax			// rax = rax * 16 ; number of values

  2563. 	leaq	(AO, %rax, SIZE), AO

  2564. 	leaq	(BO, BI, SIZE), BO

  2565. 	negq	BI

  2566. 	negq	%rax

  2567. 	ALIGN_4

  2568. 

  2569. .L1_17:

  2570. 

  2571. 	KERNEL16x1_SUB

  2572. 

  2573. 	jl	.L1_17

  2574. 	ALIGN_4

  2575. 

  2576. 

  2577. .L1_19:

  2578. 

  2579. 	SAVE16x1

  2580. 

  2581. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2582.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2583.         movq    K, %rax

  2584.         subq    KKK, %rax

  2585.         movq    %rax, BI                        //  Index for BO

  2586.         leaq    (BO, BI, SIZE), BO

  2587.         salq    $4, %rax                        // rax = rax * 16 ; number of values

  2588.         leaq    (AO, %rax, SIZE), AO

  2589. #endif

  2590. 

  2591. 

  2592. #if defined(TRMMKERNEL) && defined(LEFT)

  2593.         addq    $16, KK				

  2594. #endif

  2595. 

  2596. 	addq	$16 * SIZE, CO1		# coffset += 16

  2597. 	decq	I			# i --

  2598. 	jg	.L1_11

  2599. 	ALIGN_4	

  2600. 

  2601. /**************************************************************************

  2602. * Rest of M 

  2603. ***************************************************************************/

  2604. .L1_20:

  2605. 	// Test rest of M

  2606. 

  2607. 	testq	$15, M

  2608. 	jz	.L999

  2609. 

  2610. 	testq	$8, M		

  2611. 	jz	.L1_21pre

  2612. 	ALIGN_4

  2613. 

  2614. /**************************************************************************/

  2615. 

  2616. .L1_20_1:

  2617. #if !defined(TRMMKERNEL) || \

  2618.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2619.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2620.         leaq    BUFFER1, BO             // first buffer to BO

  2621.         addq    $4 * SIZE, BO

  2622. #else

  2623.         movq    KK, %rax

  2624.         leaq    BUFFER1, BO             // first buffer to BO

  2625.         addq    $4 * SIZE, BO

  2626.         movq    %rax, BI                        //  Index for BO

  2627.         leaq    (BO, BI, SIZE), BO

  2628.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  2629.         leaq    (AO, %rax, SIZE), AO

  2630. #endif

  2631. 

  2632. 

  2633. 	vzeroall

  2634. 

  2635. #ifndef TRMMKERNEL

  2636.         movq    K, %rax

  2637. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2638.         movq    K, %rax

  2639.         subq    KK, %rax

  2640.         movq    %rax, KKK

  2641. #else

  2642.         movq    KK, %rax

  2643. #ifdef LEFT

  2644.         addq    $8, %rax        // number of values in A

  2645. #else

  2646.         addq    $1, %rax        // number of values in BO

  2647. #endif

  2648.         movq    %rax, KKK

  2649. #endif

  2650. 

  2651. 

  2652. 	andq	$-8, %rax

  2653. 	je	.L1_20_6

  2654. 	movq    %rax, BI                        //  Index for BO

  2655. 

  2656. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  2657. 	leaq	(AO, %rax, SIZE), AO

  2658. 	leaq	(BO, BI, SIZE), BO

  2659. 	negq	BI

  2660. 	negq	%rax

  2661. 	ALIGN_4

  2662. 

  2663. .L1_20_2:

  2664. 

  2665. 	KERNEL8x1_SUB

  2666. 	KERNEL8x1_SUB

  2667. 	KERNEL8x1_SUB

  2668. 	KERNEL8x1_SUB

  2669. 

  2670. 	KERNEL8x1_SUB

  2671. 	KERNEL8x1_SUB

  2672. 	KERNEL8x1_SUB

  2673. 	KERNEL8x1_SUB

  2674. 

  2675. 	je	.L1_20_6

  2676. 

  2677. 	KERNEL8x1_SUB

  2678. 	KERNEL8x1_SUB

  2679. 	KERNEL8x1_SUB

  2680. 	KERNEL8x1_SUB

  2681. 

  2682. 	KERNEL8x1_SUB

  2683. 	KERNEL8x1_SUB

  2684. 	KERNEL8x1_SUB

  2685. 	KERNEL8x1_SUB

  2686. 

  2687. 	je	.L1_20_6

  2688. 

  2689. 	jmp	.L1_20_2

  2690. 	ALIGN_4

  2691. 

  2692. .L1_20_6:

  2693. #ifndef TRMMKERNEL

  2694.         movq    K, %rax

  2695. #else

  2696.         movq    KKK, %rax

  2697. #endif

  2698. 

  2699. 	andq	$7, %rax		# if (k & 1)

  2700. 	je .L1_20_9

  2701. 

  2702. 	movq    %rax, BI                        //  Index for BO

  2703. 

  2704. 	salq	$3, %rax			// rax = rax * 8 ; number of values

  2705. 	leaq	(AO, %rax, SIZE), AO

  2706. 	leaq	(BO, BI, SIZE), BO

  2707. 	negq	BI

  2708. 	negq	%rax

  2709. 	ALIGN_4

  2710. 

  2711. .L1_20_7:

  2712. 

  2713. 	KERNEL8x1_SUB

  2714. 

  2715. 	jl	.L1_20_7

  2716. 	ALIGN_4

  2717. 

  2718. 

  2719. .L1_20_9:

  2720. 

  2721. 	SAVE8x1

  2722. 

  2723. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2724.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2725.         movq    K, %rax 

  2726.         subq    KKK, %rax

  2727.         movq    %rax, BI                        //  Index for BO

  2728.         leaq    (BO, BI, SIZE), BO         

  2729.         salq    $3, %rax                        // rax = rax * 8 ; number of values

  2730.         leaq    (AO, %rax, SIZE), AO

  2731. #endif  

  2732. 

  2733. 

  2734. #if defined(TRMMKERNEL) && defined(LEFT)

  2735.         addq    $8, KK

  2736. #endif

  2737. 

  2738. 	addq	$8 * SIZE, CO1		# coffset += 8

  2739. 	ALIGN_4

  2740. 	

  2741. 

  2742. 

  2743. /**************************************************************************/

  2744. 

  2745. .L1_21pre:

  2746. 

  2747. 	testq	$4, M		

  2748. 	jz	.L1_30

  2749. 	ALIGN_4

  2750. 

  2751. .L1_21:

  2752. #if !defined(TRMMKERNEL) || \

  2753.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2754.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2755.         leaq    BUFFER1, BO             // first buffer to BO

  2756.         addq    $4 * SIZE, BO

  2757. #else

  2758.         movq    KK, %rax

  2759.         leaq    BUFFER1, BO             // first buffer to BO

  2760.         addq    $4 * SIZE, BO

  2761.         movq    %rax, BI                        //  Index for BO

  2762.         leaq    (BO, BI, SIZE), BO

  2763.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2764.         leaq    (AO, %rax, SIZE), AO

  2765. #endif

  2766. 

  2767. 

  2768. 	vzeroall

  2769. 

  2770. #ifndef TRMMKERNEL

  2771.         movq    K, %rax

  2772. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2773.         movq    K, %rax

  2774.         subq    KK, %rax

  2775.         movq    %rax, KKK

  2776. #else

  2777.         movq    KK, %rax

  2778. #ifdef LEFT

  2779.         addq    $4, %rax        // number of values in A

  2780. #else

  2781.         addq    $1, %rax        // number of values in BO

  2782. #endif

  2783.         movq    %rax, KKK

  2784. #endif

  2785. 

  2786. 

  2787. 	andq	$-8, %rax

  2788. 	je	.L1_26

  2789. 	movq    %rax, BI                        //  Index for BO

  2790. 

  2791. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2792. 	leaq	(AO, %rax, SIZE), AO

  2793. 	leaq	(BO, BI, SIZE), BO

  2794. 	negq	BI

  2795. 	negq	%rax

  2796. 	ALIGN_4

  2797. 

  2798. .L1_22:

  2799. 

  2800. 	KERNEL4x1_SUB

  2801. 	KERNEL4x1_SUB

  2802. 	KERNEL4x1_SUB

  2803. 	KERNEL4x1_SUB

  2804. 

  2805. 	KERNEL4x1_SUB

  2806. 	KERNEL4x1_SUB

  2807. 	KERNEL4x1_SUB

  2808. 	KERNEL4x1_SUB

  2809. 

  2810. 	je	.L1_26

  2811. 

  2812. 	KERNEL4x1_SUB

  2813. 	KERNEL4x1_SUB

  2814. 	KERNEL4x1_SUB

  2815. 	KERNEL4x1_SUB

  2816. 

  2817. 	KERNEL4x1_SUB

  2818. 	KERNEL4x1_SUB

  2819. 	KERNEL4x1_SUB

  2820. 	KERNEL4x1_SUB

  2821. 

  2822. 	je	.L1_26

  2823. 

  2824. 	jmp	.L1_22

  2825. 	ALIGN_4

  2826. 

  2827. .L1_26:

  2828. #ifndef TRMMKERNEL

  2829.         movq    K, %rax

  2830. #else

  2831.         movq    KKK, %rax

  2832. #endif

  2833. 

  2834. 	andq	$7, %rax		# if (k & 1)

  2835. 	je .L1_29

  2836. 

  2837. 	movq    %rax, BI                        //  Index for BO

  2838. 

  2839. 	salq	$2, %rax			// rax = rax * 4 ; number of values

  2840. 	leaq	(AO, %rax, SIZE), AO

  2841. 	leaq	(BO, BI, SIZE), BO

  2842. 	negq	BI

  2843. 	negq	%rax

  2844. 	ALIGN_4

  2845. 

  2846. .L1_27:

  2847. 

  2848. 	KERNEL4x1_SUB

  2849. 

  2850. 	jl	.L1_27

  2851. 	ALIGN_4

  2852. 

  2853. 

  2854. .L1_29:

  2855. 

  2856. 	SAVE4x1

  2857. 

  2858. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2859.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2860.         movq    K, %rax 

  2861.         subq    KKK, %rax

  2862.         movq    %rax, BI                        //  Index for BO

  2863.         leaq    (BO, BI, SIZE), BO         

  2864.         salq    $2, %rax                        // rax = rax * 4 ; number of values

  2865.         leaq    (AO, %rax, SIZE), AO

  2866. #endif  

  2867. 

  2868. 

  2869. #if defined(TRMMKERNEL) && defined(LEFT)

  2870.         addq    $4, KK

  2871. #endif

  2872. 

  2873. 	addq	$4 * SIZE, CO1		# coffset += 4

  2874. 	ALIGN_4

  2875. 	

  2876. 

  2877. .L1_30:

  2878. 	testq	$2, M		

  2879. 	jz	.L1_40

  2880. 

  2881. 	ALIGN_4

  2882. 

  2883. .L1_31:

  2884. #if !defined(TRMMKERNEL) || \

  2885.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2886.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2887.         leaq    BUFFER1, BO             // first buffer to BO

  2888.         addq    $4 * SIZE, BO

  2889. #else

  2890.         movq    KK, %rax

  2891.         leaq    BUFFER1, BO             // first buffer to BO

  2892.         addq    $4 * SIZE, BO

  2893.         movq    %rax, BI                        //  Index for BO

  2894.         leaq    (BO, BI, SIZE), BO

  2895.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2896.         leaq    (AO, %rax, SIZE), AO

  2897. #endif

  2898. 

  2899. 

  2900. 	vzeroall

  2901. 

  2902. #ifndef TRMMKERNEL

  2903.         movq    K, %rax

  2904. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  2905.         movq    K, %rax

  2906.         subq    KK, %rax

  2907.         movq    %rax, KKK

  2908. #else

  2909.         movq    KK, %rax

  2910. #ifdef LEFT

  2911.         addq    $2, %rax        // number of values in AO

  2912. #else

  2913.         addq    $1, %rax        // number of values in BO

  2914. #endif

  2915.         movq    %rax, KKK

  2916. #endif

  2917. 

  2918. 

  2919. 	andq	$-8, %rax

  2920. 	je	.L1_36

  2921. 	movq    %rax, BI                        //  Index for BO

  2922. 

  2923. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2924. 	leaq	(AO, %rax, SIZE), AO

  2925. 	leaq	(BO, BI, SIZE), BO

  2926. 	negq	BI

  2927. 	negq	%rax

  2928. 	ALIGN_4

  2929. 

  2930. .L1_32:

  2931. 

  2932. 	KERNEL2x1_SUB

  2933. 	KERNEL2x1_SUB

  2934. 	KERNEL2x1_SUB

  2935. 	KERNEL2x1_SUB

  2936. 

  2937. 	KERNEL2x1_SUB

  2938. 	KERNEL2x1_SUB

  2939. 	KERNEL2x1_SUB

  2940. 	KERNEL2x1_SUB

  2941. 

  2942. 	je	.L1_36

  2943. 

  2944. 	KERNEL2x1_SUB

  2945. 	KERNEL2x1_SUB

  2946. 	KERNEL2x1_SUB

  2947. 	KERNEL2x1_SUB

  2948. 

  2949. 	KERNEL2x1_SUB

  2950. 	KERNEL2x1_SUB

  2951. 	KERNEL2x1_SUB

  2952. 	KERNEL2x1_SUB

  2953. 

  2954. 	je	.L1_36

  2955. 

  2956. 	jmp	.L1_32

  2957. 	ALIGN_4

  2958. 

  2959. .L1_36:

  2960. #ifndef TRMMKERNEL

  2961.         movq    K, %rax

  2962. #else

  2963.         movq    KKK, %rax

  2964. #endif

  2965. 

  2966. 	andq	$7, %rax		# if (k & 1)

  2967. 	je .L1_39

  2968. 

  2969. 	movq    %rax, BI                        //  Index for BO

  2970. 	

  2971. 	salq	$1, %rax			// rax = rax *2 ; number of values

  2972. 	leaq	(AO, %rax, SIZE), AO

  2973. 	leaq	(BO, BI, SIZE), BO

  2974. 	negq	BI

  2975. 	negq	%rax

  2976. 	ALIGN_4

  2977. 

  2978. .L1_37:

  2979. 

  2980. 	KERNEL2x1_SUB

  2981. 

  2982. 	jl	.L1_37

  2983. 	ALIGN_4

  2984. 

  2985. 

  2986. .L1_39:

  2987. 

  2988. 	SAVE2x1

  2989. 

  2990. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  2991.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  2992.         movq    K, %rax 

  2993.         subq    KKK, %rax

  2994.         movq    %rax, BI                        //  Index for BO

  2995.         leaq    (BO, BI, SIZE), BO         

  2996.         salq    $1, %rax                        // rax = rax * 2 ; number of values

  2997.         leaq    (AO, %rax, SIZE), AO

  2998. #endif  

  2999. 

  3000. 

  3001. #if defined(TRMMKERNEL) && defined(LEFT)

  3002.         addq    $2, KK

  3003. #endif

  3004. 

  3005. 	addq	$2 * SIZE, CO1		# coffset += 2

  3006. 	ALIGN_4

  3007. 

  3008. .L1_40:

  3009. 	testq	$1, M		

  3010. 	jz	.L999

  3011. 

  3012. 	ALIGN_4

  3013. 

  3014. .L1_41:

  3015. #if !defined(TRMMKERNEL) || \

  3016.         (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3017.         (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3018.         leaq    BUFFER1, BO             // first buffer to BO

  3019.         addq    $4 * SIZE, BO

  3020. #else

  3021.         movq    KK, %rax

  3022.         leaq    BUFFER1, BO             // first buffer to BO

  3023.         addq    $4 * SIZE, BO

  3024.         movq    %rax, BI                        //  Index for BO

  3025.         leaq    (BO, BI, SIZE), BO

  3026.         leaq    (AO, %rax, SIZE), AO

  3027. #endif

  3028. 

  3029. 

  3030. 	vzeroall

  3031. 

  3032. #ifndef TRMMKERNEL

  3033.         movq    K, %rax

  3034. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  3035.         movq    K, %rax

  3036.         subq    KK, %rax

  3037.         movq    %rax, KKK

  3038. #else

  3039.         movq    KK, %rax

  3040. #ifdef LEFT

  3041.         addq    $1, %rax        // number of values in AO

  3042. #else

  3043.         addq    $1, %rax        // number of values in BO

  3044. #endif

  3045.         movq    %rax, KKK

  3046. #endif

  3047. 

  3048. 	andq	$-8, %rax

  3049. 	je	.L1_46

  3050. 	movq    %rax, BI                        //  Index for BO

  3051. 

  3052. 	leaq	(AO, %rax, SIZE), AO

  3053. 	leaq	(BO, BI, SIZE), BO

  3054. 	negq	BI

  3055. 	negq	%rax

  3056. 	ALIGN_4

  3057. 

  3058. .L1_42:

  3059. 

  3060. 	KERNEL1x1_SUB

  3061. 	KERNEL1x1_SUB

  3062. 	KERNEL1x1_SUB

  3063. 	KERNEL1x1_SUB

  3064. 

  3065. 	KERNEL1x1_SUB

  3066. 	KERNEL1x1_SUB

  3067. 	KERNEL1x1_SUB

  3068. 	KERNEL1x1_SUB

  3069. 

  3070. 	je	.L1_46

  3071. 

  3072. 	KERNEL1x1_SUB

  3073. 	KERNEL1x1_SUB

  3074. 	KERNEL1x1_SUB

  3075. 	KERNEL1x1_SUB

  3076. 

  3077. 	KERNEL1x1_SUB

  3078. 	KERNEL1x1_SUB

  3079. 	KERNEL1x1_SUB

  3080. 	KERNEL1x1_SUB

  3081. 

  3082. 	je	.L1_46

  3083. 

  3084. 	jmp	.L1_42

  3085. 	ALIGN_4

  3086. 

  3087. .L1_46:

  3088. #ifndef TRMMKERNEL

  3089.         movq    K, %rax

  3090. #else

  3091.         movq    KKK, %rax

  3092. #endif

  3093. 

  3094. 	andq	$7, %rax		# if (k & 1)

  3095. 	je .L1_49

  3096. 

  3097. 	movq    %rax, BI                        //  Index for BO

  3098. 

  3099. 	leaq	(AO, %rax, SIZE), AO

  3100. 	leaq	(BO, BI, SIZE), BO

  3101. 	negq	BI

  3102. 	negq	%rax

  3103. 	ALIGN_4

  3104. 

  3105. .L1_47:

  3106. 

  3107. 	KERNEL1x1_SUB

  3108. 

  3109. 	jl	.L1_47

  3110. 	ALIGN_4

  3111. 

  3112. 

  3113. .L1_49:

  3114. 

  3115. 	SAVE1x1

  3116. 

  3117. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  3118.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  3119.         movq    K, %rax 

  3120.         subq    KKK, %rax

  3121.         movq    %rax, BI                        //  Index for BO

  3122.         leaq    (BO, BI, SIZE), BO         

  3123.         leaq    (AO, %rax, SIZE), AO

  3124. #endif  

  3125. 

  3126. 

  3127. #if defined(TRMMKERNEL) && defined(LEFT)

  3128.         addq    $1, KK

  3129. #endif

  3130. 

  3131. 	addq	$1 * SIZE, CO1		# coffset += 1

  3132. 	ALIGN_4

  3133. 	

  3134. 

  3135. .L999:

  3136. 	movq   		SP, %rsp

  3137. 	movq	   (%rsp), %rbx

  3138. 	movq	  8(%rsp), %rbp

  3139. 	movq	 16(%rsp), %r12

  3140. 	movq	 24(%rsp), %r13

  3141. 	movq	 32(%rsp), %r14

  3142. 	movq	 40(%rsp), %r15

  3143. 

  3144. #ifdef WINDOWS_ABI

  3145. 	movq	 48(%rsp), %rdi

  3146. 	movq	 56(%rsp), %rsi

  3147. 	movups	 64(%rsp), %xmm6

  3148. 	movups	 80(%rsp), %xmm7

  3149. 	movups	 96(%rsp), %xmm8

  3150. 	movups	112(%rsp), %xmm9

  3151. 	movups	128(%rsp), %xmm10

  3152. 	movups	144(%rsp), %xmm11

  3153. 	movups	160(%rsp), %xmm12

  3154. 	movups	176(%rsp), %xmm13

  3155. 	movups	192(%rsp), %xmm14

  3156. 	movups	208(%rsp), %xmm15

  3157. #endif

  3158. 

  3159. 	addq	$STACKSIZE, %rsp

  3160. 	ret

  3161. 

  3162. 	EPILOGUE

  3163. 

  3164. 

  3165. 

  3166. 

  3167.