OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_kernel_4x8_skylakex.c

1566 lines
40 kB
Raw Blame History 

  1. /*********************************************************************************

  2. Copyright (c) 2015, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. **********************************************************************************/

  27. 

  28. /*

  29.  * This file is based on dgemm_kernel_4x8_haswell.s (original copyright above).

  30.  * The content got translated from ASM to C+intrinsics, significantly simplified,

  31.  * and AVX512 support added by Arjan van de Ven <arjan@linux.intel.com>

  32.  */

  33. 

  34. 

  35. #include "common.h"

  36. #include <immintrin.h>

  37. 

  38. 

  39. /*******************************************************************************************

  40. * Macro definitions

  41. *******************************************************************************************/

  42. 

  43. 

  44. /******************************************************************************************/

  45. 

  46. 

  47. #define INIT4x8()				\

  48. 	ymm4 = _mm256_setzero_pd();		\

  49. 	ymm5 = _mm256_setzero_pd();		\

  50. 	ymm6 = _mm256_setzero_pd();		\

  51. 	ymm7 = _mm256_setzero_pd();		\

  52. 	ymm8 = _mm256_setzero_pd();		\

  53. 	ymm9 = _mm256_setzero_pd();		\

  54. 	ymm10 = _mm256_setzero_pd();		\

  55. 	ymm11 = _mm256_setzero_pd();		\

  56. 

  57. 

  58. #define KERNEL4x8_SUB()				\

  59. 	ymm0  = _mm256_loadu_pd(AO - 16);	\

  60. /*	ymm0 [ A B C D ] */			\

  61. 	ymm1  = _mm256_loadu_pd(BO - 12);	\

  62. 	ymm2  = _mm256_loadu_pd(BO - 8);	\

  63. /* 	ymm1 [ 1 2 3 4 ] */			\

  64. /* 	ymm2 [ 5 6 7 8 ] */			\

  65. 						\

  66. 	ymm4 += ymm0 * ymm1;			\

  67. /*	ymm4 +=  [ A*1 | B*2 | C*3 | D*4 ] */	\

  68. 	ymm8 += ymm0 * ymm2;			\

  69. /*	ymm8 +=  [ A*5 | B*6 | C*7 | D*8 ] */   \

  70. 						\

  71. 	ymm0  = _mm256_permute4x64_pd(ymm0, 0xb1);	\

  72. /*	ymm0 [ B A D C ] */			\

  73. 	ymm5 += ymm0 * ymm1;			\

  74. /*	ymm5 +=  [ B*1 | A*2 | D*3 | C*4 ] */	\

  75. 	ymm9 += ymm0 * ymm2;			\

  76. /*	ymm9 +=  [ B*5 | A*6 | D*7 | C*8 ] */	\

  77. 						\

  78. 	ymm0  = _mm256_permute4x64_pd(ymm0, 0x1b);	\

  79. /*	ymm0 [ C D A B ]] */ 			\

  80. 	ymm6 += ymm0 * ymm1;			\

  81. /*	ymm6 +=  [ C*1 | D*2 | A*3 | B*4 ] */ 	\

  82. 	ymm10+= ymm0 * ymm2;			\

  83. /*	ymm10 += [ C*5 | D*6 | A*7 | B*8 ] */ 	\

  84. 						\

  85. 	ymm0  = _mm256_permute4x64_pd(ymm0, 0xb1);	\

  86. /*	ymm0 [ D C B A ] */			\

  87. 	ymm7 += ymm0 * ymm1;			\

  88. /*	ymm7  += [ D*1 | C*2 | B*3 | A*4 ] */	\

  89. 	ymm11+= ymm0 * ymm2;			\

  90. /*	ymm11 += [ D*5 | C*6 | B*7 | A*8 ] */	\

  91. 	AO += 4;				\

  92. 	BO += 8;

  93. 

  94. 

  95. #define SAVE4x8(ALPHA)					\

  96. 	ymm0 = _mm256_set1_pd(ALPHA);			\

  97. 	ymm4 *= ymm0;					\

  98. 	ymm5 *= ymm0;					\

  99. 	ymm6 *= ymm0;					\

  100. 	ymm7 *= ymm0;					\

  101. 	ymm8 *= ymm0;					\

  102. 	ymm9 *= ymm0;					\

  103. 	ymm10 *= ymm0;					\

  104. 	ymm11 *= ymm0;					\

  105. 							\

  106. /*	Entry values:  			    */		\

  107. /*	ymm4  = a [ A*1 | B*2 | C*3 | D*4 ] */		\

  108. /*	ymm5  = a [ B*1 | A*2 | D*3 | C*4 ] */		\

  109. /*	ymm6  = a [ C*1 | D*2 | A*3 | B*4 ] */ 		\

  110. /*	ymm7  = a [ D*1 | C*2 | B*3 | A*4 ] */		\

  111. /*	ymm8  = a [ A*5 | B*6 | C*7 | D*8 ] */		\

  112. /*	ymm9  = a [ B*5 | A*6 | D*7 | C*8 ] */		\

  113. /*	ymm10 = a [ C*5 | D*6 | A*7 | B*8 ] */ 		\

  114. /*	ymm11 = a [ D*5 | C*6 | B*7 | A*8 ] */		\

  115. 							\

  116. 	ymm5 = _mm256_permute4x64_pd(ymm5, 0xb1);	\

  117. /*	ymm5 =  a [ A*2 | B*1 | C*4 | D*3 ] */		\

  118. 	ymm7 = _mm256_permute4x64_pd(ymm7, 0xb1);	\

  119. /*	ymm7 =  a [ C*2 | D*1 | A*4 | B*3 ] */		\

  120. 							\

  121. 	ymm0 = _mm256_blend_pd(ymm4, ymm5, 0x0a);	\

  122. 	ymm1 = _mm256_blend_pd(ymm4, ymm5, 0x05);	\

  123. /*	ymm0 =  a [ A*1 | B*1 | C*3 | D*3 ] */		\

  124. /*	ymm1 =  a [ A*2 | B*2 | C*4 | D*4 ] */		\

  125. 	ymm2 = _mm256_blend_pd(ymm6, ymm7, 0x0a);	\

  126. 	ymm3 = _mm256_blend_pd(ymm6, ymm7, 0x05);	\

  127. /*	ymm2 =  a [ C*1 | D*1 | A*3 | B*3 ] */		\

  128. /*	ymm3 =  a [ C*2 | D*2 | A*4 | B*4 ] */		\

  129. 							\

  130. 	ymm2 = _mm256_permute4x64_pd(ymm2, 0x1b);	\

  131. 	ymm3 = _mm256_permute4x64_pd(ymm3, 0x1b);	\

  132. /*	ymm2 =  a [ B*3 | A*3 | D*1 | C*1 ] */		\

  133. /*	ymm3 =  a [ B*4 | A*4 | D*2 | C*2 ] */		\

  134. 	ymm2 = _mm256_permute4x64_pd(ymm2, 0xb1);	\

  135. 	ymm3 = _mm256_permute4x64_pd(ymm3, 0xb1);	\

  136. /*	ymm2 =  a [ A*3 | B*3 | C*1 | D*1 ] */		\

  137. /*	ymm3 =  a [ A*4 | B*4 | C*2 | D*2 ] */		\

  138. 							\

  139. 	ymm4 = _mm256_blend_pd(ymm2, ymm0, 0x03);	\

  140. 	ymm5 = _mm256_blend_pd(ymm3, ymm1, 0x03);	\

  141. /*	ymm4 =  a [ A*1 | B*1 | C*1 | D*1 ] */		\

  142. /*	ymm5 =  a [ A*2 | B*2 | C*2 | D*2 ] */		\

  143. 	ymm6 = _mm256_blend_pd(ymm0, ymm2, 0x03);	\

  144. 	ymm7 = _mm256_blend_pd(ymm1, ymm3, 0x03);	\

  145. /*	ymm5 =  a [ A*3 | B*3 | C*3 | D*3 ] */		\

  146. /*	ymm7 =  a [ A*4 | B*4 | C*4 | D*4 ] */		\

  147. 							\

  148. 	ymm4 += _mm256_loadu_pd(CO1 + (0 * ldc));	\

  149. 	ymm5 += _mm256_loadu_pd(CO1 + (1 * ldc));	\

  150. 	ymm6 += _mm256_loadu_pd(CO1 + (2 * ldc));	\

  151. 	ymm7 += _mm256_loadu_pd(CO1 + (3 * ldc));	\

  152. 	_mm256_storeu_pd(CO1 + (0 * ldc), ymm4);	\

  153. 	_mm256_storeu_pd(CO1 + (1 * ldc), ymm5);	\

  154. 	_mm256_storeu_pd(CO1 + (2 * ldc), ymm6);	\

  155. 	_mm256_storeu_pd(CO1 + (3 * ldc), ymm7);	\

  156. 							\

  157. 	ymm9 = _mm256_permute4x64_pd(ymm9, 0xb1);	\

  158. 	ymm11 = _mm256_permute4x64_pd(ymm11, 0xb1);	\

  159. 							\

  160. 	ymm0 = _mm256_blend_pd(ymm8, ymm9, 0x0a);	\

  161. 	ymm1 = _mm256_blend_pd(ymm8, ymm9, 0x05);	\

  162. 	ymm2 = _mm256_blend_pd(ymm10, ymm11, 0x0a);	\

  163. 	ymm3 = _mm256_blend_pd(ymm10, ymm11, 0x05);	\

  164. 							\

  165. 	ymm2 = _mm256_permute4x64_pd(ymm2, 0x1b);	\

  166. 	ymm3 = _mm256_permute4x64_pd(ymm3, 0x1b);	\

  167. 	ymm2 = _mm256_permute4x64_pd(ymm2, 0xb1);	\

  168. 	ymm3 = _mm256_permute4x64_pd(ymm3, 0xb1);	\

  169. 							\

  170. 	ymm4 = _mm256_blend_pd(ymm2, ymm0, 0x03);	\

  171. 	ymm5 = _mm256_blend_pd(ymm3, ymm1, 0x03);	\

  172. 	ymm6 = _mm256_blend_pd(ymm0, ymm2, 0x03);	\

  173. 	ymm7 = _mm256_blend_pd(ymm1, ymm3, 0x03);	\

  174. 							\

  175. 	ymm4 += _mm256_loadu_pd(CO1 + (4 * ldc));	\

  176. 	ymm5 += _mm256_loadu_pd(CO1 + (5 * ldc));	\

  177. 	ymm6 += _mm256_loadu_pd(CO1 + (6 * ldc));	\

  178. 	ymm7 += _mm256_loadu_pd(CO1 + (7 * ldc));	\

  179. 	_mm256_storeu_pd(CO1 + (4 * ldc), ymm4);	\

  180. 	_mm256_storeu_pd(CO1 + (5 * ldc), ymm5);	\

  181. 	_mm256_storeu_pd(CO1 + (6 * ldc), ymm6);	\

  182. 	_mm256_storeu_pd(CO1 + (7 * ldc), ymm7);	\

  183. 							\

  184. 	CO1 += 4;

  185. 

  186. /******************************************************************************************/

  187. 

  188. #define INIT2x8()				\

  189. 	xmm4 = _mm_setzero_pd(); 		\

  190. 	xmm5 = _mm_setzero_pd(); 		\

  191. 	xmm6 = _mm_setzero_pd(); 		\

  192. 	xmm7 = _mm_setzero_pd(); 		\

  193. 	xmm8 = _mm_setzero_pd(); 		\

  194. 	xmm9 = _mm_setzero_pd(); 		\

  195. 	xmm10 = _mm_setzero_pd(); 		\

  196. 	xmm11 = _mm_setzero_pd(); 		\

  197. 

  198. 

  199. #define KERNEL2x8_SUB()				\

  200. 	xmm0 = _mm_loadu_pd(AO - 16);		\

  201. 	xmm1 = _mm_set1_pd(*(BO - 12));		\

  202. 	xmm2 = _mm_set1_pd(*(BO - 11));		\

  203. 	xmm3 = _mm_set1_pd(*(BO - 10));		\

  204. 	xmm4 += xmm0 * xmm1;			\

  205. 	xmm1 = _mm_set1_pd(*(BO - 9));		\

  206. 	xmm5 += xmm0 * xmm2;			\

  207. 	xmm2 = _mm_set1_pd(*(BO - 8));		\

  208. 	xmm6 += xmm0 * xmm3;			\

  209. 	xmm3 = _mm_set1_pd(*(BO - 7));		\

  210. 	xmm7 += xmm0 * xmm1;			\

  211. 	xmm1 = _mm_set1_pd(*(BO - 6));		\

  212. 	xmm8 += xmm0 * xmm2;			\

  213. 	xmm2 = _mm_set1_pd(*(BO - 5));		\

  214. 	xmm9 += xmm0 * xmm3;			\

  215. 	xmm10 += xmm0 * xmm1;			\

  216. 	xmm11 += xmm0 * xmm2;			\

  217. 	BO += 8;				\

  218. 	AO += 2;

  219. 

  220. #define  SAVE2x8(ALPHA)					\

  221. 	xmm0 = _mm_set1_pd(ALPHA);			\

  222. 	xmm4 *= xmm0;					\

  223. 	xmm5 *= xmm0;					\

  224. 	xmm6 *= xmm0;					\

  225. 	xmm7 *= xmm0;					\

  226. 	xmm8 *= xmm0;					\

  227. 	xmm9 *= xmm0;					\

  228. 	xmm10 *= xmm0;					\

  229. 	xmm11 *= xmm0;					\

  230. 							\

  231. 	xmm4 += _mm_loadu_pd(CO1 + (0 * ldc));		\

  232. 	xmm5 += _mm_loadu_pd(CO1 + (1 * ldc));		\

  233. 	xmm6 += _mm_loadu_pd(CO1 + (2 * ldc));		\

  234. 	xmm7 += _mm_loadu_pd(CO1 + (3 * ldc));		\

  235. 							\

  236. 	_mm_storeu_pd(CO1 + (0 * ldc), xmm4);		\

  237. 	_mm_storeu_pd(CO1 + (1 * ldc), xmm5);		\

  238. 	_mm_storeu_pd(CO1 + (2 * ldc), xmm6);		\

  239. 	_mm_storeu_pd(CO1 + (3 * ldc), xmm7);		\

  240. 							\

  241. 	xmm8 += _mm_loadu_pd(CO1 + (4 * ldc));		\

  242. 	xmm9 += _mm_loadu_pd(CO1 + (5 * ldc));		\

  243. 	xmm10+= _mm_loadu_pd(CO1 + (6 * ldc));		\

  244. 	xmm11+= _mm_loadu_pd(CO1 + (7 * ldc));		\

  245. 	_mm_storeu_pd(CO1 + (4 * ldc), xmm8);		\

  246. 	_mm_storeu_pd(CO1 + (5 * ldc), xmm9);		\

  247. 	_mm_storeu_pd(CO1 + (6 * ldc), xmm10);		\

  248. 	_mm_storeu_pd(CO1 + (7 * ldc), xmm11);		\

  249. 	CO1 += 2;

  250. 

  251. 

  252. 

  253. 

  254. /******************************************************************************************/

  255. 

  256. #define INIT1x8()				\

  257. 	dbl4 = 0;	\

  258. 	dbl5 = 0;	\

  259. 	dbl6 = 0;	\

  260. 	dbl7 = 0;	\

  261. 	dbl8 = 0;	\

  262. 	dbl9 = 0;	\

  263. 	dbl10 = 0;	\

  264. 	dbl11 = 0;	

  265. 

  266. 

  267. #define KERNEL1x8_SUB()				\

  268. 	dbl0 = *(AO - 16);			\

  269. 	dbl1 = *(BO - 12);			\

  270. 	dbl2 = *(BO - 11);			\

  271. 	dbl3 = *(BO - 10);			\

  272. 	dbl4 += dbl0 * dbl1;			\

  273. 	dbl1 = *(BO - 9);			\

  274. 	dbl5 += dbl0 * dbl2;			\

  275. 	dbl2 = *(BO - 8);			\

  276. 	dbl6 += dbl0 * dbl3;			\

  277. 	dbl3 = *(BO - 7);			\

  278. 	dbl7 += dbl0 * dbl1;			\

  279. 	dbl1 = *(BO - 6);			\

  280. 	dbl8 += dbl0 * dbl2;			\

  281. 	dbl2 = *(BO - 5);			\

  282. 	dbl9  += dbl0 * dbl3;			\

  283. 	dbl10 += dbl0 * dbl1;			\

  284. 	dbl11 += dbl0 * dbl2;			\

  285. 	BO += 8;				\

  286. 	AO += 1;

  287. 

  288. 

  289. #define SAVE1x8(ALPHA)				\

  290. 	dbl0 = ALPHA;				\

  291. 	dbl4 *= dbl0;				\

  292. 	dbl5 *= dbl0;				\

  293. 	dbl6 *= dbl0;				\

  294. 	dbl7 *= dbl0;				\

  295. 	dbl8 *= dbl0;				\

  296. 	dbl9 *= dbl0;				\

  297. 	dbl10 *= dbl0;				\

  298. 	dbl11 *= dbl0;				\

  299. 						\

  300. 	dbl4 += *(CO1 + (0 * ldc));		\

  301. 	dbl5 += *(CO1 + (1 * ldc));		\

  302. 	dbl6 += *(CO1 + (2 * ldc));		\

  303. 	dbl7 += *(CO1 + (3 * ldc));		\

  304. 	*(CO1 + (0 * ldc)) = dbl4;		\

  305. 	*(CO1 + (1 * ldc)) = dbl5;		\

  306. 	*(CO1 + (2 * ldc)) = dbl6;		\

  307. 	*(CO1 + (3 * ldc)) = dbl7;		\

  308. 						\

  309. 	dbl8  += *(CO1 + (4 * ldc));		\

  310. 	dbl9  += *(CO1 + (5 * ldc));		\

  311. 	dbl10 += *(CO1 + (6 * ldc));		\

  312. 	dbl11 += *(CO1 + (7 * ldc));		\

  313. 	*(CO1 + (4 * ldc)) = dbl8;		\

  314. 	*(CO1 + (5 * ldc)) = dbl9;		\

  315. 	*(CO1 + (6 * ldc)) = dbl10;		\

  316. 	*(CO1 + (7 * ldc)) = dbl11;		\

  317. 						\

  318. 	CO1 += 1;

  319. 

  320. 

  321. 

  322. 

  323. 

  324. 

  325. /******************************************************************************************/

  326. 

  327. #define INIT4x4()				\

  328. 	ymm4 = _mm256_setzero_pd();		\

  329. 	ymm5 = _mm256_setzero_pd();		\

  330. 	ymm6 = _mm256_setzero_pd();		\

  331. 	ymm7 = _mm256_setzero_pd();		\

  332. 

  333. 

  334. #define KERNEL4x4_SUB() 				\

  335. 	ymm0  = _mm256_loadu_pd(AO - 16);		\

  336. 	ymm1  = _mm256_broadcastsd_pd(_mm_load_sd(BO - 12));	\

  337. 							\

  338. 	ymm4 += ymm0 * ymm1;				\

  339. 							\

  340. 	ymm1  = _mm256_broadcastsd_pd(_mm_load_sd(BO - 11));	\

  341. 	ymm5 += ymm0 * ymm1;				\

  342. 							\

  343. 	ymm1  = _mm256_broadcastsd_pd(_mm_load_sd(BO - 10));	\

  344. 	ymm6 += ymm0 * ymm1;				\

  345. 							\

  346. 	ymm1  = _mm256_broadcastsd_pd(_mm_load_sd(BO - 9));	\

  347. 	ymm7 += ymm0 * ymm1;				\

  348. 	AO += 4;					\

  349. 	BO += 4;

  350. 

  351. 

  352. #define SAVE4x4(ALPHA)					\

  353. 	ymm0 = _mm256_set1_pd(ALPHA);			\

  354. 	ymm4 *= ymm0;					\

  355. 	ymm5 *= ymm0;					\

  356. 	ymm6 *= ymm0;					\

  357. 	ymm7 *= ymm0;					\

  358. 							\

  359. 	ymm4 += _mm256_loadu_pd(CO1 + (0 * ldc));	\

  360. 	ymm5 += _mm256_loadu_pd(CO1 + (1 * ldc));	\

  361. 	ymm6 += _mm256_loadu_pd(CO1 + (2 * ldc));	\

  362. 	ymm7 += _mm256_loadu_pd(CO1 + (3 * ldc));	\

  363. 	_mm256_storeu_pd(CO1 + (0 * ldc), ymm4);	\

  364. 	_mm256_storeu_pd(CO1 + (1 * ldc), ymm5);	\

  365. 	_mm256_storeu_pd(CO1 + (2 * ldc), ymm6);	\

  366. 	_mm256_storeu_pd(CO1 + (3 * ldc), ymm7);	\

  367. 							\

  368. 	CO1 += 4;

  369. 

  370. 

  371. /******************************************************************************************/

  372. /******************************************************************************************/

  373. 

  374. #define  INIT2x4()				\

  375. 	xmm4 = _mm_setzero_pd(); 		\

  376. 	xmm5 = _mm_setzero_pd(); 		\

  377. 	xmm6 = _mm_setzero_pd(); 		\

  378. 	xmm7 = _mm_setzero_pd(); 		\

  379. 

  380. 

  381. 

  382. #define KERNEL2x4_SUB()				\

  383. 	xmm0 = _mm_loadu_pd(AO - 16);		\

  384. 	xmm1 = _mm_set1_pd(*(BO - 12));		\

  385. 	xmm2 = _mm_set1_pd(*(BO - 11));		\

  386. 	xmm3 = _mm_set1_pd(*(BO - 10));		\

  387. 	xmm4 += xmm0 * xmm1;			\

  388. 	xmm1 = _mm_set1_pd(*(BO - 9));		\

  389. 	xmm5 += xmm0 * xmm2;			\

  390. 	xmm6 += xmm0 * xmm3;			\

  391. 	xmm7 += xmm0 * xmm1;			\

  392. 	BO += 4;				\

  393. 	AO += 2;

  394. 

  395. 

  396. 

  397. #define  SAVE2x4(ALPHA)					\

  398. 	xmm0 = _mm_set1_pd(ALPHA);			\

  399. 	xmm4 *= xmm0;					\

  400. 	xmm5 *= xmm0;					\

  401. 	xmm6 *= xmm0;					\

  402. 	xmm7 *= xmm0;					\

  403. 							\

  404. 	xmm4 += _mm_loadu_pd(CO1 + (0 * ldc));	\

  405. 	xmm5 += _mm_loadu_pd(CO1 + (1 * ldc));	\

  406. 	xmm6 += _mm_loadu_pd(CO1 + (2 * ldc));	\

  407. 	xmm7 += _mm_loadu_pd(CO1 + (3 * ldc));	\

  408. 							\

  409. 	_mm_storeu_pd(CO1 + (0 * ldc), xmm4);		\

  410. 	_mm_storeu_pd(CO1 + (1 * ldc), xmm5);		\

  411. 	_mm_storeu_pd(CO1 + (2 * ldc), xmm6);		\

  412. 	_mm_storeu_pd(CO1 + (3 * ldc), xmm7);		\

  413. 							\

  414. 	CO1 += 2;

  415. 

  416. /******************************************************************************************/

  417. /******************************************************************************************/

  418. 

  419. #define  INIT1x4()		\

  420. 	dbl4 = 0; 		\

  421. 	dbl5 = 0; 		\

  422. 	dbl6 = 0; 		\

  423. 	dbl7 = 0; 		\

  424. 

  425. #define KERNEL1x4_SUB()				\

  426. 	dbl0 = *(AO - 16);			\

  427. 	dbl1 = *(BO - 12);			\

  428. 	dbl2 = *(BO - 11);			\

  429. 	dbl3 = *(BO - 10);			\

  430. 	dbl8  = *(BO - 9);			\

  431. 						\

  432. 	dbl4 += dbl0 * dbl1;			\

  433. 	dbl5 += dbl0 * dbl2;			\

  434. 	dbl6 += dbl0 * dbl3;			\

  435. 	dbl7 += dbl0 * dbl8;			\

  436. 	BO += 4;				\

  437. 	AO += 1;

  438. 

  439. 

  440. #define SAVE1x4(ALPHA)				\

  441. 	dbl0 = ALPHA;				\

  442. 	dbl4 *= dbl0;				\

  443. 	dbl5 *= dbl0;				\

  444. 	dbl6 *= dbl0;				\

  445. 	dbl7 *= dbl0;				\

  446. 						\

  447. 	dbl4 += *(CO1 + (0 * ldc));		\

  448. 	dbl5 += *(CO1 + (1 * ldc));		\

  449. 	dbl6 += *(CO1 + (2 * ldc));		\

  450. 	dbl7 += *(CO1 + (3 * ldc));		\

  451. 	*(CO1 + (0 * ldc)) = dbl4;		\

  452. 	*(CO1 + (1 * ldc)) = dbl5;		\

  453. 	*(CO1 + (2 * ldc)) = dbl6;		\

  454. 	*(CO1 + (3 * ldc)) = dbl7;		\

  455. 						\

  456. 						\

  457. 	CO1 += 1;

  458. 

  459. 

  460. /******************************************************************************************/

  461. /******************************************************************************************/

  462. 

  463. #define  INIT8x4()				\

  464. 	ymm10 = _mm256_setzero_pd(); 		\

  465. 	ymm11 = _mm256_setzero_pd(); 		\

  466. 	ymm12 = _mm256_setzero_pd(); 		\

  467. 	ymm13 = _mm256_setzero_pd(); 		\

  468. 	ymm14 = _mm256_setzero_pd(); 		\

  469. 	ymm15 = _mm256_setzero_pd(); 		\

  470. 	ymm16 = _mm256_setzero_pd(); 		\

  471. 	ymm17 = _mm256_setzero_pd(); 		\

  472. 

  473. 

  474. #define KERNEL8x4_SUB()				\

  475. 	ymm0 = _mm256_loadu_pd(AO - 16);	\

  476. 	ymm1 = _mm256_loadu_pd(AO - 12);	\

  477. 	ymm2 = _mm256_set1_pd(*(BO - 12));	\

  478. 	ymm3 = _mm256_set1_pd(*(BO - 11));	\

  479. 	ymm4 = _mm256_set1_pd(*(BO - 10));	\

  480. 	ymm5 = _mm256_set1_pd(*(BO - 9));	\

  481. 	ymm10 += ymm0 * ymm2;			\

  482. 	ymm11 += ymm1 * ymm2;			\

  483. 	ymm12 += ymm0 * ymm3;			\

  484. 	ymm13 += ymm1 * ymm3;			\

  485. 	ymm14 += ymm0 * ymm4;			\

  486. 	ymm15 += ymm1 * ymm4;			\

  487. 	ymm16 += ymm0 * ymm5;			\

  488. 	ymm17 += ymm1 * ymm5;			\

  489. 	BO += 4;				\

  490. 	AO += 8;

  491. 

  492. 

  493. 

  494. #define SAVE8x4(ALPHA)					\

  495. 	ymm0 = _mm256_set1_pd(ALPHA);			\

  496. 	ymm10 *= ymm0;					\

  497. 	ymm11 *= ymm0;					\

  498. 	ymm12 *= ymm0;					\

  499. 	ymm13 *= ymm0;					\

  500. 	ymm14 *= ymm0;					\

  501. 	ymm15 *= ymm0;					\

  502. 	ymm16 *= ymm0;					\

  503. 	ymm17 *= ymm0;					\

  504. 							\

  505. 	ymm10 += _mm256_loadu_pd(CO1);			\

  506. 	ymm11 += _mm256_loadu_pd(CO1 + 4);		\

  507. 	ymm12 += _mm256_loadu_pd(CO1 + (ldc));		\

  508. 	ymm13 += _mm256_loadu_pd(CO1 + (ldc) + 4);	\

  509. 	ymm14 += _mm256_loadu_pd(CO1 + (ldc*2));	\

  510. 	ymm15 += _mm256_loadu_pd(CO1 + (ldc*2) + 4);	\

  511. 	ymm16 += _mm256_loadu_pd(CO1 + (ldc*3));	\

  512. 	ymm17 += _mm256_loadu_pd(CO1 + (ldc*3) + 4);	\

  513. 							\

  514. 	_mm256_storeu_pd(CO1, ymm10);			\

  515. 	_mm256_storeu_pd(CO1 + 4, ymm11);		\

  516. 	_mm256_storeu_pd(CO1 + ldc, ymm12);		\

  517. 	_mm256_storeu_pd(CO1 + ldc + 4, ymm13);		\

  518. 	_mm256_storeu_pd(CO1 + ldc*2, ymm14);		\

  519. 	_mm256_storeu_pd(CO1 + ldc*2 + 4, ymm15);	\

  520. 	_mm256_storeu_pd(CO1 + ldc*3, ymm16);		\

  521. 	_mm256_storeu_pd(CO1 + ldc*3 + 4, ymm17);	\

  522. 							\

  523. 	CO1 += 8;

  524. 

  525. 

  526. /******************************************************************************************/

  527. /******************************************************************************************/

  528. #define  INIT8x2()				\

  529. 	ymm4 = _mm256_setzero_pd(); 		\

  530. 	ymm5 = _mm256_setzero_pd(); 		\

  531. 	ymm6 = _mm256_setzero_pd(); 		\

  532. 	ymm7 = _mm256_setzero_pd(); 		\

  533. 

  534. 

  535. #define KERNEL8x2_SUB()				\

  536. 	ymm0 = _mm256_loadu_pd(AO - 16);	\

  537. 	ymm1 = _mm256_loadu_pd(AO - 12);	\

  538. 	ymm2 = _mm256_set1_pd(*(BO - 12));	\

  539. 	ymm3 = _mm256_set1_pd(*(BO - 11));	\

  540. 	ymm4 += ymm0 * ymm2;			\

  541. 	ymm5 += ymm1 * ymm2;			\

  542. 	ymm6 += ymm0 * ymm3;			\

  543. 	ymm7 += ymm1 * ymm3;			\

  544. 	BO += 2;				\

  545. 	AO += 8;

  546. 

  547. 

  548. 

  549. #define SAVE8x2(ALPHA)					\

  550. 	ymm0 = _mm256_set1_pd(ALPHA);			\

  551. 	ymm4 *= ymm0;					\

  552. 	ymm5 *= ymm0;					\

  553. 	ymm6 *= ymm0;					\

  554. 	ymm7 *= ymm0;					\

  555. 							\

  556. 	ymm4 += _mm256_loadu_pd(CO1);			\

  557. 	ymm5 += _mm256_loadu_pd(CO1 + 4);		\

  558. 	ymm6 += _mm256_loadu_pd(CO1 + (ldc));		\

  559. 	ymm7 += _mm256_loadu_pd(CO1 + (ldc) + 4);	\

  560. 							\

  561. 	_mm256_storeu_pd(CO1, ymm4);			\

  562. 	_mm256_storeu_pd(CO1 + 4, ymm5);		\

  563. 	_mm256_storeu_pd(CO1 + ldc, ymm6);		\

  564. 	_mm256_storeu_pd(CO1 + ldc + 4, ymm7);		\

  565. 							\

  566. 	CO1 += 8;

  567. 

  568. 

  569. /******************************************************************************************/

  570. /******************************************************************************************/

  571. #define  INIT4x2()				\

  572. 	xmm4 = _mm_setzero_pd(); 		\

  573. 	xmm5 = _mm_setzero_pd(); 		\

  574. 	xmm6 = _mm_setzero_pd(); 		\

  575. 	xmm7 = _mm_setzero_pd(); 		\

  576. 

  577. 

  578. #define KERNEL4x2_SUB()				\

  579. 	xmm0 = _mm_loadu_pd(AO - 16);		\

  580. 	xmm1 = _mm_loadu_pd(AO - 14);		\

  581. 	xmm2 = _mm_set1_pd(*(BO - 12));		\

  582. 	xmm3 = _mm_set1_pd(*(BO - 11));		\

  583. 	xmm4 += xmm0 * xmm2;			\

  584. 	xmm5 += xmm1 * xmm2;			\

  585. 	xmm6 += xmm0 * xmm3;			\

  586. 	xmm7 += xmm1 * xmm3;			\

  587. 	BO += 2;				\

  588. 	AO += 4;

  589. 

  590. 

  591. 

  592. #define SAVE4x2(ALPHA)					\

  593. 	xmm0 = _mm_set1_pd(ALPHA);			\

  594. 	xmm4 *= xmm0;					\

  595. 	xmm5 *= xmm0;					\

  596. 	xmm6 *= xmm0;					\

  597. 	xmm7 *= xmm0;					\

  598. 							\

  599. 	xmm4 += _mm_loadu_pd(CO1);			\

  600. 	xmm5 += _mm_loadu_pd(CO1 + 2);			\

  601. 	xmm6 += _mm_loadu_pd(CO1 + (ldc));		\

  602. 	xmm7 += _mm_loadu_pd(CO1 + (ldc) + 2);		\

  603. 							\

  604. 	_mm_storeu_pd(CO1, xmm4);			\

  605. 	_mm_storeu_pd(CO1 + 2, xmm5);			\

  606. 	_mm_storeu_pd(CO1 + ldc, xmm6);			\

  607. 	_mm_storeu_pd(CO1 + ldc + 2, xmm7);		\

  608. 							\

  609. 	CO1 += 4;

  610. 

  611. 

  612. /******************************************************************************************/

  613. /******************************************************************************************/

  614. 

  615. #define  INIT2x2()				\

  616. 	xmm4 = _mm_setzero_pd(); 		\

  617. 	xmm6 = _mm_setzero_pd(); 		\

  618. 

  619. 

  620. 

  621. #define KERNEL2x2_SUB()				\

  622. 	xmm2 = _mm_set1_pd(*(BO - 12));		\

  623. 	xmm0 = _mm_loadu_pd(AO - 16);		\

  624. 	xmm3 = _mm_set1_pd(*(BO - 11));		\

  625. 	xmm4 += xmm0 * xmm2;			\

  626. 	xmm6 += xmm0 * xmm3;			\

  627. 	BO += 2;				\

  628. 	AO += 2;

  629. 

  630. 

  631. #define  SAVE2x2(ALPHA)					\

  632. 	xmm0 = _mm_set1_pd(ALPHA);			\

  633. 	xmm4 *= xmm0;					\

  634. 	xmm6 *= xmm0;					\

  635. 							\

  636. 	xmm4 += _mm_loadu_pd(CO1);			\

  637. 	xmm6 += _mm_loadu_pd(CO1 + ldc);		\

  638. 							\

  639. 	_mm_storeu_pd(CO1, xmm4);			\

  640. 	_mm_storeu_pd(CO1 + ldc, xmm6);			\

  641. 							\

  642. 	CO1 += 2;

  643. 

  644. 

  645. /******************************************************************************************/

  646. /******************************************************************************************/

  647. 

  648. #define INIT1x2()				\

  649. 	dbl4 = 0;				\

  650. 	dbl5 = 0;			

  651. 

  652. 

  653. #define KERNEL1x2_SUB()				\

  654. 	dbl0 = *(AO - 16);			\

  655. 	dbl1 = *(BO - 12);			\

  656. 	dbl2 = *(BO - 11);			\

  657. 	dbl4 += dbl0 * dbl1;			\

  658. 	dbl5 += dbl0 * dbl2;			\

  659. 	BO += 2;				\

  660. 	AO += 1;

  661. 

  662. 

  663. #define SAVE1x2(ALPHA)				\

  664. 	dbl0 = ALPHA;				\

  665. 	dbl4 *= dbl0;				\

  666. 	dbl5 *= dbl0;				\

  667. 						\

  668. 	dbl4 += *(CO1 + (0 * ldc));		\

  669. 	dbl5 += *(CO1 + (1 * ldc));		\

  670. 	*(CO1 + (0 * ldc)) = dbl4;		\

  671. 	*(CO1 + (1 * ldc)) = dbl5;		\

  672. 						\

  673. 						\

  674. 	CO1 += 1;

  675. 

  676. 

  677. 

  678. /******************************************************************************************/

  679. /******************************************************************************************/

  680. 

  681. #define INIT4x1()				\

  682. 	ymm4 = _mm256_setzero_pd();		\

  683. 	ymm5 = _mm256_setzero_pd();		\

  684. 	ymm6 = _mm256_setzero_pd();		\

  685. 	ymm7 = _mm256_setzero_pd();		

  686. 

  687. 

  688. #define KERNEL4x1()					\

  689. 	ymm0 =  _mm256_set1_pd(*(BO - 12));		\

  690. 	ymm1 =  _mm256_set1_pd(*(BO - 11));		\

  691. 	ymm2 =  _mm256_set1_pd(*(BO - 10));		\

  692. 	ymm3 =  _mm256_set1_pd(*(BO -  9));		\

  693. 							\

  694. 	ymm4 += _mm256_loadu_pd(AO - 16) * ymm0;	\

  695. 	ymm5 += _mm256_loadu_pd(AO - 12) * ymm1;	\

  696. 							\

  697. 	ymm0 =  _mm256_set1_pd(*(BO - 8));		\

  698. 	ymm1 =  _mm256_set1_pd(*(BO - 7));		\

  699. 							\

  700. 	ymm6 += _mm256_loadu_pd(AO - 8) * ymm2;		\

  701. 	ymm7 += _mm256_loadu_pd(AO - 4) * ymm3;		\

  702. 							\

  703. 	ymm2 =  _mm256_set1_pd(*(BO - 6));		\

  704. 	ymm3 =  _mm256_set1_pd(*(BO - 5));		\

  705. 							\

  706. 	ymm4 += _mm256_loadu_pd(AO + 0) * ymm0;		\

  707. 	ymm5 += _mm256_loadu_pd(AO + 4) * ymm1;		\

  708. 	ymm6 += _mm256_loadu_pd(AO + 8) * ymm2;		\

  709. 	ymm7 += _mm256_loadu_pd(AO + 12) * ymm3;	\

  710. 							\

  711. 	BO += 8;					\

  712. 	AO += 32;

  713. 

  714. 

  715. #define INIT8x1()				\

  716. 	zmm4 = _mm512_setzero_pd();		\

  717. 

  718. 

  719. #define KERNEL8x1_SUB() 					\

  720. 	zmm2 = _mm512_set1_pd(*(BO - 12));			\

  721. 	zmm0 = _mm512_loadu_pd(AO - 16);			\

  722. 	zmm4 += zmm0 * zmm2;					\

  723. 	BO += 1;						\

  724. 	AO += 8;

  725. 

  726. 

  727. #define SAVE8x1(ALPHA)						\

  728. 	zmm0 = _mm512_set1_pd(ALPHA);				\

  729. 	zmm4 *= zmm0;						\

  730. 								\

  731. 	zmm4 += _mm512_loadu_pd(CO1);				\

  732. 	_mm512_storeu_pd(CO1, zmm4);				\

  733. 	CO1 += 8;

  734. 

  735. #define KERNEL4x1_SUB() 					\

  736. 	ymm2 = _mm256_set1_pd(*(BO - 12));			\

  737. 	ymm0 = _mm256_loadu_pd(AO - 16);			\

  738. 	ymm4 += ymm0 * ymm2;					\

  739. 	BO += 1;						\

  740. 	AO += 4;

  741. 

  742. 

  743. #define SAVE4x1(ALPHA)						\

  744. 	ymm0 = _mm256_set1_pd(ALPHA);				\

  745. 	ymm4 += ymm5;						\

  746. 	ymm6 += ymm7;						\

  747. 	ymm4 += ymm6;						\

  748. 	ymm4 *= ymm0;						\

  749. 								\

  750. 	ymm4 += _mm256_loadu_pd(CO1);				\

  751. 	_mm256_storeu_pd(CO1, ymm4);				\

  752. 	CO1 += 4;

  753. 

  754. 

  755. /******************************************************************************************/

  756. /******************************************************************************************/

  757. 

  758. #define INIT2x1()					\

  759. 	xmm4 = _mm_setzero_pd(); 		

  760. 

  761. 

  762. #define KERNEL2x1_SUB()				\

  763. 	xmm2 = _mm_set1_pd(*(BO - 12));		\

  764. 	xmm0 = _mm_loadu_pd(AO - 16);		\

  765. 	xmm4 += xmm0 * xmm2;			\

  766. 	BO += 1;				\

  767. 	AO += 2;

  768. 

  769. 

  770. #define  SAVE2x1(ALPHA)					\

  771. 	xmm0 = _mm_set1_pd(ALPHA);			\

  772. 	xmm4 *= xmm0;					\

  773. 							\

  774. 	xmm4 += _mm_loadu_pd(CO1);			\

  775. 							\

  776. 	_mm_storeu_pd(CO1, xmm4);			\

  777. 							\

  778. 	CO1 += 2;

  779. 

  780. 

  781. /******************************************************************************************/

  782. /******************************************************************************************/

  783. 

  784. #define INIT1x1()	\

  785. 	dbl4 = 0;

  786. 

  787. #define KERNEL1x1_SUB() \

  788. 	dbl1 = *(BO - 12);	\

  789. 	dbl0 = *(AO - 16);	\

  790. 	dbl4 += dbl0 * dbl1;	\

  791. 	BO += 1;		\

  792. 	AO += 1;

  793. 

  794. #define SAVE1x1(ALPHA)	\

  795. 	dbl0 = ALPHA;	\

  796. 	dbl4 *= dbl0; 	\

  797. 	dbl4 += *CO1;	\

  798. 	*CO1 = dbl4;	\

  799. 	CO1 += 1;

  800. 

  801. 

  802. /*******************************************************************************************/

  803. 

  804. /* START */

  805. 

  806. 

  807. int __attribute__ ((noinline))

  808. CNAME(BLASLONG m, BLASLONG n, BLASLONG k, double alpha, double * __restrict__ A, double * __restrict__ B, double * __restrict__ C, BLASLONG ldc)

  809. {

  810. 	unsigned long M=m, N=n, K=k;

  811. 

  812. 	

  813. 	if (M == 0)

  814. 		return 0;

  815. 	if (N == 0)

  816. 		return 0;

  817. 	if (K == 0)

  818. 		return 0;

  819. 

  820. 	while (N >= 8) {

  821. 		double *CO1;

  822. 		double *AO;

  823. 		int i;

  824. 	

  825. 		CO1 = C;

  826. 		C += 8 * ldc;

  827. 

  828. 		AO = A + 16;

  829. 

  830. 		i = m;

  831. 

  832. 		while (i >= 24) {

  833. 			double *BO;

  834. 			double *A1, *A2;

  835. 			int kloop = K;

  836. 

  837. 			BO = B + 12;

  838. 			A1 = AO + 8 * K;

  839. 			A2 = AO + 16 * K;

  840. 			/*

  841. 			 *  This is the inner loop for the hot hot path

  842. 			 *  Written in inline asm because compilers like GCC 8 and earlier

  843. 			 *  struggle with register allocation and are not good at using

  844. 			 *  the AVX512 built in broadcast ability (1to8)

  845. 			 */

  846. 			asm(

  847. 			"vxorpd  %%zmm1, %%zmm1, %%zmm1\n"

  848. 			"vmovapd %%zmm1, %%zmm2\n"

  849. 			"vmovapd %%zmm1, %%zmm3\n"

  850. 			"vmovapd %%zmm1, %%zmm4\n"

  851. 			"vmovapd %%zmm1, %%zmm5\n"

  852. 			"vmovapd %%zmm1, %%zmm6\n"

  853. 			"vmovapd %%zmm1, %%zmm7\n"

  854. 			"vmovapd %%zmm1, %%zmm8\n"

  855. 			"vmovapd %%zmm1, %%zmm11\n"

  856. 			"vmovapd %%zmm1, %%zmm12\n"

  857. 			"vmovapd %%zmm1, %%zmm13\n"

  858. 			"vmovapd %%zmm1, %%zmm14\n"

  859. 			"vmovapd %%zmm1, %%zmm15\n"

  860. 			"vmovapd %%zmm1, %%zmm16\n"

  861. 			"vmovapd %%zmm1, %%zmm17\n"

  862. 			"vmovapd %%zmm1, %%zmm18\n"

  863. 			"vmovapd %%zmm1, %%zmm21\n"

  864. 			"vmovapd %%zmm1, %%zmm22\n"

  865. 			"vmovapd %%zmm1, %%zmm23\n"

  866. 			"vmovapd %%zmm1, %%zmm24\n"

  867. 			"vmovapd %%zmm1, %%zmm25\n"

  868. 			"vmovapd %%zmm1, %%zmm26\n"

  869. 			"vmovapd %%zmm1, %%zmm27\n"

  870. 			"vmovapd %%zmm1, %%zmm28\n"

  871. 			"jmp .label24\n"

  872. 			".p2align 5\n"

  873. 			/* Inner math loop */

  874. 			".label24:\n"

  875. 			"vmovupd     -128(%[AO]),%%zmm0\n"

  876. 			"vmovupd     -128(%[A1]),%%zmm10\n"

  877. 			"vmovupd     -128(%[A2]),%%zmm20\n"

  878. 

  879. 			"vbroadcastsd       -96(%[BO]),  %%zmm9\n"

  880. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm1\n"

  881. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm11\n"

  882. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm21\n"

  883. 

  884. 			"vbroadcastsd       -88(%[BO]),  %%zmm9\n"

  885. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm2\n"

  886. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm12\n"

  887. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm22\n"

  888. 

  889. 			"vbroadcastsd       -80(%[BO]),  %%zmm9\n"

  890. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm3\n"

  891. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm13\n"

  892. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm23\n"

  893. 

  894. 			"vbroadcastsd       -72(%[BO]),  %%zmm9\n"

  895. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm4\n"

  896. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm14\n"

  897. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm24\n"

  898. 

  899. 			"vbroadcastsd       -64(%[BO]),  %%zmm9\n"

  900. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm5\n"

  901. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm15\n"

  902. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm25\n"

  903. 

  904. 			"vbroadcastsd       -56(%[BO]),  %%zmm9\n"

  905. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm6\n"

  906. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm16\n"

  907. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm26\n"

  908. 

  909. 			"vbroadcastsd       -48(%[BO]),  %%zmm9\n"

  910. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm7\n"

  911. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm17\n"

  912. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm27\n"

  913. 

  914. 			"vbroadcastsd       -40(%[BO]),  %%zmm9\n"

  915. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm8\n"

  916. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm18\n"

  917. 			"vfmadd231pd    %%zmm9, %%zmm20, %%zmm28\n"

  918. 			"add $64, %[AO]\n"

  919. 			"add $64, %[A1]\n"

  920. 			"add $64, %[A2]\n"

  921. 			"add $64, %[BO]\n"

  922. 			"prefetch 512(%[AO])\n"

  923. 			"prefetch 512(%[A1])\n"

  924. 			"prefetch 512(%[A2])\n"

  925. 			"prefetch 512(%[BO])\n"

  926. 			"subl $1, %[kloop]\n"

  927. 			"jg .label24\n"

  928. 			/* multiply the result by alpha */

  929. 			"vbroadcastsd (%[alpha]), %%zmm9\n"

  930. 			/* And store additively in C */

  931. 			"vfmadd213pd (%[C0]), %%zmm9, %%zmm1\n"

  932. 			"vfmadd213pd (%[C1]), %%zmm9, %%zmm2\n"

  933. 			"vfmadd213pd (%[C2]), %%zmm9, %%zmm3\n"

  934. 			"vfmadd213pd (%[C3]), %%zmm9, %%zmm4\n"

  935. 			"vfmadd213pd (%[C4]), %%zmm9, %%zmm5\n"

  936. 			"vfmadd213pd (%[C5]), %%zmm9, %%zmm6\n"

  937. 			"vfmadd213pd (%[C6]), %%zmm9, %%zmm7\n"

  938. 			"vfmadd213pd (%[C7]), %%zmm9, %%zmm8\n"

  939. 			"vmovupd %%zmm1, (%[C0])\n"

  940. 			"vmovupd %%zmm2, (%[C1])\n"

  941. 			"vmovupd %%zmm3, (%[C2])\n"

  942. 			"vmovupd %%zmm4, (%[C3])\n"

  943. 			"vmovupd %%zmm5, (%[C4])\n"

  944. 			"vmovupd %%zmm6, (%[C5])\n"

  945. 			"vmovupd %%zmm7, (%[C6])\n"

  946. 			"vmovupd %%zmm8, (%[C7])\n"

  947. 

  948. 			"vfmadd213pd 64(%[C0]), %%zmm9, %%zmm11\n"

  949. 			"vfmadd213pd 64(%[C1]), %%zmm9, %%zmm12\n"

  950. 			"vfmadd213pd 64(%[C2]), %%zmm9, %%zmm13\n"

  951. 			"vfmadd213pd 64(%[C3]), %%zmm9, %%zmm14\n"

  952. 			"vfmadd213pd 64(%[C4]), %%zmm9, %%zmm15\n"

  953. 			"vfmadd213pd 64(%[C5]), %%zmm9, %%zmm16\n"

  954. 			"vfmadd213pd 64(%[C6]), %%zmm9, %%zmm17\n"

  955. 			"vfmadd213pd 64(%[C7]), %%zmm9, %%zmm18\n"

  956. 			"vmovupd %%zmm11, 64(%[C0])\n"

  957. 			"vmovupd %%zmm12, 64(%[C1])\n"

  958. 			"vmovupd %%zmm13, 64(%[C2])\n"

  959. 			"vmovupd %%zmm14, 64(%[C3])\n"

  960. 			"vmovupd %%zmm15, 64(%[C4])\n"

  961. 			"vmovupd %%zmm16, 64(%[C5])\n"

  962. 			"vmovupd %%zmm17, 64(%[C6])\n"

  963. 			"vmovupd %%zmm18, 64(%[C7])\n"

  964. 

  965. 			"vfmadd213pd 128(%[C0]), %%zmm9, %%zmm21\n"

  966. 			"vfmadd213pd 128(%[C1]), %%zmm9, %%zmm22\n"

  967. 			"vfmadd213pd 128(%[C2]), %%zmm9, %%zmm23\n"

  968. 			"vfmadd213pd 128(%[C3]), %%zmm9, %%zmm24\n"

  969. 			"vfmadd213pd 128(%[C4]), %%zmm9, %%zmm25\n"

  970. 			"vfmadd213pd 128(%[C5]), %%zmm9, %%zmm26\n"

  971. 			"vfmadd213pd 128(%[C6]), %%zmm9, %%zmm27\n"

  972. 			"vfmadd213pd 128(%[C7]), %%zmm9, %%zmm28\n"

  973. 			"vmovupd %%zmm21, 128(%[C0])\n"

  974. 			"vmovupd %%zmm22, 128(%[C1])\n"

  975. 			"vmovupd %%zmm23, 128(%[C2])\n"

  976. 			"vmovupd %%zmm24, 128(%[C3])\n"

  977. 			"vmovupd %%zmm25, 128(%[C4])\n"

  978. 			"vmovupd %%zmm26, 128(%[C5])\n"

  979. 			"vmovupd %%zmm27, 128(%[C6])\n"

  980. 			"vmovupd %%zmm28, 128(%[C7])\n"

  981. 

  982. 			   :

  983. 				[AO]	"+r" (AO),

  984. 				[A1]	"+r" (A1),

  985. 				[A2]	"+r" (A2),

  986. 				[BO]	"+r" (BO),

  987. 				[C0]	"+r" (CO1),

  988. 				[kloop]	"+r" (kloop)

  989. 			   :

  990. 				[alpha] 	"r" (&alpha),

  991. 				[C1] 	"r" (CO1 + 1 * ldc),

  992. 				[C2] 	"r" (CO1 + 2 * ldc),

  993. 				[C3] 	"r" (CO1 + 3 * ldc),

  994. 				[C4] 	"r" (CO1 + 4 * ldc),

  995. 				[C5] 	"r" (CO1 + 5 * ldc),

  996. 				[C6] 	"r" (CO1 + 6 * ldc),

  997. 				[C7] 	"r" (CO1 + 7 * ldc)

  998. 

  999. 			     :  "memory", "zmm0",  "zmm1",  "zmm2",  "zmm3",  "zmm4",  "zmm5",  "zmm6",  "zmm7",  "zmm8", "zmm9",

  1000. 					  "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", "zmm16", "zmm17", "zmm18",

  1001. 					  "zmm20", "zmm21", "zmm22", "zmm23", "zmm24", "zmm25", "zmm26", "zmm27", "zmm28"

  1002. 			);

  1003. 			CO1 += 24;

  1004. 			AO += 16 * K;

  1005. 			i-= 24;

  1006. 		}

  1007. 

  1008. 

  1009. 		while (i >= 16) {

  1010. 			double *BO;

  1011. 			double *A1;

  1012. 			int kloop = K;

  1013. 

  1014. 			BO = B + 12;

  1015. 			A1 = AO + 8 * K;

  1016. 			/*

  1017. 			 *  This is the inner loop for the hot hot path 

  1018. 			 *  Written in inline asm because compilers like GCC 8 and earlier

  1019. 			 *  struggle with register allocation and are not good at using

  1020. 		 	 *  the AVX512 built in broadcast ability (1to8)

  1021. 			 */

  1022. 			asm(

  1023. 			"vxorpd  %%zmm1, %%zmm1, %%zmm1\n"

  1024. 			"vmovapd %%zmm1, %%zmm2\n"

  1025. 			"vmovapd %%zmm1, %%zmm3\n"

  1026. 			"vmovapd %%zmm1, %%zmm4\n"

  1027. 			"vmovapd %%zmm1, %%zmm5\n"

  1028. 			"vmovapd %%zmm1, %%zmm6\n"

  1029. 			"vmovapd %%zmm1, %%zmm7\n"

  1030. 			"vmovapd %%zmm1, %%zmm8\n"

  1031. 			"vmovapd %%zmm1, %%zmm11\n"

  1032. 			"vmovapd %%zmm1, %%zmm12\n"

  1033. 			"vmovapd %%zmm1, %%zmm13\n"

  1034. 			"vmovapd %%zmm1, %%zmm14\n"

  1035. 			"vmovapd %%zmm1, %%zmm15\n"

  1036. 			"vmovapd %%zmm1, %%zmm16\n"

  1037. 			"vmovapd %%zmm1, %%zmm17\n"

  1038. 			"vmovapd %%zmm1, %%zmm18\n"

  1039. 			"jmp .label16\n"

  1040. 			".p2align 5\n"

  1041. 			/* Inner math loop */

  1042. 			".label16:\n"

  1043. 			"vmovupd     -128(%[AO]),%%zmm0\n"

  1044. 			"vmovupd     -128(%[A1]),%%zmm10\n"

  1045. 

  1046. 			"vbroadcastsd       -96(%[BO]),  %%zmm9\n"

  1047. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm1\n"

  1048. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm11\n"

  1049. 

  1050. 			"vbroadcastsd       -88(%[BO]),  %%zmm9\n"

  1051. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm2\n"

  1052. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm12\n"

  1053. 

  1054. 			"vbroadcastsd       -80(%[BO]),  %%zmm9\n"

  1055. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm3\n"

  1056. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm13\n"

  1057. 

  1058. 			"vbroadcastsd       -72(%[BO]),  %%zmm9\n"

  1059. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm4\n"

  1060. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm14\n"

  1061. 

  1062. 			"vbroadcastsd       -64(%[BO]),  %%zmm9\n"

  1063. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm5\n"

  1064. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm15\n"

  1065. 

  1066. 			"vbroadcastsd       -56(%[BO]),  %%zmm9\n"

  1067. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm6\n"

  1068. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm16\n"

  1069. 

  1070. 			"vbroadcastsd       -48(%[BO]),  %%zmm9\n"

  1071. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm7\n"

  1072. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm17\n"

  1073. 

  1074. 			"vbroadcastsd       -40(%[BO]),  %%zmm9\n"

  1075. 			"vfmadd231pd    %%zmm9, %%zmm0,  %%zmm8\n"

  1076. 			"vfmadd231pd    %%zmm9, %%zmm10, %%zmm18\n"

  1077. 			"add $64, %[AO]\n"

  1078. 			"add $64, %[A1]\n"

  1079. 			"add $64, %[BO]\n"

  1080. 			"prefetch 512(%[AO])\n"

  1081. 			"prefetch 512(%[A1])\n"

  1082. 			"prefetch 512(%[BO])\n"

  1083. 			"subl $1, %[kloop]\n"

  1084. 			"jg .label16\n"

  1085. 			/* multiply the result by alpha */

  1086. 			"vbroadcastsd (%[alpha]), %%zmm9\n"

  1087. 			/* And store additively in C */

  1088. 			"vfmadd213pd (%[C0]), %%zmm9, %%zmm1\n"

  1089. 			"vfmadd213pd (%[C1]), %%zmm9, %%zmm2\n"

  1090. 			"vfmadd213pd (%[C2]), %%zmm9, %%zmm3\n"

  1091. 			"vfmadd213pd (%[C3]), %%zmm9, %%zmm4\n"

  1092. 			"vfmadd213pd (%[C4]), %%zmm9, %%zmm5\n"

  1093. 			"vfmadd213pd (%[C5]), %%zmm9, %%zmm6\n"

  1094. 			"vfmadd213pd (%[C6]), %%zmm9, %%zmm7\n"

  1095. 			"vfmadd213pd (%[C7]), %%zmm9, %%zmm8\n"

  1096. 			"vmovupd %%zmm1, (%[C0])\n"

  1097. 			"vmovupd %%zmm2, (%[C1])\n"

  1098. 			"vmovupd %%zmm3, (%[C2])\n"

  1099. 			"vmovupd %%zmm4, (%[C3])\n"

  1100. 			"vmovupd %%zmm5, (%[C4])\n"

  1101. 			"vmovupd %%zmm6, (%[C5])\n"

  1102. 			"vmovupd %%zmm7, (%[C6])\n"

  1103. 			"vmovupd %%zmm8, (%[C7])\n"

  1104. 

  1105. 			"vfmadd213pd 64(%[C0]), %%zmm9, %%zmm11\n"

  1106. 			"vfmadd213pd 64(%[C1]), %%zmm9, %%zmm12\n"

  1107. 			"vfmadd213pd 64(%[C2]), %%zmm9, %%zmm13\n"

  1108. 			"vfmadd213pd 64(%[C3]), %%zmm9, %%zmm14\n"

  1109. 			"vfmadd213pd 64(%[C4]), %%zmm9, %%zmm15\n"

  1110. 			"vfmadd213pd 64(%[C5]), %%zmm9, %%zmm16\n"

  1111. 			"vfmadd213pd 64(%[C6]), %%zmm9, %%zmm17\n"

  1112. 			"vfmadd213pd 64(%[C7]), %%zmm9, %%zmm18\n"

  1113. 			"vmovupd %%zmm11, 64(%[C0])\n"

  1114. 			"vmovupd %%zmm12, 64(%[C1])\n"

  1115. 			"vmovupd %%zmm13, 64(%[C2])\n"

  1116. 			"vmovupd %%zmm14, 64(%[C3])\n"

  1117. 			"vmovupd %%zmm15, 64(%[C4])\n"

  1118. 			"vmovupd %%zmm16, 64(%[C5])\n"

  1119. 			"vmovupd %%zmm17, 64(%[C6])\n"

  1120. 			"vmovupd %%zmm18, 64(%[C7])\n"

  1121. 

  1122. 			   :

  1123. 				[AO]	"+r" (AO),

  1124. 				[A1]	"+r" (A1),

  1125. 				[BO]	"+r" (BO),

  1126. 				[C0]	"+r" (CO1),

  1127. 				[kloop]	"+r" (kloop)

  1128. 			   :

  1129. 				[alpha] 	"r" (&alpha),

  1130. 				[C1] 	"r" (CO1 + 1 * ldc),

  1131. 				[C2] 	"r" (CO1 + 2 * ldc),

  1132. 				[C3] 	"r" (CO1 + 3 * ldc),

  1133. 				[C4] 	"r" (CO1 + 4 * ldc),

  1134. 				[C5] 	"r" (CO1 + 5 * ldc),

  1135. 				[C6] 	"r" (CO1 + 6 * ldc),

  1136. 				[C7] 	"r" (CO1 + 7 * ldc)

  1137. 

  1138. 			     :  "memory", "zmm0",  "zmm1",  "zmm2",  "zmm3",  "zmm4",  "zmm5",  "zmm6",  "zmm7",  "zmm8", "zmm9",

  1139. 					  "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", "zmm16", "zmm17", "zmm18"

  1140. 			);

  1141. 			CO1 += 16;

  1142. 			AO += 8 * K;

  1143. 			i-= 16;

  1144. 		}

  1145. 

  1146. 		while (i >= 8) {

  1147. 			double *BO;

  1148. 			int kloop = K;

  1149. 

  1150. 			BO = B + 12;

  1151. 			/*

  1152. 			 *  This is the inner loop for the hot hot path

  1153. 			 *  Written in inline asm because compilers like GCC 8 and earlier

  1154. 			 *  struggle with register allocation and are not good at using

  1155. 			 *  the AVX512 built in broadcast ability (1to8)

  1156. 			 */

  1157. 			asm(

  1158. 			"vxorpd  %%zmm1, %%zmm1, %%zmm1\n" 

  1159. 			"vmovapd %%zmm1, %%zmm2\n"

  1160. 			"vmovapd %%zmm1, %%zmm3\n"

  1161. 			"vmovapd %%zmm1, %%zmm4\n"

  1162. 			"vmovapd %%zmm1, %%zmm5\n"

  1163. 			"vmovapd %%zmm1, %%zmm6\n"

  1164. 			"vmovapd %%zmm1, %%zmm7\n"

  1165. 			"vmovapd %%zmm1, %%zmm8\n"

  1166. 			"vbroadcastsd (%[alpha]), %%zmm9\n"

  1167. 			"jmp .label1\n"

  1168. 			".p2align 5\n"

  1169. 			/* Inner math loop */

  1170. 			".label1:\n"

  1171. 			"vmovupd     -128(%[AO]),%%zmm0\n"

  1172. 			"vfmadd231pd  -96(%[BO])%{1to8%}, %%zmm0, %%zmm1\n"

  1173. 			"vfmadd231pd  -88(%[BO])%{1to8%}, %%zmm0, %%zmm2\n"

  1174. 			"vfmadd231pd  -80(%[BO])%{1to8%}, %%zmm0, %%zmm3\n"

  1175. 			"vfmadd231pd  -72(%[BO])%{1to8%}, %%zmm0, %%zmm4\n"

  1176. 			"vfmadd231pd  -64(%[BO])%{1to8%}, %%zmm0, %%zmm5\n"

  1177. 			"vfmadd231pd  -56(%[BO])%{1to8%}, %%zmm0, %%zmm6\n"

  1178. 			"vfmadd231pd  -48(%[BO])%{1to8%}, %%zmm0, %%zmm7\n"

  1179. 			"vfmadd231pd  -40(%[BO])%{1to8%}, %%zmm0, %%zmm8\n"

  1180. 			"add $64, %[AO]\n"

  1181. 			"add $64, %[BO]\n"

  1182. 			"subl $1, %[kloop]\n"

  1183. 			"jg .label1\n"

  1184. 			/* multiply the result by alpha and add to the memory */

  1185. 			"vfmadd213pd (%[C0]), %%zmm9, %%zmm1\n"

  1186. 			"vfmadd213pd (%[C1]), %%zmm9, %%zmm2\n"

  1187. 			"vfmadd213pd (%[C2]), %%zmm9, %%zmm3\n"

  1188. 			"vfmadd213pd (%[C3]), %%zmm9, %%zmm4\n"

  1189. 			"vfmadd213pd (%[C4]), %%zmm9, %%zmm5\n"

  1190. 			"vfmadd213pd (%[C5]), %%zmm9, %%zmm6\n"

  1191. 			"vfmadd213pd (%[C6]), %%zmm9, %%zmm7\n"

  1192. 			"vfmadd213pd (%[C7]), %%zmm9, %%zmm8\n"

  1193. 			"vmovupd %%zmm1, (%[C0])\n"

  1194. 			"vmovupd %%zmm2, (%[C1])\n"

  1195. 			"vmovupd %%zmm3, (%[C2])\n"

  1196. 			"vmovupd %%zmm4, (%[C3])\n"

  1197. 			"vmovupd %%zmm5, (%[C4])\n"

  1198. 			"vmovupd %%zmm6, (%[C5])\n"

  1199. 			"vmovupd %%zmm7, (%[C6])\n"

  1200. 			"vmovupd %%zmm8, (%[C7])\n"

  1201. 			   : 

  1202.   			     [AO]	"+r" (AO),

  1203. 			     [BO]	"+r" (BO),

  1204. 			     [C0]	"+r" (CO1),

  1205. 		             [kloop]	"+r" (kloop)

  1206. 			   :

  1207. 			     [alpha] 	"r" (&alpha),

  1208. 			     [C1] 	"r" (CO1 + 1 * ldc),

  1209. 			     [C2] 	"r" (CO1 + 2 * ldc),

  1210. 			     [C3] 	"r" (CO1 + 3 * ldc),

  1211. 			     [C4] 	"r" (CO1 + 4 * ldc),

  1212. 			     [C5] 	"r" (CO1 + 5 * ldc),

  1213. 			     [C6] 	"r" (CO1 + 6 * ldc),

  1214. 			     [C7] 	"r" (CO1 + 7 * ldc)

  1215. 

  1216. 			     :  "memory", "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", "zmm6", "zmm7", "zmm8", "zmm9"

  1217. 			);

  1218. 			CO1 += 8;

  1219. 			i-= 8;

  1220. 		}

  1221. 

  1222. 

  1223. 

  1224. 		while (i >= 4) {

  1225. 			double *BO;

  1226. 			__m256d ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, ymm8, ymm9, ymm10, ymm11;

  1227. 			int kloop = K;

  1228. 

  1229. 			BO = B + 12;

  1230. 			INIT4x8()

  1231. 

  1232. 			while (kloop > 0) {

  1233. 				KERNEL4x8_SUB()

  1234. 				kloop--;

  1235. 			}				

  1236. 			SAVE4x8(alpha)

  1237. 			i-= 4;

  1238. 		}

  1239. 

  1240. 

  1241. 		while (i >= 2) {

  1242. 			double *BO;

  1243. 			__m128d xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11;

  1244. 			int kloop = K;

  1245. 

  1246. 			BO = B + 12;

  1247. 			INIT2x8()

  1248. 				

  1249. 			while (kloop > 0) {

  1250. 				KERNEL2x8_SUB()

  1251. 				kloop--;

  1252. 			}

  1253. 			SAVE2x8(alpha)

  1254. 			i -= 2;

  1255. 		}

  1256. 

  1257. 		while (i >= 1) {

  1258. 			double *BO;

  1259. 			double dbl0, dbl1, dbl2, dbl3, dbl4, dbl5, dbl6, dbl7, dbl8, dbl9, dbl10, dbl11;

  1260. 			int kloop = K;

  1261. 

  1262. 			BO = B + 12;

  1263. 			INIT1x8()

  1264. 										

  1265. 			while (kloop > 0) {

  1266. 				KERNEL1x8_SUB()

  1267. 				kloop--;

  1268. 			}

  1269. 			SAVE1x8(alpha)

  1270. 			i -= 1;

  1271. 		}

  1272. 		B += K * 8;

  1273. 		N -= 8;

  1274. 	}

  1275. 

  1276. 	if (N == 0)

  1277. 		return 0;	

  1278. 	

  1279. 

  1280. 

  1281. 	// L8_0

  1282. 	while (N >= 4) {

  1283. 		double *CO1;

  1284. 		double *AO;

  1285. 		int i;

  1286. 		// L8_10

  1287. 		CO1 = C;

  1288. 		C += 4 * ldc;

  1289. 

  1290. 		AO = A + 16;

  1291. 

  1292. 		i = m;

  1293. 		while (i >= 8) {

  1294. 			double *BO;

  1295. 			// L8_11

  1296. 			__m256d ymm0, ymm1, ymm2, ymm3, ymm4, ymm5,  ymm10, ymm11,ymm12,ymm13,ymm14,ymm15,ymm16,ymm17;

  1297. 			BO = B + 12;

  1298. 			int kloop = K;

  1299. 	

  1300. 			INIT8x4()

  1301. 

  1302. 			while (kloop > 0) {

  1303. 				// L12_17

  1304. 				KERNEL8x4_SUB()

  1305. 				kloop--;

  1306. 			}

  1307. 			// L8_19

  1308. 			SAVE8x4(alpha)

  1309. 	

  1310. 			i -= 8;

  1311. 		}

  1312. 		while (i >= 4) {

  1313. 			// L8_11

  1314. 			double *BO;

  1315. 			__m256d ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7;

  1316. 			BO = B + 12;

  1317. 			int kloop = K;

  1318. 

  1319. 			INIT4x4()

  1320. 			// L8_16

  1321. 			while (kloop > 0) {

  1322. 				// L12_17

  1323. 				KERNEL4x4_SUB()

  1324. 				kloop--;

  1325. 			}

  1326. 			// L8_19

  1327. 			SAVE4x4(alpha)

  1328. 

  1329. 			i -= 4;

  1330. 		}

  1331. 

  1332. /**************************************************************************

  1333. * Rest of M 

  1334. ***************************************************************************/

  1335. 

  1336. 		while (i >= 2) {

  1337. 			double *BO;

  1338. 			__m128d xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7;

  1339. 			BO = B;

  1340. 			BO += 12;

  1341. 

  1342. 			INIT2x4()

  1343. 			int kloop = K;

  1344. 			

  1345. 			while (kloop > 0) {

  1346. 				KERNEL2x4_SUB()

  1347. 				kloop--;

  1348. 			}

  1349. 			SAVE2x4(alpha)

  1350. 			i -= 2;

  1351. 		}

  1352. 			// L13_40

  1353. 		while (i >= 1) {

  1354. 			double *BO;

  1355. 			double dbl0, dbl1, dbl2, dbl3, dbl4, dbl5, dbl6, dbl7, dbl8;

  1356. 			int kloop = K;

  1357. 			BO = B + 12;

  1358. 			INIT1x4()

  1359. 				

  1360. 			while (kloop > 0) {

  1361. 				KERNEL1x4_SUB()

  1362. 				kloop--;

  1363. 			}

  1364. 			SAVE1x4(alpha)

  1365. 			i -= 1;

  1366. 		}

  1367. 			

  1368. 		B += K * 4;

  1369. 		N -= 4;

  1370. 	}

  1371. 

  1372. /**************************************************************************************************/

  1373. 

  1374. 		// L8_0

  1375. 	while (N >= 2) {

  1376. 		double *CO1;

  1377. 		double *AO;

  1378. 		int i;

  1379. 		// L8_10

  1380. 		CO1 = C;

  1381. 		C += 2 * ldc;

  1382. 

  1383. 		AO = A + 16;

  1384. 

  1385. 		i = m;

  1386. 		while (i >= 8) {

  1387. 			double *BO;

  1388. 			__m256d ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7;

  1389. 			// L8_11

  1390. 			BO = B + 12;

  1391. 			int kloop = K;

  1392. 

  1393. 			INIT8x2()

  1394. 

  1395. 			// L8_16

  1396. 			while (kloop > 0) {

  1397. 				// L12_17

  1398. 				KERNEL8x2_SUB()

  1399. 				kloop--;

  1400. 			}

  1401. 			// L8_19

  1402. 			SAVE8x2(alpha)

  1403. 

  1404. 			i-=8;

  1405. 		}

  1406. 

  1407. 		while (i >= 4) {

  1408. 			double *BO;

  1409. 			__m128d xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7;

  1410. 			// L8_11

  1411. 			BO = B + 12;

  1412. 			int kloop = K;

  1413. 	

  1414. 			INIT4x2()

  1415. 

  1416. 			// L8_16

  1417. 			while (kloop > 0) {

  1418. 				// L12_17

  1419. 				KERNEL4x2_SUB()

  1420. 				kloop--;

  1421. 			}

  1422. 			// L8_19

  1423. 			SAVE4x2(alpha)

  1424. 	

  1425. 			i-=4;

  1426. 		}

  1427. 

  1428. /**************************************************************************

  1429. * Rest of M 

  1430. ***************************************************************************/

  1431. 

  1432. 		while (i >= 2) {

  1433. 			double *BO;

  1434. 			__m128d xmm0, xmm2, xmm3, xmm4, xmm6;

  1435. 			int kloop = K;

  1436. 			BO = B + 12;

  1437. 

  1438. 			INIT2x2()

  1439. 				

  1440. 			while (kloop > 0) {

  1441. 				KERNEL2x2_SUB()

  1442. 				kloop--;

  1443. 			}

  1444. 			SAVE2x2(alpha)

  1445. 			i -= 2;

  1446. 		}

  1447. 			// L13_40

  1448. 		while (i >= 1) {

  1449. 			double *BO;

  1450. 			double dbl0, dbl1, dbl2, dbl4, dbl5;

  1451. 			int kloop = K;

  1452. 			BO = B + 12;

  1453. 

  1454. 			INIT1x2()

  1455. 					

  1456. 			while (kloop > 0) {

  1457. 				KERNEL1x2_SUB()

  1458. 				kloop--;

  1459. 			}

  1460. 			SAVE1x2(alpha)

  1461. 			i -= 1;

  1462. 		}

  1463. 			

  1464. 		B += K * 2;

  1465. 		N -= 2;

  1466. 	}

  1467. 

  1468. 		// L8_0

  1469. 	while (N >= 1) {

  1470. 		// L8_10

  1471. 		double *CO1;

  1472. 		double *AO;

  1473. 		int i;

  1474. 

  1475. 		CO1 = C;

  1476. 		C += ldc;

  1477. 

  1478. 		AO = A + 16;

  1479. 

  1480. 		i = m;

  1481. 		while (i >= 8) {

  1482. 			double *BO;

  1483. 			__m512d zmm0, zmm2, zmm4;

  1484. 			// L8_11

  1485. 			BO = B + 12;

  1486. 			int kloop = K;

  1487. 

  1488. 			INIT8x1()

  1489. 			// L8_16

  1490. 			while (kloop > 0) {

  1491. 				// L12_17

  1492. 				KERNEL8x1_SUB()

  1493. 				kloop--;

  1494. 			}

  1495. 			// L8_19

  1496. 			SAVE8x1(alpha)

  1497. 

  1498. 			i-= 8;

  1499. 		}

  1500. 		while (i >= 4) {

  1501. 			double *BO;

  1502. 			__m256d ymm0, ymm2, ymm4, ymm5, ymm6, ymm7;

  1503. 			// L8_11

  1504. 			BO = B + 12;

  1505. 			int kloop = K;

  1506. 

  1507. 			INIT4x1()

  1508. 			// L8_16

  1509. 			while (kloop > 0) {

  1510. 				// L12_17

  1511. 				KERNEL4x1_SUB()

  1512. 				kloop--;

  1513. 			}

  1514. 			// L8_19

  1515. 			SAVE4x1(alpha)

  1516. 

  1517. 			i-= 4;

  1518. 		}

  1519. 

  1520. /**************************************************************************

  1521. * Rest of M 

  1522. ***************************************************************************/

  1523. 

  1524. 		while (i >= 2) {

  1525. 			double *BO;

  1526. 			__m128d xmm0, xmm2, xmm4;

  1527. 			int kloop = K;

  1528. 			BO = B;

  1529. 			BO += 12;

  1530. 

  1531. 			INIT2x1()

  1532. 				

  1533. 			while (kloop > 0) {

  1534. 				KERNEL2x1_SUB()

  1535. 				kloop--;

  1536. 			}

  1537. 			SAVE2x1(alpha)

  1538. 			i -= 2;

  1539. 		}

  1540. 				// L13_40

  1541. 		while (i >= 1) {

  1542. 			double *BO;

  1543. 			double dbl0, dbl1, dbl4;

  1544. 			int kloop = K;

  1545. 

  1546. 			BO = B;

  1547. 			BO += 12;

  1548. 			INIT1x1()

  1549. 				

  1550. 

  1551. 			while (kloop > 0) {

  1552. 				KERNEL1x1_SUB()

  1553. 				kloop--;

  1554. 			}

  1555. 			SAVE1x1(alpha)

  1556. 			i -= 1;

  1557. 		}

  1558. 			

  1559. 		B += K * 1;

  1560. 		N -= 1;

  1561. 	}

  1562. 

  1563. 

  1564. 	return 0;

  1565. }