OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/sgemm_direct_skylakex.c

479 lines
14 kB
Raw Blame History 

  1. /* the direct sgemm code written by Arjan van der Ven */

  2. #include "common.h"

  3. 

  4. #if defined(SKYLAKEX) || defined (COOPERLAKE) || defined (SAPPHIRERAPIDS)

  5. 

  6. #include <immintrin.h>

  7. 

  8. 

  9. /*

  10.  * "Direct sgemm" code. This code operates directly on the inputs and outputs

  11.  * of the sgemm call, avoiding the copies, memory realignments and threading,

  12.  * and only supports alpha = 1 and beta = 0.

  13.  * This is a common case and provides value for relatively small matrixes.

  14.  * For larger matrixes the "regular" sgemm code is superior, there the cost of

  15.  * copying/shuffling the B matrix really pays off.

  16.  */

  17. 

  18. 

  19. 

  20. #define DECLARE_RESULT_512(N,M) __m512 result##N##M = _mm512_setzero_ps()

  21. #define BROADCAST_LOAD_A_512(N,M) __m512 Aval##M = _mm512_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  22. #define LOAD_B_512(N,M)  __m512 Bval##N = _mm512_loadu_ps(&B[strideB * k + j + (N*16)])

  23. #define MATMUL_512(N,M)  result##N##M = _mm512_fmadd_ps(Aval##M, Bval##N , result##N##M)

  24. #define STORE_512(N,M) _mm512_storeu_ps(&R[(i+M) * strideR + j+(N*16)], result##N##M)

  25. 

  26. 

  27. #define DECLARE_RESULT_256(N,M) __m256 result##N##M = _mm256_setzero_ps()

  28. #define BROADCAST_LOAD_A_256(N,M) __m256 Aval##M = _mm256_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  29. #define LOAD_B_256(N,M)  __m256 Bval##N = _mm256_loadu_ps(&B[strideB * k + j + (N*8)])

  30. #define MATMUL_256(N,M)  result##N##M = _mm256_fmadd_ps(Aval##M, Bval##N , result##N##M)

  31. #define STORE_256(N,M) _mm256_storeu_ps(&R[(i+M) * strideR + j+(N*8)], result##N##M)

  32. 

  33. #define DECLARE_RESULT_128(N,M) __m128 result##N##M = _mm_setzero_ps()

  34. #define BROADCAST_LOAD_A_128(N,M) __m128 Aval##M = _mm_broadcastss_ps(_mm_load_ss(&A[k  + strideA * (i+M)]))

  35. #define LOAD_B_128(N,M)  __m128 Bval##N = _mm_loadu_ps(&B[strideB * k + j + (N*4)])

  36. #define MATMUL_128(N,M)  result##N##M = _mm_fmadd_ps(Aval##M, Bval##N , result##N##M)

  37. #define STORE_128(N,M) _mm_storeu_ps(&R[(i+M) * strideR + j+(N*4)], result##N##M)

  38. 

  39. #define DECLARE_RESULT_SCALAR(N,M) float result##N##M = 0;

  40. #define BROADCAST_LOAD_A_SCALAR(N,M) float Aval##M = A[k + strideA * (i + M)];

  41. #define LOAD_B_SCALAR(N,M)  float Bval##N  = B[k * strideB + j + N];

  42. #define MATMUL_SCALAR(N,M) result##N##M +=  Aval##M * Bval##N;

  43. #define STORE_SCALAR(N,M)  R[(i+M) * strideR + j + N] = result##N##M;

  44. 

  45. #if 0

  46. int sgemm_kernel_direct_performant(BLASLONG M, BLASLONG N, BLASLONG K)

  47. {

  48. 	unsigned long long mnk = M * N * K;

  49. 	/* large matrixes -> not performant */

  50. 	if (mnk >= 28 * 512 * 512)

  51. 		return 0;

  52. 

  53. 	/*

  54. 	 * if the B matrix is not a nice multiple if 4 we get many unaligned accesses,

  55. 	 * and the regular sgemm copy/realignment of data pays off much quicker

  56. 	 */

  57. 	if ((N & 3) != 0 && (mnk >= 8 * 512 * 512))

  58. 		return 0;

  59. 

  60. #ifdef SMP

  61. 	/* if we can run multithreaded, the threading changes the based threshold */

  62. 	if (mnk > 2 * 350 * 512 && num_cpu_avail(3)> 1)

  63. 		return 0;

  64. #endif

  65. 

  66. 	return 1;

  67. }

  68. 

  69. #endif

  70. 

  71. //void sgemm_kernel_direct (BLASLONG M, BLASLONG N, BLASLONG K, float * __restrict A, BLASLONG strideA, float * __restrict B, BLASLONG strideB , float * __restrict R, BLASLONG strideR)

  72. void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float * __restrict A, BLASLONG strideA, float * __restrict B, BLASLONG strideB , float * __restrict R, BLASLONG strideR)

  73. {

  74. 	int i, j, k;

  75. 

  76.         int m4 = M & ~3;

  77. 	int m2 = M & ~1;

  78. 

  79. 	int n64 = N & ~63;

  80. 	int n32 = N & ~31;

  81. 	int n16 = N & ~15;

  82. 	int n8 = N & ~7;

  83. 	int n4 = N & ~3;

  84. 	int n2 = N & ~1;

  85. 

  86. 	i = 0;

  87. 

  88. 	for (i = 0; i < m4; i+=4) {

  89. 

  90. 		for (j = 0; j < n64; j+= 64) {

  91. 			k = 0;

  92. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  93. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);    			DECLARE_RESULT_512(2, 1);    DECLARE_RESULT_512(3, 1);

  94. 			DECLARE_RESULT_512(0, 2);    DECLARE_RESULT_512(1, 2);    			DECLARE_RESULT_512(2, 2);    DECLARE_RESULT_512(3, 2);

  95. 			DECLARE_RESULT_512(0, 3);    DECLARE_RESULT_512(1, 3);    			DECLARE_RESULT_512(2, 3);    DECLARE_RESULT_512(3, 3);

  96. 

  97. 

  98. 			for (k = 0; k < K; k++) {

  99. 				BROADCAST_LOAD_A_512(x, 0);

  100. 				BROADCAST_LOAD_A_512(x, 1);

  101. 				BROADCAST_LOAD_A_512(x, 2);

  102. 				BROADCAST_LOAD_A_512(x, 3);

  103. 

  104. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  105. 

  106. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  107. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);			MATMUL_512(2, 1);		MATMUL_512(3, 1);

  108. 				MATMUL_512(0, 2);		MATMUL_512(1, 2);			MATMUL_512(2, 2);		MATMUL_512(3, 2);

  109. 				MATMUL_512(0, 3);		MATMUL_512(1, 3);			MATMUL_512(2, 3);		MATMUL_512(3, 3);

  110. 			}

  111. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  112. 			STORE_512(0, 1);		STORE_512(1, 1);			STORE_512(2, 1);		STORE_512(3, 1);

  113. 			STORE_512(0, 2);		STORE_512(1, 2);			STORE_512(2, 2);		STORE_512(3, 2);

  114. 			STORE_512(0, 3);		STORE_512(1, 3);			STORE_512(2, 3);		STORE_512(3, 3);

  115. 		}

  116. 

  117. 		for (; j < n32; j+= 32) {

  118. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  119. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);

  120. 			DECLARE_RESULT_512(0, 2);    DECLARE_RESULT_512(1, 2);

  121. 			DECLARE_RESULT_512(0, 3);    DECLARE_RESULT_512(1, 3);

  122. 

  123. 			for (k = 0; k < K; k++) {

  124. 				BROADCAST_LOAD_A_512(x, 0);

  125. 				BROADCAST_LOAD_A_512(x, 1);

  126. 				BROADCAST_LOAD_A_512(x, 2);

  127. 				BROADCAST_LOAD_A_512(x, 3);

  128. 

  129. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  130. 

  131. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  132. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);

  133. 				MATMUL_512(0, 2);		MATMUL_512(1, 2);

  134. 				MATMUL_512(0, 3);		MATMUL_512(1, 3);

  135. 			}

  136. 			STORE_512(0, 0);		STORE_512(1, 0);

  137. 			STORE_512(0, 1);		STORE_512(1, 1);

  138. 			STORE_512(0, 2);		STORE_512(1, 2);

  139. 			STORE_512(0, 3);		STORE_512(1, 3);

  140. 		}

  141. 

  142. 		for (; j < n16; j+= 16) {

  143. 			DECLARE_RESULT_512(0, 0);

  144. 			DECLARE_RESULT_512(0, 1);

  145. 			DECLARE_RESULT_512(0, 2);

  146. 			DECLARE_RESULT_512(0, 3);

  147. 

  148. 		 	for (k = 0; k < K; k++) {

  149. 				BROADCAST_LOAD_A_512(x, 0);

  150. 				BROADCAST_LOAD_A_512(x, 1);

  151. 				BROADCAST_LOAD_A_512(x, 2);

  152. 				BROADCAST_LOAD_A_512(x, 3);

  153. 

  154. 				LOAD_B_512(0, x);

  155. 

  156. 				MATMUL_512(0, 0);

  157. 				MATMUL_512(0, 1);

  158. 				MATMUL_512(0, 2);

  159. 				MATMUL_512(0, 3);

  160. 			}

  161. 			STORE_512(0, 0);

  162. 			STORE_512(0, 1);

  163. 			STORE_512(0, 2);

  164. 			STORE_512(0, 3);

  165. 		}

  166. 

  167. 		for (; j < n8; j+= 8) {

  168. 			DECLARE_RESULT_256(0, 0);

  169. 			DECLARE_RESULT_256(0, 1);

  170. 			DECLARE_RESULT_256(0, 2);

  171. 			DECLARE_RESULT_256(0, 3);

  172. 

  173. 			for (k = 0; k < K; k++) {

  174. 				BROADCAST_LOAD_A_256(x, 0);

  175. 				BROADCAST_LOAD_A_256(x, 1);

  176. 				BROADCAST_LOAD_A_256(x, 2);

  177. 				BROADCAST_LOAD_A_256(x, 3);

  178. 

  179. 				LOAD_B_256(0, x);

  180. 

  181. 				MATMUL_256(0, 0);

  182. 				MATMUL_256(0, 1);

  183. 				MATMUL_256(0, 2);

  184. 				MATMUL_256(0, 3);

  185. 			}

  186. 			STORE_256(0, 0);

  187. 			STORE_256(0, 1);

  188. 			STORE_256(0, 2);

  189. 			STORE_256(0, 3);

  190. 		}

  191. 

  192. 		for (; j < n4; j+= 4) {

  193. 			DECLARE_RESULT_128(0, 0);

  194. 			DECLARE_RESULT_128(0, 1);

  195. 			DECLARE_RESULT_128(0, 2);

  196. 			DECLARE_RESULT_128(0, 3);

  197. 

  198. 			for (k = 0; k < K; k++) {

  199. 				BROADCAST_LOAD_A_128(x, 0);

  200. 				BROADCAST_LOAD_A_128(x, 1);

  201. 				BROADCAST_LOAD_A_128(x, 2);

  202. 				BROADCAST_LOAD_A_128(x, 3);

  203. 

  204. 				LOAD_B_128(0, x);

  205. 

  206. 				MATMUL_128(0, 0);

  207. 				MATMUL_128(0, 1);

  208. 				MATMUL_128(0, 2);

  209. 				MATMUL_128(0, 3);

  210. 			}

  211. 			STORE_128(0, 0);

  212. 			STORE_128(0, 1);

  213. 			STORE_128(0, 2);

  214. 			STORE_128(0, 3);

  215. 		}

  216. 

  217. 		for (; j < n2; j+= 2) {

  218. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  219. 			DECLARE_RESULT_SCALAR(0, 1);	DECLARE_RESULT_SCALAR(1, 1);

  220. 			DECLARE_RESULT_SCALAR(0, 2);	DECLARE_RESULT_SCALAR(1, 2);

  221. 			DECLARE_RESULT_SCALAR(0, 3);	DECLARE_RESULT_SCALAR(1, 3);

  222. 

  223. 			for (k = 0; k < K; k++) {

  224. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  225. 				BROADCAST_LOAD_A_SCALAR(x, 1);

  226. 				BROADCAST_LOAD_A_SCALAR(x, 2);

  227. 				BROADCAST_LOAD_A_SCALAR(x, 3);

  228. 

  229. 				LOAD_B_SCALAR(0, x);	LOAD_B_SCALAR(1, x);

  230. 

  231. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  232. 				MATMUL_SCALAR(0, 1);	MATMUL_SCALAR(1, 1);

  233. 				MATMUL_SCALAR(0, 2);	MATMUL_SCALAR(1, 2);

  234. 				MATMUL_SCALAR(0, 3);	MATMUL_SCALAR(1, 3);

  235. 			}

  236. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  237. 			STORE_SCALAR(0, 1);	STORE_SCALAR(1, 1);

  238. 			STORE_SCALAR(0, 2);	STORE_SCALAR(1, 2);

  239. 			STORE_SCALAR(0, 3);	STORE_SCALAR(1, 3);

  240. 		}

  241. 

  242. 		for (; j < N; j++) {

  243. 			DECLARE_RESULT_SCALAR(0, 0)

  244. 			DECLARE_RESULT_SCALAR(0, 1)

  245. 			DECLARE_RESULT_SCALAR(0, 2)

  246. 			DECLARE_RESULT_SCALAR(0, 3)

  247. 

  248. 			for (k = 0; k < K; k++) {

  249. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  250. 				BROADCAST_LOAD_A_SCALAR(0, 1);

  251. 				BROADCAST_LOAD_A_SCALAR(0, 2);

  252. 				BROADCAST_LOAD_A_SCALAR(0, 3);

  253. 

  254. 				LOAD_B_SCALAR(0, 0);

  255. 

  256. 				MATMUL_SCALAR(0, 0);

  257. 				MATMUL_SCALAR(0, 1);

  258. 				MATMUL_SCALAR(0, 2);

  259. 				MATMUL_SCALAR(0, 3);

  260. 			}

  261. 			STORE_SCALAR(0, 0);

  262. 			STORE_SCALAR(0, 1);

  263. 			STORE_SCALAR(0, 2);

  264. 			STORE_SCALAR(0, 3);

  265. 		}

  266. 	}

  267. 

  268. 	for (; i < m2; i+=2) {

  269. 		j = 0;

  270. 

  271. 		for (; j < n64; j+= 64) {

  272. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  273. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);    			DECLARE_RESULT_512(2, 1);    DECLARE_RESULT_512(3, 1);

  274. 

  275. 

  276. 			for (k = 0; k < K; k++) {

  277. 				BROADCAST_LOAD_A_512(x, 0);

  278. 				BROADCAST_LOAD_A_512(x, 1);

  279. 

  280. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  281. 

  282. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  283. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);			MATMUL_512(2, 1);		MATMUL_512(3, 1);

  284. 			}

  285. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  286. 			STORE_512(0, 1);		STORE_512(1, 1);			STORE_512(2, 1);		STORE_512(3, 1);

  287. 		}

  288. 

  289. 		for (; j < n32; j+= 32) {

  290. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  291. 			DECLARE_RESULT_512(0, 1);    DECLARE_RESULT_512(1, 1);

  292. 

  293. 			for (k = 0; k < K; k++) {

  294. 				BROADCAST_LOAD_A_512(x, 0);

  295. 				BROADCAST_LOAD_A_512(x, 1);

  296. 

  297. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  298. 

  299. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  300. 				MATMUL_512(0, 1);		MATMUL_512(1, 1);

  301. 			}

  302. 			STORE_512(0, 0);		STORE_512(1, 0);

  303. 			STORE_512(0, 1);		STORE_512(1, 1);

  304. 		}

  305. 

  306. 

  307. 		for (; j < n16; j+= 16) {

  308. 			DECLARE_RESULT_512(0, 0);

  309. 			DECLARE_RESULT_512(0, 1);

  310. 

  311. 			for (k = 0; k < K; k++) {

  312. 				BROADCAST_LOAD_A_512(x, 0);

  313. 				BROADCAST_LOAD_A_512(x, 1);

  314. 

  315. 				LOAD_B_512(0, x);

  316. 

  317. 				MATMUL_512(0, 0);

  318. 				MATMUL_512(0, 1);

  319. 			}

  320. 			STORE_512(0, 0);

  321. 			STORE_512(0, 1);

  322. 		}

  323. 

  324. 		for (; j < n8; j+= 8) {

  325. 			DECLARE_RESULT_256(0, 0);

  326. 			DECLARE_RESULT_256(0, 1);

  327. 

  328. 			for (k = 0; k < K; k++) {

  329. 				BROADCAST_LOAD_A_256(x, 0);

  330. 				BROADCAST_LOAD_A_256(x, 1);

  331. 

  332. 				LOAD_B_256(0, x);

  333. 

  334. 				MATMUL_256(0, 0);

  335. 				MATMUL_256(0, 1);

  336. 			}

  337. 			STORE_256(0, 0);

  338. 			STORE_256(0, 1);

  339. 		}

  340. 

  341. 		for (; j < n4; j+= 4) {

  342. 			DECLARE_RESULT_128(0, 0);

  343. 			DECLARE_RESULT_128(0, 1);

  344. 

  345. 			for (k = 0; k < K; k++) {

  346. 				BROADCAST_LOAD_A_128(x, 0);

  347. 				BROADCAST_LOAD_A_128(x, 1);

  348. 

  349. 				LOAD_B_128(0, x);

  350. 

  351. 				MATMUL_128(0, 0);

  352. 				MATMUL_128(0, 1);

  353. 			}

  354. 			STORE_128(0, 0);

  355. 			STORE_128(0, 1);

  356. 		}

  357. 		for (; j < n2; j+= 2) {

  358. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  359. 			DECLARE_RESULT_SCALAR(0, 1);	DECLARE_RESULT_SCALAR(1, 1);

  360. 

  361. 			for (k = 0; k < K; k++) {

  362. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  363. 				BROADCAST_LOAD_A_SCALAR(x, 1);

  364. 

  365. 				LOAD_B_SCALAR(0, x);	LOAD_B_SCALAR(1, x);

  366. 

  367. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  368. 				MATMUL_SCALAR(0, 1);	MATMUL_SCALAR(1, 1);

  369. 			}

  370. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  371. 			STORE_SCALAR(0, 1);	STORE_SCALAR(1, 1);

  372. 		}

  373. 

  374. 		for (; j < N; j++) {

  375. 			DECLARE_RESULT_SCALAR(0, 0);

  376. 			DECLARE_RESULT_SCALAR(0, 1);

  377. 

  378. 			for (k = 0; k < K; k++) {

  379. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  380. 				BROADCAST_LOAD_A_SCALAR(0, 1);

  381. 

  382. 				LOAD_B_SCALAR(0, 0);

  383. 

  384. 				MATMUL_SCALAR(0, 0);

  385. 				MATMUL_SCALAR(0, 1);

  386. 			}

  387. 			STORE_SCALAR(0, 0);

  388. 			STORE_SCALAR(0, 1);

  389. 		}

  390. 	}

  391. 

  392. 	for (; i < M; i+=1) {

  393. 		j = 0;

  394. 		for (; j < n64; j+= 64) {

  395. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);    			DECLARE_RESULT_512(2, 0);    DECLARE_RESULT_512(3, 0);

  396. 

  397. 			for (k = 0; k < K; k++) {

  398. 				BROADCAST_LOAD_A_512(x, 0);

  399. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);			LOAD_B_512(2, x);		LOAD_B_512(3, x);

  400. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);			MATMUL_512(2, 0);		MATMUL_512(3, 0);

  401. 			}

  402. 			STORE_512(0, 0);		STORE_512(1, 0);			STORE_512(2, 0);		STORE_512(3, 0);

  403. 		}

  404. 		for (; j < n32; j+= 32) {

  405. 			DECLARE_RESULT_512(0, 0);    DECLARE_RESULT_512(1, 0);

  406. 

  407. 			for (k = 0; k < K; k++) {

  408. 				BROADCAST_LOAD_A_512(x, 0);

  409. 				LOAD_B_512(0, x);		LOAD_B_512(1, x);

  410. 				MATMUL_512(0, 0);		MATMUL_512(1, 0);

  411. 			}

  412. 			STORE_512(0, 0);		STORE_512(1, 0);

  413. 		}

  414. 

  415. 

  416. 		for (; j < n16; j+= 16) {

  417. 			DECLARE_RESULT_512(0, 0);

  418. 

  419. 			for (k = 0; k < K; k++) {

  420. 				BROADCAST_LOAD_A_512(x, 0);

  421. 

  422. 				LOAD_B_512(0, x);

  423. 

  424. 				MATMUL_512(0, 0);

  425. 			}

  426. 			STORE_512(0, 0);

  427. 		}

  428. 

  429. 		for (; j < n8; j+= 8) {

  430. 			DECLARE_RESULT_256(0, 0);

  431. 

  432. 			for (k = 0; k < K; k++) {

  433. 				BROADCAST_LOAD_A_256(x, 0);

  434. 				LOAD_B_256(0, x);

  435. 				MATMUL_256(0, 0);

  436. 			}

  437. 			STORE_256(0, 0);

  438. 		}

  439. 

  440. 		for (; j < n4; j+= 4) {

  441. 			DECLARE_RESULT_128(0, 0);

  442. 

  443. 			for (k = 0; k < K; k++) {

  444. 				BROADCAST_LOAD_A_128(x, 0);

  445. 				LOAD_B_128(0, x);

  446. 				MATMUL_128(0, 0);

  447. 			}

  448. 			STORE_128(0, 0);

  449. 		}

  450. 

  451. 		for (; j < n2; j+= 2) {

  452. 			DECLARE_RESULT_SCALAR(0, 0);	DECLARE_RESULT_SCALAR(1, 0);

  453. 

  454. 			for (k = 0; k < K; k++) {

  455. 				BROADCAST_LOAD_A_SCALAR(x, 0);

  456. 				LOAD_B_SCALAR(0, 0);	LOAD_B_SCALAR(1, 0);

  457. 				MATMUL_SCALAR(0, 0);	MATMUL_SCALAR(1, 0);

  458. 			}

  459. 			STORE_SCALAR(0, 0);	STORE_SCALAR(1, 0);

  460. 		}

  461. 

  462. 		for (; j < N; j++) {

  463. 			DECLARE_RESULT_SCALAR(0, 0);

  464. 

  465. 			for (k = 0; k < K; k++) {

  466. 				BROADCAST_LOAD_A_SCALAR(0, 0);

  467. 				LOAD_B_SCALAR(0, 0);

  468. 				MATMUL_SCALAR(0, 0);

  469. 			}

  470. 			STORE_SCALAR(0, 0);

  471. 		}

  472. 	}

  473. }

  474. #else

  475. 

  476. void CNAME (BLASLONG M, BLASLONG N, BLASLONG K, float * __restrict A, BLASLONG strideA, float * __restrict B, BLASLONG strideB , float * __restrict R, BLASLONG strideR)

  477. {}

  478. #endif