OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/sbgemm_otcopy_16_spr.c

303 lines
12 kB
Raw Blame History 

  1. /***************************************************************************

  2.  * Copyright (c) 2021, The OpenBLAS Project

  3.  * All rights reserved.

  4.  * Redistribution and use in source and binary forms, with or without

  5.  * modification, are permitted provided that the following conditions are

  6.  * met:

  7.  * 1. Redistributions of source code must retain the above copyright

  8.  * notice, this list of conditions and the following disclaimer.

  9.  * 2. Redistributions in binary form must reproduce the above copyright

  10.  * notice, this list of conditions and the following disclaimer in

  11.  * the documentation and/or other materials provided with the

  12.  * distribution.

  13.  * 3. Neither the name of the OpenBLAS project nor the names of

  14.  * its contributors may be used to endorse or promote products

  15.  * derived from this software without specific prior written permission.

  16.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17.  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18.  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19.  * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20.  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21.  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22.  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23.  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24.  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25.  * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26.  * *****************************************************************************/

  27. 

  28. #include <immintrin.h>

  29. #include "common.h"

  30. 

  31. #define LOAD_A_8VEC(aptr)  \

  32. 	r0 = _mm256_loadu_si256((__m256i *)(aptr + lda*0)); \

  33. 	r1 = _mm256_loadu_si256((__m256i *)(aptr + lda*1)); \

  34. 	r2 = _mm256_loadu_si256((__m256i *)(aptr + lda*2)); \

  35. 	r3 = _mm256_loadu_si256((__m256i *)(aptr + lda*3)); \

  36. 	r4 = _mm256_loadu_si256((__m256i *)(aptr + lda*4)); \

  37. 	r5 = _mm256_loadu_si256((__m256i *)(aptr + lda*5)); \

  38. 	r6 = _mm256_loadu_si256((__m256i *)(aptr + lda*6)); \

  39. 	r7 = _mm256_loadu_si256((__m256i *)(aptr + lda*7));

  40. 

  41. #define MASK_LOAD_A_8VEC(aptr)  \

  42. 	r0 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*0)); \

  43. 	r1 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*1)); \

  44. 	r2 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*2)); \

  45. 	r3 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*3)); \

  46. 	r4 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*4)); \

  47. 	r5 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*5)); \

  48. 	r6 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*6)); \

  49. 	r7 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*7));

  50. 

  51. #define SWITCH_LOAD_A_8VEC(aptr, cond) \

  52. 	switch((cond)) { \

  53. 		case 8: r7 = _mm256_loadu_si256((__m256i *)(aptr + lda*7)); \

  54. 		case 7: r6 = _mm256_loadu_si256((__m256i *)(aptr + lda*6)); \

  55. 		case 6: r5 = _mm256_loadu_si256((__m256i *)(aptr + lda*5)); \

  56. 		case 5: r4 = _mm256_loadu_si256((__m256i *)(aptr + lda*4)); \

  57. 		case 4: r3 = _mm256_loadu_si256((__m256i *)(aptr + lda*3)); \

  58. 		case 3: r2 = _mm256_loadu_si256((__m256i *)(aptr + lda*2)); \

  59. 		case 2: r1 = _mm256_loadu_si256((__m256i *)(aptr + lda*1)); \

  60. 		case 1: r0 = _mm256_loadu_si256((__m256i *)(aptr + lda*0)); \

  61. 	}

  62. 

  63. #define SWITCH_MASK_LOAD_A_8VEC(aptr, cond) \

  64. 	switch((cond)) { \

  65. 		case 8: r7 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*7)); \

  66. 		case 7: r6 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*6)); \

  67. 		case 6: r5 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*5)); \

  68. 		case 5: r4 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*4)); \

  69. 		case 4: r3 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*3)); \

  70. 		case 3: r2 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*2)); \

  71. 		case 2: r1 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*1)); \

  72. 		case 1: r0 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aptr + lda*0)); \

  73. 	}

  74. 

  75. #define REORDER_8x16(t0, t1, t2, t3, t4, t5, t6, t7) \

  76. 	t0 = _mm256_unpacklo_epi16(r0, r1); \

  77. 	t1 = _mm256_unpackhi_epi16(r0, r1); \

  78. 	t2 = _mm256_unpacklo_epi16(r2, r3); \

  79. 	t3 = _mm256_unpackhi_epi16(r2, r3); \

  80. 	t4 = _mm256_unpacklo_epi16(r4, r5); \

  81. 	t5 = _mm256_unpackhi_epi16(r4, r5); \

  82. 	t6 = _mm256_unpacklo_epi16(r6, r7); \

  83. 	t7 = _mm256_unpackhi_epi16(r6, r7); \

  84. 	r0 = _mm256_unpacklo_epi32(t0, t2); \

  85. 	r1 = _mm256_unpacklo_epi32(t1, t3); \

  86. 	r2 = _mm256_unpacklo_epi32(t4, t6); \

  87. 	r3 = _mm256_unpacklo_epi32(t5, t7); \

  88. 	r4 = _mm256_unpackhi_epi32(t0, t2); \

  89. 	r5 = _mm256_unpackhi_epi32(t1, t3); \

  90. 	r6 = _mm256_unpackhi_epi32(t4, t6); \

  91. 	r7 = _mm256_unpackhi_epi32(t5, t7); \

  92. 	t0 = _mm256_unpacklo_epi64(r0, r2); \

  93. 	t1 = _mm256_unpackhi_epi64(r0, r2); \

  94. 	t2 = _mm256_unpacklo_epi64(r4, r6); \

  95. 	t3 = _mm256_unpackhi_epi64(r4, r6); \

  96. 	t4 = _mm256_unpacklo_epi64(r1, r3); \

  97. 	t5 = _mm256_unpackhi_epi64(r1, r3); \

  98. 	t6 = _mm256_unpacklo_epi64(r5, r7); \

  99. 	t7 = _mm256_unpackhi_epi64(r5, r7);

  100. 

  101. #define STORE_256_LO(x) \

  102. 	v = _mm256_permute2x128_si256(t0##x, t1##x, 0x20); \

  103. 	_mm256_storeu_si256((__m256i *)(boffset + x*32), v);

  104. 

  105. #define STORE_256_HI(x) \

  106. 	v = _mm256_permute2x128_si256(t0##x, t1##x, 0x31); \

  107. 	_mm256_storeu_si256((__m256i *)(boffset + (x + 8)*32), v);

  108. 

  109. #define MASK_STORE_256_LO(x) \

  110. 	v = _mm256_permute2x128_si256(t0##x, t1##x, 0x20); \

  111. 	_mm256_mask_storeu_epi16(boffset + x*m_load, mmask, v);

  112. 

  113. #define MASK_STORE_256_HI(x) \

  114. 	v = _mm256_permute2x128_si256(t0##x, t1##x, 0x31); \

  115. 	_mm256_mask_storeu_epi16(boffset + (x + 8)*m_load, mmask, v);

  116. 

  117. #define STORE_256(x, y) {\

  118. 	__m256i v; \

  119. 	if (x == 0) { STORE_256_LO(y); } \

  120. 	else { STORE_256_HI(y); } \

  121. }

  122. 

  123. #define MASK_STORE_256(x, y) {\

  124. 	__m256i v; \

  125. 	if (x == 0) { MASK_STORE_256_LO(y); } \

  126. 	else { MASK_STORE_256_HI(y); } \

  127. }

  128. 

  129. #define SWITCH_STORE_16x(cond, func) \

  130. 	switch((cond)) {\

  131. 		case 15: func(1, 6); \

  132. 		case 14: func(1, 5); \

  133. 		case 13: func(1, 4); \

  134. 		case 12: func(1, 3); \

  135. 		case 11: func(1, 2); \

  136. 		case 10: func(1, 1); \

  137. 		case 9: func(1, 0); \

  138. 		case 8: func(0, 7); \

  139. 		case 7: func(0, 6); \

  140. 		case 6: func(0, 5); \

  141. 		case 5: func(0, 4); \

  142. 		case 4: func(0, 3); \

  143. 		case 3: func(0, 2); \

  144. 		case 2: func(0, 1); \

  145. 		case 1: func(0, 0); \

  146. 	}

  147. 

  148. 

  149. int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {

  150. 	IFLOAT *aoffset, *boffset;

  151. 	IFLOAT *aoffset00, *aoffset01, *aoffset10, *aoffset11;

  152. 	IFLOAT *boffset0;

  153. 

  154. 	__m256i r0, r1, r2, r3, r4, r5, r6, r7;

  155. 	__m256i t00, t01, t02, t03, t04, t05, t06, t07;

  156. 	__m256i t10, t11, t12, t13, t14, t15, t16, t17;

  157. 

  158. 	aoffset = a;

  159. 	boffset = b;

  160. 	BLASLONG n_count = n;

  161. 	BLASLONG m_count = m;

  162. 	for (; n_count > 15; n_count -= 16) {

  163. 		aoffset00 = aoffset;

  164. 		aoffset01 = aoffset00 + 8 * lda;

  165. 		aoffset10 = aoffset01 + 8 * lda;

  166. 		aoffset11 = aoffset10 + 8 * lda;

  167. 		aoffset += 16;

  168. 		m_count = m;

  169. 		for (; m_count > 31; m_count -= 32) {

  170. 			// first 16 rows

  171. 			LOAD_A_8VEC(aoffset00);

  172. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  173. 			LOAD_A_8VEC(aoffset01);

  174. 			REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  175. 			STORE_256(0, 0); STORE_256(0, 1); STORE_256(0, 2); STORE_256(0, 3);

  176. 			STORE_256(0, 4); STORE_256(0, 5); STORE_256(0, 6); STORE_256(0, 7);

  177. 			STORE_256(1, 0); STORE_256(1, 1); STORE_256(1, 2); STORE_256(1, 3);

  178. 			STORE_256(1, 4); STORE_256(1, 5); STORE_256(1, 6); STORE_256(1, 7);

  179. 			// last 16 rows

  180. 			boffset += 16;

  181. 			LOAD_A_8VEC(aoffset10);

  182. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  183. 			LOAD_A_8VEC(aoffset11);

  184. 			REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  185. 			STORE_256(0, 0); STORE_256(0, 1); STORE_256(0, 2); STORE_256(0, 3);

  186. 			STORE_256(0, 4); STORE_256(0, 5); STORE_256(0, 6); STORE_256(0, 7);

  187. 			STORE_256(1, 0); STORE_256(1, 1); STORE_256(1, 2); STORE_256(1, 3);

  188. 			STORE_256(1, 4); STORE_256(1, 5); STORE_256(1, 6); STORE_256(1, 7);

  189. 			aoffset00 += 32 * lda;

  190. 			aoffset01 += 32 * lda;

  191. 			aoffset10 += 32 * lda;

  192. 			aoffset11 += 32 * lda;

  193. 			boffset += 31 * 16;

  194. 		}

  195. 		if (m_count > 1) {

  196. 			int m_load = m_count & ~1;

  197. 			m_count -= m_load;

  198. 			__mmask16 mmask;

  199. 			SWITCH_LOAD_A_8VEC(aoffset00, m_load > 8 ? 8: m_load);

  200. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  201. 			if (m_load > 8) {

  202. 				SWITCH_LOAD_A_8VEC(aoffset01, m_load > 16 ? 8: m_load - 8);

  203. 				REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  204. 			}

  205. 			int this_load = m_load > 16 ? 16 : m_load;

  206. 			mmask = (1UL << this_load) - 1;

  207. 			MASK_STORE_256(0, 0); MASK_STORE_256(0, 1); MASK_STORE_256(0, 2); MASK_STORE_256(0, 3);

  208. 			MASK_STORE_256(0, 4); MASK_STORE_256(0, 5); MASK_STORE_256(0, 6); MASK_STORE_256(0, 7);

  209. 			MASK_STORE_256(1, 0); MASK_STORE_256(1, 1); MASK_STORE_256(1, 2); MASK_STORE_256(1, 3);

  210. 			MASK_STORE_256(1, 4); MASK_STORE_256(1, 5); MASK_STORE_256(1, 6); MASK_STORE_256(1, 7);

  211. 			boffset0 = boffset;

  212. 			if (m_load > 16) {

  213. 				boffset += this_load;

  214. 				SWITCH_LOAD_A_8VEC(aoffset10, m_load > 24 ? 8: m_load - 16);

  215. 				REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  216. 				if (m_load > 24) {

  217. 					SWITCH_LOAD_A_8VEC(aoffset11, m_load - 24);

  218. 					REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  219. 				}

  220. 				this_load = m_load - 16;

  221. 				mmask = (1UL << this_load) - 1;

  222. 				MASK_STORE_256(0, 0); MASK_STORE_256(0, 1); MASK_STORE_256(0, 2); MASK_STORE_256(0, 3);

  223. 				MASK_STORE_256(0, 4); MASK_STORE_256(0, 5); MASK_STORE_256(0, 6); MASK_STORE_256(0, 7);

  224. 				MASK_STORE_256(1, 0); MASK_STORE_256(1, 1); MASK_STORE_256(1, 2); MASK_STORE_256(1, 3);

  225. 				MASK_STORE_256(1, 4); MASK_STORE_256(1, 5); MASK_STORE_256(1, 6); MASK_STORE_256(1, 7);

  226. 			}

  227. 			boffset = boffset0 + 16 * m_load;

  228. 			aoffset00 += m_load * lda;

  229. 		}

  230. 		if (m_count > 0) {

  231. 			// just copy lask K to B directly

  232. 			r0 = _mm256_loadu_si256((__m256i *)(aoffset00));

  233. 			_mm256_storeu_si256((__m256i *)(boffset), r0);

  234. 			boffset += 16;

  235. 		}

  236. 	}

  237. 	if (n_count > 0) {

  238. 		__mmask16 nmask = (1UL << n_count) - 1;

  239. 		aoffset00 = aoffset;

  240. 		aoffset01 = aoffset00 + 8 * lda;

  241. 		aoffset10 = aoffset01 + 8 * lda;

  242. 		aoffset11 = aoffset10 + 8 * lda;

  243. 		m_count = m;

  244. 		for (; m_count > 31; m_count -= 32) {

  245. 			// first 16 rows

  246. 			MASK_LOAD_A_8VEC(aoffset00);

  247. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  248. 			MASK_LOAD_A_8VEC(aoffset01);

  249. 			REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  250. 			SWITCH_STORE_16x(n_count, STORE_256);

  251. 			// last 16 rows

  252. 			boffset0 = boffset;

  253. 			boffset += 16;

  254. 			MASK_LOAD_A_8VEC(aoffset10);

  255. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  256. 			MASK_LOAD_A_8VEC(aoffset11);

  257. 			REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  258. 			SWITCH_STORE_16x(n_count, STORE_256);

  259. 			aoffset00 += 32 * lda;

  260. 			aoffset01 += 32 * lda;

  261. 			aoffset10 += 32 * lda;

  262. 			aoffset11 += 32 * lda;

  263. 			boffset = 32 * n_count + boffset0;

  264. 		}

  265. 		if (m_count > 1) {

  266. 			int m_load = m_count & ~1;

  267. 			m_count -= m_load;

  268. 			__mmask16 mmask;

  269. 			SWITCH_MASK_LOAD_A_8VEC(aoffset00, m_load > 8 ? 8: m_load);

  270. 			REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  271. 			if (m_load > 8) {

  272. 				SWITCH_MASK_LOAD_A_8VEC(aoffset01, m_load > 16 ? 8: m_load - 8);

  273. 				REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  274. 			}

  275. 			int this_load = m_load > 16 ? 16 : m_load;

  276. 			mmask = (1UL << this_load) - 1;

  277. 			SWITCH_STORE_16x(n_count, MASK_STORE_256);

  278. 			boffset0 = boffset;

  279. 			if (m_load > 16) {

  280. 				boffset += this_load;

  281. 				SWITCH_MASK_LOAD_A_8VEC(aoffset10, m_load > 24 ? 8: m_load - 16);

  282. 				REORDER_8x16(t00, t01, t02, t03, t04, t05, t06, t07);

  283. 				if (m_load > 24) {

  284. 					SWITCH_MASK_LOAD_A_8VEC(aoffset11, m_load - 24);

  285. 					REORDER_8x16(t10, t11, t12, t13, t14, t15, t16, t17);

  286. 				}

  287. 				this_load = m_load - 16;

  288. 				mmask = (1UL << this_load) - 1;

  289. 				SWITCH_STORE_16x(n_count, MASK_STORE_256);

  290. 			}

  291. 			boffset = boffset0 + n_count * m_load;

  292. 			aoffset00 += m_load * lda;

  293. 		}

  294. 		if (m_count > 0) {

  295. 			// just copy lask K to B directly

  296. 			r0 = _mm256_maskz_loadu_epi16(nmask, (__m256i *)(aoffset00));

  297. 			_mm256_mask_storeu_epi16((__m256i *)(boffset), nmask, r0);

  298. 			boffset += 16;

  299. 		}

  300. 	}

  301. 	return 0;

  302. }