OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: cd8ac192a9
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cd8ac192a9'
${ noResults }
OpenBLAS/kernel/x86_64/dgemm_tcopy_8_skylakex.c

418 lines
11 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #include <stdio.h>

  40. #include "common.h"

  41. #include <immintrin.h>

  42. 

  43. int CNAME(BLASLONG m, BLASLONG n, FLOAT * __restrict a, BLASLONG lda, FLOAT * __restrict b){

  44. 

  45.   BLASLONG i, j;

  46. 

  47.   FLOAT *aoffset;

  48.   FLOAT *aoffset1, *aoffset2, *aoffset3, *aoffset4;

  49.   FLOAT *aoffset5, *aoffset6, *aoffset7, *aoffset8;

  50. 

  51.   FLOAT *boffset,  *boffset1, *boffset2, *boffset3, *boffset4;

  52. 

  53.   FLOAT ctemp01, ctemp02, ctemp03, ctemp04;

  54.   FLOAT ctemp05, ctemp06, ctemp07, ctemp08;

  55. 

  56.   aoffset   = a;

  57.   boffset   = b;

  58. 

  59. #if 0

  60.   fprintf(stderr, "M = %d N = %d\n", m, n);

  61. #endif

  62. 

  63.   boffset2  = b + m  * (n & ~7);

  64.   boffset3  = b + m  * (n & ~3);

  65.   boffset4  = b + m  * (n & ~1);

  66. 

  67.   j = (m >> 3);

  68.   if (j > 0){

  69.     do{

  70.       aoffset1  = aoffset;

  71.       aoffset2  = aoffset1 + lda;

  72.       aoffset3  = aoffset2 + lda;

  73.       aoffset4  = aoffset3 + lda;

  74.       aoffset5  = aoffset4 + lda;

  75.       aoffset6  = aoffset5 + lda;

  76.       aoffset7  = aoffset6 + lda;

  77.       aoffset8  = aoffset7 + lda;

  78.       aoffset += 8 * lda;

  79. 

  80.       boffset1  = boffset;

  81.       boffset  += 64;

  82. 

  83.       i = (n >> 3);

  84.       if (i > 0){

  85. 	do{

  86. 	  __m512d row1, row2, row3, row4, row5, row6, row7, row8;

  87. 	  row1 = _mm512_loadu_pd(aoffset1);

  88. 	  aoffset1 += 8;

  89. 	  row2 = _mm512_loadu_pd(aoffset2);

  90. 	  aoffset2 += 8;

  91. 	  row3 = _mm512_loadu_pd(aoffset3);

  92. 	  aoffset3 += 8;

  93. 	  row4 = _mm512_loadu_pd(aoffset4);

  94. 	  aoffset4 += 8;

  95. 	  row5 = _mm512_loadu_pd(aoffset5);

  96. 	  aoffset5 += 8;

  97. 	  row6 = _mm512_loadu_pd(aoffset6);

  98. 	  aoffset6 += 8;

  99. 	  row7 = _mm512_loadu_pd(aoffset7);

  100. 	  aoffset7 += 8;

  101. 	  row8 = _mm512_loadu_pd(aoffset8);

  102. 	  aoffset8 += 8;

  103. 

  104. 	  _mm512_storeu_pd(boffset1 +  0, row1);

  105. 	  _mm512_storeu_pd(boffset1 +  8, row2);

  106. 	  _mm512_storeu_pd(boffset1 + 16, row3);

  107. 	  _mm512_storeu_pd(boffset1 + 24, row4);

  108. 	  _mm512_storeu_pd(boffset1 + 32, row5);

  109. 	  _mm512_storeu_pd(boffset1 + 40, row6);

  110. 	  _mm512_storeu_pd(boffset1 + 48, row7);

  111. 	  _mm512_storeu_pd(boffset1 + 56, row8);

  112. 	  boffset1 += m * 8;

  113. 	  i --;

  114. 	}while(i > 0);

  115.       }

  116. 

  117.       if (n & 4){

  118. 	__m256d row1, row2, row3, row4, row5, row6, row7, row8;

  119. 	row1 = _mm256_loadu_pd(aoffset1);

  120. 	aoffset1 += 4;

  121. 	row2 = _mm256_loadu_pd(aoffset2);

  122. 	aoffset2 += 4;

  123. 	row3 = _mm256_loadu_pd(aoffset3);

  124. 	aoffset3 += 4;

  125. 	row4 = _mm256_loadu_pd(aoffset4);

  126. 	aoffset4 += 4;

  127. 	row5 = _mm256_loadu_pd(aoffset5);

  128. 	aoffset5 += 4;

  129. 	row6 = _mm256_loadu_pd(aoffset6);

  130. 	aoffset6 += 4;

  131. 	row7 = _mm256_loadu_pd(aoffset7);

  132. 	aoffset7 += 4;

  133. 	row8 = _mm256_loadu_pd(aoffset8);

  134. 	aoffset8 += 4;

  135. 

  136. 	_mm256_storeu_pd(boffset2 +   0, row1);

  137. 	_mm256_storeu_pd(boffset2 +   4, row2);

  138. 	_mm256_storeu_pd(boffset2 +   8, row3);

  139. 	_mm256_storeu_pd(boffset2 +  12, row4);

  140. 	_mm256_storeu_pd(boffset2 +  16, row5);

  141. 	_mm256_storeu_pd(boffset2 +  20, row6);

  142. 	_mm256_storeu_pd(boffset2 +  24, row7);

  143. 	_mm256_storeu_pd(boffset2 +  28, row8);

  144. 	boffset2 += 32;

  145.       }

  146. 

  147.       if (n & 2){

  148. 	__m128d row1, row2, row3, row4, row5, row6, row7, row8;

  149. 	row1 = _mm_loadu_pd(aoffset1);

  150. 	aoffset1 += 2;

  151. 

  152. 	row2 = _mm_loadu_pd(aoffset2);

  153. 	aoffset2 += 2;

  154. 

  155. 	row3 = _mm_loadu_pd(aoffset3);

  156. 	aoffset3 += 2;

  157. 

  158. 	row4 = _mm_loadu_pd(aoffset4);

  159. 	aoffset4 += 2;

  160. 

  161. 	row5 = _mm_loadu_pd(aoffset5);

  162. 	aoffset5 += 2;

  163. 

  164. 	row6 = _mm_loadu_pd(aoffset6);

  165. 	aoffset6 += 2;

  166. 

  167. 	row7 = _mm_loadu_pd(aoffset7);

  168. 	aoffset7 += 2;

  169. 

  170. 	row8 = _mm_loadu_pd(aoffset8);

  171. 	aoffset8 += 2;

  172. 

  173. 	_mm_storeu_pd(boffset3 +   0, row1);

  174. 	_mm_storeu_pd(boffset3 +   2, row2);

  175. 	_mm_storeu_pd(boffset3 +   4, row3);

  176. 	_mm_storeu_pd(boffset3 +   6, row4);

  177. 	_mm_storeu_pd(boffset3 +   8, row5);

  178. 	_mm_storeu_pd(boffset3 +  10, row6);

  179. 	_mm_storeu_pd(boffset3 +  12, row7);

  180. 	_mm_storeu_pd(boffset3 +  14, row8);

  181. 	boffset3 += 16;

  182.       }

  183. 

  184.       if (n & 1){

  185. 	ctemp01 = *(aoffset1 + 0);

  186. 	aoffset1 ++;

  187. 	ctemp02 = *(aoffset2 + 0);

  188. 	aoffset2 ++;

  189. 	ctemp03 = *(aoffset3 + 0);

  190. 	aoffset3 ++;

  191. 	ctemp04 = *(aoffset4 + 0);

  192. 	aoffset4 ++;

  193. 	ctemp05 = *(aoffset5 + 0);

  194. 	aoffset5 ++;

  195. 	ctemp06 = *(aoffset6 + 0);

  196. 	aoffset6 ++;

  197. 	ctemp07 = *(aoffset7 + 0);

  198. 	aoffset7 ++;

  199. 	ctemp08 = *(aoffset8 + 0);

  200. 	aoffset8 ++;

  201. 

  202. 	*(boffset4 +  0) = ctemp01;

  203. 	*(boffset4 +  1) = ctemp02;

  204. 	*(boffset4 +  2) = ctemp03;

  205. 	*(boffset4 +  3) = ctemp04;

  206. 	*(boffset4 +  4) = ctemp05;

  207. 	*(boffset4 +  5) = ctemp06;

  208. 	*(boffset4 +  6) = ctemp07;

  209. 	*(boffset4 +  7) = ctemp08;

  210. 	boffset4 += 8;

  211.       }

  212. 

  213.       j--;

  214.     }while(j > 0);

  215.   }

  216. 

  217.   if (m & 4){

  218. 

  219.     aoffset1  = aoffset;

  220.     aoffset2  = aoffset1 + lda;

  221.     aoffset3  = aoffset2 + lda;

  222.     aoffset4  = aoffset3 + lda;

  223.     aoffset += 4 * lda;

  224. 

  225.     boffset1  = boffset;

  226.     boffset  += 32;

  227. 

  228.     i = (n >> 3);

  229.     if (i > 0){

  230. 

  231.       do{

  232. 	  __m512d row1, row2, row3, row4;

  233. 	  row1 = _mm512_loadu_pd(aoffset1);

  234. 	  aoffset1 += 8;

  235. 	  row2 = _mm512_loadu_pd(aoffset2);

  236. 	  aoffset2 += 8;

  237. 	  row3 = _mm512_loadu_pd(aoffset3);

  238. 	  aoffset3 += 8;

  239. 	  row4 = _mm512_loadu_pd(aoffset4);

  240. 	  aoffset4 += 8;

  241. 

  242. 	  _mm512_storeu_pd(boffset1 +  0, row1);

  243. 	  _mm512_storeu_pd(boffset1 +  8, row2);

  244. 	  _mm512_storeu_pd(boffset1 + 16, row3);

  245. 	  _mm512_storeu_pd(boffset1 + 24, row4);

  246. 

  247. 	  boffset1 += 8 * m;

  248. 	  i --;

  249.       }while(i > 0);

  250.     }

  251. 

  252.     if (n & 4) {

  253. 	__m256d row1, row2, row3, row4;

  254. 	row1 = _mm256_loadu_pd(aoffset1);

  255. 	aoffset1 += 4;

  256. 	row2 = _mm256_loadu_pd(aoffset2);

  257. 	aoffset2 += 4;

  258. 	row3 = _mm256_loadu_pd(aoffset3);

  259. 	aoffset3 += 4;

  260. 	row4 = _mm256_loadu_pd(aoffset4);

  261. 	aoffset4 += 4;

  262. 	_mm256_storeu_pd(boffset2 +   0, row1);

  263. 	_mm256_storeu_pd(boffset2 +   4, row2);

  264. 	_mm256_storeu_pd(boffset2 +   8, row3);

  265. 	_mm256_storeu_pd(boffset2 +  12, row4);

  266.         boffset2 += 16;

  267.     }

  268. 

  269.     if (n & 2){

  270. 	__m128d row1, row2, row3, row4;

  271. 	row1 = _mm_loadu_pd(aoffset1);

  272. 	aoffset1 += 2;

  273. 

  274. 	row2 = _mm_loadu_pd(aoffset2);

  275. 	aoffset2 += 2;

  276. 

  277. 	row3 = _mm_loadu_pd(aoffset3);

  278. 	aoffset3 += 2;

  279. 

  280. 	row4 = _mm_loadu_pd(aoffset4);

  281. 	aoffset4 += 2;

  282. 

  283. 

  284. 	_mm_storeu_pd(boffset3 +   0, row1);

  285. 	_mm_storeu_pd(boffset3 +   2, row2);

  286. 	_mm_storeu_pd(boffset3 +   4, row3);

  287. 	_mm_storeu_pd(boffset3 +   6, row4);

  288.         boffset3 += 8;

  289.     }

  290. 

  291.     if (n & 1){

  292.       ctemp01 = *(aoffset1 + 0);

  293.       aoffset1 ++;

  294.       ctemp02 = *(aoffset2 + 0);

  295.       aoffset2 ++;

  296.       ctemp03 = *(aoffset3 + 0);

  297.       aoffset3 ++;

  298.       ctemp04 = *(aoffset4 + 0);

  299.       aoffset4 ++;

  300. 

  301.       *(boffset4 +  0) = ctemp01;

  302.       *(boffset4 +  1) = ctemp02;

  303.       *(boffset4 +  2) = ctemp03;

  304.       *(boffset4 +  3) = ctemp04;

  305.       boffset4 += 4;

  306.     }

  307.   }

  308. 

  309.   if (m & 2){

  310.     aoffset1  = aoffset;

  311.     aoffset2  = aoffset1 + lda;

  312.     aoffset += 2 * lda;

  313. 

  314.     boffset1  = boffset;

  315.     boffset  += 16;

  316. 

  317.     i = (n >> 3);

  318.     if (i > 0){

  319.       do{

  320. 	  __m512d row1, row2;

  321. 	  row1 = _mm512_loadu_pd(aoffset1);

  322. 	  aoffset1 += 8;

  323. 	  row2 = _mm512_loadu_pd(aoffset2);

  324. 	  aoffset2 += 8;

  325. 

  326. 	  _mm512_storeu_pd(boffset1 +  0, row1);

  327. 	  _mm512_storeu_pd(boffset1 +  8, row2);

  328. 	  boffset1 += 8 * m;

  329. 	  i --;

  330.       }while(i > 0);

  331.     }

  332. 

  333.     if (n & 4){

  334. 	__m256d row1, row2;

  335. 	row1 = _mm256_loadu_pd(aoffset1);

  336. 	aoffset1 += 4;

  337. 	row2 = _mm256_loadu_pd(aoffset2);

  338. 	aoffset2 += 4;

  339. 	_mm256_storeu_pd(boffset2 +   0, row1);

  340. 	_mm256_storeu_pd(boffset2 +   4, row2);

  341.         boffset2 += 8;

  342.     }

  343. 

  344.     if (n & 2){

  345. 	__m128d row1, row2;

  346. 	row1 = _mm_loadu_pd(aoffset1);

  347. 	aoffset1 += 2;

  348. 

  349. 	row2 = _mm_loadu_pd(aoffset2);

  350. 	aoffset2 += 2;

  351. 

  352. 

  353. 	_mm_storeu_pd(boffset3 +   0, row1);

  354. 	_mm_storeu_pd(boffset3 +   2, row2);

  355.        boffset3 += 4;

  356.     }

  357. 

  358.     if (n & 1){

  359.       ctemp01 = *(aoffset1 + 0);

  360.       aoffset1 ++;

  361.       ctemp02 = *(aoffset2 + 0);

  362.       aoffset2 ++;

  363. 

  364.       *(boffset4 +  0) = ctemp01;

  365.       *(boffset4 +  1) = ctemp02;

  366.       boffset4 += 2;

  367.     }

  368.   }

  369. 

  370.   if (m & 1){

  371.     aoffset1  = aoffset;

  372.     // aoffset += lda;

  373. 

  374.     boffset1  = boffset;

  375.     // boffset  += 8;

  376. 

  377.     i = (n >> 3);

  378.     if (i > 0){

  379.       do{

  380. 	__m512d row1;

  381. 	  row1 = _mm512_loadu_pd(aoffset1);

  382. 	  aoffset1 += 8;

  383. 

  384. 	  _mm512_storeu_pd(boffset1 +  0, row1);

  385.   	  boffset1 += 8 * m;

  386. 	  i --;

  387.        }while(i > 0);

  388.      }

  389. 

  390.      if (n & 4){

  391. 	__m256d row1;

  392. 	row1 = _mm256_loadu_pd(aoffset1);

  393. 	aoffset1 += 4;

  394. 	_mm256_storeu_pd(boffset2 +   0, row1);

  395.        // boffset2 += 4;

  396.      }

  397. 

  398.      if (n & 2){

  399. 	__m128d row1;

  400. 	row1 = _mm_loadu_pd(aoffset1);

  401. 	aoffset1 += 2;

  402. 

  403. 	_mm_storeu_pd(boffset3 +   0, row1);

  404. 

  405.        // boffset3 += 2;

  406.      }

  407. 

  408.      if (n & 1){

  409.        ctemp01 = *(aoffset1 + 0);

  410.        aoffset1 ++;

  411.       *(boffset4 +  0) = ctemp01;

  412.       boffset4 ++;

  413.     }

  414.   }

  415. 

  416.   return 0;

  417. }