OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 39c90f9859
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '39c90f9859'
${ noResults }
OpenBLAS/kernel/arm64/sbgemm_tcopy_4_neoversev1.c

362 lines
13 kB
Raw Blame History 

  1. /***************************************************************************

  2.  * Copyright (c) 2024-2025, The OpenBLAS Project

  3.  * All rights reserved.

  4.  * Redistribution and use in source and binary forms, with or without

  5.  * modification, are permitted provided that the following conditions are

  6.  * met:

  7.  * 1. Redistributions of source code must retain the above copyright

  8.  * notice, this list of conditions and the following disclaimer.

  9.  * 2. Redistributions in binary form must reproduce the above copyright

  10.  * notice, this list of conditions and the following disclaimer in

  11.  * the documentation and/or other materials provided with the

  12.  * distribution.

  13.  * 3. Neither the name of the OpenBLAS project nor the names of

  14.  * its contributors may be used to endorse or promote products

  15.  * derived from this software without specific prior written permission.

  16.  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17.  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18.  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19.  * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20.  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR

  21.  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

  22.  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

  23.  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

  24.  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

  25.  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

  26.  * POSSIBILITY OF SUCH DAMAGE.

  27.  * *****************************************************************************/

  28. #include "common.h"

  29. #include <arm_neon.h>

  30. #include <arm_sve.h>

  31. 

  32. int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) {

  33.   BLASLONG pad_m = ((m + 7) & ~7);

  34.   BLASLONG rest = (m & 7); // rest along m dim

  35. 

  36.   IFLOAT *a_offset;

  37.   IFLOAT *a_offset0, *a_offset1, *a_offset2, *a_offset3;

  38.   IFLOAT *a_offset4, *a_offset5, *a_offset6, *a_offset7;

  39. 

  40.   IFLOAT *b_offset;

  41.   IFLOAT *b_offset0, *b_offset1;

  42. 

  43.   a_offset = a;

  44.   b_offset = b;

  45. 

  46.   svuint16_t c0, c1, c2, c3, c4, c5, c6, c7;

  47.   svuint16_t t0, t1, t2, t3;

  48.   svuint32_t m00, m01, m10, m11;

  49.   svuint64_t st_offsets_0, st_offsets_1;

  50. 

  51.   svbool_t pg16_first_4 = svwhilelt_b16(0, 4);

  52.   svbool_t pg16_first_8 = svwhilelt_b16(0, 8);

  53. 

  54.   svbool_t pg64_first_4 = svwhilelt_b64(0, 4);

  55. 

  56.   uint32_t sizeof_u64 = 8;

  57.   uint64_t _st_offsets_0[4] = {

  58.       0 * sizeof_u64,

  59.       1 * sizeof_u64,

  60.       4 * sizeof_u64,

  61.       5 * sizeof_u64,

  62.   };

  63. 

  64.   uint64_t _st_offsets_1[4] = {

  65.       2 * sizeof_u64,

  66.       3 * sizeof_u64,

  67.       6 * sizeof_u64,

  68.       7 * sizeof_u64,

  69.   };

  70. 

  71.   st_offsets_0 = svld1_u64(pg64_first_4, _st_offsets_0);

  72.   st_offsets_1 = svld1_u64(pg64_first_4, _st_offsets_1);

  73. 

  74.   for (BLASLONG j = 0; j < n / 8; j++) {

  75.     a_offset0 = a_offset;

  76.     a_offset1 = a_offset0 + lda;

  77.     a_offset2 = a_offset1 + lda;

  78.     a_offset3 = a_offset2 + lda;

  79.     a_offset4 = a_offset3 + lda;

  80.     a_offset5 = a_offset4 + lda;

  81.     a_offset6 = a_offset5 + lda;

  82.     a_offset7 = a_offset6 + lda;

  83.     a_offset += 8;

  84. 

  85.     b_offset0 = b_offset;

  86.     b_offset1 = b_offset0 + 4 * pad_m;

  87. 

  88.     b_offset += 8 * pad_m;

  89.     for (BLASLONG i = 0; i < m / 8; i++) {

  90.       // transpose 8x8 matrix and pack into two 4x8 block consists of two 2x4

  91.       // small blocks

  92.       c0 = svld1_u16(pg16_first_8, a_offset0);

  93.       c1 = svld1_u16(pg16_first_8, a_offset1);

  94.       c2 = svld1_u16(pg16_first_8, a_offset2);

  95.       c3 = svld1_u16(pg16_first_8, a_offset3);

  96.       c4 = svld1_u16(pg16_first_8, a_offset4);

  97.       c5 = svld1_u16(pg16_first_8, a_offset5);

  98.       c6 = svld1_u16(pg16_first_8, a_offset6);

  99.       c7 = svld1_u16(pg16_first_8, a_offset7);

  100. 

  101.       t0 = svzip1_u16(c0, c1);

  102.       t1 = svzip1_u16(c2, c3);

  103.       t2 = svzip1_u16(c4, c5);

  104.       t3 = svzip1_u16(c6, c7);

  105. 

  106.       m00 = svzip1_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  107.       m10 = svzip2_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  108.       m01 = svzip1_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  109.       m11 = svzip2_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  110. 

  111.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  112.                                   st_offsets_0, svreinterpret_u64_u32(m00));

  113.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  114.                                   st_offsets_1, svreinterpret_u64_u32(m01));

  115.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset1,

  116.                                   st_offsets_0, svreinterpret_u64_u32(m10));

  117.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset1,

  118.                                   st_offsets_1, svreinterpret_u64_u32(m11));

  119. 

  120.       a_offset0 += 8 * lda;

  121.       a_offset1 += 8 * lda;

  122.       a_offset2 += 8 * lda;

  123.       a_offset3 += 8 * lda;

  124.       a_offset4 += 8 * lda;

  125.       a_offset5 += 8 * lda;

  126.       a_offset6 += 8 * lda;

  127.       a_offset7 += 8 * lda;

  128. 

  129.       b_offset0 += 32;

  130.       b_offset1 += 32;

  131.     }

  132. 

  133.     if (rest) {

  134.       c0 = svld1_u16(pg16_first_8, a_offset0);

  135.       c1 = (rest >= 2 ? svld1_u16(pg16_first_8, a_offset1) : svdup_u16(0));

  136.       c2 = (rest >= 3 ? svld1_u16(pg16_first_8, a_offset2) : svdup_u16(0));

  137.       c3 = (rest >= 4 ? svld1_u16(pg16_first_8, a_offset3) : svdup_u16(0));

  138.       c4 = (rest >= 5 ? svld1_u16(pg16_first_8, a_offset4) : svdup_u16(0));

  139.       c5 = (rest >= 6 ? svld1_u16(pg16_first_8, a_offset5) : svdup_u16(0));

  140.       c6 = (rest == 7 ? svld1_u16(pg16_first_8, a_offset6) : svdup_u16(0));

  141.       c7 = (svdup_u16(0));

  142. 

  143.       t0 = svzip1_u16(c0, c1);

  144.       t1 = svzip1_u16(c2, c3);

  145.       t2 = svzip1_u16(c4, c5);

  146.       t3 = svzip1_u16(c6, c7);

  147. 

  148.       m00 = svzip1_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  149.       m10 = svzip2_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  150.       m01 = svzip1_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  151.       m11 = svzip2_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  152. 

  153.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  154.                                   st_offsets_0, svreinterpret_u64_u32(m00));

  155.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  156.                                   st_offsets_1, svreinterpret_u64_u32(m01));

  157.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset1,

  158.                                   st_offsets_0, svreinterpret_u64_u32(m10));

  159.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset1,

  160.                                   st_offsets_1, svreinterpret_u64_u32(m11));

  161.     }

  162.   }

  163. 

  164.   if (n & 4) {

  165.     a_offset0 = a_offset;

  166.     a_offset1 = a_offset0 + lda;

  167.     a_offset2 = a_offset1 + lda;

  168.     a_offset3 = a_offset2 + lda;

  169.     a_offset4 = a_offset3 + lda;

  170.     a_offset5 = a_offset4 + lda;

  171.     a_offset6 = a_offset5 + lda;

  172.     a_offset7 = a_offset6 + lda;

  173.     a_offset += 4;

  174. 

  175.     b_offset0 = b_offset;

  176.     b_offset += 4 * pad_m;

  177. 

  178.     for (BLASLONG i = 0; i < m / 8; i++) {

  179.       // transpose 8x8 matrix and pack into two 4x8 block consists of two 2x4

  180.       // small blocks

  181.       c0 = svld1_u16(pg16_first_4, a_offset0);

  182.       c1 = svld1_u16(pg16_first_4, a_offset1);

  183.       c2 = svld1_u16(pg16_first_4, a_offset2);

  184.       c3 = svld1_u16(pg16_first_4, a_offset3);

  185.       c4 = svld1_u16(pg16_first_4, a_offset4);

  186.       c5 = svld1_u16(pg16_first_4, a_offset5);

  187.       c6 = svld1_u16(pg16_first_4, a_offset6);

  188.       c7 = svld1_u16(pg16_first_4, a_offset7);

  189. 

  190.       t0 = svzip1_u16(c0, c1);

  191.       t1 = svzip1_u16(c2, c3);

  192.       t2 = svzip1_u16(c4, c5);

  193.       t3 = svzip1_u16(c6, c7);

  194. 

  195.       m00 = svzip1_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  196.       m01 = svzip1_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  197.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  198.                                   st_offsets_0, svreinterpret_u64_u32(m00));

  199.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  200.                                   st_offsets_1, svreinterpret_u64_u32(m01));

  201. 

  202.       a_offset0 += 8 * lda;

  203.       a_offset1 += 8 * lda;

  204.       a_offset2 += 8 * lda;

  205.       a_offset3 += 8 * lda;

  206.       a_offset4 += 8 * lda;

  207.       a_offset5 += 8 * lda;

  208.       a_offset6 += 8 * lda;

  209.       a_offset7 += 8 * lda;

  210. 

  211.       b_offset0 += 32;

  212.     }

  213. 

  214.     if (rest) {

  215.       c0 = svld1_u16(pg16_first_4, a_offset0); // rest >= 1

  216.       c1 = (rest >= 2 ? svld1_u16(pg16_first_4, a_offset1) : svdup_u16(0));

  217.       c2 = (rest >= 3 ? svld1_u16(pg16_first_4, a_offset2) : svdup_u16(0));

  218.       c3 = (rest >= 4 ? svld1_u16(pg16_first_4, a_offset3) : svdup_u16(0));

  219.       c4 = (rest >= 5 ? svld1_u16(pg16_first_4, a_offset4) : svdup_u16(0));

  220.       c5 = (rest >= 6 ? svld1_u16(pg16_first_4, a_offset5) : svdup_u16(0));

  221.       c6 = (rest == 7 ? svld1_u16(pg16_first_4, a_offset6) : svdup_u16(0));

  222.       c7 = (svdup_u16(0));

  223. 

  224.       t0 = svzip1_u16(c0, c1);

  225.       t1 = svzip1_u16(c2, c3);

  226.       t2 = svzip1_u16(c4, c5);

  227.       t3 = svzip1_u16(c6, c7);

  228. 

  229.       m00 = svzip1_u32(svreinterpret_u32_u16(t0), svreinterpret_u32_u16(t1));

  230.       m01 = svzip1_u32(svreinterpret_u32_u16(t2), svreinterpret_u32_u16(t3));

  231. 

  232.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  233.                                   st_offsets_0, svreinterpret_u64_u32(m00));

  234.       svst1_scatter_u64offset_u64(pg64_first_4, (uint64_t *)b_offset0,

  235.                                   st_offsets_1, svreinterpret_u64_u32(m01));

  236.     }

  237.   }

  238. 

  239.   if (n & 2) {

  240.     a_offset0 = a_offset;

  241.     a_offset1 = a_offset0 + lda;

  242.     a_offset2 = a_offset1 + lda;

  243.     a_offset3 = a_offset2 + lda;

  244.     a_offset4 = a_offset3 + lda;

  245.     a_offset5 = a_offset4 + lda;

  246.     a_offset6 = a_offset5 + lda;

  247.     a_offset7 = a_offset6 + lda;

  248.     a_offset += 2;

  249. 

  250.     b_offset0 = b_offset;

  251.     b_offset1 = b_offset0 + 8;

  252. 

  253.     b_offset += 2 * pad_m;

  254. 

  255.     for (BLASLONG i = 0; i < m / 8; i++) {

  256.       for (BLASLONG line = 0; line < 2; line++) {

  257.         b_offset0[line * 4] = a_offset0[line];

  258.         b_offset0[line * 4 + 1] = a_offset1[line];

  259.         b_offset0[line * 4 + 2] = a_offset2[line];

  260.         b_offset0[line * 4 + 3] = a_offset3[line];

  261. 

  262.         b_offset1[line * 4] = a_offset4[line];

  263.         b_offset1[line * 4 + 1] = a_offset5[line];

  264.         b_offset1[line * 4 + 2] = a_offset6[line];

  265.         b_offset1[line * 4 + 3] = a_offset7[line];

  266.       }

  267.       b_offset0 += 16;

  268.       b_offset1 += 16;

  269. 

  270.       a_offset0 += 8 * lda;

  271.       a_offset1 += 8 * lda;

  272.       a_offset2 += 8 * lda;

  273.       a_offset3 += 8 * lda;

  274.       a_offset4 += 8 * lda;

  275.       a_offset5 += 8 * lda;

  276.       a_offset6 += 8 * lda;

  277.       a_offset7 += 8 * lda;

  278.     }

  279. 

  280.     if (rest) {

  281.       for (BLASLONG line = 0; line < 2; line++) {

  282.         b_offset0[line * 4] = a_offset0[line];

  283.         b_offset0[line * 4 + 1] = rest == 1 ? 0 : a_offset1[line];

  284.         b_offset0[line * 4 + 2] = rest <= 2 ? 0 : a_offset2[line];

  285.         b_offset0[line * 4 + 3] = rest <= 3 ? 0 : a_offset3[line];

  286. 

  287.         b_offset1[line * 4] = rest <= 4 ? 0 : a_offset4[line];

  288.         b_offset1[line * 4 + 1] = rest <= 5 ? 0 : a_offset5[line];

  289.         b_offset1[line * 4 + 2] = rest <= 6 ? 0 : a_offset6[line];

  290.         b_offset1[line * 4 + 3] = 0;

  291.       }

  292.     }

  293.   }

  294. 

  295.   if (n & 1) {

  296.     a_offset0 = a_offset;

  297.     a_offset1 = a_offset0 + lda;

  298.     a_offset2 = a_offset1 + lda;

  299.     a_offset3 = a_offset2 + lda;

  300.     a_offset4 = a_offset3 + lda;

  301.     a_offset5 = a_offset4 + lda;

  302.     a_offset6 = a_offset5 + lda;

  303.     a_offset7 = a_offset6 + lda;

  304. 

  305.     for (BLASLONG i = 0; i < m / 8; i++) {

  306.       b_offset[0] = a_offset0[0];

  307.       b_offset[1] = a_offset1[0];

  308.       b_offset[2] = a_offset2[0];

  309.       b_offset[3] = a_offset3[0];

  310. 

  311.       b_offset[4] = 0;

  312.       b_offset[5] = 0;

  313.       b_offset[6] = 0;

  314.       b_offset[7] = 0;

  315. 

  316.       b_offset[8] = a_offset4[0];

  317.       b_offset[9] = a_offset5[0];

  318.       b_offset[10] = a_offset6[0];

  319.       b_offset[11] = a_offset7[0];

  320. 

  321.       b_offset[12] = 0;

  322.       b_offset[13] = 0;

  323.       b_offset[14] = 0;

  324.       b_offset[15] = 0;

  325. 

  326.       b_offset += 16;

  327.       a_offset0 += 8 * lda;

  328.       a_offset1 += 8 * lda;

  329.       a_offset2 += 8 * lda;

  330.       a_offset3 += 8 * lda;

  331.       a_offset4 += 8 * lda;

  332.       a_offset5 += 8 * lda;

  333.       a_offset6 += 8 * lda;

  334.       a_offset7 += 8 * lda;

  335.     }

  336. 

  337.     if (rest) {

  338.       b_offset[0] = *a_offset0;

  339.       b_offset[1] = rest == 1 ? 0 : *a_offset1;

  340.       b_offset[2] = rest <= 2 ? 0 : *a_offset2;

  341.       b_offset[3] = rest <= 3 ? 0 : *a_offset3;

  342. 

  343.       b_offset[4] = 0;

  344.       b_offset[5] = 0;

  345.       b_offset[6] = 0;

  346.       b_offset[7] = 0;

  347. 

  348.       b_offset[8] = rest <= 4 ? 0 : *a_offset4;

  349.       b_offset[9] = rest <= 5 ? 0 : *a_offset5;

  350.       b_offset[10] = rest <= 6 ? 0 : *a_offset6;

  351.       b_offset[11] = 0;

  352. 

  353.       b_offset[12] = 0;

  354.       b_offset[13] = 0;

  355.       b_offset[14] = 0;

  356.       b_offset[15] = 0;

  357.     }

  358.   }

  359. 

  360.   return 0;

  361. }