OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 855945befb
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '855945befb'
${ noResults }
OpenBLAS/kernel/zarch/sgemv_t_4.c

754 lines
22 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2019, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #include "common.h"

  29. 

  30. #define NBMAX 2048

  31. 

  32. static void sgemv_kernel_4x4(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {

  33.   register FLOAT *ap0 = ap[0];

  34.   register FLOAT *ap1 = ap[1];

  35.   register FLOAT *ap2 = ap[2];

  36.   register FLOAT *ap3 = ap[3];

  37. 

  38.   __asm__("vzero %%v0\n\t"

  39.     "vzero %%v1\n\t"

  40.     "vzero %%v2\n\t"

  41.     "vzero %%v3\n\t"

  42.     "vzero %%v4\n\t"

  43.     "vzero %%v5\n\t"

  44.     "vzero %%v6\n\t"

  45.     "vzero %%v7\n\t"

  46.     "xgr   %%r1,%%r1\n\t"

  47.     "lghi    %%r0,-32\n\t"

  48.     "ngr     %%r0,%[n]\n\t"

  49.     "ltgr    %%r0,%%r0\n\t"

  50.     "jz      1f\n\t"

  51.     "srlg  %%r0,%%r0,5\n\t"

  52.     "0:\n\t"

  53.     "pfd 1,1024(%%r1,%[ap0])\n\t"

  54.     "pfd 1,1024(%%r1,%[ap1])\n\t"

  55.     "pfd 1,1024(%%r1,%[ap2])\n\t"

  56.     "pfd 1,1024(%%r1,%[ap3])\n\t"

  57.     "pfd 1,1024(%%r1,%[x])\n\t"

  58.     "vl  %%v16,0(%%r1,%[x])\n\t"

  59.     "vl  %%v17,16(%%r1,%[x])\n\t"

  60.     "vl  %%v18,32(%%r1,%[x])\n\t"

  61.     "vl  %%v19,48(%%r1,%[x])\n\t"

  62.     "vl  %%v20,64(%%r1,%[x])\n\t"

  63.     "vl  %%v21,80(%%r1,%[x])\n\t"

  64.     "vl  %%v22,96(%%r1,%[x])\n\t"

  65.     "vl  %%v23,112(%%r1,%[x])\n\t"

  66.     "vl  %%v24,0(%%r1,%[ap0])\n\t"

  67.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  68.     "vl  %%v25,0(%%r1,%[ap1])\n\t"

  69.     "vfmasb   %%v1,%%v16,%%v25,%%v1\n\t"

  70.     "vl  %%v26,0(%%r1,%[ap2])\n\t"

  71.     "vfmasb   %%v2,%%v16,%%v26,%%v2\n\t"

  72.     "vl  %%v27,0(%%r1,%[ap3])\n\t"

  73.     "vfmasb   %%v3,%%v16,%%v27,%%v3\n\t"

  74.     "vl  %%v28,16(%%r1,%[ap0])\n\t"

  75.     "vfmasb   %%v4,%%v17,%%v28,%%v4\n\t"

  76.     "vl  %%v29,16(%%r1,%[ap1])\n\t"

  77.     "vfmasb   %%v5,%%v17,%%v29,%%v5\n\t"

  78.     "vl  %%v30,16(%%r1,%[ap2])\n\t"

  79.     "vfmasb   %%v6,%%v17,%%v30,%%v6\n\t"

  80.     "vl  %%v31,16(%%r1,%[ap3])\n\t"

  81.     "vfmasb   %%v7,%%v17,%%v31,%%v7\n\t"

  82.     "vl  %%v24,32(%%r1,%[ap0])\n\t"

  83.     "vfmasb   %%v0,%%v18,%%v24,%%v0\n\t"

  84.     "vl  %%v25,32(%%r1,%[ap1])\n\t"

  85.     "vfmasb   %%v1,%%v18,%%v25,%%v1\n\t"

  86.     "vl  %%v26,32(%%r1,%[ap2])\n\t"

  87.     "vfmasb   %%v2,%%v18,%%v26,%%v2\n\t"

  88.     "vl  %%v27,32(%%r1,%[ap3])\n\t"

  89.     "vfmasb   %%v3,%%v18,%%v27,%%v3\n\t"

  90.     "vl  %%v28,48(%%r1,%[ap0])\n\t"

  91.     "vfmasb   %%v4,%%v19,%%v28,%%v4\n\t"

  92.     "vl  %%v29,48(%%r1,%[ap1])\n\t"

  93.     "vfmasb   %%v5,%%v19,%%v29,%%v5\n\t"

  94.     "vl  %%v30,48(%%r1,%[ap2])\n\t"

  95.     "vfmasb   %%v6,%%v19,%%v30,%%v6\n\t"

  96.     "vl  %%v31,48(%%r1,%[ap3])\n\t"

  97.     "vfmasb   %%v7,%%v19,%%v31,%%v7\n\t"

  98.     "vl  %%v24,64(%%r1,%[ap0])\n\t"

  99.     "vfmasb   %%v0,%%v20,%%v24,%%v0\n\t"

  100.     "vl  %%v25,64(%%r1,%[ap1])\n\t"

  101.     "vfmasb   %%v1,%%v20,%%v25,%%v1\n\t"

  102.     "vl  %%v26,64(%%r1,%[ap2])\n\t"

  103.     "vfmasb   %%v2,%%v20,%%v26,%%v2\n\t"

  104.     "vl  %%v27,64(%%r1,%[ap3])\n\t"

  105.     "vfmasb   %%v3,%%v20,%%v27,%%v3\n\t"

  106.     "vl  %%v28,80(%%r1,%[ap0])\n\t"

  107.     "vfmasb   %%v4,%%v21,%%v28,%%v4\n\t"

  108.     "vl  %%v29,80(%%r1,%[ap1])\n\t"

  109.     "vfmasb   %%v5,%%v21,%%v29,%%v5\n\t"

  110.     "vl  %%v30,80(%%r1,%[ap2])\n\t"

  111.     "vfmasb   %%v6,%%v21,%%v30,%%v6\n\t"

  112.     "vl  %%v31,80(%%r1,%[ap3])\n\t"

  113.     "vfmasb   %%v7,%%v21,%%v31,%%v7\n\t"

  114.     "vl  %%v24,96(%%r1,%[ap0])\n\t"

  115.     "vfmasb   %%v0,%%v22,%%v24,%%v0\n\t"

  116.     "vl  %%v25,96(%%r1,%[ap1])\n\t"

  117.     "vfmasb   %%v1,%%v22,%%v25,%%v1\n\t"

  118.     "vl  %%v26,96(%%r1,%[ap2])\n\t"

  119.     "vfmasb   %%v2,%%v22,%%v26,%%v2\n\t"

  120.     "vl  %%v27,96(%%r1,%[ap3])\n\t"

  121.     "vfmasb   %%v3,%%v22,%%v27,%%v3\n\t"

  122.     "vl  %%v28,112(%%r1,%[ap0])\n\t"

  123.     "vfmasb   %%v4,%%v23,%%v28,%%v4\n\t"

  124.     "vl  %%v29,112(%%r1,%[ap1])\n\t"

  125.     "vfmasb   %%v5,%%v23,%%v29,%%v5\n\t"

  126.     "vl  %%v30,112(%%r1,%[ap2])\n\t"

  127.     "vfmasb   %%v6,%%v23,%%v30,%%v6\n\t"

  128.     "vl  %%v31,112(%%r1,%[ap3])\n\t"

  129.     "vfmasb   %%v7,%%v23,%%v31,%%v7\n\t"

  130.     "agfi   %%r1,128\n\t"

  131.     "brctg  %%r0,0b\n\t"

  132.     "1:\n\t"

  133.     "lghi    %%r0,28\n\t"

  134.     "ngr     %%r0,%[n]\n\t"

  135.     "ltgr    %%r0,%%r0\n\t"

  136.     "jz      3f\n\t"

  137.     "srlg  %%r0,%%r0,2\n\t"

  138.     "2:\n\t"

  139.     "vl  %%v16,0(%%r1,%[x])\n\t"

  140.     "vl  %%v24,0(%%r1,%[ap0])\n\t"

  141.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  142.     "vl  %%v25,0(%%r1,%[ap1])\n\t"

  143.     "vfmasb   %%v1,%%v16,%%v25,%%v1\n\t"

  144.     "vl  %%v26,0(%%r1,%[ap2])\n\t"

  145.     "vfmasb   %%v2,%%v16,%%v26,%%v2\n\t"

  146.     "vl  %%v27,0(%%r1,%[ap3])\n\t"

  147.     "vfmasb   %%v3,%%v16,%%v27,%%v3\n\t"

  148.     "agfi   %%r1,16\n\t"

  149.     "brctg  %%r0,2b\n\t"

  150.     "3:\n\t"

  151.     "vfasb  %%v0,%%v0,%%v4\n\t"

  152.     "vfasb  %%v1,%%v1,%%v5\n\t"

  153.     "vfasb  %%v2,%%v2,%%v6\n\t"

  154.     "vfasb  %%v3,%%v3,%%v7\n\t"

  155.     "veslg  %%v4,%%v0,32\n\t"

  156.     "vfasb  %%v0,%%v0,%%v4\n\t"

  157.     "vrepg  %%v4,%%v0,1\n\t"

  158.     "aebr   %%f0,%%f4\n\t"

  159.     "ste    %%f0,0(%[y])\n\t"

  160.     "veslg  %%v4,%%v1,32\n\t"

  161.     "vfasb  %%v1,%%v1,%%v4\n\t"

  162.     "vrepg  %%v4,%%v1,1\n\t"

  163.     "aebr   %%f1,%%f4\n\t"

  164.     "ste    %%f1,4(%[y])\n\t"

  165.     "veslg  %%v4,%%v2,32\n\t"

  166.     "vfasb  %%v2,%%v2,%%v4\n\t"

  167.     "vrepg  %%v4,%%v2,1\n\t"

  168.     "aebr   %%f2,%%f4\n\t"

  169.     "ste    %%f2,8(%[y])\n\t"

  170.     "veslg  %%v4,%%v3,32\n\t"

  171.     "vfasb  %%v3,%%v3,%%v4\n\t"

  172.     "vrepg  %%v4,%%v3,1\n\t"

  173.     "aebr   %%f3,%%f4\n\t"

  174.     "ste    %%f3,12(%[y])"

  175.     : "=m"(*(FLOAT (*)[4]) y)

  176.     : [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),

  177.        "m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),

  178.        "m"(*(const FLOAT (*)[n]) ap2),[ap2] "a"(ap2),

  179.        "m"(*(const FLOAT (*)[n]) ap3),[ap3] "a"(ap3),

  180.        "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)

  181.     : "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",

  182.        "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",

  183.        "v26", "v27", "v28", "v29", "v30", "v31");

  184. }

  185. 

  186. static void sgemv_kernel_4x2(BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) {

  187.   register FLOAT *ap0 = ap[0];

  188.   register FLOAT *ap1 = ap[1];

  189. 

  190.   __asm__("vzero %%v0\n\t"

  191.     "vzero %%v1\n\t"

  192.     "vzero %%v2\n\t"

  193.     "vzero %%v3\n\t"

  194.     "vzero %%v4\n\t"

  195.     "vzero %%v5\n\t"

  196.     "vzero %%v6\n\t"

  197.     "vzero %%v7\n\t"

  198.     "xgr   %%r1,%%r1\n\t"

  199.     "lghi    %%r0,-32\n\t"

  200.     "ngr     %%r0,%[n]\n\t"

  201.     "ltgr    %%r0,%%r0\n\t"

  202.     "jz      1f\n\t"

  203.     "srlg  %%r0,%%r0,5\n\t"

  204.     "0:\n\t"

  205.     "pfd 1,1024(%%r1,%[ap0])\n\t"

  206.     "pfd 1,1024(%%r1,%[ap1])\n\t"

  207.     "pfd 1,1024(%%r1,%[x])\n\t"

  208.     "vl  %%v16,0(%%r1,%[x])\n\t"

  209.     "vl  %%v17,16(%%r1,%[x])\n\t"

  210.     "vl  %%v18,32(%%r1,%[x])\n\t"

  211.     "vl  %%v19,48(%%r1,%[x])\n\t"

  212.     "vl  %%v20,64(%%r1,%[x])\n\t"

  213.     "vl  %%v21,80(%%r1,%[x])\n\t"

  214.     "vl  %%v22,96(%%r1,%[x])\n\t"

  215.     "vl  %%v23,112(%%r1,%[x])\n\t"

  216.     "vl  %%v24,0(%%r1,%[ap0])\n\t"

  217.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  218.     "vl  %%v25,0(%%r1,%[ap1])\n\t"

  219.     "vfmasb   %%v1,%%v16,%%v25,%%v1\n\t"

  220.     "vl  %%v26,16(%%r1,%[ap0])\n\t"

  221.     "vfmasb   %%v2,%%v17,%%v26,%%v2\n\t"

  222.     "vl  %%v27,16(%%r1,%[ap1])\n\t"

  223.     "vfmasb   %%v3,%%v17,%%v27,%%v3\n\t"

  224.     "vl  %%v28,32(%%r1,%[ap0])\n\t"

  225.     "vfmasb   %%v4,%%v18,%%v28,%%v4\n\t"

  226.     "vl  %%v29,32(%%r1,%[ap1])\n\t"

  227.     "vfmasb   %%v5,%%v18,%%v29,%%v5\n\t"

  228.     "vl  %%v30,48(%%r1,%[ap0])\n\t"

  229.     "vfmasb   %%v6,%%v19,%%v30,%%v6\n\t"

  230.     "vl  %%v31,48(%%r1,%[ap1])\n\t"

  231.     "vfmasb   %%v7,%%v19,%%v31,%%v7\n\t"

  232.     "vl  %%v24,64(%%r1,%[ap0])\n\t"

  233.     "vfmasb   %%v0,%%v20,%%v24,%%v0\n\t"

  234.     "vl  %%v25,64(%%r1,%[ap1])\n\t"

  235.     "vfmasb   %%v1,%%v20,%%v25,%%v1\n\t"

  236.     "vl  %%v26,80(%%r1,%[ap0])\n\t"

  237.     "vfmasb   %%v2,%%v21,%%v26,%%v2\n\t"

  238.     "vl  %%v27,80(%%r1,%[ap1])\n\t"

  239.     "vfmasb   %%v3,%%v21,%%v27,%%v3\n\t"

  240.     "vl  %%v28,96(%%r1,%[ap0])\n\t"

  241.     "vfmasb   %%v4,%%v22,%%v28,%%v4\n\t"

  242.     "vl  %%v29,96(%%r1,%[ap1])\n\t"

  243.     "vfmasb   %%v5,%%v22,%%v29,%%v5\n\t"

  244.     "vl  %%v30,112(%%r1,%[ap0])\n\t"

  245.     "vfmasb   %%v6,%%v23,%%v30,%%v6\n\t"

  246.     "vl  %%v31,112(%%r1,%[ap1])\n\t"

  247.     "vfmasb   %%v7,%%v23,%%v31,%%v7\n\t"

  248.     "agfi   %%r1,128\n\t"

  249.     "brctg  %%r0,0b\n\t"

  250.     "1:\n\t"

  251.     "lghi    %%r0,28\n\t"

  252.     "ngr     %%r0,%[n]\n\t"

  253.     "ltgr    %%r0,%%r0\n\t"

  254.     "jz      3f\n\t"

  255.     "srlg  %%r0,%%r0,2\n\t"

  256.     "2:\n\t"

  257.     "vl  %%v16,0(%%r1,%[x])\n\t"

  258.     "vl  %%v24,0(%%r1,%[ap0])\n\t"

  259.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  260.     "vl  %%v25,0(%%r1,%[ap1])\n\t"

  261.     "vfmasb   %%v1,%%v16,%%v25,%%v1\n\t"

  262.     "agfi   %%r1,16\n\t"

  263.     "brctg  %%r0,2b\n\t"

  264.     "3:\n\t"

  265.     "vfasb  %%v0,%%v0,%%v2\n\t"

  266.     "vfasb  %%v0,%%v0,%%v4\n\t"

  267.     "vfasb  %%v0,%%v0,%%v6\n\t"

  268.     "vfasb  %%v1,%%v1,%%v3\n\t"

  269.     "vfasb  %%v1,%%v1,%%v5\n\t"

  270.     "vfasb  %%v1,%%v1,%%v7\n\t"

  271.     "veslg  %%v2,%%v0,32\n\t"

  272.     "vfasb  %%v0,%%v0,%%v2\n\t"

  273.     "vrepg  %%v2,%%v0,1\n\t"

  274.     "aebr   %%f0,%%f2\n\t"

  275.     "ste    %%f0,0(%[y])\n\t"

  276.     "veslg  %%v2,%%v1,32\n\t"

  277.     "vfasb  %%v1,%%v1,%%v2\n\t"

  278.     "vrepg  %%v2,%%v1,1\n\t"

  279.     "aebr   %%f1,%%f2\n\t"

  280.     "ste    %%f1,4(%[y])"

  281.     : "=m"(*(FLOAT (*)[2]) y)

  282.     : [y] "a"(y), "m"(*(const FLOAT (*)[n]) ap0),[ap0] "a"(ap0),

  283.        "m"(*(const FLOAT (*)[n]) ap1),[ap1] "a"(ap1),

  284.        "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)

  285.     : "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",

  286.        "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",

  287.        "v26", "v27", "v28", "v29", "v30", "v31");

  288. }

  289. 

  290. static void sgemv_kernel_4x1(BLASLONG n, FLOAT *a0, FLOAT *x, FLOAT *y) {

  291.   __asm__("vzero %%v0\n\t"

  292.     "vzero %%v1\n\t"

  293.     "vzero %%v2\n\t"

  294.     "vzero %%v3\n\t"

  295.     "vzero %%v4\n\t"

  296.     "vzero %%v5\n\t"

  297.     "vzero %%v6\n\t"

  298.     "vzero %%v7\n\t"

  299.     "xgr   %%r1,%%r1\n\t"

  300.     "lghi    %%r0,-32\n\t"

  301.     "ngr     %%r0,%[n]\n\t"

  302.     "ltgr    %%r0,%%r0\n\t"

  303.     "jz      1f\n\t"

  304.     "srlg  %%r0,%%r0,5\n\t"

  305.     "0:\n\t"

  306.     "pfd 1,1024(%%r1,%[a0])\n\t"

  307.     "pfd 1,1024(%%r1,%[x])\n\t"

  308.     "vl  %%v16,0(%%r1,%[x])\n\t"

  309.     "vl  %%v17,16(%%r1,%[x])\n\t"

  310.     "vl  %%v18,32(%%r1,%[x])\n\t"

  311.     "vl  %%v19,48(%%r1,%[x])\n\t"

  312.     "vl  %%v20,64(%%r1,%[x])\n\t"

  313.     "vl  %%v21,80(%%r1,%[x])\n\t"

  314.     "vl  %%v22,96(%%r1,%[x])\n\t"

  315.     "vl  %%v23,112(%%r1,%[x])\n\t"

  316.     "vl  %%v24,0(%%r1,%[a0])\n\t"

  317.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  318.     "vl  %%v25,16(%%r1,%[a0])\n\t"

  319.     "vfmasb   %%v1,%%v17,%%v25,%%v1\n\t"

  320.     "vl  %%v26,32(%%r1,%[a0])\n\t"

  321.     "vfmasb   %%v2,%%v18,%%v26,%%v2\n\t"

  322.     "vl  %%v27,48(%%r1,%[a0])\n\t"

  323.     "vfmasb   %%v3,%%v19,%%v27,%%v3\n\t"

  324.     "vl  %%v28,64(%%r1,%[a0])\n\t"

  325.     "vfmasb   %%v4,%%v20,%%v28,%%v4\n\t"

  326.     "vl  %%v29,80(%%r1,%[a0])\n\t"

  327.     "vfmasb   %%v5,%%v21,%%v29,%%v5\n\t"

  328.     "vl  %%v30,96(%%r1,%[a0])\n\t"

  329.     "vfmasb   %%v6,%%v22,%%v30,%%v6\n\t"

  330.     "vl  %%v31,112(%%r1,%[a0])\n\t"

  331.     "vfmasb   %%v7,%%v23,%%v31,%%v7\n\t"

  332.     "agfi   %%r1,128\n\t"

  333.     "brctg  %%r0,0b\n\t"

  334.     "1:\n\t"

  335.     "lghi    %%r0,28\n\t"

  336.     "ngr     %%r0,%[n]\n\t"

  337.     "ltgr    %%r0,%%r0\n\t"

  338.     "jz      3f\n\t"

  339.     "srlg  %%r0,%%r0,2\n\t"

  340.     "2:\n\t"

  341.     "vl  %%v16,0(%%r1,%[x])\n\t"

  342.     "vl  %%v24,0(%%r1,%[a0])\n\t"

  343.     "vfmasb   %%v0,%%v16,%%v24,%%v0\n\t"

  344.     "agfi   %%r1,16\n\t"

  345.     "brctg  %%r0,2b\n\t"

  346.     "3:\n\t"

  347.     "vfasb  %%v0,%%v0,%%v1\n\t"

  348.     "vfasb  %%v0,%%v0,%%v2\n\t"

  349.     "vfasb  %%v0,%%v0,%%v3\n\t"

  350.     "vfasb  %%v0,%%v0,%%v4\n\t"

  351.     "vfasb  %%v0,%%v0,%%v5\n\t"

  352.     "vfasb  %%v0,%%v0,%%v6\n\t"

  353.     "vfasb  %%v0,%%v0,%%v7\n\t"

  354.     "veslg  %%v1,%%v0,32\n\t"

  355.     "vfasb  %%v0,%%v0,%%v1\n\t"

  356.     "vrepg  %%v1,%%v0,1\n\t"

  357.     "aebr   %%f0,%%f1\n\t"

  358.     "ste    %%f0,0(%[y])"

  359.     : "=m"(*(FLOAT (*)[1]) y)

  360.     : [y] "a"(y), "m"(*(const FLOAT (*)[n]) a0),[a0] "a"(a0),

  361.        "m"(*(const FLOAT (*)[n]) x),[x] "a"(x),[n] "r"(n)

  362.     : "cc", "r0", "r1", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",

  363.        "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",

  364.        "v26", "v27", "v28", "v29", "v30", "v31");

  365. }

  366. 

  367. static void copy_x(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_src) {

  368.   BLASLONG i;

  369.   for (i = 0; i < n; i++) {

  370.     dest[i] = *src;

  371.     src += inc_src;

  372.   }

  373. }

  374. 

  375. static void add_y_kernel_4(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest) {

  376.   __asm__("vlrepf %%v0,%[da]\n\t"

  377.     "xgr   %%r1,%%r1\n\t"

  378.     "lghi    %%r0,-32\n\t"

  379.     "ngr     %%r0,%[n]\n\t"

  380.     "ltgr    %%r0,%%r0\n\t"

  381.     "jz      1f\n\t"

  382.     "srlg  %%r0,%%r0,5\n\t"

  383.     "0:\n\t"

  384.     "pfd 1,1024(%%r1,%[src])\n\t"

  385.     "pfd 2,1024(%%r1,%[dest])\n\t"

  386.     "vl  %%v16,0(%%r1,%[src])\n\t"

  387.     "vl  %%v17,16(%%r1,%[src])\n\t"

  388.     "vl  %%v18,32(%%r1,%[src])\n\t"

  389.     "vl  %%v19,48(%%r1,%[src])\n\t"

  390.     "vl  %%v20,64(%%r1,%[src])\n\t"

  391.     "vl  %%v21,80(%%r1,%[src])\n\t"

  392.     "vl  %%v22,96(%%r1,%[src])\n\t"

  393.     "vl  %%v23,112(%%r1,%[src])\n\t"

  394.     "vl  %%v24, 0(%%r1,%[dest])\n\t"

  395.     "vfmasb   %%v24,%%v16,%%v0,%%v24\n\t"

  396.     "vst  %%v24, 0(%%r1,%[dest])\n\t"

  397.     "vl  %%v25, 16(%%r1,%[dest])\n\t"

  398.     "vfmasb   %%v25,%%v17,%%v0,%%v25\n\t"

  399.     "vst  %%v25, 16(%%r1,%[dest])\n\t"

  400.     "vl  %%v26, 32(%%r1,%[dest])\n\t"

  401.     "vfmasb   %%v26,%%v18,%%v0,%%v26\n\t"

  402.     "vst  %%v26, 32(%%r1,%[dest])\n\t"

  403.     "vl  %%v27, 48(%%r1,%[dest])\n\t"

  404.     "vfmasb   %%v27,%%v19,%%v0,%%v27\n\t"

  405.     "vst  %%v27, 48(%%r1,%[dest])\n\t"

  406.     "vl  %%v28, 64(%%r1,%[dest])\n\t"

  407.     "vfmasb   %%v28,%%v20,%%v0,%%v28\n\t"

  408.     "vst  %%v28, 64(%%r1,%[dest])\n\t"

  409.     "vl  %%v29, 80(%%r1,%[dest])\n\t"

  410.     "vfmasb   %%v29,%%v21,%%v0,%%v29\n\t"

  411.     "vst  %%v29, 80(%%r1,%[dest])\n\t"

  412.     "vl  %%v30, 96(%%r1,%[dest])\n\t"

  413.     "vfmasb   %%v30,%%v22,%%v0,%%v30\n\t"

  414.     "vst  %%v30, 96(%%r1,%[dest])\n\t"

  415.     "vl  %%v31, 112(%%r1,%[dest])\n\t"

  416.     "vfmasb   %%v31,%%v23,%%v0,%%v31\n\t"

  417.     "vst  %%v31, 112(%%r1,%[dest])\n\t"

  418.     "agfi   %%r1,128\n\t"

  419.     "brctg  %%r0,0b\n\t"

  420.     "1:\n\t"

  421.     "lghi    %%r0,28\n\t"

  422.     "ngr     %%r0,%[n]\n\t"

  423.     "ltgr    %%r0,%%r0\n\t"

  424.     "jz      3f\n\t"

  425.     "srlg  %%r0,%%r0,2\n\t"

  426.     "2:\n\t"

  427.     "vl  %%v16,0(%%r1,%[src])\n\t"

  428.     "vl  %%v24, 0(%%r1,%[dest])\n\t"

  429.     "vfmasb   %%v24,%%v16,%%v0,%%v24\n\t"

  430.     "vst  %%v24, 0(%%r1,%[dest])\n\t"

  431.     "agfi   %%r1,16\n\t"

  432.     "brctg  %%r0,2b\n\t"

  433.     "3:\n\t"

  434.     "nop 0"

  435.     : "+m"(*(FLOAT (*)[n]) dest)

  436.     : [dest] "a"(dest),[da] "Q"(da), "m"(*(const FLOAT (*)[n]) src),

  437.        [src] "a"(src),[n] "r"(n)

  438.     : "cc", "r0", "r1", "v0", "v16", "v17", "v18", "v19", "v20", "v21",

  439.        "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30",

  440.        "v31");

  441. }

  442. static void add_y(BLASLONG n, FLOAT da, FLOAT *src, FLOAT *dest,

  443.                   BLASLONG inc_dest) {

  444.   if (inc_dest == 1)

  445.     add_y_kernel_4(n, da, src, dest);

  446.   else {

  447.     BLASLONG i;

  448.     for (i = 0; i < n; i++) {

  449.       *dest += src[i] * da;

  450.       dest += inc_dest;

  451.     }

  452.   }

  453. }

  454. 

  455. int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT alpha, FLOAT *a,

  456.           BLASLONG lda, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y,

  457.           FLOAT *buffer) {

  458.   BLASLONG register i;

  459.   BLASLONG register j;

  460.   FLOAT *a_ptr;

  461.   FLOAT *x_ptr;

  462.   FLOAT *y_ptr;

  463.   BLASLONG n0;

  464.   BLASLONG n1;

  465.   BLASLONG m1;

  466.   BLASLONG m2;

  467.   BLASLONG m3;

  468.   BLASLONG n2;

  469.   FLOAT ybuffer[2] __attribute__ ((aligned(16)));

  470.   FLOAT *xbuffer;

  471.   FLOAT *ytemp;

  472. 

  473.   if (m < 1)

  474.     return (0);

  475.   if (n < 1)

  476.     return (0);

  477. 

  478.   xbuffer = buffer;

  479.   ytemp = buffer + (m < NBMAX ? m : NBMAX);

  480. 

  481.   n0 = n / NBMAX;

  482.   n1 = (n % NBMAX) >> 2;

  483.   n2 = n & 3;

  484. 

  485.   m3 = m & 3;

  486.   m1 = m & -4;

  487.   m2 = (m & (NBMAX - 1)) - m3;

  488. 

  489.   BLASLONG NB = NBMAX;

  490. 

  491.   while (NB == NBMAX) {

  492.     m1 -= NB;

  493.     if (m1 < 0) {

  494.       if (m2 == 0)

  495.         break;

  496.       NB = m2;

  497.     }

  498. 

  499.     y_ptr = y;

  500.     a_ptr = a;

  501.     x_ptr = x;

  502. 

  503.     if (inc_x == 1)

  504.       xbuffer = x_ptr;

  505.     else

  506.       copy_x(NB, x_ptr, xbuffer, inc_x);

  507. 

  508.     FLOAT *ap[4];

  509.     FLOAT *yp;

  510.     BLASLONG register lda4 = 4 * lda;

  511.     ap[0] = a_ptr;

  512.     ap[1] = a_ptr + lda;

  513.     ap[2] = ap[1] + lda;

  514.     ap[3] = ap[2] + lda;

  515. 

  516.     if (n0 > 0) {

  517.       BLASLONG nb1 = NBMAX / 4;

  518.       for (j = 0; j < n0; j++) {

  519. 

  520.         yp = ytemp;

  521.         for (i = 0; i < nb1; i++) {

  522.           sgemv_kernel_4x4(NB, ap, xbuffer, yp);

  523.           ap[0] += lda4;

  524.           ap[1] += lda4;

  525.           ap[2] += lda4;

  526.           ap[3] += lda4;

  527.           yp += 4;

  528.         }

  529.         add_y(nb1 * 4, alpha, ytemp, y_ptr, inc_y);

  530.         y_ptr += nb1 * inc_y * 4;

  531.         a_ptr += nb1 * lda4;

  532. 

  533.       }

  534. 

  535.     }

  536. 

  537.     yp = ytemp;

  538. 

  539.     for (i = 0; i < n1; i++) {

  540.       sgemv_kernel_4x4(NB, ap, xbuffer, yp);

  541.       ap[0] += lda4;

  542.       ap[1] += lda4;

  543.       ap[2] += lda4;

  544.       ap[3] += lda4;

  545.       yp += 4;

  546.     }

  547.     if (n1 > 0) {

  548.       add_y(n1 * 4, alpha, ytemp, y_ptr, inc_y);

  549.       y_ptr += n1 * inc_y * 4;

  550.       a_ptr += n1 * lda4;

  551.     }

  552. 

  553.     if (n2 & 2) {

  554. 

  555.       sgemv_kernel_4x2(NB, ap, xbuffer, ybuffer);

  556.       a_ptr += lda * 2;

  557.       *y_ptr += ybuffer[0] * alpha;

  558.       y_ptr += inc_y;

  559.       *y_ptr += ybuffer[1] * alpha;

  560.       y_ptr += inc_y;

  561. 

  562.     }

  563. 

  564.     if (n2 & 1) {

  565. 

  566.       sgemv_kernel_4x1(NB, a_ptr, xbuffer, ybuffer);

  567.       // a_ptr  += lda;

  568.       *y_ptr += ybuffer[0] * alpha;

  569.       // y_ptr  += inc_y;

  570. 

  571.     }

  572.     a += NB;

  573.     x += NB * inc_x;

  574.   }

  575. 

  576.   if (m3 == 0)

  577.     return (0);

  578. 

  579.   x_ptr = x;

  580.   a_ptr = a;

  581.   if (m3 == 3) {

  582.     FLOAT xtemp0 = *x_ptr * alpha;

  583.     x_ptr += inc_x;

  584.     FLOAT xtemp1 = *x_ptr * alpha;

  585.     x_ptr += inc_x;

  586.     FLOAT xtemp2 = *x_ptr * alpha;

  587. 

  588.     FLOAT *aj = a_ptr;

  589.     y_ptr = y;

  590. 

  591.     if (lda == 3 && inc_y == 1) {

  592. 

  593.       for (j = 0; j < (n & -4); j += 4) {

  594. 

  595.         y_ptr[j] += aj[0] * xtemp0 + aj[1] * xtemp1 + aj[2] * xtemp2;

  596.         y_ptr[j + 1] += aj[3] * xtemp0 + aj[4] * xtemp1 + aj[5] * xtemp2;

  597.         y_ptr[j + 2] += aj[6] * xtemp0 + aj[7] * xtemp1 + aj[8] * xtemp2;

  598.         y_ptr[j + 3] += aj[9] * xtemp0 + aj[10] * xtemp1 + aj[11] * xtemp2;

  599.         aj += 12;

  600.       }

  601. 

  602.       for (; j < n; j++) {

  603.         y_ptr[j] += aj[0] * xtemp0 + aj[1] * xtemp1 + aj[2] * xtemp2;

  604.         aj += 3;

  605.       }

  606. 

  607.     } else {

  608. 

  609.       if (inc_y == 1) {

  610. 

  611.         BLASLONG register lda2 = lda << 1;

  612.         BLASLONG register lda4 = lda << 2;

  613.         BLASLONG register lda3 = lda2 + lda;

  614. 

  615.         for (j = 0; j < (n & -4); j += 4) {

  616. 

  617.           y_ptr[j] += *aj * xtemp0 + *(aj + 1) * xtemp1 + *(aj + 2) * xtemp2;

  618.           y_ptr[j + 1] +=

  619.             *(aj + lda) * xtemp0 + *(aj + lda + 1) * xtemp1 + *(aj + lda +

  620.                                                                 2) * xtemp2;

  621.           y_ptr[j + 2] +=

  622.             *(aj + lda2) * xtemp0 + *(aj + lda2 + 1) * xtemp1 + *(aj + lda2 +

  623.                                                                   2) * xtemp2;

  624.           y_ptr[j + 3] +=

  625.             *(aj + lda3) * xtemp0 + *(aj + lda3 + 1) * xtemp1 + *(aj + lda3 +

  626.                                                                   2) * xtemp2;

  627.           aj += lda4;

  628.         }

  629. 

  630.         for (; j < n; j++) {

  631. 

  632.           y_ptr[j] += *aj * xtemp0 + *(aj + 1) * xtemp1 + *(aj + 2) * xtemp2;

  633.           aj += lda;

  634.         }

  635. 

  636.       } else {

  637. 

  638.         for (j = 0; j < n; j++) {

  639.           *y_ptr += *aj * xtemp0 + *(aj + 1) * xtemp1 + *(aj + 2) * xtemp2;

  640.           y_ptr += inc_y;

  641.           aj += lda;

  642.         }

  643. 

  644.       }

  645. 

  646.     }

  647.     return (0);

  648.   }

  649. 

  650.   if (m3 == 2) {

  651.     FLOAT xtemp0 = *x_ptr * alpha;

  652.     x_ptr += inc_x;

  653.     FLOAT xtemp1 = *x_ptr * alpha;

  654. 

  655.     FLOAT *aj = a_ptr;

  656.     y_ptr = y;

  657. 

  658.     if (lda == 2 && inc_y == 1) {

  659. 

  660.       for (j = 0; j < (n & -4); j += 4) {

  661.         y_ptr[j] += aj[0] * xtemp0 + aj[1] * xtemp1;

  662.         y_ptr[j + 1] += aj[2] * xtemp0 + aj[3] * xtemp1;

  663.         y_ptr[j + 2] += aj[4] * xtemp0 + aj[5] * xtemp1;

  664.         y_ptr[j + 3] += aj[6] * xtemp0 + aj[7] * xtemp1;

  665.         aj += 8;

  666. 

  667.       }

  668. 

  669.       for (; j < n; j++) {

  670.         y_ptr[j] += aj[0] * xtemp0 + aj[1] * xtemp1;

  671.         aj += 2;

  672.       }

  673. 

  674.     } else {

  675.       if (inc_y == 1) {

  676. 

  677.         BLASLONG register lda2 = lda << 1;

  678.         BLASLONG register lda4 = lda << 2;

  679.         BLASLONG register lda3 = lda2 + lda;

  680. 

  681.         for (j = 0; j < (n & -4); j += 4) {

  682. 

  683.           y_ptr[j] += *aj * xtemp0 + *(aj + 1) * xtemp1;

  684.           y_ptr[j + 1] += *(aj + lda) * xtemp0 + *(aj + lda + 1) * xtemp1;

  685.           y_ptr[j + 2] += *(aj + lda2) * xtemp0 + *(aj + lda2 + 1) * xtemp1;

  686.           y_ptr[j + 3] += *(aj + lda3) * xtemp0 + *(aj + lda3 + 1) * xtemp1;

  687.           aj += lda4;

  688.         }

  689. 

  690.         for (; j < n; j++) {

  691. 

  692.           y_ptr[j] += *aj * xtemp0 + *(aj + 1) * xtemp1;

  693.           aj += lda;

  694.         }

  695. 

  696.       } else {

  697.         for (j = 0; j < n; j++) {

  698.           *y_ptr += *aj * xtemp0 + *(aj + 1) * xtemp1;

  699.           y_ptr += inc_y;

  700.           aj += lda;

  701.         }

  702.       }

  703. 

  704.     }

  705.     return (0);

  706. 

  707.   }

  708. 

  709.   FLOAT xtemp = *x_ptr * alpha;

  710.   FLOAT *aj = a_ptr;

  711.   y_ptr = y;

  712.   if (lda == 1 && inc_y == 1) {

  713.     for (j = 0; j < (n & -4); j += 4) {

  714.       y_ptr[j] += aj[j] * xtemp;

  715.       y_ptr[j + 1] += aj[j + 1] * xtemp;

  716.       y_ptr[j + 2] += aj[j + 2] * xtemp;

  717.       y_ptr[j + 3] += aj[j + 3] * xtemp;

  718.     }

  719.     for (; j < n; j++) {

  720.       y_ptr[j] += aj[j] * xtemp;

  721.     }

  722. 

  723.   } else {

  724.     if (inc_y == 1) {

  725. 

  726.       BLASLONG register lda2 = lda << 1;

  727.       BLASLONG register lda4 = lda << 2;

  728.       BLASLONG register lda3 = lda2 + lda;

  729.       for (j = 0; j < (n & -4); j += 4) {

  730.         y_ptr[j] += *aj * xtemp;

  731.         y_ptr[j + 1] += *(aj + lda) * xtemp;

  732.         y_ptr[j + 2] += *(aj + lda2) * xtemp;

  733.         y_ptr[j + 3] += *(aj + lda3) * xtemp;

  734.         aj += lda4;

  735.       }

  736. 

  737.       for (; j < n; j++) {

  738.         y_ptr[j] += *aj * xtemp;

  739.         aj += lda;

  740.       }

  741. 

  742.     } else {

  743.       for (j = 0; j < n; j++) {

  744.         *y_ptr += *aj * xtemp;

  745.         y_ptr += inc_y;

  746.         aj += lda;

  747.       }

  748. 

  749.     }

  750.   }

  751. 

  752.   return (0);

  753. }