Browse Source
Merge pull request #4946 from XiWeiGu/la64_omatcopy_lasx
LoongArch64: Opt somatcopy with LASXtags/v0.3.29
Martin Kroeker
GitHub
11 months ago
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 951 additions and 1 deletions
Split View
Diff Options
-
+38 -1benchmark/Makefile
-
+122 -0benchmark/omatcopy.c
-
+5 -0kernel/loongarch64/KERNEL.LA464
-
+193 -0kernel/loongarch64/somatcopy_cn_lasx.c
-
+200 -0kernel/loongarch64/somatcopy_ct_lasx.c
-
+193 -0kernel/loongarch64/somatcopy_rn_lasx.c
-
+200 -0kernel/loongarch64/somatcopy_rt_lasx.c
+ 38
- 1
benchmark/Makefile
View File
| @@ -103,6 +103,7 @@ goto :: slinpack.goto dlinpack.goto clinpack.goto zlinpack.goto \ | |||
| sgetri.goto dgetri.goto cgetri.goto zgetri.goto \ | |||
| spotrf.goto dpotrf.goto cpotrf.goto zpotrf.goto \ | |||
| ssymm.goto dsymm.goto csymm.goto zsymm.goto \ | |||
| somatcopy.goto domatcopy.goto comatcopy.goto zomatcopy.goto \ | |||
| saxpby.goto daxpby.goto caxpby.goto zaxpby.goto $(GOTO_HALF_TARGETS) | |||
| acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \ | |||
| @@ -276,6 +277,7 @@ goto :: sgemm.goto dgemm.goto cgemm.goto zgemm.goto \ | |||
| samin.goto damin.goto camin.goto zamin.goto \ | |||
| smin.goto dmin.goto \ | |||
| saxpby.goto daxpby.goto caxpby.goto zaxpby.goto \ | |||
| somatcopy.goto domatcopy.goto comatcopy.goto zomatcopy.goto \ | |||
| snrm2.goto dnrm2.goto scnrm2.goto dznrm2.goto $(GOTO_LAPACK_TARGETS) $(GOTO_HALF_TARGETS) | |||
| acml :: slinpack.acml dlinpack.acml clinpack.acml zlinpack.acml \ | |||
| @@ -2906,6 +2908,29 @@ dznrm2.goto : dznrm2.$(SUFFIX) ../$(LIBNAME) | |||
| dznrm2.atlas : dznrm2.$(SUFFIX) | |||
| -$(CC) $(CFLAGS) -o $(@F) $^ $(LIBATLAS) $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) | |||
| ################################################################################################### | |||
| ############################################ SOMATCOPY ############################################ | |||
| somatcopy.goto : somatcopy.$(SUFFIX) ../$(LIBNAME) | |||
| $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm | |||
| ################################################################################################### | |||
| ############################################ DOMATCOPY ############################################ | |||
| domatcopy.goto : domatcopy.$(SUFFIX) ../$(LIBNAME) | |||
| $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm | |||
| ################################################################################################### | |||
| ############################################ COMATCOPY ############################################ | |||
| comatcopy.goto : comatcopy.$(SUFFIX) ../$(LIBNAME) | |||
| $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm | |||
| ################################################################################################### | |||
| ############################################ ZOMATCOPY ############################################ | |||
| zomatcopy.goto : zomatcopy.$(SUFFIX) ../$(LIBNAME) | |||
| $(CC) $(CFLAGS) -o $(@F) $^ $(CEXTRALIB) $(EXTRALIB) $(FEXTRALIB) -lm | |||
| ################################################################################################### | |||
| @@ -3435,6 +3460,18 @@ scnrm2.$(SUFFIX) : nrm2.c | |||
| dznrm2.$(SUFFIX) : nrm2.c | |||
| $(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^ | |||
| somatcopy.$(SUFFIX) : omatcopy.c | |||
| $(CC) $(CFLAGS) -c -UCOMPLEX -UDOUBLE -o $(@F) $^ | |||
| domatcopy.$(SUFFIX) : omatcopy.c | |||
| $(CC) $(CFLAGS) -c -UCOMPLEX -DDOUBLE -o $(@F) $^ | |||
| comatcopy.$(SUFFIX) : omatcopy.c | |||
| $(CC) $(CFLAGS) -c -DCOMPLEX -UDOUBLE -o $(@F) $^ | |||
| zomatcopy.$(SUFFIX) : omatcopy.c | |||
| $(CC) $(CFLAGS) -c -DCOMPLEX -DDOUBLE -o $(@F) $^ | |||
| smallscaling: smallscaling.c ../$(LIBNAME) | |||
| $(CC) $(CFLAGS) -o $(@F) $^ $(EXTRALIB) -fopenmp -lm -lpthread | |||
| @@ -3442,4 +3479,4 @@ smallscaling: smallscaling.c ../$(LIBNAME) | |||
| clean :: | |||
| @rm -f *.goto *.mkl *.acml *.atlas *.veclib *.essl smallscaling | |||
| include $(TOPDIR)/Makefile.tail | |||
| include $(TOPDIR)/Makefile.tail | |||
+ 122
- 0
benchmark/omatcopy.c
View File
| @@ -0,0 +1,122 @@ | |||
| /*************************************************************************** | |||
| Copyright (c) 2024, The OpenBLAS Project | |||
| All rights reserved. | |||
| Redistribution and use in source and binary forms, with or without | |||
| modification, are permitted provided that the following conditions are | |||
| met: | |||
| 1. Redistributions of source code must retain the above copyright | |||
| notice, this list of conditions and the following disclaimer. | |||
| 2. Redistributions in binary form must reproduce the above copyright | |||
| notice, this list of conditions and the following disclaimer in | |||
| the documentation and/or other materials provided with the | |||
| distribution. | |||
| 3. Neither the name of the OpenBLAS project nor the names of | |||
| its contributors may be used to endorse or promote products | |||
| derived from this software without specific prior written permission. | |||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
| *****************************************************************************/ | |||
| #include "bench.h" | |||
| #undef OMATCOPY | |||
| #ifndef COMPLEX | |||
| #ifdef DOUBLE | |||
| #define OMATCOPY BLASFUNC(domatcopy) | |||
| #else | |||
| #define OMATCOPY BLASFUNC(somatcopy) | |||
| #endif | |||
| #else | |||
| #ifdef DOUBLE | |||
| #define OMATCOPY BLASFUNC(zomatcopy) | |||
| #else | |||
| #define OMATCOPY BLASFUNC(comatcopy) | |||
| #endif | |||
| #endif | |||
| int main(int argc, char *argv[]){ | |||
| FLOAT *a, *b; | |||
| FLOAT alpha[] = {1.0, 0.0}; | |||
| char trans = 'N'; | |||
| char order = 'C'; | |||
| blasint crows, ccols, clda, cldb; | |||
| int loops = 1; | |||
| char *p; | |||
| int from = 1; | |||
| int to = 200; | |||
| int step = 1; | |||
| int i, j; | |||
| double time1, timeg; | |||
| argc--;argv++; | |||
| if (argc > 0) { from = atol(*argv); argc--; argv++; } | |||
| if (argc > 0) { to = MAX(atol(*argv), from); argc--; argv++; } | |||
| if (argc > 0) { step = atol(*argv); argc--; argv++; } | |||
| if ((p = getenv("OPENBLAS_TRANS"))) { | |||
| trans=*p; | |||
| } | |||
| if ((p = getenv("OPENBLAS_ORDER"))) { | |||
| order=*p; | |||
| } | |||
| TOUPPER(trans); | |||
| TOUPPER(order); | |||
| fprintf(stderr, "From : %3d To : %3d Step=%d : Trans=%c : Order=%c\n", from, to, step, trans, order); | |||
| p = getenv("OPENBLAS_LOOPS"); | |||
| if ( p != NULL ) { | |||
| loops = atoi(p); | |||
| } | |||
| if (( a = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL) { | |||
| fprintf(stderr,"Out of Memory!!\n");exit(1); | |||
| } | |||
| if (( b = (FLOAT *)malloc(sizeof(FLOAT) * to * to * COMPSIZE)) == NULL) { | |||
| fprintf(stderr,"Out of Memory!!\n");exit(1); | |||
| } | |||
| #ifdef __linux | |||
| srandom(getpid()); | |||
| #endif | |||
| for (i = 0; i < to * to * COMPSIZE; i++) { | |||
| a[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5; | |||
| } | |||
| for (i = 0; i < to * to * COMPSIZE; i++) { | |||
| b[i] = ((FLOAT) rand() / (FLOAT) RAND_MAX) - 0.5; | |||
| } | |||
| fprintf(stderr, " SIZE Flops Time\n"); | |||
| for (i = from; i <= to; i += step) { | |||
| cldb = clda = crows = ccols = i; | |||
| fprintf(stderr, " ROWS=%4d, COLS=%4d : ", (int)crows, (int)ccols); | |||
| begin(); | |||
| for (j=0; j<loops; j++) { | |||
| OMATCOPY (&order, &trans, &crows, &ccols, alpha, a, &clda, b, &cldb); | |||
| } | |||
| end(); | |||
| time1 = getsec(); | |||
| timeg = time1/loops; | |||
| fprintf(stderr, | |||
| " %10.2f MFlops %10.6f sec\n", | |||
| COMPSIZE * COMPSIZE * (double)ccols * (double)crows / timeg * 1.e-6, time1); | |||
| } | |||
| free(a); | |||
| free(b); | |||
| return 0; | |||
| } | |||
+ 5
- 0
kernel/loongarch64/KERNEL.LA464
View File
| @@ -172,4 +172,9 @@ DGEMM_SMALL_K_TN = dgemm_small_kernel_tn_lasx.S | |||
| DGEMM_SMALL_K_B0_TN = dgemm_small_kernel_tn_lasx.S | |||
| DGEMM_SMALL_K_TT = dgemm_small_kernel_tt_lasx.S | |||
| DGEMM_SMALL_K_B0_TT = dgemm_small_kernel_tt_lasx.S | |||
| SOMATCOPY_RT = somatcopy_rt_lasx.c | |||
| SOMATCOPY_RN = somatcopy_rn_lasx.c | |||
| SOMATCOPY_CT = somatcopy_ct_lasx.c | |||
| SOMATCOPY_CN = somatcopy_cn_lasx.c | |||
| endif | |||
+ 193
- 0
kernel/loongarch64/somatcopy_cn_lasx.c
View File
| @@ -0,0 +1,193 @@ | |||
| /*************************************************************************** | |||
| Copyright (c) 2024, The OpenBLAS Project | |||
| All rights reserved. | |||
| Redistribution and use in source and binary forms, with or without | |||
| modification, are permitted provided that the following conditions are | |||
| met: | |||
| 1. Redistributions of source code must retain the above copyright | |||
| notice, this list of conditions and the following disclaimer. | |||
| 2. Redistributions in binary form must reproduce the above copyright | |||
| notice, this list of conditions and the following disclaimer in | |||
| the documentation and/or other materials provided with the | |||
| distribution. | |||
| 3. Neither the name of the OpenBLAS project nor the names of | |||
| its contributors may be used to endorse or promote products | |||
| derived from this software without specific prior written permission. | |||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
| *****************************************************************************/ | |||
| #include "common.h" | |||
| #define SAVE_4x1(b1, b2, b3, b4) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \ | |||
| "vstelm.w $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x2(b1, b2, b3, b4) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \ | |||
| "vstelm.d $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x4(b1, b2, b3, b4) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t" \ | |||
| "vst $vr"#b3",%1,0;add.d %1,%1,%3;vst $vr"#b4",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x8(b1, b2, b3, b4) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;add.d %1,%1,%3;xvst $xr"#b4",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x16(b1, b2, b3, b4, b5, b6, b7, b8) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b5",%1,0;xvst $xr"#b6",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b7",%1,0;xvst $xr"#b8",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x16(b1, b2, b3, b4) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x8(b1, b2) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x4(b1, b2) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x2(b1, b2) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x1(b1, b2) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x16(b1, b2) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x8(b1) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x4(b1) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x2(b1) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x1(b1) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define COPY_4x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| "xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \ | |||
| SAVE_4x16(0,4,1,5,2,6,3,7) | |||
| #define COPY_4x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| SAVE_4x8(0,1,2,3) | |||
| #define COPY_4x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| SAVE_4x4(0,1,2,3) | |||
| #define COPY_4x2 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| SAVE_4x2(0,1,2,3) | |||
| #define COPY_4x1 \ | |||
| "fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15;fmul.s $f2,$f2,$f15;fmul.s $f3,$f3,$f15 \n\t" \ | |||
| SAVE_4x1(0,1,2,3) | |||
| #define COPY_2x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| SAVE_2x16(0,2,1,3) | |||
| #define COPY_2x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \ | |||
| SAVE_2x8(0,1) | |||
| #define COPY_2x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \ | |||
| SAVE_2x4(0,1) | |||
| #define COPY_2x2 \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \ | |||
| SAVE_2x2(0,1) | |||
| #define COPY_2x1 \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15; \n\t" \ | |||
| SAVE_2x1(0,1) | |||
| #define COPY_1x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr1,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \ | |||
| SAVE_1x16(0,1) | |||
| #define COPY_1x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15; \n\t" \ | |||
| SAVE_1x8(0) | |||
| #define COPY_1x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_1x4(0) | |||
| #define COPY_1x2 \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_1x2(0) | |||
| #define COPY_1x1 \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15; \n\t" \ | |||
| SAVE_1x1(0) | |||
| #define ROWS_OF_BLOCK 128 | |||
| #define COMPUTE(ndim) \ | |||
| src = src_base; dst = dst_base; \ | |||
| __asm__ __volatile__( \ | |||
| "xvldrepl.w $xr15, %6, 0 \n\t" \ | |||
| "srli.d $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"3f \n\t" \ | |||
| #ndim"4: \n\t" \ | |||
| COPY_4x##ndim \ | |||
| "addi.d $r6, $r6, -1 \n\t" \ | |||
| "bnez $r6, "#ndim"4b \n\t" \ | |||
| #ndim"3: \n\t" \ | |||
| "andi $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"1f \n\t" \ | |||
| #ndim"2: \n\t" \ | |||
| COPY_2x##ndim \ | |||
| #ndim"1: \n\t" \ | |||
| "andi $r6, %5, 1 \n\t" \ | |||
| "beqz $r6, "#ndim"0f \n\t" \ | |||
| COPY_1x##ndim \ | |||
| #ndim"0: \n\t" \ | |||
| :"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \ | |||
| :"r"(num_cols),"r"(&ALPHA) \ | |||
| :"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \ | |||
| ); | |||
| int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){ | |||
| float *src, *dst, *dst_tmp=0, *src_base, *dst_base; | |||
| uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_cols = 0; | |||
| BLASLONG rows_left, cols_done; float ALPHA = alpha; | |||
| if (ALPHA == 0.0) { | |||
| dst_base = b; | |||
| for (rows_left = rows; rows_left > 0; rows_left--) {memset(dst_base, 0, cols * sizeof(float)); dst_base += ldb;} | |||
| return 0; | |||
| } | |||
| for (cols_done = 0; cols_done < cols; cols_done += num_cols) { | |||
| num_cols = cols - cols_done; | |||
| if (num_cols > ROWS_OF_BLOCK) num_cols = ROWS_OF_BLOCK; | |||
| rows_left = rows; src_base = a + (int64_t)lda * (int64_t)cols_done; dst_base = b + cols_done; | |||
| for (;rows_left > 15; rows_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16;} | |||
| for (;rows_left > 7; rows_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8;} | |||
| for (;rows_left > 3; rows_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4;} | |||
| for (;rows_left > 1; rows_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2;} | |||
| if (rows_left > 0) {COMPUTE(1) src_base ++; dst_base ++;} | |||
| } | |||
| } | |||
+ 200
- 0
kernel/loongarch64/somatcopy_ct_lasx.c
View File
| @@ -0,0 +1,200 @@ | |||
| /*************************************************************************** | |||
| Copyright (c) 2024, The OpenBLAS Project | |||
| All rights reserved. | |||
| Redistribution and use in source and binary forms, with or without | |||
| modification, are permitted provided that the following conditions are | |||
| met: | |||
| 1. Redistributions of source code must retain the above copyright | |||
| notice, this list of conditions and the following disclaimer. | |||
| 2. Redistributions in binary form must reproduce the above copyright | |||
| notice, this list of conditions and the following disclaimer in | |||
| the documentation and/or other materials provided with the | |||
| distribution. | |||
| 3. Neither the name of the OpenBLAS project nor the names of | |||
| its contributors may be used to endorse or promote products | |||
| derived from this software without specific prior written permission. | |||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
| *****************************************************************************/ | |||
| #include "common.h" | |||
| #define SAVE1x4(c1) \ | |||
| "vfmul.s $vr"#c1",$vr"#c1",$vr15;vstelm.w $vr"#c1",%4,0,0;add.d %4,%4,%3;\n\t" \ | |||
| "vstelm.w $vr"#c1",%4,0,1;add.d %4,%4,%3;vstelm.w $vr"#c1",%4,0,2;add.d %4,%4,%3; \n\t" \ | |||
| "vstelm.w $vr"#c1",%4,0,3;add.d %4,%4,%3;\n\t" | |||
| #define SAVE_2x4(c1, c2, t1, t2) \ | |||
| "vilvl.w $vr"#t1",$vr"#c2",$vr"#c1";vfmul.s $vr"#t1",$vr"#t1",$vr15; \n\t" \ | |||
| "vstelm.d $vr"#t1",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t1",%4,0,1;add.d %4,%4,%3 \n\t" \ | |||
| "vilvh.w $vr"#t2",$vr"#c2",$vr"#c1";vfmul.s $vr"#t2",$vr"#t2",$vr15; \n\t" \ | |||
| "vstelm.d $vr"#t2",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t2",%4,0,1;add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x1(b1) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x2(b1, b2) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x4(b1, b2, b3, b4) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t" \ | |||
| "vst $vr"#b3",%4,0; add.d %4,%4,%3; vst $vr"#b4",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x8(b1, b2, b3, b4) \ | |||
| SAVE_4x4(b1, b2, b3, b4) \ | |||
| "xvpermi.q $xr"#b1",$xr"#b1",1; xvpermi.q $xr"#b2",$xr"#b2",1; xvpermi.q $xr"#b3",$xr"#b3",1; xvpermi.q $xr"#b4",$xr"#b4",1; \n\t" \ | |||
| SAVE_4x4(b1, b2, b3, b4) | |||
| #define TRANS_4x4(a1, a2, a3, a4, t1, t2, t3, t4) \ | |||
| "vilvl.w $vr"#t1",$vr"#a2",$vr"#a1";vilvh.w $vr"#t2",$vr"#a2",$vr"#a1"; \n\t" \ | |||
| "vilvl.w $vr"#t3",$vr"#a4",$vr"#a3";vilvh.w $vr"#t4",$vr"#a4",$vr"#a3"; \n\t" \ | |||
| "vilvl.d $vr"#a1",$vr"#t3",$vr"#t1";vilvh.d $vr"#a2",$vr"#t3",$vr"#t1"; \n\t" \ | |||
| "vilvl.d $vr"#a3",$vr"#t4",$vr"#t2";vilvh.d $vr"#a4",$vr"#t4",$vr"#t2"; \n\t" | |||
| #define TRANS_4x8(a1, a2, a3, a4, t1, t2, t3, t4) \ | |||
| "xvilvl.w $xr"#t1",$xr"#a2",$xr"#a1"; xvilvh.w $xr"#t2",$xr"#a2",$xr"#a1"; \n\t" \ | |||
| "xvilvl.w $xr"#t3",$xr"#a4",$xr"#a3"; xvilvh.w $xr"#t4",$xr"#a4",$xr"#a3"; \n\t" \ | |||
| "xvilvl.d $xr"#a1",$xr"#t3",$xr"#t1"; xvilvh.d $xr"#a2",$xr"#t3",$xr"#t1"; \n\t" \ | |||
| "xvilvl.d $xr"#a3",$xr"#t4",$xr"#t2"; xvilvh.d $xr"#a4",$xr"#t4",$xr"#t2"; \n\t" | |||
| #define COPY_4x16 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| "xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \ | |||
| TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3) \ | |||
| TRANS_4x8(4,5,6,7,8,9,10,11) SAVE_4x8(4,5,6,7) | |||
| #define COPY_4x8 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3) | |||
| #define COPY_4x4 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x4(0,1,2,3) | |||
| #define COPY_4x2 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x2(0,1) | |||
| #define COPY_4x1 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.w $xr0,$xr1,1;xvinsve0.w $xr0,$xr2,2;xvinsve0.w $xr0,$xr3,3; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_4x1(0) | |||
| #define COPY_2x16 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvpermi.q $xr4,$xr0,1;xvpermi.q $xr6,$xr2,1;xvpermi.q $xr5,$xr1,1;xvpermi.q $xr7,$xr3,1; \n\t" \ | |||
| SAVE_2x4(0,1,8,9) SAVE_2x4(4,5,8,9) SAVE_2x4(2,3,8,9) SAVE_2x4(6,7,8,9) | |||
| #define COPY_2x8 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvpermi.q $xr2,$xr0,1;xvpermi.q $xr3,$xr1,1; \n\t" \ | |||
| SAVE_2x4(0,1,4,5) SAVE_2x4(2,3,4,5) | |||
| #define COPY_2x4 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| SAVE_2x4(0,1,4,5) | |||
| #define COPY_2x2 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.d $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15;vshuf4i.w $vr0,$vr0,0xd8 \n\t" \ | |||
| "vstelm.d $vr0,%4,0,0;add.d %4,%4,%3;vstelm.d $vr0,%4,0,1 \n\t" | |||
| #define COPY_2x1 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.w $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| "vstelm.d $vr0,%4,0,0; \n\t" | |||
| #define COPY_1x16 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \ | |||
| "vld $vr1,%0,32;" SAVE1x4(1) "vld $vr2,%0,48;" SAVE1x4(2) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x8 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x4 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x2 \ | |||
| "move %4,%1;fld.d $f1,%0,0;add.d %0,%0,%2;vfmul.s $vr1,$vr1,$vr15;vstelm.w $vr1,%4,0,0;add.d %4,%4,%3;vstelm.w $vr1,%4,0,1;\n\t" \ | |||
| "addi.d %1,%1,4;\n\t" | |||
| #define COPY_1x1 \ | |||
| "fld.s $f1,%0,0;fmul.s $f1,$f1,$f15;fst.s $f1,%1,0;add.d %0,%0,%2;addi.d %1,%1,4;\n\t" | |||
| #define ROWS_OF_BLOCK 128 | |||
| #define COMPUTE(ndim) \ | |||
| src = src_base; dst = dst_base; \ | |||
| __asm__ __volatile__( \ | |||
| "xvldrepl.w $xr15, %6, 0 \n\t" \ | |||
| "srli.d $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"3f \n\t" \ | |||
| #ndim"4: \n\t" \ | |||
| COPY_4x##ndim \ | |||
| "addi.d $r6, $r6, -1 \n\t" \ | |||
| "bnez $r6, "#ndim"4b \n\t" \ | |||
| #ndim"3: \n\t" \ | |||
| "andi $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"1f \n\t" \ | |||
| #ndim"2: \n\t" \ | |||
| COPY_2x##ndim \ | |||
| #ndim"1: \n\t" \ | |||
| "andi $r6, %5, 1 \n\t" \ | |||
| "beqz $r6, "#ndim"0f \n\t" \ | |||
| COPY_1x##ndim \ | |||
| #ndim"0: \n\t" \ | |||
| :"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \ | |||
| :"r"(num_cols),"r"(&ALPHA) \ | |||
| :"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \ | |||
| ); | |||
| int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){ | |||
| float *src, *dst, *dst_tmp=0, *src_base, *dst_base; | |||
| uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_cols = 0; | |||
| BLASLONG rows_left, cols_done; float ALPHA = alpha; | |||
| if (ALPHA == 0.0) { | |||
| dst_base = b; | |||
| for (rows_left = rows; rows_left > 0; rows_left--) {memset(dst_base, 0, cols * sizeof(float)); dst_base += ldb;} | |||
| return 0; | |||
| } | |||
| for (cols_done = 0; cols_done < cols; cols_done += num_cols) { | |||
| num_cols = cols - cols_done; | |||
| if (num_cols > ROWS_OF_BLOCK) num_cols = ROWS_OF_BLOCK; | |||
| rows_left = rows; src_base = a + (int64_t)lda * (int64_t)cols_done; dst_base = b + cols_done; | |||
| for (;rows_left > 15; rows_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16 * ldb;} | |||
| for (;rows_left > 7; rows_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8 * ldb;} | |||
| for (;rows_left > 3; rows_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4 * ldb;} | |||
| for (;rows_left > 1; rows_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2 * ldb;} | |||
| if (rows_left > 0) {COMPUTE(1) src_base ++; dst_base += ldb;} | |||
| } | |||
| } | |||
+ 193
- 0
kernel/loongarch64/somatcopy_rn_lasx.c
View File
| @@ -0,0 +1,193 @@ | |||
| /*************************************************************************** | |||
| Copyright (c) 2024, The OpenBLAS Project | |||
| All rights reserved. | |||
| Redistribution and use in source and binary forms, with or without | |||
| modification, are permitted provided that the following conditions are | |||
| met: | |||
| 1. Redistributions of source code must retain the above copyright | |||
| notice, this list of conditions and the following disclaimer. | |||
| 2. Redistributions in binary form must reproduce the above copyright | |||
| notice, this list of conditions and the following disclaimer in | |||
| the documentation and/or other materials provided with the | |||
| distribution. | |||
| 3. Neither the name of the OpenBLAS project nor the names of | |||
| its contributors may be used to endorse or promote products | |||
| derived from this software without specific prior written permission. | |||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
| *****************************************************************************/ | |||
| #include "common.h" | |||
| #define SAVE_4x1(b1, b2, b3, b4) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \ | |||
| "vstelm.w $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x2(b1, b2, b3, b4) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" \ | |||
| "vstelm.d $vr"#b3",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b4",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x4(b1, b2, b3, b4) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t" \ | |||
| "vst $vr"#b3",%1,0;add.d %1,%1,%3;vst $vr"#b4",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x8(b1, b2, b3, b4) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;add.d %1,%1,%3;xvst $xr"#b4",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_4x16(b1, b2, b3, b4, b5, b6, b7, b8) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b5",%1,0;xvst $xr"#b6",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b7",%1,0;xvst $xr"#b8",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x16(b1, b2, b3, b4) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" \ | |||
| "xvst $xr"#b3",%1,0;xvst $xr"#b4",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x8(b1, b2) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;xvst $xr"#b2",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x4(b1, b2) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;vst $vr"#b2",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x2(b1, b2) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.d $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_2x1(b1, b2) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;vstelm.w $vr"#b2",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x16(b1, b2) \ | |||
| "xvst $xr"#b1",%1,0;xvst $xr"#b2",%1,32;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x8(b1) \ | |||
| "xvst $xr"#b1",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x4(b1) \ | |||
| "vst $vr"#b1",%1,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x2(b1) \ | |||
| "vstelm.d $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define SAVE_1x1(b1) \ | |||
| "vstelm.w $vr"#b1",%1,0,0;add.d %1,%1,%3;\n\t" | |||
| #define COPY_4x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| "xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \ | |||
| SAVE_4x16(0,4,1,5,2,6,3,7) | |||
| #define COPY_4x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| SAVE_4x8(0,1,2,3) | |||
| #define COPY_4x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| SAVE_4x4(0,1,2,3) | |||
| #define COPY_4x2 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| SAVE_4x2(0,1,2,3) | |||
| #define COPY_4x1 \ | |||
| "fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15;fmul.s $f2,$f2,$f15;fmul.s $f3,$f3,$f15 \n\t" \ | |||
| SAVE_4x1(0,1,2,3) | |||
| #define COPY_2x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| SAVE_2x16(0,2,1,3) | |||
| #define COPY_2x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \ | |||
| SAVE_2x8(0,1) | |||
| #define COPY_2x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \ | |||
| SAVE_2x4(0,1) | |||
| #define COPY_2x2 \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15; \n\t" \ | |||
| SAVE_2x2(0,1) | |||
| #define COPY_2x1 \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15;fmul.s $f1,$f1,$f15; \n\t" \ | |||
| SAVE_2x1(0,1) | |||
| #define COPY_1x16 \ | |||
| "xvld $xr0,%0,0; xvld $xr1,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15; \n\t" \ | |||
| SAVE_1x16(0,1) | |||
| #define COPY_1x8 \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15; \n\t" \ | |||
| SAVE_1x8(0) | |||
| #define COPY_1x4 \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_1x4(0) | |||
| #define COPY_1x2 \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_1x2(0) | |||
| #define COPY_1x1 \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fmul.s $f0,$f0,$f15; \n\t" \ | |||
| SAVE_1x1(0) | |||
| #define ROWS_OF_BLOCK 128 | |||
| #define COMPUTE(ndim) \ | |||
| src = src_base; dst = dst_base; \ | |||
| __asm__ __volatile__( \ | |||
| "xvldrepl.w $xr15, %6, 0 \n\t" \ | |||
| "srli.d $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"3f \n\t" \ | |||
| #ndim"4: \n\t" \ | |||
| COPY_4x##ndim \ | |||
| "addi.d $r6, $r6, -1 \n\t" \ | |||
| "bnez $r6, "#ndim"4b \n\t" \ | |||
| #ndim"3: \n\t" \ | |||
| "andi $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"1f \n\t" \ | |||
| #ndim"2: \n\t" \ | |||
| COPY_2x##ndim \ | |||
| #ndim"1: \n\t" \ | |||
| "andi $r6, %5, 1 \n\t" \ | |||
| "beqz $r6, "#ndim"0f \n\t" \ | |||
| COPY_1x##ndim \ | |||
| #ndim"0: \n\t" \ | |||
| :"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \ | |||
| :"r"(num_rows),"r"(&ALPHA) \ | |||
| :"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \ | |||
| ); | |||
| int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){ | |||
| float *src, *dst, *dst_tmp=0, *src_base, *dst_base; | |||
| uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_rows = 0; | |||
| BLASLONG cols_left, rows_done; float ALPHA = alpha; | |||
| if (ALPHA == 0.0) { | |||
| dst_base = b; | |||
| for (cols_left = cols; cols_left > 0; cols_left--) {memset(dst_base, 0, rows * sizeof(float)); dst_base += ldb;} | |||
| return 0; | |||
| } | |||
| for (rows_done = 0; rows_done < rows; rows_done += num_rows) { | |||
| num_rows = rows - rows_done; | |||
| if (num_rows > ROWS_OF_BLOCK) num_rows = ROWS_OF_BLOCK; | |||
| cols_left = cols; src_base = a + (int64_t)lda * (int64_t)rows_done; dst_base = b + rows_done; | |||
| for (;cols_left > 15; cols_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16;} | |||
| for (;cols_left > 7; cols_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8;} | |||
| for (;cols_left > 3; cols_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4;} | |||
| for (;cols_left > 1; cols_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2;} | |||
| if (cols_left > 0) {COMPUTE(1) src_base ++; dst_base ++;} | |||
| } | |||
| } | |||
+ 200
- 0
kernel/loongarch64/somatcopy_rt_lasx.c
View File
| @@ -0,0 +1,200 @@ | |||
| /*************************************************************************** | |||
| Copyright (c) 2024, The OpenBLAS Project | |||
| All rights reserved. | |||
| Redistribution and use in source and binary forms, with or without | |||
| modification, are permitted provided that the following conditions are | |||
| met: | |||
| 1. Redistributions of source code must retain the above copyright | |||
| notice, this list of conditions and the following disclaimer. | |||
| 2. Redistributions in binary form must reproduce the above copyright | |||
| notice, this list of conditions and the following disclaimer in | |||
| the documentation and/or other materials provided with the | |||
| distribution. | |||
| 3. Neither the name of the OpenBLAS project nor the names of | |||
| its contributors may be used to endorse or promote products | |||
| derived from this software without specific prior written permission. | |||
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | |||
| AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | |||
| IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | |||
| ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE | |||
| LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | |||
| DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR | |||
| SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER | |||
| CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, | |||
| OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE | |||
| USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |||
| *****************************************************************************/ | |||
| #include "common.h" | |||
| #define SAVE1x4(c1) \ | |||
| "vfmul.s $vr"#c1",$vr"#c1",$vr15;vstelm.w $vr"#c1",%4,0,0;add.d %4,%4,%3;\n\t" \ | |||
| "vstelm.w $vr"#c1",%4,0,1;add.d %4,%4,%3;vstelm.w $vr"#c1",%4,0,2;add.d %4,%4,%3; \n\t" \ | |||
| "vstelm.w $vr"#c1",%4,0,3;add.d %4,%4,%3;\n\t" | |||
| #define SAVE_2x4(c1, c2, t1, t2) \ | |||
| "vilvl.w $vr"#t1",$vr"#c2",$vr"#c1";vfmul.s $vr"#t1",$vr"#t1",$vr15; \n\t" \ | |||
| "vstelm.d $vr"#t1",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t1",%4,0,1;add.d %4,%4,%3 \n\t" \ | |||
| "vilvh.w $vr"#t2",$vr"#c2",$vr"#c1";vfmul.s $vr"#t2",$vr"#t2",$vr15; \n\t" \ | |||
| "vstelm.d $vr"#t2",%4,0,0;add.d %4,%4,%3;vstelm.d $vr"#t2",%4,0,1;add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x1(b1) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x2(b1, b2) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x4(b1, b2, b3, b4) \ | |||
| "vst $vr"#b1",%4,0; add.d %4,%4,%3; vst $vr"#b2",%4,0; add.d %4,%4,%3 \n\t" \ | |||
| "vst $vr"#b3",%4,0; add.d %4,%4,%3; vst $vr"#b4",%4,0; add.d %4,%4,%3 \n\t" | |||
| #define SAVE_4x8(b1, b2, b3, b4) \ | |||
| SAVE_4x4(b1, b2, b3, b4) \ | |||
| "xvpermi.q $xr"#b1",$xr"#b1",1; xvpermi.q $xr"#b2",$xr"#b2",1; xvpermi.q $xr"#b3",$xr"#b3",1; xvpermi.q $xr"#b4",$xr"#b4",1; \n\t" \ | |||
| SAVE_4x4(b1, b2, b3, b4) | |||
| #define TRANS_4x4(a1, a2, a3, a4, t1, t2, t3, t4) \ | |||
| "vilvl.w $vr"#t1",$vr"#a2",$vr"#a1";vilvh.w $vr"#t2",$vr"#a2",$vr"#a1"; \n\t" \ | |||
| "vilvl.w $vr"#t3",$vr"#a4",$vr"#a3";vilvh.w $vr"#t4",$vr"#a4",$vr"#a3"; \n\t" \ | |||
| "vilvl.d $vr"#a1",$vr"#t3",$vr"#t1";vilvh.d $vr"#a2",$vr"#t3",$vr"#t1"; \n\t" \ | |||
| "vilvl.d $vr"#a3",$vr"#t4",$vr"#t2";vilvh.d $vr"#a4",$vr"#t4",$vr"#t2"; \n\t" | |||
| #define TRANS_4x8(a1, a2, a3, a4, t1, t2, t3, t4) \ | |||
| "xvilvl.w $xr"#t1",$xr"#a2",$xr"#a1"; xvilvh.w $xr"#t2",$xr"#a2",$xr"#a1"; \n\t" \ | |||
| "xvilvl.w $xr"#t3",$xr"#a4",$xr"#a3"; xvilvh.w $xr"#t4",$xr"#a4",$xr"#a3"; \n\t" \ | |||
| "xvilvl.d $xr"#a1",$xr"#t3",$xr"#t1"; xvilvh.d $xr"#a2",$xr"#t3",$xr"#t1"; \n\t" \ | |||
| "xvilvl.d $xr"#a3",$xr"#t4",$xr"#t2"; xvilvh.d $xr"#a4",$xr"#t4",$xr"#t2"; \n\t" | |||
| #define COPY_4x16 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "xvld $xr0,%0,0; xvld $xr4,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr5,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; xvld $xr6,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; xvld $xr7,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| "xvfmul.s $xr4,$xr4,$xr15;xvfmul.s $xr5,$xr5,$xr15;xvfmul.s $xr6,$xr6,$xr15;xvfmul.s $xr7,$xr7,$xr15 \n\t" \ | |||
| TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3) \ | |||
| TRANS_4x8(4,5,6,7,8,9,10,11) SAVE_4x8(4,5,6,7) | |||
| #define COPY_4x8 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvfmul.s $xr0,$xr0,$xr15;xvfmul.s $xr1,$xr1,$xr15;xvfmul.s $xr2,$xr2,$xr15;xvfmul.s $xr3,$xr3,$xr15 \n\t" \ | |||
| TRANS_4x8(0,1,2,3,8,9,10,11) SAVE_4x8(0,1,2,3) | |||
| #define COPY_4x4 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x4(0,1,2,3) | |||
| #define COPY_4x2 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15;vfmul.s $vr1,$vr1,$vr15;vfmul.s $vr2,$vr2,$vr15;vfmul.s $vr3,$vr3,$vr15 \n\t" \ | |||
| TRANS_4x4(0,1,2,3,8,9,10,11) SAVE_4x2(0,1) | |||
| #define COPY_4x1 \ | |||
| "move %4,%1; addi.d %1,%1,16 \n\t" \ | |||
| "fld.s $f0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f2,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f3,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.w $xr0,$xr1,1;xvinsve0.w $xr0,$xr2,2;xvinsve0.w $xr0,$xr3,3; \n\t" \ | |||
| "vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| SAVE_4x1(0) | |||
| #define COPY_2x16 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "xvld $xr0,%0,0; xvld $xr2,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; xvld $xr3,%0,32; add.d %0,%0,%2; \n\t" \ | |||
| "xvpermi.q $xr4,$xr0,1;xvpermi.q $xr6,$xr2,1;xvpermi.q $xr5,$xr1,1;xvpermi.q $xr7,$xr3,1; \n\t" \ | |||
| SAVE_2x4(0,1,8,9) SAVE_2x4(4,5,8,9) SAVE_2x4(2,3,8,9) SAVE_2x4(6,7,8,9) | |||
| #define COPY_2x8 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "xvld $xr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvld $xr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "xvpermi.q $xr2,$xr0,1;xvpermi.q $xr3,$xr1,1; \n\t" \ | |||
| SAVE_2x4(0,1,4,5) SAVE_2x4(2,3,4,5) | |||
| #define COPY_2x4 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "vld $vr0,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| "vld $vr1,%0,0; add.d %0,%0,%2; \n\t" \ | |||
| SAVE_2x4(0,1,4,5) | |||
| #define COPY_2x2 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "fld.d $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.d $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.d $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15;vshuf4i.w $vr0,$vr0,0xd8 \n\t" \ | |||
| "vstelm.d $vr0,%4,0,0;add.d %4,%4,%3;vstelm.d $vr0,%4,0,1 \n\t" | |||
| #define COPY_2x1 \ | |||
| "move %4,%1; addi.d %1,%1,8 \n\t" \ | |||
| "fld.s $f0,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "fld.s $f1,%0,0;add.d %0,%0,%2; \n\t" \ | |||
| "xvinsve0.w $xr0,$xr1,1;vfmul.s $vr0,$vr0,$vr15; \n\t" \ | |||
| "vstelm.d $vr0,%4,0,0; \n\t" | |||
| #define COPY_1x16 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \ | |||
| "vld $vr1,%0,32;" SAVE1x4(1) "vld $vr2,%0,48;" SAVE1x4(2) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x8 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) "vld $vr2,%0,16;" SAVE1x4(2) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x4 \ | |||
| "move %4,%1; addi.d %1,%1,4 \n\t" \ | |||
| "vld $vr1,%0,0;" SAVE1x4(1) \ | |||
| "add.d %0,%0,%2 \n\t" | |||
| #define COPY_1x2 \ | |||
| "move %4,%1;fld.d $f1,%0,0;add.d %0,%0,%2;vfmul.s $vr1,$vr1,$vr15;vstelm.w $vr1,%4,0,0;add.d %4,%4,%3;vstelm.w $vr1,%4,0,1;\n\t" \ | |||
| "addi.d %1,%1,4;\n\t" | |||
| #define COPY_1x1 \ | |||
| "fld.s $f1,%0,0;fmul.s $f1,$f1,$f15;fst.s $f1,%1,0;add.d %0,%0,%2;addi.d %1,%1,4;\n\t" | |||
| #define ROWS_OF_BLOCK 128 | |||
| #define COMPUTE(ndim) \ | |||
| src = src_base; dst = dst_base; \ | |||
| __asm__ __volatile__( \ | |||
| "xvldrepl.w $xr15, %6, 0 \n\t" \ | |||
| "srli.d $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"3f \n\t" \ | |||
| #ndim"4: \n\t" \ | |||
| COPY_4x##ndim \ | |||
| "addi.d $r6, $r6, -1 \n\t" \ | |||
| "bnez $r6, "#ndim"4b \n\t" \ | |||
| #ndim"3: \n\t" \ | |||
| "andi $r6, %5, 2 \n\t" \ | |||
| "beqz $r6, "#ndim"1f \n\t" \ | |||
| #ndim"2: \n\t" \ | |||
| COPY_2x##ndim \ | |||
| #ndim"1: \n\t" \ | |||
| "andi $r6, %5, 1 \n\t" \ | |||
| "beqz $r6, "#ndim"0f \n\t" \ | |||
| COPY_1x##ndim \ | |||
| #ndim"0: \n\t" \ | |||
| :"+r"(src),"+r"(dst),"+r"(src_ld_bytes),"+r"(dst_ld_bytes),"+r"(dst_tmp) \ | |||
| :"r"(num_rows),"r"(&ALPHA) \ | |||
| :"memory", "$r6", "$f0", "$f1", "$f2", "$f3", "$f4", "$f5", "$f6", "$f7", "$f8", "$f9", "$f10", "$f11", "$f15" \ | |||
| ); | |||
| int CNAME(BLASLONG rows, BLASLONG cols, FLOAT alpha, FLOAT *a, BLASLONG lda, FLOAT *b, BLASLONG ldb){ | |||
| float *src, *dst, *dst_tmp=0, *src_base, *dst_base; | |||
| uint64_t src_ld_bytes = (uint64_t)lda * sizeof(float), dst_ld_bytes = (uint64_t)ldb * sizeof(float), num_rows = 0; | |||
| BLASLONG cols_left, rows_done; float ALPHA = alpha; | |||
| if (ALPHA == 0.0) { | |||
| dst_base = b; | |||
| for (cols_left = cols; cols_left > 0; cols_left--) {memset(dst_base, 0, rows * sizeof(float)); dst_base += ldb;} | |||
| return 0; | |||
| } | |||
| for (rows_done = 0; rows_done < rows; rows_done += num_rows) { | |||
| num_rows = rows - rows_done; | |||
| if (num_rows > ROWS_OF_BLOCK) num_rows = ROWS_OF_BLOCK; | |||
| cols_left = cols; src_base = a + (int64_t)lda * (int64_t)rows_done; dst_base = b + rows_done; | |||
| for (;cols_left > 15; cols_left -= 16) {COMPUTE(16) src_base += 16; dst_base += 16 * ldb;} | |||
| for (;cols_left > 7; cols_left -= 8) {COMPUTE(8) src_base += 8; dst_base += 8 * ldb;} | |||
| for (;cols_left > 3; cols_left -= 4) {COMPUTE(4) src_base += 4; dst_base += 4 * ldb;} | |||
| for (;cols_left > 1; cols_left -= 2) {COMPUTE(2) src_base += 2; dst_base += 2 * ldb;} | |||
| if (cols_left > 0) {COMPUTE(1) src_base ++; dst_base += ldb;} | |||
| } | |||
| } | |||