OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

loongarch64: Add optimization for amin.

tags/v0.3.26
yancheng Shiyou Yin 1 year ago
parent
265b5f2e80
commit
ff2ecc6cda
6 changed files with 714 additions and 0 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +3
    -0
      kernel/loongarch64/KERNEL.LOONGSON2K1000
  2. +3
    -0
      kernel/loongarch64/KERNEL.LOONGSON3R5
  3. +178
    -0
      kernel/loongarch64/damin_lasx.S
  4. +145
    -0
      kernel/loongarch64/damin_lsx.S
  5. +208
    -0
      kernel/loongarch64/samin_lasx.S
  6. +177
    -0
      kernel/loongarch64/samin_lsx.S

+ 3
- 0
kernel/loongarch64/KERNEL.LOONGSON2K1000 View File

@@ -10,4 +10,7 @@ DSCALKERNEL = dscal_lsx.S
SAMAXKERNEL = samax_lsx.S SAMAXKERNEL = samax_lsx.S
DAMAXKERNEL = damax_lsx.S DAMAXKERNEL = damax_lsx.S


SAMINKERNEL = samin_lsx.S
DAMINKERNEL = damin_lsx.S

endif endif

+ 3
- 0
kernel/loongarch64/KERNEL.LOONGSON3R5 View File

@@ -10,6 +10,9 @@ DSCALKERNEL = dscal_lasx.S
SAMAXKERNEL = samax_lasx.S SAMAXKERNEL = samax_lasx.S
DAMAXKERNEL = damax_lasx.S DAMAXKERNEL = damax_lasx.S


SAMINKERNEL = samin_lasx.S
DAMINKERNEL = damin_lasx.S

DGEMMKERNEL = dgemm_kernel_16x4.S DGEMMKERNEL = dgemm_kernel_16x4.S
DGEMMINCOPY = dgemm_ncopy_16.S DGEMMINCOPY = dgemm_ncopy_16.S
DGEMMITCOPY = dgemm_tcopy_16.S DGEMMITCOPY = dgemm_tcopy_16.S


+ 178
- 0
kernel/loongarch64/damin_lasx.S View File

@@ -0,0 +1,178 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define J $r13
#define t1 $r14
#define t2 $r18
#define t3 $r15
#define t4 $r17
#define TEMP $r16
#define m0 $xr8
#define x1 $xr9
#define x2 $xr10
#define x3 $xr11
#define x4 $xr12
#define VX0 $xr20
#define VX1 $xr21
#define VM0 $xr22
#define VM1 $xr23
#define VM2 $xr19

PROLOGUE

bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
xvld VM0, X, 0
srai.d I, N, 3
bge $r0, I, .L12
.align 3

.L10:
xvld VX0, X, 0 * SIZE
addi.d I, I, -1
xvld VX1, X, 4 * SIZE
xvfmina.d VM1, VX1, VX0
addi.d X, X, 8 * SIZE
xvfmina.d VM0, VM0, VM1
blt $r0, I, .L10
.align 3

.L11:
xvpickve.d x1, VM0, 0
xvpickve.d x2, VM0, 1
xvpickve.d x3, VM0, 2
xvpickve.d x4, VM0, 3
xvfmina.d VM1, x1, x2
xvfmina.d VM2, x3, x4
xvfmina.d VM0, VM1, VM2
.align 3

.L12: //INCX==1 and N<8
andi I, N, 7
li.d J, 4
bge J, I, .L13 // 4<N<8
xvld VX0, X, 0
slli.d J, J, 1 // 8
sub.d I, J, I
slli.d I, I, BASE_SHIFT
xvldx VX1, X, I
xvfmina.d m0, VX0, VX1 //patial repeat read
xvpickve.d x1, m0, 0
xvpickve.d x2, m0, 1
xvpickve.d x3, m0, 2
xvpickve.d x4, m0, 3
xvfmina.d VM1, x1, x2
xvfmina.d m0, x3, x4
xvfmina.d m0, m0, VM1
xvfmina.d VM0, m0, VM0
fabs.d $f22, $f22
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L13: //INCX==1 and 0<=N<=4
bge $r0, I, .L15
.align 3

.L14:
xvld x1, X, 0
addi.d I, I, -1
xvfmina.d VM0, VM0, x1
addi.d X, X, SIZE
blt $r0, I, .L14
.align 3

.L15:
fabs.d $f22, $f22
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L20: // INCX!=1
move TEMP, X // initialize the mina value
ld.d t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.d VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L23
ld.d t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.d t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.d t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.d VM0, t2, 1
xvinsgr2vr.d VM0, t3, 2
xvinsgr2vr.d VM0, t4, 3
.align 3

.L21:
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.d VX0, t1, 0
xvinsgr2vr.d VX0, t2, 1
xvinsgr2vr.d VX0, t3, 2
xvinsgr2vr.d VX0, t4, 3
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.d VX1, t1, 0
xvinsgr2vr.d VX1, t2, 1
xvinsgr2vr.d VX1, t3, 2
xvinsgr2vr.d VX1, t4, 3
addi.d I, I, -1
xvfmina.d VM1, VX1, VX0
xvfmina.d VM0, VM1, VM0
blt $r0, I, .L21
.align 3

.L22:
xvpickve.d x1, VM0, 0
xvpickve.d x2, VM0, 1
xvpickve.d x3, VM0, 2
xvpickve.d x4, VM0, 3
xvfmina.d VM1, x1, x2
xvfmina.d VM2, x3, x4
xvfmina.d VM0, VM1, VM2
.align 3

.L23: //INCX!=1 and N<8
andi I, N, 7
bge $r0, I, .L999
.align 3

.L24:
xvld x1, X, 0
addi.d I, I, -1
xvfmina.d VM0, VM0, x1
add.d X, X, INCX
blt $r0, I, .L24
.align 3

.L999:
fabs.d $f22, $f22
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 145
- 0
kernel/loongarch64/damin_lsx.S View File

@@ -0,0 +1,145 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define t1 $r14
#define t2 $r18
#define t3 $r15
#define t4 $r17
#define TEMP $r16
#define x1 $vr9
#define x2 $vr10
#define VX0 $vr20
#define VX1 $vr21
#define VM0 $vr22
#define VM1 $vr23
#define VM2 $vr18
#define VM3 $vr19

PROLOGUE

bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
vld VM0, X, 0
srai.d I, N, 3
bge $r0, I, .L12
.align 3

.L10:
vld VX0, X, 0 * SIZE
vld VX1, X, 2 * SIZE
addi.d I, I, -1
vfmina.d VM1, VX1, VX0
vld VX0, X, 4 * SIZE
vld VX1, X, 6 * SIZE
vfmina.d VM2, VX1, VX0
vfmina.d VM3, VM1, VM2
addi.d X, X, 8 * SIZE
vfmina.d VM0, VM0, VM3
blt $r0, I, .L10
.align 3

.L11:
vreplvei.d x1, VM0, 0
vreplvei.d x2, VM0, 1
vfmina.d VM0, x1, x2
.align 3

.L12: //INCX==1 and N<8
andi I, N, 7
bge $r0, I, .L14
.align 3

.L13:
vld x1, X, 0
addi.d I, I, -1
vfmina.d VM0, VM0, x1
addi.d X, X, SIZE
blt $r0, I, .L13
.align 3

.L14:
fabs.d $f22, $f22
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L20: // INCX!=1
move TEMP, X // initialize the mina value
ld.d t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.d VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L23
ld.d t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.d VM0, t2, 1
.align 3

.L21:
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX0, t1, 0
vinsgr2vr.d VX0, t2, 1
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX1, t3, 0
vinsgr2vr.d VX1, t4, 1
vfmina.d VM1, VX0, VX1
ld.d t1, X, 0 * SIZE
add.d X, X, INCX
ld.d t2, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX0, t1, 0
vinsgr2vr.d VX0, t2, 1
ld.d t3, X, 0 * SIZE
add.d X, X, INCX
ld.d t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.d VX1, t3, 0
vinsgr2vr.d VX1, t4, 1
addi.d I, I, -1
vfmina.d VM2, VX0, VX1
vfmina.d VM3, VM1, VM2
vfmina.d VM0, VM0, VM3
blt $r0, I, .L21
.align 3

.L22:
vreplvei.d x1, VM0, 0
vreplvei.d x2, VM0, 1
vfmina.d VM0, x1, x2
.align 3

.L23: //INCX!=1 and N<8
andi I, N, 7
bge $r0, I, .L999
.align 3

.L24:
vld x1, X, 0
vfmina.d VM0, VM0, x1
addi.d I, I, -1
add.d X, X, INCX
blt $r0, I, .L24
.align 3

.L999:
fabs.d $f22, $f22
fmov.d $f0, $f22
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 208
- 0
kernel/loongarch64/samin_lasx.S View File

@@ -0,0 +1,208 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define J $r13
#define t1 $r14
#define t2 $r18
#define t3 $r15
#define t4 $r17
#define TEMP $r16
#define m0 $xr8
#define x1 $xr9
#define x2 $xr10
#define x3 $xr11
#define x4 $xr12
#define x5 $xr13
#define x6 $xr14
#define x7 $xr15
#define x8 $xr16
#define VX0 $xr20
#define VX1 $xr21
#define VM0 $xr22
#define VM1 $xr23
#define VM2 $xr18
#define VM3 $xr19

PROLOGUE

bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
xvld VM0, X, 0
srai.d I, N, 3
bge $r0, I, .L12
.align 3

.L10:
xvld VX0, X, 0 * SIZE
addi.d I, I, -1
xvfmina.s VM0, VM0, VX0
addi.d X, X, 8 * SIZE
blt $r0, I, .L10
.align 3

.L11:
xvpickve.w x1, VM0, 0
xvpickve.w x2, VM0, 1
xvpickve.w x3, VM0, 2
xvpickve.w x4, VM0, 3
xvpickve.w x5, VM0, 4
xvpickve.w x6, VM0, 5
xvpickve.w x7, VM0, 6
xvpickve.w x8, VM0, 7
xvfmina.s VM3, x1, x2
xvfmina.s VM2, x3, x4
xvfmina.s VM1, x5, x6
xvfmina.s VM0, x7, x8
xvfmina.s VM2, VM2, VM3
xvfmina.s VM0, VM0, VM1
xvfmina.s VM0, VM0, VM2
.align 3

.L12: //INCX==1 and N<8
andi I, N, 7
li.d J, 4
bge J, I, .L13 // 4<N<8
xvld VX0, X, 0
slli.d J, J, 1 // 8
sub.d I, J, I
slli.d I, I, BASE_SHIFT
xvldx VX1, X, I
xvfmina.s m0, VX0, VX1 //patial repeat read
xvpickve.w x1, m0, 0
xvpickve.w x2, m0, 1
xvpickve.w x3, m0, 2
xvpickve.w x4, m0, 3
xvfmina.s m0, x1, x2
xvfmina.s VM1, x3, x4
xvfmina.s m0, m0, VM1
xvfmina.s VM0, m0, VM0
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L13: //INCX==1 and 0<=N<=4
bge $r0, I, .L15
.align 3

.L14:
xvld x1, X, 0
addi.d I, I, -1
xvfmina.s VM0, VM0, x1
addi.d X, X, SIZE
blt $r0, I, .L14
.align 3

.L15:
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L20: // INCX!=1
move TEMP, X // initialize the mina value
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.w VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L23
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.w VM0, t2, 1
xvinsgr2vr.w VM0, t3, 2
xvinsgr2vr.w VM0, t4, 3
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
xvinsgr2vr.w VM0, t1, 4
xvinsgr2vr.w VM0, t2, 5
xvinsgr2vr.w VM0, t3, 6
xvinsgr2vr.w VM0, t4, 7
.align 3

.L21:
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.w VX0, t1, 0
xvinsgr2vr.w VX0, t2, 1
xvinsgr2vr.w VX0, t3, 2
xvinsgr2vr.w VX0, t4, 3
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
xvinsgr2vr.w VX0, t1, 4
xvinsgr2vr.w VX0, t2, 5
xvinsgr2vr.w VX0, t3, 6
xvinsgr2vr.w VX0, t4, 7
addi.d I, I, -1
xvfmina.s VM0, VM0, VX0
blt $r0, I, .L21
.align 3

.L22:
xvpickve.w x1, VM0, 0
xvpickve.w x2, VM0, 1
xvpickve.w x3, VM0, 2
xvpickve.w x4, VM0, 3
xvpickve.w x5, VM0, 4
xvpickve.w x6, VM0, 5
xvpickve.w x7, VM0, 6
xvpickve.w x8, VM0, 7
xvfmina.s VM3, x1, x2
xvfmina.s VM2, x3, x4
xvfmina.s VM1, x5, x6
xvfmina.s VM0, x7, x8
xvfmina.s VM2, VM2, VM3
xvfmina.s VM0, VM0, VM1
xvfmina.s VM0, VM0, VM2
.align 3

.L23: //INCX!=1 and N<8
andi I, N, 7
bge $r0, I, .L999
.align 3

.L24:
xvld x1, X, 0
addi.d I, I, -1
xvfmina.s VM0, VM0, x1
add.d X, X, INCX
blt $r0, I, .L24
.align 3

.L999:
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

+ 177
- 0
kernel/loongarch64/samin_lsx.S View File

@@ -0,0 +1,177 @@
#define ASSEMBLER

#include "common.h"

#define N $r4
#define X $r5
#define INCX $r6
#define I $r12
#define J $r13
#define t1 $r14
#define t2 $r18
#define t3 $r15
#define t4 $r17
#define TEMP $r16
#define m0 $vr8
#define x1 $vr9
#define x2 $vr10
#define x3 $vr11
#define x4 $vr12
#define VX0 $vr20
#define VX1 $vr21
#define VM0 $vr22
#define VM1 $vr23

PROLOGUE

bge $r0, N, .L999
bge $r0, INCX, .L999
li.d TEMP, 1
slli.d TEMP, TEMP, BASE_SHIFT
slli.d INCX, INCX, BASE_SHIFT
bne INCX, TEMP, .L20
vld VM0, X, 0
srai.d I, N, 3
bge $r0, I, .L12
.align 3

.L10:
vld VX0, X, 0 * SIZE
vld VX1, X, 4 * SIZE
addi.d I, I, -1
vfmina.s VM1, VX0, VX1
addi.d X, X, 8 * SIZE
vfmina.s VM0, VM0, VM1
blt $r0, I, .L10
.align 3

.L11:
vreplvei.w x1, VM0, 0
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfmina.s VM1, x1, x2
vfmina.s VM0, x3, x4
vfmina.s VM0, VM0, VM1
.align 3

.L12: //INCX==1 and N<8
andi I, N, 7
li.d J, 4
bge J, I, .L13 // 4<N<8
vld VX0, X, 0
slli.d J, J, 1 // 8
sub.d I, J, I
slli.d I, I, BASE_SHIFT
vldx VX1, X, I
vfmina.s m0, VX0, VX1 //patial repeat read
vreplvei.w x1, m0, 0
vreplvei.w x2, m0, 1
vreplvei.w x3, m0, 2
vreplvei.w x4, m0, 3
vfmina.s m0, x1, x2
vfmina.s VM1, x3, x4
vfmina.s m0, m0, VM1
vfmina.s VM0, m0, VM0
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L13: //INCX==1 and 0<=N<=4
bge $r0, I, .L15
.align 3

.L14:
vld x1, X, 0
addi.d I, I, -1
vfmina.s VM0, VM0, x1
addi.d X, X, SIZE
blt $r0, I, .L14
.align 3

.L15:
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

.L20: // INCX!=1
move TEMP, X
ld.w t1, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.w VM0, t1, 0
srai.d I, N, 3
bge $r0, I, .L23
ld.w t2, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t3, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
ld.w t4, TEMP, 0 * SIZE
add.d TEMP, TEMP, INCX
vinsgr2vr.w VM0, t2, 1
vinsgr2vr.w VM0, t3, 2
vinsgr2vr.w VM0, t4, 3
.align 3

.L21:
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.w VX0, t1, 0
vinsgr2vr.w VX0, t2, 1
vinsgr2vr.w VX0, t3, 2
vinsgr2vr.w VX0, t4, 3
ld.w t1, X, 0 * SIZE
add.d X, X, INCX
ld.w t2, X, 0 * SIZE
add.d X, X, INCX
ld.w t3, X, 0 * SIZE
add.d X, X, INCX
ld.w t4, X, 0 * SIZE
add.d X, X, INCX
vinsgr2vr.w VX1, t1, 0
vinsgr2vr.w VX1, t2, 1
vinsgr2vr.w VX1, t3, 2
vinsgr2vr.w VX1, t4, 3
addi.d I, I, -1
vfmina.s VM1, VX0, VX1
vfmina.s VM0, VM0, VM1
blt $r0, I, .L21
.align 3

.L22:
vreplvei.w x1, VM0, 0
vreplvei.w x2, VM0, 1
vreplvei.w x3, VM0, 2
vreplvei.w x4, VM0, 3
vfmina.s VM1, x1, x2
vfmina.s VM0, x3, x4
vfmina.s VM0, VM0, VM1
.align 3

.L23: //INCX!=1 and N<8
andi I, N, 7
bge $r0, I, .L999
.align 3

.L24:
vld x1, X, 0
addi.d I, I, -1
vfmina.s VM0, VM0, x1
add.d X, X, INCX
blt $r0, I, .L24
.align 3

.L999:
fabs.s $f22, $f22
fmov.s $f0, $f22
jirl $r0, $r1, 0x0
.align 3

EPILOGUE

Write Preview
Loading…
Cancel
Save