OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
Browse Source

Updated data prefetch in TRSM, ASUM, DOT functions 

Signed-off-by: kaustubh <kaustubh.raste@imgtec.com>
tags/v0.2.20^2
kaustubh 9 years ago
parent
db3efb2e14
commit
3eaff85191
17 changed files with 939 additions and 652 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +71
    -30
      kernel/mips/casum_msa.c
  2. +119
    -63
      kernel/mips/cdot_msa.c
  3. +71
    -30
      kernel/mips/dasum_msa.c
  4. +29
    -14
      kernel/mips/ddot_msa.c
  5. +57
    -86
      kernel/mips/dtrsm_kernel_LN_8x4_msa.c
  6. +57
    -85
      kernel/mips/dtrsm_kernel_LT_8x4_msa.c
  7. +45
    -67
      kernel/mips/dtrsm_kernel_RN_8x4_msa.c
  8. +47
    -69
      kernel/mips/dtrsm_kernel_RT_8x4_msa.c
  9. +17
    -0
      kernel/mips/macros_msa.h
  10. +45
    -29
      kernel/mips/sasum_msa.c
  11. +29
    -14
      kernel/mips/sdot_msa.c
  12. +19
    -14
      kernel/mips/strsm_kernel_LN_8x8_msa.c
  13. +17
    -11
      kernel/mips/strsm_kernel_LT_8x8_msa.c
  14. +17
    -10
      kernel/mips/strsm_kernel_RN_8x8_msa.c
  15. +18
    -13
      kernel/mips/strsm_kernel_RT_8x8_msa.c
  16. +71
    -30
      kernel/mips/zasum_msa.c
  17. +210
    -87
      kernel/mips/zdot_msa.c

+ 71
- 30
kernel/mips/casum_msa.c View File

@@ -47,39 +47,55 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)


if (1 == inc_x) if (1 == inc_x)
{ {
#ifdef ENABLE_PREFETCH
FLOAT *x_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
if (n > 31)
{ {
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
x_pref = x + pref_offset + 128;
#endif
FLOAT *x_pref;
BLASLONG pref_offset;


for (i = (n >> 5); i--;)
{
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 128(%[x_pref])\n\t"
"pref 0, 160(%[x_pref])\n\t"
"pref 0, 192(%[x_pref])\n\t"
"pref 0, 224(%[x_pref])\n\t"

: : [x_pref] "r" (x_pref)
);

x_pref += 64;
#endif
pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 128 + 32;


LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7); LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 5) - 1; i--;)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(x_pref, 128);
PREF_OFFSET(x_pref, 160);
PREF_OFFSET(x_pref, 192);
PREF_OFFSET(x_pref, 224);
x_pref += 64;

LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_W(src0);
sum_abs1 += AND_VEC_W(src1);
sum_abs2 += AND_VEC_W(src2);
sum_abs3 += AND_VEC_W(src3);
sum_abs0 += AND_VEC_W(src4);
sum_abs1 += AND_VEC_W(src5);
sum_abs2 += AND_VEC_W(src6);
sum_abs3 += AND_VEC_W(src7);

LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_W(src8);
sum_abs1 += AND_VEC_W(src9);
sum_abs2 += AND_VEC_W(src10);
sum_abs3 += AND_VEC_W(src11);
sum_abs0 += AND_VEC_W(src12);
sum_abs1 += AND_VEC_W(src13);
sum_abs2 += AND_VEC_W(src14);
sum_abs3 += AND_VEC_W(src15);
}

LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15); LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_W(src0); sum_abs0 += AND_VEC_W(src0);
@@ -159,9 +175,34 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
{ {
inc_x2 = 2 * inc_x; inc_x2 = 2 * inc_x;


for (i = (n >> 4); i--;)
if (n > 16)
{ {
LD_SP8_INC(x, inc_x2, src0, src1, src2, src3, src4, src5, src6, src7); LD_SP8_INC(x, inc_x2, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 4) - 1; i--;)
{
LD_SP8_INC(x, inc_x2, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_W(src0);
sum_abs1 += AND_VEC_W(src1);
sum_abs2 += AND_VEC_W(src2);
sum_abs3 += AND_VEC_W(src3);
sum_abs0 += AND_VEC_W(src4);
sum_abs1 += AND_VEC_W(src5);
sum_abs2 += AND_VEC_W(src6);
sum_abs3 += AND_VEC_W(src7);

LD_SP8_INC(x, inc_x2, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_W(src8);
sum_abs1 += AND_VEC_W(src9);
sum_abs2 += AND_VEC_W(src10);
sum_abs3 += AND_VEC_W(src11);
sum_abs0 += AND_VEC_W(src12);
sum_abs1 += AND_VEC_W(src13);
sum_abs2 += AND_VEC_W(src14);
sum_abs3 += AND_VEC_W(src15);
}

LD_SP8_INC(x, inc_x2, src8, src9, src10, src11, src12, src13, src14, src15); LD_SP8_INC(x, inc_x2, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_W(src0); sum_abs0 += AND_VEC_W(src0);


+ 119
- 63
kernel/mips/cdot_msa.c View File

@@ -45,10 +45,9 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA
BLASLONG i = 0; BLASLONG i = 0;
FLOAT dot[2]; FLOAT dot[2];
BLASLONG inc_x2, inc_y2; BLASLONG inc_x2, inc_y2;
FLOAT x0, x1, x2, x3, x4, x5, x6, x7;
FLOAT y0, y1, y2, y3, y4, y5, y6, y7;
v4f32 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7;
v4f32 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7;
FLOAT x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, y6, y7;
v4f32 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7, vx8, vx9, vx10, vx11;
v4f32 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7, vy8, vy9, vy10, vy11;
v4f32 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i; v4f32 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i;
v4f32 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i; v4f32 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i;
v4f32 dot0 = {0, 0, 0, 0}; v4f32 dot0 = {0, 0, 0, 0};
@@ -71,78 +70,135 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA


if ((1 == inc_x) && (1 == inc_y)) if ((1 == inc_x) && (1 == inc_y))
{ {
#ifdef ENABLE_PREFETCH
FLOAT *x_pref, *y_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
if (n > 15)
{ {
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
x_pref = x + pref_offset + 64;
FLOAT *x_pref, *y_pref;
BLASLONG pref_offset;


pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
y_pref = y + pref_offset + 64;
#endif
pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 64 + 16;


for (i = (n >> 4); i--;)
{
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 0(%[y_pref])\n\t"
"pref 0, 32(%[y_pref])\n\t"
"pref 0, 64(%[y_pref])\n\t"
"pref 0, 96(%[y_pref])\n\t"

: : [x_pref] "r" (x_pref), [y_pref] "r" (y_pref)
);

x_pref += 32;
y_pref += 32;
#endif
pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
y_pref = y + pref_offset + 64 + 16;


LD_SP8_INC(x, 4, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_SP8_INC(y, 4, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
LD_SP4_INC(x, 4, vx0, vx1, vx2, vx3);
LD_SP4_INC(y, 4, vy0, vy1, vy2, vy3);


PCKEVOD_W2_SP(vx1, vx0, vx0r, vx0i); PCKEVOD_W2_SP(vx1, vx0, vx0r, vx0i);
PCKEVOD_W2_SP(vx3, vx2, vx1r, vx1i);
PCKEVOD_W2_SP(vx5, vx4, vx2r, vx2i);
PCKEVOD_W2_SP(vx7, vx6, vx3r, vx3i);

PCKEVOD_W2_SP(vy1, vy0, vy0r, vy0i); PCKEVOD_W2_SP(vy1, vy0, vy0r, vy0i);
PCKEVOD_W2_SP(vy3, vy2, vy1r, vy1i);
PCKEVOD_W2_SP(vy5, vy4, vy2r, vy2i);
PCKEVOD_W2_SP(vy7, vy6, vy3r, vy3i);


dot0 += (vx0r * vy0r);
dot0 OP1 (vx0i * vy0i);
dot1 OP2 (vx0i * vy0r);
dot1 += (vx0r * vy0i);
for (i = (n >> 4) - 1; i--;)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(y_pref, 0);
PREF_OFFSET(y_pref, 32);
PREF_OFFSET(y_pref, 64);
PREF_OFFSET(y_pref, 96);
x_pref += 32;
y_pref += 32;

vx4 = LD_SP(x); x += 4;
vx1r = (v4f32) __msa_pckev_w((v4i32) vx3, (v4i32) vx2);
dot0 += (vx0r * vy0r);
vx5 = LD_SP(x); x += 4;
vx1i = (v4f32) __msa_pckod_w((v4i32) vx3, (v4i32) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_SP(y); y += 4;
vy1r = (v4f32) __msa_pckev_w((v4i32) vy3, (v4i32) vy2);
dot2 += (vx1r * vy1r);
vy5 = LD_SP(y); y += 4;
vy1i = (v4f32) __msa_pckod_w((v4i32) vy3, (v4i32) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_SP(x); x += 4;
vx7 = LD_SP(x); x += 4;
vy6 = LD_SP(y); y += 4;
vy7 = LD_SP(y); y += 4;
vx8 = LD_SP(x); x += 4;
dot0 -= (vx0i * vy0i);
vx9 = LD_SP(x); x += 4;
vx2r = (v4f32) __msa_pckev_w((v4i32) vx5, (v4i32) vx4);
dot1 += (vx0r * vy0i);
vy8 = LD_SP(y); y += 4;
vx2i = (v4f32) __msa_pckod_w((v4i32) vx5, (v4i32) vx4);
dot2 -= (vx1i * vy1i);
vy9 = LD_SP(y); y += 4;
vy2r = (v4f32) __msa_pckev_w((v4i32) vy5, (v4i32) vy4);
dot3 += (vx1r * vy1i);
vx10 = LD_SP(x); x += 4;
vy2i = (v4f32) __msa_pckod_w((v4i32) vy5, (v4i32) vy4);
vx11 = LD_SP(x); x += 4;
vx3r = (v4f32) __msa_pckev_w((v4i32) vx7, (v4i32) vx6);
dot4 += (vx2r * vy2r);
vy10 = LD_SP(y); y += 4;
vx3i = (v4f32) __msa_pckod_w((v4i32) vx7, (v4i32) vx6);
dot5 += (vx2i * vy2r);
vy11 = LD_SP(y); y += 4;
vy3r = (v4f32) __msa_pckev_w((v4i32) vy7, (v4i32) vy6);
vy3i = (v4f32) __msa_pckod_w((v4i32) vy7, (v4i32) vy6);
dot6 += (vx3r * vy3r);
vx0r = (v4f32) __msa_pckev_w((v4i32) vx9, (v4i32) vx8);
dot7 += (vx3i * vy3r);
vx0i = (v4f32) __msa_pckod_w((v4i32) vx9, (v4i32) vx8);
vy0r = (v4f32) __msa_pckev_w((v4i32) vy9, (v4i32) vy8);
vx2 = vx10;
vy0i = (v4f32) __msa_pckod_w((v4i32) vy9, (v4i32) vy8);
vx3 = vx11;
dot4 -= (vx2i * vy2i);
vy2 = vy10;
dot5 += (vx2r * vy2i);
vy3 = vy11;
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}


vx4 = LD_SP(x); x += 4;
vx1r = (v4f32) __msa_pckev_w((v4i32) vx3, (v4i32) vx2);
dot0 += (vx0r * vy0r);
vx5 = LD_SP(x); x += 4;
vx1i = (v4f32) __msa_pckod_w((v4i32) vx3, (v4i32) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_SP(y); y += 4;
vy1r = (v4f32) __msa_pckev_w((v4i32) vy3, (v4i32) vy2);
dot2 += (vx1r * vy1r); dot2 += (vx1r * vy1r);
dot2 OP1 (vx1i * vy1i);
dot3 OP2 (vx1i * vy1r);
vy5 = LD_SP(y); y += 4;
vy1i = (v4f32) __msa_pckod_w((v4i32) vy3, (v4i32) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_SP(x); x += 4;
vx7 = LD_SP(x); x += 4;
vy6 = LD_SP(y); y += 4;
vy7 = LD_SP(y); y += 4;
dot0 -= (vx0i * vy0i);
vx2r = (v4f32) __msa_pckev_w((v4i32) vx5, (v4i32) vx4);
dot1 += (vx0r * vy0i);
vx2i = (v4f32) __msa_pckod_w((v4i32) vx5, (v4i32) vx4);
dot2 -= (vx1i * vy1i);
vy2r = (v4f32) __msa_pckev_w((v4i32) vy5, (v4i32) vy4);
dot3 += (vx1r * vy1i); dot3 += (vx1r * vy1i);

vy2i = (v4f32) __msa_pckod_w((v4i32) vy5, (v4i32) vy4);
vx3r = (v4f32) __msa_pckev_w((v4i32) vx7, (v4i32) vx6);
dot4 += (vx2r * vy2r); dot4 += (vx2r * vy2r);
dot4 OP1 (vx2i * vy2i);
dot5 OP2 (vx2i * vy2r);
dot5 += (vx2r * vy2i);

vx3i = (v4f32) __msa_pckod_w((v4i32) vx7, (v4i32) vx6);
dot5 += (vx2i * vy2r);
vy3r = (v4f32) __msa_pckev_w((v4i32) vy7, (v4i32) vy6);
vy3i = (v4f32) __msa_pckod_w((v4i32) vy7, (v4i32) vy6);
dot6 += (vx3r * vy3r); dot6 += (vx3r * vy3r);
dot6 OP1 (vx3i * vy3i);
dot7 OP2 (vx3i * vy3r);
dot7 += (vx3i * vy3r);
dot4 -= (vx2i * vy2i);
dot5 += (vx2r * vy2i);
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i); dot7 += (vx3r * vy3i);
} }




+ 71
- 30
kernel/mips/dasum_msa.c View File

@@ -47,39 +47,55 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)


if (1 == inc_x) if (1 == inc_x)
{ {
#ifdef ENABLE_PREFETCH
FLOAT *x_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
if (n > 31)
{ {
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
x_pref = x + pref_offset + 64;
#endif
FLOAT *x_pref;
BLASLONG pref_offset;


for (i = (n >> 5); i--;)
{
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 128(%[x_pref])\n\t"
"pref 0, 160(%[x_pref])\n\t"
"pref 0, 192(%[x_pref])\n\t"
"pref 0, 224(%[x_pref])\n\t"

: : [x_pref] "r" (x_pref)
);

x_pref += 32;
#endif
pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 64 + 16;


LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7); LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 5) - 1; i--;)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(x_pref, 128);
PREF_OFFSET(x_pref, 160);
PREF_OFFSET(x_pref, 192);
PREF_OFFSET(x_pref, 224);
x_pref += 32;

LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_D(src0);
sum_abs1 += AND_VEC_D(src1);
sum_abs2 += AND_VEC_D(src2);
sum_abs3 += AND_VEC_D(src3);
sum_abs0 += AND_VEC_D(src4);
sum_abs1 += AND_VEC_D(src5);
sum_abs2 += AND_VEC_D(src6);
sum_abs3 += AND_VEC_D(src7);

LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_D(src8);
sum_abs1 += AND_VEC_D(src9);
sum_abs2 += AND_VEC_D(src10);
sum_abs3 += AND_VEC_D(src11);
sum_abs0 += AND_VEC_D(src12);
sum_abs1 += AND_VEC_D(src13);
sum_abs2 += AND_VEC_D(src14);
sum_abs3 += AND_VEC_D(src15);
}

LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15); LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_D(src0); sum_abs0 += AND_VEC_D(src0);
@@ -153,9 +169,34 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
} }
else else
{ {
for (i = (n >> 4); i--;)
if (n > 16)
{ {
LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7); LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 4) - 1; i--;)
{
LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_D(src0);
sum_abs1 += AND_VEC_D(src1);
sum_abs2 += AND_VEC_D(src2);
sum_abs3 += AND_VEC_D(src3);
sum_abs0 += AND_VEC_D(src4);
sum_abs1 += AND_VEC_D(src5);
sum_abs2 += AND_VEC_D(src6);
sum_abs3 += AND_VEC_D(src7);

LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_D(src8);
sum_abs1 += AND_VEC_D(src9);
sum_abs2 += AND_VEC_D(src10);
sum_abs3 += AND_VEC_D(src11);
sum_abs0 += AND_VEC_D(src12);
sum_abs1 += AND_VEC_D(src13);
sum_abs2 += AND_VEC_D(src14);
sum_abs3 += AND_VEC_D(src15);
}

LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15); LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_D(src0); sum_abs0 += AND_VEC_D(src0);


+ 29
- 14
kernel/mips/ddot_msa.c View File

@@ -44,25 +44,40 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)


if ((1 == inc_x) && (1 == inc_y)) if ((1 == inc_x) && (1 == inc_y))
{ {
FLOAT *x_pref, *y_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 32;

pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
y_pref = y + pref_offset + 32;

for (i = (n >> 4); i--;) for (i = (n >> 4); i--;)
{ {
LD_DP8_INC(x, 2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7); LD_DP8_INC(x, 2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_DP8_INC(y, 2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7); LD_DP8_INC(y, 2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);


#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 256(%[x])\n\t"
"pref 0, 288(%[x])\n\t"
"pref 0, 320(%[x])\n\t"
"pref 0, 352(%[x])\n\t"
"pref 0, 256(%[y])\n\t"
"pref 0, 288(%[y])\n\t"
"pref 0, 320(%[y])\n\t"
"pref 0, 352(%[y])\n\t"

: : [x] "r" (x), [y] "r" (y)
);
#endif
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(y_pref, 0);
PREF_OFFSET(y_pref, 32);
PREF_OFFSET(y_pref, 64);
PREF_OFFSET(y_pref, 96);
x_pref += 16;
y_pref += 16;


dot0 += (vy0 * vx0); dot0 += (vy0 * vx0);
dot1 += (vy1 * vx1); dot1 += (vy1 * vx1);


+ 57
- 86
kernel/mips/dtrsm_kernel_LN_8x4_msa.c View File

@@ -45,25 +45,18 @@ void dsolve_8x4_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
FLOAT *c_nxt2line = c + 2 * ldc; FLOAT *c_nxt2line = c + 2 * ldc;
FLOAT *c_nxt3line = c + 3 * ldc; FLOAT *c_nxt3line = c + 3 * ldc;


#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, -96(%[a]) \n\t"
"pref 0, -32(%[a]) \n\t"
"pref 0, -160(%[a]) \n\t"
"pref 0, -224(%[a]) \n\t"
"pref 0, -64(%[a]) \n\t"
"pref 0, -128(%[a]) \n\t"
"pref 0, -192(%[a]) \n\t"
"pref 0, -256(%[a]) \n\t"
"pref 0, -320(%[a]) \n\t"
"pref 0, -384(%[a]) \n\t"
"pref 0, -448(%[a]) \n\t"
"pref 0, -512(%[a]) \n\t"

:
: [a] "r"(a)
);
#endif
PREF_OFFSET(a, -96);
PREF_OFFSET(a, -32);
PREF_OFFSET(a, -160);
PREF_OFFSET(a, -224);
PREF_OFFSET(a, -64);
PREF_OFFSET(a, -128);
PREF_OFFSET(a, -192);
PREF_OFFSET(a, -256);
PREF_OFFSET(a, -320);
PREF_OFFSET(a, -384);
PREF_OFFSET(a, -448);
PREF_OFFSET(a, -512);


LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3); LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3);
LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7); LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7);
@@ -72,29 +65,29 @@ void dsolve_8x4_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)


if (bk > 0) if (bk > 0)
{ {
BLASLONG i;
FLOAT *pba = a, *pbb = b;
v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
BLASLONG i, pref_offset;
FLOAT *pba = a, *pbb = b, *pa0_pref;
v2f64 src_b, src_b0, src_b1;


LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);


for (i = (bk - 1) >> 1; i--;)
if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 128(%[pba]) \n\t"
"pref 0, 160(%[pba]) \n\t"
"pref 0, 192(%[pba]) \n\t"
"pref 0, 224(%[pba]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}

pa0_pref = a + pref_offset;


:
: [pba] "r"(pba)
);
#endif
for (i = bk >> 1; i--;)
{
PREF_OFFSET(pa0_pref, 128);
PREF_OFFSET(pa0_pref, 160);
PREF_OFFSET(pa0_pref, 192);
PREF_OFFSET(pa0_pref, 224);


LD_DP4_INC(pba, 2, src_a8, src_a9, src_a16, src_a17);
LD_DP2_INC(pbb, 2, src_b2, src_b3);
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
@@ -123,33 +116,6 @@ void dsolve_8x4_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1); LD_DP2_INC(pbb, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
src_c0 -= src_a8 * src_b;
src_c1 -= src_a9 * src_b;
src_c2 -= src_a16 * src_b;
src_c3 -= src_a17 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
src_c4 -= src_a8 * src_b;
src_c5 -= src_a9 * src_b;
src_c6 -= src_a16 * src_b;
src_c7 -= src_a17 * src_b;

src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
src_c8 -= src_a8 * src_b;
src_c9 -= src_a9 * src_b;
src_c10 -= src_a16 * src_b;
src_c11 -= src_a17 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
src_c12 -= src_a8 * src_b;
src_c13 -= src_a9 * src_b;
src_c14 -= src_a16 * src_b;
src_c15 -= src_a17 * src_b;
}

if ((bk - 1) & 1)
{
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b; src_c1 -= src_a1 * src_b;
@@ -174,33 +140,38 @@ void dsolve_8x4_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
src_c14 -= src_a2 * src_b; src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b; src_c15 -= src_a3 * src_b;


pa0_pref += 16;
}

if (bk & 1)
{
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1); LD_DP2_INC(pbb, 2, src_b0, src_b1);
}


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
}
} }


a -= 64; a -= 64;


+ 57
- 85
kernel/mips/dtrsm_kernel_LT_8x4_msa.c View File

@@ -44,27 +44,20 @@ void dsolve_8x4_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
FLOAT *c_nxt2line = c + 2 * ldc; FLOAT *c_nxt2line = c + 2 * ldc;
FLOAT *c_nxt3line = c + 3 * ldc; FLOAT *c_nxt3line = c + 3 * ldc;


#ifdef ENABLE_PREFETCH
a += bk * 8; a += bk * 8;
__asm__ __volatile__(
"pref 0, (%[a]) \n\t"
"pref 0, 32(%[a]) \n\t"
"pref 0, 72(%[a]) \n\t"
"pref 0, 104(%[a]) \n\t"
"pref 0, 144(%[a]) \n\t"
"pref 0, 176(%[a]) \n\t"
"pref 0, 216(%[a]) \n\t"
"pref 0, 248(%[a]) \n\t"
"pref 0, 288(%[a]) \n\t"
"pref 0, 360(%[a]) \n\t"
"pref 0, 504(%[a]) \n\t"
"pref 0, 432(%[a]) \n\t"

:
: [a] "r"(a)
);
PREF_OFFSET(a, 0);
PREF_OFFSET(a, 32);
PREF_OFFSET(a, 72);
PREF_OFFSET(a, 104);
PREF_OFFSET(a, 144);
PREF_OFFSET(a, 176);
PREF_OFFSET(a, 216);
PREF_OFFSET(a, 248);
PREF_OFFSET(a, 288);
PREF_OFFSET(a, 360);
PREF_OFFSET(a, 504);
PREF_OFFSET(a, 432);
a -= bk * 8; a -= bk * 8;
#endif


LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3); LD_DP4(c, 2, src_c0, src_c1, src_c2, src_c3);
LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7); LD_DP4(c_nxt1line, 2, src_c4, src_c5, src_c6, src_c7);
@@ -73,28 +66,29 @@ void dsolve_8x4_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)


if (bk) if (bk)
{ {
BLASLONG i;
v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
BLASLONG i, pref_offset;
FLOAT *pa0_pref;
v2f64 src_b, src_b0, src_b1;


LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);


for (i = ((bk - 1) >> 1); i--;)
if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 128(%[a]) \n\t"
"pref 0, 160(%[a]) \n\t"
"pref 0, 192(%[a]) \n\t"
"pref 0, 224(%[a]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}


:
: [a] "r"(a)
);
#endif
pa0_pref = a + pref_offset;


LD_DP4_INC(a, 2, src_a4, src_a5, src_a6, src_a7);
LD_DP2_INC(b, 2, src_b2, src_b3);
for (i = (bk >> 1); i--;)
{
PREF_OFFSET(pa0_pref, 128);
PREF_OFFSET(pa0_pref, 160);
PREF_OFFSET(pa0_pref, 192);
PREF_OFFSET(pa0_pref, 224);

LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
@@ -123,33 +117,6 @@ void dsolve_8x4_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1); LD_DP2_INC(b, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
src_c0 -= src_a4 * src_b;
src_c1 -= src_a5 * src_b;
src_c2 -= src_a6 * src_b;
src_c3 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
src_c4 -= src_a4 * src_b;
src_c5 -= src_a5 * src_b;
src_c6 -= src_a6 * src_b;
src_c7 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
src_c8 -= src_a4 * src_b;
src_c9 -= src_a5 * src_b;
src_c10 -= src_a6 * src_b;
src_c11 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
src_c12 -= src_a4 * src_b;
src_c13 -= src_a5 * src_b;
src_c14 -= src_a6 * src_b;
src_c15 -= src_a7 * src_b;
}

if ((bk - 1) & 1)
{
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b; src_c1 -= src_a1 * src_b;
@@ -174,33 +141,38 @@ void dsolve_8x4_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
src_c14 -= src_a2 * src_b; src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b; src_c15 -= src_a3 * src_b;


pa0_pref += 16;
}

if (bk & 1)
{
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1); LD_DP2_INC(b, 2, src_b0, src_b1);
}


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
}
} }


ILVRL_D2_DP(src_c4, src_c0, res_c0, res_c1); ILVRL_D2_DP(src_c4, src_c0, res_c0, res_c1);


+ 45
- 67
kernel/mips/dtrsm_kernel_RN_8x4_msa.c View File

@@ -46,29 +46,29 @@ void dsolve_8x4_rn_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)


if (bk) if (bk)
{ {
BLASLONG i;
v2f64 src_a0, src_a1, src_a2, src_a3, src_a4, src_a5, src_a6, src_a7;
v2f64 src_b;
BLASLONG i, pref_offset;
FLOAT *pa0_pref;
v2f64 src_a0, src_a1, src_a2, src_a3, src_b;


LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);


for (i = ((bk - 1) >> 1); i--;)
if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 128(%[a]) \n\t"
"pref 0, 160(%[a]) \n\t"
"pref 0, 192(%[a]) \n\t"
"pref 0, 224(%[a]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}

pa0_pref = a + pref_offset;


:
: [a] "r"(a)
);
#endif
for (i = (bk >> 1); i--;)
{
PREF_OFFSET(pa0_pref, 128);
PREF_OFFSET(pa0_pref, 160);
PREF_OFFSET(pa0_pref, 192);
PREF_OFFSET(pa0_pref, 224);


LD_DP4_INC(a, 2, src_a4, src_a5, src_a6, src_a7);
LD_DP2_INC(b, 2, src_b2, src_b3);
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
@@ -97,33 +97,6 @@ void dsolve_8x4_rn_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1); LD_DP2_INC(b, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
src_c0 -= src_a4 * src_b;
src_c1 -= src_a5 * src_b;
src_c2 -= src_a6 * src_b;
src_c3 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
src_c4 -= src_a4 * src_b;
src_c5 -= src_a5 * src_b;
src_c6 -= src_a6 * src_b;
src_c7 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
src_c8 -= src_a4 * src_b;
src_c9 -= src_a5 * src_b;
src_c10 -= src_a6 * src_b;
src_c11 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
src_c12 -= src_a4 * src_b;
src_c13 -= src_a5 * src_b;
src_c14 -= src_a6 * src_b;
src_c15 -= src_a7 * src_b;
}

if ((bk - 1) & 1)
{
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b; src_c1 -= src_a1 * src_b;
@@ -148,33 +121,38 @@ void dsolve_8x4_rn_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
src_c14 -= src_a2 * src_b; src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b; src_c15 -= src_a3 * src_b;


pa0_pref += 16;
}

if (bk & 1)
{
LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(a, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(b, 2, src_b0, src_b1); LD_DP2_INC(b, 2, src_b0, src_b1);
}


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
}
} }


src_b0 = LD_DP(b + 0); src_b0 = LD_DP(b + 0);


+ 47
- 69
kernel/mips/dtrsm_kernel_RT_8x4_msa.c View File

@@ -46,29 +46,29 @@ void dsolve_8x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)


if (bk > 0) if (bk > 0)
{ {
BLASLONG i;
FLOAT *pba = a, *pbb = b;
v2f64 src_b, src_b0, src_b1, src_b2, src_b3;
v2f64 src_a0, src_a1, src_a2, src_a3, src_a4, src_a5, src_a6, src_a7;
BLASLONG i, pref_offset;
FLOAT *pba = a, *pbb = b, *pa0_pref;
v2f64 src_b, src_b0, src_b1, src_a0, src_a1, src_a2, src_a3;

pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);

if (pref_offset)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}


LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);
pa0_pref = a + pref_offset;


for (i = ((bk - 1) >> 1); i--;)
for (i = (bk >> 1); i--;)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 128(%[pba]) \n\t"
"pref 0, 160(%[pba]) \n\t"
"pref 0, 192(%[pba]) \n\t"
"pref 0, 224(%[pba]) \n\t"

:
: [pba] "r"(pba)
);
#endif
LD_DP4_INC(pba, 2, src_a4, src_a5, src_a6, src_a7);
LD_DP2_INC(pbb, 2, src_b2, src_b3);
PREF_OFFSET(pa0_pref, 128);
PREF_OFFSET(pa0_pref, 160);
PREF_OFFSET(pa0_pref, 192);
PREF_OFFSET(pa0_pref, 224);

LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
@@ -97,33 +97,6 @@ void dsolve_8x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1); LD_DP2_INC(pbb, 2, src_b0, src_b1);


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b2, (v2i64) src_b2);
src_c0 -= src_a4 * src_b;
src_c1 -= src_a5 * src_b;
src_c2 -= src_a6 * src_b;
src_c3 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b2, (v2i64) src_b2);
src_c4 -= src_a4 * src_b;
src_c5 -= src_a5 * src_b;
src_c6 -= src_a6 * src_b;
src_c7 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvr_d((v2i64) src_b3, (v2i64) src_b3);
src_c8 -= src_a4 * src_b;
src_c9 -= src_a5 * src_b;
src_c10 -= src_a6 * src_b;
src_c11 -= src_a7 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b3, (v2i64) src_b3);
src_c12 -= src_a4 * src_b;
src_c13 -= src_a5 * src_b;
src_c14 -= src_a6 * src_b;
src_c15 -= src_a7 * src_b;
}

if ((bk - 1) & 1)
{
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0); src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b; src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b; src_c1 -= src_a1 * src_b;
@@ -148,33 +121,38 @@ void dsolve_8x4_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG bk)
src_c14 -= src_a2 * src_b; src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b; src_c15 -= src_a3 * src_b;


pa0_pref += 16;
}

if (bk & 1)
{
LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3); LD_DP4_INC(pba, 2, src_a0, src_a1, src_a2, src_a3);
LD_DP2_INC(pbb, 2, src_b0, src_b1); LD_DP2_INC(pbb, 2, src_b0, src_b1);
}


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b0, (v2i64) src_b0);
src_c0 -= src_a0 * src_b;
src_c1 -= src_a1 * src_b;
src_c2 -= src_a2 * src_b;
src_c3 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvl_d((v2i64) src_b0, (v2i64) src_b0);
src_c4 -= src_a0 * src_b;
src_c5 -= src_a1 * src_b;
src_c6 -= src_a2 * src_b;
src_c7 -= src_a3 * src_b;


src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
src_b = (v2f64) __msa_ilvr_d((v2i64) src_b1, (v2i64) src_b1);
src_c8 -= src_a0 * src_b;
src_c9 -= src_a1 * src_b;
src_c10 -= src_a2 * src_b;
src_c11 -= src_a3 * src_b;

src_b = (v2f64) __msa_ilvl_d((v2i64) src_b1, (v2i64) src_b1);
src_c12 -= src_a0 * src_b;
src_c13 -= src_a1 * src_b;
src_c14 -= src_a2 * src_b;
src_c15 -= src_a3 * src_b;
}
} }


a -= 32; a -= 32;


+ 17
- 0
kernel/mips/macros_msa.h View File

@@ -32,6 +32,23 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


#define ENABLE_PREFETCH #define ENABLE_PREFETCH


#ifdef ENABLE_PREFETCH
inline static void prefetch_load_lf(unsigned char *src)
{
__asm__ __volatile__("pref 0, 0(%[src]) \n\t" : : [src] "r" (src));
}

#define PREFETCH(PTR) prefetch_load_lf((unsigned char *)(PTR));

#define STRNG(X) #X
#define PREF_OFFSET(src_ptr, offset) \
__asm__ __volatile__("pref 0, " STRNG(offset) "(%[src]) \n\t" : : [src] "r" (src_ptr));

#else
#define PREFETCH(PTR)
#define PREF_OFFSET(src_ptr, offset)
#endif

#define LD_W(RTYPE, psrc) *((RTYPE *)(psrc)) #define LD_W(RTYPE, psrc) *((RTYPE *)(psrc))
#define LD_SP(...) LD_W(v4f32, __VA_ARGS__) #define LD_SP(...) LD_W(v4f32, __VA_ARGS__)




+ 45
- 29
kernel/mips/sasum_msa.c View File

@@ -48,39 +48,55 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)


if (1 == inc_x) if (1 == inc_x)
{ {
#ifdef ENABLE_PREFETCH
FLOAT *x_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
if (n > 63)
{ {
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
x_pref = x + pref_offset + 128;
#endif
FLOAT *x_pref;
BLASLONG pref_offset;


for (i = 0; i < (n >> 6); i++)
{
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 128(%[x_pref])\n\t"
"pref 0, 160(%[x_pref])\n\t"
"pref 0, 192(%[x_pref])\n\t"
"pref 0, 224(%[x_pref])\n\t"

: : [x_pref] "r" (x_pref)
);

x_pref += 64;
#endif
pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 128 + 32;


LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7); LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = 0; i < (n >> 6) - 1; i++)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(x_pref, 128);
PREF_OFFSET(x_pref, 160);
PREF_OFFSET(x_pref, 192);
PREF_OFFSET(x_pref, 224);
x_pref += 64;

LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_W(src0);
sum_abs1 += AND_VEC_W(src1);
sum_abs2 += AND_VEC_W(src2);
sum_abs3 += AND_VEC_W(src3);
sum_abs0 += AND_VEC_W(src4);
sum_abs1 += AND_VEC_W(src5);
sum_abs2 += AND_VEC_W(src6);
sum_abs3 += AND_VEC_W(src7);

LD_SP8_INC(x, 4, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_W(src8);
sum_abs1 += AND_VEC_W(src9);
sum_abs2 += AND_VEC_W(src10);
sum_abs3 += AND_VEC_W(src11);
sum_abs0 += AND_VEC_W(src12);
sum_abs1 += AND_VEC_W(src13);
sum_abs2 += AND_VEC_W(src14);
sum_abs3 += AND_VEC_W(src15);
}

LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15); LD_SP8_INC(x, 4, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_W(src0); sum_abs0 += AND_VEC_W(src0);


+ 29
- 14
kernel/mips/sdot_msa.c View File

@@ -48,25 +48,40 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)


if ((1 == inc_x) && (1 == inc_y)) if ((1 == inc_x) && (1 == inc_y))
{ {
FLOAT *x_pref, *y_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 64;

pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
y_pref = y + pref_offset + 64;

for (i = (n >> 5); i--;) for (i = (n >> 5); i--;)
{ {
LD_SP8_INC(x, 4, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7); LD_SP8_INC(x, 4, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_SP8_INC(y, 4, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7); LD_SP8_INC(y, 4, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);


#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 256(%[x])\n\t"
"pref 0, 288(%[x])\n\t"
"pref 0, 320(%[x])\n\t"
"pref 0, 352(%[x])\n\t"
"pref 0, 256(%[y])\n\t"
"pref 0, 288(%[y])\n\t"
"pref 0, 320(%[y])\n\t"
"pref 0, 352(%[y])\n\t"

: : [x] "r" (x), [y] "r" (y)
);
#endif
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(y_pref, 0);
PREF_OFFSET(y_pref, 32);
PREF_OFFSET(y_pref, 64);
PREF_OFFSET(y_pref, 96);
x_pref += 32;
y_pref += 32;


dot0 += (vy0 * vx0); dot0 += (vy0 * vx0);
dot1 += (vy1 * vx1); dot1 += (vy1 * vx1);


+ 19
- 14
kernel/mips/strsm_kernel_LN_8x8_msa.c View File

@@ -58,21 +58,24 @@ static void ssolve_8x8_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO


if (bk > 0) if (bk > 0)
{ {
BLASLONG k;
FLOAT *aa = a, *bb = b;
v4f32 src_bb0, src_bb1, src_b0, src_b1, src_b2, src_b3, src_a1;
BLASLONG k, pref_offset;
FLOAT *aa = a, *bb = b, *pa0_pref;
v4f32 src_a1, src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;


for (k = 0; k < (bk >> 1); k++)
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);

if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 64(%[aa]) \n\t"
"pref 0, 96(%[aa]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}


:
: [aa] "r" (aa)
);
#endif
pa0_pref = a + pref_offset;

for (k = 0; k < (bk >> 1); k++)
{
PREF_OFFSET(pa0_pref, 64);
PREF_OFFSET(pa0_pref, 96);


LD_SP2_INC(aa, 4, src_a0, src_a1); LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1); LD_SP2_INC(bb, 4, src_bb0, src_bb1);
@@ -119,12 +122,14 @@ static void ssolve_8x8_ln_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
src_c13 -= src_a1 * src_b2; src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3; src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3; src_c15 -= src_a1 * src_b3;

pa0_pref += 16;
} }


if (bk & 1) if (bk & 1)
{ {
LD_SP2(aa, 4, src_a0, src_a1);
LD_SP2(bb, 4, src_bb0, src_bb1);
LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1);


SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3); SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3);
src_c0 -= src_a0 * src_b0; src_c0 -= src_a0 * src_b0;


+ 17
- 11
kernel/mips/strsm_kernel_LT_8x8_msa.c View File

@@ -58,20 +58,24 @@ static void ssolve_8x8_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO


if (bk > 0) if (bk > 0)
{ {
BLASLONG k;
v4f32 src_b, src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;
BLASLONG k, pref_offset;
FLOAT *pa0_pref;
v4f32 src_b0, src_b1, src_b2, src_b3, src_bb0, src_bb1;


for (k = 0; k < (bk >> 1); k++)
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);

if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 64(%[a]) \n\t"
"pref 0, 96(%[a]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}


:
: [a] "r" (a)
);
#endif
pa0_pref = a + pref_offset;

for (k = 0; k < (bk >> 1); k++)
{
PREF_OFFSET(pa0_pref, 64);
PREF_OFFSET(pa0_pref, 96);


LD_SP2_INC(a, 4, src_a0, src_a1); LD_SP2_INC(a, 4, src_a0, src_a1);
LD_SP2_INC(b, 4, src_bb0, src_bb1); LD_SP2_INC(b, 4, src_bb0, src_bb1);
@@ -118,6 +122,8 @@ static void ssolve_8x8_lt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
src_c13 -= src_a1 * src_b2; src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3; src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3; src_c15 -= src_a1 * src_b3;

pa0_pref += 16;
} }


if (bk & 1) if (bk & 1)


+ 17
- 10
kernel/mips/strsm_kernel_RN_8x8_msa.c View File

@@ -56,20 +56,25 @@ static void ssolve_8x8_rn_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO


if (bk > 0) if (bk > 0)
{ {
BLASLONG k;
BLASLONG k, pref_offset;
FLOAT *pa0_pref;
v4f32 src_a0, src_a1, src_bb0, src_bb1; v4f32 src_a0, src_a1, src_bb0, src_bb1;


pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);

if (pref_offset)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}

pa0_pref = a + pref_offset;

for (k = 0; k < (bk >> 1); k++) for (k = 0; k < (bk >> 1); k++)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 64(%[a]) \n\t"
"pref 0, 96(%[a]) \n\t"

:
: [a] "r" (a)
);
#endif
PREF_OFFSET(pa0_pref, 64);
PREF_OFFSET(pa0_pref, 96);

LD_SP2_INC(a, 4, src_a0, src_a1); LD_SP2_INC(a, 4, src_a0, src_a1);
LD_SP2_INC(b, 4, src_bb0, src_bb1); LD_SP2_INC(b, 4, src_bb0, src_bb1);


@@ -115,6 +120,8 @@ static void ssolve_8x8_rn_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
src_c13 -= src_a1 * src_b2; src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3; src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3; src_c15 -= src_a1 * src_b3;

pa0_pref += 16;
} }


if (bk & 1) if (bk & 1)


+ 18
- 13
kernel/mips/strsm_kernel_RT_8x8_msa.c View File

@@ -56,21 +56,24 @@ static void ssolve_8x8_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO


if (bk > 0) if (bk > 0)
{ {
BLASLONG k;
FLOAT *aa = a, *bb = b;
BLASLONG k, pref_offset;
FLOAT *aa = a, *bb = b, *pa0_pref;
v4f32 src_a0, src_a1, src_b1, src_b2, src_b3, src_bb0, src_bb1; v4f32 src_a0, src_a1, src_b1, src_b2, src_b3, src_bb0, src_bb1;


for (k = 0; k < (bk >> 1); k++)
pref_offset = (uintptr_t)a & (L1_DATA_LINESIZE - 1);

if (pref_offset)
{ {
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 64(%[aa]) \n\t"
"pref 0, 96(%[aa]) \n\t"
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}


:
: [aa] "r" (aa)
);
#endif
pa0_pref = a + pref_offset;

for (k = 0; k < (bk >> 1); k++)
{
PREF_OFFSET(pa0_pref, 64);
PREF_OFFSET(pa0_pref, 96);


LD_SP2_INC(aa, 4, src_a0, src_a1); LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1); LD_SP2_INC(bb, 4, src_bb0, src_bb1);
@@ -117,12 +120,14 @@ static void ssolve_8x8_rt_msa(FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLO
src_c13 -= src_a1 * src_b2; src_c13 -= src_a1 * src_b2;
src_c14 -= src_a0 * src_b3; src_c14 -= src_a0 * src_b3;
src_c15 -= src_a1 * src_b3; src_c15 -= src_a1 * src_b3;

pa0_pref += 16;
} }


if (bk & 1) if (bk & 1)
{ {
LD_SP2(aa, 4, src_a0, src_a1);
LD_SP2(bb, 4, src_bb0, src_bb1);
LD_SP2_INC(aa, 4, src_a0, src_a1);
LD_SP2_INC(bb, 4, src_bb0, src_bb1);


SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3); SPLATI_W4_SP(src_bb0, src_b0, src_b1, src_b2, src_b3);
src_c0 -= src_a0 * src_b0; src_c0 -= src_a0 * src_b0;


+ 71
- 30
kernel/mips/zasum_msa.c View File

@@ -47,39 +47,55 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)


if (1 == inc_x) if (1 == inc_x)
{ {
#ifdef ENABLE_PREFETCH
FLOAT *x_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
if (n > 16)
{ {
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(FLOAT);
x_pref = x + pref_offset + 64;
#endif
FLOAT *x_pref;
BLASLONG pref_offset;


for (i = (n >> 4); i--;)
{
#ifdef ENABLE_PREFETCH
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 128(%[x_pref])\n\t"
"pref 0, 160(%[x_pref])\n\t"
"pref 0, 192(%[x_pref])\n\t"
"pref 0, 224(%[x_pref])\n\t"

: : [x_pref] "r" (x_pref)
);

x_pref += 32;
#endif
pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 64 + 16;


LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7); LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 4) - 1; i--;)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(x_pref, 128);
PREF_OFFSET(x_pref, 160);
PREF_OFFSET(x_pref, 192);
PREF_OFFSET(x_pref, 224);
x_pref += 32;

LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_D(src0);
sum_abs1 += AND_VEC_D(src1);
sum_abs2 += AND_VEC_D(src2);
sum_abs3 += AND_VEC_D(src3);
sum_abs0 += AND_VEC_D(src4);
sum_abs1 += AND_VEC_D(src5);
sum_abs2 += AND_VEC_D(src6);
sum_abs3 += AND_VEC_D(src7);

LD_DP8_INC(x, 2, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_D(src8);
sum_abs1 += AND_VEC_D(src9);
sum_abs2 += AND_VEC_D(src10);
sum_abs3 += AND_VEC_D(src11);
sum_abs0 += AND_VEC_D(src12);
sum_abs1 += AND_VEC_D(src13);
sum_abs2 += AND_VEC_D(src14);
sum_abs3 += AND_VEC_D(src15);
}

LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15); LD_DP8_INC(x, 2, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_D(src0); sum_abs0 += AND_VEC_D(src0);
@@ -149,9 +165,34 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x)
{ {
inc_x *= 2; inc_x *= 2;


for (i = (n >> 4); i--;)
if (n > 16)
{ {
LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7); LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);
for (i = (n >> 4) - 1; i--;)
{
LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);

sum_abs0 += AND_VEC_D(src0);
sum_abs1 += AND_VEC_D(src1);
sum_abs2 += AND_VEC_D(src2);
sum_abs3 += AND_VEC_D(src3);
sum_abs0 += AND_VEC_D(src4);
sum_abs1 += AND_VEC_D(src5);
sum_abs2 += AND_VEC_D(src6);
sum_abs3 += AND_VEC_D(src7);

LD_DP8_INC(x, inc_x, src0, src1, src2, src3, src4, src5, src6, src7);

sum_abs0 += AND_VEC_D(src8);
sum_abs1 += AND_VEC_D(src9);
sum_abs2 += AND_VEC_D(src10);
sum_abs3 += AND_VEC_D(src11);
sum_abs0 += AND_VEC_D(src12);
sum_abs1 += AND_VEC_D(src13);
sum_abs2 += AND_VEC_D(src14);
sum_abs3 += AND_VEC_D(src15);
}

LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15); LD_DP8_INC(x, inc_x, src8, src9, src10, src11, src12, src13, src14, src15);


sum_abs0 += AND_VEC_D(src0); sum_abs0 += AND_VEC_D(src0);


+ 210
- 87
kernel/mips/zdot_msa.c View File

@@ -45,8 +45,8 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA
BLASLONG i = 0; BLASLONG i = 0;
FLOAT dot[2]; FLOAT dot[2];
BLASLONG inc_x2, inc_y2; BLASLONG inc_x2, inc_y2;
v2f64 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7;
v2f64 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7;
v2f64 vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7, vx8, vx9, vx10, vx11;
v2f64 vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7, vy8, vy9, vy10, vy11;
v2f64 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i; v2f64 vx0r, vx0i, vx1r, vx1i, vx2r, vx2i, vx3r, vx3i;
v2f64 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i; v2f64 vy0r, vy0i, vy1r, vy1i, vy2r, vy2i, vy3r, vy3i;
v2f64 dot0 = {0, 0}; v2f64 dot0 = {0, 0};
@@ -71,116 +71,239 @@ OPENBLAS_COMPLEX_FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLA
inc_x2 = 2 * inc_x; inc_x2 = 2 * inc_x;
inc_y2 = 2 * inc_y; inc_y2 = 2 * inc_y;



#ifdef ENABLE_PREFETCH
if ((1 == inc_x) && (1 == inc_y)) if ((1 == inc_x) && (1 == inc_y))
{ {
double *x_pref, *y_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(double);
x_pref = x + pref_offset + 32;

pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
}
pref_offset = pref_offset / sizeof(double);
y_pref = y + pref_offset + 32;

for (i = (n >> 3); i--;)
if (n > 7)
{ {
__asm__ __volatile__(
"pref 0, 0(%[x_pref])\n\t"
"pref 0, 32(%[x_pref])\n\t"
"pref 0, 64(%[x_pref])\n\t"
"pref 0, 96(%[x_pref])\n\t"
"pref 0, 0(%[y_pref])\n\t"
"pref 0, 32(%[y_pref])\n\t"
"pref 0, 64(%[y_pref])\n\t"
"pref 0, 96(%[y_pref])\n\t"

: : [x_pref] "r" (x_pref), [y_pref] "r" (y_pref)
);

x_pref += 16;
y_pref += 16;

LD_DP8_INC(x, 2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_DP8_INC(y, 2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
FLOAT *x_pref, *y_pref;
BLASLONG pref_offset;

pref_offset = (BLASLONG)x & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
x_pref = x + pref_offset + 32 + 8;

pref_offset = (BLASLONG)y & (L1_DATA_LINESIZE - 1);
if (pref_offset > 0)
{
pref_offset = L1_DATA_LINESIZE - pref_offset;
pref_offset = pref_offset / sizeof(FLOAT);
}
y_pref = y + pref_offset + 32 + 8;

LD_DP4_INC(x, 2, vx0, vx1, vx2, vx3);
LD_DP4_INC(y, 2, vy0, vy1, vy2, vy3);


PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i); PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i);
PCKEVOD_D2_DP(vx3, vx2, vx1r, vx1i);
PCKEVOD_D2_DP(vx5, vx4, vx2r, vx2i);
PCKEVOD_D2_DP(vx7, vx6, vx3r, vx3i);

PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i); PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i);
PCKEVOD_D2_DP(vy3, vy2, vy1r, vy1i);
PCKEVOD_D2_DP(vy5, vy4, vy2r, vy2i);
PCKEVOD_D2_DP(vy7, vy6, vy3r, vy3i);


for (i = (n >> 3) - 1; i--;)
{
PREF_OFFSET(x_pref, 0);
PREF_OFFSET(x_pref, 32);
PREF_OFFSET(x_pref, 64);
PREF_OFFSET(x_pref, 96);
PREF_OFFSET(y_pref, 0);
PREF_OFFSET(y_pref, 32);
PREF_OFFSET(y_pref, 64);
PREF_OFFSET(y_pref, 96);
x_pref += 16;
y_pref += 16;

vx4 = LD_DP(x); x += 2;
vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
dot0 += (vx0r * vy0r);
vx5 = LD_DP(x); x += 2;
vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_DP(y); y += 2;
vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r);
vy5 = LD_DP(y); y += 2;
vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_DP(x); x += 2;
vx7 = LD_DP(x); x += 2;
vy6 = LD_DP(y); y += 2;
vy7 = LD_DP(y); y += 2;
vx8 = LD_DP(x); x += 2;
dot0 -= (vx0i * vy0i);
vx9 = LD_DP(x); x += 2;
vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
dot1 += (vx0r * vy0i);
vy8 = LD_DP(y); y += 2;
vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
dot2 -= (vx1i * vy1i);
vy9 = LD_DP(y); y += 2;
vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i);
vx10 = LD_DP(x); x += 2;
vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
vx11 = LD_DP(x); x += 2;
vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r);
vy10 = LD_DP(y); y += 2;
vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
dot5 += (vx2i * vy2r);
vy11 = LD_DP(y); y += 2;
vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r);
vx0r = (v2f64) __msa_pckev_d((v2i64) vx9, (v2i64) vx8);
dot7 += (vx3i * vy3r);
vx0i = (v2f64) __msa_pckod_d((v2i64) vx9, (v2i64) vx8);
vy0r = (v2f64) __msa_pckev_d((v2i64) vy9, (v2i64) vy8);
vx2 = vx10;
vy0i = (v2f64) __msa_pckod_d((v2i64) vy9, (v2i64) vy8);
vx3 = vx11;
dot4 -= (vx2i * vy2i);
vy2 = vy10;
dot5 += (vx2r * vy2i);
vy3 = vy11;
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}

vx4 = LD_DP(x); x += 2;
vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
dot0 += (vx0r * vy0r); dot0 += (vx0r * vy0r);
dot0 OP1 (vx0i * vy0i);
dot1 OP2 (vx0i * vy0r);
dot1 += (vx0r * vy0i);

vx5 = LD_DP(x); x += 2;
vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_DP(y); y += 2;
vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r); dot2 += (vx1r * vy1r);
dot2 OP1 (vx1i * vy1i);
dot3 OP2 (vx1i * vy1r);
vy5 = LD_DP(y); y += 2;
vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_DP(x); x += 2;
vx7 = LD_DP(x); x += 2;
vy6 = LD_DP(y); y += 2;
vy7 = LD_DP(y); y += 2;
dot0 -= (vx0i * vy0i);
vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
dot1 += (vx0r * vy0i);
vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
dot2 -= (vx1i * vy1i);
vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i); dot3 += (vx1r * vy1i);

vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r); dot4 += (vx2r * vy2r);
dot4 OP1 (vx2i * vy2i);
dot5 OP2 (vx2i * vy2r);
dot5 += (vx2r * vy2i);

vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
dot5 += (vx2i * vy2r);
vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r); dot6 += (vx3r * vy3r);
dot6 OP1 (vx3i * vy3i);
dot7 OP2 (vx3i * vy3r);
dot7 += (vx3i * vy3r);
dot4 -= (vx2i * vy2i);
dot5 += (vx2r * vy2i);
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i); dot7 += (vx3r * vy3i);
} }
} }
else
#endif
for (i = (n >> 3); i--;)
else if (n > 7)
{ {
LD_DP8_INC(x, inc_x2, vx0, vx1, vx2, vx3, vx4, vx5, vx6, vx7);
LD_DP8_INC(y, inc_y2, vy0, vy1, vy2, vy3, vy4, vy5, vy6, vy7);
LD_DP4_INC(x, inc_x2, vx0, vx1, vx2, vx3);
LD_DP4_INC(y, inc_y2, vy0, vy1, vy2, vy3);


PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i); PCKEVOD_D2_DP(vx1, vx0, vx0r, vx0i);
PCKEVOD_D2_DP(vx3, vx2, vx1r, vx1i);
PCKEVOD_D2_DP(vx5, vx4, vx2r, vx2i);
PCKEVOD_D2_DP(vx7, vx6, vx3r, vx3i);

PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i); PCKEVOD_D2_DP(vy1, vy0, vy0r, vy0i);
PCKEVOD_D2_DP(vy3, vy2, vy1r, vy1i);
PCKEVOD_D2_DP(vy5, vy4, vy2r, vy2i);
PCKEVOD_D2_DP(vy7, vy6, vy3r, vy3i);


dot0 += (vx0r * vy0r);
dot0 OP1 (vx0i * vy0i);
dot1 OP2 (vx0i * vy0r);
dot1 += (vx0r * vy0i);
for (i = (n >> 3) - 1; i--;)
{
vx4 = LD_DP(x); x += inc_x2;
vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
dot0 += (vx0r * vy0r);
vx5 = LD_DP(x); x += inc_x2;
vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_DP(y); y += inc_y2;
vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r);
vy5 = LD_DP(y); y += inc_y2;
vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_DP(x); x += inc_x2;
vx7 = LD_DP(x); x += inc_x2;
vy6 = LD_DP(y); y += inc_y2;
vy7 = LD_DP(y); y += inc_y2;
vx8 = LD_DP(x); x += inc_x2;
dot0 -= (vx0i * vy0i);
vx9 = LD_DP(x); x += inc_x2;
vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
dot1 += (vx0r * vy0i);
vy8 = LD_DP(y); y += inc_y2;
vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
dot2 -= (vx1i * vy1i);
vy9 = LD_DP(y); y += inc_y2;
vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i);
vx10 = LD_DP(x); x += inc_x2;
vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
vx11 = LD_DP(x); x += inc_x2;
vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r);
vy10 = LD_DP(y); y += inc_y2;
vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
dot5 += (vx2i * vy2r);
vy11 = LD_DP(y); y += inc_y2;
vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r);
vx0r = (v2f64) __msa_pckev_d((v2i64) vx9, (v2i64) vx8);
dot7 += (vx3i * vy3r);
vx0i = (v2f64) __msa_pckod_d((v2i64) vx9, (v2i64) vx8);
vy0r = (v2f64) __msa_pckev_d((v2i64) vy9, (v2i64) vy8);
vx2 = vx10;
vy0i = (v2f64) __msa_pckod_d((v2i64) vy9, (v2i64) vy8);
vx3 = vx11;
dot4 -= (vx2i * vy2i);
vy2 = vy10;
dot5 += (vx2r * vy2i);
vy3 = vy11;
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i);
}


vx4 = LD_DP(x); x += inc_x2;
vx1r = (v2f64) __msa_pckev_d((v2i64) vx3, (v2i64) vx2);
dot0 += (vx0r * vy0r);
vx5 = LD_DP(x); x += inc_x2;
vx1i = (v2f64) __msa_pckod_d((v2i64) vx3, (v2i64) vx2);
dot1 += (vx0i * vy0r);
vy4 = LD_DP(y); y += inc_y2;
vy1r = (v2f64) __msa_pckev_d((v2i64) vy3, (v2i64) vy2);
dot2 += (vx1r * vy1r); dot2 += (vx1r * vy1r);
dot2 OP1 (vx1i * vy1i);
dot3 OP2 (vx1i * vy1r);
vy5 = LD_DP(y); y += inc_y2;
vy1i = (v2f64) __msa_pckod_d((v2i64) vy3, (v2i64) vy2);
dot3 += (vx1i * vy1r);
vx6 = LD_DP(x); x += inc_x2;
vx7 = LD_DP(x); x += inc_x2;
vy6 = LD_DP(y); y += inc_y2;
vy7 = LD_DP(y); y += inc_y2;
dot0 -= (vx0i * vy0i);
vx2r = (v2f64) __msa_pckev_d((v2i64) vx5, (v2i64) vx4);
dot1 += (vx0r * vy0i);
vx2i = (v2f64) __msa_pckod_d((v2i64) vx5, (v2i64) vx4);
dot2 -= (vx1i * vy1i);
vy2r = (v2f64) __msa_pckev_d((v2i64) vy5, (v2i64) vy4);
dot3 += (vx1r * vy1i); dot3 += (vx1r * vy1i);

vy2i = (v2f64) __msa_pckod_d((v2i64) vy5, (v2i64) vy4);
vx3r = (v2f64) __msa_pckev_d((v2i64) vx7, (v2i64) vx6);
dot4 += (vx2r * vy2r); dot4 += (vx2r * vy2r);
dot4 OP1 (vx2i * vy2i);
dot5 OP2 (vx2i * vy2r);
dot5 += (vx2r * vy2i);

vx3i = (v2f64) __msa_pckod_d((v2i64) vx7, (v2i64) vx6);
dot5 += (vx2i * vy2r);
vy3r = (v2f64) __msa_pckev_d((v2i64) vy7, (v2i64) vy6);
vy3i = (v2f64) __msa_pckod_d((v2i64) vy7, (v2i64) vy6);
dot6 += (vx3r * vy3r); dot6 += (vx3r * vy3r);
dot6 OP1 (vx3i * vy3i);
dot7 OP2 (vx3i * vy3r);
dot7 += (vx3i * vy3r);
dot4 -= (vx2i * vy2i);
dot5 += (vx2r * vy2i);
dot6 -= (vx3i * vy3i);
dot7 += (vx3r * vy3i); dot7 += (vx3r * vy3i);
} }




Write Preview
Loading…
Cancel
Save