From ea2faf0c9acf561617cb5d62e4b009ce7458b69a Mon Sep 17 00:00:00 2001 From: Chris Sidebottom Date: Mon, 21 Jul 2025 17:09:47 +0000 Subject: [PATCH] Add optimized BGEMM for NEOVERSEN2 target This re-uses the existing NEOVERSEN2 8x4 `sbgemm` kernel to implement `bgemm`. --- kernel/arm64/KERNEL.NEOVERSEN2 | 17 +- kernel/arm64/KERNEL.NEOVERSEV2 | 7 +- kernel/arm64/sbgemm_kernel_8x4_neoversen2.c | 18 +- .../arm64/sbgemm_kernel_8x4_neoversen2_impl.c | 243 +++++++++++------- param.h | 10 +- test/compare_sgemm_bgemm.c | 8 +- test/compare_sgemv_bgemv.c | 4 +- 7 files changed, 189 insertions(+), 118 deletions(-) diff --git a/kernel/arm64/KERNEL.NEOVERSEN2 b/kernel/arm64/KERNEL.NEOVERSEN2 index c8d511f20..5ada34e6b 100644 --- a/kernel/arm64/KERNEL.NEOVERSEN2 +++ b/kernel/arm64/KERNEL.NEOVERSEN2 @@ -188,6 +188,20 @@ ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) +ifeq ($(BUILD_BFLOAT16), 1) +BGEMM_BETA = sbgemm_beta_neoversen2.c +BGEMMKERNEL = sbgemm_kernel_$(BGEMM_UNROLL_M)x$(BGEMM_UNROLL_N)_neoversen2.c +BGEMMINCOPY = sbgemm_ncopy_$(BGEMM_UNROLL_M)_neoversen2.c +BGEMMITCOPY = sbgemm_tcopy_$(BGEMM_UNROLL_M)_neoversen2.c +BGEMMONCOPY = sbgemm_ncopy_$(BGEMM_UNROLL_N)_neoversen2.c +BGEMMOTCOPY = sbgemm_tcopy_$(BGEMM_UNROLL_N)_neoversen2.c +BGEMMINCOPYOBJ = bgemm_incopy$(TSUFFIX).$(SUFFIX) +BGEMMITCOPYOBJ = bgemm_itcopy$(TSUFFIX).$(SUFFIX) +BGEMMONCOPYOBJ = bgemm_oncopy$(TSUFFIX).$(SUFFIX) +BGEMMOTCOPYOBJ = bgemm_otcopy$(TSUFFIX).$(SUFFIX) +BGEMVTKERNEL = sbgemv_t_bfdot.c +BGEMVNKERNEL = bgemv_n_sve_v3x4.c + SBGEMM_BETA = sbgemm_beta_neoversen2.c SBGEMMKERNEL = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c SBGEMMINCOPY = sbgemm_ncopy_$(SBGEMM_UNROLL_M)_neoversen2.c @@ -199,4 +213,5 @@ SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX) SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX) SBGEMMOTCOPYOBJ = sbgemm_otcopy$(TSUFFIX).$(SUFFIX) SBGEMVTKERNEL = sbgemv_t_bfdot.c -SBGEMVNKERNEL = sbgemv_n_neon.c \ No newline at end of file +SBGEMVNKERNEL = sbgemv_n_neon.c +endif diff --git a/kernel/arm64/KERNEL.NEOVERSEV2 b/kernel/arm64/KERNEL.NEOVERSEV2 index e08efdb9d..8f58833cd 100644 --- a/kernel/arm64/KERNEL.NEOVERSEV2 +++ b/kernel/arm64/KERNEL.NEOVERSEV2 @@ -1,6 +1 @@ -include $(KERNELDIR)/KERNEL.ARMV8SVE - -ifeq ($(BUILD_BFLOAT16), 1) -SBGEMVTKERNEL = sbgemv_t_bfdot.c -SBGEMVNKERNEL = sbgemv_n_neon.c -endif \ No newline at end of file +include $(KERNELDIR)/KERNEL.NEOVERSEN2 diff --git a/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c b/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c index 4c1385fbe..42ecf3719 100644 --- a/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c +++ b/kernel/arm64/sbgemm_kernel_8x4_neoversen2.c @@ -1,5 +1,5 @@ /*************************************************************************** - * Copyright (c) 2022, The OpenBLAS Project + * Copyright (c) 2022,2025 The OpenBLAS Project * All rights reserved. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -33,13 +33,23 @@ #define ALPHA_ONE #include "sbgemm_kernel_8x4_neoversen2_impl.c" #undef ALPHA_ONE +#undef UPDATE_C #include "sbgemm_kernel_8x4_neoversen2_impl.c" int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B, FLOAT *C, BLASLONG ldc) { - if (alpha == 1.0f) - return sbgemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc); +#ifdef BGEMM + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + float alpha_f32 = vcvtah_f32_bf16(alpha_bf16); +#else + float alpha_f32 = alpha; +#endif + + if (alpha_f32 == 1.0f) + return gemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc); else - return sbgemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc); + return gemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc); + return 0; } diff --git a/kernel/arm64/sbgemm_kernel_8x4_neoversen2_impl.c b/kernel/arm64/sbgemm_kernel_8x4_neoversen2_impl.c index 26ea7ee61..712949f2c 100644 --- a/kernel/arm64/sbgemm_kernel_8x4_neoversen2_impl.c +++ b/kernel/arm64/sbgemm_kernel_8x4_neoversen2_impl.c @@ -1,5 +1,5 @@ /*************************************************************************** - * Copyright (c) 2022, The OpenBLAS Project + * Copyright (c) 2022,2025 The OpenBLAS Project * All rights reserved. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are @@ -46,49 +46,94 @@ INIT_C(3, 1); \ } while (0); +#ifdef BGEMM #ifdef ALPHA_ONE -#define UPDATE_C(PG, PTR, DST, SRC) \ - do { \ - DST = svld1_f32((PG), (PTR)); \ - DST = svadd_z((PG), SRC, DST); \ - svst1_f32((PG), (PTR), DST); \ +#define UPDATE_C(PG16, PG32, PTR, SRC) \ + do { \ + tmp32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + tmp32 = svadd_z((PG32), SRC, tmp32); \ + tmp16 = svcvt_bf16_f32_z((PG32), tmp32); \ + tmp16 = svuzp1_bf16(tmp16, tmp16); \ + svst1_bf16((PG16), (PTR), tmp16); \ + } while (0) +#else +#define UPDATE_C(PG16, PG32, PTR, SRC) \ + do { \ + tmp32 = svreinterpret_f32_u32(svld1uh_u32((PG16), (uint16_t*)PTR)); \ + tmp32 = svmad_z((PG32), svalpha, SRC, tmp32); \ + tmp16 = svcvt_bf16_f32_z((PG32), tmp32); \ + tmp16 = svuzp1_bf16(tmp16, tmp16); \ + svst1_bf16((PG16), (PTR), tmp16); \ + } while (0) +#endif +#else +#ifdef ALPHA_ONE +#define UPDATE_C(PG16, PG32, PTR, SRC) \ + do { \ + tmp32 = svld1_f32((PG32), (PTR)); \ + tmp32 = svadd_z((PG32), SRC, tmp32); \ + svst1_f32((PG32), (PTR), tmp32); \ } while (0); #else -#define UPDATE_C(PG, PTR, DST, SRC) \ - do { \ - DST = svld1_f32((PG), (PTR)); \ - DST = svmad_z((PG), svalpha, SRC, DST); \ - svst1_f32((PG), (PTR), DST); \ +#define UPDATE_C(PG16, PG32, PTR, SRC) \ + do { \ + tmp32 = svld1_f32((PG32), (PTR)); \ + tmp32 = svmad_z((PG32), svalpha, SRC, tmp32); \ + svst1_f32((PG32), (PTR), tmp32); \ } while (0); + #endif + #endif + +#ifdef BGEMM +#define OUTPUT_FLOAT bfloat16_t +#else +#define OUTPUT_FLOAT float #endif #ifdef ALPHA_ONE -int sbgemm_kernel_neoversen2_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc) +static int gemm_kernel_neoversen2_alpha_one(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc) #else -int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc) +static int gemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT * A, IFLOAT * B, FLOAT * C, BLASLONG ldc) #endif { BLASLONG pad_k = (k + 3) & ~3; svbfloat16_t ma0, ma1, ma2, ma3, mb0, mb1; svfloat32_t mc00, mc01, mc10, mc11, mc20, mc21, mc30, mc31, - vc0, vc1, vc2, vc3, vc4, vc5, vc6, vc7, - oc0, oc1, oc2, oc3, oc4, oc5, oc6, oc7; + vc0, vc1, vc2, vc3, vc4, vc5, vc6, vc7; + +#ifndef ALPHA_ONE +#ifdef BGEMM + bfloat16_t alpha_bf16; + memcpy(&alpha_bf16, &alpha, sizeof(bfloat16_t)); + svfloat32_t svalpha = svdup_f32(vcvtah_f32_bf16(alpha_bf16)); +#else svfloat32_t svalpha = svdup_f32(alpha); +#endif +#endif - svbool_t pg16 = svptrue_b16(); - svbool_t pg16_low = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0); - svbool_t pg32 = svptrue_b32(); - svbool_t pg32_low = svdupq_b32(1, 1, 0, 0); - svbool_t pg32_first = svdupq_b32(1, 0, 0, 0); + svbool_t pg32_first_4 = svdupq_b32(1, 1, 1, 1); + svbool_t pg32_first_2 = svdupq_b32(1, 1, 0, 0); + svbool_t pg32_first_1 = svdupq_b32(1, 0, 0, 0); + svbool_t pg16_first_8 = svdupq_b16(1, 1, 1, 1, 1, 1, 1, 1); + svbool_t pg16_first_4 = svdupq_b16(1, 1, 1, 1, 0, 0, 0, 0); +#ifdef BGEMM + svbool_t pg16_first_2 = svdupq_b16(1, 1, 0, 0, 0, 0, 0, 0); + svbool_t pg16_first_1 = svdupq_b16(1, 0, 0, 0, 0, 0, 0, 0); +#endif bfloat16_t *ptr_a = (bfloat16_t *)A; bfloat16_t *ptr_b = (bfloat16_t *)B; - FLOAT *ptr_c = C; + OUTPUT_FLOAT *ptr_c = (OUTPUT_FLOAT*)C; + + bfloat16_t *ptr_a0; + bfloat16_t *ptr_b0; + OUTPUT_FLOAT *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3; - bfloat16_t *ptr_a0, *ptr_a1, *ptr_a2, *ptr_a3; - bfloat16_t *ptr_b0, *ptr_b1; - FLOAT *ptr_c0, *ptr_c1, *ptr_c2, *ptr_c3; + svfloat32_t tmp32; +#ifdef BGEMM + svbfloat16_t tmp16; +#endif for (BLASLONG j = 0; j < n / 4; j++) { ptr_c0 = ptr_c; @@ -107,13 +152,13 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C_8x4; for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - ma2 = svld1_bf16(pg16, ptr_a0 + 16); - ma3 = svld1_bf16(pg16, ptr_a0 + 24); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + ma2 = svld1_bf16(pg16_first_8, ptr_a0 + 16); + ma3 = svld1_bf16(pg16_first_8, ptr_a0 + 24); - mb0 = svld1_bf16(pg16, ptr_b0); - mb1 = svld1_bf16(pg16, ptr_b0 + 8); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); + mb1 = svld1_bf16(pg16_first_8, ptr_b0 + 8); MATMUL(0, 0); MATMUL(0, 1); MATMUL(1, 0); MATMUL(1, 1); @@ -133,14 +178,14 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc6 = svuzp2(mc01, mc11); vc7 = svuzp2(mc21, mc31); - UPDATE_C(pg32, ptr_c0, oc0, vc0); - UPDATE_C(pg32, ptr_c0+4, oc1, vc1); - UPDATE_C(pg32, ptr_c1, oc2, vc2); - UPDATE_C(pg32, ptr_c1+4, oc3, vc3); - UPDATE_C(pg32, ptr_c2, oc4, vc4) - UPDATE_C(pg32, ptr_c2+4, oc5, vc5); - UPDATE_C(pg32, ptr_c3, oc6, vc6) - UPDATE_C(pg32, ptr_c3+4, oc7, vc7); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0+4, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, vc2); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1+4, vc3); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2, vc4); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2+4, vc5); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3, vc6); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3+4, vc7); ptr_c0 += 8; ptr_c1 += 8; @@ -157,10 +202,10 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(1, 0); INIT_C(1, 1); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - mb0 = svld1_bf16(pg16, ptr_b0); - mb1 = svld1_bf16(pg16, ptr_b0 + 8); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); + mb1 = svld1_bf16(pg16_first_8, ptr_b0 + 8); MATMUL(0, 0); MATMUL(0, 1); MATMUL(1, 0); MATMUL(1, 1); @@ -174,10 +219,10 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc2 = svuzp1(mc01, mc11); vc3 = svuzp2(mc01, mc11); - UPDATE_C(pg32, ptr_c0, oc0, vc0); - UPDATE_C(pg32, ptr_c1, oc1, vc1); - UPDATE_C(pg32, ptr_c2, oc2, vc2); - UPDATE_C(pg32, ptr_c3, oc3, vc3); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c2, vc2); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c3, vc3); ptr_c0 += 4; ptr_c1 += 4; @@ -192,9 +237,9 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(0, 0); INIT_C(0, 1); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - mb0 = svld1_bf16(pg16, ptr_b0); - mb1 = svld1_bf16(pg16, ptr_b0 + 8); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); + mb1 = svld1_bf16(pg16_first_8, ptr_b0 + 8); MATMUL(0, 0); MATMUL(0, 1); @@ -207,10 +252,10 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc2 = svuzp1(mc01, mc01); vc3 = svuzp2(mc01, mc01); - UPDATE_C(pg32_low, ptr_c0, oc0, vc0); - UPDATE_C(pg32_low, ptr_c1, oc1, vc1); - UPDATE_C(pg32_low, ptr_c2, oc2, vc2); - UPDATE_C(pg32_low, ptr_c3, oc3, vc3); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, vc1); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c2, vc2); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c3, vc3); ptr_c0 += 2; ptr_c1 += 2; @@ -224,9 +269,9 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(0, 0); INIT_C(0, 1); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16_low, ptr_a0); - mb0 = svld1_bf16(pg16, ptr_b0); - mb1 = svld1_bf16(pg16, ptr_b0 + 8); + ma0 = svld1_bf16(pg16_first_4, ptr_a0); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); + mb1 = svld1_bf16(pg16_first_8, ptr_b0 + 8); MATMUL(0, 0); MATMUL(0, 1); @@ -237,10 +282,10 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc1 = svuzp2(mc00, mc00); vc3 = svuzp2(mc01, mc01); - UPDATE_C(pg32_first, ptr_c0, oc0, mc00); - UPDATE_C(pg32_first, ptr_c1, oc1, vc1); - UPDATE_C(pg32_first, ptr_c2, oc2, mc01); - UPDATE_C(pg32_first, ptr_c3, oc3, vc3); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, vc1); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c2, mc01); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c3, vc3); } @@ -265,12 +310,12 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(3, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - ma2 = svld1_bf16(pg16, ptr_a0 + 16); - ma3 = svld1_bf16(pg16, ptr_a0 + 24); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + ma2 = svld1_bf16(pg16_first_8, ptr_a0 + 16); + ma3 = svld1_bf16(pg16_first_8, ptr_a0 + 24); - mb0 = svld1_bf16(pg16, ptr_b0); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); MATMUL(0, 0); MATMUL(1, 0); @@ -286,10 +331,10 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc2 = svuzp2(mc00, mc10); vc3 = svuzp2(mc20, mc30); - UPDATE_C(pg32, ptr_c0, oc0, vc0); - UPDATE_C(pg32, ptr_c0 + 4, oc1, vc1); - UPDATE_C(pg32, ptr_c1, oc2, vc2); - UPDATE_C(pg32, ptr_c1 + 4, oc3, vc3); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0 + 4, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, vc2); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1 + 4, vc3); ptr_c0 += 8; ptr_c1 += 8; @@ -304,9 +349,9 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(1, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - mb0 = svld1_bf16(pg16, ptr_b0); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); MATMUL(0, 0); MATMUL(1, 0); ptr_a0 += 16; @@ -316,8 +361,8 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc0 = svuzp1(mc00, mc10); vc1 = svuzp2(mc00, mc10); - UPDATE_C(pg32, ptr_c0, oc0, vc0); - UPDATE_C(pg32, ptr_c1, oc1, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c1, vc1); ptr_c0 += 4; ptr_c1 += 4; @@ -331,8 +376,8 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(0, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - mb0 = svld1_bf16(pg16, ptr_b0); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); MATMUL(0, 0); @@ -342,8 +387,8 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc0 = svuzp1(mc00, mc00); vc1 = svuzp2(mc00, mc00); - UPDATE_C(pg32_low, ptr_c0, oc0, vc0); - UPDATE_C(pg32_low, ptr_c1, oc1, vc1); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c1, vc1); ptr_c0 += 2; ptr_c1 += 2; @@ -355,16 +400,16 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp ptr_b0 = ptr_b; INIT_C(0, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16_low, ptr_a0); - mb0 = svld1_bf16(pg16, ptr_b0); + ma0 = svld1_bf16(pg16_first_4, ptr_a0); + mb0 = svld1_bf16(pg16_first_8, ptr_b0); MATMUL(0, 0); ptr_a0 += 4; ptr_b0 += 8; } vc1 = svuzp2(mc00, mc00); - UPDATE_C(pg32_first, ptr_c0, oc0, mc00); - UPDATE_C(pg32_first, ptr_c1, oc1, vc1); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c1, vc1); } ptr_b += 2 * pad_k; @@ -386,12 +431,12 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(3, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - ma2 = svld1_bf16(pg16, ptr_a0 + 16); - ma3 = svld1_bf16(pg16, ptr_a0 + 24); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + ma2 = svld1_bf16(pg16_first_8, ptr_a0 + 16); + ma3 = svld1_bf16(pg16_first_8, ptr_a0 + 24); - mb0 = svld1_bf16(pg16_low, ptr_b0); + mb0 = svld1_bf16(pg16_first_4, ptr_b0); MATMUL(0, 0); MATMUL(1, 0); @@ -405,8 +450,8 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp vc0 = svuzp1(mc00, mc10); vc1 = svuzp1(mc20, mc30); - UPDATE_C(pg32, ptr_c0, oc0, vc0); - UPDATE_C(pg32, ptr_c0 + 4, oc1, vc1); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0 + 4, vc1); ptr_c0 += 8; } @@ -418,16 +463,16 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(0, 0); INIT_C(1, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - ma1 = svld1_bf16(pg16, ptr_a0 + 8); - mb0 = svld1_bf16(pg16_low, ptr_b0); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + ma1 = svld1_bf16(pg16_first_8, ptr_a0 + 8); + mb0 = svld1_bf16(pg16_first_4, ptr_b0); MATMUL(0, 0); MATMUL(1, 0); ptr_a0 += 16; ptr_b0 += 4; } vc0 = svuzp1(mc00, mc10); - UPDATE_C(pg32, ptr_c0, oc0, vc0); + UPDATE_C(pg16_first_4, pg32_first_4, ptr_c0, vc0); ptr_c0 += 4; } @@ -439,8 +484,8 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp INIT_C(0, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16, ptr_a0); - mb0 = svld1_bf16(pg16_low, ptr_b0); + ma0 = svld1_bf16(pg16_first_8, ptr_a0); + mb0 = svld1_bf16(pg16_first_4, ptr_b0); MATMUL(0, 0); @@ -448,7 +493,7 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp ptr_b0 += 4; } vc0 = svuzp1(mc00, mc00); - UPDATE_C(pg32_low, ptr_c0, oc0, vc0); + UPDATE_C(pg16_first_2, pg32_first_2, ptr_c0, vc0); ptr_c0 += 2; } @@ -457,13 +502,13 @@ int sbgemm_kernel_neoversen2_alpha(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alp ptr_b0 = ptr_b; INIT_C(0, 0); for (BLASLONG p = 0; p < pad_k; p += 4) { - ma0 = svld1_bf16(pg16_low, ptr_a0); - mb0 = svld1_bf16(pg16_low, ptr_b0); + ma0 = svld1_bf16(pg16_first_4, ptr_a0); + mb0 = svld1_bf16(pg16_first_4, ptr_b0); MATMUL(0, 0); ptr_a0 += 4; ptr_b0 += 4; } - UPDATE_C(pg32_first, ptr_c0, oc0, mc00); + UPDATE_C(pg16_first_1, pg32_first_1, ptr_c0, mc00); } } diff --git a/param.h b/param.h index 662d08449..1a44cd0e4 100644 --- a/param.h +++ b/param.h @@ -3644,11 +3644,17 @@ is a big desktop or server with abundant cache rather than a phone or embedded d #define SWITCH_RATIO 16 #endif -#undef SBGEMM_ALIGN_K -#define SBGEMM_ALIGN_K 4 +#undef BGEMM_ALIGN_K +#undef BGEMM_DEFAULT_UNROLL_M +#undef BGEMM_DEFAULT_UNROLL_N +#define BGEMM_ALIGN_K 4 +#define BGEMM_DEFAULT_UNROLL_M 8 +#define BGEMM_DEFAULT_UNROLL_N 4 +#undef SBGEMM_ALIGN_K #undef SBGEMM_DEFAULT_UNROLL_M #undef SBGEMM_DEFAULT_UNROLL_N +#define SBGEMM_ALIGN_K 4 #define SBGEMM_DEFAULT_UNROLL_M 8 #define SBGEMM_DEFAULT_UNROLL_N 4 diff --git a/test/compare_sgemm_bgemm.c b/test/compare_sgemm_bgemm.c index 5358150d0..be7c538b6 100644 --- a/test/compare_sgemm_bgemm.c +++ b/test/compare_sgemm_bgemm.c @@ -100,7 +100,7 @@ main (int argc, char *argv[]) SGEMM (&transA, &transB, &m, &n, &k, &alpha, A, &m, B, &k, &beta, C, &m); - BGEMM (&transA, &transB, &m, &n, &k, &alpha_bf16, (bfloat16*) AA, + BGEMM (&transA, &transB, &m, &n, &k, &alpha_bf16, (bfloat16*)AA, &m, (bfloat16*)BB, &k, &beta_bf16, (bfloat16*)CC, &m); for (i = 0; i < n; i++) @@ -126,15 +126,15 @@ main (int argc, char *argv[]) } if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j]), 0.01, 0.001)) { #ifdef DEBUG - printf("Mismatch at i=%d, j=%d, k=%ld: CC=%.6f, C=%.6f\n", + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(C[i * m + j])); #endif ret++; } - if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]), 0.0001, 0.00001)) { + if (!is_close(float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j]), 0.01, 0.001)) { #ifdef DEBUG - printf("Mismatch at i=%d, j=%d, k=%ld: CC=%.6f, DD=%.6f\n", + printf("Mismatch at i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n", i, j, k, float16to32(CC[i * m + j]), truncate_float32_to_bfloat16(DD[i * m + j])); #endif ret++; diff --git a/test/compare_sgemv_bgemv.c b/test/compare_sgemv_bgemv.c index 17011a4b7..014c7da50 100644 --- a/test/compare_sgemv_bgemv.c +++ b/test/compare_sgemv_bgemv.c @@ -119,7 +119,7 @@ int main(int argc, char *argv[]) if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l]), 0.01, 0.001)) { #ifdef DEBUG - printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%ld: CC=%.6f, C=%.6f\n", + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, C=%.6f\n", transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(C[j << l])); #endif ret++; @@ -127,7 +127,7 @@ int main(int argc, char *argv[]) if (!is_close(float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j]), 0.001, 0.0001)) { #ifdef DEBUG - printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%ld: CC=%.6f, DD=%.6f\n", + printf("Mismatch at trans=%c, alpha=%.2f, beta=%.2f, i=%d, j=%d, k=%d: CC=%.6f, DD=%.6f\n", transA, alpha, beta, i, j, k, float16to32(CC[j << l]), truncate_float32_to_bfloat16(DD[j])); #endif ret++;