Further tweak small GEMM for AArch64

kernel/arm64/dgemm_small_kernel_nn_sve.c

@@ -46,13 +46,27 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
+#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
+#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
+
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
 
 #define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
 #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
@@ -112,8 +126,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat64_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
@@ -140,26 +153,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -169,13 +179,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -223,12 +226,29 @@ CNAME(BLASLONG M,
   FLOAT* packed_b =
     (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
 
+  FLOAT* b_offset = B;
+  FLOAT* a_offset = A;
+  FLOAT* c_offset = C;
+
   BLASLONG j = 0;
   for (; j < n4; j += 4) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_C_POINTER(2, 2);
+    CREATE_C_POINTER(3, 3);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+    CREATE_B_POINTER(2, 2);
+    CREATE_B_POINTER(3, 3);
+
     BLASLONG i = 0;
     for (; i < v_m2; i += v_size2) {
 
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -372,9 +392,16 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 1, 1);
       VECTOR_STORE(pg_true, 1, 2);
       VECTOR_STORE(pg_true, 1, 3);
+      INCR_C_POINTER(0, v_size2);
+      INCR_C_POINTER(1, v_size2);
+      INCR_C_POINTER(2, v_size2);
+      INCR_C_POINTER(3, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -431,9 +458,15 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
+      INCR_C_POINTER(2, v_size);
+      INCR_C_POINTER(3, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -491,13 +524,30 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
+      INCR_C_POINTER(2, 0);
+      INCR_C_POINTER(3, 0);
     }
+
+    UPDATE_B_POINTER(4);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(4);
   }
   for (; j < n2; j += 2) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+
     BLASLONG i = 0;
     for (; i < v_m2; i += v_size2) {
 
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -538,9 +588,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 1, 0);
       VECTOR_STORE(pg_true, 1, 1);
+      INCR_C_POINTER(0, v_size2);
+      INCR_C_POINTER(1, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -568,9 +623,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -599,13 +658,26 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
     }
+
+    UPDATE_B_POINTER(2);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(2);
   }
   for (; j < N; j++) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_B_POINTER(0, 0);
+
     BLASLONG i = 0;
     for (; i < v_m2; i += v_size2) {
 
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(1, 0);
@@ -620,9 +692,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 1, 0);
+      INCR_C_POINTER(0, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -633,9 +709,12 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -647,11 +726,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 0);
     }
+
+    UPDATE_B_POINTER(1);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(1);
   }
 
   if (pack_b)
     free(packed_b);
 
   return 0;
-}
+}

kernel/arm64/dgemm_small_kernel_nt_sve.c

@@ -46,13 +46,27 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
+#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
+#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
+
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
 
 #define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
 #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
@@ -97,8 +111,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat64_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
@@ -111,26 +124,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -140,13 +150,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -176,8 +179,7 @@ CNAME(BLASLONG M,
 #endif
 {
   const uint64_t v_size = svcntd();
-  const uint64_t v_size32 = v_size * 32;
-  const uint64_t v_size3 = v_size * 3;
+  const uint64_t v_size2 = v_size * 2;
   const svbool_t pg_true = svptrue_b64();
   const svbool_t pg_quad = svwhilelt_b64(0, 2);
   const svfloat64_t alpha_vec = svdup_f64(alpha);
@@ -186,14 +188,31 @@ CNAME(BLASLONG M,
 #endif
   const BLASLONG n4 = N & -4;
   const BLASLONG n2 = N & -2;
-  const BLASLONG v_m3 = M - (M % v_size3);
+  const BLASLONG v_m2 = M & -v_size2;
   const BLASLONG v_m1 = M & -v_size;
 
+  FLOAT* b_offset = B;
+  FLOAT* a_offset = A;
+  FLOAT* c_offset = C;
+
   BLASLONG j = 0;
   for (; j < n4; j += 4) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_C_POINTER(2, 2);
+    CREATE_C_POINTER(3, 3);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+    CREATE_B_POINTER(2, 2);
+    CREATE_B_POINTER(3, 3);
+
     BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
+    for (; i < v_m2; i += v_size2) {
+
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -204,10 +223,6 @@ CNAME(BLASLONG M,
       DECLARE_RESULT_VECTOR(1, 1);
       DECLARE_RESULT_VECTOR(1, 2);
       DECLARE_RESULT_VECTOR(1, 3);
-      DECLARE_RESULT_VECTOR(2, 0);
-      DECLARE_RESULT_VECTOR(2, 1);
-      DECLARE_RESULT_VECTOR(2, 2);
-      DECLARE_RESULT_VECTOR(2, 3);
 
       for (; k < K; k++) {
 
@@ -223,11 +238,6 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 2, 0, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 2, 1, 0);
-        VECTOR_LOAD_A(pg_true, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 2, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 2, 1, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
@@ -237,13 +247,16 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 1, 1);
       VECTOR_STORE(pg_true, 1, 2);
       VECTOR_STORE(pg_true, 1, 3);
-      VECTOR_STORE(pg_true, 2, 0);
-      VECTOR_STORE(pg_true, 2, 1);
-      VECTOR_STORE(pg_true, 2, 2);
-      VECTOR_STORE(pg_true, 2, 3);
+      INCR_C_POINTER(0, v_size2);
+      INCR_C_POINTER(1, v_size2);
+      INCR_C_POINTER(2, v_size2);
+      INCR_C_POINTER(3, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -264,9 +277,15 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
+      INCR_C_POINTER(2, v_size);
+      INCR_C_POINTER(3, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -288,20 +307,35 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
+      INCR_C_POINTER(2, 0);
+      INCR_C_POINTER(3, 0);
     }
+
+    UPDATE_B_POINTER(4);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(4);
   }
   for (; j < n2; j += 2) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+
     BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
+    for (; i < v_m2; i += v_size2) {
+
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(1, 0);
       DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(2, 0);
-      DECLARE_RESULT_VECTOR(2, 1);
 
       for (; k < K; k++) {
 
@@ -312,19 +346,19 @@ CNAME(BLASLONG M,
         VECTOR_LOAD_A(pg_true, 1, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-        VECTOR_LOAD_A(pg_true, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 1, 0);
       VECTOR_STORE(pg_true, 1, 1);
-      VECTOR_STORE(pg_true, 2, 0);
-      VECTOR_STORE(pg_true, 2, 1);
+      INCR_C_POINTER(0, v_size2);
+      INCR_C_POINTER(1, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -338,9 +372,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -355,17 +393,29 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
     }
+
+    UPDATE_B_POINTER(2);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(2);
   }
   for (; j < N; j++) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_B_POINTER(0, 0);
+
     BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
+    for (; i < v_m2; i += v_size2) {
+
+      CREATE_A_POINTER(0, 0);
+      CREATE_A_POINTER(1, v_size);
+      UPDATE_A_POINTER(v_size2);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(2, 0);
 
       for (; k < K; k++) {
 
@@ -374,15 +424,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
         VECTOR_LOAD_A(pg_true, 1, 0);
         UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-        VECTOR_LOAD_A(pg_true, 2, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 2, 0);
+      INCR_C_POINTER(0, v_size2);
     }
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -393,9 +444,12 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -407,8 +461,13 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 0);
     }
+
+    UPDATE_B_POINTER(1);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(1);
   }
 
   return 0;
-}
+}

kernel/arm64/dgemm_small_kernel_tn_sve.c

@@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
+#define UPDATE_C_POINTER(scale) c_offset += scale;
+#define C_ELEMENT(m, n)                                                        \
+  *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
+
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
 
-#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m]
+#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
 #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
 
 // ASIMD
@@ -112,14 +127,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat64_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
 #define GATHER_LOAD_A(pg, m, offset_k)                                         \
   svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec);
+    svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
 #define PACK_A(m, offset_k)                                                    \
   svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
 #define VECTOR_PACK_A(m, offset_k)                                             \
@@ -143,26 +157,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -172,13 +183,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -226,14 +230,29 @@ CNAME(BLASLONG M,
 
   const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
   FLOAT* packed_a =
-    (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL;
+    (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
+
+  FLOAT* a_offset = A;
+  FLOAT* b_offset = B;
+  FLOAT* c_offset = C;
 
   BLASLONG i = 0;
   for (; i < v_m2; i += v_size2) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, v_size);
+    CREATE_A_POINTER(0, 0);
+    CREATE_A_POINTER(1, v_size);
+
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -408,9 +427,15 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 1, 1);
       VECTOR_STORE(pg_true, 1, 2);
       VECTOR_STORE(pg_true, 1, 3);
+      INCR_C_POINTER(0, 4);
+      INCR_C_POINTER(1, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -484,9 +509,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 1, 0);
       VECTOR_STORE(pg_true, 1, 1);
+      INCR_C_POINTER(0, 2);
+      INCR_C_POINTER(1, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(1, 0);
@@ -512,13 +542,28 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 1, 0);
+      INCR_C_POINTER(0, 1);
+      INCR_C_POINTER(1, 1);
     }
+
+    UPDATE_A_POINTER(v_size2);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(v_size2);
   }
   for (; i < v_m1; i += v_size) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
+
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -562,9 +607,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -592,9 +642,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
+      INCR_C_POINTER(0, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -605,14 +659,27 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
+
+    UPDATE_A_POINTER(v_size);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(v_size);
   }
   for (; i < M; i += v_size) {
     const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
 
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -656,9 +723,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -686,9 +758,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
+      INCR_C_POINTER(0, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -699,11 +775,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
+
+    UPDATE_A_POINTER(0);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(0);
   }
 
   if (pack_a)
     free(packed_a);
 
   return 0;
-}
+}

kernel/arm64/dgemm_small_kernel_tt_sve.c

@@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
+#define UPDATE_C_POINTER(scale) c_offset += scale;
+#define C_ELEMENT(m, n)                                                        \
+  *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
+
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
 
-#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * 2 * v_size + m]
+#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
 #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
 
 // ASIMD
@@ -103,14 +118,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat64_t b##s##n##_k##offset_k = svdup_f64(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat64_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat64_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
 #define GATHER_LOAD_A(pg, m, offset_k)                                         \
   svfloat64_t a##s##m##_k##offset_k =                                          \
-    svld1_gather_index(pg, &A_ELEMENT_K(v_size * m, offset_k), lda_vec);
+    svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
 #define PACK_A(m, offset_k)                                                    \
   svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
 #define VECTOR_PACK_A(m, offset_k)                                             \
@@ -134,26 +148,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u64(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u64(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -163,13 +174,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -216,14 +220,29 @@ CNAME(BLASLONG M,
 
   const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
   FLOAT* packed_a =
-    (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL;
+    (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size2 * sizeof(FLOAT)) : NULL;
+
+  FLOAT* a_offset = A;
+  FLOAT* b_offset = B;
+  FLOAT* c_offset = C;
 
   BLASLONG i = 0;
   for (; i < v_m2; i += v_size2) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, v_size);
+    CREATE_A_POINTER(0, 0);
+    CREATE_A_POINTER(1, v_size);
+
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -295,9 +314,15 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 1, 1);
       VECTOR_STORE(pg_true, 1, 2);
       VECTOR_STORE(pg_true, 1, 3);
+      INCR_C_POINTER(0, 4);
+      INCR_C_POINTER(1, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -331,9 +356,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 1, 0);
       VECTOR_STORE(pg_true, 1, 1);
+      INCR_C_POINTER(0, 2);
+      INCR_C_POINTER(1, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(1, 0);
@@ -359,13 +389,28 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 1, 0);
+      INCR_C_POINTER(0, 1);
+      INCR_C_POINTER(1, 1);
     }
+
+    UPDATE_A_POINTER(v_size2);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(v_size2);
   }
   for (; i < v_m1; i += v_size) {
 
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
+
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -386,9 +431,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -402,9 +452,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
+      INCR_C_POINTER(0, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -415,14 +469,27 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
+
+    UPDATE_A_POINTER(v_size);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(v_size);
   }
   for (; i < M; i += v_size) {
     const svbool_t pg_tail = svwhilelt_b64((uint64_t)i, (uint64_t)(M));
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
 
     BLASLONG j = 0;
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -443,9 +510,14 @@ CNAME(BLASLONG M,
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < n2; j += 2) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      UPDATE_B_POINTER(2);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -459,9 +531,13 @@ CNAME(BLASLONG M,
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
+      INCR_C_POINTER(0, 2);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -472,11 +548,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
+
+    UPDATE_A_POINTER(0);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(0);
   }
 
   if (pack_a)
     free(packed_a);
 
   return 0;
-}
+}

kernel/arm64/gemm_small_kernel_permit_sve.c

@@ -32,16 +32,14 @@ int CNAME(int transa, int transb, BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alph
   BLASLONG MNK = M * N * K;
 
 #if defined(DOUBLE) // dgemm
-  // TN prefers full copies much earlier
-  if (transa && !transb && MNK > 16*16*16) {
-    return 0;
-  }
+  if (MNK <= 64*64*64)
+    return 1;
 #else // sgemm
-  // TODO!
+  if (MNK <= 256*256*256)
+    return 1;
 #endif
 
-  if (MNK <= 64*64*64)
-    return 1;
+
 
   return 0;
 }

kernel/arm64/sgemm_small_kernel_nn_sve.c

@@ -46,15 +46,29 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) + (j + (n)) * ldb]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale * ldb;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale * ldb;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k))
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
+#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
+#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
+
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
 
-#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n]
+#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
 #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
 
 // ASIMD
@@ -141,8 +155,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat32_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat32_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
@@ -169,26 +182,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -198,13 +208,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -234,7 +237,6 @@ CNAME(BLASLONG M,
 #endif
 {
   const uint64_t v_size = svcntw();
-  const uint64_t v_size2 = v_size * 2;
   const svbool_t pg_true = svptrue_b32();
   const svbool_t pg_quad = svwhilelt_b32(0, 4);
   const svbool_t pg_first = svwhilelt_b32(0, 1);
@@ -242,39 +244,41 @@ CNAME(BLASLONG M,
 #ifndef B0
   const svfloat32_t beta_vec = svdup_f32(beta);
 #endif
-  const BLASLONG n8 = N & -8;
   const BLASLONG n4 = N & -4;
-  const BLASLONG v_m2 = M & -v_size2;
   const BLASLONG v_m1 = M & -v_size;
   const BLASLONG k4 = K & -4;
 
-  const int pack_b = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
+  const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0;
   FLOAT* packed_b =
-    (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL;
+    (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
+
+  FLOAT* b_offset = B;
+  FLOAT* a_offset = A;
+  FLOAT* c_offset = C;
 
   BLASLONG j = 0;
-  for (; j < n8; j += 8) {
+  for (; j < n4; j += 4) {
+
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_C_POINTER(2, 2);
+    CREATE_C_POINTER(3, 3);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+    CREATE_B_POINTER(2, 2);
+    CREATE_B_POINTER(3, 3);
 
     BLASLONG i = 0;
-    for (; i < v_m2; i += v_size2) {
+    for (; i < v_m1; i += v_size) {
+
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(0, 2);
       DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-      DECLARE_RESULT_VECTOR(1, 4);
-      DECLARE_RESULT_VECTOR(1, 5);
-      DECLARE_RESULT_VECTOR(1, 6);
-      DECLARE_RESULT_VECTOR(1, 7);
 
       if (LIKELY(packed_b != NULL)) {
         if (i == 0) {
@@ -310,68 +314,6 @@ CNAME(BLASLONG M,
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-            VECTOR_LOAD_B_K4(4, 0);
-            VECTOR_LOAD_B_K4(5, 0);
-            VECTOR_LOAD_B_K4(6, 0);
-            VECTOR_LOAD_B_K4(7, 0);
-            TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3);
-            SCALE_B4_K4(4, 0, 1, 2, 3);
-            VECTOR_PACK_B4(4, 0);
-            VECTOR_PACK_B4(4, 1);
-            VECTOR_PACK_B4(4, 2);
-            VECTOR_PACK_B4(4, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3);
-            VECTOR_LOAD_A(pg_true, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 1, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1);
-            VECTOR_LOAD_A(pg_true, 1, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2);
-            VECTOR_LOAD_A(pg_true, 1, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3);
           }
           for (; k < K; k++) {
 
@@ -382,33 +324,12 @@ CNAME(BLASLONG M,
             BROADCAST_LOAD_B(1, 0);
             PACK_B(1, 0);
             UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
-            VECTOR_LOAD_A(pg_true, 1, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
             BROADCAST_LOAD_B(2, 0);
             PACK_B(2, 0);
             UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
             BROADCAST_LOAD_B(3, 0);
             PACK_B(3, 0);
             UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
-            BROADCAST_LOAD_B(4, 0);
-            PACK_B(4, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0);
-            BROADCAST_LOAD_B(5, 0);
-            PACK_B(5, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0);
-            BROADCAST_LOAD_B(6, 0);
-            PACK_B(6, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0);
-            BROADCAST_LOAD_B(7, 0);
-            PACK_B(7, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
-            UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0);
           }
         } else {
           for (; k < K; k++) {
@@ -419,20 +340,6 @@ CNAME(BLASLONG M,
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-            UNPACK_QUADWORD_B(4, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
           }
         }
       } else {
@@ -464,64 +371,6 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-          VECTOR_LOAD_B_K4(4, 0);
-          VECTOR_LOAD_B_K4(5, 0);
-          VECTOR_LOAD_B_K4(6, 0);
-          VECTOR_LOAD_B_K4(7, 0);
-          TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3);
-          SCALE_B4_K4(4, 0, 1, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3);
-          VECTOR_LOAD_A(pg_true, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-          VECTOR_LOAD_A(pg_true, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 1);
-          VECTOR_LOAD_A(pg_true, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 2);
-          VECTOR_LOAD_A(pg_true, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 3);
         }
         for (; k < K; k++) {
 
@@ -530,167 +379,31 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
           BROADCAST_LOAD_B(1, 0);
           UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
-          VECTOR_LOAD_A(pg_true, 1, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
           BROADCAST_LOAD_B(2, 0);
           UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
           BROADCAST_LOAD_B(3, 0);
           UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
-          BROADCAST_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 4, 0);
-          BROADCAST_LOAD_B(5, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 5, 0);
-          BROADCAST_LOAD_B(6, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 6, 0);
-          BROADCAST_LOAD_B(7, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 7, 0);
         }
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 0, 4);
-      VECTOR_STORE(pg_true, 0, 5);
-      VECTOR_STORE(pg_true, 0, 6);
-      VECTOR_STORE(pg_true, 0, 7);
-      VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 1, 1);
-      VECTOR_STORE(pg_true, 1, 2);
-      VECTOR_STORE(pg_true, 1, 3);
-      VECTOR_STORE(pg_true, 1, 4);
-      VECTOR_STORE(pg_true, 1, 5);
-      VECTOR_STORE(pg_true, 1, 6);
-      VECTOR_STORE(pg_true, 1, 7);
-    }
-    for (; i < v_m1; i += v_size) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-
-      if (LIKELY(packed_b != NULL)) {
-        for (; k < K; k++) {
-
-          UNPACK_QUADWORD_B(0, 0);
-          VECTOR_LOAD_A(pg_true, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          UNPACK_QUADWORD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-        }
-      } else {
-        for (; k < k4; k += 4) {
-
-          VECTOR_LOAD_B_K4(0, 0);
-          VECTOR_LOAD_B_K4(1, 0);
-          VECTOR_LOAD_B_K4(2, 0);
-          VECTOR_LOAD_B_K4(3, 0);
-          TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
-          SCALE_B4_K4(0, 0, 1, 2, 3);
-          VECTOR_LOAD_A(pg_true, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          VECTOR_LOAD_A(pg_true, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
-          VECTOR_LOAD_A(pg_true, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
-          VECTOR_LOAD_A(pg_true, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-          VECTOR_LOAD_B_K4(4, 0);
-          VECTOR_LOAD_B_K4(5, 0);
-          VECTOR_LOAD_B_K4(6, 0);
-          VECTOR_LOAD_B_K4(7, 0);
-          TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3);
-          SCALE_B4_K4(4, 0, 1, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3);
-        }
-        for (; k < K; k++) {
-
-          BROADCAST_LOAD_B(0, 0);
-          VECTOR_LOAD_A(pg_true, 0, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-          BROADCAST_LOAD_B(1, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
-          BROADCAST_LOAD_B(2, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
-          BROADCAST_LOAD_B(3, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
-          BROADCAST_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 4, 0);
-          BROADCAST_LOAD_B(5, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 5, 0);
-          BROADCAST_LOAD_B(6, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 6, 0);
-          BROADCAST_LOAD_B(7, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 0, 7, 0);
-        }
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 0, 4);
-      VECTOR_STORE(pg_true, 0, 5);
-      VECTOR_STORE(pg_true, 0, 6);
-      VECTOR_STORE(pg_true, 0, 7);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
+      INCR_C_POINTER(2, v_size);
+      INCR_C_POINTER(3, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(0, 2);
       DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
 
       if (LIKELY(packed_b != NULL)) {
         for (; k < K; k++) {
@@ -701,11 +414,6 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          UNPACK_QUADWORD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
         }
       } else {
         for (; k < k4; k += 4) {
@@ -736,28 +444,6 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-          VECTOR_LOAD_B_K4(4, 0);
-          VECTOR_LOAD_B_K4(5, 0);
-          VECTOR_LOAD_B_K4(6, 0);
-          VECTOR_LOAD_B_K4(7, 0);
-          TRANSPOSE_B4_K4(4, 5, 6, 7, 0, 1, 2, 3);
-          SCALE_B4_K4(4, 0, 1, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 1);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 2);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 3);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 3);
         }
         for (; k < K; k++) {
 
@@ -770,248 +456,33 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
           BROADCAST_LOAD_B(3, 0);
           UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
-          BROADCAST_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR(pg_tail, 0, 4, 0);
-          BROADCAST_LOAD_B(5, 0);
-          UPDATE_RESULT_VECTOR(pg_tail, 0, 5, 0);
-          BROADCAST_LOAD_B(6, 0);
-          UPDATE_RESULT_VECTOR(pg_tail, 0, 6, 0);
-          BROADCAST_LOAD_B(7, 0);
-          UPDATE_RESULT_VECTOR(pg_tail, 0, 7, 0);
         }
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
-      VECTOR_STORE(pg_tail, 0, 4);
-      VECTOR_STORE(pg_tail, 0, 5);
-      VECTOR_STORE(pg_tail, 0, 6);
-      VECTOR_STORE(pg_tail, 0, 7);
-    }
-  }
-  for (; j < n4; j += 4) {
-
-    BLASLONG i = 0;
-    for (; i < v_m2; i += v_size2) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-
-      for (; k < k4; k += 4) {
-
-        VECTOR_LOAD_B_K4(0, 0);
-        VECTOR_LOAD_B_K4(1, 0);
-        VECTOR_LOAD_B_K4(2, 0);
-        VECTOR_LOAD_B_K4(3, 0);
-        TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
-        SCALE_B4_K4(0, 0, 1, 2, 3);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_true, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
-        VECTOR_LOAD_A(pg_true, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
-        VECTOR_LOAD_A(pg_true, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-        VECTOR_LOAD_A(pg_true, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_true, 1, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 1);
-        VECTOR_LOAD_A(pg_true, 1, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 2);
-        VECTOR_LOAD_A(pg_true, 1, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 3);
-      }
-      for (; k < K; k++) {
-
-        BROADCAST_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-        BROADCAST_LOAD_B(1, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
-        VECTOR_LOAD_A(pg_true, 1, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 1, 0);
-        BROADCAST_LOAD_B(2, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 2, 0);
-        BROADCAST_LOAD_B(3, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 1, 1);
-      VECTOR_STORE(pg_true, 1, 2);
-      VECTOR_STORE(pg_true, 1, 3);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
+      INCR_C_POINTER(2, 0);
+      INCR_C_POINTER(3, 0);
     }
-    for (; i < v_m1; i += v_size) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-
-      for (; k < k4; k += 4) {
-
-        VECTOR_LOAD_B_K4(0, 0);
-        VECTOR_LOAD_B_K4(1, 0);
-        VECTOR_LOAD_B_K4(2, 0);
-        VECTOR_LOAD_B_K4(3, 0);
-        TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
-        SCALE_B4_K4(0, 0, 1, 2, 3);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_true, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
-        VECTOR_LOAD_A(pg_true, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
-        VECTOR_LOAD_A(pg_true, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-      }
-      for (; k < K; k++) {
 
-        BROADCAST_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-        BROADCAST_LOAD_B(1, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 1, 0);
-        BROADCAST_LOAD_B(2, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 2, 0);
-        BROADCAST_LOAD_B(3, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-    }
-    for (; i < M; i += v_size) {
-      const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-
-      for (; k < k4; k += 4) {
-
-        VECTOR_LOAD_B_K4(0, 0);
-        VECTOR_LOAD_B_K4(1, 0);
-        VECTOR_LOAD_B_K4(2, 0);
-        VECTOR_LOAD_B_K4(3, 0);
-        TRANSPOSE_B4_K4(0, 1, 2, 3, 0, 1, 2, 3);
-        SCALE_B4_K4(0, 0, 1, 2, 3);
-        VECTOR_LOAD_A(pg_tail, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_tail, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 1);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 1);
-        VECTOR_LOAD_A(pg_tail, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 2);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 2);
-        VECTOR_LOAD_A(pg_tail, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 3);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 3);
-      }
-      for (; k < K; k++) {
-
-        BROADCAST_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_tail, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
-        BROADCAST_LOAD_B(1, 0);
-        UPDATE_RESULT_VECTOR(pg_tail, 0, 1, 0);
-        BROADCAST_LOAD_B(2, 0);
-        UPDATE_RESULT_VECTOR(pg_tail, 0, 2, 0);
-        BROADCAST_LOAD_B(3, 0);
-        UPDATE_RESULT_VECTOR(pg_tail, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_tail, 0, 0);
-      VECTOR_STORE(pg_tail, 0, 1);
-      VECTOR_STORE(pg_tail, 0, 2);
-      VECTOR_STORE(pg_tail, 0, 3);
-    }
+    UPDATE_B_POINTER(4);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(4);
   }
   for (; j < N; j++) {
 
-    BLASLONG i = 0;
-    for (; i < v_m2; i += v_size2) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(1, 0);
-
-      for (; k < K; k++) {
+    CREATE_C_POINTER(0, 0);
+    CREATE_B_POINTER(0, 0);
 
-        BROADCAST_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-        VECTOR_LOAD_A(pg_true, 1, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 1, 0);
-    }
+    BLASLONG i = 0;
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -1022,9 +493,12 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -1036,11 +510,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 0);
     }
+
+    UPDATE_B_POINTER(1);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(1);
   }
 
   if (pack_b)
     free(packed_b);
 
   return 0;
-}
+}

kernel/arm64/sgemm_small_kernel_nt_sve.c

@@ -46,15 +46,29 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) + (k + offset_k) * lda]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k) * lda)
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(n, scale) FLOAT* c_offset##n = c_offset + scale * ldc;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr;
+#define UPDATE_C_POINTER(scale) c_offset = c_offset + scale * ldc;
+#define C_ELEMENT(m, n) *(c_offset##n + ((m * v_size) + i))
 
-#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 8 + n]
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
+
+#define PACK_ELEMENT_K(n, offset_k) packed_b[(k + offset_k) * 4 + n]
 #define PACK_ELEMENT(n) PACK_ELEMENT_K(n, 0)
 
 // ASIMD
@@ -113,8 +127,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
 #define VECTOR_LOAD_A(pg, m, offset_k)                                         \
-  svfloat32_t a##s##m##_k##offset_k =                                          \
-    svld1(pg, &A_ELEMENT_K(v_size * m, offset_k));
+  svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat32_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
@@ -141,26 +154,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifdef B0
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
 #define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(v_size * m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(v_size* m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -170,13 +180,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -206,7 +209,6 @@ CNAME(BLASLONG M,
 #endif
 {
   const uint64_t v_size = svcntw();
-  const uint64_t v_size3 = v_size * 3;
   const svbool_t pg_true = svptrue_b32();
   const svbool_t pg_quad = svwhilelt_b32(0, 4);
   const svbool_t pg_first = svwhilelt_b32(0, 1);
@@ -214,46 +216,40 @@ CNAME(BLASLONG M,
 #ifndef B0
   const svfloat32_t beta_vec = svdup_f32(beta);
 #endif
-  const BLASLONG n8 = N & -8;
   const BLASLONG n4 = N & -4;
-  const BLASLONG v_m3 = M - (M % v_size3);
   const BLASLONG v_m1 = M & -v_size;
 
-  const int pack_b = M >= v_size3 && N >= 8 && K >= 8 ? 1 : 0;
+  const int pack_b = M >= v_size && N >= 8 && K >= 8 ? 1 : 0;
   FLOAT* packed_b =
-    (pack_b) ? packed_b = (FLOAT*)malloc(K * 8 * sizeof(FLOAT)) : NULL;
+    (pack_b) ? packed_b = (FLOAT*)malloc(K * 4 * sizeof(FLOAT)) : NULL;
+
+  FLOAT* b_offset = B;
+  FLOAT* a_offset = A;
+  FLOAT* c_offset = C;
 
   BLASLONG j = 0;
-  for (; j < n8; j += 8) {
+  for (; j < n4; j += 4) {
+
+    CREATE_C_POINTER(0, 0);
+    CREATE_C_POINTER(1, 1);
+    CREATE_C_POINTER(2, 2);
+    CREATE_C_POINTER(3, 3);
+    CREATE_B_POINTER(0, 0);
+    CREATE_B_POINTER(1, 1);
+    CREATE_B_POINTER(2, 2);
+    CREATE_B_POINTER(3, 3);
 
     BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
+    for (; i < v_m1; i += v_size) {
+
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(0, 2);
       DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-      DECLARE_RESULT_VECTOR(1, 4);
-      DECLARE_RESULT_VECTOR(1, 5);
-      DECLARE_RESULT_VECTOR(1, 6);
-      DECLARE_RESULT_VECTOR(1, 7);
-      DECLARE_RESULT_VECTOR(2, 0);
-      DECLARE_RESULT_VECTOR(2, 1);
-      DECLARE_RESULT_VECTOR(2, 2);
-      DECLARE_RESULT_VECTOR(2, 3);
-      DECLARE_RESULT_VECTOR(2, 4);
-      DECLARE_RESULT_VECTOR(2, 5);
-      DECLARE_RESULT_VECTOR(2, 6);
-      DECLARE_RESULT_VECTOR(2, 7);
 
       if (LIKELY(packed_b != NULL)) {
         if (i == 0) {
@@ -266,30 +262,6 @@ CNAME(BLASLONG M,
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-            QUADWORD_LOAD_B(4, 0);
-            QUADWORD_PACK_B(4, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0);
           }
         } else {
           for (; k < K; k++) {
@@ -300,29 +272,6 @@ CNAME(BLASLONG M,
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-            UNPACK_QUADWORD_B(4, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-            VECTOR_LOAD_A(pg_true, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0);
           }
         }
       } else {
@@ -334,120 +283,27 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          QUADWORD_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-          VECTOR_LOAD_A(pg_true, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-          VECTOR_LOAD_A(pg_true, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(2, 7, 4, 3, 0);
-        }
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 0, 4);
-      VECTOR_STORE(pg_true, 0, 5);
-      VECTOR_STORE(pg_true, 0, 6);
-      VECTOR_STORE(pg_true, 0, 7);
-      VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 1, 1);
-      VECTOR_STORE(pg_true, 1, 2);
-      VECTOR_STORE(pg_true, 1, 3);
-      VECTOR_STORE(pg_true, 1, 4);
-      VECTOR_STORE(pg_true, 1, 5);
-      VECTOR_STORE(pg_true, 1, 6);
-      VECTOR_STORE(pg_true, 1, 7);
-      VECTOR_STORE(pg_true, 2, 0);
-      VECTOR_STORE(pg_true, 2, 1);
-      VECTOR_STORE(pg_true, 2, 2);
-      VECTOR_STORE(pg_true, 2, 3);
-      VECTOR_STORE(pg_true, 2, 4);
-      VECTOR_STORE(pg_true, 2, 5);
-      VECTOR_STORE(pg_true, 2, 6);
-      VECTOR_STORE(pg_true, 2, 7);
-    }
-    for (; i < v_m1; i += v_size) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-
-      if (LIKELY(packed_b != NULL)) {
-        for (; k < K; k++) {
-
-          UNPACK_QUADWORD_B(0, 0);
-          VECTOR_LOAD_A(pg_true, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          UNPACK_QUADWORD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-        }
-      } else {
-        for (; k < K; k++) {
-
-          QUADWORD_LOAD_B(0, 0);
-          VECTOR_LOAD_A(pg_true, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          QUADWORD_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
         }
       }
       VECTOR_STORE(pg_true, 0, 0);
       VECTOR_STORE(pg_true, 0, 1);
       VECTOR_STORE(pg_true, 0, 2);
       VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 0, 4);
-      VECTOR_STORE(pg_true, 0, 5);
-      VECTOR_STORE(pg_true, 0, 6);
-      VECTOR_STORE(pg_true, 0, 7);
+      INCR_C_POINTER(0, v_size);
+      INCR_C_POINTER(1, v_size);
+      INCR_C_POINTER(2, v_size);
+      INCR_C_POINTER(3, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(0, 2);
       DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
 
       if (LIKELY(packed_b != NULL)) {
         for (; k < K; k++) {
@@ -458,11 +314,6 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          UNPACK_QUADWORD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
         }
       } else {
         for (; k < K; k++) {
@@ -473,146 +324,33 @@ CNAME(BLASLONG M,
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          QUADWORD_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
         }
       }
       VECTOR_STORE(pg_tail, 0, 0);
       VECTOR_STORE(pg_tail, 0, 1);
       VECTOR_STORE(pg_tail, 0, 2);
       VECTOR_STORE(pg_tail, 0, 3);
-      VECTOR_STORE(pg_tail, 0, 4);
-      VECTOR_STORE(pg_tail, 0, 5);
-      VECTOR_STORE(pg_tail, 0, 6);
-      VECTOR_STORE(pg_tail, 0, 7);
+      INCR_C_POINTER(0, 0);
+      INCR_C_POINTER(1, 0);
+      INCR_C_POINTER(2, 0);
+      INCR_C_POINTER(3, 0);
     }
-  }
-  for (; j < n4; j += 4) {
-
-    BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-      DECLARE_RESULT_VECTOR(2, 0);
-      DECLARE_RESULT_VECTOR(2, 1);
-      DECLARE_RESULT_VECTOR(2, 2);
-      DECLARE_RESULT_VECTOR(2, 3);
-
-      for (; k < K; k++) {
 
-        QUADWORD_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_true, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-        VECTOR_LOAD_A(pg_true, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(2, 3, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-      VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 1, 1);
-      VECTOR_STORE(pg_true, 1, 2);
-      VECTOR_STORE(pg_true, 1, 3);
-      VECTOR_STORE(pg_true, 2, 0);
-      VECTOR_STORE(pg_true, 2, 1);
-      VECTOR_STORE(pg_true, 2, 2);
-      VECTOR_STORE(pg_true, 2, 3);
-    }
-    for (; i < v_m1; i += v_size) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-
-      for (; k < K; k++) {
-
-        QUADWORD_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 0, 1);
-      VECTOR_STORE(pg_true, 0, 2);
-      VECTOR_STORE(pg_true, 0, 3);
-    }
-    for (; i < M; i += v_size) {
-      const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-
-      for (; k < K; k++) {
-
-        QUADWORD_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_tail, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_tail, 0, 0);
-      VECTOR_STORE(pg_tail, 0, 1);
-      VECTOR_STORE(pg_tail, 0, 2);
-      VECTOR_STORE(pg_tail, 0, 3);
-    }
+    UPDATE_B_POINTER(4);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(4);
   }
   for (; j < N; j++) {
 
-    BLASLONG i = 0;
-    for (; i < v_m3; i += v_size3) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(2, 0);
+    CREATE_C_POINTER(0, 0);
+    CREATE_B_POINTER(0, 0);
 
-      for (; k < K; k++) {
-
-        BROADCAST_LOAD_B(0, 0);
-        VECTOR_LOAD_A(pg_true, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-        VECTOR_LOAD_A(pg_true, 1, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
-        VECTOR_LOAD_A(pg_true, 2, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 2, 0, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0);
-      VECTOR_STORE(pg_true, 1, 0);
-      VECTOR_STORE(pg_true, 2, 0);
-    }
+    BLASLONG i = 0;
     for (; i < v_m1; i += v_size) {
 
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(v_size);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
@@ -623,9 +361,12 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
       }
       VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, v_size);
     }
     for (; i < M; i += v_size) {
       const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
+      CREATE_A_POINTER(0, 0);
+      UPDATE_A_POINTER(0);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
@@ -637,11 +378,16 @@ CNAME(BLASLONG M,
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
       VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 0);
     }
+
+    UPDATE_B_POINTER(1);
+    RESET_A_POINTER();
+    UPDATE_C_POINTER(1);
   }
 
   if (pack_b)
     free(packed_b);
 
   return 0;
-}
+}

kernel/arm64/sgemm_small_kernel_tt_sve.c

@@ -46,15 +46,30 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   })
 #endif
 
-#define A_ELEMENT_K(m, offset_k) A[(i + (m)) * lda + (k + offset_k)]
+#define RESET_A_POINTER() a_offset = A;
+
+#define CREATE_A_POINTER(m, scale) FLOAT* a_offset##m = a_offset + scale * lda;
+#define UPDATE_A_POINTER(scale) a_offset = a_offset + scale * lda;
+#define A_ELEMENT_K(m, offset_k) *(a_offset##m + (k + offset_k))
 #define A_ELEMENT(m) A_ELEMENT_K(m, 0)
 
-#define B_ELEMENT_K(n, offset_k) B[(k + offset_k) * ldb + (j + (n))]
+#define RESET_B_POINTER() b_offset = B;
+
+#define CREATE_B_POINTER(n, scale) FLOAT* b_offset##n = b_offset + scale;
+#define UPDATE_B_POINTER(scale) b_offset = b_offset + scale;
+#define B_ELEMENT_K(n, offset_k) *(b_offset##n + (k + offset_k) * ldb)
 #define B_ELEMENT(n) B_ELEMENT_K(n, 0)
 
-#define C_ELEMENT(m, n) C[(i + (m)) + (j + (n)) * ldc]
+#define CREATE_C_POINTER(m, scale) FLOAT* c_offset##m = c_offset + scale;
+#define INCR_C_POINTER(m, incr) // c_offset ## m += incr * ldc;
+#define UPDATE_C_POINTER(scale) c_offset += scale;
+#define C_ELEMENT(m, n)                                                        \
+  *(c_offset##m + ((j + n) * ldc)) // C[(i+(m))+(j+(n))*ldc]
 
-#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size2 + m]
+// #undef C_ELEMENT
+// #define C_ELEMENT(m, n)             C[(i+(m))+(j+(n))*ldc]
+
+#define PACK_ELEMENT_K(m, offset_k) packed_a[(k + offset_k) * v_size + m]
 #define PACK_ELEMENT(m) PACK_ELEMENT_K(m, 0)
 
 // ASIMD
@@ -106,22 +121,23 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   svfloat32_t a##s##m##_k##offset_k = svdup_f32(A_ELEMENT_K(m, offset_k));
 #define BROADCAST_LOAD_B(n, offset_k)                                          \
   svfloat32_t b##s##n##_k##offset_k = svdup_f32(B_ELEMENT_K(n, offset_k));
-
+#define VECTOR_LOAD_A(pg, m, offset_k)                                         \
+  svfloat32_t a##s##m##_k##offset_k = svld1(pg, &A_ELEMENT_K(m, offset_k));
 #define QUADWORD_LOAD_B(n, offset_k)                                           \
   svfloat32_t b##s##n##_k##offset_k =                                          \
     svld1rq(pg_true, &B_ELEMENT_K(n, offset_k));
-#define GATHER_LOAD_A(pg, offset_m, m, offset_k)                                         \
+#define GATHER_LOAD_A(pg, m, offset_k)                                         \
   svfloat32_t a##s##m##_k##offset_k =                                          \
-    svld1_gather_index(pg, &A_ELEMENT_K(offset_m, offset_k), lda_vec);
+    svld1_gather_index(pg, &A_ELEMENT_K(m, offset_k), lda_vec);
 #define PACK_A(m, offset_k)                                                    \
   svst1(pg_first, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
-#define VECTOR_PACK_A(offset_m, m, offset_k)                                             \
+#define VECTOR_PACK_A(m, offset_k)                                             \
   svst1(pg_true, &PACK_ELEMENT_K(m* v_size, offset_k), a##s##m##_k##offset_k);
 #define QUADWORD_PACK_A(m, offset_k)                                           \
   svst1(pg_quad, &PACK_ELEMENT_K(m, offset_k), a##s##m##_k##offset_k);
-#define UNPACK_VECTOR_A(offset_m, m, offset_k)                                           \
+#define UNPACK_VECTOR_A(m, offset_k)                                           \
   svfloat32_t a##s##m##_k##offset_k =                                          \
-    svld1(pg_true, &PACK_ELEMENT_K(offset_m, offset_k));
+    svld1(pg_true, &PACK_ELEMENT_K(m * v_size, offset_k));
 #define UNPACK_BROADCAST_A(m, offset_k)                                        \
   svfloat32_t a##s##m##_k##offset_k = svdup_f32(PACK_ELEMENT_K(m, offset_k));
 #define UNPACK_QUADWORD_A(m, offset_k)                                         \
@@ -134,28 +150,25 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   result##m##n = svmla_lane(                                                   \
     result##m##n, a##s##m##_k##offset_k, b##s##outer##_k##offset_k, lane);
 #ifdef B0
-#define VECTOR_STORE(pg, offset_m, m, n)                                                 \
+#define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1(pg, &C_ELEMENT(offset_m, n), result##m##n);
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #else
-#define VECTOR_STORE(pg, offset_m, m, n)                                                 \
+#define VECTOR_STORE(pg, m, n)                                                 \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
   result##m##n =                                                               \
-    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(offset_m, n)), beta_vec); \
-  svst1(pg, &C_ELEMENT(offset_m, n), result##m##n);
+    svmla_m(pg, result##m##n, svld1(pg, &C_ELEMENT(m, n)), beta_vec);          \
+  svst1(pg, &C_ELEMENT(m, n), result##m##n);
 #define SCATTER_STORE(pg, m, n)                                                \
   result##m##n = svmul_m(pg, result##m##n, alpha_vec);                         \
-  result##m##n = svmla_m(                                                      \
-    pg,                                                                        \
-    result##m##n,                                                              \
-    svld1_gather_index(pg, &C_ELEMENT(v_size * m, n), svindex_u32(0LL, ldc)),  \
-    beta_vec);                                                                 \
-  svst1_scatter_index(                                                         \
-    pg, &C_ELEMENT(v_size* m, n), svindex_u32(0LL, ldc), result##m##n);
+  result##m##n = svmla_m(pg,                                                   \
+                         result##m##n,                                         \
+                         svld1_gather_index(pg, &C_ELEMENT(m, n), ldc_vec),    \
+                         beta_vec);                                            \
+  svst1_scatter_index(pg, &C_ELEMENT(m, n), ldc_vec, result##m##n);
 #endif
 
 #ifndef LIKELY
@@ -165,13 +178,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define LIKELY(x) (x)
 #endif
 #endif
-#ifndef UNLIKELY
-#ifdef __GNUC__
-#define UNLIKELY(x) __builtin_expect(!!(x), 0)
-#else
-#define UNLIKELY(x) (x)
-#endif
-#endif
 
 #ifdef B0
 int
@@ -201,337 +207,132 @@ CNAME(BLASLONG M,
 #endif
 {
   const uint64_t v_size = svcntw();
-  const uint64_t v_size2 = v_size * 2;
   const svbool_t pg_true = svptrue_b32();
+  const svbool_t pg_quad = svwhilelt_b32(0, 4);
+  const svbool_t pg_first = svwhilelt_b32(0, 1);
   const svfloat32_t alpha_vec = svdup_f32(alpha);
 #ifndef B0
   const svfloat32_t beta_vec = svdup_f32(beta);
 #endif
   const svuint32_t lda_vec = svindex_u32(0LL, lda);
 
-  const BLASLONG v_m2 = M & -v_size2;
   const BLASLONG v_m1 = M & -v_size;
-  const BLASLONG n8 = N & -8;
   const BLASLONG n4 = N & -4;
 
-  const int pack_a = M >= v_size2 && N >= 8 && K >= 8 ? 1 : 0;
+  const int pack_a = M >= v_size && N >= 8 && K >= 8 ? 1 : 0;
   FLOAT* packed_a =
-    (pack_a) ? packed_a = (FLOAT*)malloc(K * 2 * v_size * sizeof(FLOAT)) : NULL;
+    (pack_a) ? packed_a = (FLOAT*)malloc(K * v_size * sizeof(FLOAT)) : NULL;
+
+  FLOAT* a_offset = A;
+  FLOAT* b_offset = B;
+  FLOAT* c_offset = C;
 
   BLASLONG i = 0;
-  for (; i < v_m2; i += v_size2) {
+  for (; i < v_m1; i += v_size) {
+
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
 
     BLASLONG j = 0;
-    for (; j < n8; j += 8) {
+    for (; j < n4; j += 4) {
+
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
 
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
       DECLARE_RESULT_VECTOR(0, 2);
       DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-      DECLARE_RESULT_VECTOR(1, 4);
-      DECLARE_RESULT_VECTOR(1, 5);
-      DECLARE_RESULT_VECTOR(1, 6);
-      DECLARE_RESULT_VECTOR(1, 7);
 
       if (LIKELY(packed_a != NULL)) {
         if (j == 0) {
           for (; k < K; k++) {
 
             QUADWORD_LOAD_B(0, 0);
-            GATHER_LOAD_A(pg_true, 0, 0, 0);
-            VECTOR_PACK_A(0, 0, 0);
+            GATHER_LOAD_A(pg_true, 0, 0);
+            VECTOR_PACK_A(0, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-            QUADWORD_LOAD_B(4, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            GATHER_LOAD_A(pg_true, v_size, 1, 0);
-            VECTOR_PACK_A(v_size, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
           }
         } else {
           for (; k < K; k++) {
 
             QUADWORD_LOAD_B(0, 0);
-            UNPACK_VECTOR_A(0, 0, 0);
+            UNPACK_VECTOR_A(0, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
             UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-            QUADWORD_LOAD_B(4, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-            UNPACK_VECTOR_A(v_size, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-            UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
           }
         }
       } else {
         for (; k < K; k++) {
 
           QUADWORD_LOAD_B(0, 0);
-          GATHER_LOAD_A(pg_true, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          QUADWORD_LOAD_B(4, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-          GATHER_LOAD_A(pg_true, v_size, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 4, 4, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 5, 4, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 6, 4, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 7, 4, 3, 0);
-        }
-      }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-      VECTOR_STORE(pg_true, 0, 0, 1);
-      VECTOR_STORE(pg_true, 0, 0, 2);
-      VECTOR_STORE(pg_true, 0, 0, 3);
-      VECTOR_STORE(pg_true, 0, 0, 4);
-      VECTOR_STORE(pg_true, 0, 0, 5);
-      VECTOR_STORE(pg_true, 0, 0, 6);
-      VECTOR_STORE(pg_true, 0, 0, 7);
-      VECTOR_STORE(pg_true, v_size, 1, 0);
-      VECTOR_STORE(pg_true, v_size, 1, 1);
-      VECTOR_STORE(pg_true, v_size, 1, 2);
-      VECTOR_STORE(pg_true, v_size, 1, 3);
-      VECTOR_STORE(pg_true, v_size, 1, 4);
-      VECTOR_STORE(pg_true, v_size, 1, 5);
-      VECTOR_STORE(pg_true, v_size, 1, 6);
-      VECTOR_STORE(pg_true, v_size, 1, 7);
-    }
-    for (; j < n4; j += 4) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(1, 0);
-      DECLARE_RESULT_VECTOR(1, 1);
-      DECLARE_RESULT_VECTOR(1, 2);
-      DECLARE_RESULT_VECTOR(1, 3);
-
-      if (LIKELY(packed_a != NULL)) {
-        for (; k < K; k++) {
-
-          QUADWORD_LOAD_B(0, 0);
-          UNPACK_VECTOR_A(0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          UNPACK_VECTOR_A(v_size, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
-        }
-      } else {
-        for (; k < K; k++) {
-
-          QUADWORD_LOAD_B(0, 0);
-          GATHER_LOAD_A(pg_true, 0, 0, 0);
+          GATHER_LOAD_A(pg_true, 0, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
           UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-          GATHER_LOAD_A(pg_true, v_size, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 0, 0, 0, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 1, 0, 1, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 2, 0, 2, 0);
-          UPDATE_RESULT_VECTOR_QUADWORD(1, 3, 0, 3, 0);
         }
       }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-      VECTOR_STORE(pg_true, 0, 0, 1);
-      VECTOR_STORE(pg_true, 0, 0, 2);
-      VECTOR_STORE(pg_true, 0, 0, 3);
-      VECTOR_STORE(pg_true, v_size, 1, 0);
-      VECTOR_STORE(pg_true, v_size, 1, 1);
-      VECTOR_STORE(pg_true, v_size, 1, 2);
-      VECTOR_STORE(pg_true, v_size, 1, 3);
+      VECTOR_STORE(pg_true, 0, 0);
+      VECTOR_STORE(pg_true, 0, 1);
+      VECTOR_STORE(pg_true, 0, 2);
+      VECTOR_STORE(pg_true, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(1, 0);
 
       if (LIKELY(packed_a != NULL)) {
         for (; k < K; k++) {
 
           BROADCAST_LOAD_B(0, 0);
-          UNPACK_VECTOR_A(0, 0, 0);
+          UNPACK_VECTOR_A(0, 0);
           UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-          UNPACK_VECTOR_A(v_size, 1, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
         }
       } else {
         for (; k < K; k++) {
 
           BROADCAST_LOAD_B(0, 0);
-          GATHER_LOAD_A(pg_true, 0, 0, 0);
+          GATHER_LOAD_A(pg_true, 0, 0);
           UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-          GATHER_LOAD_A(pg_true, v_size, 1, 0);
-          UPDATE_RESULT_VECTOR(pg_true, 1, 0, 0);
         }
       }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-      VECTOR_STORE(pg_true, v_size, 1, 0);
+      VECTOR_STORE(pg_true, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
-  }
-  for (; i < v_m1; i += v_size) {
-
-    BLASLONG j = 0;
-    for (; j < n8; j += 8) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-
-      for (; k < K; k++) {
 
-        QUADWORD_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_true, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        QUADWORD_LOAD_B(4, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-      VECTOR_STORE(pg_true, 0, 0, 1);
-      VECTOR_STORE(pg_true, 0, 0, 2);
-      VECTOR_STORE(pg_true, 0, 0, 3);
-      VECTOR_STORE(pg_true, 0, 0, 4);
-      VECTOR_STORE(pg_true, 0, 0, 5);
-      VECTOR_STORE(pg_true, 0, 0, 6);
-      VECTOR_STORE(pg_true, 0, 0, 7);
-    }
-    for (; j < n4; j += 4) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-
-      for (; k < K; k++) {
-
-        QUADWORD_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_true, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-      VECTOR_STORE(pg_true, 0, 0, 1);
-      VECTOR_STORE(pg_true, 0, 0, 2);
-      VECTOR_STORE(pg_true, 0, 0, 3);
-    }
-    for (; j < N; j++) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-
-      for (; k < K; k++) {
-
-        BROADCAST_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_true, 0, 0, 0);
-        UPDATE_RESULT_VECTOR(pg_true, 0, 0, 0);
-      }
-      VECTOR_STORE(pg_true, 0, 0, 0);
-    }
+    UPDATE_A_POINTER(v_size);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(v_size);
   }
   for (; i < M; i += v_size) {
     const svbool_t pg_tail = svwhilelt_b32((uint32_t)i, (uint32_t)(M));
+    CREATE_C_POINTER(0, 0);
+    CREATE_A_POINTER(0, 0);
 
     BLASLONG j = 0;
-    for (; j < n8; j += 8) {
-
-      BLASLONG k = 0;
-      DECLARE_RESULT_VECTOR(0, 0);
-      DECLARE_RESULT_VECTOR(0, 1);
-      DECLARE_RESULT_VECTOR(0, 2);
-      DECLARE_RESULT_VECTOR(0, 3);
-      DECLARE_RESULT_VECTOR(0, 4);
-      DECLARE_RESULT_VECTOR(0, 5);
-      DECLARE_RESULT_VECTOR(0, 6);
-      DECLARE_RESULT_VECTOR(0, 7);
-
-      for (; k < K; k++) {
-
-        QUADWORD_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_tail, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
-        QUADWORD_LOAD_B(4, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 4, 4, 0, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 5, 4, 1, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 6, 4, 2, 0);
-        UPDATE_RESULT_VECTOR_QUADWORD(0, 7, 4, 3, 0);
-      }
-      VECTOR_STORE(pg_tail, 0, 0, 0);
-      VECTOR_STORE(pg_tail, 0, 0, 1);
-      VECTOR_STORE(pg_tail, 0, 0, 2);
-      VECTOR_STORE(pg_tail, 0, 0, 3);
-      VECTOR_STORE(pg_tail, 0, 0, 4);
-      VECTOR_STORE(pg_tail, 0, 0, 5);
-      VECTOR_STORE(pg_tail, 0, 0, 6);
-      VECTOR_STORE(pg_tail, 0, 0, 7);
-    }
     for (; j < n4; j += 4) {
 
+      CREATE_B_POINTER(0, 0);
+      CREATE_B_POINTER(1, 1);
+      CREATE_B_POINTER(2, 2);
+      CREATE_B_POINTER(3, 3);
+      UPDATE_B_POINTER(4);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
       DECLARE_RESULT_VECTOR(0, 1);
@@ -541,34 +342,43 @@ CNAME(BLASLONG M,
       for (; k < K; k++) {
 
         QUADWORD_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_tail, 0, 0, 0);
+        GATHER_LOAD_A(pg_tail, 0, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(0, 0, 0, 0, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(0, 1, 0, 1, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(0, 2, 0, 2, 0);
         UPDATE_RESULT_VECTOR_QUADWORD(0, 3, 0, 3, 0);
       }
-      VECTOR_STORE(pg_tail, 0, 0, 0);
-      VECTOR_STORE(pg_tail, 0, 0, 1);
-      VECTOR_STORE(pg_tail, 0, 0, 2);
-      VECTOR_STORE(pg_tail, 0, 0, 3);
+      VECTOR_STORE(pg_tail, 0, 0);
+      VECTOR_STORE(pg_tail, 0, 1);
+      VECTOR_STORE(pg_tail, 0, 2);
+      VECTOR_STORE(pg_tail, 0, 3);
+      INCR_C_POINTER(0, 4);
     }
     for (; j < N; j++) {
 
+      CREATE_B_POINTER(0, 0);
+      UPDATE_B_POINTER(1);
+
       BLASLONG k = 0;
       DECLARE_RESULT_VECTOR(0, 0);
 
       for (; k < K; k++) {
 
         BROADCAST_LOAD_B(0, 0);
-        GATHER_LOAD_A(pg_tail, 0, 0, 0);
+        GATHER_LOAD_A(pg_tail, 0, 0);
         UPDATE_RESULT_VECTOR(pg_tail, 0, 0, 0);
       }
-      VECTOR_STORE(pg_tail, 0, 0, 0);
+      VECTOR_STORE(pg_tail, 0, 0);
+      INCR_C_POINTER(0, 1);
     }
+
+    UPDATE_A_POINTER(0);
+    RESET_B_POINTER();
+    UPDATE_C_POINTER(0);
   }
 
   if (pack_a)
     free(packed_a);
 
   return 0;
-}
+}