Merge pull request #486 from wernsaar/develop

Optimizations for steamroller

Makefile.rule

@@ -15,7 +15,7 @@ VERSION = 0.2.13
 # TARGET = PENRYN
 
 # If you want to support multiple architecture in one binary
-# DYNAMIC_ARCH = 1
+#DYNAMIC_ARCH = 1
 
 # C compiler including binary type(32bit / 64bit). Default is gcc.
 # Don't use Intel Compiler or PGI, it won't generate right codes as I expect.

benchmark/Makefile

@@ -10,7 +10,7 @@ include $(TOPDIR)/Makefile.system
 #LIBACML = -fopenmp $(ACML)/libacml_mp.a -lgfortran -lm
 
 # ACML 6.1 custom
-ACML=/home/saar/acml6.1/gfortran64_mp/lib
+ACML=/home/werner/project/acml6.1/gfortran64_mp/lib
 LIBACML = -fopenmp $(ACML)/libacml_mp.so -lgfortran -lm
  
 

common.h

@@ -327,6 +327,14 @@ typedef int blasint;
 #endif
 #endif
 
+/*
+#ifdef STEAMROLLER
+#ifndef YIELDING
+#define YIELDING        __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop;\n");
+#endif
+#endif
+*/
+
 #ifndef YIELDING
 #define YIELDING	sched_yield()
 #endif

kernel/x86_64/KERNEL.STEAMROLLER

@@ -1,15 +1,27 @@
+DAXPYKERNEL = daxpy.c
+CAXPYKERNEL = caxpy.c
+ZAXPYKERNEL = zaxpy.c
+
+SDOTKERNEL     = sdot.c
+DDOTKERNEL     = ddot.c
+
+DSYMV_U_KERNEL = dsymv_U.c
+DSYMV_L_KERNEL = dsymv_L.c
+SSYMV_U_KERNEL = ssymv_U.c
+SSYMV_L_KERNEL = ssymv_L.c
+
 SGEMVNKERNEL = sgemv_n_4.c
 SGEMVTKERNEL = sgemv_t_4.c
 
+DGEMVNKERNEL = dgemv_n_4.c
+DGEMVTKERNEL = dgemv_t_4.c
+
 ZGEMVNKERNEL = zgemv_n_dup.S
 ZGEMVTKERNEL = zgemv_t_4.c
 
-DGEMVNKERNEL = dgemv_n_bulldozer.S
-DGEMVTKERNEL = dgemv_t_bulldozer.S
-
-DDOTKERNEL   = ddot_bulldozer.S
 DCOPYKERNEL  = dcopy_bulldozer.S
 
+
 SGEMMKERNEL    =  sgemm_kernel_16x2_piledriver.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_16.c
 SGEMMITCOPY    =  ../generic/gemm_tcopy_16.c
@@ -21,8 +33,8 @@ SGEMMONCOPYOBJ =  sgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SGEMMOTCOPYOBJ =  sgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 DGEMMKERNEL    =  dgemm_kernel_8x2_piledriver.S
-DGEMMINCOPY    =  dgemm_ncopy_8_bulldozer.S
-DGEMMITCOPY    =  dgemm_tcopy_8_bulldozer.S
+DGEMMINCOPY    =  ../generic/gemm_ncopy_8.c
+DGEMMITCOPY    =  ../generic/gemm_tcopy_8.c
 DGEMMONCOPY    =  gemm_ncopy_2_bulldozer.S
 DGEMMOTCOPY    =  gemm_tcopy_2_bulldozer.S
 DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)

kernel/x86_64/caxpy.c

@@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "caxpy_microk_bulldozer-2.c"
 #endif
 

kernel/x86_64/daxpy.c

@@ -31,7 +31,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(NEHALEM)
 #include "daxpy_microk_nehalem-2.c"
-#elif defined(BULLDOZER)
+#elif defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "daxpy_microk_bulldozer-2.c"
 #endif
 

kernel/x86_64/dgemv_n_4.c

@@ -31,7 +31,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(NEHALEM)
 #include "dgemv_n_microk_nehalem-4.c"
-#elif defined(HASWELL)
+#elif defined(HASWELL) || defined(STEAMROLLER)
 #include "dgemv_n_microk_haswell-4.c"
 #endif
 

kernel/x86_64/dgemv_n_microk_piledriver-4.c

@@ -0,0 +1,247 @@
+/***************************************************************************
+Copyright (c) 2014, The OpenBLAS Project
+All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in
+the documentation and/or other materials provided with the
+distribution.
+3. Neither the name of the OpenBLAS project nor the names of
+its contributors may be used to endorse or promote products
+derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*****************************************************************************/
+
+
+
+#define HAVE_KERNEL_4x8 1
+static void dgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha) __attribute__ ((noinline));
+
+static void dgemv_kernel_4x8( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, BLASLONG lda4, FLOAT *alpha)
+{
+
+	BLASLONG register i = 0;
+
+	__asm__  __volatile__
+	(
+	"vzeroupper			 \n\t"
+	"vbroadcastsd    (%2), %%ymm12	 \n\t"	// x0 
+	"vbroadcastsd   8(%2), %%ymm13	 \n\t"	// x1 
+	"vbroadcastsd  16(%2), %%ymm14	 \n\t"	// x2 
+	"vbroadcastsd  24(%2), %%ymm15	 \n\t"	// x3 
+	"vbroadcastsd  32(%2), %%ymm0 	 \n\t"	// x4 
+	"vbroadcastsd  40(%2), %%ymm1 	 \n\t"	// x5 
+	"vbroadcastsd  48(%2), %%ymm2 	 \n\t"	// x6 
+	"vbroadcastsd  56(%2), %%ymm3 	 \n\t"	// x7 
+
+	"vbroadcastsd    (%9), %%ymm6 	 \n\t"	// alpha 
+
+        "testq          $0x04, %1                      \n\t"
+        "jz             2f                     \n\t"
+
+	"vmovupd	(%3,%0,8), %%ymm7	       \n\t"	// 4 * y
+	"vxorpd		%%ymm4 , %%ymm4, %%ymm4        \n\t"
+	"vxorpd		%%ymm5 , %%ymm5, %%ymm5        \n\t"
+
+	"vfmadd231pd   (%4,%0,8), %%ymm12, %%ymm4      \n\t" 
+	"vfmadd231pd   (%5,%0,8), %%ymm13, %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%0,8), %%ymm14, %%ymm4      \n\t" 
+	"vfmadd231pd   (%7,%0,8), %%ymm15, %%ymm5      \n\t" 
+
+	"vfmadd231pd   (%4,%8,8), %%ymm0 , %%ymm4      \n\t" 
+	"vfmadd231pd   (%5,%8,8), %%ymm1 , %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%8,8), %%ymm2 , %%ymm4      \n\t" 
+	"vfmadd231pd   (%7,%8,8), %%ymm3 , %%ymm5      \n\t" 
+
+	"vaddpd		%%ymm4 , %%ymm5 , %%ymm5       \n\t"
+	"vmulpd		%%ymm6 , %%ymm5 , %%ymm5       \n\t"
+	"vaddpd		%%ymm7 , %%ymm5 , %%ymm5       \n\t"
+
+
+	"vmovupd  %%ymm5,   (%3,%0,8)		       \n\t"	// 4 * y
+
+        "addq		$4 , %8	  	 	       \n\t"
+        "addq		$4 , %0	  	 	       \n\t"
+	"subq	        $4 , %1			       \n\t"		
+
+        "2:                                   \n\t"
+
+        "cmpq           $0, %1                         \n\t"
+        "je             3f                      \n\t"
+
+
+	".align 16				 \n\t"
+	"1:				 \n\t"
+
+	"vxorpd		%%ymm4 , %%ymm4, %%ymm4        \n\t"
+	"vxorpd		%%ymm5 , %%ymm5, %%ymm5        \n\t"
+	"vmovupd	(%3,%0,8), %%ymm8	       \n\t"	// 4 * y
+	"vmovupd      32(%3,%0,8), %%ymm9	       \n\t"	// 4 * y
+
+	"vfmadd231pd   (%4,%0,8), %%ymm12, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%4,%0,8), %%ymm12, %%ymm5      \n\t" 
+	"vfmadd231pd   (%5,%0,8), %%ymm13, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%5,%0,8), %%ymm13, %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%0,8), %%ymm14, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%6,%0,8), %%ymm14, %%ymm5      \n\t" 
+	"vfmadd231pd   (%7,%0,8), %%ymm15, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%7,%0,8), %%ymm15, %%ymm5      \n\t" 
+
+	"vfmadd231pd   (%4,%8,8), %%ymm0 , %%ymm4      \n\t" 
+        "addq		$8 , %0	  	 	       \n\t"
+	"vfmadd231pd 32(%4,%8,8), %%ymm0 , %%ymm5      \n\t" 
+	"vfmadd231pd   (%5,%8,8), %%ymm1 , %%ymm4      \n\t" 
+	"vfmadd231pd 32(%5,%8,8), %%ymm1 , %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%8,8), %%ymm2 , %%ymm4      \n\t" 
+	"vfmadd231pd 32(%6,%8,8), %%ymm2 , %%ymm5      \n\t" 
+	"vfmadd231pd   (%7,%8,8), %%ymm3 , %%ymm4      \n\t" 
+	"vfmadd231pd 32(%7,%8,8), %%ymm3 , %%ymm5      \n\t" 
+
+	"vfmadd231pd     %%ymm6 , %%ymm4 , %%ymm8      \n\t"
+	"vfmadd231pd     %%ymm6 , %%ymm5 , %%ymm9      \n\t"
+
+        "addq		$8 , %8	  	 	      \n\t"
+	"vmovupd  %%ymm8,-64(%3,%0,8)		      \n\t"	// 4 * y
+	"subq	        $8 , %1			      \n\t"		
+	"vmovupd  %%ymm9,-32(%3,%0,8)		      \n\t"	// 4 * y
+
+	"jnz		1b		      \n\t"
+
+        "3:                             \n\t"
+	"vzeroupper			        \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x),      // 2
+          "r" (y),      // 3
+          "r" (ap[0]),  // 4
+          "r" (ap[1]),  // 5
+          "r" (ap[2]),  // 6
+          "r" (ap[3]),  // 7
+          "r" (lda4),   // 8
+          "r" (alpha)   // 9
+	: "cc", 
+	  "%xmm0", "%xmm1", 
+	  "%xmm2", "%xmm3", 
+	  "%xmm4", "%xmm5", 
+	  "%xmm6", "%xmm7", 
+	  "%xmm8", "%xmm9", 
+	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+	  "memory"
+	);
+
+} 
+
+
+
+#define HAVE_KERNEL_4x4 1
+static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));
+
+static void dgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)
+{
+
+	BLASLONG register i = 0;
+
+	__asm__  __volatile__
+	(
+	"vzeroupper			 \n\t"
+	"vbroadcastsd    (%2), %%ymm12	 \n\t"	// x0 
+	"vbroadcastsd   8(%2), %%ymm13	 \n\t"	// x1 
+	"vbroadcastsd  16(%2), %%ymm14	 \n\t"	// x2 
+	"vbroadcastsd  24(%2), %%ymm15	 \n\t"	// x3 
+
+	"vbroadcastsd    (%8), %%ymm6 	 \n\t"	// alpha 
+
+        "testq          $0x04, %1                      \n\t"
+        "jz             2f                     \n\t"
+
+	"vxorpd		%%ymm4 , %%ymm4, %%ymm4        \n\t"
+	"vxorpd		%%ymm5 , %%ymm5, %%ymm5        \n\t"
+	"vmovupd	(%3,%0,8), %%ymm7	       \n\t"	// 4 * y
+
+	"vfmadd231pd   (%4,%0,8), %%ymm12, %%ymm4      \n\t" 
+	"vfmadd231pd   (%5,%0,8), %%ymm13, %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%0,8), %%ymm14, %%ymm4      \n\t" 
+	"vfmadd231pd   (%7,%0,8), %%ymm15, %%ymm5      \n\t" 
+
+	"vaddpd		%%ymm4 , %%ymm5 , %%ymm5       \n\t"
+	"vmulpd		%%ymm6 , %%ymm5 , %%ymm5       \n\t"
+	"vaddpd		%%ymm7 , %%ymm5 , %%ymm5       \n\t"
+
+	"vmovupd  %%ymm5,   (%3,%0,8)		       \n\t"	// 4 * y
+
+        "addq		$4 , %0	  	 	       \n\t"
+	"subq	        $4 , %1			       \n\t"		
+
+        "2:                                   \n\t"
+
+        "cmpq           $0, %1                         \n\t"
+        "je             3f                       \n\t"
+
+
+	".align 16				 \n\t"
+	"1:				 \n\t"
+	"vxorpd		%%ymm4 , %%ymm4, %%ymm4        \n\t"
+	"vxorpd		%%ymm5 , %%ymm5, %%ymm5        \n\t"
+	"vmovupd	(%3,%0,8), %%ymm8	       \n\t"	// 4 * y
+	"vmovupd      32(%3,%0,8), %%ymm9	       \n\t"	// 4 * y
+
+	"vfmadd231pd   (%4,%0,8), %%ymm12, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%4,%0,8), %%ymm12, %%ymm5      \n\t" 
+	"vfmadd231pd   (%5,%0,8), %%ymm13, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%5,%0,8), %%ymm13, %%ymm5      \n\t" 
+	"vfmadd231pd   (%6,%0,8), %%ymm14, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%6,%0,8), %%ymm14, %%ymm5      \n\t" 
+	"vfmadd231pd   (%7,%0,8), %%ymm15, %%ymm4      \n\t" 
+	"vfmadd231pd 32(%7,%0,8), %%ymm15, %%ymm5      \n\t" 
+
+	"vfmadd231pd     %%ymm6 , %%ymm4 , %%ymm8      \n\t"
+	"vfmadd231pd     %%ymm6 , %%ymm5 , %%ymm9      \n\t"
+
+	"vmovupd  %%ymm8,   (%3,%0,8)		      \n\t"	// 4 * y
+	"vmovupd  %%ymm9, 32(%3,%0,8)		      \n\t"	// 4 * y
+
+        "addq		$8 , %0	  	 	      \n\t"
+	"subq	        $8 , %1			      \n\t"		
+	"jnz		1b		      \n\t"
+
+        "3:                                    \n\t"
+	"vzeroupper			              \n\t"
+
+	:
+        : 
+          "r" (i),	// 0	
+	  "r" (n),  	// 1
+          "r" (x),      // 2
+          "r" (y),      // 3
+          "r" (ap[0]),  // 4
+          "r" (ap[1]),  // 5
+          "r" (ap[2]),  // 6
+          "r" (ap[3]),  // 7
+          "r" (alpha)   // 8
+	: "cc", 
+	  "%xmm4", "%xmm5", 
+	  "%xmm6", "%xmm7", 
+	  "%xmm8", "%xmm9", 
+	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+	  "memory"
+	);
+
+} 
+
+

kernel/x86_64/dgemv_t_4.c

@@ -28,7 +28,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(HASWELL)
+#if defined(HASWELL) || defined(STEAMROLLER)
 #include "dgemv_t_microk_haswell-4.c"
 #endif
 

kernel/x86_64/dsymv_L.c

@@ -28,7 +28,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "dsymv_L_microk_bulldozer-2.c"
 #elif defined(NEHALEM)
 #include "dsymv_L_microk_nehalem-2.c"

kernel/x86_64/dsymv_U.c

@@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "dsymv_U_microk_bulldozer-2.c"
 #elif defined(NEHALEM)
 #include "dsymv_U_microk_nehalem-2.c"

kernel/x86_64/sgemv_n_4.c

@@ -39,8 +39,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "sgemv_n_microk_haswell-4.c"
 #endif
 
-
+#if defined(STEAMROLLER)
+#define NBMAX 2048
+#else
 #define NBMAX 4096
+#endif
 
 #ifndef HAVE_KERNEL_4x8
 

kernel/x86_64/sgemv_t_4.c

@@ -38,7 +38,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "sgemv_t_microk_haswell-4.c"
 #endif
 
+#if defined(STEAMROLLER)
+#define NBMAX 2048
+#else
 #define NBMAX 4096
+#endif
 
 #ifndef HAVE_KERNEL_4x4
 

kernel/x86_64/ssymv_L.c

@@ -28,7 +28,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "common.h"
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "ssymv_L_microk_bulldozer-2.c"
 #elif defined(NEHALEM)
 #include "ssymv_L_microk_nehalem-2.c"

kernel/x86_64/ssymv_U.c

@@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "ssymv_U_microk_bulldozer-2.c"
 #elif defined(NEHALEM)
 #include "ssymv_U_microk_nehalem-2.c"

kernel/x86_64/zaxpy.c

@@ -29,7 +29,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "common.h"
 
 
-#if defined(BULLDOZER)
+#if defined(BULLDOZER) || defined(PILEDRIVER) || defined(STEAMROLLER)
 #include "zaxpy_microk_bulldozer-2.c"
 #endif
 

param.h

@@ -448,9 +448,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(ARCH_X86_64)
 #define SGEMM_DEFAULT_P 768
-#define DGEMM_DEFAULT_P 768
-#define ZGEMM_DEFAULT_P 384
-#define CGEMM_DEFAULT_P 768
+#define DGEMM_DEFAULT_P 576
+#define ZGEMM_DEFAULT_P 288
+#define CGEMM_DEFAULT_P 576
 #else
 #define SGEMM_DEFAULT_P 448
 #define DGEMM_DEFAULT_P 480
@@ -462,9 +462,9 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #if defined(ARCH_X86_64)
 #define SGEMM_DEFAULT_Q 192
-#define DGEMM_DEFAULT_Q 168
-#define ZGEMM_DEFAULT_Q 168
-#define CGEMM_DEFAULT_Q 168
+#define DGEMM_DEFAULT_Q 160
+#define ZGEMM_DEFAULT_Q 160
+#define CGEMM_DEFAULT_Q 160
 #else
 #define SGEMM_DEFAULT_Q 224
 #define DGEMM_DEFAULT_Q 224