|
|
@@ -143,7 +143,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
#if defined(BARCELONA) || defined(SHANGHAI) || defined(BOBCAT) || defined(BULLDOZER) |
|
|
|
#if defined(BARCELONA) || defined(SHANGHAI) || defined(BOBCAT) |
|
|
|
|
|
|
|
#define SNUMOPT 8 |
|
|
|
#define DNUMOPT 4 |
|
|
@@ -152,13 +152,8 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
#define GEMM_DEFAULT_OFFSET_B 832 |
|
|
|
#define GEMM_DEFAULT_ALIGN 0x0fffUL |
|
|
|
|
|
|
|
#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) |
|
|
|
#define SGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#else |
|
|
|
#define SGEMM_DEFAULT_UNROLL_N 4 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_N 4 |
|
|
|
#endif |
|
|
|
#define QGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_N 2 |
|
|
@@ -166,28 +161,18 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
|
|
|
|
#ifdef ARCH_X86 |
|
|
|
#define SGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define QGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#define XGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#else |
|
|
|
#if defined(BULLDOZER) && !defined(COMPLEX) |
|
|
|
#define SGEMM_DEFAULT_UNROLL_M 16 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 8 |
|
|
|
#else |
|
|
|
#define SGEMM_DEFAULT_UNROLL_M 8 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#endif |
|
|
|
#define QGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define XGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#define CGEMM3M_DEFAULT_UNROLL_N 4 |
|
|
|
#define CGEMM3M_DEFAULT_UNROLL_M 8 |
|
|
|
#define ZGEMM3M_DEFAULT_UNROLL_N 4 |
|
|
|
#define ZGEMM3M_DEFAULT_UNROLL_M 4 |
|
|
|
#define GEMV_UNROLL 8 |
|
|
|
#endif |
|
|
|
|
|
|
|
#if 0 |
|
|
@@ -207,7 +192,80 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
|
|
|
|
#else |
|
|
|
|
|
|
|
#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) |
|
|
|
#define SGEMM_DEFAULT_P 448 |
|
|
|
#define DGEMM_DEFAULT_P 224 |
|
|
|
#define QGEMM_DEFAULT_P 112 |
|
|
|
#define CGEMM_DEFAULT_P 224 |
|
|
|
#define ZGEMM_DEFAULT_P 112 |
|
|
|
#define XGEMM_DEFAULT_P 56 |
|
|
|
|
|
|
|
#define SGEMM_DEFAULT_Q 224 |
|
|
|
#define DGEMM_DEFAULT_Q 224 |
|
|
|
#define QGEMM_DEFAULT_Q 224 |
|
|
|
#define CGEMM_DEFAULT_Q 224 |
|
|
|
#define ZGEMM_DEFAULT_Q 224 |
|
|
|
#define XGEMM_DEFAULT_Q 224 |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
#define SGEMM_DEFAULT_R sgemm_r |
|
|
|
#define QGEMM_DEFAULT_R qgemm_r |
|
|
|
#define DGEMM_DEFAULT_R dgemm_r |
|
|
|
#define CGEMM_DEFAULT_R cgemm_r |
|
|
|
#define ZGEMM_DEFAULT_R zgemm_r |
|
|
|
#define XGEMM_DEFAULT_R xgemm_r |
|
|
|
|
|
|
|
#define SYMV_P 16 |
|
|
|
#define HAVE_EXCLUSIVE_CACHE |
|
|
|
|
|
|
|
#define GEMM_THREAD gemm_thread_mn |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
|
|
#ifdef BULLDOZER |
|
|
|
|
|
|
|
#define SNUMOPT 8 |
|
|
|
#define DNUMOPT 4 |
|
|
|
|
|
|
|
#define GEMM_DEFAULT_OFFSET_A 64 |
|
|
|
#define GEMM_DEFAULT_OFFSET_B 832 |
|
|
|
#define GEMM_DEFAULT_ALIGN 0x0fffUL |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#define QGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define XGEMM_DEFAULT_UNROLL_N 1 |
|
|
|
|
|
|
|
#ifdef ARCH_X86 |
|
|
|
#define SGEMM_DEFAULT_UNROLL_N 4 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_N 4 |
|
|
|
#define SGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define QGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#define XGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#else |
|
|
|
#define SGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_N 2 |
|
|
|
#define SGEMM_DEFAULT_UNROLL_M 16 |
|
|
|
#define DGEMM_DEFAULT_UNROLL_M 8 |
|
|
|
#define QGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define CGEMM_DEFAULT_UNROLL_M 4 |
|
|
|
#define ZGEMM_DEFAULT_UNROLL_M 2 |
|
|
|
#define XGEMM_DEFAULT_UNROLL_M 1 |
|
|
|
#define CGEMM3M_DEFAULT_UNROLL_N 4 |
|
|
|
#define CGEMM3M_DEFAULT_UNROLL_M 8 |
|
|
|
#define ZGEMM3M_DEFAULT_UNROLL_N 4 |
|
|
|
#define ZGEMM3M_DEFAULT_UNROLL_M 4 |
|
|
|
#define GEMV_UNROLL 8 |
|
|
|
#endif |
|
|
|
|
|
|
|
|
|
|
|
#if defined(ARCH_X86_64) |
|
|
|
#define SGEMM_DEFAULT_P 768 |
|
|
|
#define DGEMM_DEFAULT_P 384 |
|
|
|
#else |
|
|
@@ -219,7 +277,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
#define ZGEMM_DEFAULT_P 112 |
|
|
|
#define XGEMM_DEFAULT_P 56 |
|
|
|
|
|
|
|
#if defined(BULLDOZER) && defined(ARCH_X86_64) && !defined(COMPLEX) |
|
|
|
#if defined(ARCH_X86_64) |
|
|
|
#define SGEMM_DEFAULT_Q 168 |
|
|
|
#define DGEMM_DEFAULT_Q 168 |
|
|
|
#else |
|
|
@@ -231,8 +289,6 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
|
|
#define ZGEMM_DEFAULT_Q 224 |
|
|
|
#define XGEMM_DEFAULT_Q 224 |
|
|
|
|
|
|
|
#endif |
|
|
|
|
|
|
|
#define SGEMM_DEFAULT_R sgemm_r |
|
|
|
#define QGEMM_DEFAULT_R qgemm_r |
|
|
|
#define DGEMM_DEFAULT_R dgemm_r |
|
|
|