added dcopy_bulldozer.S

kernel/x86_64/KERNEL.BULLDOZER

@@ -3,8 +3,9 @@ ZGEMVTKERNEL = zgemv_t_dup.S
 
 DGEMVNKERNEL = dgemv_n_bulldozer.S
 DGEMVTKERNEL = dgemv_t_bulldozer.S
-DAXPYKERNEL =  daxpy_bulldozer.S
-DDOTKERNEL =  ddot_bulldozer.S
+DAXPYKERNEL  = daxpy_bulldozer.S
+DDOTKERNEL   = ddot_bulldozer.S
+DCOPYKERNEL  = dcopy_bulldozer.S
 
 SGEMMKERNEL    =  sgemm_kernel_16x2_bulldozer.S
 SGEMMINCOPY    =  ../generic/gemm_ncopy_16.c

kernel/x86_64/dcopy_bulldozer.S

@@ -0,0 +1,291 @@
+/*********************************************************************/
+/* Copyright 2009, 2010 The University of Texas at Austin.           */
+/* All rights reserved.                                              */
+/*                                                                   */
+/* Redistribution and use in source and binary forms, with or        */
+/* without modification, are permitted provided that the following   */
+/* conditions are met:                                               */
+/*                                                                   */
+/*   1. Redistributions of source code must retain the above         */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer.                                                  */
+/*                                                                   */
+/*   2. Redistributions in binary form must reproduce the above      */
+/*      copyright notice, this list of conditions and the following  */
+/*      disclaimer in the documentation and/or other materials       */
+/*      provided with the distribution.                              */
+/*                                                                   */
+/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
+/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
+/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
+/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
+/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
+/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
+/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
+/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
+/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
+/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
+/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
+/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
+/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
+/*    POSSIBILITY OF SUCH DAMAGE.                                    */
+/*                                                                   */
+/* The views and conclusions contained in the software and           */
+/* documentation are those of the authors and should not be          */
+/* interpreted as representing official policies, either expressed   */
+/* or implied, of The University of Texas at Austin.                 */
+/*********************************************************************/
+
+#define ASSEMBLER
+#include "common.h"
+
+#define M	ARG1	/* rdi */
+#define X	ARG2	/* rsi */
+#define INCX	ARG3	/* rdx */
+#define Y	ARG4	/* rcx */
+#ifndef WINDOWS_ABI
+#define INCY	ARG5	/* r8  */
+#else
+#define INCY	%r10
+#endif
+
+#include "l1param.h"
+
+#define VLOAD(OFFSET, ADDR, REG)		vmovups	OFFSET(ADDR), REG
+#define VSHUFPD_1(REG1 , REG2)			vshufpd $0x01, REG1, REG2, REG2
+#define A_PRE 640
+#define B_PRE 640
+
+	PROLOGUE
+	PROFCODE
+
+#ifdef WINDOWS_ABI
+	movq	40(%rsp), INCY
+#endif
+
+	SAVEREGISTERS
+
+	leaq	(, INCX, SIZE), INCX
+	leaq	(, INCY, SIZE), INCY
+
+	cmpq	$SIZE, INCX
+	jne	.L40
+	cmpq	$SIZE, INCY
+	jne	.L40
+
+	testq	$SIZE, X
+	je	.L10
+
+	vmovsd	(X), %xmm0
+	vmovsd	%xmm0, (Y)
+	addq	$1 * SIZE, X
+	addq	$1 * SIZE, Y
+	decq	M
+	jle	.L19
+	ALIGN_4
+
+.L10:
+	subq	$-16 * SIZE, X
+	subq	$-16 * SIZE, Y
+
+
+	movq	M,  %rax
+	sarq	$4, %rax
+	jle	.L13
+
+	vmovups	-16 * SIZE(X), %xmm0
+	vmovups	-14 * SIZE(X), %xmm1
+	vmovups	-12 * SIZE(X), %xmm2
+	vmovups	-10 * SIZE(X), %xmm3
+	vmovups	 -8 * SIZE(X), %xmm4
+	vmovups	 -6 * SIZE(X), %xmm5
+	vmovups	 -4 * SIZE(X), %xmm6
+	vmovups	 -2 * SIZE(X), %xmm7
+
+	decq	%rax
+	jle .L12
+	ALIGN_4
+
+.L11:
+
+	prefetchnta	A_PRE(X)
+	nop
+	vmovups	%xmm0, -16 * SIZE(Y)
+	vmovups	%xmm1, -14 * SIZE(Y)
+	prefetchnta	B_PRE(Y)
+	nop
+	vmovups	%xmm2, -12 * SIZE(Y)
+	vmovups	%xmm3, -10 * SIZE(Y)
+
+	VLOAD( 0 * SIZE, X, %xmm0)
+	VLOAD( 2 * SIZE, X, %xmm1)
+	VLOAD( 4 * SIZE, X, %xmm2)
+	VLOAD( 6 * SIZE, X, %xmm3)
+
+	prefetchnta	A_PRE+64(X)
+	nop
+	vmovups	%xmm4, -8 * SIZE(Y)
+	vmovups	%xmm5, -6 * SIZE(Y)
+	prefetchnta	B_PRE+64(Y)
+	nop
+	vmovups	%xmm6, -4 * SIZE(Y)
+	vmovups	%xmm7, -2 * SIZE(Y)
+
+	VLOAD( 8 * SIZE, X, %xmm4)
+	VLOAD(10 * SIZE, X, %xmm5)
+	subq	$-16 * SIZE, Y
+	VLOAD(12 * SIZE, X, %xmm6)
+	VLOAD(14 * SIZE, X, %xmm7)
+
+	subq	$-16 * SIZE, X
+	decq	%rax
+	jg	.L11
+	ALIGN_3
+
+.L12:
+	vmovups	%xmm0, -16 * SIZE(Y)
+	vmovups	%xmm1, -14 * SIZE(Y)
+	vmovups	%xmm2, -12 * SIZE(Y)
+	vmovups	%xmm3, -10 * SIZE(Y)
+	vmovups	%xmm4,  -8 * SIZE(Y)
+	vmovups	%xmm5,  -6 * SIZE(Y)
+	vmovups	%xmm6,  -4 * SIZE(Y)
+	vmovups	%xmm7,  -2 * SIZE(Y)
+
+	subq	$-16 * SIZE, Y
+	subq	$-16 * SIZE, X
+	ALIGN_3
+
+.L13:
+	testq	$8, M
+	jle	.L14
+	ALIGN_3
+
+	vmovups	-16 * SIZE(X), %xmm0
+	vmovups	-14 * SIZE(X), %xmm1
+	vmovups	-12 * SIZE(X), %xmm2
+	vmovups	-10 * SIZE(X), %xmm3
+
+	vmovups	%xmm0, -16 * SIZE(Y)
+	vmovups	%xmm1, -14 * SIZE(Y)
+	vmovups	%xmm2, -12 * SIZE(Y)
+	vmovups	%xmm3, -10 * SIZE(Y)
+
+	addq	$8 * SIZE, X
+	addq	$8 * SIZE, Y
+	ALIGN_3
+
+.L14:
+	testq	$4, M
+	jle	.L15
+	ALIGN_3
+
+	vmovups	-16 * SIZE(X), %xmm0
+	vmovups	-14 * SIZE(X), %xmm1
+
+	vmovups	%xmm0, -16 * SIZE(Y)
+	vmovups	%xmm1, -14 * SIZE(Y)
+
+	addq	$4 * SIZE, X
+	addq	$4 * SIZE, Y
+	ALIGN_3
+
+.L15:
+	testq	$2, M
+	jle	.L16
+	ALIGN_3
+
+	vmovups	-16 * SIZE(X), %xmm0
+	vmovups	%xmm0, -16 * SIZE(Y)
+
+	addq	$2 * SIZE, X
+	addq	$2 * SIZE, Y
+	ALIGN_3
+
+.L16:
+	testq	$1, M
+	jle	.L19
+	ALIGN_3
+
+	vmovsd	-16 * SIZE(X), %xmm0
+	vmovsd	%xmm0, 	-16 * SIZE(Y)
+	ALIGN_3
+
+.L19:
+	xorq	%rax,%rax
+
+	RESTOREREGISTERS
+
+	ret
+	ALIGN_3
+
+
+
+.L40:
+	movq	M,  %rax
+	sarq	$3, %rax
+	jle	.L45
+	ALIGN_3
+
+.L41:
+	vmovsd	(X), %xmm0
+	addq	INCX, X
+	vmovsd	(X), %xmm4
+	addq	INCX, X
+	vmovsd	(X), %xmm1
+	addq	INCX, X
+	vmovsd	(X), %xmm5
+	addq	INCX, X
+	vmovsd	(X), %xmm2
+	addq	INCX, X
+	vmovsd	(X), %xmm6
+	addq	INCX, X
+	vmovsd	(X), %xmm3
+	addq	INCX, X
+	vmovsd	(X), %xmm7
+	addq	INCX, X
+
+	vmovsd	%xmm0, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm4, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm1, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm5, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm2, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm6, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm3, (Y)
+	addq	INCY, Y
+	vmovsd	%xmm7, (Y)
+	addq	INCY, Y
+
+	decq	%rax
+	jg	.L41
+	ALIGN_3
+
+.L45:
+	movq	M,  %rax
+	andq	$7, %rax
+	jle	.L47
+	ALIGN_3
+
+.L46:
+	vmovsd	(X), %xmm0
+	addq	INCX, X
+	vmovsd	%xmm0, (Y)
+	addq	INCY, Y
+	decq	%rax
+	jg	.L46
+	ALIGN_3
+
+.L47:
+	xorq	%rax, %rax
+
+	RESTOREREGISTERS
+
+	ret
+
+	EPILOGUE