OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/zgemv_t_microk_bulldozer-4.c

458 lines
21 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary froms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary from must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define HAVE_KERNEL_4x4 1

  29. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  30. 

  31. static void zgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  32. {

  33. 

  34. 	BLASLONG register i = 0;

  35. 

  36. 	__asm__  __volatile__

  37. 	(

  38. 	"vzeroupper			 \n\t"

  39. 

  40. 	"vxorpd		%%xmm8 , %%xmm8 , %%xmm8 	\n\t" // temp

  41. 	"vxorpd		%%xmm9 , %%xmm9 , %%xmm9 	\n\t" // temp

  42. 	"vxorpd		%%xmm10, %%xmm10, %%xmm10	\n\t" // temp

  43. 	"vxorpd		%%xmm11, %%xmm11, %%xmm11	\n\t" // temp

  44. 	"vxorpd		%%xmm12, %%xmm12, %%xmm12	\n\t" // temp

  45. 	"vxorpd		%%xmm13, %%xmm13, %%xmm13	\n\t"

  46. 	"vxorpd		%%xmm14, %%xmm14, %%xmm14	\n\t"

  47. 	"vxorpd		%%xmm15, %%xmm15, %%xmm15	\n\t"

  48. 

  49. 	".align 16				        \n\t"

  50. 	"1:				        \n\t"

  51. 

  52. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  53. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  54. 

  55. 	"prefetcht0  192(%4,%0,8)                       \n\t"

  56. 	"vmovups	(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  57. 	"prefetcht0  192(%5,%0,8)                       \n\t"

  58. 	"vmovups	(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  59. 	"prefetcht0  192(%6,%0,8)                       \n\t"

  60. 	"vmovups	(%6,%0,8), %%xmm6	        \n\t" // 1 complex values from a2

  61. 	"prefetcht0  192(%7,%0,8)                       \n\t"

  62. 	"vmovups	(%7,%0,8), %%xmm7               \n\t" // 1 complex values from a3

  63. 

  64. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  65. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  66. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  67. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  68. 	"vfmaddpd   %%xmm12,   %%xmm6 , %%xmm0, %%xmm12      \n\t" // ar0*xr0,al0*xr0 

  69. 	"vfmaddpd   %%xmm13,   %%xmm6 , %%xmm1, %%xmm13      \n\t" // ar0*xl0,al0*xl0 

  70. 	"vfmaddpd   %%xmm14,   %%xmm7 , %%xmm0, %%xmm14      \n\t" // ar0*xr0,al0*xr0 

  71. 	"vfmaddpd   %%xmm15,   %%xmm7 , %%xmm1, %%xmm15      \n\t" // ar0*xl0,al0*xl0 

  72. 

  73. 	"vmovddup	 16(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  74. 	"vmovddup	 24(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  75. 

  76. 	"vmovups      16(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  77. 	"vmovups      16(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  78. 	"vmovups      16(%6,%0,8), %%xmm6	        \n\t" // 1 complex values from a2

  79. 	"vmovups      16(%7,%0,8), %%xmm7               \n\t" // 1 complex values from a3

  80. 

  81. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  82. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  83. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  84. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  85. 	"vfmaddpd   %%xmm12,   %%xmm6 , %%xmm0, %%xmm12      \n\t" // ar0*xr0,al0*xr0 

  86. 	"vfmaddpd   %%xmm13,   %%xmm6 , %%xmm1, %%xmm13      \n\t" // ar0*xl0,al0*xl0 

  87. 	"vfmaddpd   %%xmm14,   %%xmm7 , %%xmm0, %%xmm14      \n\t" // ar0*xr0,al0*xr0 

  88. 	"vfmaddpd   %%xmm15,   %%xmm7 , %%xmm1, %%xmm15      \n\t" // ar0*xl0,al0*xl0 

  89. 

  90. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  91. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  92. 

  93. 	"vmovups      32(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  94. 	"vmovups      32(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  95. 	"vmovups      32(%6,%0,8), %%xmm6	        \n\t" // 1 complex values from a2

  96. 	"vmovups      32(%7,%0,8), %%xmm7               \n\t" // 1 complex values from a3

  97. 

  98. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  99. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  100. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  101. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  102. 	"vfmaddpd   %%xmm12,   %%xmm6 , %%xmm0, %%xmm12      \n\t" // ar0*xr0,al0*xr0 

  103. 	"vfmaddpd   %%xmm13,   %%xmm6 , %%xmm1, %%xmm13      \n\t" // ar0*xl0,al0*xl0 

  104. 	"vfmaddpd   %%xmm14,   %%xmm7 , %%xmm0, %%xmm14      \n\t" // ar0*xr0,al0*xr0 

  105. 	"vfmaddpd   %%xmm15,   %%xmm7 , %%xmm1, %%xmm15      \n\t" // ar0*xl0,al0*xl0 

  106. 

  107. 	"vmovddup	 48(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  108. 	"vmovddup	 56(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  109. 

  110. 	"vmovups      48(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  111. 	"vmovups      48(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  112. 	"vmovups      48(%6,%0,8), %%xmm6	        \n\t" // 1 complex values from a2

  113. 	"vmovups      48(%7,%0,8), %%xmm7               \n\t" // 1 complex values from a3

  114. 

  115. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  116. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  117. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  118. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  119. 	"vfmaddpd   %%xmm12,   %%xmm6 , %%xmm0, %%xmm12      \n\t" // ar0*xr0,al0*xr0 

  120. 	"vfmaddpd   %%xmm13,   %%xmm6 , %%xmm1, %%xmm13      \n\t" // ar0*xl0,al0*xl0 

  121. 	"vfmaddpd   %%xmm14,   %%xmm7 , %%xmm0, %%xmm14      \n\t" // ar0*xr0,al0*xr0 

  122. 	"vfmaddpd   %%xmm15,   %%xmm7 , %%xmm1, %%xmm15      \n\t" // ar0*xl0,al0*xl0 

  123. 

  124.         "addq		$8 , %0	  	 	        \n\t"

  125. 	"subq	        $4 , %1			        \n\t"		

  126. 	"jnz		1b		        \n\t"

  127. 

  128. 	"vmovddup               (%8)  , %%xmm0                \n\t"  // value from alpha

  129. 	"vmovddup	       8(%8)  , %%xmm1                \n\t"  // value from alpha

  130. 

  131. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  132.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  133.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  134.         "vpermilpd      $0x1 , %%xmm13, %%xmm13               \n\t"

  135.         "vpermilpd      $0x1 , %%xmm15, %%xmm15               \n\t"

  136.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  137.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  138.         "vaddsubpd      %%xmm13, %%xmm12, %%xmm12             \n\t"

  139.         "vaddsubpd      %%xmm15, %%xmm14, %%xmm14             \n\t"

  140. #else

  141.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  142.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  143.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  144.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  145.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  146.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  147.         "vaddsubpd      %%xmm12, %%xmm13, %%xmm12             \n\t"

  148.         "vaddsubpd      %%xmm14, %%xmm15, %%xmm14             \n\t"

  149.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  150.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  151.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  152.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  153. #endif

  154. 

  155. 	"vmulpd		%%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  156. 	"vmulpd		%%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  157. 	"vmulpd		%%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  158. 	"vmulpd		%%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  159. 	"vmulpd		%%xmm12, %%xmm1 , %%xmm13             \n\t"  // t_r * alpha_i , t_i * alpha_i

  160. 	"vmulpd		%%xmm12, %%xmm0 , %%xmm12             \n\t"  // t_r * alpha_r , t_i * alpha_r

  161. 	"vmulpd		%%xmm14, %%xmm1 , %%xmm15             \n\t"  // t_r * alpha_i , t_i * alpha_i

  162. 	"vmulpd		%%xmm14, %%xmm0 , %%xmm14             \n\t"  // t_r * alpha_r , t_i * alpha_r

  163. 

  164. #if !defined(XCONJ) 

  165.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  166.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  167.         "vpermilpd      $0x1 , %%xmm13, %%xmm13               \n\t"

  168.         "vpermilpd      $0x1 , %%xmm15, %%xmm15               \n\t"

  169.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  170.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  171.         "vaddsubpd      %%xmm13, %%xmm12, %%xmm12             \n\t"

  172.         "vaddsubpd      %%xmm15, %%xmm14, %%xmm14             \n\t"

  173. #else

  174.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  175.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  176.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  177.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  178.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  179.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  180.         "vaddsubpd      %%xmm12, %%xmm13, %%xmm12             \n\t"

  181.         "vaddsubpd      %%xmm14, %%xmm15, %%xmm14             \n\t"

  182.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  183.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  184.         "vpermilpd      $0x1 , %%xmm12, %%xmm12               \n\t"

  185.         "vpermilpd      $0x1 , %%xmm14, %%xmm14               \n\t"

  186. #endif

  187. 

  188. 	"vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  189. 	"vaddpd         16(%3) , %%xmm10, %%xmm10             \n\t"

  190. 	"vaddpd         32(%3) , %%xmm12, %%xmm12             \n\t"

  191. 	"vaddpd         48(%3) , %%xmm14, %%xmm14             \n\t"

  192. 

  193. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  194. 	"vmovups	%%xmm10, 16(%3)			\n\t"

  195. 	"vmovups	%%xmm12, 32(%3)			\n\t"

  196. 	"vmovups	%%xmm14, 48(%3)			\n\t"

  197. 

  198. 	"vzeroupper			 \n\t"

  199. 

  200. 	:

  201.           "+r" (i),	// 0	

  202. 	  "+r" (n)  	// 1

  203.         : 

  204.           "r" (x),      // 2

  205.           "r" (y),      // 3

  206.           "r" (ap[0]),  // 4

  207.           "r" (ap[1]),  // 5

  208.           "r" (ap[2]),  // 6

  209.           "r" (ap[3]),  // 7

  210.           "r" (alpha)   // 8

  211. 	: "cc", 

  212. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  213. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  214. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  215. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  216. 	  "memory"

  217. 	);

  218. 

  219. } 

  220. 

  221. #define HAVE_KERNEL_4x2 1

  222. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  223. 

  224. static void zgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  225. {

  226. 

  227. 	BLASLONG register i = 0;

  228. 

  229. 	__asm__  __volatile__

  230. 	(

  231. 	"vzeroupper			 \n\t"

  232. 

  233. 	"vxorpd		%%xmm8 , %%xmm8 , %%xmm8 	\n\t" // temp

  234. 	"vxorpd		%%xmm9 , %%xmm9 , %%xmm9 	\n\t" // temp

  235. 	"vxorpd		%%xmm10, %%xmm10, %%xmm10	\n\t" // temp

  236. 	"vxorpd		%%xmm11, %%xmm11, %%xmm11	\n\t" // temp

  237. 

  238. 	".align 16				        \n\t"

  239. 	"1:				        \n\t"

  240. 

  241. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  242. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  243. 

  244. 	"prefetcht0  192(%4,%0,8)                       \n\t"

  245. 	"vmovups	(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  246. 	"prefetcht0  192(%5,%0,8)                       \n\t"

  247. 	"vmovups	(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  248. 

  249. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  250. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  251. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  252. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  253. 

  254. 	"vmovddup	 16(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  255. 	"vmovddup	 24(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  256. 

  257. 	"vmovups      16(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  258. 	"vmovups      16(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  259. 

  260. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  261. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  262. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  263. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  264. 

  265. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  266. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  267. 

  268. 	"vmovups      32(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  269. 	"vmovups      32(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  270. 

  271. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  272. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  273. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  274. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  275. 

  276. 	"vmovddup	 48(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  277. 	"vmovddup	 56(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  278. 

  279. 	"vmovups      48(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  280. 	"vmovups      48(%5,%0,8), %%xmm5               \n\t" // 1 complex values from a1

  281. 

  282. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  283. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  284. 	"vfmaddpd   %%xmm10,   %%xmm5 , %%xmm0, %%xmm10      \n\t" // ar0*xr0,al0*xr0

  285. 	"vfmaddpd   %%xmm11,   %%xmm5 , %%xmm1, %%xmm11      \n\t" // ar0*xl0,al0*xl0 

  286. 

  287.         "addq		$8 , %0	  	 	        \n\t"

  288. 	"subq	        $4 , %1			        \n\t"		

  289. 	"jnz		1b		        \n\t"

  290. 

  291. 	"vmovddup               (%6)  , %%xmm0                \n\t"  // value from alpha

  292. 	"vmovddup	       8(%6)  , %%xmm1                \n\t"  // value from alpha

  293. 

  294. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  295.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  296.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  297.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  298.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  299. #else

  300.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  301.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  302.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  303.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  304.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  305.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  306. #endif

  307. 

  308. 	"vmulpd		%%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  309. 	"vmulpd		%%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  310. 	"vmulpd		%%xmm10, %%xmm1 , %%xmm11             \n\t"  // t_r * alpha_i , t_i * alpha_i

  311. 	"vmulpd		%%xmm10, %%xmm0 , %%xmm10             \n\t"  // t_r * alpha_r , t_i * alpha_r

  312. 

  313. #if !defined(XCONJ) 

  314.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  315.         "vpermilpd      $0x1 , %%xmm11, %%xmm11               \n\t"

  316.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  317.         "vaddsubpd      %%xmm11, %%xmm10, %%xmm10             \n\t"

  318. #else

  319.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  320.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  321.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  322.         "vaddsubpd      %%xmm10, %%xmm11, %%xmm10             \n\t"

  323.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  324.         "vpermilpd      $0x1 , %%xmm10, %%xmm10               \n\t"

  325. #endif

  326. 

  327. 	"vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  328. 	"vaddpd         16(%3) , %%xmm10, %%xmm10             \n\t"

  329. 

  330. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  331. 	"vmovups	%%xmm10, 16(%3)			\n\t"

  332. 

  333. 	"vzeroupper			 \n\t"

  334. 

  335. 	:

  336.           "+r" (i),	// 0	

  337. 	  "+r" (n)  	// 1

  338.         : 

  339.           "r" (x),      // 2

  340.           "r" (y),      // 3

  341.           "r" (ap[0]),  // 4

  342.           "r" (ap[1]),  // 5

  343.           "r" (alpha)   // 6

  344. 	: "cc", 

  345. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  346. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  347. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  348. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  349. 	  "memory"

  350. 	);

  351. 

  352. } 

  353. 

  354. 

  355. 

  356. #define HAVE_KERNEL_4x1 1

  357. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha) __attribute__ ((noinline));

  358. 

  359. static void zgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y, FLOAT *alpha)

  360. {

  361. 

  362. 	BLASLONG register i = 0;

  363. 

  364. 	__asm__  __volatile__

  365. 	(

  366. 	"vzeroupper			 \n\t"

  367. 

  368. 	"vxorpd		%%xmm8 , %%xmm8 , %%xmm8 	\n\t" // temp

  369. 	"vxorpd		%%xmm9 , %%xmm9 , %%xmm9 	\n\t" // temp

  370. 

  371. 	".align 16				        \n\t"

  372. 	"1:				        \n\t"

  373. 

  374. 	"vmovddup	   (%2,%0,8), %%xmm0            \n\t"  // real value from x0

  375. 	"vmovddup	  8(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  376. 

  377. 	"prefetcht0  192(%4,%0,8)                       \n\t"

  378. 	"vmovups	(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  379. 	"vmovups      16(%4,%0,8), %%xmm5	        \n\t" // 1 complex values from a0

  380. 

  381. 	"vmovddup	 16(%2,%0,8), %%xmm2            \n\t"  // real value from x0

  382. 	"vmovddup	 24(%2,%0,8), %%xmm3            \n\t"  // imag value from x0

  383. 

  384. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  385. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  386. 

  387. 	"vmovddup	 32(%2,%0,8), %%xmm0            \n\t"  // real value from x0

  388. 	"vmovddup	 40(%2,%0,8), %%xmm1            \n\t"  // imag value from x0

  389. 

  390. 	"vfmaddpd   %%xmm8 ,   %%xmm5 , %%xmm2, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  391. 	"vfmaddpd   %%xmm9 ,   %%xmm5 , %%xmm3, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  392. 

  393. 	"vmovups      32(%4,%0,8), %%xmm4	        \n\t" // 1 complex values from a0

  394. 	"vmovups      48(%4,%0,8), %%xmm5	        \n\t" // 1 complex values from a0

  395. 

  396. 	"vmovddup	 48(%2,%0,8), %%xmm2            \n\t"  // real value from x0

  397. 	"vmovddup	 56(%2,%0,8), %%xmm3            \n\t"  // imag value from x0

  398. 

  399.         "addq		$8 , %0	  	 	        \n\t"

  400. 	"vfmaddpd   %%xmm8 ,   %%xmm4 , %%xmm0, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  401. 	"vfmaddpd   %%xmm9 ,   %%xmm4 , %%xmm1, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  402. 

  403. 	"subq	        $4 , %1			        \n\t"		

  404. 	"vfmaddpd   %%xmm8 ,   %%xmm5 , %%xmm2, %%xmm8       \n\t" // ar0*xr0,al0*xr0 

  405. 	"vfmaddpd   %%xmm9 ,   %%xmm5 , %%xmm3, %%xmm9       \n\t" // ar0*xl0,al0*xl0 

  406. 

  407. 	"jnz		1b		        \n\t"

  408. 

  409. 	"vmovddup               (%5)  , %%xmm0                \n\t"  // value from alpha

  410. 	"vmovddup	       8(%5)  , %%xmm1                \n\t"  // value from alpha

  411. 

  412. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  413.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  414.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  415. #else

  416.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  417.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  418.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  419. #endif

  420. 

  421. 	"vmulpd		%%xmm8 , %%xmm1 , %%xmm9              \n\t"  // t_r * alpha_i , t_i * alpha_i

  422. 	"vmulpd		%%xmm8 , %%xmm0 , %%xmm8              \n\t"  // t_r * alpha_r , t_i * alpha_r

  423. 

  424. #if !defined(XCONJ) 

  425.         "vpermilpd      $0x1 , %%xmm9 , %%xmm9                \n\t"

  426.         "vaddsubpd      %%xmm9 , %%xmm8, %%xmm8               \n\t" 

  427. #else

  428.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  429.         "vaddsubpd      %%xmm8 , %%xmm9 , %%xmm8              \n\t"

  430.         "vpermilpd      $0x1 , %%xmm8 , %%xmm8                \n\t"

  431. #endif

  432. 

  433. 	"vaddpd           (%3) , %%xmm8 , %%xmm8              \n\t"

  434. 

  435. 	"vmovups	%%xmm8 ,   (%3)			\n\t"

  436. 

  437. 	"vzeroupper			 \n\t"

  438. 

  439. 	:

  440.           "+r" (i),	// 0	

  441. 	  "+r" (n)  	// 1

  442.         : 

  443.           "r" (x),      // 2

  444.           "r" (y),      // 3

  445.           "r" (ap),     // 4

  446.           "r" (alpha)   // 5

  447. 	: "cc", 

  448. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  449. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  450. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  451. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  452. 	  "memory"

  453. 	);

  454. 

  455. } 

  456.