OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 5e1a429eab
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '5e1a429eab'
${ noResults }
OpenBLAS/kernel/x86_64/cgemv_n_microk_bulldozer-4.c

542 lines
22 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2014, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. #define HAVE_KERNEL_4x4 1

  29. static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  30. 

  31. static void cgemv_kernel_4x4( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  32. {

  33. 

  34. 	BLASLONG register i = 0;

  35. 	BLASLONG register n1 = n & -8 ;

  36. 	BLASLONG register n2 = n &  4 ;

  37. 

  38. 	__asm__  __volatile__

  39. 	(

  40. 

  41. 	"vbroadcastss	  (%2), %%ymm0                  \n\t"  // real part x0

  42. 	"vbroadcastss	 4(%2), %%ymm1                  \n\t"  // imag part x0

  43. 	"vbroadcastss	 8(%2), %%ymm2                  \n\t"  // real part x1

  44. 	"vbroadcastss	12(%2), %%ymm3                  \n\t"  // imag part x1

  45. 	"vbroadcastss	16(%2), %%ymm4                  \n\t"  // real part x2

  46. 	"vbroadcastss	20(%2), %%ymm5                  \n\t"  // imag part x2

  47. 	"vbroadcastss	24(%2), %%ymm6                  \n\t"  // real part x3

  48. 	"vbroadcastss	28(%2), %%ymm7                  \n\t"  // imag part x3

  49. 

  50. 	"cmpq		$0 , %1				\n\t"

  51. 	"je		2f			        \n\t"

  52. 

  53. 	".align 16				        \n\t"

  54. 	"1:				        \n\t"

  55. 	"prefetcht0      384(%4,%0,4)			\n\t"

  56. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  57. 	"vmovups      32(%4,%0,4), %%ymm9	        \n\t" // 4 complex values form a0

  58. 

  59. 	"prefetcht0      384(%5,%0,4)			\n\t"

  60. 	"vmovups	(%5,%0,4), %%ymm10              \n\t" // 4 complex values form a1

  61. 	"vmovups      32(%5,%0,4), %%ymm11              \n\t" // 4 complex values form a1

  62. 

  63. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  64. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  65. 	"vmulps      %%ymm9 , %%ymm0, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  66. 	"vmulps      %%ymm9 , %%ymm1, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  67. 

  68. 	"prefetcht0      384(%6,%0,4)			\n\t"

  69. 	"vmovups	(%6,%0,4), %%ymm8	        \n\t" // 4 complex values form a2

  70. 	"vmovups      32(%6,%0,4), %%ymm9	        \n\t" // 4 complex values form a2

  71. 

  72. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm2, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  73. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm3, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  74. 	"vfmaddps      %%ymm14, %%ymm11, %%ymm2, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  75. 	"vfmaddps      %%ymm15, %%ymm11, %%ymm3, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  76. 

  77. 	"prefetcht0      384(%7,%0,4)			\n\t"

  78. 	"vmovups	(%7,%0,4), %%ymm10              \n\t" // 4 complex values form a3

  79. 	"vmovups      32(%7,%0,4), %%ymm11              \n\t" // 4 complex values form a3

  80. 

  81. 	"vfmaddps      %%ymm12, %%ymm8 , %%ymm4, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  82. 	"vfmaddps      %%ymm13, %%ymm8 , %%ymm5, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  83. 	"vfmaddps      %%ymm14, %%ymm9 , %%ymm4, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  84. 	"vfmaddps      %%ymm15, %%ymm9 , %%ymm5, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  85. 

  86. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm6, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  87. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm7, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  88. 	"vfmaddps      %%ymm14, %%ymm11, %%ymm6, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  89. 	"vfmaddps      %%ymm15, %%ymm11, %%ymm7, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  90. 

  91. 	"prefetcht0      384(%3,%0,4)			\n\t"

  92. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  93. 	"vmovups	32(%3,%0,4),  %%ymm11           \n\t"

  94. 

  95. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  96.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  97.         "vpermilps      $0xb1 , %%ymm15, %%ymm15               \n\t"

  98.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  99.         "vaddsubps      %%ymm15, %%ymm14, %%ymm9              \n\t"

  100. #else

  101.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  102.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  103.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  104.         "vaddsubps      %%ymm14, %%ymm15, %%ymm9              \n\t"

  105.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  106.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  107. #endif

  108. 

  109.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  110.         "vaddps         %%ymm9, %%ymm11, %%ymm13              \n\t"

  111. 

  112. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  113. 	"vmovups  %%ymm13, 32(%3,%0,4)		        \n\t"	

  114. 

  115.         "addq		$16, %0	  	 	        \n\t"

  116. 	"subq	        $8 , %1			        \n\t"		

  117. 	"jnz		1b		        \n\t"

  118. 

  119. 	"2:				        \n\t"

  120. 

  121. 	"cmpq		$4, %8				\n\t"

  122. 	"jne		3f				\n\t"

  123. 

  124. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  125. 	"vmovups	(%5,%0,4), %%ymm10              \n\t" // 4 complex values form a1

  126. 

  127. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  128. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  129. 

  130. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm2, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  131. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm3, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  132. 

  133. 	"vmovups	(%6,%0,4), %%ymm8	        \n\t" // 4 complex values form a2

  134. 	"vmovups	(%7,%0,4), %%ymm10              \n\t" // 4 complex values form a3

  135. 

  136. 	"vfmaddps      %%ymm12, %%ymm8 , %%ymm4, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  137. 	"vfmaddps      %%ymm13, %%ymm8 , %%ymm5, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  138. 

  139. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm6, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  140. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm7, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  141. 

  142. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  143. 

  144. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  145.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  146.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  147. #else

  148.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  149.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  150.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  151. #endif

  152. 

  153.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  154. 

  155. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  156. 

  157. 	"3:				        \n\t"

  158. 	"vzeroupper			 \n\t"

  159. 

  160. 	:

  161.           "+r" (i),	// 0	

  162. 	  "+r" (n1)  	// 1

  163. 	:

  164.           "r" (x),      // 2

  165.           "r" (y),      // 3

  166.           "r" (ap[0]),  // 4

  167.           "r" (ap[1]),  // 5

  168.           "r" (ap[2]),  // 6

  169.           "r" (ap[3]),  // 7

  170.           "r" (n2)      // 8

  171. 	: "cc", 

  172. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  173. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  174. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  175. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  176. 	  "memory"

  177. 	);

  178. 

  179. } 

  180. 

  181. 

  182. #define HAVE_KERNEL_4x2 1

  183. static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  184. 

  185. static void cgemv_kernel_4x2( BLASLONG n, FLOAT **ap, FLOAT *x, FLOAT *y)

  186. {

  187. 

  188. 	BLASLONG register i = 0;

  189. 	BLASLONG register n1 = n & -8 ;

  190. 	BLASLONG register n2 = n &  4 ;

  191. 

  192. 	__asm__  __volatile__

  193. 	(

  194. 	"vzeroupper			 \n\t"

  195. 

  196. 	"vbroadcastss	  (%2), %%ymm0                  \n\t"  // real part x0

  197. 	"vbroadcastss	 4(%2), %%ymm1                  \n\t"  // imag part x0

  198. 	"vbroadcastss	 8(%2), %%ymm2                  \n\t"  // real part x1

  199. 	"vbroadcastss	12(%2), %%ymm3                  \n\t"  // imag part x1

  200. 

  201. 	"cmpq		$0 , %1				\n\t"

  202. 	"je		2f			        \n\t"

  203. 

  204. 	//	".align 16				        \n\t"

  205. 	"1:				        \n\t"

  206. 	"prefetcht0      384(%4,%0,4)			\n\t"

  207. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  208. 	"vmovups      32(%4,%0,4), %%ymm9	        \n\t" // 4 complex values form a0

  209. 

  210. 	"prefetcht0      384(%5,%0,4)			\n\t"

  211. 	"vmovups	(%5,%0,4), %%ymm10              \n\t" // 4 complex values form a1

  212. 	"vmovups      32(%5,%0,4), %%ymm11              \n\t" // 4 complex values form a1

  213. 

  214. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  215. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  216. 	"vmulps      %%ymm9 , %%ymm0, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  217. 	"vmulps      %%ymm9 , %%ymm1, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  218. 

  219. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm2, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  220. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm3, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  221. 	"vfmaddps      %%ymm14, %%ymm11, %%ymm2, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  222. 	"vfmaddps      %%ymm15, %%ymm11, %%ymm3, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  223. 

  224. 	"prefetcht0      384(%3,%0,4)			\n\t"

  225. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  226. 	"vmovups	32(%3,%0,4),  %%ymm11           \n\t"

  227. 

  228. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  229.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  230.         "vpermilps      $0xb1 , %%ymm15, %%ymm15               \n\t"

  231.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  232.         "vaddsubps      %%ymm15, %%ymm14, %%ymm9              \n\t"

  233. #else

  234.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  235.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  236.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  237.         "vaddsubps      %%ymm14, %%ymm15, %%ymm9              \n\t"

  238.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  239.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  240. #endif

  241. 

  242.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  243.         "vaddps         %%ymm9, %%ymm11, %%ymm13              \n\t"

  244. 

  245. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  246. 	"vmovups  %%ymm13, 32(%3,%0,4)		        \n\t"	

  247. 

  248.         "addq		$16, %0	  	 	        \n\t"

  249. 	"subq	        $8 , %1			        \n\t"		

  250. 	"jnz		1b		        \n\t"

  251. 

  252. 	"2:				        \n\t"

  253. 

  254. 	"cmpq		$4, %6				\n\t"

  255. 	"jne		3f				\n\t"

  256. 

  257. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  258. 	"vmovups	(%5,%0,4), %%ymm10              \n\t" // 4 complex values form a1

  259. 

  260. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  261. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  262. 

  263. 	"vfmaddps      %%ymm12, %%ymm10, %%ymm2, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  264. 	"vfmaddps      %%ymm13, %%ymm10, %%ymm3, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  265. 

  266. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  267. 

  268. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  269.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  270.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  271. #else

  272.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  273.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  274.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  275. #endif

  276. 

  277.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  278. 

  279. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  280. 

  281. 	"3:				        \n\t"

  282. 	"vzeroupper			 \n\t"

  283. 

  284. 	:

  285.           "+r" (i),	// 0	

  286. 	  "+r" (n1)  	// 1

  287. 	:

  288.           "r" (x),      // 2

  289.           "r" (y),      // 3

  290.           "r" (ap[0]),  // 4

  291.           "r" (ap[1]),  // 5

  292.           "r" (n2)      // 6

  293. 	: "cc", 

  294. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  295. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  296. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  297. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  298. 	  "memory"

  299. 	);

  300. 

  301. } 

  302. 

  303. 

  304. #define HAVE_KERNEL_4x1 1

  305. static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y) __attribute__ ((noinline));

  306. 

  307. static void cgemv_kernel_4x1( BLASLONG n, FLOAT *ap, FLOAT *x, FLOAT *y)

  308. {

  309. 

  310. 	BLASLONG register i = 0;

  311. 	BLASLONG register n1 = n & -8 ;

  312. 	BLASLONG register n2 = n &  4 ;

  313. 

  314. 	__asm__  __volatile__

  315. 	(

  316. 	"vzeroupper			 \n\t"

  317. 

  318. 	"vbroadcastss	  (%2), %%ymm0                  \n\t"  // real part x0

  319. 	"vbroadcastss	 4(%2), %%ymm1                  \n\t"  // imag part x0

  320. 

  321. 	"cmpq		$0 , %1				\n\t"

  322. 	"je		2f			        \n\t"

  323. 

  324. 	//	".align 16				        \n\t"

  325. 	"1:				        \n\t"

  326. 	"prefetcht0      384(%4,%0,4)			\n\t"

  327. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  328. 	"vmovups      32(%4,%0,4), %%ymm9	        \n\t" // 4 complex values form a0

  329. 

  330. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  331. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  332. 	"vmulps      %%ymm9 , %%ymm0, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  333. 	"vmulps      %%ymm9 , %%ymm1, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  334. 

  335. 	"prefetcht0      384(%3,%0,4)			\n\t"

  336. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  337. 	"vmovups	32(%3,%0,4),  %%ymm11           \n\t"

  338. 

  339. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  340.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  341.         "vpermilps      $0xb1 , %%ymm15, %%ymm15               \n\t"

  342.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  343.         "vaddsubps      %%ymm15, %%ymm14, %%ymm9              \n\t"

  344. #else

  345.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  346.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  347.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  348.         "vaddsubps      %%ymm14, %%ymm15, %%ymm9              \n\t"

  349.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  350.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  351. #endif

  352. 

  353.         "addq		$16, %0	  	 	        \n\t"

  354.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  355.         "vaddps         %%ymm9, %%ymm11, %%ymm13              \n\t"

  356. 

  357. 	"subq	        $8 , %1			        \n\t"		

  358. 	"vmovups  %%ymm12,-64(%3,%0,4)		        \n\t" // 4 complex values to y	

  359. 	"vmovups  %%ymm13,-32(%3,%0,4)		        \n\t"	

  360. 

  361. 	"jnz		1b		        \n\t"

  362. 

  363. 	"2:				        \n\t"

  364. 

  365. 	"cmpq		$4, %5				\n\t"

  366. 	"jne		3f				\n\t"

  367. 

  368. 	"vmovups	(%4,%0,4), %%ymm8	        \n\t" // 4 complex values form a0

  369. 

  370. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  371. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  372. 

  373. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  374. 

  375. #if ( !defined(CONJ) && !defined(XCONJ) ) || ( defined(CONJ) && defined(XCONJ) )

  376.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  377.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  378. #else

  379.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  380.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  381.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  382. #endif

  383. 

  384.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  385. 

  386. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  387. 

  388. 	"3:				        \n\t"

  389. 	"vzeroupper			 \n\t"

  390. 

  391. 	:

  392.           "+r" (i),	// 0	

  393. 	  "+r" (n1)  	// 1

  394. 	:

  395.           "r" (x),      // 2

  396.           "r" (y),      // 3

  397.           "r" (ap),     // 4

  398.           "r" (n2)      // 5

  399. 	: "cc", 

  400. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  401. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  402. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  403. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  404. 	  "memory"

  405. 	);

  406. 

  407. } 

  408. 

  409. 

  410. #define HAVE_KERNEL_ADDY 1

  411. 

  412. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)  __attribute__ ((noinline));

  413. 

  414. static void add_y(BLASLONG n, FLOAT *src, FLOAT *dest, BLASLONG inc_dest,FLOAT alpha_r, FLOAT alpha_i)

  415. {

  416. 	BLASLONG i;

  417. 

  418. 	if ( inc_dest != 2 )

  419. 	{

  420. 

  421. 		FLOAT temp_r;

  422. 		FLOAT temp_i;

  423. 		for ( i=0; i<n; i++ )

  424. 		{

  425. #if !defined(XCONJ) 

  426. 			temp_r = alpha_r * src[0] - alpha_i * src[1];

  427. 			temp_i = alpha_r * src[1] + alpha_i * src[0];

  428. #else

  429. 			temp_r =  alpha_r * src[0] + alpha_i * src[1];

  430. 			temp_i = -alpha_r * src[1] + alpha_i * src[0];

  431. #endif

  432. 

  433. 			*dest += temp_r;

  434. 			*(dest+1) += temp_i;

  435. 

  436. 			src+=2;

  437. 			dest += inc_dest;

  438. 		}

  439. 		return;

  440. 	}

  441. 

  442. 	i=0;

  443. 	BLASLONG register n1 = n & -8 ;

  444. 	BLASLONG register n2 = n &  4 ;

  445. 

  446. 	__asm__  __volatile__

  447. 	(

  448. 	"vzeroupper			 \n\t"

  449. 

  450. 	"vbroadcastss	  (%4), %%ymm0                  \n\t"  // alpha_r

  451. 	"vbroadcastss	  (%5), %%ymm1                  \n\t"  // alpha_i

  452. 

  453. 	"cmpq		$0 , %1				\n\t"

  454. 	"je		2f			        \n\t"

  455. 

  456. 	//	".align 16				        \n\t"

  457. 	"1:				        \n\t"

  458. 	"vmovups	(%2,%0,4), %%ymm8	        \n\t" // 4 complex values from src

  459. 	"vmovups      32(%2,%0,4), %%ymm9	        \n\t" 

  460. 

  461. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  462. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  463. 	"vmulps      %%ymm9 , %%ymm0, %%ymm14      \n\t" // a_r[2] * x_r , a_i[2] * x_r, a_r[3] * x_r, a_i[3] * x_r

  464. 	"vmulps      %%ymm9 , %%ymm1, %%ymm15      \n\t" // a_r[2] * x_i , a_i[2] * x_i, a_r[3] * x_i, a_i[3] * x_i

  465. 

  466. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t" // 4 complex values from dest

  467. 	"vmovups	32(%3,%0,4),  %%ymm11           \n\t"

  468. 

  469. #if  !defined(XCONJ)  

  470.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  471.         "vpermilps      $0xb1 , %%ymm15, %%ymm15               \n\t"

  472.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  473.         "vaddsubps      %%ymm15, %%ymm14, %%ymm9              \n\t"

  474. #else

  475.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  476.         "vpermilps      $0xb1 , %%ymm14, %%ymm14               \n\t"

  477.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  478.         "vaddsubps      %%ymm14, %%ymm15, %%ymm9              \n\t"

  479.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  480.         "vpermilps      $0xb1 , %%ymm9 , %%ymm9                \n\t"

  481. #endif

  482. 

  483.         "addq		$16, %0	  	 	        \n\t"

  484.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  485.         "vaddps         %%ymm9, %%ymm11, %%ymm13              \n\t"

  486. 

  487. 	"subq	        $8 , %1			        \n\t"		

  488. 	"vmovups  %%ymm12,-64(%3,%0,4)		        \n\t" // 4 complex values to y	

  489. 	"vmovups  %%ymm13,-32(%3,%0,4)		        \n\t"	

  490. 

  491. 	"jnz		1b		        \n\t"

  492. 

  493. 	"2:				        \n\t"

  494. 

  495. 	"cmpq		$4, %6				\n\t"

  496. 	"jne		3f				\n\t"

  497. 

  498. 	"vmovups	(%2,%0,4), %%ymm8	        \n\t" // 4 complex values src

  499. 

  500. 	"vmulps      %%ymm8 , %%ymm0, %%ymm12      \n\t" // a_r[0] * x_r , a_i[0] * x_r, a_r[1] * x_r, a_i[1] * x_r

  501. 	"vmulps      %%ymm8 , %%ymm1, %%ymm13      \n\t" // a_r[0] * x_i , a_i[0] * x_i, a_r[1] * x_i, a_i[1] * x_i

  502. 

  503. 	"vmovups	  (%3,%0,4),  %%ymm10           \n\t"

  504. 

  505. #if !defined(XCONJ)

  506.         "vpermilps      $0xb1 , %%ymm13, %%ymm13               \n\t"

  507.         "vaddsubps      %%ymm13, %%ymm12, %%ymm8              \n\t"

  508. #else

  509.         "vpermilps      $0xb1 , %%ymm12, %%ymm12               \n\t"

  510.         "vaddsubps      %%ymm12, %%ymm13, %%ymm8              \n\t"

  511.         "vpermilps      $0xb1 , %%ymm8 , %%ymm8                \n\t"

  512. #endif

  513. 

  514.         "vaddps         %%ymm8, %%ymm10, %%ymm12              \n\t"

  515. 

  516. 	"vmovups  %%ymm12,   (%3,%0,4)		        \n\t" // 4 complex values to y	

  517. 

  518. 	"3:				        \n\t"

  519. 	"vzeroupper			 \n\t"

  520. 

  521. 	:

  522.           "+r" (i),	      // 0	

  523. 	  "+r" (n1)  	      // 1

  524. 	:

  525.           "r" (src),          // 2

  526.           "r" (dest),         // 3

  527.           "r" (&alpha_r),     // 4

  528.           "r" (&alpha_i),     // 5

  529.           "r" (n2)            // 6

  530. 	: "cc", 

  531. 	  "%xmm0", "%xmm1", "%xmm2", "%xmm3", 

  532. 	  "%xmm4", "%xmm5", "%xmm6", "%xmm7", 

  533. 	  "%xmm8", "%xmm9", "%xmm10", "%xmm11", 

  534. 	  "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  535. 	  "memory"

  536. 	);

  537. 

  538. 	return;

  539. 

  540. }