OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 64629cb5c7
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '64629cb5c7'
${ noResults }
OpenBLAS/kernel/x86_64/strsm_kernel_LN_bulldozer.c

757 lines
35 kB
Raw Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #include "common.h"

  40. 

  41. static FLOAT dm1 = -1.;

  42. 

  43. #ifdef CONJ

  44. #define GEMM_KERNEL   GEMM_KERNEL_L

  45. #else

  46. #define GEMM_KERNEL   GEMM_KERNEL_N

  47. #endif

  48. 

  49. #if GEMM_DEFAULT_UNROLL_M == 1

  50. #define GEMM_UNROLL_M_SHIFT 0

  51. #endif

  52. 

  53. #if GEMM_DEFAULT_UNROLL_M == 2

  54. #define GEMM_UNROLL_M_SHIFT 1

  55. #endif

  56. 

  57. #if GEMM_DEFAULT_UNROLL_M == 4

  58. #define GEMM_UNROLL_M_SHIFT 2

  59. #endif

  60. 

  61. #if GEMM_DEFAULT_UNROLL_M == 6

  62. #define GEMM_UNROLL_M_SHIFT 2

  63. #endif

  64. 

  65. #if GEMM_DEFAULT_UNROLL_M == 8

  66. #define GEMM_UNROLL_M_SHIFT 3

  67. #endif

  68. 

  69. #if GEMM_DEFAULT_UNROLL_M == 16

  70. #define GEMM_UNROLL_M_SHIFT 4

  71. #endif

  72. 

  73. #if GEMM_DEFAULT_UNROLL_N == 1

  74. #define GEMM_UNROLL_N_SHIFT 0

  75. #endif

  76. 

  77. #if GEMM_DEFAULT_UNROLL_N == 2

  78. #define GEMM_UNROLL_N_SHIFT 1

  79. #endif

  80. 

  81. #if GEMM_DEFAULT_UNROLL_N == 4

  82. #define GEMM_UNROLL_N_SHIFT 2

  83. #endif

  84. 

  85. #if GEMM_DEFAULT_UNROLL_N == 8

  86. #define GEMM_UNROLL_N_SHIFT 3

  87. #endif

  88. 

  89. #if GEMM_DEFAULT_UNROLL_N == 16

  90. #define GEMM_UNROLL_N_SHIFT 4

  91. #endif

  92. 

  93. #ifndef COMPLEX

  94. 

  95. 

  96. 

  97. static void strsm_LN_solve_opt(BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, FLOAT *as, FLOAT *bs)  __attribute__ ((noinline));

  98. 

  99. static void strsm_LN_solve_opt(BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, FLOAT *as, FLOAT *bs)

  100. {

  101. 

  102. 	FLOAT *c1 = c + ldc ;

  103. 	BLASLONG n1 = n * 8;

  104. 	BLASLONG i=0;

  105. 

  106.   	as += (16 - 1) * 16;

  107.   	bs += (16 - 1) * 2;

  108. 

  109.         __asm__  __volatile__

  110.         (

  111. 	"	vzeroupper							\n\t"

  112. 	"	prefetcht0	(%4)						\n\t"

  113. 	"	prefetcht0	(%5)						\n\t"

  114. 	"	vxorps	%%xmm8 , %%xmm8 , %%xmm8				\n\t"

  115. 	"	vxorps	%%xmm9 , %%xmm9 , %%xmm9				\n\t"

  116. 	"	vxorps	%%xmm10, %%xmm10, %%xmm10				\n\t"

  117. 	"	vxorps	%%xmm11, %%xmm11, %%xmm11				\n\t"

  118. 	"	vxorps	%%xmm12, %%xmm12, %%xmm12				\n\t"

  119. 	"	vxorps	%%xmm13, %%xmm13, %%xmm13				\n\t"

  120. 	"	vxorps	%%xmm14, %%xmm14, %%xmm14				\n\t"

  121. 	"	vxorps	%%xmm15, %%xmm15, %%xmm15				\n\t"

  122. 

  123. 	"	cmpq	       $0, %0						\n\t"

  124. 	"	je	       2f						\n\t"

  125. 

  126. 	"	.align 16							\n\t"

  127. 	"1:									\n\t"

  128. 

  129. 	"	vbroadcastss	(%7,%1,1), %%xmm0				\n\t"	// read b

  130. 	"	vmovups         (%6,%1,8), %%xmm4				\n\t"

  131. 	"	vbroadcastss   4(%7,%1,1), %%xmm1				\n\t"	

  132. 	"	vmovups       16(%6,%1,8), %%xmm5				\n\t"

  133. 	"	vmovups       32(%6,%1,8), %%xmm6				\n\t"

  134. 	"	vmovups       48(%6,%1,8), %%xmm7				\n\t"

  135. 

  136. 	"	vfmaddps	%%xmm8 , %%xmm0 , %%xmm4 , %%xmm8		\n\t"

  137. 	"	vfmaddps	%%xmm12, %%xmm1 , %%xmm4 , %%xmm12		\n\t"

  138. 	"	vfmaddps	%%xmm9 , %%xmm0 , %%xmm5 , %%xmm9		\n\t"

  139. 	"	vfmaddps	%%xmm13, %%xmm1 , %%xmm5 , %%xmm13		\n\t"

  140. 	"	vfmaddps	%%xmm10, %%xmm0 , %%xmm6 , %%xmm10		\n\t"

  141. 	"	vfmaddps	%%xmm14, %%xmm1 , %%xmm6 , %%xmm14		\n\t"

  142. 	"	addq		$8, %1						\n\t"

  143. 	"	vfmaddps	%%xmm11, %%xmm0 , %%xmm7 , %%xmm11		\n\t"

  144. 	"	vfmaddps	%%xmm15, %%xmm1 , %%xmm7 , %%xmm15		\n\t"

  145. 	"	cmpq		%1, %0						\n\t"

  146. 

  147. 	"	jnz		1b						\n\t"

  148. 

  149. 	"2:									\n\t"

  150. 

  151. 

  152. 	"	vmovups		  (%4) , %%xmm0					\n\t"

  153. 	"	vmovups		16(%4) , %%xmm1					\n\t"

  154. 	"	vmovups		32(%4) , %%xmm2					\n\t"

  155. 	"	vmovups		48(%4) , %%xmm3					\n\t"

  156. 

  157. 	"	vmovups		  (%5) , %%xmm4					\n\t"

  158. 	"	vmovups		16(%5) , %%xmm5					\n\t"

  159. 	"	vmovups		32(%5) , %%xmm6					\n\t"

  160. 	"	vmovups		48(%5) , %%xmm7					\n\t"

  161. 

  162. 	"	vsubps		%%xmm8 , %%xmm0 , %%xmm8			\n\t"

  163. 	"	vsubps		%%xmm9 , %%xmm1 , %%xmm9			\n\t"

  164. 	"	vsubps		%%xmm10, %%xmm2 , %%xmm10			\n\t"

  165. 	"	vsubps		%%xmm11, %%xmm3 , %%xmm11			\n\t"

  166. 

  167. 	"	vsubps		%%xmm12, %%xmm4 , %%xmm12			\n\t"

  168. 	"	vsubps		%%xmm13, %%xmm5 , %%xmm13			\n\t"

  169. 	"	vsubps		%%xmm14, %%xmm6 , %%xmm14			\n\t"

  170. 	"	vsubps		%%xmm15, %%xmm7 , %%xmm15			\n\t"

  171. 

  172. 	"3:									\n\t"	

  173. 

  174. 	"	vbroadcastss    60(%2) , %%xmm0					\n\t" // i=15, read aa[i]		

  175. 	"	vshufps		$0xff  , %%xmm11 , %%xmm11 , %%xmm1		\n\t" // extract bb0

  176. 	"	vshufps		$0xff  , %%xmm15 , %%xmm15 , %%xmm2		\n\t" // extract bb1

  177. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  178. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  179.         "       vmovss          %%xmm1  , 60(%4)                                \n\t"   // c[i] = bb0 * aa

  180.         "       vmovss          %%xmm2  , 60(%5)                                \n\t"   // c[i] = bb1 * aa

  181.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  182.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  183. 

  184. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  185. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  186. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  187. 	"       vmovups        48(%2)  , %%xmm7                                 \n\t"   // read a[k]

  188. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  189.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  190.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  191.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  192.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  193.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  194.         "       vfnmaddps       %%xmm11 , %%xmm1 , %%xmm7 , %%xmm11             \n\t"

  195.         "       vfnmaddps       %%xmm15 , %%xmm2 , %%xmm7 , %%xmm15             \n\t"

  196. 

  197. 	"	subq		$64 , %2					\n\t"   // a -= m

  198. 	"	subq		$8  , %3					\n\t"   // b -= n

  199. 

  200. 	"	vbroadcastss    56(%2) , %%xmm0					\n\t" // i=14, read aa[i]		

  201. 	"	vshufps		$0xaa  , %%xmm11 , %%xmm11 , %%xmm1		\n\t" // extract bb0

  202. 	"	vshufps		$0xaa  , %%xmm15 , %%xmm15 , %%xmm2		\n\t" // extract bb1

  203. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  204. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  205.         "       vmovss          %%xmm1  , 56(%4)                                \n\t"   // c[i] = bb0 * aa

  206.         "       vmovss          %%xmm2  , 56(%5)                                \n\t"   // c[i] = bb1 * aa

  207.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  208.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  209. 

  210. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  211. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  212. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  213. 	"       vmovups        48(%2)  , %%xmm7                                 \n\t"   // read a[k]

  214. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  215.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  216.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  217.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  218.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  219.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  220.         "       vfnmaddps       %%xmm11 , %%xmm1 , %%xmm7 , %%xmm11             \n\t"

  221.         "       vfnmaddps       %%xmm15 , %%xmm2 , %%xmm7 , %%xmm15             \n\t"

  222. 

  223. 	"	subq		$64 , %2					\n\t"   // a -= m

  224. 	"	subq		$8  , %3					\n\t"   // b -= n

  225. 

  226. 	"	vbroadcastss    52(%2) , %%xmm0					\n\t" // i=13, read aa[i]		

  227. 	"	vshufps		$0x55  , %%xmm11 , %%xmm11 , %%xmm1		\n\t" // extract bb0

  228. 	"	vshufps		$0x55  , %%xmm15 , %%xmm15 , %%xmm2		\n\t" // extract bb1

  229. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  230. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  231.         "       vmovss          %%xmm1  , 52(%4)                                \n\t"   // c[i] = bb0 * aa

  232.         "       vmovss          %%xmm2  , 52(%5)                                \n\t"   // c[i] = bb1 * aa

  233.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  234.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  235. 

  236. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  237. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  238. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  239. 	"       vmovups        48(%2)  , %%xmm7                                 \n\t"   // read a[k]

  240. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  241.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  242.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  243.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  244.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  245.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  246.         "       vfnmaddps       %%xmm11 , %%xmm1 , %%xmm7 , %%xmm11             \n\t"

  247.         "       vfnmaddps       %%xmm15 , %%xmm2 , %%xmm7 , %%xmm15             \n\t"

  248. 

  249. 	"	subq		$64 , %2					\n\t"   // a -= m

  250. 	"	subq		$8  , %3					\n\t"   // b -= n

  251. 

  252. 	"	vbroadcastss    48(%2) , %%xmm0					\n\t" // i=12, read aa[i]		

  253. 	"	vshufps		$0x00  , %%xmm11 , %%xmm11 , %%xmm1		\n\t" // extract bb0

  254. 	"	vshufps		$0x00  , %%xmm15 , %%xmm15 , %%xmm2		\n\t" // extract bb1

  255. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  256. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  257.         "       vmovss          %%xmm1  , 48(%4)                                \n\t"   // c[i] = bb0 * aa

  258.         "       vmovss          %%xmm2  , 48(%5)                                \n\t"   // c[i] = bb1 * aa

  259.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  260.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  261. 

  262. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  263. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  264. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  265. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  266.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  267.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  268.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  269.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  270.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  271. 

  272. 	"	subq		$64 , %2					\n\t"   // a -= m

  273. 	"	subq		$8  , %3					\n\t"   // b -= n

  274. 

  275. 	"	vbroadcastss    44(%2) , %%xmm0					\n\t" // i=11, read aa[i]		

  276. 	"	vshufps		$0xff  , %%xmm10 , %%xmm10 , %%xmm1		\n\t" // extract bb0

  277. 	"	vshufps		$0xff  , %%xmm14 , %%xmm14 , %%xmm2		\n\t" // extract bb1

  278. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  279. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  280.         "       vmovss          %%xmm1  , 44(%4)                                \n\t"   // c[i] = bb0 * aa

  281.         "       vmovss          %%xmm2  , 44(%5)                                \n\t"   // c[i] = bb1 * aa

  282.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  283.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  284. 

  285. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  286. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  287. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  288. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  289.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  290.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  291.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  292.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  293.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  294. 

  295. 	"	subq		$64 , %2					\n\t"   // a -= m

  296. 	"	subq		$8  , %3					\n\t"   // b -= n

  297. 

  298. 	"	vbroadcastss    40(%2) , %%xmm0					\n\t" // i=10, read aa[i]		

  299. 	"	vshufps		$0xaa  , %%xmm10 , %%xmm10 , %%xmm1		\n\t" // extract bb0

  300. 	"	vshufps		$0xaa  , %%xmm14 , %%xmm14 , %%xmm2		\n\t" // extract bb1

  301. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  302. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  303.         "       vmovss          %%xmm1  , 40(%4)                                \n\t"   // c[i] = bb0 * aa

  304.         "       vmovss          %%xmm2  , 40(%5)                                \n\t"   // c[i] = bb1 * aa

  305.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  306.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  307. 

  308. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  309. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  310. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  311. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  312.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  313.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  314.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  315.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  316.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  317. 

  318. 	"	subq		$64 , %2					\n\t"   // a -= m

  319. 	"	subq		$8  , %3					\n\t"   // b -= n

  320. 

  321. 	"	vbroadcastss    36(%2) , %%xmm0					\n\t" // i=9 , read aa[i]		

  322. 	"	vshufps		$0x55  , %%xmm10 , %%xmm10 , %%xmm1		\n\t" // extract bb0

  323. 	"	vshufps		$0x55  , %%xmm14 , %%xmm14 , %%xmm2		\n\t" // extract bb1

  324. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  325. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  326.         "       vmovss          %%xmm1  , 36(%4)                                \n\t"   // c[i] = bb0 * aa

  327.         "       vmovss          %%xmm2  , 36(%5)                                \n\t"   // c[i] = bb1 * aa

  328.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  329.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  330. 

  331. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  332. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  333. 	"       vmovups        32(%2)  , %%xmm6                                 \n\t"   // read a[k]

  334. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  335.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  336.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  337.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  338.         "       vfnmaddps       %%xmm10 , %%xmm1 , %%xmm6 , %%xmm10             \n\t"

  339.         "       vfnmaddps       %%xmm14 , %%xmm2 , %%xmm6 , %%xmm14             \n\t"

  340. 

  341. 	"	subq		$64 , %2					\n\t"   // a -= m

  342. 	"	subq		$8  , %3					\n\t"   // b -= n

  343. 

  344. 	"	vbroadcastss    32(%2) , %%xmm0					\n\t" // i=8 , read aa[i]		

  345. 	"	vshufps		$0x00  , %%xmm10 , %%xmm10 , %%xmm1		\n\t" // extract bb0

  346. 	"	vshufps		$0x00  , %%xmm14 , %%xmm14 , %%xmm2		\n\t" // extract bb1

  347. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  348. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  349.         "       vmovss          %%xmm1  , 32(%4)                                \n\t"   // c[i] = bb0 * aa

  350.         "       vmovss          %%xmm2  , 32(%5)                                \n\t"   // c[i] = bb1 * aa

  351.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  352.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  353. 

  354. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  355. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  356. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  357.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  358.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  359.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  360. 

  361. 	"	subq		$64 , %2					\n\t"   // a -= m

  362. 	"	subq		$8  , %3					\n\t"   // b -= n

  363. 

  364. 	"	vbroadcastss    28(%2) , %%xmm0					\n\t" // i=7 , read aa[i]		

  365. 	"	vshufps		$0xff  , %%xmm9  , %%xmm9  , %%xmm1		\n\t" // extract bb0

  366. 	"	vshufps		$0xff  , %%xmm13 , %%xmm13 , %%xmm2		\n\t" // extract bb1

  367. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  368. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  369.         "       vmovss          %%xmm1  , 28(%4)                                \n\t"   // c[i] = bb0 * aa

  370.         "       vmovss          %%xmm2  , 28(%5)                                \n\t"   // c[i] = bb1 * aa

  371.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  372.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  373. 

  374. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  375. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  376. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  377.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  378.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  379.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  380. 

  381. 	"	subq		$64 , %2					\n\t"   // a -= m

  382. 	"	subq		$8  , %3					\n\t"   // b -= n

  383. 

  384. 	"	vbroadcastss    24(%2) , %%xmm0					\n\t" // i=6 , read aa[i]		

  385. 	"	vshufps		$0xaa  , %%xmm9  , %%xmm9  , %%xmm1		\n\t" // extract bb0

  386. 	"	vshufps		$0xaa  , %%xmm13 , %%xmm13 , %%xmm2		\n\t" // extract bb1

  387. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  388. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  389.         "       vmovss          %%xmm1  , 24(%4)                                \n\t"   // c[i] = bb0 * aa

  390.         "       vmovss          %%xmm2  , 24(%5)                                \n\t"   // c[i] = bb1 * aa

  391.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  392.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  393. 

  394. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  395. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  396. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  397.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  398.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  399.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  400. 

  401. 	"	subq		$64 , %2					\n\t"   // a -= m

  402. 	"	subq		$8  , %3					\n\t"   // b -= n

  403. 

  404. 	"	vbroadcastss    20(%2) , %%xmm0					\n\t" // i=5 , read aa[i]		

  405. 	"	vshufps		$0x55  , %%xmm9  , %%xmm9  , %%xmm1		\n\t" // extract bb0

  406. 	"	vshufps		$0x55  , %%xmm13 , %%xmm13 , %%xmm2		\n\t" // extract bb1

  407. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  408. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  409.         "       vmovss          %%xmm1  , 20(%4)                                \n\t"   // c[i] = bb0 * aa

  410.         "       vmovss          %%xmm2  , 20(%5)                                \n\t"   // c[i] = bb1 * aa

  411.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  412.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  413. 

  414. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  415. 	"       vmovups        16(%2)  , %%xmm5                                 \n\t"   // read a[k]

  416. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  417.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  418.         "       vfnmaddps       %%xmm9  , %%xmm1 , %%xmm5 , %%xmm9              \n\t"

  419.         "       vfnmaddps       %%xmm13 , %%xmm2 , %%xmm5 , %%xmm13             \n\t"

  420. 

  421. 	"	subq		$64 , %2					\n\t"   // a -= m

  422. 	"	subq		$8  , %3					\n\t"   // b -= n

  423. 

  424. 	"	vbroadcastss    16(%2) , %%xmm0					\n\t" // i=4 , read aa[i]		

  425. 	"	vshufps		$0x00  , %%xmm9  , %%xmm9  , %%xmm1		\n\t" // extract bb0

  426. 	"	vshufps		$0x00  , %%xmm13 , %%xmm13 , %%xmm2		\n\t" // extract bb1

  427. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  428. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  429.         "       vmovss          %%xmm1  , 16(%4)                                \n\t"   // c[i] = bb0 * aa

  430.         "       vmovss          %%xmm2  , 16(%5)                                \n\t"   // c[i] = bb1 * aa

  431.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  432.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  433. 

  434. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  435. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  436.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  437. 

  438. 	"	subq		$64 , %2					\n\t"   // a -= m

  439. 	"	subq		$8  , %3					\n\t"   // b -= n

  440. 

  441. 	"	vbroadcastss    12(%2) , %%xmm0					\n\t" // i=3 , read aa[i]		

  442. 	"	vshufps		$0xff  , %%xmm8  , %%xmm8  , %%xmm1		\n\t" // extract bb0

  443. 	"	vshufps		$0xff  , %%xmm12 , %%xmm12 , %%xmm2		\n\t" // extract bb1

  444. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  445. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  446.         "       vmovss          %%xmm1  , 12(%4)                                \n\t"   // c[i] = bb0 * aa

  447.         "       vmovss          %%xmm2  , 12(%5)                                \n\t"   // c[i] = bb1 * aa

  448.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  449.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  450. 

  451. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  452. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  453.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  454. 

  455. 	"	subq		$64 , %2					\n\t"   // a -= m

  456. 	"	subq		$8  , %3					\n\t"   // b -= n

  457. 

  458. 	"	vbroadcastss     8(%2) , %%xmm0					\n\t" // i=2 , read aa[i]		

  459. 	"	vshufps		$0xaa  , %%xmm8  , %%xmm8  , %%xmm1		\n\t" // extract bb0

  460. 	"	vshufps		$0xaa  , %%xmm12 , %%xmm12 , %%xmm2		\n\t" // extract bb1

  461. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  462. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  463.         "       vmovss          %%xmm1  ,  8(%4)                                \n\t"   // c[i] = bb0 * aa

  464.         "       vmovss          %%xmm2  ,  8(%5)                                \n\t"   // c[i] = bb1 * aa

  465.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  466.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  467. 

  468. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  469. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  470.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  471. 

  472. 	"	subq		$64 , %2					\n\t"   // a -= m

  473. 	"	subq		$8  , %3					\n\t"   // b -= n

  474. 

  475. 	"	vbroadcastss     4(%2) , %%xmm0					\n\t" // i=1 , read aa[i]		

  476. 	"	vshufps		$0x55  , %%xmm8  , %%xmm8  , %%xmm1		\n\t" // extract bb0

  477. 	"	vshufps		$0x55  , %%xmm12 , %%xmm12 , %%xmm2		\n\t" // extract bb1

  478. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  479. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  480.         "       vmovss          %%xmm1  ,  4(%4)                                \n\t"   // c[i] = bb0 * aa

  481.         "       vmovss          %%xmm2  ,  4(%5)                                \n\t"   // c[i] = bb1 * aa

  482.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  483.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  484. 

  485. 	"       vmovups         0(%2)  , %%xmm4                                 \n\t"   // read a[k]

  486. 	"       vfnmaddps       %%xmm8  , %%xmm1 , %%xmm4 , %%xmm8              \n\t"

  487.         "       vfnmaddps       %%xmm12 , %%xmm2 , %%xmm4 , %%xmm12             \n\t"

  488. 

  489. 	"	subq		$64 , %2					\n\t"   // a -= m

  490. 	"	subq		$8  , %3					\n\t"   // b -= n

  491. 

  492. 	"	vbroadcastss     0(%2) , %%xmm0					\n\t" // i=0 , read aa[i]		

  493. 	"	vshufps		$0x00  , %%xmm8  , %%xmm8  , %%xmm1		\n\t" // extract bb0

  494. 	"	vshufps		$0x00  , %%xmm12 , %%xmm12 , %%xmm2		\n\t" // extract bb1

  495. 	"       vmulps          %%xmm0  , %%xmm1 , %%xmm1                       \n\t"   // bb0 * aa

  496. 	"       vmulps          %%xmm0  , %%xmm2 , %%xmm2                       \n\t"   // bb1 * aa

  497.         "       vmovss          %%xmm1  ,  0(%4)                                \n\t"   // c[i] = bb0 * aa

  498.         "       vmovss          %%xmm2  ,  0(%5)                                \n\t"   // c[i] = bb1 * aa

  499.         "       vmovss          %%xmm1  ,   (%3)                        	\n\t"   // b[0] = bb0 * aa

  500.         "       vmovss          %%xmm2  ,  4(%3)                        	\n\t"   // b[1] = bb1 * aa

  501. 

  502. 	"	vzeroupper							\n\t"

  503. 

  504.         :

  505.           "+r" (n1),     // 0    

  506.           "+a" (i),      // 1    

  507.           "+r" (as),     // 2

  508.           "+r" (bs)      // 3

  509.         :

  510.           "r" (c),      // 4

  511.           "r" (c1),     // 5

  512.           "r" (a),      // 6

  513.           "r" (b)       // 7

  514.         : "cc",

  515.           "%xmm0", "%xmm1", "%xmm2", "%xmm3",

  516.           "%xmm4", "%xmm5", "%xmm6", "%xmm7",

  517.           "%xmm8", "%xmm9", "%xmm10", "%xmm11",

  518.           "%xmm12", "%xmm13", "%xmm14", "%xmm15",

  519.           "memory"

  520.         );

  521. 

  522. }

  523. 

  524. 

  525. 

  526. 

  527. static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) {

  528. 

  529.   FLOAT aa,  bb;

  530. 

  531.   int i, j, k;

  532. 

  533.   a += (m - 1) * m;

  534.   b += (m - 1) * n;

  535. 

  536.   for (i = m - 1; i >= 0; i--) {

  537. 

  538.     aa = *(a + i);

  539. 

  540.     for (j = 0; j < n; j ++) {

  541.       bb = *(c + i + j * ldc);

  542.       bb *= aa;

  543.       *b             = bb;

  544.       *(c + i + j * ldc) = bb;

  545.       b ++;

  546. 

  547.       for (k = 0; k < i; k ++){

  548. 	*(c + k + j * ldc) -= bb * *(a + k);

  549.       }

  550. 

  551.     }

  552.     a -= m;

  553.     b -= 2 * n;

  554.   }

  555. 

  556. }

  557. 

  558. #else

  559. 

  560. static inline void solve(BLASLONG m, BLASLONG n, FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc) {

  561. 

  562.   FLOAT aa1, aa2;

  563.   FLOAT bb1, bb2;

  564.   FLOAT cc1, cc2;

  565. 

  566.   int i, j, k;

  567. 

  568.   ldc *= 2;

  569.   a += (m - 1) * m * 2;

  570.   b += (m - 1) * n * 2;

  571. 

  572.   for (i = m - 1; i >= 0; i--) {

  573. 

  574.     aa1 = *(a + i * 2 + 0);

  575.     aa2 = *(a + i * 2 + 1);

  576. 

  577.     for (j = 0; j < n; j ++) {

  578.       bb1 = *(c + i * 2 + 0 + j * ldc);

  579.       bb2 = *(c + i * 2 + 1 + j * ldc);

  580. 

  581. #ifndef CONJ

  582.       cc1 = aa1 * bb1 - aa2 * bb2;

  583.       cc2 = aa1 * bb2 + aa2 * bb1;

  584. #else

  585.       cc1 = aa1 * bb1 + aa2 * bb2;

  586.       cc2 = aa1 * bb2 - aa2 * bb1;

  587. #endif

  588. 

  589. 

  590.       *(b + 0) = cc1;

  591.       *(b + 1) = cc2;

  592.       *(c + i * 2 + 0 + j * ldc) = cc1;

  593.       *(c + i * 2 + 1 + j * ldc) = cc2;

  594.       b += 2;

  595. 

  596.       for (k = 0; k < i; k ++){

  597. #ifndef CONJ

  598. 	*(c + k * 2 + 0 + j * ldc) -= cc1 * *(a + k * 2 + 0) - cc2 * *(a + k * 2 + 1);

  599. 	*(c + k * 2 + 1 + j * ldc) -= cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0);

  600. #else

  601. 	*(c + k * 2 + 0 + j * ldc) -=   cc1 * *(a + k * 2 + 0) + cc2 * *(a + k * 2 + 1);

  602. 	*(c + k * 2 + 1 + j * ldc) -= - cc1 * *(a + k * 2 + 1) + cc2 * *(a + k * 2 + 0);

  603. #endif

  604.       }

  605. 

  606.     }

  607.     a -= m * 2;

  608.     b -= 4 * n;

  609.   }

  610. 

  611. }

  612. 

  613. #endif

  614. 

  615. 

  616. int CNAME(BLASLONG m, BLASLONG n, BLASLONG k,  FLOAT dummy1,

  617. #ifdef COMPLEX

  618. 	   FLOAT dummy2,

  619. #endif

  620. 	   FLOAT *a, FLOAT *b, FLOAT *c, BLASLONG ldc, BLASLONG offset){

  621. 

  622.   BLASLONG i, j;

  623.   FLOAT *aa, *cc;

  624.   BLASLONG  kk;

  625. 

  626. #if 0

  627.   fprintf(stderr, "TRSM KERNEL LN : m = %3ld  n = %3ld  k = %3ld offset = %3ld\n",

  628. 	  m, n, k, offset);

  629. #endif

  630. 

  631.   j = (n >> GEMM_UNROLL_N_SHIFT);

  632. 

  633.   while (j > 0) {

  634. 

  635.     kk = m + offset;

  636. 

  637.     if (m & (GEMM_UNROLL_M - 1)) {

  638.       for (i = 1; i < GEMM_UNROLL_M; i *= 2){

  639. 	if (m & i) {

  640. 	  aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;

  641. 	  cc = c + ((m & ~(i - 1)) - i)     * COMPSIZE;

  642. 

  643. 	  if (k - kk > 0) {

  644. 	    GEMM_KERNEL(i, GEMM_UNROLL_N, k - kk, dm1,

  645. #ifdef COMPLEX

  646. 			ZERO,

  647. #endif

  648. 			aa + i             * kk * COMPSIZE,

  649. 			b  + GEMM_UNROLL_N * kk * COMPSIZE,

  650. 			cc,

  651. 			ldc);

  652. 	  }

  653. 

  654. 	  solve(i, GEMM_UNROLL_N,

  655. 		aa + (kk - i) * i             * COMPSIZE,

  656. 		b  + (kk - i) * GEMM_UNROLL_N * COMPSIZE,

  657. 		cc, ldc);

  658. 

  659. 	  kk -= i;

  660. 	}

  661.       }

  662.     }

  663. 

  664.     i = (m >> GEMM_UNROLL_M_SHIFT);

  665.     if (i > 0) {

  666.       aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE;

  667.       cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M)     * COMPSIZE;

  668. 

  669.       do {

  670. 	strsm_LN_solve_opt(k-kk, aa + GEMM_UNROLL_M * kk * COMPSIZE, b +  GEMM_UNROLL_N * kk * COMPSIZE, cc, ldc,

  671.                            aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE,b  + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_N * COMPSIZE);

  672. 

  673. 	aa -= GEMM_UNROLL_M * k * COMPSIZE;

  674. 	cc -= GEMM_UNROLL_M     * COMPSIZE;

  675. 	kk -= GEMM_UNROLL_M;

  676. 	i --;

  677.       } while (i > 0);

  678.     }

  679. 

  680.     b += GEMM_UNROLL_N * k * COMPSIZE;

  681.     c += GEMM_UNROLL_N * ldc * COMPSIZE;

  682.     j --;

  683.   }

  684. 

  685.   if (n & (GEMM_UNROLL_N - 1)) {

  686. 

  687.     j = (GEMM_UNROLL_N >> 1);

  688.     while (j > 0) {

  689.       if (n & j) {

  690. 

  691. 	kk = m + offset;

  692. 

  693. 	if (m & (GEMM_UNROLL_M - 1)) {

  694. 	  for (i = 1; i < GEMM_UNROLL_M; i *= 2){

  695. 	    if (m & i) {

  696. 	      aa = a + ((m & ~(i - 1)) - i) * k * COMPSIZE;

  697. 	      cc = c + ((m & ~(i - 1)) - i)     * COMPSIZE;

  698. 

  699. 	      if (k - kk > 0) {

  700. 		GEMM_KERNEL(i, j, k - kk, dm1,

  701. #ifdef COMPLEX

  702. 			    ZERO,

  703. #endif

  704. 			    aa + i * kk * COMPSIZE,

  705. 			    b  + j * kk * COMPSIZE,

  706. 			    cc, ldc);

  707. 	      }

  708. 

  709. 	      solve(i, j,

  710. 		    aa + (kk - i) * i * COMPSIZE,

  711. 		    b  + (kk - i) * j * COMPSIZE,

  712. 		    cc, ldc);

  713. 

  714. 	      kk -= i;

  715. 	    }

  716. 	  }

  717. 	}

  718. 

  719. 	i = (m >> GEMM_UNROLL_M_SHIFT);

  720. 	if (i > 0) {

  721. 	  aa = a + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M) * k * COMPSIZE;

  722. 	  cc = c + ((m & ~(GEMM_UNROLL_M - 1)) - GEMM_UNROLL_M)     * COMPSIZE;

  723. 

  724. 	  do {

  725. 	    if (k - kk > 0) {

  726. 	      GEMM_KERNEL(GEMM_UNROLL_M, j, k - kk, dm1,

  727. #ifdef COMPLEX

  728. 			  ZERO,

  729. #endif

  730. 			  aa + GEMM_UNROLL_M * kk * COMPSIZE,

  731. 			  b +  j             * kk * COMPSIZE,

  732. 			  cc,

  733. 			  ldc);

  734. 	    }

  735. 

  736. 	    solve(GEMM_UNROLL_M, j,

  737. 		  aa + (kk - GEMM_UNROLL_M) * GEMM_UNROLL_M * COMPSIZE,

  738. 		  b  + (kk - GEMM_UNROLL_M) * j             * COMPSIZE,

  739. 		  cc, ldc);

  740. 

  741. 	    aa -= GEMM_UNROLL_M * k * COMPSIZE;

  742. 	    cc -= GEMM_UNROLL_M     * COMPSIZE;

  743. 	    kk -= GEMM_UNROLL_M;

  744. 	    i --;

  745. 	  } while (i > 0);

  746. 	}

  747. 

  748. 	b += j * k   * COMPSIZE;

  749. 	c += j * ldc * COMPSIZE;

  750.       }

  751.       j >>= 1;

  752.     }

  753.   }

  754. 

  755.   return 0;

  756. }