OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branch: develop
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'develop'
${ noResults }
OpenBLAS/kernel/x86/zgemm_kernel_1x2.S

814 lines
17 kB
Raw Permalink Blame History 

  1. /*********************************************************************/

  2. /* Copyright 2009, 2010 The University of Texas at Austin.           */

  3. /* All rights reserved.                                              */

  4. /*                                                                   */

  5. /* Redistribution and use in source and binary forms, with or        */

  6. /* without modification, are permitted provided that the following   */

  7. /* conditions are met:                                               */

  8. /*                                                                   */

  9. /*   1. Redistributions of source code must retain the above         */

  10. /*      copyright notice, this list of conditions and the following  */

  11. /*      disclaimer.                                                  */

  12. /*                                                                   */

  13. /*   2. Redistributions in binary form must reproduce the above      */

  14. /*      copyright notice, this list of conditions and the following  */

  15. /*      disclaimer in the documentation and/or other materials       */

  16. /*      provided with the distribution.                              */

  17. /*                                                                   */

  18. /*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */

  19. /*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */

  20. /*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */

  21. /*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */

  22. /*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */

  23. /*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */

  24. /*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */

  25. /*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */

  26. /*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */

  27. /*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */

  28. /*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */

  29. /*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */

  30. /*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */

  31. /*    POSSIBILITY OF SUCH DAMAGE.                                    */

  32. /*                                                                   */

  33. /* The views and conclusions contained in the software and           */

  34. /* documentation are those of the authors and should not be          */

  35. /* interpreted as representing official policies, either expressed   */

  36. /* or implied, of The University of Texas at Austin.                 */

  37. /*********************************************************************/

  38. 

  39. #define ASSEMBLER

  40. #include "common.h"

  41. 

  42. #define STACK	16

  43. #define ARGS	16

  44. 

  45. #define J	 0 + STACK(%esp)

  46. #define I	 4 + STACK(%esp)

  47. #define KK	 8 + STACK(%esp)

  48. #define KKK	12 + STACK(%esp)

  49. 

  50. #define M	 4 + STACK + ARGS(%esp)

  51. #define N	 8 + STACK + ARGS(%esp)

  52. #define K	12 + STACK + ARGS(%esp)

  53. #ifdef DOUBLE

  54. #define ALPHA_R	16 + STACK + ARGS(%esp)

  55. #define ALPHA_I	24 + STACK + ARGS(%esp)

  56. #define STACK_A	32 + STACK + ARGS(%esp)

  57. #define STACK_B	36 + STACK + ARGS(%esp)

  58. #define C	40 + STACK + ARGS(%esp)

  59. #define STACK_LDC	44 + STACK + ARGS(%esp)

  60. #define OFFSET	48 + STACK + ARGS(%esp)

  61. #else

  62. #define ALPHA_R	16 + STACK + ARGS(%esp)

  63. #define ALPHA_I	20 + STACK + ARGS(%esp)

  64. #define STACK_A	24 + STACK + ARGS(%esp)

  65. #define STACK_B	28 + STACK + ARGS(%esp)

  66. #define C	32 + STACK + ARGS(%esp)

  67. #define STACK_LDC	36 + STACK + ARGS(%esp)

  68. #define OFFSET	40 + STACK + ARGS(%esp)

  69. #endif

  70. 

  71. #define A	%edx

  72. #define B	%ecx

  73. #define BB	%ebx

  74. #define LDC	%ebp

  75. #define BX	%esi

  76. 

  77. #define ADD1	faddp

  78. 

  79. #if defined(NN) || defined(CN)

  80. #define ADD2	faddp

  81. #else

  82. #define ADD2	fsubrp

  83. #endif

  84. 

  85. #if defined(NN) || defined(CC)

  86. #define ADD3	fsubrp

  87. #else

  88. #define ADD3	faddp

  89. #endif

  90. 

  91. #if defined(NN) || defined(NC)

  92. #define ADD4	faddp

  93. #else

  94. #define ADD4	fsubrp

  95. #endif

  96. 

  97. #define PREFETCHSIZE (8 * 5 + 4)

  98. 

  99. #define AOFFSET   1

  100. #define BOFFSET   1

  101. 

  102. #ifdef HAVE_3DNOW

  103. #define PREFETCH	prefetch

  104. #else

  105. #define PREFETCH	prefetcht0

  106. #endif

  107. 

  108. #define KERNEL \

  109. 	PREFETCH	PREFETCHSIZE * SIZE + AOFFSET(A, %eax, 2);\

  110. 	fmul	%st(1), %st;\

  111. 	ADD1	%st, %st(4);\

  112. 	FLD	-15 * SIZE + BOFFSET(B, %eax, 4);\

  113. 	fmul	%st(1), %st;\

  114. 	ADD2	%st, %st(5);\

  115. 	FLD	-14 * SIZE + BOFFSET(B, %eax, 4);\

  116. 	fmul	%st(1), %st;\

  117. 	ADD1	%st, %st(6);\

  118. 	FMUL	-13 * SIZE + BOFFSET(B, %eax, 4);\

  119. 	ADD2	%st, %st(6);\

  120. 	FLD	-15 * SIZE + AOFFSET(A, %eax, 2);\

  121. 	FLD	-15 * SIZE + BOFFSET(B, %eax, 4);\

  122. 	fmul	%st(1), %st;\

  123. 	ADD3	%st, %st(4);\

  124. 	FLD	-16 * SIZE + BOFFSET(B, %eax, 4);\

  125. 	fmul	%st(1), %st;\

  126. 	ADD4	%st, %st(5);\

  127. 	FLD	-13 * SIZE + BOFFSET(B, %eax, 4);\

  128. 	fmul	%st(1), %st;\

  129. 	ADD3	%st, %st(6);\

  130. 	FMUL	-14 * SIZE + BOFFSET(B, %eax, 4);\

  131. 	ADD4	%st, %st(6);\

  132. 	FLD	-14 * SIZE + AOFFSET(A, %eax, 2);\

  133. 	FLD	-12 * SIZE + BOFFSET(B, %eax, 4);\

  134. 	fmul	%st(1), %st;\

  135. 	ADD1	%st, %st(4);\

  136. 	FLD	-11 * SIZE + BOFFSET(B, %eax, 4);\

  137. 	fmul	%st(1), %st;\

  138. 	ADD2	%st, %st(5);\

  139. 	FLD	-10 * SIZE + BOFFSET(B, %eax, 4);\

  140. 	fmul	%st(1), %st;\

  141. 	ADD1	%st, %st(6);\

  142. 	FMUL	 -9 * SIZE + BOFFSET(B, %eax, 4);\

  143. 	ADD2	%st, %st(6);\

  144. 	FLD	-13 * SIZE + AOFFSET(A, %eax, 2);\

  145. 	FLD	-11 * SIZE + BOFFSET(B, %eax, 4);\

  146. 	fmul	%st(1), %st;\

  147. 	ADD3	%st, %st(4);\

  148. 	FLD	-12 * SIZE + BOFFSET(B, %eax, 4);\

  149. 	fmul	%st(1), %st;\

  150. 	ADD4	%st, %st(5);\

  151. 	FLD	 -9 * SIZE + BOFFSET(B, %eax, 4);\

  152. 	fmul	%st(1), %st;\

  153. 	ADD3	%st, %st(6);\

  154. 	FMUL	-10 * SIZE + BOFFSET(B, %eax, 4);\

  155. 	ADD4	%st, %st(6);\

  156. 	FLD	-12 * SIZE + AOFFSET(A, %eax, 2);\

  157. 	FLD	 -8 * SIZE + BOFFSET(B, %eax, 4);\

  158. 	fmul	%st(1), %st;\

  159. 	ADD1	%st, %st(4);\

  160. 	FLD	 -7 * SIZE + BOFFSET(B, %eax, 4);\

  161. 	fmul	%st(1), %st;\

  162. 	ADD2	%st, %st(5);\

  163. 	FLD	 -6 * SIZE + BOFFSET(B, %eax, 4);\

  164. 	fmul	%st(1), %st;\

  165. 	ADD1	%st, %st(6);\

  166. 	FMUL	 -5 * SIZE + BOFFSET(B, %eax, 4);\

  167. 	ADD2	%st, %st(6);\

  168. 	FLD	-11 * SIZE + AOFFSET(A, %eax, 2);\

  169. 	FLD	 -7 * SIZE + BOFFSET(B, %eax, 4);\

  170. 	fmul	%st(1), %st;\

  171. 	ADD3	%st, %st(4);\

  172. 	FLD	 -8 * SIZE + BOFFSET(B, %eax, 4);\

  173. 	fmul	%st(1), %st;\

  174. 	ADD4	%st, %st(5);\

  175. 	FLD	 -5 * SIZE + BOFFSET(B, %eax, 4);\

  176. 	fmul	%st(1), %st;\

  177. 	ADD3	%st, %st(6);\

  178. 	FMUL	 -6 * SIZE + BOFFSET(B, %eax, 4);\

  179. 	ADD4	%st, %st(6);\

  180. 	FLD	-10 * SIZE + AOFFSET(A, %eax, 2);\

  181. 	FLD	 -4 * SIZE + BOFFSET(B, %eax, 4);\

  182. 	fmul	%st(1), %st;\

  183. 	ADD1	%st, %st(4);\

  184. 	FLD	 -3 * SIZE + BOFFSET(B, %eax, 4);\

  185. 	fmul	%st(1), %st;\

  186. 	ADD2	%st, %st(5);\

  187. 	FLD	 -2 * SIZE + BOFFSET(B, %eax, 4);\

  188. 	fmul	%st(1), %st;\

  189. 	ADD1	%st, %st(6);\

  190. 	FMUL	 -1 * SIZE + BOFFSET(B, %eax, 4);\

  191. 	ADD2	%st, %st(6);\

  192. 	FLD	 -9 * SIZE + AOFFSET(A, %eax, 2);\

  193. 	FLD	 -3 * SIZE + BOFFSET(B, %eax, 4);\

  194. 	fmul	%st(1), %st;\

  195. 	ADD3	%st, %st(4);\

  196. 	FLD	 -4 * SIZE + BOFFSET(B, %eax, 4);\

  197. 	fmul	%st(1), %st;\

  198. 	ADD4	%st, %st(5);\

  199. 	FLD	 -1 * SIZE + BOFFSET(B, %eax, 4);\

  200. 	fmul	%st(1), %st;\

  201. 	ADD3	%st, %st(6);\

  202. 	FMUL	 -2 * SIZE + BOFFSET(B, %eax, 4);\

  203. 	ADD4	%st, %st(6);\

  204. 	FLD	  8 * SIZE + AOFFSET(A, %eax, 2);\

  205. 	fxch	%st(1);\

  206. 	FLD	  0 * SIZE + BOFFSET(B, %eax, 4);\

  207. 	fmul	%st(1), %st;\

  208. 	ADD1	%st, %st(4);\

  209. 	FLD	  1 * SIZE + BOFFSET(B, %eax, 4);\

  210. 	fmul	%st(1), %st;\

  211. 	PREFETCH	(PREFETCHSIZE + 8) * SIZE + AOFFSET(A, %eax, 2);\

  212. 	ADD2	%st, %st(5);\

  213. 	FLD	  2 * SIZE + BOFFSET(B, %eax, 4);\

  214. 	fmul	%st(1), %st;\

  215. 	ADD1	%st, %st(6);\

  216. 	FMUL	  3 * SIZE + BOFFSET(B, %eax, 4);\

  217. 	ADD2	%st, %st(6);\

  218. 	FLD	 -7 * SIZE + AOFFSET(A, %eax, 2);\

  219. 	FLD	  1 * SIZE + BOFFSET(B, %eax, 4);\

  220. 	fmul	%st(1), %st;\

  221. 	ADD3	%st, %st(4);\

  222. 	FLD	  0 * SIZE + BOFFSET(B, %eax, 4);\

  223. 	fmul	%st(1), %st;\

  224. 	ADD4	%st, %st(5);\

  225. 	FLD	  3 * SIZE + BOFFSET(B, %eax, 4);\

  226. 	fmul	%st(1), %st;\

  227. 	ADD3	%st, %st(6);\

  228. 	FMUL	  2 * SIZE + BOFFSET(B, %eax, 4);\

  229. 	ADD4	%st, %st(6);\

  230. 	FLD	 -6 * SIZE + AOFFSET(A, %eax, 2);\

  231. 	FLD	  4 * SIZE + BOFFSET(B, %eax, 4);\

  232. 	fmul	%st(1), %st;\

  233. 	ADD1	%st, %st(4);\

  234. 	FLD	  5 * SIZE + BOFFSET(B, %eax, 4);\

  235. 	fmul	%st(1), %st;\

  236. 	ADD2	%st, %st(5);\

  237. 	FLD	  6 * SIZE + BOFFSET(B, %eax, 4);\

  238. 	fmul	%st(1), %st;\

  239. 	ADD1	%st, %st(6);\

  240. 	FMUL	  7 * SIZE + BOFFSET(B, %eax, 4);\

  241. 	ADD2	%st, %st(6);\

  242. 	FLD	 -5 * SIZE + AOFFSET(A, %eax, 2);\

  243. 	FLD	  5 * SIZE + BOFFSET(B, %eax, 4);\

  244. 	fmul	%st(1), %st;\

  245. 	ADD3	%st, %st(4);\

  246. 	FLD	  4 * SIZE + BOFFSET(B, %eax, 4);\

  247. 	fmul	%st(1), %st;\

  248. 	ADD4	%st, %st(5);\

  249. 	FLD	  7 * SIZE + BOFFSET(B, %eax, 4);\

  250. 	fmul	%st(1), %st;\

  251. 	ADD3	%st, %st(6);\

  252. 	FMUL	  6 * SIZE + BOFFSET(B, %eax, 4);\

  253. 	ADD4	%st, %st(6);\

  254. 	FLD	 -4 * SIZE + AOFFSET(A, %eax, 2);\

  255. 	FLD	  8 * SIZE + BOFFSET(B, %eax, 4);\

  256. 	fmul	%st(1), %st;\

  257. 	ADD1	%st, %st(4);\

  258. 	FLD	  9 * SIZE + BOFFSET(B, %eax, 4);\

  259. 	fmul	%st(1), %st;\

  260. 	ADD2	%st, %st(5);\

  261. 	FLD	 10 * SIZE + BOFFSET(B, %eax, 4);\

  262. 	fmul	%st(1), %st;\

  263. 	ADD1	%st, %st(6);\

  264. 	FMUL	 11 * SIZE + BOFFSET(B, %eax, 4);\

  265. 	ADD2	%st, %st(6);\

  266. 	FLD	 -3 * SIZE + AOFFSET(A, %eax, 2);\

  267. 	FLD	  9 * SIZE + BOFFSET(B, %eax, 4);\

  268. 	fmul	%st(1), %st;\

  269. 	ADD3	%st, %st(4);\

  270. 	FLD	  8 * SIZE + BOFFSET(B, %eax, 4);\

  271. 	fmul	%st(1), %st;\

  272. 	ADD4	%st, %st(5);\

  273. 	FLD	 11 * SIZE + BOFFSET(B, %eax, 4);\

  274. 	fmul	%st(1), %st;\

  275. 	ADD3	%st, %st(6);\

  276. 	FMUL	 10 * SIZE + BOFFSET(B, %eax, 4);\

  277. 	ADD4	%st, %st(6);\

  278. 	FLD	 -2 * SIZE + AOFFSET(A, %eax, 2);\

  279. 	FLD	 12 * SIZE + BOFFSET(B, %eax, 4);\

  280. 	fmul	%st(1), %st;\

  281. 	ADD1	%st, %st(4);\

  282. 	FLD	 13 * SIZE + BOFFSET(B, %eax, 4);\

  283. 	fmul	%st(1), %st;\

  284. 	ADD2	%st, %st(5);\

  285. 	FLD	 14 * SIZE + BOFFSET(B, %eax, 4);\

  286. 	fmul	%st(1), %st;\

  287. 	ADD1	%st, %st(6);\

  288. 	FMUL	 15 * SIZE + BOFFSET(B, %eax, 4);\

  289. 	ADD2	%st, %st(6);\

  290. 	FLD	 -1 * SIZE + AOFFSET(A, %eax, 2);\

  291. 	FLD	 13 * SIZE + BOFFSET(B, %eax, 4);\

  292. 	fmul	%st(1), %st;\

  293. 	ADD3	%st, %st(4);\

  294. 	FLD	 12 * SIZE + BOFFSET(B, %eax, 4);\

  295. 	fmul	%st(1), %st;\

  296. 	ADD4	%st, %st(5);\

  297. 	FLD	 15 * SIZE + BOFFSET(B, %eax, 4);\

  298. 	fmul	%st(1), %st;\

  299. 	ADD3	%st, %st(6);\

  300. 	FMUL	 14 * SIZE + BOFFSET(B, %eax, 4);\

  301. 	ADD4	%st, %st(6);\

  302. 	FLD	 16 * SIZE + AOFFSET(A, %eax, 2);\

  303. 	fxch	%st(2);\

  304. 	FLD	  0 * SIZE + BOFFSET(BB, %eax, 4);\

  305. 	subl	 $-8 * SIZE, %eax

  306. 

  307. /*

  308. 

  309.   A hint of scheduling is received from following URL

  310. 

  311.   http://www.netlib.org/atlas/atlas-comm/msg00260.html

  312. 

  313. */

  314. 

  315. 	PROLOGUE

  316. 

  317. 	subl	$ARGS, %esp	# Generate Stack Frame

  318. 

  319. 	pushl	%ebp

  320. 	pushl	%edi

  321. 	pushl	%esi

  322. 	pushl	%ebx

  323. 

  324. 	PROFCODE

  325. 

  326. #if defined(TRMMKERNEL) && !defined(LEFT)

  327. 	movl	OFFSET, %eax

  328. 	negl	%eax

  329. 	movl	%eax, KK

  330. #endif

  331. 

  332. 	movl	STACK_LDC, LDC

  333. 	sall	$ZBASE_SHIFT, LDC

  334. 

  335. 	subl	$(AOFFSET - 16 * SIZE), STACK_A

  336. 	subl	$(BOFFSET - 16 * SIZE), STACK_B

  337. 

  338. 	movl	M, %eax

  339. 	testl	%eax, %eax

  340. 	jle	.L999

  341. 

  342. 	movl	N, %eax

  343. 	testl	%eax, %eax

  344. 	jle	.L999

  345. 

  346. 	movl	K, %eax

  347. 	testl	%eax, %eax

  348. 	jle	.L999

  349. 

  350. 	movl	N,   %eax

  351. 	sarl	$1,  %eax

  352. 	movl	%eax, J

  353. 	je	.L20

  354. 	ALIGN_3

  355. 

  356. .L11:

  357. #if defined(TRMMKERNEL) && defined(LEFT)

  358. 	movl	OFFSET, %eax

  359. 	movl	%eax, KK

  360. #endif

  361. 

  362. 	movl	STACK_A, A

  363. 	movl	STACK_B, B

  364. 	movl	C, %edi

  365. 

  366. 	movl	K, BX

  367. 	sall	$ZBASE_SHIFT + 1, BX

  368. 	addl	B, BX

  369. 

  370. 	movl	M, %eax

  371. 	movl	%eax, I

  372. 	ALIGN_3

  373. 

  374. .L14:

  375. 	prefetchnta	-16 * SIZE + BOFFSET(BX)

  376. 	prefetchnta	 -8 * SIZE + BOFFSET(BX)

  377. 	subl	$-16 * SIZE, BX

  378. 

  379. 	movl	STACK_B, B

  380. 

  381. #if !defined(TRMMKERNEL) || \

  382. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  383. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  384. #else

  385. 	movl	KK,   %eax

  386. 	leal	(, %eax, SIZE), %eax

  387. 	leal	(A, %eax, 2), A

  388. 	leal	(B, %eax, 4), B

  389. #endif

  390. 

  391. 	fldz

  392. 	fldz

  393. 	fldz

  394. 	fldz

  395. 

  396. 	FLD	   0 * SIZE + AOFFSET(A)

  397. 	FLD	  -8 * SIZE + AOFFSET(A)

  398. 	FLD	 -16 * SIZE + AOFFSET(A)

  399. 	FLD	 -16 * SIZE + BOFFSET(B)

  400. 

  401. #ifdef HAVE_3DNOW

  402. 	prefetchw	1 * SIZE(%edi)

  403. 	prefetchw	2 * SIZE(%edi, LDC)

  404. #elif defined(HAVE_SSE)

  405. 	prefetcht0	1 * SIZE(%edi)

  406. 	prefetcht0	2 * SIZE(%edi, LDC)

  407. #endif

  408. 

  409. #ifndef TRMMKERNEL

  410. 	movl	K,  %eax

  411. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  412. 	movl	K, %eax

  413. 	subl	KK, %eax

  414. 	movl	%eax, KKK

  415. #else

  416. 	movl	KK, %eax

  417. #ifdef LEFT

  418. 	addl	$1, %eax

  419. #else

  420. 	addl	$2, %eax

  421. #endif

  422. 	movl	%eax, KKK

  423. #endif

  424. 

  425. 	andl	$-8, %eax

  426. 

  427. 	leal	(, %eax, SIZE), %eax

  428. 	leal	(A, %eax, 2), A

  429. 	leal	16 * SIZE(B, %eax, 4), BB

  430. 	leal	(B, %eax, 4), B

  431. 	negl	%eax

  432. 	NOBRANCH

  433.  	je	.L16

  434. 	ALIGN_4

  435. 

  436. .L15:

  437. 	KERNEL

  438. 	jge	.L16

  439. 	KERNEL

  440. 	jge	.L16

  441. 	KERNEL

  442. 	jge	.L16

  443. 	KERNEL

  444. 	jl	.L15

  445. 	ALIGN_4

  446. 

  447. .L16:

  448. #ifndef TRMMKERNEL

  449. 	movl	K, %eax

  450. #else

  451. 	movl	KKK, %eax

  452. #endif

  453. 	and	$7, %eax

  454. 	je	.L19

  455. 	ALIGN_4

  456. 

  457. 

  458. .L17:

  459. 	fmul	%st(1), %st

  460. 	ADD1	%st, %st(4)

  461. 	FLD	-15 * SIZE + BOFFSET(B)

  462. 

  463. 	fmul	%st(1), %st

  464. 	ADD2	%st, %st(5)

  465. 	FLD	-14 * SIZE + BOFFSET(B)

  466. 

  467. 	fmul	%st(1), %st

  468. 	ADD1	%st, %st(6)

  469. 	FMUL	-13 * SIZE + BOFFSET(B)

  470. 

  471. 	ADD2	%st, %st(6)

  472. 	FLD	-15 * SIZE + AOFFSET(A)

  473. 	FLD	-15 * SIZE + BOFFSET(B)

  474. 

  475. 	fmul	%st(1), %st

  476. 	ADD3	%st, %st(4)

  477. 	FLD	-16 * SIZE + BOFFSET(B)

  478. 

  479. 	fmul	%st(1), %st

  480. 	ADD4	%st, %st(5)

  481. 	FLD	-13 * SIZE + BOFFSET(B)

  482. 

  483. 	fmul	%st(1), %st

  484. 	ADD3	%st, %st(6)

  485. 	FMUL	-14 * SIZE + BOFFSET(B)

  486. 

  487. 	ADD4	%st, %st(6)

  488. 	FLD	-14 * SIZE + AOFFSET(A)

  489. 	FLD	-12 * SIZE + BOFFSET(B)

  490. 

  491. 	addl	$2 * SIZE,A

  492. 	addl	$4 * SIZE,B

  493. 

  494. 	decl	%eax

  495. 	jne	 .L17

  496. 	ALIGN_4

  497. 

  498. .L19:

  499. 	ffreep	%st(0)

  500. 	ffreep	%st(0)

  501. 	ffreep	%st(0)

  502. 	ffreep	%st(0)

  503. 

  504. 	FLD	ALPHA_R

  505. 	fmul	%st(1), %st

  506. 	FLD	ALPHA_I

  507. 	fmul	%st(3), %st

  508. 	fsubrp	%st, %st(1)

  509. 	fxch	%st(2)

  510. 	FMUL	ALPHA_R

  511. 	fxch	%st(1)

  512. 	FMUL	ALPHA_I

  513. 	faddp	%st, %st(1)

  514. 

  515. #ifndef TRMMKERNEL

  516. 	FADD	1 * SIZE(%edi)

  517. 	FST	1 * SIZE(%edi)

  518. 	FADD	0 * SIZE(%edi)

  519. 	FST	0 * SIZE(%edi)

  520. #else

  521. 	FST	1 * SIZE(%edi)

  522. 	FST	0 * SIZE(%edi)

  523. #endif

  524. 

  525. 	FLD	ALPHA_R

  526. 	fmul	%st(1), %st

  527. 	FLD	ALPHA_I

  528. 	fmul	%st(3), %st

  529. 	fsubrp	%st, %st(1)

  530. 	fxch	%st(2)

  531. 	FMUL	ALPHA_R

  532. 	fxch	%st(1)

  533. 	FMUL	ALPHA_I

  534. 	faddp	%st, %st(1)

  535. 

  536. #ifndef TRMMKERNEL

  537. 	FADD	1 * SIZE(%edi,LDC)

  538. 	FST	1 * SIZE(%edi,LDC)

  539. 	FADD	0 * SIZE(%edi,LDC)

  540. 	FST	0 * SIZE(%edi,LDC)

  541. #else

  542. 	FST	1 * SIZE(%edi,LDC)

  543. 	FST	0 * SIZE(%edi,LDC)

  544. #endif

  545. 

  546. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  547.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  548. 	movl	K, %eax

  549. 	subl	KKK, %eax

  550. 	leal	(,%eax, SIZE), %eax

  551. 	leal	(A, %eax, 2), A

  552. 	leal	(B, %eax, 4), B

  553. #endif

  554. 

  555. #if defined(TRMMKERNEL) && defined(LEFT)

  556. 	addl	$1, KK

  557. #endif

  558. 

  559. 	addl	$2 * SIZE, %edi

  560. 	decl	I

  561. 	jne	.L14

  562. 

  563. #if defined(TRMMKERNEL) && !defined(LEFT)

  564. 	addl	$2, KK

  565. #endif

  566. 

  567. 	leal	(, LDC, 2), %eax

  568. 	addl	%eax, C

  569. 	movl	B, STACK_B

  570. 	decl	J

  571. 	jne	.L11

  572. 	ALIGN_4

  573. 

  574. .L20:

  575. 	movl	N,   %eax

  576. 	andl	$1,  %eax

  577. 	je	.L999

  578. 	ALIGN_3

  579. 

  580. #if defined(TRMMKERNEL) && defined(LEFT)

  581. 	movl	OFFSET, %eax

  582. 	movl	%eax, KK

  583. #endif

  584. 

  585. 	movl	STACK_A, A

  586. 	movl	STACK_B, B

  587. 	movl	C, %edi

  588. 

  589. 	movl	M, %eax

  590. 	movl	%eax, I

  591. 	ALIGN_3

  592. 

  593. .L24:

  594. 	movl	STACK_B, B

  595. #if !defined(TRMMKERNEL) || \

  596. 	(defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  597. 	(defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  598. #else

  599. 	movl	KK,   %eax

  600. 	leal	(, %eax, SIZE), %eax

  601. 	leal	(A,      %eax, 2), A

  602. 	leal	(B, %eax, 2), B

  603. #endif

  604. 

  605. 	fldz

  606. 	fldz

  607. 	fldz

  608. 	fldz

  609. 

  610. 	FLD	-16 * SIZE + AOFFSET(A)

  611. 	FLD	-16 * SIZE + BOFFSET(B)

  612. 

  613. 	prefetchw	1 * SIZE(%edi)

  614. 

  615. #ifndef TRMMKERNEL

  616. 	movl	K,  %eax

  617. #elif (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  618. 	movl	K, %eax

  619. 	subl	KK, %eax

  620. 	movl	%eax, KKK

  621. #else

  622. 	movl	KK, %eax

  623. #ifdef LEFT

  624. 	addl	$1, %eax

  625. #else

  626. 	addl	$1, %eax

  627. #endif

  628. 	movl	%eax, KKK

  629. #endif

  630. 	sarl	$2, %eax

  631.  	je	.L26

  632. 	ALIGN_3

  633. 

  634. .L25:

  635. 	fmul	%st(1), %st

  636. 	PADDING

  637. 	ADD1	%st, %st(2)

  638. 	FMUL	-15 * SIZE + BOFFSET(B)

  639. 

  640. 	ADD2	%st, %st(2)

  641. 	FLD	-15 * SIZE + AOFFSET(A)

  642. 	FLD	-16 * SIZE + BOFFSET(B)

  643. 

  644. 	fmul	%st(1), %st

  645. 	PADDING

  646. 	ADD4	%st, %st(4)

  647. 	FMUL	-15 * SIZE + BOFFSET(B)

  648. 

  649. 	ADD3	%st, %st(4)

  650. 	FLD	-14 * SIZE + AOFFSET(A)

  651. 	FLD	-14 * SIZE + BOFFSET(B)

  652. 

  653. 	fmul	%st(1), %st

  654. 	PADDING

  655. 	ADD1	%st, %st(2)

  656. 	FMUL	-13 * SIZE + BOFFSET(B)

  657. 

  658. 	ADD2	%st, %st(2)

  659. 	FLD	-13 * SIZE + AOFFSET(A)

  660. 	FLD	-14 * SIZE + BOFFSET(B)

  661. 

  662. 	fmul	%st(1), %st

  663. 	PADDING

  664. 	ADD4	%st, %st(4)

  665. 	FMUL	-13 * SIZE + BOFFSET(B)

  666. 

  667. 	ADD3	%st, %st(4)

  668. 	FLD	-12 * SIZE + AOFFSET(A)

  669. 	FLD	-12 * SIZE + BOFFSET(B)

  670. 

  671. 	fmul	%st(1), %st

  672. 	PADDING

  673. 	ADD1	%st, %st(2)

  674. 	FMUL	-11 * SIZE + BOFFSET(B)

  675. 

  676. 	ADD2	%st, %st(2)

  677. 	FLD	-11 * SIZE + AOFFSET(A)

  678. 	FLD	-12 * SIZE + BOFFSET(B)

  679. 

  680. 	fmul	%st(1), %st

  681. 	PADDING

  682. 	ADD4	%st, %st(4)

  683. 	FMUL	-11 * SIZE + BOFFSET(B)

  684. 

  685. 	ADD3	%st, %st(4)

  686. 	FLD	-10 * SIZE + AOFFSET(A)

  687. 	FLD	-10 * SIZE + BOFFSET(B)

  688. 

  689. 	fmul	%st(1), %st

  690. 	PADDING

  691. 	ADD1	%st, %st(2)

  692. 	FMUL	 -9 * SIZE + BOFFSET(B)

  693. 

  694. 	ADD2	%st, %st(2)

  695. 	FLD	 -9 * SIZE + AOFFSET(A)

  696. 	FLD	-10 * SIZE + BOFFSET(B)

  697. 

  698. 	fmul	%st(1), %st

  699. 	PADDING

  700. 	ADD4	%st, %st(4)

  701. 	FMUL	 -9 * SIZE + BOFFSET(B)

  702. 

  703. 	ADD3	%st, %st(4)

  704. 	FLD	 -8 * SIZE + AOFFSET(A)

  705. 	FLD	 -8 * SIZE + BOFFSET(B)

  706. 

  707. 	addl	$8 * SIZE,A

  708. 	addl	$8 * SIZE,B

  709. 

  710. 	decl	%eax

  711. 	jne	.L25

  712. 	ALIGN_4

  713. 

  714. .L26:

  715. #ifndef TRMMKERNEL

  716. 	movl	K, %eax

  717. #else

  718. 	movl	KKK, %eax

  719. #endif

  720. 	and	$3, %eax

  721. 	je	.L29

  722. 	ALIGN_4

  723. 

  724. .L27:

  725. 	fmul	%st(1), %st

  726. 	PADDING

  727. 	ADD1	%st, %st(2)

  728. 	FMUL	-15 * SIZE + BOFFSET(B)

  729. 

  730. 	ADD2	%st, %st(2)

  731. 	FLD	-15 * SIZE + AOFFSET(A)

  732. 	FLD	-16 * SIZE + BOFFSET(B)

  733. 

  734. 	fmul	%st(1), %st

  735. 	PADDING

  736. 	ADD4	%st, %st(4)

  737. 	FMUL	-15 * SIZE + BOFFSET(B)

  738. 

  739. 	ADD3	%st, %st(4)

  740. 	FLD	-14 * SIZE + AOFFSET(A)

  741. 	FLD	-14 * SIZE + BOFFSET(B)

  742. 

  743. 	addl	$2 * SIZE,A

  744. 	addl	$2 * SIZE,B

  745. 

  746. 	decl	%eax

  747. 	jne	 .L27

  748. 	ALIGN_4

  749. 

  750. .L29:

  751. 	ffreep	%st(0)

  752. 	ffreep	%st(0)

  753. 

  754. 	faddp	%st, %st(3)

  755. 	faddp	%st, %st(1)

  756. 

  757. 	fxch	%st(1)

  758. 

  759. 	FLD	ALPHA_R

  760. 	fmul	%st(1), %st

  761. 	FLD	ALPHA_I

  762. 	fmul	%st(3), %st

  763. 	fsubrp	%st, %st(1)

  764. 	fxch	%st(2)

  765. 	FMUL	ALPHA_R

  766. 	fxch	%st(1)

  767. 	FMUL	ALPHA_I

  768. 	faddp	%st, %st(1)

  769. 

  770. #ifndef TRMMKERNEL

  771. 	FADD	1 * SIZE(%edi)

  772. 	FST	1 * SIZE(%edi)

  773. 	FADD	0 * SIZE(%edi)

  774. 	FST	0 * SIZE(%edi)

  775. #else

  776. 	FST	1 * SIZE(%edi)

  777. 	FST	0 * SIZE(%edi)

  778. #endif

  779. 

  780. #if (defined(TRMMKERNEL) &&  defined(LEFT) &&  defined(TRANSA)) || \

  781.     (defined(TRMMKERNEL) && !defined(LEFT) && !defined(TRANSA))

  782. 	movl	K, %eax

  783. 	subl	KKK, %eax

  784. 	leal	(,%eax, SIZE), %eax

  785. 	leal	(A, %eax, 2), A

  786. 	leal	(B, %eax, 2), B

  787. #endif

  788. 

  789. #if defined(TRMMKERNEL) && defined(LEFT)

  790. 	addl	$1, KK

  791. #endif

  792. 

  793. 	addl	$2 * SIZE, %edi

  794. 	decl	I

  795. 	jne	.L24

  796. 

  797. #if defined(TRMMKERNEL) && !defined(LEFT)

  798. 	addl	$1, KK

  799. #endif

  800. 

  801. 	addl	LDC, C

  802. 	movl	B, STACK_B

  803. 	ALIGN_4

  804. 

  805. .L999:

  806. 	popl	%ebx

  807. 	popl	%esi

  808. 	popl	%edi

  809. 	popl	%ebp

  810. 	addl	$ARGS, %esp

  811. 	ret

  812. 

  813. 	EPILOGUE