OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: 6b6c9b1441
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '6b6c9b1441'
${ noResults }
OpenBLAS/kernel/power/sgemm_macros_power9.S

5575 lines
144 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2019, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27.  

  28. #define unit_size 4

  29. #define DISP64(ind,disp) (ind*unit_size*64+disp)

  30. #define DISP32(ind,disp) (ind*unit_size*32+disp)

  31. #define DISP16(ind,disp) (ind*unit_size*16+disp)

  32. #define DISP8(ind,disp) (ind*unit_size*8+disp)

  33. #define DISP4(ind,disp) (ind*unit_size*4+disp)

  34. #define DISP2(ind,disp) (ind*unit_size*2+disp)

  35. #define DISP1(ind,disp) (ind*unit_size+disp)

  36. 

  37. /**********************************************************************************************

  38. * Macros for N=8 and M=16

  39. **********************************************************************************************/

  40. 

  41.  

  42. 

  43. .macro KERNEL8x16_L1_L4  Index,IsLast

  44.   KERNEL8x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  45. .endm

  46. 

  47. .macro KERNEL8x16_I1_L4  OffsetA,OffsetB, Index,IsLast

  48.   KERNEL8x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  49. .endm

  50. 

  51. .macro KERNEL8x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  52.   KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  53. .endm

  54. 

  55. .macro KERNEL8x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  56.   KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  57. .endm

  58.  

  59. .macro KERNEL8x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  60.   KERNEL8x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  61. .endm

  62. 

  63. .macro KERNEL8x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  64.   KERNEL8x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  65. .endm

  66. 

  67. .macro Zero8X16

  68.     xxlxor		vs32,	vs32,	vs32

  69.     xxlxor		vs33,	vs33,	vs33

  70. 	xxlxor		vs34,	vs34,	vs34

  71. 	xxlxor		vs35,	vs35,	vs35

  72. 	xxlxor		vs36,	vs36,	vs36

  73. 	xxlxor		vs37,	vs37,	vs37

  74. 	xxlxor		vs38,	vs38,	vs38

  75. 	xxlxor		vs39,	vs39,	vs39

  76. 	xxlxor		vs40,	vs40,	vs40

  77. 	xxlxor		vs41,	vs41,	vs41

  78. 	xxlxor		vs42,	vs42,	vs42

  79. 	xxlxor		vs43,	vs43,	vs43

  80. 	xxlxor		vs44,	vs44,	vs44

  81. 	xxlxor		vs45,	vs45,	vs45

  82. 	xxlxor		vs46,	vs46,	vs46

  83. 	xxlxor		vs47,	vs47,	vs47

  84. 	xxlxor		vs48,	vs48,	vs48

  85. 	xxlxor		vs49,	vs49,	vs49

  86. 	xxlxor		vs50,	vs50,	vs50

  87. 	xxlxor		vs51,	vs51,	vs51 

  88. 	xxlxor		vs52,	vs52,	vs52

  89. 	xxlxor		vs53,	vs53,	vs53

  90. 	xxlxor		vs54,	vs54,	vs54

  91. 	xxlxor		vs55,	vs55,	vs55 

  92. 	xxlxor		vs56,	vs56,	vs56

  93. 	xxlxor		vs57,	vs57,	vs57

  94. 	xxlxor		vs58,	vs58,	vs58

  95. 	xxlxor		vs59,	vs59,	vs59 

  96. 	xxlxor		vs60,	vs60,	vs60

  97. 	xxlxor		vs61,	vs61,	vs61

  98. 	xxlxor		vs62,	vs62,	vs62

  99. 	xxlxor		vs63,	vs63,	vs63	

  100. .endm

  101. 

  102. .macro LOAD8x16  OffsetA,OffsetB

  103. 

  104. 	lxv	vs24,	(\OffsetB+0)(BO)

  105. 	lxv	vs28,	(\OffsetB+16)(BO)

  106. 	xxperm  	vs26,	vs24,		permute_mask

  107. 	xxperm  	vs30,	vs28,		permute_mask	  

  108. 	lxv	vs0,	(\OffsetA+0)(AO)

  109. 	lxv	vs1,	(\OffsetA+16)(AO)

  110. 	xxpermdi	vs25,	vs24,	vs24,2	   

  111. 	xxpermdi	vs29,	vs28,	vs28,2	  

  112. 	lxv	vs2,	(\OffsetA+32)(AO)

  113. 	lxv	vs3,	(\OffsetA+48)(AO) 

  114. 	xxpermdi	vs27,	vs26,	vs26,2	

  115. 	xxpermdi	vs31,	vs30,	vs30,2	 	

  116. 

  117. .endm

  118. 

  119. .macro END8x16_NORMAL

  120.   END8x16 0, AO, BO, 64,32 

  121. .endm

  122. 

  123. .macro END8x16_WITHOUT_ADD

  124. 	END8x16 0, AO,BO,0,0

  125. .endm

  126. 

  127. .macro END8x16 First, AREG, BREG, OffsetA, OffsetB

  128. 

  129. .if \OffsetB != 0 

  130.     addi        \BREG, \BREG, \OffsetB 

  131. .endif

  132. .if \OffsetA != 0 

  133.     addi        \AREG, \AREG, \OffsetA 

  134. .endif  

  135. 

  136. .if \First==1

  137.     xvmulsp     vs32, vs0,vs24

  138.     xvmulsp     vs33, vs1,vs24

  139.     xvmulsp     vs34, vs2,vs24  

  140.     xvmulsp     vs35, vs3,vs24  

  141. 

  142.     xvmulsp     vs36, vs0,vs25

  143.     xvmulsp     vs37, vs1,vs25

  144.     xvmulsp     vs38, vs2,vs25  

  145.     xvmulsp     vs39, vs3,vs25

  146. 

  147.     xvmulsp     vs40, vs0,vs26

  148.     xvmulsp     vs41, vs1,vs26

  149.     xvmulsp     vs42, vs2,vs26  

  150.     xvmulsp     vs43, vs3,vs26

  151. 

  152.     xvmulsp     vs44, vs0,vs27

  153.     xvmulsp     vs45, vs1,vs27

  154.     xvmulsp     vs46, vs2,vs27  

  155.     xvmulsp     vs47, vs3,vs27

  156. 

  157.     xvmulsp     vs48, vs0,vs28

  158.     xvmulsp     vs49, vs1,vs28

  159.     xvmulsp     vs50, vs2,vs28  

  160.     xvmulsp     vs51, vs3,vs28  

  161. 

  162.     xvmulsp     vs52, vs0,vs29

  163.     xvmulsp     vs53, vs1,vs29

  164.     xvmulsp     vs54, vs2,vs29  

  165.     xvmulsp     vs55, vs3,vs29

  166. 

  167.     xvmulsp     vs56, vs0,vs30

  168.     xvmulsp     vs57, vs1,vs30

  169.     xvmulsp     vs58, vs2,vs30  

  170.     xvmulsp     vs59, vs3,vs30

  171. 

  172.     xvmulsp     vs60, vs0,vs31

  173.     xvmulsp     vs61, vs1,vs31

  174.     xvmulsp     vs62, vs2,vs31  

  175.     xvmulsp     vs63, vs3,vs31

  176. 

  177. .else

  178.     xvmaddasp       vs32, vs0,vs24

  179.     xvmaddasp       vs33, vs1,vs24

  180.     xvmaddasp       vs34, vs2,vs24  

  181.     xvmaddasp       vs35, vs3,vs24  

  182. 

  183.     xvmaddasp       vs36, vs0,vs25

  184.     xvmaddasp       vs37, vs1,vs25

  185.     xvmaddasp       vs38, vs2,vs25  

  186.     xvmaddasp       vs39, vs3,vs25 

  187.     xvmaddasp       vs40, vs0,vs26

  188.     xvmaddasp       vs41, vs1,vs26

  189.     xvmaddasp       vs42, vs2,vs26  

  190.     xvmaddasp       vs43, vs3,vs26

  191. 

  192.     xvmaddasp       vs44, vs0,vs27

  193.     xvmaddasp       vs45, vs1,vs27

  194.     xvmaddasp       vs46, vs2,vs27  

  195.     xvmaddasp       vs47, vs3,vs27

  196. 

  197.     xvmaddasp       vs48, vs0,vs28

  198.     xvmaddasp       vs49, vs1,vs28

  199.     xvmaddasp       vs50, vs2,vs28  

  200.     xvmaddasp       vs51, vs3,vs28  

  201. 

  202.     xvmaddasp       vs52, vs0,vs29

  203.     xvmaddasp       vs53, vs1,vs29

  204.     xvmaddasp       vs54, vs2,vs29  

  205.     xvmaddasp       vs55, vs3,vs29

  206. 

  207.     xvmaddasp       vs56, vs0,vs30

  208.     xvmaddasp       vs57, vs1,vs30

  209.     xvmaddasp       vs58, vs2,vs30  

  210.     xvmaddasp       vs59, vs3,vs30

  211. 

  212.     xvmaddasp       vs60, vs0,vs31

  213.     xvmaddasp       vs61, vs1,vs31

  214.     xvmaddasp       vs62, vs2,vs31  

  215.     xvmaddasp       vs63, vs3,vs31 

  216. 

  217. .endif

  218. .endm  

  219. 

  220. .macro KERNEL8x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  221. 

  222. KERNEL8x16_2  \AREG,\BREG, \OffsetA,\OffsetB, (\Index*2),0 ,0

  223. KERNEL8x16_2  \AREG,\BREG,\OffsetA,\OffsetB, (\Index*2+1),\IsLast ,\Complete

  224. 

  225. .endm

  226. 

  227. .macro KERNEL8x16 First

  228. 

  229.   LOAD8x16 0,0

  230.   END8x16 \First, AO, BO, 64,32 

  231. .endm

  232. 

  233. .macro LOAD8x16_2

  234.     LOAD8x16_2O AO,BO, 0,0

  235. .endm	

  236. 

  237. .macro LOAD8x16_2O  AREG,BREG, OffsetA,OffsetB

  238.   lxv	vs8,	(\OffsetB)(\BREG)

  239.   lxv	vs12,	(16+\OffsetB)(\BREG)

  240.   lxv	vs24,	(32+\OffsetB)(\BREG)

  241.   lxv	vs28,	(32+16+\OffsetB)(\BREG)

  242.   lxv	vs4,	(0+\OffsetA)(\AREG)

  243.   lxv	vs5,	(16+\OffsetA)(\AREG)

  244.   xxperm  	vs10,	vs8,		permute_mask

  245.   xxperm  	vs14,	vs12,		permute_mask	

  246.   lxv	vs6,	(32+\OffsetA)(\AREG)

  247.   lxv	vs7,	(48+\OffsetA)(\AREG) 

  248.   xxpermdi	vs9,	vs8,	 vs8,2	 

  249.   xxpermdi	vs13,	vs12,	vs12,2	 

  250.   lxv	vs0,	(64+\OffsetA)(\AREG)

  251.   lxv	vs1,	(64+16+\OffsetA)(\AREG) 

  252.   xxpermdi	vs11,	vs10,	vs10,2	

  253.   xxpermdi	vs15,	vs14,	vs14,2	

  254.   lxv	vs2,	(64+32+\OffsetA)(\AREG)

  255.   lxv	vs3,	(64+48+\OffsetA)(\AREG)

  256. 

  257.   xxperm  	vs26,	vs24,	permute_mask

  258.   xxperm  	vs30,	vs28,	permute_mask	

  259.   xxpermdi	vs25,	vs24,	vs24,2 

  260.   xxpermdi	vs29,	vs28,	vs28,2	      

  261.   xxpermdi	vs27,	vs26,	vs26,2	

  262.   xxpermdi	vs31,	vs30,	vs30,2	 

  263. .endm

  264. 

  265. .macro END8x16_2	  

  266.   /*for load2 offset will be 128 and 64*/

  267.    KERNEL8x16_2	AO,BO,	128,64,0 ,1,1 

  268. .endm

  269.  

  270. 

  271. 

  272. .macro KERNEL8x16_E2	OffsetA,OffsetB, Index,IsLast 

  273.   KERNEL8x16_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,1 

  274. .endm

  275. 

  276. 

  277. .macro KERNEL8x16_L2	OffsetA,OffsetB, Index,IsLast

  278.   KERNEL8x16_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,0 

  279. .endm

  280. 

  281. 

  282. .macro KERNEL8x16_2  AREG,BREG,  OffsetA,OffsetB, Index,IsLast ,Complete

  283.   xvmaddasp		vs32, vs4,vs8

  284.   xvmaddasp		vs33, vs5,vs8

  285.   xvmaddasp		vs48, vs4,vs12

  286.   xvmaddasp		vs49, vs5,vs12

  287. 

  288.   xvmaddasp		vs40, vs4,vs10

  289.   xvmaddasp		vs41, vs5,vs10

  290.   xvmaddasp		vs56, vs4,vs14

  291.   xvmaddasp		vs57, vs5,vs14

  292. 

  293.   xvmaddasp		vs36, vs4,vs9

  294.   xvmaddasp		vs37, vs5,vs9

  295.   xvmaddasp		vs52, vs4,vs13

  296.   xvmaddasp		vs53, vs5,vs13

  297. 

  298.   xvmaddasp		vs44, vs4,vs11

  299.   xvmaddasp		vs45, vs5,vs11

  300.   xvmaddasp		vs60, vs4,vs15

  301.   xvmaddasp		vs61, vs5,vs15

  302. 

  303. .if \Complete==0	

  304.    lxv	vs4,	DISP32(\Index,0+\OffsetA)(\AREG)

  305.    lxv	vs5,	DISP32(\Index,16+\OffsetA)(\AREG)

  306. .endif

  307. 

  308.   xvmaddasp		vs34, vs6,vs8	

  309.   xvmaddasp		vs35, vs7,vs8	

  310.   xvmaddasp		vs50, vs6,vs12

  311.   xvmaddasp		vs51, vs7,vs12

  312. .if \Complete==0  

  313.   lxv vs8,  DISP16(\Index,\OffsetB)(\BREG)

  314.   lxv vs12, DISP16(\Index,16+\OffsetB)(\BREG)

  315. .endif    

  316.   xvmaddasp		vs42, vs6,vs10

  317.   xvmaddasp		vs43, vs7,vs10

  318.   xvmaddasp		vs58, vs6,vs14

  319.   xvmaddasp		vs59, vs7,vs14

  320. .if \Complete==0  

  321.   xxperm    vs10, vs8,    permute_mask

  322.   xxperm    vs14, vs12,   permute_mask    

  323. .endif    

  324.   xvmaddasp		vs38, vs6,vs9	

  325.   xvmaddasp		vs39, vs7,vs9	

  326.   xvmaddasp   vs54, vs6,vs13

  327.   xvmaddasp   vs55, vs7,vs13

  328. .if \Complete==0

  329.   xxpermdi  vs9,  vs8,   vs8,2   

  330.   xxpermdi  vs13, vs12, vs12,2   

  331. .endif    

  332.   xvmaddasp		vs46, vs6,vs11

  333.   xvmaddasp		vs47, vs7,vs11

  334.   xvmaddasp		vs62, vs6,vs15

  335.   xvmaddasp		vs63, vs7,vs15

  336. .if \Complete==0

  337.   xxpermdi  vs11, vs10, vs10,2  

  338.   xxpermdi  vs15, vs14, vs14,2  

  339. .endif  

  340. 

  341. .if \Complete==0

  342.    lxv	vs6,	DISP32(\Index,32+\OffsetA)(\AREG)

  343.    lxv	vs7,	DISP32(\Index,48+\OffsetA)(\AREG) 

  344. .endif 

  345. 

  346.   xvmaddasp		vs32, vs0,vs24

  347.   xvmaddasp		vs33, vs1,vs24

  348.   xvmaddasp		vs48, vs0,vs28

  349.   xvmaddasp		vs49, vs1,vs28

  350.   xvmaddasp		vs40, vs0,vs26

  351.   xvmaddasp		vs41, vs1,vs26

  352.   xvmaddasp		vs56, vs0,vs30

  353.   xvmaddasp		vs57, vs1,vs30

  354.   xvmaddasp		vs36, vs0,vs25

  355.   xvmaddasp		vs37, vs1,vs25

  356.   xvmaddasp		vs52, vs0,vs29

  357.   xvmaddasp		vs53, vs1,vs29

  358.   xvmaddasp		vs44, vs0,vs27

  359.   xvmaddasp		vs45, vs1,vs27

  360.   xvmaddasp		vs60, vs0,vs31

  361.   xvmaddasp		vs61, vs1,vs31 

  362. .if \Complete==0

  363.   lxv	vs0,	DISP32(\Index,64+\OffsetA)(\AREG)

  364.   lxv	vs1,	DISP32(\Index,64+16+\OffsetA)(\AREG) 

  365. .endif

  366. 

  367.   xvmaddasp		vs34, vs2,vs24

  368.   xvmaddasp		vs35, vs3,vs24	  

  369.   xvmaddasp		vs50, vs2,vs28

  370.   xvmaddasp		vs51, vs3,vs28

  371. .if \Complete==0

  372.   lxv vs24, DISP16(\Index,32+\OffsetB)(\BREG)

  373.   lxv vs28, DISP16(\Index,32+16+\OffsetB)(\BREG)

  374. .endif  

  375.   xvmaddasp		vs42, vs2,vs26

  376.   xvmaddasp		vs43, vs3,vs26

  377.   xvmaddasp		vs58, vs2,vs30

  378.   xvmaddasp		vs59, vs3,vs30

  379. .if \Complete==0

  380.   xxperm    vs26, vs24, permute_mask

  381.   xxperm    vs30, vs28, permute_mask  

  382. .endif  

  383.   xvmaddasp		vs38, vs2,vs25

  384.   xvmaddasp		vs39, vs3,vs25

  385.   xvmaddasp		vs54, vs2,vs29

  386.   xvmaddasp		vs55, vs3,vs29

  387. .if \Complete==0

  388.   xxpermdi  vs25, vs24, vs24,2 

  389.   xxpermdi  vs29, vs28, vs28,2    

  390. .endif  

  391.   xvmaddasp		vs46, vs2,vs27

  392.   xvmaddasp		vs47, vs3,vs27

  393.   xvmaddasp		vs62, vs2,vs31	

  394.   xvmaddasp		vs63, vs3,vs31

  395. .if \Complete==0

  396.   xxpermdi  vs27, vs26, vs26,2  

  397.   xxpermdi  vs31, vs30, vs30,2   

  398. .endif

  399. .if \Complete==0

  400.   lxv	vs2,	DISP32(\Index,64+32+\OffsetA)(\AREG)

  401.   lxv	vs3,	DISP32(\Index,64+48+\OffsetA)(\AREG)

  402. .endif

  403. 

  404. 

  405. .if \IsLast==1	

  406. .if \Complete==1

  407. 	addi		\BREG, \BREG,  DISP16(\Index,\OffsetB)

  408.   addi    \AREG, \AREG, DISP32(\Index,\OffsetA)  

  409. 

  410. .else

  411. 	addi		\BREG, \BREG,  DISP16(\Index,64)

  412.   addi    \AREG, \AREG, DISP32(\Index,128)  

  413. 

  414. .endif

  415. .endif   

  416. 

  417. 

  418. .endm

  419. 

  420.  

  421. .macro SAVE8x16

  422. 

  423.   slwi    T10, LDC ,   1 

  424.   add     T1, CO, LDC 

  425. 

  426.   add     T2, CO, T10  

  427.   add     T3, T1, T10  

  428. 

  429.   add     T4, T2, T10  

  430.   add     T5, T3, T10 

  431. 

  432.   add     T6, T4, T10 

  433.   add     T7, T5, T10 

  434. 

  435. 

  436. 

  437.    /* permute to restore butterfly rank 1 updateto normal promoted one */  

  438.     /* permute 16 vs8 MEM(CO) vs9 MEM(CO+LDC) vs10 MEM(CO+2*LDC)  vs11 MEM(CO+3*LDC) */

  439.     /* permute 16 vs12 MEM(16+CO) vs13 MEM(16+CO+LDC) vs14 MEM(16+CO+2*LDC)  vs15 MEM(16+CO+3*LDC) */

  440.     /* permute 16 vs16 MEM(32+CO) vs17 MEM(32+CO+LDC) vs18 MEM(32+CO+2*LDC)  vs19 MEM(32+CO+3*LDC) */

  441.     /* permute 16 vs24 MEM(32+CO) vs25 MEM(32+CO+LDC) vs26 MEM(32+CO+2*LDC)  vs27 MEM(32+CO+3*LDC) */

  442. 

  443.     xxmrglw     vs8,    vs32,   vs44

  444.     xxmrglw     vs10,   vs36,   vs40  

  445. 

  446.     xxmrghw     vs1,    vs32,   vs44

  447.     xxmrghw     vs0,    vs36,   vs40

  448. 

  449.     xxmrglw     vs12,   vs33,   vs45

  450.     xxmrglw     vs14,   vs37,   vs41  

  451. 

  452.     xxmrghw     vs2,    vs37,   vs41

  453.     xxmrghw     vs3,    vs33,   vs45

  454. #ifndef TRMMKERNEL    

  455.     lxv        vs32, 0(CO)

  456.     lxv        vs33, 16(CO) 

  457. #endif 

  458.     xxmrglw     vs16,   vs34,   vs46

  459.     xxmrglw     vs18,   vs38,   vs42   

  460. 

  461.     xxlor      vs9, vs8,    vs8

  462.     xxlor      vs11,    vs10,   vs10 

  463. 

  464.     xxmrghw     vs4,    vs38,   vs42

  465.     xxmrghw     vs5,    vs34,   vs46

  466. 

  467.     xxlor      vs13,    vs12,   vs12

  468.     xxlor      vs15,    vs14,   vs14

  469. 

  470.     xxmrglw     vs24,   vs35,   vs47

  471.     xxmrglw     vs26,   vs39,   vs43  

  472. 

  473.     xxlor      vs17,    vs16,   vs16

  474.     xxlor      vs19,    vs18,   vs18

  475. 

  476.     xxmrghw     vs30,   vs39,   vs43 

  477.     xxmrghw     vs31,   vs35,   vs47

  478. #ifndef TRMMKERNEL       

  479.     lxv        vs34, 32(CO)  

  480.     lxv        vs35, 48(CO)      

  481. #endif

  482.     xxperm      vs8,    vs0,    save_permute_1

  483.     xxperm      vs10,   vs1,    save_permute_1

  484. #ifndef TRMMKERNEL    

  485.     lxv        vs36, 0(T1)

  486.     lxv        vs37, 16(T1) 

  487. #endif

  488.     xxperm      vs9,    vs0,    save_permute_2  

  489.     xxperm      vs11,   vs1,    save_permute_2      

  490. 

  491. #ifndef TRMMKERNEL      

  492.     lxv        vs38, 32(T1)  

  493.     lxv        vs39, 48(T1)     

  494. #endif

  495. 

  496.     xxlor      vs25,    vs24,   vs24

  497.     xxlor      vs27,    vs26,   vs26 

  498. 

  499. 

  500. 

  501. #ifndef TRMMKERNEL       

  502.     lxv        vs40, 0(T2)

  503.     lxv        vs41, 16(T2) 

  504. #endif

  505. 

  506.     xxperm     vs12,    vs2,    save_permute_1

  507.     xxperm     vs14,    vs3,    save_permute_1

  508. #ifndef TRMMKERNEL     

  509.     lxv        vs42, 32(T2)  

  510.     lxv        vs43, 48(T2)     

  511. #endif  

  512.        

  513.     xxperm     vs13,    vs2,    save_permute_2   

  514.     xxperm     vs15,    vs3,    save_permute_2  

  515. #ifndef TRMMKERNEL    

  516.     lxv        vs44, 0(T3)

  517.     lxv        vs45, 16(T3)

  518. #endif

  519.     xxperm     vs16,    vs4,    save_permute_1

  520.     xxperm     vs18,    vs5,    save_permute_1

  521. #ifndef TRMMKERNEL      

  522.     lxv        vs46, 32(T3)  

  523.     lxv        vs47, 48(T3)                 

  524. #endif  

  525. 

  526.     

  527. 

  528. 

  529.       

  530.     xxperm     vs17,    vs4,    save_permute_2   

  531.     xxperm     vs19,    vs5,    save_permute_2      

  532. #ifdef TRMMKERNEL

  533.     xvmulsp     vs32,   vs8,    alpha_r 

  534.     xvmulsp     vs33,   vs12,   alpha_r                 

  535. #else 

  536.     xvmaddasp   vs32,   vs8,    alpha_r 

  537.     xvmaddasp   vs33,   vs12,   alpha_r            

  538. #endif 

  539.     xxperm     vs24,    vs30,   save_permute_1

  540.     xxperm     vs26,    vs31,   save_permute_1 

  541. 

  542.  

  543.     stxv        vs32, 0(CO)

  544.     stxv        vs33, 16(CO)     

  545. #ifdef TRMMKERNEL   

  546.     xvmulsp     vs34,   vs16,   alpha_r 

  547.     xvmulsp     vs35,   vs24,   alpha_r                 

  548. #else    

  549.     xvmaddasp   vs34,   vs16,   alpha_r 

  550.     xvmaddasp   vs35,   vs24,   alpha_r           

  551. #endif 

  552.          

  553.     xxperm     vs25,    vs30,   save_permute_2   

  554.     xxperm     vs27,    vs31,   save_permute_2  

  555. 

  556. 

  557.     stxv        vs34, 32(CO)  

  558.     stxv        vs35, 48(CO)  

  559. #ifdef TRMMKERNEL  

  560.     xvmulsp     vs36,   vs9,    alpha_r 

  561.     xvmulsp     vs37,   vs13,   alpha_r                

  562. #else   

  563.     xvmaddasp   vs36,   vs9,    alpha_r 

  564.     xvmaddasp   vs37,   vs13,   alpha_r           

  565. #endif 

  566.     stxv        vs36, 0(T1)

  567.     stxv        vs37, 16(T1)

  568. #ifdef TRMMKERNEL  

  569.     xvmulsp     vs38,   vs17,   alpha_r 

  570.     xvmulsp     vs39,   vs25,   alpha_r               

  571. #else   

  572.     xvmaddasp   vs38,   vs17,   alpha_r 

  573.     xvmaddasp   vs39,   vs25,   alpha_r         

  574. #endif 

  575.     stxv        vs38, 32(T1)  

  576.     stxv        vs39, 48(T1)

  577. 

  578. #ifdef TRMMKERNEL

  579.     xvmulsp     vs40,   vs10,   alpha_r 

  580.     xvmulsp     vs41,   vs14,   alpha_r                    

  581. #else 

  582.     xvmaddasp   vs40,   vs10,   alpha_r 

  583.     xvmaddasp   vs41,   vs14,   alpha_r   

  584. #endif   

  585. 

  586.     stxv        vs40, 0(T2)

  587.     stxv        vs41, 16(T2)  

  588. #ifdef TRMMKERNEL 

  589.     xvmulsp     vs42,   vs18,   alpha_r 

  590.     xvmulsp     vs43,   vs26,   alpha_r                     

  591. #else   

  592.     xvmaddasp   vs42,   vs18,   alpha_r 

  593.     xvmaddasp   vs43,   vs26,   alpha_r

  594. #endif      

  595.     stxv        vs42, 32(T2)  

  596.     stxv        vs43, 48(T2)  

  597. #ifdef TRMMKERNEL  

  598.     xvmulsp     vs44,   vs11,   alpha_r 

  599.     xvmulsp     vs45,   vs15,   alpha_r                    

  600. #else

  601.     xvmaddasp   vs44,   vs11,   alpha_r 

  602.     xvmaddasp   vs45,   vs15,   alpha_r    

  603. #endif      

  604.     stxv        vs44, 0(T3)

  605.     stxv        vs45, 16(T3) 

  606. #ifdef TRMMKERNEL 

  607.     xvmulsp     vs46,   vs19,   alpha_r 

  608.     xvmulsp     vs47,   vs27,   alpha_r                   

  609. #else 

  610.     xvmaddasp   vs46,   vs19,   alpha_r 

  611.     xvmaddasp   vs47,   vs27,   alpha_r 

  612. #endif      

  613.     stxv        vs46, 32(T3)  

  614.     stxv        vs47, 48(T3)

  615.   

  616.  /*****the same with the second 8X8 ****/

  617.  #ifndef TRMMKERNEL 

  618.     lxv        vs32, 0(T4)

  619.     lxv        vs33, 16(T4) 

  620. #endif  

  621.     xxmrglw     vs8,    vs48,   vs60

  622.     xxmrglw     vs10,   vs52,   vs56  

  623. #ifndef TRMMKERNEL    

  624.     lxv        vs34, 32(T4)  

  625.     lxv        vs35, 48(T4)  

  626. #endif  

  627.     xxmrghw     vs1,    vs48,   vs60

  628.     xxmrghw     vs0,    vs52,   vs56

  629. #ifndef TRMMKERNEL        

  630.     lxv        vs36, 0(T5)

  631.     lxv        vs37, 16(T5) 

  632. #endif  

  633.     xxmrglw     vs12,   vs49,   vs61

  634.     xxmrglw     vs14,   vs53,   vs57  

  635. #ifndef TRMMKERNEL    

  636.     lxv        vs38,32(T5)  

  637.     lxv        vs39, 48(T5)     

  638. #endif   

  639.  

  640.     xxmrghw     vs2,    vs53,   vs57

  641.     xxmrghw     vs3,    vs49,   vs61

  642. #ifndef TRMMKERNEL   

  643.     lxv        vs40, 0(T6)

  644.     lxv        vs41, 16(T6)

  645. #endif  

  646.     xxmrglw     vs16,   vs50,   vs62

  647.     xxmrglw     vs18,   vs54,   vs58   

  648. #ifndef TRMMKERNEL      

  649.     lxv        vs42, 32(T6)  

  650.     lxv        vs43, 48(T6) 

  651. #endif  

  652.     xxlor      vs9, vs8,    vs8

  653.     xxlor      vs11,    vs10,   vs10 

  654.     xxmrghw     vs4,    vs54,   vs58

  655.     xxmrghw     vs5,    vs50,   vs62

  656. #ifndef TRMMKERNEL              

  657.     lxv        vs44, 0(T7)

  658.     lxv        vs45, 16(T7) 

  659. #endif  

  660.     xxlor      vs13,    vs12,   vs12

  661.     xxlor      vs15,    vs14,   vs14

  662.  

  663.     xxmrglw     vs24,   vs51,   vs63

  664.     xxmrglw     vs26,   vs55,   vs59 

  665. #ifndef TRMMKERNEL    

  666.     lxv        vs46, 32(T7)  

  667.     lxv        vs47, 48(T7)     

  668. #endif  

  669.     xxlor      vs17,    vs16,   vs16

  670.     xxlor      vs19,    vs18,   vs18

  671.     xxmrghw     vs30,   vs55,   vs59 

  672.     xxmrghw     vs31,   vs51,   vs63 

  673. 

  674.  

  675. 

  676.     xxperm      vs8,    vs0,    save_permute_1

  677.     xxperm      vs10,   vs1,    save_permute_1

  678.      

  679.     xxperm      vs9,    vs0,    save_permute_2  

  680.     xxperm      vs11,   vs1,    save_permute_2      

  681. 

  682.     xxlor      vs25,    vs24,   vs24

  683.     xxlor      vs27,    vs26,   vs26 

  684.     xxperm     vs12,    vs2,    save_permute_1

  685.     xxperm     vs14,    vs3,    save_permute_1

  686. 

  687.     xxperm     vs13,    vs2,    save_permute_2   

  688.     xxperm     vs15,    vs3,    save_permute_2      

  689.  #ifdef TRMMKERNEL

  690.     xvmulsp     vs32,   vs8,    alpha_r 

  691.     xvmulsp     vs33,   vs12,   alpha_r                 

  692. #else 

  693.     xvmaddasp   vs32,   vs8,    alpha_r 

  694.     xvmaddasp   vs33,   vs12,   alpha_r            

  695. #endif  

  696.     xxperm     vs16,    vs4,    save_permute_1

  697.     xxperm     vs18,    vs5,    save_permute_1

  698.     stxv        vs32, 0(T4)

  699.     stxv        vs33, 16(T4) 

  700.     xxperm     vs17,    vs4,    save_permute_2   

  701.     xxperm     vs19,    vs5,    save_permute_2      

  702.     xxperm     vs24,    vs30,   save_permute_1

  703.     xxperm     vs26,    vs31,   save_permute_1 

  704.     xxperm     vs25,    vs30,   save_permute_2   

  705.     xxperm     vs27,    vs31,   save_permute_2      

  706. 

  707. #ifdef TRMMKERNEL   

  708.     xvmulsp     vs34,   vs16,   alpha_r 

  709.     xvmulsp     vs35,   vs24,   alpha_r                 

  710. #else    

  711.     xvmaddasp   vs34,   vs16,   alpha_r 

  712.     xvmaddasp   vs35,   vs24,   alpha_r           

  713. #endif 

  714.     stxv        vs34, 32(T4)  

  715.     stxv        vs35, 48(T4)  

  716. 

  717. #ifdef TRMMKERNEL  

  718.     xvmulsp     vs36,   vs9,    alpha_r 

  719.     xvmulsp     vs37,   vs13,   alpha_r                

  720. #else   

  721.     xvmaddasp   vs36,   vs9,    alpha_r 

  722.     xvmaddasp   vs37,   vs13,   alpha_r           

  723. #endif 

  724.     stxv        vs36, 0(T5)

  725.     stxv        vs37, 16(T5) 

  726. 

  727. #ifdef TRMMKERNEL  

  728.     xvmulsp     vs38,   vs17,   alpha_r 

  729.     xvmulsp     vs39,   vs25,   alpha_r               

  730. #else  

  731.     xvmaddasp   vs38,   vs17,   alpha_r 

  732.     xvmaddasp   vs39,   vs25,   alpha_r         

  733. #endif 

  734. 

  735. 

  736. 

  737.  

  738.     stxv        vs38, 32(T5)  

  739.     stxv        vs39, 48(T5)

  740. 

  741. 

  742. #ifdef TRMMKERNEL

  743.     xvmulsp     vs40,   vs10,   alpha_r 

  744.     xvmulsp     vs41,   vs14,   alpha_r                    

  745. #else 

  746.     xvmaddasp   vs40,   vs10,   alpha_r 

  747.     xvmaddasp   vs41,   vs14,   alpha_r   

  748. #endif  

  749.     stxv        vs40, 0(T6)

  750.     stxv        vs41, 16(T6) 

  751. #ifdef TRMMKERNEL 

  752.     xvmulsp     vs42,   vs18,   alpha_r 

  753.     xvmulsp     vs43,   vs26,   alpha_r                     

  754. #else   

  755.     xvmaddasp   vs42,   vs18,   alpha_r 

  756.     xvmaddasp   vs43,   vs26,   alpha_r

  757. #endif  

  758.     stxv        vs42, 32(T6)  

  759.     stxv        vs43, 48(T6)  

  760. #ifdef TRMMKERNEL  

  761.     xvmulsp     vs44,   vs11,   alpha_r 

  762.     xvmulsp     vs45,   vs15,   alpha_r                    

  763. #else

  764.     xvmaddasp   vs44,   vs11,   alpha_r 

  765.     xvmaddasp   vs45,   vs15,   alpha_r    

  766. #endif  

  767. 

  768.     stxv        vs44, 0(T7)

  769.     stxv        vs45, 16(T7) 

  770. #ifdef TRMMKERNEL 

  771.     xvmulsp     vs46,   vs19,   alpha_r 

  772.     xvmulsp     vs47,   vs27,   alpha_r                   

  773. #else 

  774.     xvmaddasp   vs46,   vs19,   alpha_r 

  775.     xvmaddasp   vs47,   vs27,   alpha_r 

  776. #endif  

  777.  

  778.     stxv        vs46, 32(T7)  

  779.     stxv        vs47, 48(T7)

  780.   

  781. 

  782.     addi CO,CO,64

  783. 

  784. 

  785. .endm

  786. 

  787. 

  788. 

  789. /**********************************************************************************************

  790. * Macros for N=8 and M=8

  791. **********************************************************************************************/

  792. 

  793. .macro LOAD8x8_1

  794.    LOAD8x8 1

  795. .endm

  796. 

  797. .macro LOAD8x8_0

  798.    LOAD8x8 0

  799. .endm

  800. 

  801. .macro KERNEL8x8_L1_L4  Index,IsLast

  802.   KERNEL8x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  803. .endm

  804. 

  805. .macro KERNEL8x8_I1_L4  OffsetA,OffsetB, Index,IsLast

  806.   KERNEL8x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  807. .endm

  808. 

  809. .macro KERNEL8x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  810.   KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  811. .endm

  812. 

  813. .macro KERNEL8x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  814.   KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  815. .endm

  816. .macro KERNEL8x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast

  817.    KERNEL8x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1

  818. .endm

  819. 

  820. .macro KERNEL8x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  821.   KERNEL8x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  822. .endm

  823. 

  824. .macro KERNEL8x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  825.   KERNEL8x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  826. .endm

  827. 

  828. .macro END8x8_NORMAL

  829.   END8x8 0, AO, BO, 32,32 

  830. .endm

  831. 

  832. .macro Zero8X8

  833.     xxlxor      vs32,   vs32,   vs32

  834.     xxlxor      vs33,   vs33,   vs33

  835.  

  836.     xxlxor      vs36,   vs36,   vs36

  837.     xxlxor      vs37,   vs37,   vs37

  838.  

  839.     xxlxor      vs40,   vs40,   vs40

  840.     xxlxor      vs41,   vs41,   vs41

  841.  

  842.     xxlxor      vs44,   vs44,   vs44

  843.     xxlxor      vs45,   vs45,   vs45

  844.  

  845.     xxlxor      vs48,   vs48,   vs48

  846.     xxlxor      vs49,   vs49,   vs49

  847.  

  848.     xxlxor      vs52,   vs52,   vs52

  849.     xxlxor      vs53,   vs53,   vs53

  850.  

  851.     xxlxor      vs56,   vs56,   vs56

  852.     xxlxor      vs57,   vs57,   vs57

  853.   

  854.     xxlxor      vs60,   vs60,   vs60

  855.     xxlxor      vs61,   vs61,   vs61

  856.     

  857. .endm

  858. 

  859. .macro LOAD8x8  Zero

  860. 

  861.     lxv vs24,   0(BO)

  862.     lxv vs28,   16(BO)

  863.     lxv vs0,     0(AO)

  864.     lxv vs1,    16(AO)

  865. 

  866.     xxperm      vs26,   vs24,       permute_mask

  867.     xxperm      vs30,   vs28,       permute_mask    

  868.     xxpermdi    vs25,   vs24,   vs24,2     

  869.     xxpermdi    vs29,   vs28,   vs28,2    

  870. 

  871.     xxpermdi    vs27,   vs26,   vs26,2  

  872.     xxpermdi    vs31,   vs30,   vs30,2      

  873. 

  874. .if \Zero==1 

  875.     xxlxor      vs32,   vs32,   vs32

  876.     xxlxor      vs33,   vs33,   vs33 

  877.     xxlxor      vs36,   vs36,   vs36

  878.     xxlxor      vs37,   vs37,   vs37

  879.     xxlxor      vs40,   vs40,   vs40

  880.     xxlxor      vs41,   vs41,   vs41 

  881.     xxlxor      vs44,   vs44,   vs44

  882.     xxlxor      vs45,   vs45,   vs45 

  883.     xxlxor      vs48,   vs48,   vs48

  884.     xxlxor      vs49,   vs49,   vs49 

  885.     xxlxor      vs52,   vs52,   vs52

  886.     xxlxor      vs53,   vs53,   vs53 

  887.     xxlxor      vs56,   vs56,   vs56

  888.     xxlxor      vs57,   vs57,   vs57  

  889.     xxlxor      vs60,   vs60,   vs60

  890.     xxlxor      vs61,   vs61,   vs61  

  891. .endif

  892. .endm

  893. 

  894. 

  895. .macro END8x8 First, AREG, BREG, OffsetA, OffsetB

  896. 

  897. .if \OffsetB != 0 

  898.     addi        \BREG, \BREG, \OffsetB 

  899. .endif

  900. .if \OffsetA != 0 

  901.     addi        \AREG, \AREG, \OffsetA 

  902. .endif  

  903. 

  904. .if \First==1

  905.     xvmulsp     vs32, vs0,vs24

  906.     xvmulsp     vs33, vs1,vs24

  907. 

  908.     xvmulsp     vs36, vs0,vs25

  909.     xvmulsp     vs37, vs1,vs25

  910. 

  911.     xvmulsp     vs40, vs0,vs26

  912.     xvmulsp     vs41, vs1,vs26

  913. 

  914.     xvmulsp     vs44, vs0,vs27

  915.     xvmulsp     vs45, vs1,vs27

  916. 

  917.     xvmulsp     vs48, vs0,vs28

  918.     xvmulsp     vs49, vs1,vs28

  919. 

  920.     xvmulsp     vs52, vs0,vs29

  921.     xvmulsp     vs53, vs1,vs29

  922. 

  923.     xvmulsp     vs56, vs0,vs30

  924.     xvmulsp     vs57, vs1,vs30

  925. 

  926.     xvmulsp     vs60, vs0,vs31

  927.     xvmulsp     vs61, vs1,vs31

  928. 

  929. .else

  930.     xvmaddasp       vs32, vs0,vs24

  931.     xvmaddasp       vs33, vs1,vs24

  932. 

  933.     xvmaddasp       vs36, vs0,vs25

  934.     xvmaddasp       vs37, vs1,vs25

  935. 

  936.     xvmaddasp       vs40, vs0,vs26

  937.     xvmaddasp       vs41, vs1,vs26

  938. 

  939.     xvmaddasp       vs44, vs0,vs27

  940.     xvmaddasp       vs45, vs1,vs27

  941. 

  942.     xvmaddasp       vs48, vs0,vs28

  943.     xvmaddasp       vs49, vs1,vs28

  944. 

  945.     xvmaddasp       vs52, vs0,vs29

  946.     xvmaddasp       vs53, vs1,vs29

  947. 

  948.     xvmaddasp       vs56, vs0,vs30

  949.     xvmaddasp       vs57, vs1,vs30

  950. 

  951.     xvmaddasp       vs60, vs0,vs31

  952.     xvmaddasp       vs61, vs1,vs31

  953. 

  954. .endif

  955. .endm  

  956. 

  957. .macro KERNEL8x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  958. 

  959.     lxv vs8,    DISP32(\Index, 0+\OffsetB)(\BREG)

  960.     lxv vs12,   DISP32(\Index,16+\OffsetB)(\BREG)

  961. 

  962.     lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)

  963.     lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)

  964. 

  965.     xxperm      vs10,   vs8,        permute_mask

  966.     xxperm      vs14,   vs12,       permute_mask    

  967. 

  968.     xvmaddasp       vs32, vs0,vs24

  969.     xvmaddasp       vs33, vs1,vs24

  970. 

  971.     xxpermdi    vs9,    vs8,    vs8,2    

  972.     xxpermdi    vs13,   vs12,   vs12,2   

  973. 

  974. 

  975.     xvmaddasp       vs36, vs0,vs25

  976.     xvmaddasp       vs37, vs1,vs25

  977. 

  978.     xxpermdi    vs11,   vs10,   vs10,2  

  979.     xxpermdi    vs15,   vs14,   vs14,2  

  980. 

  981.     xvmaddasp       vs40, vs0,vs26

  982.     xvmaddasp       vs41, vs1,vs26

  983. 

  984.     xvmaddasp       vs44, vs0,vs27

  985.     xvmaddasp       vs45, vs1,vs27

  986. 

  987.     xvmaddasp       vs48, vs0,vs28

  988.     xvmaddasp       vs49, vs1,vs28

  989. 

  990.     xvmaddasp       vs52, vs0,vs29

  991.     xvmaddasp       vs53, vs1,vs29

  992.     lxv vs24,   DISP32(\Index,32+\OffsetB)(\BREG)

  993.     lxv vs28,   DISP32(\Index,32+16+\OffsetB)(\BREG)

  994.     xvmaddasp       vs56, vs0,vs30

  995.     xvmaddasp       vs57, vs1,vs30

  996. 

  997.     xvmaddasp       vs60, vs0,vs31

  998.     xvmaddasp       vs61, vs1,vs31

  999. 

  1000.     xxperm      vs26,   vs24,       permute_mask

  1001.     xxperm      vs30,   vs28,       permute_mask    

  1002. 

  1003.     lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)

  1004.     lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)

  1005. 

  1006. 

  1007.     xxpermdi    vs25,   vs24,   vs24,2     

  1008.     xxpermdi    vs29,   vs28,   vs28,2    

  1009. 

  1010.     xvmaddasp       vs32, vs4,vs8

  1011.     xvmaddasp       vs33, vs5,vs8

  1012. 

  1013.     xvmaddasp       vs36, vs4,vs9

  1014.     xvmaddasp       vs37, vs5,vs9

  1015. 

  1016.     xxpermdi    vs27,   vs26,   vs26,2  

  1017.     xxpermdi    vs31,   vs30,   vs30,2      

  1018. 

  1019.     xvmaddasp       vs40, vs4,vs10

  1020.     xvmaddasp       vs41, vs5,vs10

  1021. 

  1022.     xvmaddasp       vs44, vs4,vs11

  1023.     xvmaddasp       vs45, vs5,vs11

  1024. 

  1025.     xvmaddasp       vs48, vs4,vs12

  1026.     xvmaddasp       vs49, vs5,vs12

  1027. 

  1028.     xvmaddasp       vs52, vs4,vs13

  1029.     xvmaddasp       vs53, vs5,vs13

  1030.     lxv vs8,    DISP32(\Index,64+\OffsetB)(\BREG)

  1031.     lxv vs12,   DISP32(\Index,64+16+\OffsetB)(\BREG)

  1032.     xvmaddasp       vs56, vs4,vs14

  1033.     xvmaddasp       vs57, vs5,vs14

  1034. 

  1035.     xvmaddasp       vs60, vs4,vs15

  1036.     xvmaddasp       vs61, vs5,vs15

  1037. 

  1038.     xxperm      vs10,   vs8,        permute_mask

  1039.     xxperm      vs14,   vs12,       permute_mask   

  1040.  

  1041. 

  1042.     lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)

  1043.     lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)

  1044. 

  1045.  

  1046.     xxpermdi    vs9,    vs8,    vs8,2    

  1047.     xxpermdi    vs13,   vs12,   vs12,2  

  1048. 

  1049.     xvmaddasp       vs32, vs0,vs24

  1050.     xvmaddasp       vs33, vs1,vs24

  1051. 

  1052.     xvmaddasp       vs36, vs0,vs25

  1053.     xvmaddasp       vs37, vs1,vs25

  1054. 

  1055.     xxpermdi    vs11,   vs10,   vs10,2  

  1056.     xxpermdi    vs15,   vs14,   vs14,2  

  1057. 

  1058.     xvmaddasp       vs40, vs0,vs26

  1059.     xvmaddasp       vs41, vs1,vs26

  1060. 

  1061.     xvmaddasp       vs44, vs0,vs27

  1062.     xvmaddasp       vs45, vs1,vs27

  1063. 

  1064.     xvmaddasp       vs48, vs0,vs28

  1065.     xvmaddasp       vs49, vs1,vs28

  1066. 

  1067.     xvmaddasp       vs52, vs0,vs29

  1068.     xvmaddasp       vs53, vs1,vs29

  1069. .if \Complete==0

  1070.     lxv vs24,   DISP32(\Index,96+\OffsetB)(\BREG)

  1071.     lxv vs28,   DISP32(\Index,96+16+\OffsetB)(\BREG)

  1072. .endif 

  1073.     xvmaddasp       vs56, vs0,vs30

  1074.     xvmaddasp       vs57, vs1,vs30

  1075. .if \Complete==0

  1076.     xxperm      vs26,   vs24,   permute_mask

  1077.     xxperm      vs30,   vs28,   permute_mask   

  1078. .endif 

  1079.     xvmaddasp       vs60, vs0,vs31

  1080.     xvmaddasp       vs61, vs1,vs31

  1081. 

  1082. 

  1083. .if \Complete==0

  1084.     lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)

  1085.     lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) 

  1086. .endif 

  1087. 

  1088. .if \Complete==0     

  1089.     xxpermdi    vs25,   vs24,   vs24,2 

  1090.     xxpermdi    vs29,   vs28,   vs28,2      

  1091. 

  1092. .endif 

  1093. .if \IsLast==1  

  1094. .if \Complete==1

  1095.   

  1096.     addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)

  1097.     addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)

  1098. .else

  1099.   

  1100.     addi        \BREG, \BREG,  DISP32(\Index,128)

  1101.     addi        \AREG, \AREG, DISP32(\Index,128)

  1102. .endif

  1103. .endif   

  1104.  

  1105.     xvmaddasp       vs32, vs4,vs8

  1106.     xvmaddasp       vs33, vs5,vs8

  1107. 

  1108.     xvmaddasp       vs36, vs4,vs9

  1109.     xvmaddasp       vs37, vs5,vs9

  1110. 

  1111. .if \Complete==0        

  1112.     xxpermdi    vs27,   vs26,   vs26,2  

  1113.     xxpermdi    vs31,   vs30,   vs30,2  

  1114.     

  1115. .endif

  1116.  

  1117.     xvmaddasp       vs40, vs4,vs10

  1118.     xvmaddasp       vs41, vs5,vs10

  1119. 

  1120.     xvmaddasp       vs44, vs4,vs11

  1121.     xvmaddasp       vs45, vs5,vs11

  1122. 

  1123.     xvmaddasp       vs48, vs4,vs12

  1124.     xvmaddasp       vs49, vs5,vs12

  1125. 

  1126.     xvmaddasp       vs52, vs4,vs13

  1127.     xvmaddasp       vs53, vs5,vs13

  1128. 

  1129.     xvmaddasp       vs56, vs4,vs14

  1130.     xvmaddasp       vs57, vs5,vs14

  1131. 

  1132.     xvmaddasp       vs60, vs4,vs15

  1133.     xvmaddasp       vs61, vs5,vs15

  1134. 

  1135. .endm

  1136. 

  1137. .macro KERNEL8x8 First

  1138. 

  1139.   LOAD8x8 0

  1140.   END8x8 \First, AO, BO, 32,32  

  1141. .endm

  1142. 

  1143. .macro KERNEL8x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

  1144.     

  1145.     lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG)

  1146.     lxv vs12,   DISP16(\Index,16+\OffsetB)(\BREG)

  1147. 

  1148.     lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)

  1149.     lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)

  1150. 

  1151.     xxperm      vs10,   vs8,        permute_mask

  1152.     xxperm      vs14,   vs12,       permute_mask    

  1153.     xxpermdi    vs9,    vs8,    vs8,2    

  1154.     xxpermdi    vs13,   vs12,   vs12,2   

  1155. .if \First==1

  1156.     xvmulsp     vs32, vs0,vs24

  1157.     xvmulsp     vs33, vs1,vs24

  1158. 

  1159.     xvmulsp     vs36, vs0,vs25

  1160.     xvmulsp     vs37, vs1,vs25

  1161. 

  1162. .else

  1163.     xvmaddasp       vs32, vs0,vs24

  1164.     xvmaddasp       vs33, vs1,vs24

  1165. 

  1166.     xvmaddasp       vs36, vs0,vs25

  1167.     xvmaddasp       vs37, vs1,vs25

  1168. 

  1169. .endif

  1170. 

  1171.     xxpermdi    vs11,   vs10,   vs10,2  

  1172.     xxpermdi    vs15,   vs14,   vs14,2  

  1173.  

  1174. .if \First==1  

  1175.     xvmulsp     vs40, vs0,vs26

  1176.     xvmulsp     vs41, vs1,vs26

  1177. 

  1178.     xvmulsp     vs44, vs0,vs27

  1179.     xvmulsp     vs45, vs1,vs27

  1180. 

  1181.     xvmulsp     vs48, vs0,vs28

  1182.     xvmulsp     vs49, vs1,vs28

  1183. 

  1184.     xvmulsp     vs52, vs0,vs29

  1185.     xvmulsp     vs53, vs1,vs29

  1186. 

  1187.     xvmulsp     vs56, vs0,vs30

  1188.     xvmulsp     vs57, vs1,vs30

  1189. 

  1190.     xvmulsp     vs60, vs0,vs31

  1191.     xvmulsp     vs61, vs1,vs31

  1192. 

  1193. .else 

  1194.     xvmaddasp       vs40, vs0,vs26

  1195.     xvmaddasp       vs41, vs1,vs26

  1196. 

  1197.     xvmaddasp       vs44, vs0,vs27

  1198.     xvmaddasp       vs45, vs1,vs27

  1199. 

  1200.     xvmaddasp       vs48, vs0,vs28

  1201.     xvmaddasp       vs49, vs1,vs28

  1202. 

  1203.     xvmaddasp       vs52, vs0,vs29

  1204.     xvmaddasp       vs53, vs1,vs29

  1205. 

  1206.     xvmaddasp       vs56, vs0,vs30

  1207.     xvmaddasp       vs57, vs1,vs30

  1208. 

  1209.     xvmaddasp       vs60, vs0,vs31

  1210.     xvmaddasp       vs61, vs1,vs31

  1211. 

  1212. .endif

  1213. .if \Complete==0

  1214.     lxv vs24,   DISP16(\Index,32+\OffsetB)(\BREG)

  1215.     lxv vs28,   DISP16(\Index,32+16+\OffsetB)(\BREG)

  1216. 

  1217.     lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)

  1218.     lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)

  1219. 

  1220.     xxperm      vs26,   vs24,   permute_mask

  1221.     xxperm      vs30,   vs28,   permute_mask    

  1222.     xxpermdi    vs25,   vs24,   vs24,2   

  1223.     xxpermdi    vs29,   vs28,   vs28,2  

  1224. .endif    

  1225. .if \IsLast==1  

  1226. .if \Complete==1

  1227.     addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB) 

  1228.     addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)

  1229. 

  1230. .else

  1231.     addi        \BREG, \BREG,  DISP16(\Index,64)

  1232.     addi        \AREG, \AREG,  DISP16(\Index,64) 

  1233. .endif

  1234. .endif

  1235. 

  1236. .if \First==1

  1237.     xvmulsp     vs32, vs4,vs8

  1238.     xvmulsp     vs33, vs5,vs8

  1239. 

  1240.     xvmulsp     vs36, vs4,vs9

  1241.     xvmulsp     vs37, vs5,vs9

  1242. 

  1243. .else

  1244.     xvmaddasp       vs32, vs4,vs8

  1245.     xvmaddasp       vs33, vs5,vs8

  1246. 

  1247.     xvmaddasp       vs36, vs4,vs9

  1248.     xvmaddasp       vs37, vs5,vs9

  1249. 

  1250. .endif 

  1251.  

  1252. .if \Complete==0        

  1253.     xxpermdi    vs27,   vs26,   vs26,2  

  1254.     xxpermdi    vs31,   vs30,   vs30,2  

  1255.  

  1256. .endif

  1257. .if \First==1  

  1258.     xvmulsp     vs40, vs4,vs10

  1259.     xvmulsp     vs41, vs5,vs10

  1260. 

  1261.     xvmulsp     vs44, vs4,vs11

  1262.     xvmulsp     vs45, vs5,vs11

  1263. 

  1264.     xvmulsp     vs48, vs4,vs12

  1265.     xvmulsp     vs49, vs5,vs12

  1266. 

  1267.     xvmulsp     vs52, vs4,vs13

  1268.     xvmulsp     vs53, vs5,vs13

  1269. 

  1270.     xvmulsp     vs56, vs4,vs14

  1271.     xvmulsp     vs57, vs5,vs14

  1272. 

  1273.     xvmulsp     vs60, vs4,vs15

  1274.     xvmulsp     vs61, vs5,vs15

  1275. 

  1276. .else 

  1277.     xvmaddasp       vs40, vs4,vs10

  1278.     xvmaddasp       vs41, vs5,vs10

  1279. 

  1280.     xvmaddasp       vs44, vs4,vs11

  1281.     xvmaddasp       vs45, vs5,vs11

  1282. 

  1283.     xvmaddasp       vs48, vs4,vs12

  1284.     xvmaddasp       vs49, vs5,vs12

  1285. 

  1286.     xvmaddasp       vs52, vs4,vs13

  1287.     xvmaddasp       vs53, vs5,vs13

  1288. 

  1289.     xvmaddasp       vs56, vs4,vs14

  1290.     xvmaddasp       vs57, vs5,vs14

  1291. 

  1292.     xvmaddasp       vs60, vs4,vs15

  1293.     xvmaddasp       vs61, vs5,vs15

  1294. 

  1295. .endif

  1296. 

  1297. .endm

  1298. 

  1299. 

  1300. .macro SAVE8x8 

  1301.  

  1302.   slwi    T10, LDC ,   1 

  1303.   add     T1, CO, LDC 

  1304. 

  1305.   add     T2, CO, T10  

  1306.   add     T3, T1, T10  

  1307. 

  1308.   add     T4, T2, T10  

  1309.   add     T5, T3, T10 

  1310. 

  1311.   add     T6, T4, T10 

  1312.   add     T7, T5, T10 

  1313. 

  1314. #ifndef TRMMKERNEL    

  1315.     lxv        vs34, 0(CO)

  1316.     lxv        vs35, 16(CO)      

  1317.     lxv        vs38, 0(T1)

  1318.     lxv        vs39, 16(T1)  

  1319.     lxv        vs42, 0(T2)

  1320.     lxv        vs43, 16(T2)     

  1321.     lxv        vs46, 0(T3)

  1322.     lxv        vs47, 16(T3)  

  1323. 

  1324.     lxv        vs50, 0(T4)

  1325.     lxv        vs51, 16(T4)      

  1326.     lxv        vs54, 0(T5)

  1327.     lxv        vs55, 16(T5)  

  1328.     lxv        vs58, 0(T6)

  1329.     lxv        vs59, 16(T6)     

  1330.     lxv        vs62, 0(T7)

  1331.     lxv        vs63, 16(T7) 

  1332. #endif  

  1333. 

  1334.     xxmrglw     vs8,    vs32,   vs44

  1335.     xxmrglw     vs10,   vs36,   vs40  

  1336. 

  1337.     xxmrghw     vs1,    vs32,   vs44

  1338.     xxmrghw     vs0,    vs36,   vs40

  1339. 

  1340.     xxmrglw     vs12,   vs33,   vs45

  1341.     xxmrglw     vs14,   vs37,   vs41  

  1342. 

  1343.     xxmrghw     vs2,    vs37,   vs41

  1344.     xxmrghw     vs3,    vs33,   vs45

  1345. 

  1346.     xxlor      vs9, vs8,    vs8

  1347.     xxlor      vs11,    vs10,   vs10 

  1348.  

  1349.     xxlor      vs13,    vs12,   vs12

  1350.     xxlor      vs15,    vs14,   vs14

  1351. 

  1352.     xxperm      vs8,    vs0,    save_permute_1

  1353.     xxperm      vs10,   vs1,    save_permute_1

  1354.     xxperm      vs9,    vs0,    save_permute_2  

  1355.     xxperm      vs11,   vs1,    save_permute_2      

  1356. 

  1357.     xxperm     vs12,    vs2,    save_permute_1

  1358.     xxperm     vs14,    vs3,    save_permute_1

  1359.       

  1360.     xxperm     vs13,    vs2,    save_permute_2   

  1361.     xxperm     vs15,    vs3,    save_permute_2      

  1362. 

  1363. 

  1364.     /* multiply add normal way */

  1365.  

  1366. #ifdef TRMMKERNEL

  1367.     xvmulsp     vs34,   vs8,    alpha_r 

  1368.     xvmulsp     vs35,   vs12,   alpha_r 

  1369.     xvmulsp     vs38,   vs9,    alpha_r 

  1370.     xvmulsp     vs39,   vs13,   alpha_r 

  1371.     xvmulsp     vs42,   vs10,   alpha_r 

  1372.     xvmulsp     vs43,   vs14,   alpha_r 

  1373.     xvmulsp     vs46,   vs11,   alpha_r 

  1374.     xvmulsp     vs47,   vs15,   alpha_r                    

  1375. #else 

  1376.     xvmaddasp   vs34,   vs8,    alpha_r 

  1377.     xvmaddasp   vs35,   vs12,   alpha_r 

  1378.     xvmaddasp   vs38,   vs9,    alpha_r 

  1379.     xvmaddasp   vs39,   vs13,   alpha_r  

  1380.     xvmaddasp   vs42,   vs10,   alpha_r 

  1381.     xvmaddasp   vs43,   vs14,   alpha_r   

  1382.     xvmaddasp   vs46,   vs11,   alpha_r 

  1383.     xvmaddasp   vs47,   vs15,   alpha_r                     

  1384. #endif     

  1385.  

  1386.    

  1387.     xxmrglw     vs8,    vs48,   vs60

  1388.     xxmrglw     vs10,   vs52,   vs56  

  1389. 

  1390.     xxmrghw     vs1,    vs48,   vs60

  1391.     xxmrghw     vs0,    vs52,   vs56

  1392.     stxv        vs34, 0(CO)

  1393.     stxv        vs35, 16(CO) 

  1394.     xxmrglw     vs12,   vs49,   vs61

  1395.     xxmrglw     vs14,   vs53,   vs57  

  1396.     stxv        vs38, 0(T1)

  1397.     stxv        vs39, 16(T1) 

  1398.     xxmrghw     vs2,    vs53,   vs57

  1399.     xxmrghw     vs3,    vs49,   vs61

  1400.     stxv        vs42, 0(T2)

  1401.     stxv        vs43, 16(T2)   

  1402.     xxlor      vs9, vs8,    vs8

  1403.     xxlor      vs11,    vs10,   vs10  

  1404.     stxv        vs46, 0(T3)

  1405.     stxv        vs47, 16(T3)  

  1406.     xxlor      vs13,    vs12,   vs12

  1407.     xxlor      vs15,    vs14,   vs14

  1408.    

  1409.     xxperm      vs8,    vs0,    save_permute_1

  1410.     xxperm      vs10,   vs1,    save_permute_1

  1411.     

  1412.  

  1413.     xxperm      vs9,    vs0,    save_permute_2  

  1414.     xxperm      vs11,   vs1,    save_permute_2      

  1415.  

  1416.     xxperm     vs12,    vs2,    save_permute_1

  1417.     xxperm     vs14,    vs3,    save_permute_1

  1418.     xxperm     vs13,    vs2,    save_permute_2   

  1419.     xxperm     vs15,    vs3,    save_permute_2      

  1420.     

  1421.  #ifdef TRMMKERNEL

  1422.     xvmulsp     vs50,   vs8,    alpha_r 

  1423.     xvmulsp     vs51,   vs12,   alpha_r 

  1424.     xvmulsp     vs54,   vs9,    alpha_r 

  1425.     xvmulsp     vs55,   vs13,   alpha_r 

  1426.     xvmulsp     vs58,   vs10,   alpha_r 

  1427.     xvmulsp     vs59,   vs14,   alpha_r 

  1428.     xvmulsp     vs62,   vs11,   alpha_r 

  1429.     xvmulsp     vs63,   vs15,   alpha_r                    

  1430. #else 

  1431.     xvmaddasp     vs50,   vs8,    alpha_r 

  1432.     xvmaddasp     vs51,   vs12,   alpha_r 

  1433.     xvmaddasp     vs54,   vs9,    alpha_r 

  1434.     xvmaddasp     vs55,   vs13,   alpha_r 

  1435.     xvmaddasp     vs58,   vs10,   alpha_r 

  1436.     xvmaddasp     vs59,   vs14,   alpha_r 

  1437.     xvmaddasp     vs62,   vs11,   alpha_r 

  1438.     xvmaddasp     vs63,   vs15,   alpha_r                     

  1439. #endif  

  1440. 

  1441.     stxv        vs50, 0(T4)

  1442.     stxv        vs51, 16(T4)      

  1443.     stxv        vs54, 0(T5)

  1444.     stxv        vs55, 16(T5)  

  1445.     stxv        vs58, 0(T6)

  1446.     stxv        vs59, 16(T6)     

  1447.     stxv        vs62, 0(T7)

  1448.     stxv        vs63, 16(T7)   

  1449. 

  1450.     addi CO,CO,32

  1451. 

  1452. .endm

  1453. 

  1454. 

  1455. /**********************************************************************************************

  1456. * Macros for N=8 and M=4

  1457. **********************************************************************************************/

  1458. 

  1459. .macro LOAD8x4_1

  1460.    LOAD8x4 1

  1461. .endm

  1462. 

  1463. .macro LOAD8x4_0

  1464.    LOAD8x4 0

  1465. .endm

  1466. 

  1467. .macro KERNEL8x4_L1_L4  Index,IsLast

  1468.   KERNEL8x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  1469. .endm

  1470. 

  1471. .macro KERNEL8x4_I1_L4  OffsetA,OffsetB, Index,IsLast

  1472.   KERNEL8x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  1473. .endm

  1474. 

  1475. .macro KERNEL8x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  1476.   KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  1477. .endm

  1478. 

  1479. .macro KERNEL8x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  1480.   KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  1481. .endm

  1482. .macro KERNEL8x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast

  1483.    KERNEL8x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1

  1484. .endm

  1485. 

  1486. .macro KERNEL8x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  1487.   KERNEL8x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  1488. .endm

  1489. 

  1490. .macro KERNEL8x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  1491.   KERNEL8x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  1492. .endm

  1493. 

  1494. .macro Zero8X4

  1495.     xxlxor      vs32,   vs32,   vs32

  1496.     xxlxor      vs33,   vs33,   vs33 

  1497.     xxlxor      vs34,   vs34,   vs34

  1498.     xxlxor      vs35,   vs35,   vs35

  1499.     

  1500.     xxlxor      vs48,   vs48,   vs48

  1501.     xxlxor      vs49,   vs49,   vs49

  1502.     xxlxor      vs50,   vs50,   vs50

  1503.     xxlxor      vs51,   vs51,   vs51  

  1504.     

  1505. .endm

  1506. 

  1507. .macro LOAD8x4  Zero

  1508. 

  1509.     lxv vs0,     0(AO)

  1510.     lxv vs24,   0(BO)

  1511.     lxv vs25,   16(BO)

  1512. 

  1513. 

  1514. 

  1515.     xxperm      vs2,   vs0,       permute_mask  

  1516.     xxpermdi    vs1,   vs0,   vs0,2      

  1517.     xxpermdi    vs3,   vs2,   vs2,2        

  1518. 

  1519. .if \Zero==1 

  1520.     xxlxor      vs32,   vs32,   vs32

  1521.     xxlxor      vs33,   vs33,   vs33 

  1522.     xxlxor      vs34,   vs34,   vs34

  1523.     xxlxor      vs35,   vs35,   vs35

  1524. 

  1525.     xxlxor      vs48,   vs48,   vs48

  1526.     xxlxor      vs49,   vs49,   vs49

  1527.     xxlxor      vs50,   vs50,   vs50

  1528.     xxlxor      vs51,   vs51,   vs51  

  1529. .endif

  1530. .endm

  1531. 

  1532. .macro END8x4_NORMAL

  1533.   END8x4 0, AO, BO, 16,32 

  1534. .endm

  1535. 

  1536. .macro END8x4 First, AREG, BREG, OffsetA, OffsetB

  1537. 

  1538. .if \OffsetB != 0 

  1539.     addi        \BREG, \BREG, \OffsetB 

  1540. .endif

  1541. .if \OffsetA != 0 

  1542.     addi        \AREG, \AREG, \OffsetA 

  1543. .endif  

  1544. 

  1545. .if \First==1

  1546.     xvmulsp      vs32,   vs24,   vs0

  1547.     xvmulsp      vs33,   vs24,   vs1 

  1548.     xvmulsp      vs34,   vs24,   vs2

  1549.     xvmulsp      vs35,   vs24,   vs3

  1550. 

  1551.     xvmulsp      vs48,   vs25,   vs0

  1552.     xvmulsp      vs49,   vs25,   vs1

  1553.     xvmulsp      vs50,   vs25,   vs2

  1554.     xvmulsp      vs51,   vs25,   vs3  

  1555. .else

  1556.     xvmaddasp      vs32,   vs24,   vs0

  1557.     xvmaddasp      vs33,   vs24,   vs1 

  1558.     xvmaddasp      vs34,   vs24,   vs2

  1559.     xvmaddasp      vs35,   vs24,   vs3

  1560. 

  1561.     xvmaddasp      vs48,   vs25,   vs0

  1562.     xvmaddasp      vs49,   vs25,   vs1

  1563.     xvmaddasp      vs50,   vs25,   vs2

  1564.     xvmaddasp      vs51,   vs25,   vs3 

  1565. 

  1566. .endif

  1567. .endm  

  1568. 

  1569. .macro KERNEL8x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  1570. 

  1571.     lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)

  1572.     lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)

  1573.     lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)

  1574. 

  1575.     xxperm      vs6,   vs4,       permute_mask  

  1576.     xxpermdi    vs5,   vs4,   vs4,2      

  1577.     xxpermdi    vs7,   vs6,   vs6,2 

  1578.  

  1579.     xvmaddasp      vs32,   vs24,   vs0

  1580.     xvmaddasp      vs33,   vs24,   vs1 

  1581.     xvmaddasp      vs34,   vs24,   vs2

  1582.     xvmaddasp      vs35,   vs24,   vs3

  1583. 

  1584.     xvmaddasp      vs48,   vs25,   vs0

  1585.     xvmaddasp      vs49,   vs25,   vs1

  1586.     xvmaddasp      vs50,   vs25,   vs2

  1587.     xvmaddasp      vs51,   vs25,   vs3 

  1588. 

  1589.     lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)

  1590.     lxv vs24,   DISP32(\Index, 32+\OffsetB)(\BREG)

  1591.     lxv vs25,   DISP32(\Index, 48+\OffsetB)(\BREG) 

  1592. 

  1593.     xxperm      vs2,   vs0,       permute_mask  

  1594.     xxpermdi    vs1,   vs0,   vs0,2      

  1595.     xxpermdi    vs3,   vs2,   vs2,2   

  1596. 

  1597.     xvmaddasp      vs32,   vs26,   vs4

  1598.     xvmaddasp      vs33,   vs26,   vs5 

  1599.     xvmaddasp      vs34,   vs26,   vs6

  1600.     xvmaddasp      vs35,   vs26,   vs7

  1601. 

  1602.     xvmaddasp      vs48,   vs27,   vs4

  1603.     xvmaddasp      vs49,   vs27,   vs5

  1604.     xvmaddasp      vs50,   vs27,   vs6

  1605.     xvmaddasp      vs51,   vs27,   vs7

  1606.  

  1607. 

  1608.     lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)

  1609.     lxv vs26,   DISP32(\Index, 64+\OffsetB)(\BREG)

  1610.     lxv vs27,   DISP32(\Index, 80+\OffsetB)(\BREG)

  1611. 

  1612.     xxperm      vs6,   vs4,       permute_mask  

  1613.     xxpermdi    vs5,   vs4,   vs4,2      

  1614.     xxpermdi    vs7,   vs6,   vs6,2 

  1615.  

  1616.     xvmaddasp      vs32,   vs24,   vs0

  1617.     xvmaddasp      vs33,   vs24,   vs1 

  1618.     xvmaddasp      vs34,   vs24,   vs2

  1619.     xvmaddasp      vs35,   vs24,   vs3

  1620. 

  1621.     xvmaddasp      vs48,   vs25,   vs0

  1622.     xvmaddasp      vs49,   vs25,   vs1

  1623.     xvmaddasp      vs50,   vs25,   vs2

  1624.     xvmaddasp      vs51,   vs25,   vs3 

  1625. 

  1626. .if \Complete==0 

  1627. 

  1628.     lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)

  1629.     lxv vs24,   DISP32(\Index, 96+\OffsetB)(\BREG)

  1630.     lxv vs25,   DISP32(\Index, 96+16+\OffsetB)(\BREG) 

  1631. 

  1632.     xxperm      vs2,   vs0,       permute_mask  

  1633.     xxpermdi    vs1,   vs0,   vs0,2      

  1634.     xxpermdi    vs3,   vs2,   vs2,2   

  1635. .endif

  1636.     xvmaddasp      vs32,   vs26,   vs4

  1637.     xvmaddasp      vs33,   vs26,   vs5 

  1638.     xvmaddasp      vs34,   vs26,   vs6

  1639.     xvmaddasp      vs35,   vs26,   vs7

  1640. 

  1641.     xvmaddasp      vs48,   vs27,   vs4

  1642.     xvmaddasp      vs49,   vs27,   vs5

  1643.     xvmaddasp      vs50,   vs27,   vs6

  1644.     xvmaddasp      vs51,   vs27,   vs7

  1645. 

  1646.  

  1647.  

  1648. .if \IsLast==1  

  1649. .if \Complete==1

  1650.     addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  

  1651.     addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)

  1652. 

  1653. .else

  1654.     addi        \AREG, \AREG, DISP16(\Index,64)  

  1655.     addi        \BREG, \BREG,  DISP32(\Index,128)

  1656. 

  1657. .endif

  1658. .endif   

  1659.  

  1660.  

  1661. .endm

  1662. 

  1663. .macro KERNEL8x4 First

  1664.     LOAD8x4 0

  1665.     END8x4 \First, AO, BO, 16,32  

  1666. .endm

  1667. 

  1668. .macro KERNEL8x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

  1669. 

  1670.     lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)

  1671.     lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)

  1672.     lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)

  1673. 

  1674.     xxperm      vs6,   vs4,       permute_mask  

  1675.     xxpermdi    vs5,   vs4,   vs4,2      

  1676.     xxpermdi    vs7,   vs6,   vs6,2 

  1677. .if \First==1

  1678.     xvmulsp      vs32,   vs24,   vs0

  1679.     xvmulsp      vs33,   vs24,   vs1 

  1680.     xvmulsp      vs34,   vs24,   vs2

  1681.     xvmulsp      vs35,   vs24,   vs3

  1682. 

  1683.     xvmulsp      vs48,   vs25,   vs0

  1684.     xvmulsp      vs49,   vs25,   vs1

  1685.     xvmulsp      vs50,   vs25,   vs2

  1686.     xvmulsp      vs51,   vs25,   vs3  

  1687. .else 

  1688.     xvmaddasp      vs32,   vs24,   vs0

  1689.     xvmaddasp      vs33,   vs24,   vs1 

  1690.     xvmaddasp      vs34,   vs24,   vs2

  1691.     xvmaddasp      vs35,   vs24,   vs3

  1692. 

  1693.     xvmaddasp      vs48,   vs25,   vs0

  1694.     xvmaddasp      vs49,   vs25,   vs1

  1695.     xvmaddasp      vs50,   vs25,   vs2

  1696.     xvmaddasp      vs51,   vs25,   vs3 

  1697. .endif

  1698. 

  1699. .if \Complete==0 

  1700. 

  1701.     lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)

  1702.     lxv vs24,   DISP16(\Index, 32+\OffsetB)(\BREG)

  1703.     lxv vs25,   DISP16(\Index, 48+\OffsetB)(\BREG) 

  1704. 

  1705.     xxperm      vs2,   vs0,       permute_mask  

  1706.     xxpermdi    vs1,   vs0,   vs0,2      

  1707.     xxpermdi    vs3,   vs2,   vs2,2   

  1708. .endif

  1709. 

  1710. .if \First==1

  1711.     xvmulsp      vs32,   vs26,   vs4

  1712.     xvmulsp      vs33,   vs26,   vs5 

  1713.     xvmulsp      vs34,   vs26,   vs6

  1714.     xvmulsp      vs35,   vs26,   vs7

  1715. 

  1716.     xvmulsp      vs48,   vs27,   vs4

  1717.     xvmulsp      vs49,   vs27,   vs5

  1718.     xvmulsp      vs50,   vs27,   vs6

  1719.     xvmulsp      vs51,   vs27,   vs7

  1720. 

  1721. 

  1722. .else

  1723.     xvmaddasp      vs32,   vs26,   vs4

  1724.     xvmaddasp      vs33,   vs26,   vs5 

  1725.     xvmaddasp      vs34,   vs26,   vs6

  1726.     xvmaddasp      vs35,   vs26,   vs7

  1727. 

  1728.     xvmaddasp      vs48,   vs27,   vs4

  1729.     xvmaddasp      vs49,   vs27,   vs5

  1730.     xvmaddasp      vs50,   vs27,   vs6

  1731.     xvmaddasp      vs51,   vs27,   vs7

  1732. .endif

  1733.  

  1734.  

  1735. .if \IsLast==1  

  1736. .if \Complete==1

  1737.     addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  

  1738.     addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB)

  1739. 

  1740. .else

  1741.     addi        \AREG, \AREG, DISP8(\Index,32)  

  1742.     addi        \BREG, \BREG,  DISP16(\Index,64)

  1743. 

  1744. .endif

  1745. .endif   

  1746.      

  1747.   

  1748. .endm

  1749. 

  1750. 

  1751. .macro SAVE8x4

  1752.   slwi    T10, LDC ,   1 

  1753.   add     T1, CO, LDC 

  1754. #if !defined(TRMMKERNEL)  

  1755.   lxv        vs36, 0(CO)

  1756.   lxv        vs37, 0(T1)

  1757. #endif  

  1758.   add     T2, CO, T10  

  1759.   add     T3, T1, T10 

  1760. #if !defined(TRMMKERNEL)    

  1761.   lxv        vs38, 0(T2)

  1762.   lxv        vs39, 0(T3)   

  1763. #endif   

  1764.   add     T4, T2, T10 

  1765.   add     T5, T3, T10

  1766. #if !defined(TRMMKERNEL)    

  1767.   lxv        vs40, 0(T4)

  1768.   lxv        vs41, 0(T5)

  1769. #endif  

  1770.   add     T6, T4, T10 

  1771.   add     T7, T5, T10

  1772. #if !defined(TRMMKERNEL)    

  1773.   lxv        vs42, 0(T6)

  1774.   lxv        vs43, 0(T7)

  1775. #endif

  1776.   xxmrglw  vs0, vs35,vs32

  1777.   xxmrglw  vs1, vs34,vs33 

  1778.   xxmrglw  vs4, vs32,vs35

  1779.   xxmrglw  vs5, vs33,vs34 

  1780. 

  1781. 

  1782.   xxmrghw  vs2, vs35,vs32

  1783.   xxmrghw  vs3, vs34,vs33 

  1784.   xxmrghw  vs6, vs32,vs35

  1785.   xxmrghw  vs7, vs33,vs34  

  1786. 

  1787.   xxmrgld  vs24, vs1, vs0  

  1788.   xxmrghd  vs25,vs5,vs4 

  1789. 

  1790.   xxmrgld  vs26, vs2, vs3  

  1791.   xxmrghd  vs27,vs6,vs7

  1792. 

  1793. 

  1794.   xxmrglw  vs0, vs51,vs48

  1795.   xxmrglw  vs1, vs50,vs49  

  1796.   xxmrglw  vs4, vs48,vs51

  1797.   xxmrglw  vs5, vs49,vs50 

  1798. 

  1799.   xxmrghw  vs2, vs51,vs48

  1800.   xxmrghw  vs3, vs50,vs49  

  1801.   xxmrghw  vs6, vs48,vs51

  1802.   xxmrghw  vs7, vs49,vs50   

  1803. 

  1804.   xxmrgld  vs28, vs1, vs0  

  1805.   xxmrghd  vs29,vs5,vs4

  1806. 

  1807.   xxmrgld  vs30, vs2, vs3   

  1808.   xxmrghd  vs31,vs6,vs7

  1809. #if defined(TRMMKERNEL)

  1810. 

  1811.   xvmulsp        vs36, vs24, alpha_r

  1812.   xvmulsp        vs37, vs25, alpha_r 

  1813.   xvmulsp        vs38, vs26, alpha_r

  1814.   xvmulsp        vs39, vs27, alpha_r   

  1815.   xvmulsp        vs40, vs28, alpha_r

  1816.   xvmulsp        vs41, vs29, alpha_r 

  1817.   xvmulsp        vs42, vs30, alpha_r

  1818.   xvmulsp        vs43, vs31, alpha_r

  1819. #else

  1820.   xvmaddasp        vs36, vs24, alpha_r

  1821.   xvmaddasp        vs37, vs25, alpha_r 

  1822.   xvmaddasp        vs38, vs26, alpha_r

  1823.   xvmaddasp        vs39, vs27, alpha_r   

  1824.   xvmaddasp        vs40, vs28, alpha_r

  1825.   xvmaddasp        vs41, vs29, alpha_r 

  1826.   xvmaddasp        vs42, vs30, alpha_r

  1827.   xvmaddasp        vs43, vs31, alpha_r

  1828. #endif

  1829. 

  1830.   stxv        vs36, 0(CO)

  1831.   stxv        vs37, 0(T1) 

  1832.   stxv        vs38, 0(T2)

  1833.   stxv        vs39, 0(T3)   

  1834.   stxv        vs40, 0(T4)

  1835.   stxv        vs41, 0(T5) 

  1836.   stxv        vs42, 0(T6)

  1837.   stxv        vs43, 0(T7)

  1838. 

  1839. 

  1840.   addi CO,CO,16

  1841. .endm

  1842. 

  1843. 

  1844. /**********************************************************************************************

  1845. * Macros for N=8 and M=2

  1846. **********************************************************************************************/

  1847. 

  1848.  

  1849. .macro KERNEL8x2_2   OffsetA,OffsetB, Index,IsLast

  1850.   KERNEL8x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast

  1851. .endm

  1852. 

  1853.  

  1854. 

  1855. .macro Zero8x2

  1856.     xxlxor      vs0,   vs0,   vs0

  1857.     xxlxor      vs1,   vs1,   vs1 

  1858.     xxlxor      vs2,   vs2,   vs2

  1859.     xxlxor      vs3,   vs3,   vs3

  1860.        

  1861. .endm

  1862.  

  1863. .macro KERNEL8x2

  1864.   KERNEL8x2_1 AO,BO, 0, 0,0,0

  1865. .endm

  1866. .macro KERNEL8x2_1 AREG,BREG,First,OffsetA,OffsetB,Index

  1867. 

  1868. 

  1869.     lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)

  1870.     lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)

  1871.     lxv vs27,   DISP8(\Index,16+\OffsetB)(\BREG)      

  1872.     xxspltw   vs8,  vs36, 0 

  1873.     xxspltw   vs9,  vs36, 1  

  1874.  

  1875. .if \First==1

  1876.     xvmulsp      vs0,   vs26,   vs8

  1877.     xvmulsp      vs1,   vs27,   vs8 

  1878.     xvmulsp      vs2,   vs26,   vs9

  1879.     xvmulsp      vs3,   vs27,   vs9 

  1880.      

  1881. .else 

  1882.     xvmaddasp      vs0,   vs26,   vs8

  1883.     xvmaddasp      vs1,   vs27,   vs8 

  1884.     xvmaddasp      vs2,   vs26,   vs9

  1885.     xvmaddasp      vs3,   vs27,   vs9

  1886.  

  1887.  .endif

  1888.    

  1889.     addi        \AREG, \AREG, DISP2(\Index,8)  

  1890.     addi        \BREG, \BREG, DISP8(\Index,32)

  1891.  

  1892. .endm

  1893. 

  1894. .macro KERNEL8x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

  1895. 

  1896.     lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)

  1897.     lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)

  1898.     lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)

  1899.     lxv vs28,   DISP16(\Index,32+\OffsetB)(\BREG)

  1900.     lxv vs29,   DISP16(\Index,48+\OffsetB)(\BREG)      

  1901.     xxspltw   vs8,  vs4, 2  

  1902.     xxspltw   vs9,  vs4, 3 

  1903.     xxspltw   vs10, vs4, 0 

  1904.     xxspltw   vs11, vs4, 1

  1905.  

  1906. .if \First==1

  1907.     xvmulsp      vs0,   vs26,   vs8

  1908.     xvmulsp      vs1,   vs27,   vs8 

  1909.     xvmulsp      vs2,   vs26,   vs9

  1910.     xvmulsp      vs3,   vs27,   vs9 

  1911. 

  1912.     xvmulsp      vs0,   vs28,   vs10

  1913.     xvmulsp      vs1,   vs29,   vs10 

  1914.     xvmulsp      vs2,   vs28,   vs11

  1915.     xvmulsp      vs3,   vs29,   vs11     

  1916. .else 

  1917.     xvmaddasp      vs0,   vs26,   vs8

  1918.     xvmaddasp      vs1,   vs27,   vs8 

  1919.     xvmaddasp      vs2,   vs26,   vs9

  1920.     xvmaddasp      vs3,   vs27,   vs9

  1921. 

  1922.     xvmaddasp      vs0,   vs28,   vs10

  1923.     xvmaddasp      vs1,   vs29,   vs10 

  1924.     xvmaddasp      vs2,   vs28,   vs11

  1925.     xvmaddasp      vs3,   vs29,   vs11  

  1926.  .endif

  1927. 

  1928.  

  1929. .if \IsLast==1   

  1930.     addi        \AREG, \AREG, DISP4(\Index,16)  

  1931.     addi        \BREG, \BREG, DISP16(\Index,64)

  1932. .endif 

  1933.   

  1934. .endm

  1935. 

  1936. 

  1937. .macro SAVE8x2

  1938.   slwi    T10, LDC ,   1 

  1939.   add     T1, CO, LDC  

  1940.   add     T2, CO, T10  

  1941.   add     T3, T1, T10     

  1942.   add     T4, T2, T10 

  1943.   add     T5, T3, T10 

  1944.   add     T6, T4, T10 

  1945.   add     T7, T5, T10 

  1946.   /*convert alpha_r for multiply*/

  1947.   xscvspdp  vs4,alpha_r

  1948. /* v0 corresponds to vs32, do not forget*/

  1949. #if !defined(TRMMKERNEL)

  1950.   lxssp  v0,0(CO) 

  1951.   lxssp  v1,4(CO) 

  1952. 

  1953.   lxssp  v2,0(T1)

  1954.   lxssp  v3,4(T1)

  1955. 

  1956.   lxssp  v4,0(T2)

  1957.   lxssp  v5,4(T2)

  1958. 

  1959.   lxssp  v6,0(T3)

  1960.   lxssp  v7,4(T3)

  1961. 

  1962.   lxssp  v8,0(T4)

  1963.   lxssp  v9,4(T4)

  1964. 

  1965.   lxssp  v10,0(T5)

  1966.   lxssp  v11,4(T5)

  1967. 

  1968.   lxssp  v12,0(T6)

  1969.   lxssp  v13,4(T6)

  1970. 

  1971.   lxssp  v14,0(T7)

  1972.   lxssp  v15,4(T7)

  1973. #endif

  1974.   xscvspdp  vs5, vs2

  1975.   xxspltw   vs6, vs2, 1 

  1976.   xxspltw   vs7, vs2, 2 

  1977.   xxspltw   vs8, vs2, 3  

  1978.   xscvspdp  vs6,vs6

  1979.   xscvspdp  vs7,vs7

  1980.   xscvspdp  vs8,vs8

  1981. 

  1982.   xscvspdp  vs24, vs0

  1983.   xxspltw   vs25, vs0, 1 

  1984.   xxspltw   vs26, vs0, 2 

  1985.   xxspltw   vs27, vs0, 3  

  1986.   xscvspdp  vs25,vs25

  1987.   xscvspdp  vs26,vs26

  1988.   xscvspdp  vs27,vs27

  1989. 

  1990.   xscvspdp  vs9, vs3

  1991.   xxspltw   vs10, vs3, 1 

  1992.   xxspltw   vs11, vs3, 2 

  1993.   xxspltw   vs12, vs3, 3  

  1994.   xscvspdp  vs10,vs10

  1995.   xscvspdp  vs11,vs11

  1996.   xscvspdp  vs12,vs12

  1997. 

  1998.   xscvspdp  vs28, vs1

  1999.   xxspltw   vs29, vs1, 1 

  2000.   xxspltw   vs30, vs1, 2 

  2001.   xxspltw   vs31, vs1, 3  

  2002.   xscvspdp  vs29,vs29

  2003.   xscvspdp  vs30,vs30

  2004.   xscvspdp  vs31,vs31

  2005. 

  2006. 

  2007. 

  2008. 

  2009. #if defined(TRMMKERNEL)

  2010.   xsmuldp  vs32,vs8, vs4 

  2011.   xsmuldp  vs33,vs27, vs4 

  2012. 

  2013.   xsmuldp  vs34,vs7, vs4 

  2014.   xsmuldp  vs35,vs26, vs4 

  2015. 

  2016.   xsmuldp  vs36,vs6, vs4 

  2017.   xsmuldp  vs37,vs25, vs4  

  2018. 

  2019.   xsmuldp  vs38,vs5, vs4 

  2020.   xsmuldp  vs39,vs24, vs4  

  2021. 

  2022.   xsmuldp  vs40,vs12, vs4 

  2023.   xsmuldp  vs41,vs31, vs4

  2024. 

  2025.   xsmuldp  vs42,vs11, vs4 

  2026.   xsmuldp  vs43,vs30, vs4  

  2027. 

  2028.   xsmuldp  vs44,vs10, vs4 

  2029.   xsmuldp  vs45,vs29, vs4 

  2030. 

  2031.   xsmuldp  vs46,vs9, vs4 

  2032.   xsmuldp  vs47,vs28, vs4      

  2033. #else

  2034.   xsmaddadp  vs32,vs8, vs4 

  2035.   xsmaddadp  vs33,vs27, vs4 

  2036. 

  2037.   xsmaddadp  vs34,vs7, vs4 

  2038.   xsmaddadp  vs35,vs26, vs4 

  2039. 

  2040.   xsmaddadp  vs36,vs6, vs4 

  2041.   xsmaddadp  vs37,vs25, vs4  

  2042. 

  2043.   xsmaddadp  vs38,vs5, vs4 

  2044.   xsmaddadp  vs39,vs24, vs4  

  2045. 

  2046.   xsmaddadp  vs40,vs12, vs4 

  2047.   xsmaddadp  vs41,vs31, vs4

  2048. 

  2049.   xsmaddadp  vs42,vs11, vs4 

  2050.   xsmaddadp  vs43,vs30, vs4  

  2051. 

  2052.   xsmaddadp  vs44,vs10, vs4 

  2053.   xsmaddadp  vs45,vs29, vs4 

  2054. 

  2055.   xsmaddadp  vs46,vs9, vs4 

  2056.   xsmaddadp  vs47,vs28, vs4     

  2057. #endif  

  2058. 

  2059.   stxssp  v0,0(CO) 

  2060.   stxssp  v1,4(CO) 

  2061. 

  2062.   stxssp  v2,0(T1)

  2063.   stxssp  v3,4(T1)

  2064. 

  2065.   stxssp  v4,0(T2)

  2066.   stxssp  v5,4(T2)

  2067. 

  2068.   stxssp  v6,0(T3)

  2069.   stxssp  v7,4(T3)

  2070. 

  2071.   stxssp  v8,0(T4)

  2072.   stxssp  v9,4(T4)

  2073. 

  2074.   stxssp  v10,0(T5)

  2075.   stxssp  v11,4(T5)

  2076. 

  2077.   stxssp  v12,0(T6)

  2078.   stxssp  v13,4(T6)

  2079. 

  2080.   stxssp  v14,0(T7)

  2081.   stxssp  v15,4(T7)

  2082.  

  2083. 

  2084.   addi CO,CO,8

  2085. .endm

  2086. 

  2087. 

  2088. /**********************************************************************************************

  2089. * Macros for N=8 and M=1

  2090. **********************************************************************************************/

  2091. .macro KERNEL8x1_4   OffsetA,OffsetB, Index,IsLast

  2092.   KERNEL8x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast

  2093. .endm

  2094. 

  2095. .macro Zero8x1

  2096.     xxlxor      vs0,   vs0,   vs0

  2097.     xxlxor      vs1,   vs1,   vs1  

  2098. .endm

  2099. 

  2100. .macro KERNEL8x1

  2101.   KERNEL8x1_1 AO,BO, 0 

  2102. .endm

  2103. 

  2104. .macro KERNEL8x1_2

  2105.   KERNEL8x1_2_1 AO,BO, 0 

  2106. .endm

  2107. 

  2108. .macro KERNEL8x1_1 AREG,BREG,First 

  2109.     lxvwsx vs8,  0, \AREG

  2110.     lxv vs26,   0(\BREG)

  2111.     lxv vs27,   16(\BREG)      

  2112. .if \First==1

  2113.     xvmulsp      vs0,   vs26,   vs8

  2114.     xvmulsp      vs1,   vs27,   vs8  

  2115. .else 

  2116.     xvmaddasp      vs0,   vs26,   vs8

  2117.     xvmaddasp      vs1,   vs27,   vs8  

  2118.  .endif

  2119.     addi        \AREG, \AREG,  4  

  2120.     addi        \BREG, \BREG,  32

  2121. .endm

  2122. 

  2123. .macro KERNEL8x1_2_1 AREG,BREG,First 

  2124.     lxsd v4,    0(\AREG)

  2125.     lxv vs26,   0(\BREG)

  2126.     lxv vs27,  16(\BREG)      

  2127.     lxv vs28,  32(\BREG)

  2128.     lxv vs29,  48(\BREG) 

  2129.     xxspltw   vs8,  vs36, 1 

  2130.     xxspltw   vs9,  vs36, 0  

  2131. .if \First==1

  2132.     xvmulsp      vs0,   vs26,   vs8

  2133.     xvmulsp      vs1,   vs27,   vs8  

  2134.     xvmulsp      vs0,   vs28,   vs9

  2135.     xvmulsp      vs1,   vs29,   vs9     

  2136. .else 

  2137.     xvmaddasp      vs0,   vs26,   vs8

  2138.     xvmaddasp      vs1,   vs27,   vs8  

  2139.     xvmaddasp      vs0,   vs28,   vs9

  2140.     xvmaddasp      vs1,   vs29,   vs9 

  2141.  .endif

  2142.     addi        \AREG, \AREG,  8 

  2143.     addi        \BREG, \BREG,  64

  2144. .endm

  2145. 

  2146. .macro KERNEL8x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

  2147.     lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)

  2148.     xxspltw   vs8,  vs4, 3 

  2149.     xxspltw   vs9,  vs4, 2 

  2150.     xxspltw   vs10, vs4, 1 

  2151.     xxspltw   vs11, vs4, 0

  2152.     lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)

  2153.     lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)

  2154.     lxv vs28,   DISP32(\Index,32+\OffsetB)(\BREG)

  2155.     lxv vs29,   DISP32(\Index,48+\OffsetB)(\BREG) 

  2156.     lxv vs30,   DISP32(\Index,64+ 0+\OffsetB)(\BREG)

  2157.     lxv vs31,   DISP32(\Index,64+16+\OffsetB)(\BREG)

  2158.     lxv vs32,   DISP32(\Index,64+32+\OffsetB)(\BREG)

  2159.     lxv vs33,   DISP32(\Index,64+48+\OffsetB)(\BREG)         

  2160. .if \First==1

  2161.     xvmulsp      vs0,   vs26,   vs8

  2162.     xvmulsp      vs1,   vs27,   vs8  

  2163.     xvmulsp      vs0,   vs28,   vs9

  2164.     xvmulsp      vs1,   vs29,   vs9     

  2165.     xvmulsp      vs0,   vs30,   vs10

  2166.     xvmulsp      vs1,   vs31,   vs10  

  2167.     xvmulsp      vs0,   vs32,   vs11

  2168.     xvmulsp      vs1,   vs33,   vs11     

  2169. .else 

  2170.     xvmaddasp      vs0,   vs26,   vs8

  2171.     xvmaddasp      vs1,   vs27,   vs8  

  2172.     xvmaddasp      vs0,   vs28,   vs9

  2173.     xvmaddasp      vs1,   vs29,   vs9     

  2174.     xvmaddasp      vs0,   vs30,   vs10

  2175.     xvmaddasp      vs1,   vs31,   vs10  

  2176.     xvmaddasp      vs0,   vs32,   vs11

  2177.     xvmaddasp      vs1,   vs33,   vs11  

  2178.  .endif

  2179. .if \IsLast==1   

  2180.     addi        \AREG, \AREG, DISP4(\Index,16)  

  2181.     addi        \BREG, \BREG, DISP32(\Index,128)

  2182. .endif 

  2183. .endm

  2184. 

  2185. .macro SAVE8x1

  2186.   slwi    T10, LDC ,   1 

  2187.   add     T1, CO, LDC  

  2188.   add     T2, CO, T10  

  2189.   add     T3, T1, T10     

  2190.   add     T4, T2, T10 

  2191.   add     T5, T3, T10 

  2192.   add     T6, T4, T10 

  2193.   add     T7, T5, T10 

  2194.   /*convert alpha_r for multiply*/

  2195.   xscvspdp  vs4,alpha_r

  2196. /* v0 corresponds to vs32, do not forget*/

  2197. #if !defined(TRMMKERNEL)

  2198.   lxssp  v0,0(CO)  

  2199.   lxssp  v2,0(T1) 

  2200.   lxssp  v4,0(T2) 

  2201.   lxssp  v6,0(T3) 

  2202.   lxssp  v8,0(T4) 

  2203.   lxssp  v10,0(T5) 

  2204.   lxssp  v12,0(T6) 

  2205.   lxssp  v14,0(T7)

  2206. #endif

  2207.   xscvspdp  vs24, vs0

  2208.   xxspltw   vs25, vs0, 1 

  2209.   xxspltw   vs26, vs0, 2 

  2210.   xxspltw   vs27, vs0, 3  

  2211.   xscvspdp  vs25,vs25

  2212.   xscvspdp  vs26,vs26

  2213.   xscvspdp  vs27,vs27

  2214.   xscvspdp  vs28, vs1

  2215.   xxspltw   vs29, vs1, 1 

  2216.   xxspltw   vs30, vs1, 2 

  2217.   xxspltw   vs31, vs1, 3  

  2218.   xscvspdp  vs29,vs29

  2219.   xscvspdp  vs30,vs30

  2220.   xscvspdp  vs31,vs31

  2221. #if defined(TRMMKERNEL)

  2222.   xsmuldp  vs32,vs27, vs4 

  2223.   xsmuldp  vs34,vs26, vs4 

  2224.   xsmuldp  vs36,vs25, vs4 

  2225.   xsmuldp  vs38,vs24, vs4 

  2226.   xsmuldp  vs40,vs31, vs4 

  2227.   xsmuldp  vs42,vs30, vs4 

  2228.   xsmuldp  vs44,vs29, vs4 

  2229.   xsmuldp  vs46,vs28, vs4 

  2230. #else

  2231.   xsmaddadp  vs32,vs27, vs4 

  2232.   xsmaddadp  vs34,vs26, vs4 

  2233.   xsmaddadp  vs36,vs25, vs4 

  2234.   xsmaddadp  vs38,vs24, vs4 

  2235.   xsmaddadp  vs40,vs31, vs4 

  2236.   xsmaddadp  vs42,vs30, vs4 

  2237.   xsmaddadp  vs44,vs29, vs4 

  2238.   xsmaddadp  vs46,vs28, vs4  

  2239. #endif  

  2240.   stxssp  v0,0(CO)  

  2241.   stxssp  v2,0(T1) 

  2242.   stxssp  v4,0(T2) 

  2243.   stxssp  v6,0(T3) 

  2244.   stxssp  v8,0(T4) 

  2245.   stxssp  v10,0(T5) 

  2246.   stxssp  v12,0(T6) 

  2247.   stxssp  v14,0(T7) 

  2248.   addi CO,CO,4

  2249. .endm

  2250. 

  2251. 

  2252. 

  2253. /**********************************************************************************************

  2254. * Macros for N=4 and M=16

  2255. **********************************************************************************************/

  2256. 

  2257. .macro LOAD4x16_1

  2258.    LOAD4x16 1

  2259. .endm

  2260. 

  2261. .macro LOAD4x16_0

  2262.    LOAD4x16 0

  2263. .endm

  2264. 

  2265. .macro KERNEL4x16_L1_L4  Index,IsLast

  2266.   KERNEL4x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  2267. .endm

  2268. 

  2269. .macro KERNEL4x16_I1_L4  OffsetA,OffsetB, Index,IsLast

  2270.   KERNEL4x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  2271. .endm

  2272. 

  2273. .macro KERNEL4x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  2274.   KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  2275. .endm

  2276. 

  2277. .macro KERNEL4x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  2278.   KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2279. .endm

  2280. .macro KERNEL4x16_I1_L2_3  OffsetA,OffsetB, Index,IsLast

  2281.    KERNEL4x16_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2282. .endm

  2283. 

  2284. .macro KERNEL4x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  2285.   KERNEL4x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  2286. .endm

  2287. 

  2288. .macro KERNEL4x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  2289.   KERNEL4x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2290. .endm

  2291. 

  2292. .macro Zero4X16

  2293.     xxlxor		vs32,	vs32,	vs32

  2294.     xxlxor		vs33,	vs33,	vs33

  2295. 	xxlxor		vs34,	vs34,	vs34

  2296. 	xxlxor		vs35,	vs35,	vs35

  2297. 	xxlxor		vs36,	vs36,	vs36

  2298. 	xxlxor		vs37,	vs37,	vs37

  2299. 	xxlxor		vs38,	vs38,	vs38

  2300. 	xxlxor		vs39,	vs39,	vs39

  2301. 	xxlxor		vs40,	vs40,	vs40

  2302. 	xxlxor		vs41,	vs41,	vs41

  2303. 	xxlxor		vs42,	vs42,	vs42

  2304. 	xxlxor		vs43,	vs43,	vs43

  2305. 	xxlxor		vs44,	vs44,	vs44

  2306. 	xxlxor		vs45,	vs45,	vs45

  2307. 	xxlxor		vs46,	vs46,	vs46

  2308. 	xxlxor		vs47,	vs47,	vs47	

  2309. .endm

  2310. 

  2311. .macro LOAD4x16  Zero

  2312. 

  2313. 	lxv	vs24,	0(BO) 

  2314. 	lxv	vs0,	 0(AO)

  2315. 	lxv	vs1,	16(AO)

  2316. 	lxv	vs2,	32(AO)

  2317. 	lxv	vs3,	48(AO)

  2318. 	xxperm  	vs26,	vs24,		permute_mask 	

  2319. 	xxpermdi	vs25,	vs24,	vs24,2 

  2320. 	xxpermdi	vs27,	vs26,	vs26,2	 	

  2321. 

  2322. .if \Zero==1 

  2323.     xxlxor		vs32,	vs32,	vs32

  2324.     xxlxor		vs33,	vs33,	vs33

  2325. 	xxlxor		vs34,	vs34,	vs34

  2326. 	xxlxor		vs35,	vs35,	vs35

  2327. 	xxlxor		vs36,	vs36,	vs36

  2328. 	xxlxor		vs37,	vs37,	vs37

  2329. 	xxlxor		vs38,	vs38,	vs38

  2330. 	xxlxor		vs39,	vs39,	vs39

  2331. 	xxlxor		vs40,	vs40,	vs40

  2332. 	xxlxor		vs41,	vs41,	vs41

  2333. 	xxlxor		vs42,	vs42,	vs42

  2334. 	xxlxor		vs43,	vs43,	vs43

  2335. 	xxlxor		vs44,	vs44,	vs44

  2336. 	xxlxor		vs45,	vs45,	vs45

  2337. 	xxlxor		vs46,	vs46,	vs46

  2338. 	xxlxor		vs47,	vs47,	vs47

  2339.  

  2340. .endif

  2341. .endm

  2342. 

  2343. .macro END4x16_NORMAL

  2344.   END4x16 0, AO, BO, 64,16 

  2345. .endm

  2346. 

  2347. .macro END4x16 First, AREG, BREG, OffsetA, OffsetB

  2348. 

  2349. .if \OffsetB != 0 

  2350.     addi        \BREG, \BREG, \OffsetB 

  2351. .endif

  2352. .if \OffsetA != 0 

  2353.     addi        \AREG, \AREG, \OffsetA 

  2354. .endif  

  2355. 

  2356. .if \First==1

  2357.     xvmulsp     vs32, vs0,vs24

  2358.     xvmulsp     vs33, vs1,vs24

  2359.     xvmulsp     vs34, vs2,vs24  

  2360.     xvmulsp     vs35, vs3,vs24  

  2361. 

  2362.     xvmulsp     vs36, vs0,vs25

  2363.     xvmulsp     vs37, vs1,vs25

  2364.     xvmulsp     vs38, vs2,vs25  

  2365.     xvmulsp     vs39, vs3,vs25

  2366. 

  2367.     xvmulsp     vs40, vs0,vs26

  2368.     xvmulsp     vs41, vs1,vs26

  2369.     xvmulsp     vs42, vs2,vs26  

  2370.     xvmulsp     vs43, vs3,vs26

  2371. 

  2372.     xvmulsp     vs44, vs0,vs27

  2373.     xvmulsp     vs45, vs1,vs27

  2374.     xvmulsp     vs46, vs2,vs27  

  2375.     xvmulsp     vs47, vs3,vs27

  2376. 

  2377. .else

  2378.     xvmaddasp       vs32, vs0,vs24

  2379.     xvmaddasp       vs33, vs1,vs24

  2380.     xvmaddasp       vs34, vs2,vs24  

  2381.     xvmaddasp       vs35, vs3,vs24  

  2382. 

  2383.     xvmaddasp       vs36, vs0,vs25

  2384.     xvmaddasp       vs37, vs1,vs25

  2385.     xvmaddasp       vs38, vs2,vs25  

  2386.     xvmaddasp       vs39, vs3,vs25 

  2387.     xvmaddasp       vs40, vs0,vs26

  2388.     xvmaddasp       vs41, vs1,vs26

  2389.     xvmaddasp       vs42, vs2,vs26  

  2390.     xvmaddasp       vs43, vs3,vs26

  2391. 

  2392.     xvmaddasp       vs44, vs0,vs27

  2393.     xvmaddasp       vs45, vs1,vs27

  2394.     xvmaddasp       vs46, vs2,vs27  

  2395.     xvmaddasp       vs47, vs3,vs27

  2396. 

  2397. .endif

  2398. .endm  

  2399. 

  2400. .macro KERNEL4x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  2401. 

  2402. 	lxv	vs8,	DISP16(\Index, 0+\OffsetB)(\BREG) 

  2403. 

  2404.  	lxv	vs4,	DISP64(\Index, 0+\OffsetA)(\AREG)

  2405. 	lxv	vs5,	DISP64(\Index,16+\OffsetA)(\AREG)

  2406. 	lxv	vs6,	DISP64(\Index,32+\OffsetA)(\AREG)

  2407. 	lxv	vs7,	DISP64(\Index,48+\OffsetA)(\AREG) 

  2408. 

  2409. 	xxperm  	vs10,	vs8,		permute_mask 

  2410. 	xxpermdi	vs9,	vs8,	vs8,2	  

  2411. 

  2412.     xvmaddasp		vs32, vs0,vs24

  2413. 	xvmaddasp		vs33, vs1,vs24

  2414. 	xvmaddasp		vs34, vs2,vs24	

  2415. 	xvmaddasp		vs35, vs3,vs24	 

  2416. 

  2417.     xvmaddasp		vs36, vs0,vs25

  2418. 	xvmaddasp		vs37, vs1,vs25

  2419. 	xvmaddasp		vs38, vs2,vs25	

  2420. 	xvmaddasp		vs39, vs3,vs25 

  2421. 

  2422.  	xxpermdi	vs11,	vs10,	vs10,2	 

  2423. 

  2424.     xvmaddasp		vs40, vs0,vs26

  2425. 	xvmaddasp		vs41, vs1,vs26

  2426. 	xvmaddasp		vs42, vs2,vs26	

  2427. 	xvmaddasp		vs43, vs3,vs26

  2428. 

  2429.     xvmaddasp		vs44, vs0,vs27

  2430. 	xvmaddasp		vs45, vs1,vs27

  2431. 	xvmaddasp		vs46, vs2,vs27	

  2432. 	xvmaddasp		vs47, vs3,vs27

  2433. 

  2434. 

  2435. 

  2436. 	lxv	vs24,	DISP16(\Index,16+\OffsetB)(\BREG) 

  2437. 

  2438. 	lxv	vs0,	DISP64(\Index,64+\OffsetA)(\AREG)

  2439. 	lxv	vs1,	DISP64(\Index,64+16+\OffsetA)(\AREG)

  2440. 	lxv	vs2,	DISP64(\Index,64+32+\OffsetA)(\AREG)

  2441. 	lxv	vs3,	DISP64(\Index,64+48+\OffsetA)(\AREG)

  2442. 

  2443. 	xxperm  	vs26,	vs24,		permute_mask 

  2444. 	xxpermdi	vs25,	vs24,	vs24,2	    

  2445.  

  2446. 

  2447.     xvmaddasp		vs32, vs4,vs8

  2448. 	xvmaddasp		vs33, vs5,vs8

  2449. 	xvmaddasp		vs34, vs6,vs8	

  2450. 	xvmaddasp		vs35, vs7,vs8	

  2451.  

  2452.     xvmaddasp		vs36, vs4,vs9

  2453. 	xvmaddasp		vs37, vs5,vs9

  2454. 	xvmaddasp		vs38, vs6,vs9	

  2455. 	xvmaddasp		vs39, vs7,vs9

  2456.          

  2457. 	xxpermdi	vs27,	vs26,	vs26,2	 	

  2458. 

  2459.     xvmaddasp		vs40, vs4,vs10

  2460. 	xvmaddasp		vs41, vs5,vs10

  2461. 	xvmaddasp		vs42, vs6,vs10	

  2462. 	xvmaddasp		vs43, vs7,vs10

  2463. 

  2464.     xvmaddasp		vs44, vs4,vs11

  2465. 	xvmaddasp		vs45, vs5,vs11

  2466. 	xvmaddasp		vs46, vs6,vs11	

  2467. 	xvmaddasp		vs47, vs7,vs11

  2468.  

  2469. 

  2470. 	lxv	vs8,	DISP16(\Index,32+\OffsetB)(\BREG) 

  2471. 

  2472.  	lxv	vs4,	DISP64(\Index,128+0+\OffsetA)(\AREG)

  2473. 	lxv	vs5,	DISP64(\Index,128+16+\OffsetA)(\AREG)

  2474. 	lxv	vs6,	DISP64(\Index,128+32+\OffsetA)(\AREG)

  2475. 	lxv	vs7,	DISP64(\Index,128+48+\OffsetA)(\AREG) 

  2476. 

  2477. 	xxperm  	vs10,	vs8,		permute_mask 

  2478. 	xxpermdi	vs9,	vs8,	vs8,2	  

  2479. 

  2480.     xvmaddasp		vs32, vs0,vs24

  2481. 	xvmaddasp		vs33, vs1,vs24

  2482. 	xvmaddasp		vs34, vs2,vs24	

  2483. 	xvmaddasp		vs35, vs3,vs24	 

  2484. 

  2485.     xvmaddasp		vs36, vs0,vs25

  2486. 	xvmaddasp		vs37, vs1,vs25

  2487. 	xvmaddasp		vs38, vs2,vs25	

  2488. 	xvmaddasp		vs39, vs3,vs25

  2489. 

  2490.  	xxpermdi	vs11,	vs10,	vs10,2	 

  2491. 

  2492.     xvmaddasp		vs40, vs0,vs26

  2493. 	xvmaddasp		vs41, vs1,vs26

  2494. 	xvmaddasp		vs42, vs2,vs26	

  2495. 	xvmaddasp		vs43, vs3,vs26

  2496. 

  2497.     xvmaddasp		vs44, vs0,vs27

  2498. 	xvmaddasp		vs45, vs1,vs27

  2499. 	xvmaddasp		vs46, vs2,vs27	

  2500. 	xvmaddasp		vs47, vs3,vs27

  2501. 

  2502.  

  2503.  

  2504. .if \Complete==0

  2505. 	lxv	vs24,	DISP16(\Index,48+\OffsetB)(\BREG) 

  2506. 

  2507. 	lxv	vs0,	DISP64(\Index,192+\OffsetA)(\AREG)

  2508. 	lxv	vs1,	DISP64(\Index,192+16+\OffsetA)(\AREG) 

  2509. 	lxv	vs2,	DISP64(\Index,192+32+\OffsetA)(\AREG)

  2510. 	lxv	vs3,	DISP64(\Index,192+48+\OffsetA)(\AREG)

  2511. 

  2512. 	xxperm  	vs26,	vs24,	permute_mask 	

  2513. 	xxpermdi	vs25,	vs24,	vs24,2  	

  2514. 

  2515. .endif 

  2516. .if \IsLast==1	

  2517. .if \Complete==1

  2518.   

  2519. 	addi		\BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)

  2520. 	addi		\AREG, \AREG, DISP64(\Index,64*3+\OffsetA)

  2521. .else

  2522.   

  2523. 	addi		\BREG, \BREG,  DISP16(\Index,64)

  2524. 	addi		\AREG, \AREG, DISP64(\Index,256)

  2525. .endif

  2526. .endif   

  2527.  

  2528.     xvmaddasp		vs32, vs4,vs8

  2529. 	xvmaddasp		vs33, vs5,vs8

  2530. 	xvmaddasp		vs34, vs6,vs8	

  2531. 	xvmaddasp		vs35, vs7,vs8	 

  2532.  

  2533.     xvmaddasp		vs36, vs4,vs9

  2534. 	xvmaddasp		vs37, vs5,vs9

  2535. 	xvmaddasp		vs38, vs6,vs9	

  2536. 	xvmaddasp		vs39, vs7,vs9

  2537.   

  2538. .if \Complete==0        

  2539. 	xxpermdi	vs27,	vs26,	vs26,2	 

  2540.  	

  2541. .endif

  2542.  

  2543.     xvmaddasp		vs40, vs4,vs10

  2544. 	xvmaddasp		vs41, vs5,vs10

  2545. 	xvmaddasp		vs42, vs6,vs10	

  2546. 	xvmaddasp		vs43, vs7,vs10

  2547. 

  2548.     xvmaddasp		vs44, vs4,vs11

  2549. 	xvmaddasp		vs45, vs5,vs11

  2550. 	xvmaddasp		vs46, vs6,vs11	

  2551. 	xvmaddasp		vs47, vs7,vs11

  2552. 

  2553.  

  2554. 

  2555. .endm

  2556. 

  2557. .macro KERNEL4x16 First

  2558. 

  2559.   LOAD4x16 0

  2560.   END4x16 \First, AO, BO, 64,16 

  2561. .endm

  2562. 

  2563. .macro KERNEL4x16_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

  2564. 	

  2565. 	lxv	vs8,	DISP8(\Index, 0+\OffsetB)(\BREG) 

  2566.  	lxv	vs4,	DISP32(\Index, 0+\OffsetA)(\AREG)

  2567. 	lxv	vs5,	DISP32(\Index,16+\OffsetA)(\AREG)

  2568. 	lxv	vs6,	DISP32(\Index,32+\OffsetA)(\AREG)

  2569. 	lxv	vs7,	DISP32(\Index,48+\OffsetA)(\AREG) 

  2570. 

  2571. 	xxperm  	vs10,	vs8,		permute_mask 

  2572. 	xxpermdi	vs9,	vs8,	vs8,2	  

  2573. .if \First==1

  2574.     xvmulsp		vs32, vs0,vs24

  2575. 	xvmulsp		vs33, vs1,vs24

  2576. 	xvmulsp		vs34, vs2,vs24	

  2577. 	xvmulsp		vs35, vs3,vs24	

  2578. 

  2579.     xvmulsp		vs36, vs0,vs25

  2580. 	xvmulsp		vs37, vs1,vs25

  2581. 	xvmulsp		vs38, vs2,vs25	

  2582. 	xvmulsp		vs39, vs3,vs25	

  2583. .else

  2584.     xvmaddasp		vs32, vs0,vs24

  2585. 	xvmaddasp		vs33, vs1,vs24

  2586. 	xvmaddasp		vs34, vs2,vs24	

  2587. 	xvmaddasp		vs35, vs3,vs24

  2588. 

  2589.     xvmaddasp		vs36, vs0,vs25

  2590. 	xvmaddasp		vs37, vs1,vs25

  2591. 	xvmaddasp		vs38, vs2,vs25	

  2592. 	xvmaddasp		vs39, vs3,vs25		

  2593. .endif

  2594. 

  2595.  	xxpermdi	vs11,	vs10,	vs10,2	 	

  2596.  

  2597. .if \First==1  

  2598.     xvmulsp		vs40, vs0,vs26

  2599. 	xvmulsp		vs41, vs1,vs26

  2600. 	xvmulsp		vs42, vs2,vs26	

  2601. 	xvmulsp		vs43, vs3,vs26

  2602. 

  2603.     xvmulsp		vs44, vs0,vs27

  2604. 	xvmulsp		vs45, vs1,vs27

  2605. 	xvmulsp		vs46, vs2,vs27	

  2606. 	xvmulsp		vs47, vs3,vs27

  2607. 

  2608.   

  2609. .else 

  2610.     xvmaddasp		vs40, vs0,vs26

  2611. 	xvmaddasp		vs41, vs1,vs26

  2612. 	xvmaddasp		vs42, vs2,vs26	

  2613. 	xvmaddasp		vs43, vs3,vs26

  2614. 

  2615.     xvmaddasp		vs44, vs0,vs27

  2616. 	xvmaddasp		vs45, vs1,vs27

  2617. 	xvmaddasp		vs46, vs2,vs27	

  2618. 	xvmaddasp		vs47, vs3,vs27

  2619.  

  2620. 

  2621. .endif

  2622. .if \Complete==0

  2623. 	lxv	vs24,	DISP8(\Index,16+\OffsetB)(\BREG) 

  2624. 	lxv	vs0,	DISP32(\Index,64+\OffsetA)(\AREG)

  2625. 	lxv	vs1,	DISP32(\Index,64+16+\OffsetA)(\AREG)

  2626. 	lxv	vs2,	DISP32(\Index,64+32+\OffsetA)(\AREG)

  2627. 	lxv	vs3,	DISP32(\Index,64+48+\OffsetA)(\AREG)

  2628. 

  2629. 	xxperm  	vs26,	vs24,	permute_mask 

  2630. 	xxpermdi	vs25,	vs24,	vs24,2	  

  2631. .endif    

  2632. .if \IsLast==1	

  2633. .if \Complete==1

  2634.  	addi		\BREG, \BREG,  DISP8(\Index,16+\OffsetB) 

  2635. 	addi		\AREG, \AREG, DISP32(\Index,64+\OffsetA)

  2636. 

  2637. .else

  2638.   	addi		\BREG, \BREG,  DISP8(\Index,32)

  2639. 	addi		\AREG, \AREG, DISP32(\Index,128) 

  2640. .endif

  2641. .endif

  2642. 

  2643. .if \First==1

  2644.     xvmulsp		vs32, vs4,vs8

  2645. 	xvmulsp		vs33, vs5,vs8

  2646. 	xvmulsp		vs34, vs6,vs8	

  2647. 	xvmulsp		vs35, vs7,vs8

  2648. 

  2649.     xvmulsp		vs36, vs4,vs9

  2650. 	xvmulsp		vs37, vs5,vs9

  2651. 	xvmulsp		vs38, vs6,vs9	

  2652. 	xvmulsp		vs39, vs7,vs9

  2653. .else

  2654.     xvmaddasp		vs32, vs4,vs8

  2655. 	xvmaddasp		vs33, vs5,vs8

  2656. 	xvmaddasp		vs34, vs6,vs8	

  2657. 	xvmaddasp		vs35, vs7,vs8	

  2658. 

  2659.     xvmaddasp		vs36, vs4,vs9

  2660. 	xvmaddasp		vs37, vs5,vs9

  2661. 	xvmaddasp		vs38, vs6,vs9	

  2662. 	xvmaddasp		vs39, vs7,vs9

  2663. .endif 

  2664.  

  2665. .if \Complete==0        

  2666. 	xxpermdi	vs27,	vs26,	vs26,2	 

  2667.  

  2668. .endif

  2669. .if \First==1  

  2670.     xvmulsp		vs40, vs4,vs10

  2671. 	xvmulsp		vs41, vs5,vs10

  2672. 	xvmulsp		vs42, vs6,vs10	

  2673. 	xvmulsp		vs43, vs7,vs10

  2674. 

  2675.     xvmulsp		vs44, vs4,vs11

  2676. 	xvmulsp		vs45, vs5,vs11

  2677. 	xvmulsp		vs46, vs6,vs11	

  2678. 	xvmulsp		vs47, vs7,vs11

  2679. 

  2680.  

  2681. 

  2682. .else 

  2683.     xvmaddasp		vs40, vs4,vs10

  2684. 	xvmaddasp		vs41, vs5,vs10

  2685. 	xvmaddasp		vs42, vs6,vs10	

  2686. 	xvmaddasp		vs43, vs7,vs10

  2687. 

  2688.     xvmaddasp		vs44, vs4,vs11

  2689. 	xvmaddasp		vs45, vs5,vs11

  2690. 	xvmaddasp		vs46, vs6,vs11	

  2691. 	xvmaddasp		vs47, vs7,vs11

  2692. 

  2693.  

  2694. 

  2695. .endif

  2696. 

  2697. .endm

  2698. 

  2699.  

  2700. .macro SAVE4x16

  2701. 

  2702.   slwi    T10, LDC ,   1 

  2703.   add     T1, CO, LDC 

  2704. 

  2705.   add     T2, CO, T10  

  2706.   add     T3, T1, T10  

  2707. 

  2708.   

  2709.  

  2710.     xxmrglw     vs8,    vs32,   vs44

  2711.     xxmrglw     vs10,   vs36,   vs40  

  2712. 

  2713.     xxmrghw     vs1,    vs32,   vs44

  2714.     xxmrghw     vs0,    vs36,   vs40

  2715. 

  2716.     xxmrglw     vs12,   vs33,   vs45

  2717.     xxmrglw     vs14,   vs37,   vs41  

  2718. 

  2719.     xxmrghw     vs2,    vs37,   vs41

  2720.     xxmrghw     vs3,    vs33,   vs45

  2721. 

  2722.     xxmrglw     vs16,   vs34,   vs46

  2723.     xxmrglw     vs18,   vs38,   vs42   

  2724. 

  2725.     xxlor      vs9, vs8,    vs8

  2726.     xxlor      vs11,    vs10,   vs10 

  2727. 

  2728.     xxmrghw     vs4,    vs38,   vs42

  2729.     xxmrghw     vs5,    vs34,   vs46

  2730. 

  2731.     xxlor      vs13,    vs12,   vs12

  2732.     xxlor      vs15,    vs14,   vs14

  2733. 

  2734.     xxmrglw     vs24,   vs35,   vs47

  2735.     xxmrglw     vs26,   vs39,   vs43  

  2736. 

  2737.     xxlor      vs17,    vs16,   vs16

  2738.     xxlor      vs19,    vs18,   vs18

  2739. 

  2740.     xxmrghw     vs30,   vs39,   vs43 

  2741.     xxmrghw     vs31,   vs35,   vs47

  2742. 

  2743.     xxperm      vs8,    vs0,    save_permute_1

  2744.     xxperm      vs10,   vs1,    save_permute_1

  2745.     xxperm      vs9,    vs0,    save_permute_2  

  2746.     xxperm      vs11,   vs1,    save_permute_2      

  2747. 

  2748. #ifndef TRMMKERNEL    

  2749.     lxv        vs32, 0(CO)

  2750.     lxv        vs33, 16(CO) 

  2751.     lxv        vs34, 32(CO)  

  2752.     lxv        vs35, 48(CO)      

  2753. #endif

  2754.     xxlor      vs25,    vs24,   vs24

  2755.     xxlor      vs27,    vs26,   vs26 

  2756. 

  2757. #ifndef TRMMKERNEL    

  2758.     lxv        vs36, 0(T1)

  2759.     lxv        vs37, 16(T1) 

  2760.     lxv        vs38, 32(T1)  

  2761.     lxv        vs39, 48(T1)     

  2762. #endif

  2763. #ifndef TRMMKERNEL       

  2764.     lxv        vs40, 0(T2)

  2765.     lxv        vs41, 16(T2) 

  2766.     lxv        vs42, 32(T2)  

  2767.     lxv        vs43, 48(T2)     

  2768. #endif  

  2769. #ifndef TRMMKERNEL    

  2770.     lxv        vs44, 0(T3)

  2771.     lxv        vs45, 16(T3) 

  2772.     lxv        vs46, 32(T3)  

  2773.     lxv        vs47, 48(T3)                 

  2774. #endif  

  2775. 

  2776.     xxperm     vs12,    vs2,    save_permute_1

  2777.     xxperm     vs14,    vs3,    save_permute_1

  2778.        

  2779.     xxperm     vs13,    vs2,    save_permute_2   

  2780.     xxperm     vs15,    vs3,    save_permute_2      

  2781. 

  2782.     xxperm     vs16,    vs4,    save_permute_1

  2783.     xxperm     vs18,    vs5,    save_permute_1

  2784.       

  2785.     xxperm     vs17,    vs4,    save_permute_2   

  2786.     xxperm     vs19,    vs5,    save_permute_2      

  2787. 

  2788.     xxperm     vs24,    vs30,   save_permute_1

  2789.     xxperm     vs26,    vs31,   save_permute_1 

  2790.          

  2791.     xxperm     vs25,    vs30,   save_permute_2   

  2792.     xxperm     vs27,    vs31,   save_permute_2  

  2793. 

  2794. 

  2795.     /* multiply add normal way */

  2796.  

  2797. #ifdef TRMMKERNEL

  2798.     xvmulsp     vs32,   vs8,    alpha_r 

  2799.     xvmulsp     vs33,   vs12,   alpha_r   

  2800.     xvmulsp     vs34,   vs16,   alpha_r 

  2801.     xvmulsp     vs35,   vs24,   alpha_r  

  2802.     xvmulsp     vs36,   vs9,    alpha_r 

  2803.     xvmulsp     vs37,   vs13,   alpha_r  

  2804.     xvmulsp     vs38,   vs17,   alpha_r 

  2805.     xvmulsp     vs39,   vs25,   alpha_r               

  2806. #else 

  2807.     xvmaddasp   vs32,   vs8,    alpha_r 

  2808.     xvmaddasp   vs33,   vs12,   alpha_r   

  2809.     xvmaddasp   vs34,   vs16,   alpha_r 

  2810.     xvmaddasp   vs35,   vs24,   alpha_r  

  2811.     xvmaddasp   vs36,   vs9,    alpha_r 

  2812.     xvmaddasp   vs37,   vs13,   alpha_r   

  2813.     xvmaddasp   vs38,   vs17,   alpha_r 

  2814.     xvmaddasp   vs39,   vs25,   alpha_r         

  2815. #endif 

  2816. 

  2817. 

  2818. 

  2819. #ifdef TRMMKERNEL

  2820.     xvmulsp     vs40,   vs10,   alpha_r 

  2821.     xvmulsp     vs41,   vs14,   alpha_r 

  2822.     xvmulsp     vs42,   vs18,   alpha_r 

  2823.     xvmulsp     vs43,   vs26,   alpha_r  

  2824.     xvmulsp     vs44,   vs11,   alpha_r 

  2825.     xvmulsp     vs45,   vs15,   alpha_r  

  2826.     xvmulsp     vs46,   vs19,   alpha_r 

  2827.     xvmulsp     vs47,   vs27,   alpha_r                   

  2828. #else

  2829. 

  2830.     xvmaddasp   vs40,   vs10,   alpha_r 

  2831.     xvmaddasp   vs41,   vs14,   alpha_r   

  2832.     xvmaddasp   vs42,   vs18,   alpha_r 

  2833.     xvmaddasp   vs43,   vs26,   alpha_r  

  2834.     xvmaddasp   vs44,   vs11,   alpha_r 

  2835.     xvmaddasp   vs45,   vs15,   alpha_r 

  2836.     xvmaddasp   vs46,   vs19,   alpha_r 

  2837.     xvmaddasp   vs47,   vs27,   alpha_r  

  2838.         

  2839. #endif  

  2840. 

  2841.     stxv        vs32, 0(CO)

  2842.     stxv        vs33, 16(CO) 

  2843.     stxv        vs34, 32(CO)  

  2844.     stxv        vs35, 48(CO)  

  2845. 

  2846.     stxv        vs36, 0(T1)

  2847.     stxv        vs37, 16(T1)  

  2848.     stxv        vs38, 32(T1)  

  2849.     stxv        vs39, 48(T1)

  2850. 

  2851.     stxv        vs40, 0(T2)

  2852.     stxv        vs41, 16(T2)  

  2853.     stxv        vs42, 32(T2)  

  2854.     stxv        vs43, 48(T2)  

  2855.     stxv        vs44, 0(T3)

  2856.     stxv        vs45, 16(T3) 

  2857.     stxv        vs46, 32(T3)  

  2858.     stxv        vs47, 48(T3)

  2859.    

  2860.     addi CO,CO,64

  2861. 

  2862. 

  2863. .endm

  2864. 

  2865. 

  2866. 

  2867. /**********************************************************************************************

  2868. * Macros for N=4 and M=8

  2869. **********************************************************************************************/

  2870. 

  2871. .macro LOAD4x8_1

  2872.    LOAD4x8 1

  2873. .endm

  2874. 

  2875. .macro LOAD4x8_0

  2876.    LOAD4x8 0

  2877. .endm

  2878. 

  2879. .macro KERNEL4x8_L1_L4  Index,IsLast

  2880.   KERNEL4x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  2881. .endm

  2882. 

  2883. .macro KERNEL4x8_I1_L4  OffsetA,OffsetB, Index,IsLast

  2884.   KERNEL4x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  2885. .endm

  2886. 

  2887. .macro KERNEL4x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  2888.   KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  2889. .endm

  2890. 

  2891. .macro KERNEL4x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  2892.   KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2893. .endm

  2894. .macro KERNEL4x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast

  2895.    KERNEL4x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2896. .endm

  2897. 

  2898. .macro KERNEL4x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  2899.   KERNEL4x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  2900. .endm

  2901. 

  2902. .macro KERNEL4x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  2903.   KERNEL4x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  2904. .endm

  2905. 

  2906. .macro END4x8_NORMAL

  2907.   END4x8 0, AO, BO, 32,16 

  2908. .endm

  2909. 

  2910. .macro Zero4X8

  2911.     xxlxor      vs32,   vs32,   vs32

  2912.     xxlxor      vs33,   vs33,   vs33

  2913.  

  2914.     xxlxor      vs36,   vs36,   vs36

  2915.     xxlxor      vs37,   vs37,   vs37

  2916.  

  2917.     xxlxor      vs40,   vs40,   vs40

  2918.     xxlxor      vs41,   vs41,   vs41

  2919.  

  2920.     xxlxor      vs44,   vs44,   vs44

  2921.     xxlxor      vs45,   vs45,   vs45

  2922.     

  2923. .endm

  2924. 

  2925. .macro LOAD4x8  Zero

  2926. 

  2927.     lxv vs24,   0(BO) 

  2928.     lxv vs0,     0(AO)

  2929.     lxv vs1,    16(AO)

  2930. 

  2931.     xxperm      vs26,   vs24,       permute_mask    

  2932.     xxpermdi    vs25,   vs24,   vs24,2      

  2933. 

  2934.     xxpermdi    vs27,   vs26,   vs26,2      

  2935. 

  2936. .if \Zero==1 

  2937.     xxlxor      vs32,   vs32,   vs32

  2938.     xxlxor      vs33,   vs33,   vs33 

  2939.     xxlxor      vs36,   vs36,   vs36

  2940.     xxlxor      vs37,   vs37,   vs37

  2941.     xxlxor      vs40,   vs40,   vs40

  2942.     xxlxor      vs41,   vs41,   vs41 

  2943.     xxlxor      vs44,   vs44,   vs44

  2944.     xxlxor      vs45,   vs45,   vs45 

  2945.  

  2946. .endif

  2947. .endm

  2948. 

  2949. 

  2950. .macro END4x8 First, AREG, BREG, OffsetA, OffsetB

  2951. 

  2952. .if \OffsetB != 0 

  2953.     addi        \BREG, \BREG, \OffsetB 

  2954. .endif

  2955. .if \OffsetA != 0 

  2956.     addi        \AREG, \AREG, \OffsetA 

  2957. .endif  

  2958. 

  2959. .if \First==1

  2960.     xvmulsp     vs32, vs0,vs24

  2961.     xvmulsp     vs33, vs1,vs24

  2962. 

  2963.     xvmulsp     vs36, vs0,vs25

  2964.     xvmulsp     vs37, vs1,vs25

  2965. 

  2966.     xvmulsp     vs40, vs0,vs26

  2967.     xvmulsp     vs41, vs1,vs26

  2968. 

  2969.     xvmulsp     vs44, vs0,vs27

  2970.     xvmulsp     vs45, vs1,vs27

  2971.  

  2972. 

  2973. .else

  2974.     xvmaddasp       vs32, vs0,vs24

  2975.     xvmaddasp       vs33, vs1,vs24

  2976. 

  2977.     xvmaddasp       vs36, vs0,vs25

  2978.     xvmaddasp       vs37, vs1,vs25

  2979. 

  2980.     xvmaddasp       vs40, vs0,vs26

  2981.     xvmaddasp       vs41, vs1,vs26

  2982. 

  2983.     xvmaddasp       vs44, vs0,vs27

  2984.     xvmaddasp       vs45, vs1,vs27

  2985.  

  2986. 

  2987. .endif

  2988. .endm  

  2989. 

  2990. .macro KERNEL4x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  2991. 

  2992.     lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG) 

  2993. 

  2994.     lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)

  2995.     lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)

  2996. 

  2997.     xxperm      vs10,   vs8,        permute_mask    

  2998.     xxpermdi    vs9,    vs8,    vs8,2     

  2999. 

  3000.     xvmaddasp       vs32, vs0,vs24

  3001.     xvmaddasp       vs33, vs1,vs24

  3002. 

  3003.     xvmaddasp       vs36, vs0,vs25

  3004.     xvmaddasp       vs37, vs1,vs25

  3005. 

  3006.     xxpermdi    vs11,   vs10,   vs10,2   

  3007. 

  3008.     xvmaddasp       vs40, vs0,vs26

  3009.     xvmaddasp       vs41, vs1,vs26

  3010. 

  3011.     xvmaddasp       vs44, vs0,vs27

  3012.     xvmaddasp       vs45, vs1,vs27

  3013. 

  3014.  

  3015. 

  3016.     lxv vs24,   DISP16(\Index,16+\OffsetB)(\BREG) 

  3017. 

  3018.     lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)

  3019.     lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)

  3020. 

  3021.     xxperm      vs26,   vs24,       permute_mask   

  3022.     xxpermdi    vs25,   vs24,   vs24,2      

  3023. 

  3024.     xvmaddasp       vs32, vs4,vs8

  3025.     xvmaddasp       vs33, vs5,vs8

  3026. 

  3027.     xvmaddasp       vs36, vs4,vs9

  3028.     xvmaddasp       vs37, vs5,vs9

  3029. 

  3030.     xxpermdi    vs27,   vs26,   vs26,2       

  3031. 

  3032.     xvmaddasp       vs40, vs4,vs10

  3033.     xvmaddasp       vs41, vs5,vs10

  3034. 

  3035.     xvmaddasp       vs44, vs4,vs11

  3036.     xvmaddasp       vs45, vs5,vs11

  3037. 

  3038.  

  3039. 

  3040.     lxv vs8,    DISP16(\Index,32+\OffsetB)(\BREG) 

  3041. 

  3042.     lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)

  3043.     lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)

  3044. 

  3045.     xxperm      vs10,   vs8,        permute_mask     

  3046.     xxpermdi    vs9,    vs8,    vs8,2     

  3047. 

  3048.     xvmaddasp       vs32, vs0,vs24

  3049.     xvmaddasp       vs33, vs1,vs24

  3050. 

  3051.     xvmaddasp       vs36, vs0,vs25

  3052.     xvmaddasp       vs37, vs1,vs25

  3053. 

  3054.     xxpermdi    vs11,   vs10,   vs10,2   

  3055. 

  3056.     xvmaddasp       vs40, vs0,vs26

  3057.     xvmaddasp       vs41, vs1,vs26

  3058. 

  3059.     xvmaddasp       vs44, vs0,vs27

  3060.     xvmaddasp       vs45, vs1,vs27

  3061. 

  3062.  

  3063. 

  3064. .if \Complete==0

  3065.     lxv vs24,   DISP16(\Index,48+\OffsetB)(\BREG) 

  3066. 

  3067.     lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)

  3068.     lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) 

  3069. 

  3070.     xxperm      vs26,   vs24,   permute_mask     

  3071.     xxpermdi    vs25,   vs24,   vs24,2      

  3072. 

  3073. .endif 

  3074. .if \IsLast==1  

  3075. .if \Complete==1

  3076.   

  3077.     addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)

  3078.     addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)

  3079. .else

  3080.   

  3081.     addi        \BREG, \BREG,  DISP16(\Index,64)

  3082.     addi        \AREG, \AREG, DISP32(\Index,128)

  3083. .endif

  3084. .endif   

  3085.  

  3086.     xvmaddasp       vs32, vs4,vs8

  3087.     xvmaddasp       vs33, vs5,vs8

  3088. 

  3089.     xvmaddasp       vs36, vs4,vs9

  3090.     xvmaddasp       vs37, vs5,vs9

  3091. 

  3092. .if \Complete==0        

  3093.     xxpermdi    vs27,   vs26,   vs26,2    

  3094.     

  3095. .endif

  3096.  

  3097.     xvmaddasp       vs40, vs4,vs10

  3098.     xvmaddasp       vs41, vs5,vs10

  3099. 

  3100.     xvmaddasp       vs44, vs4,vs11

  3101.     xvmaddasp       vs45, vs5,vs11

  3102. 

  3103.  

  3104. 

  3105. .endm

  3106. 

  3107. .macro KERNEL4x8 First

  3108. 

  3109.   LOAD4x8 0

  3110.   END4x8 \First, AO, BO, 32,16  

  3111. .endm

  3112. 

  3113. .macro KERNEL4x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

  3114.     

  3115.     lxv vs8,     DISP8(\Index, 0+\OffsetB)(\BREG) 

  3116.     lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)

  3117.     lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)

  3118. 

  3119.     xxperm      vs10,   vs8,        permute_mask  

  3120.     xxpermdi    vs9,    vs8,    vs8,2     

  3121. .if \First==1

  3122.     xvmulsp     vs32, vs0,vs24

  3123.     xvmulsp     vs33, vs1,vs24

  3124. 

  3125.     xvmulsp     vs36, vs0,vs25

  3126.     xvmulsp     vs37, vs1,vs25

  3127. 

  3128. .else

  3129.     xvmaddasp       vs32, vs0,vs24

  3130.     xvmaddasp       vs33, vs1,vs24

  3131. 

  3132.     xvmaddasp       vs36, vs0,vs25

  3133.     xvmaddasp       vs37, vs1,vs25

  3134. 

  3135. .endif

  3136. 

  3137.     xxpermdi    vs11,   vs10,   vs10,2    

  3138.  

  3139. .if \First==1  

  3140.     xvmulsp     vs40, vs0,vs26

  3141.     xvmulsp     vs41, vs1,vs26

  3142. 

  3143.     xvmulsp     vs44, vs0,vs27

  3144.     xvmulsp     vs45, vs1,vs27

  3145.  

  3146. 

  3147. .else 

  3148.     xvmaddasp       vs40, vs0,vs26

  3149.     xvmaddasp       vs41, vs1,vs26

  3150. 

  3151.     xvmaddasp       vs44, vs0,vs27

  3152.     xvmaddasp       vs45, vs1,vs27

  3153.  

  3154. 

  3155. .endif

  3156. .if \Complete==0

  3157.     lxv vs24,    DISP8(\Index,16+\OffsetB)(\BREG) 

  3158. 

  3159.     lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)

  3160.     lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)

  3161. 

  3162.     xxperm      vs26,   vs24,   permute_mask   

  3163.     xxpermdi    vs25,   vs24,   vs24,2    

  3164. .endif    

  3165. .if \IsLast==1  

  3166. .if \Complete==1

  3167.     addi        \BREG, \BREG,   DISP8(\Index,16+\OffsetB) 

  3168.     addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)

  3169. 

  3170. .else

  3171.     addi        \BREG, \BREG,   DISP8(\Index,32)

  3172.     addi        \AREG, \AREG,  DISP16(\Index,64) 

  3173. .endif

  3174. .endif

  3175. 

  3176. .if \First==1

  3177.     xvmulsp     vs32, vs4,vs8

  3178.     xvmulsp     vs33, vs5,vs8

  3179. 

  3180.     xvmulsp     vs36, vs4,vs9

  3181.     xvmulsp     vs37, vs5,vs9

  3182. 

  3183. .else

  3184.     xvmaddasp       vs32, vs4,vs8

  3185.     xvmaddasp       vs33, vs5,vs8

  3186. 

  3187.     xvmaddasp       vs36, vs4,vs9

  3188.     xvmaddasp       vs37, vs5,vs9

  3189. 

  3190. .endif 

  3191.  

  3192. .if \Complete==0        

  3193.     xxpermdi    vs27,   vs26,   vs26,2   

  3194.  

  3195. .endif

  3196. .if \First==1  

  3197.     xvmulsp     vs40, vs4,vs10

  3198.     xvmulsp     vs41, vs5,vs10

  3199. 

  3200.     xvmulsp     vs44, vs4,vs11

  3201.     xvmulsp     vs45, vs5,vs11

  3202.  

  3203. .else 

  3204.     xvmaddasp       vs40, vs4,vs10

  3205.     xvmaddasp       vs41, vs5,vs10

  3206. 

  3207.     xvmaddasp       vs44, vs4,vs11

  3208.     xvmaddasp       vs45, vs5,vs11 

  3209. 

  3210. .endif

  3211. 

  3212. .endm

  3213. 

  3214. 

  3215. .macro SAVE4x8 

  3216.  

  3217.   slwi    T10, LDC ,   1 

  3218.   add     T1, CO, LDC 

  3219. 

  3220.   add     T2, CO, T10  

  3221.   add     T3, T1, T10  

  3222. 

  3223.  

  3224. 

  3225. #ifndef TRMMKERNEL    

  3226.     lxv        vs34, 0(CO)

  3227.     lxv        vs35, 16(CO)      

  3228.     lxv        vs38, 0(T1)

  3229.     lxv        vs39, 16(T1)  

  3230.     lxv        vs42, 0(T2)

  3231.     lxv        vs43, 16(T2)     

  3232.     lxv        vs46, 0(T3)

  3233.     lxv        vs47, 16(T3)  

  3234. 

  3235.  

  3236. #endif  

  3237. 

  3238.     xxmrglw     vs8,    vs32,   vs44

  3239.     xxmrglw     vs10,   vs36,   vs40  

  3240. 

  3241.     xxmrghw     vs1,    vs32,   vs44

  3242.     xxmrghw     vs0,    vs36,   vs40

  3243. 

  3244.     xxmrglw     vs12,   vs33,   vs45

  3245.     xxmrglw     vs14,   vs37,   vs41  

  3246. 

  3247.     xxmrghw     vs2,    vs37,   vs41

  3248.     xxmrghw     vs3,    vs33,   vs45

  3249. 

  3250.     xxlor      vs9, vs8,    vs8

  3251.     xxlor      vs11,    vs10,   vs10 

  3252.  

  3253.     xxlor      vs13,    vs12,   vs12

  3254.     xxlor      vs15,    vs14,   vs14

  3255. 

  3256.     xxperm      vs8,    vs0,    save_permute_1

  3257.     xxperm      vs10,   vs1,    save_permute_1

  3258.     xxperm      vs9,    vs0,    save_permute_2  

  3259.     xxperm      vs11,   vs1,    save_permute_2      

  3260. 

  3261.     xxperm     vs12,    vs2,    save_permute_1

  3262.     xxperm     vs14,    vs3,    save_permute_1

  3263.       

  3264.     xxperm     vs13,    vs2,    save_permute_2   

  3265.     xxperm     vs15,    vs3,    save_permute_2      

  3266. 

  3267. 

  3268.     /* multiply add normal way */

  3269.  

  3270. #ifdef TRMMKERNEL

  3271.     xvmulsp     vs34,   vs8,    alpha_r 

  3272.     xvmulsp     vs35,   vs12,   alpha_r 

  3273.     xvmulsp     vs38,   vs9,    alpha_r 

  3274.     xvmulsp     vs39,   vs13,   alpha_r 

  3275.     xvmulsp     vs42,   vs10,   alpha_r 

  3276.     xvmulsp     vs43,   vs14,   alpha_r 

  3277.     xvmulsp     vs46,   vs11,   alpha_r 

  3278.     xvmulsp     vs47,   vs15,   alpha_r                    

  3279. #else 

  3280.     xvmaddasp   vs34,   vs8,    alpha_r 

  3281.     xvmaddasp   vs35,   vs12,   alpha_r 

  3282.     xvmaddasp   vs38,   vs9,    alpha_r 

  3283.     xvmaddasp   vs39,   vs13,   alpha_r  

  3284.     xvmaddasp   vs42,   vs10,   alpha_r 

  3285.     xvmaddasp   vs43,   vs14,   alpha_r   

  3286.     xvmaddasp   vs46,   vs11,   alpha_r 

  3287.     xvmaddasp   vs47,   vs15,   alpha_r                     

  3288. #endif     

  3289.  

  3290.     

  3291.     stxv        vs34, 0(CO)

  3292.     stxv        vs35, 16(CO)  

  3293.     stxv        vs38, 0(T1)

  3294.     stxv        vs39, 16(T1)  

  3295.     stxv        vs42, 0(T2)

  3296.     stxv        vs43, 16(T2)     

  3297.     stxv        vs46, 0(T3)

  3298.     stxv        vs47, 16(T3)  

  3299.   

  3300. 

  3301.     addi CO,CO,32

  3302. 

  3303. .endm

  3304. 

  3305. 

  3306. /**********************************************************************************************

  3307. * Macros for N=4 and M=4

  3308. **********************************************************************************************/

  3309. 

  3310. .macro LOAD4x4_1

  3311.    LOAD4x4 1

  3312. .endm

  3313. 

  3314. .macro LOAD4x4_0

  3315.    LOAD4x4 0

  3316. .endm

  3317. 

  3318. .macro KERNEL4x4_L1_L4  Index,IsLast

  3319.   KERNEL4x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0

  3320. .endm

  3321. 

  3322. .macro KERNEL4x4_I1_L4  OffsetA,OffsetB, Index,IsLast

  3323.   KERNEL4x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0

  3324. .endm

  3325. 

  3326. .macro KERNEL4x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast

  3327.   KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0

  3328. .endm

  3329. 

  3330. .macro KERNEL4x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast

  3331.   KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1

  3332. .endm

  3333. .macro KERNEL4x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast

  3334.    KERNEL4x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1

  3335. .endm

  3336. 

  3337. .macro KERNEL4x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  3338.   KERNEL4x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0

  3339. .endm

  3340. 

  3341. .macro KERNEL4x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast

  3342.   KERNEL4x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1

  3343. .endm

  3344. 

  3345. .macro Zero4X4

  3346.     xxlxor      vs32,   vs32,   vs32

  3347.     xxlxor      vs33,   vs33,   vs33 

  3348.     xxlxor      vs34,   vs34,   vs34

  3349.     xxlxor      vs35,   vs35,   vs35

  3350.  

  3351. .endm

  3352. 

  3353. .macro LOAD4x4  Zero

  3354. 

  3355.     lxv vs0,     0(AO)

  3356.     lxv vs24,   0(BO) 

  3357. 

  3358. 

  3359. 

  3360.     xxperm      vs2,   vs0,       permute_mask  

  3361.     xxpermdi    vs1,   vs0,   vs0,2      

  3362.     xxpermdi    vs3,   vs2,   vs2,2        

  3363. 

  3364. .if \Zero==1 

  3365.     xxlxor      vs32,   vs32,   vs32

  3366.     xxlxor      vs33,   vs33,   vs33 

  3367.     xxlxor      vs34,   vs34,   vs34

  3368.     xxlxor      vs35,   vs35,   vs35

  3369.  

  3370. .endif

  3371. .endm

  3372. 

  3373. .macro END4x4_NORMAL

  3374.   END4x4 0, AO, BO, 16,16 

  3375. .endm

  3376. 

  3377. .macro END4x4 First, AREG, BREG, OffsetA, OffsetB

  3378. 

  3379. .if \OffsetB != 0 

  3380.     addi        \BREG, \BREG, \OffsetB 

  3381. .endif

  3382. .if \OffsetA != 0 

  3383.     addi        \AREG, \AREG, \OffsetA 

  3384. .endif  

  3385. 

  3386. .if \First==1

  3387.     xvmulsp      vs32,   vs24,   vs0

  3388.     xvmulsp      vs33,   vs24,   vs1 

  3389.     xvmulsp      vs34,   vs24,   vs2

  3390.     xvmulsp      vs35,   vs24,   vs3  

  3391. .else

  3392.     xvmaddasp      vs32,   vs24,   vs0

  3393.     xvmaddasp      vs33,   vs24,   vs1 

  3394.     xvmaddasp      vs34,   vs24,   vs2

  3395.     xvmaddasp      vs35,   vs24,   vs3

  3396.  

  3397. 

  3398. .endif

  3399. .endm  

  3400. 

  3401. .macro KERNEL4x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

  3402. 

  3403.     lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)

  3404.     lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) 

  3405. 

  3406.     xxperm      vs6,   vs4,       permute_mask  

  3407.     xxpermdi    vs5,   vs4,   vs4,2      

  3408.     xxpermdi    vs7,   vs6,   vs6,2 

  3409.  

  3410.     xvmaddasp      vs32,   vs24,   vs0

  3411.     xvmaddasp      vs33,   vs24,   vs1 

  3412.     xvmaddasp      vs34,   vs24,   vs2

  3413.     xvmaddasp      vs35,   vs24,   vs3

  3414.  

  3415. 

  3416.     lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)

  3417.     lxv vs24,   DISP16(\Index, 16+\OffsetB)(\BREG)  

  3418. 

  3419.     xxperm      vs2,   vs0,       permute_mask  

  3420.     xxpermdi    vs1,   vs0,   vs0,2      

  3421.     xxpermdi    vs3,   vs2,   vs2,2   

  3422. 

  3423.     xvmaddasp      vs32,   vs26,   vs4

  3424.     xvmaddasp      vs33,   vs26,   vs5 

  3425.     xvmaddasp      vs34,   vs26,   vs6

  3426.     xvmaddasp      vs35,   vs26,   vs7

  3427.  

  3428.  

  3429. 

  3430.     lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)

  3431.     lxv vs26,   DISP16(\Index, 32+\OffsetB)(\BREG) 

  3432. 

  3433.     xxperm      vs6,   vs4,       permute_mask  

  3434.     xxpermdi    vs5,   vs4,   vs4,2      

  3435.     xxpermdi    vs7,   vs6,   vs6,2 

  3436.  

  3437.     xvmaddasp      vs32,   vs24,   vs0

  3438.     xvmaddasp      vs33,   vs24,   vs1 

  3439.     xvmaddasp      vs34,   vs24,   vs2

  3440.     xvmaddasp      vs35,   vs24,   vs3

  3441.  

  3442. 

  3443. .if \Complete==0 

  3444. 

  3445.     lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)

  3446.     lxv vs24,   DISP16(\Index, 48+\OffsetB)(\BREG) 

  3447. 

  3448.     xxperm      vs2,   vs0,       permute_mask  

  3449.     xxpermdi    vs1,   vs0,   vs0,2      

  3450.     xxpermdi    vs3,   vs2,   vs2,2   

  3451. .endif

  3452.     xvmaddasp      vs32,   vs26,   vs4

  3453.     xvmaddasp      vs33,   vs26,   vs5 

  3454.     xvmaddasp      vs34,   vs26,   vs6

  3455.     xvmaddasp      vs35,   vs26,   vs7

  3456.  

  3457. 

  3458.  

  3459.  

  3460. .if \IsLast==1  

  3461. .if \Complete==1

  3462.     addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  

  3463.     addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)

  3464. 

  3465. .else

  3466.     addi        \AREG, \AREG, DISP16(\Index,64)  

  3467.     addi        \BREG, \BREG,  DISP16(\Index,64)

  3468. 

  3469. .endif

  3470. .endif   

  3471.  

  3472.  

  3473. .endm

  3474. 

  3475. .macro KERNEL4x4 First

  3476.     LOAD4x4 0

  3477.     END4x4 \First, AO, BO, 16,16  

  3478. .endm

  3479. 

  3480. .macro KERNEL4x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

  3481. 

  3482.     lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)

  3483.     lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) 

  3484. 

  3485.     xxperm      vs6,   vs4,       permute_mask  

  3486.     xxpermdi    vs5,   vs4,   vs4,2      

  3487.     xxpermdi    vs7,   vs6,   vs6,2 

  3488. .if \First==1

  3489.     xvmulsp      vs32,   vs24,   vs0

  3490.     xvmulsp      vs33,   vs24,   vs1 

  3491.     xvmulsp      vs34,   vs24,   vs2

  3492.     xvmulsp      vs35,   vs24,   vs3

  3493.  

  3494. .else 

  3495.     xvmaddasp      vs32,   vs24,   vs0

  3496.     xvmaddasp      vs33,   vs24,   vs1 

  3497.     xvmaddasp      vs34,   vs24,   vs2

  3498.     xvmaddasp      vs35,   vs24,   vs3

  3499.  

  3500. .endif

  3501. 

  3502. .if \Complete==0 

  3503. 

  3504.     lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)

  3505.     lxv vs24,   DISP8(\Index, 16+\OffsetB)(\BREG) 

  3506. 

  3507.     xxperm      vs2,   vs0,       permute_mask  

  3508.     xxpermdi    vs1,   vs0,   vs0,2      

  3509.     xxpermdi    vs3,   vs2,   vs2,2   

  3510. .endif

  3511. 

  3512. .if \First==1

  3513.     xvmulsp      vs32,   vs26,   vs4

  3514.     xvmulsp      vs33,   vs26,   vs5 

  3515.     xvmulsp      vs34,   vs26,   vs6

  3516.     xvmulsp      vs35,   vs26,   vs7 

  3517. 

  3518. 

  3519. .else

  3520.     xvmaddasp      vs32,   vs26,   vs4

  3521.     xvmaddasp      vs33,   vs26,   vs5 

  3522.     xvmaddasp      vs34,   vs26,   vs6

  3523.     xvmaddasp      vs35,   vs26,   vs7

  3524.  

  3525. .endif

  3526.  

  3527.  

  3528. .if \IsLast==1  

  3529. .if \Complete==1

  3530.     addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  

  3531.     addi        \BREG, \BREG,  DISP8(\Index,16+\OffsetB)

  3532. 

  3533. .else

  3534.     addi        \AREG, \AREG, DISP8(\Index,32)  

  3535.     addi        \BREG, \BREG,  DISP8(\Index,32)

  3536. 

  3537. .endif

  3538. .endif   

  3539.      

  3540.   

  3541. .endm

  3542. 

  3543. 

  3544. .macro SAVE4x4

  3545.   slwi    T10, LDC ,   1 

  3546.   add     T1, CO, LDC 

  3547. #if !defined(TRMMKERNEL)  

  3548.   lxv        vs36, 0(CO)

  3549.   lxv        vs37, 0(T1)

  3550. #endif

  3551.   add     T2, CO, T10  

  3552.   add     T3, T1, T10 

  3553. #if !defined(TRMMKERNEL)   

  3554.   lxv        vs38, 0(T2)

  3555.   lxv        vs39, 0(T3)    

  3556. #endif   

  3557. 

  3558.   xxmrglw  vs0, vs35,vs32

  3559.   xxmrglw  vs1, vs34,vs33 

  3560.   xxmrglw  vs4, vs32,vs35

  3561.   xxmrglw  vs5, vs33,vs34 

  3562. 

  3563. 

  3564.   xxmrghw  vs2, vs35,vs32

  3565.   xxmrghw  vs3, vs34,vs33 

  3566.   xxmrghw  vs6, vs32,vs35

  3567.   xxmrghw  vs7, vs33,vs34  

  3568. 

  3569.   xxmrgld  vs24, vs1, vs0  

  3570.   xxmrghd  vs25,vs5,vs4 

  3571. 

  3572.   xxmrgld  vs26, vs2, vs3  

  3573.   xxmrghd  vs27,vs6,vs7

  3574. 

  3575.  #if defined(TRMMKERNEL)

  3576.   xvmulsp        vs36, vs24, alpha_r

  3577.   xvmulsp        vs37, vs25, alpha_r 

  3578.   xvmulsp        vs38, vs26, alpha_r

  3579.   xvmulsp        vs39, vs27, alpha_r 

  3580. #else

  3581.   xvmaddasp        vs36, vs24, alpha_r

  3582.   xvmaddasp        vs37, vs25, alpha_r 

  3583.   xvmaddasp        vs38, vs26, alpha_r

  3584.   xvmaddasp        vs39, vs27, alpha_r   

  3585.  #endif

  3586.   stxv        vs36, 0(CO)

  3587.   stxv        vs37, 0(T1) 

  3588.   stxv        vs38, 0(T2)

  3589.   stxv        vs39, 0(T3)   

  3590.  

  3591. 

  3592. 

  3593.   addi CO,CO,16

  3594. .endm

  3595. 

  3596. 

  3597. /**********************************************************************************************

  3598. * Macros for N=4 and M=2

  3599. **********************************************************************************************/

  3600. 

  3601.  

  3602. .macro KERNEL4x2_2   OffsetA,OffsetB, Index,IsLast

  3603.   KERNEL4x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast

  3604. .endm

  3605. 

  3606.  

  3607. 

  3608. .macro Zero4x2

  3609.     xxlxor      vs0,   vs0,   vs0 

  3610.     xxlxor      vs2,   vs2,   vs2 

  3611.        

  3612. .endm

  3613.  

  3614. .macro KERNEL4x2

  3615.   KERNEL4x2_1 AO,BO, 0, 0,0,0

  3616. .endm

  3617. .macro KERNEL4x2_1 AREG,BREG,First,OffsetA,OffsetB,Index

  3618. 

  3619. 

  3620.     lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)

  3621.     lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)      

  3622.     xxspltw   vs8,  vs36, 0 

  3623.     xxspltw   vs9,  vs36, 1  

  3624.  

  3625. .if \First==1

  3626.     xvmulsp      vs0,   vs26,   vs8 

  3627.     xvmulsp      vs2,   vs26,   vs9 

  3628.      

  3629. .else 

  3630.     xvmaddasp      vs0,   vs26,   vs8  

  3631.     xvmaddasp      vs2,   vs26,   vs9 

  3632.  

  3633.  .endif

  3634.    

  3635.     addi        \AREG, \AREG, DISP2(\Index,8)  

  3636.     addi        \BREG, \BREG, DISP4(\Index,16)

  3637.  

  3638. .endm

  3639. 

  3640. .macro KERNEL4x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

  3641. 

  3642.     lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)

  3643.     lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) 

  3644.     lxv vs28,   DISP8(\Index,16+\OffsetB)(\BREG)       

  3645.     xxspltw   vs8,  vs4, 2  

  3646.     xxspltw   vs9,  vs4, 3 

  3647.     xxspltw   vs10, vs4, 0 

  3648.     xxspltw   vs11, vs4, 1

  3649.  

  3650. .if \First==1

  3651.     xvmulsp      vs0,   vs26,   vs8 

  3652.     xvmulsp      vs2,   vs26,   vs9  

  3653. 

  3654.     xvmulsp      vs0,   vs28,   vs10 

  3655.     xvmulsp      vs2,   vs28,   vs11     

  3656. .else 

  3657.     xvmaddasp      vs0,   vs26,   vs8 

  3658.     xvmaddasp      vs2,   vs26,   vs9 

  3659. 

  3660.     xvmaddasp      vs0,   vs28,   vs10 

  3661.     xvmaddasp      vs2,   vs28,   vs11   

  3662.  .endif

  3663. 

  3664.  

  3665. .if \IsLast==1   

  3666.     addi        \AREG, \AREG, DISP4(\Index,16)  

  3667.     addi        \BREG, \BREG, DISP8(\Index,32)

  3668. .endif 

  3669.   

  3670. .endm

  3671. 

  3672. 

  3673. .macro SAVE4x2

  3674.   slwi    T10, LDC ,   1 

  3675.   add     T1, CO, LDC  

  3676.   add     T2, CO, T10  

  3677.   add     T3, T1, T10     

  3678.   /*convert alpha_r for multiply*/

  3679.   xscvspdp  vs4,alpha_r

  3680. /* v0 corresponds to vs32, do not forget*/

  3681. #if !defined(TRMMKERNEL)

  3682.   lxssp  v0,0(CO) 

  3683.   lxssp  v1,4(CO) 

  3684. 

  3685.   lxssp  v2,0(T1)

  3686.   lxssp  v3,4(T1)

  3687. 

  3688.   lxssp  v4,0(T2)

  3689.   lxssp  v5,4(T2)

  3690. 

  3691.   lxssp  v6,0(T3)

  3692.   lxssp  v7,4(T3)

  3693. 

  3694.    

  3695. #endif

  3696.   xscvspdp  vs5, vs2

  3697.   xxspltw   vs6, vs2, 1 

  3698.   xxspltw   vs7, vs2, 2 

  3699.   xxspltw   vs8, vs2, 3  

  3700.   xscvspdp  vs6,vs6

  3701.   xscvspdp  vs7,vs7

  3702.   xscvspdp  vs8,vs8

  3703. 

  3704.   xscvspdp  vs24, vs0

  3705.   xxspltw   vs25, vs0, 1 

  3706.   xxspltw   vs26, vs0, 2 

  3707.   xxspltw   vs27, vs0, 3  

  3708.   xscvspdp  vs25,vs25

  3709.   xscvspdp  vs26,vs26

  3710.   xscvspdp  vs27,vs27

  3711.  

  3712. 

  3713. #if defined(TRMMKERNEL)

  3714.   xsmuldp  vs32,vs8, vs4 

  3715.   xsmuldp  vs33,vs27, vs4 

  3716. 

  3717.   xsmuldp  vs34,vs7, vs4 

  3718.   xsmuldp  vs35,vs26, vs4 

  3719. 

  3720.   xsmuldp  vs36,vs6, vs4 

  3721.   xsmuldp  vs37,vs25, vs4  

  3722. 

  3723.   xsmuldp  vs38,vs5, vs4 

  3724.   xsmuldp  vs39,vs24, vs4  

  3725. 

  3726.       

  3727. #else

  3728.   xsmaddadp  vs32,vs8, vs4 

  3729.   xsmaddadp  vs33,vs27, vs4 

  3730. 

  3731.   xsmaddadp  vs34,vs7, vs4 

  3732.   xsmaddadp  vs35,vs26, vs4 

  3733. 

  3734.   xsmaddadp  vs36,vs6, vs4 

  3735.   xsmaddadp  vs37,vs25, vs4  

  3736. 

  3737.   xsmaddadp  vs38,vs5, vs4 

  3738.   xsmaddadp  vs39,vs24, vs4  

  3739. 

  3740.     

  3741. #endif  

  3742. 

  3743.   stxssp  v0,0(CO) 

  3744.   stxssp  v1,4(CO) 

  3745. 

  3746.   stxssp  v2,0(T1)

  3747.   stxssp  v3,4(T1)

  3748. 

  3749.   stxssp  v4,0(T2)

  3750.   stxssp  v5,4(T2)

  3751. 

  3752.   stxssp  v6,0(T3)

  3753.   stxssp  v7,4(T3)

  3754. 

  3755.  

  3756.  

  3757. 

  3758.   addi CO,CO,8

  3759. .endm

  3760. 

  3761. 

  3762. /**********************************************************************************************

  3763. * Macros for N=4 and M=1

  3764. **********************************************************************************************/

  3765. .macro KERNEL4x1_4   OffsetA,OffsetB, Index,IsLast

  3766.   KERNEL4x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast

  3767. .endm

  3768. 

  3769. .macro Zero4x1

  3770.     xxlxor      vs0,   vs0,   vs0 

  3771. .endm

  3772. 

  3773. .macro KERNEL4x1

  3774.   KERNEL4x1_1 AO,BO, 0 

  3775. .endm

  3776. 

  3777. .macro KERNEL4x1_2

  3778.   KERNEL4x1_2_1 AO,BO, 0 

  3779. .endm

  3780. 

  3781. .macro KERNEL4x1_1 AREG,BREG,First 

  3782.     lxvwsx vs8,  0, \AREG

  3783.     lxv vs26,   0(\BREG)       

  3784. .if \First==1

  3785.     xvmulsp      vs0,   vs26,   vs8  

  3786. .else 

  3787.     xvmaddasp      vs0,   vs26,   vs8  

  3788.  .endif

  3789.     addi        \AREG, \AREG,  4  

  3790.     addi        \BREG, \BREG,  16

  3791. .endm

  3792. 

  3793. .macro KERNEL4x1_2_1 AREG,BREG,First 

  3794.     lxsd v4,    0(\AREG)

  3795.     lxv vs26,   0(\BREG)      

  3796.     lxv vs28,  16(\BREG) 

  3797.     xxspltw   vs8,  vs36, 1 

  3798.     xxspltw   vs9,  vs36, 0  

  3799. .if \First==1

  3800.     xvmulsp      vs0,   vs26,   vs8 

  3801.     xvmulsp      vs0,   vs28,   vs9     

  3802. .else 

  3803.     xvmaddasp      vs0,   vs26,   vs8  

  3804.     xvmaddasp      vs0,   vs28,   vs9  

  3805.  .endif

  3806.     addi        \AREG, \AREG,  8 

  3807.     addi        \BREG, \BREG,  32

  3808. .endm

  3809. 

  3810. .macro KERNEL4x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

  3811.     lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)

  3812.     xxspltw   vs8,  vs4, 3 

  3813.     xxspltw   vs9,  vs4, 2 

  3814.     xxspltw   vs10, vs4, 1 

  3815.     xxspltw   vs11, vs4, 0

  3816.     lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) 

  3817.     lxv vs28,   DISP16(\Index,16+\OffsetB)(\BREG)  

  3818.     lxv vs30,   DISP16(\Index,32+\OffsetB)(\BREG) 

  3819.     lxv vs32,   DISP16(\Index,48+\OffsetB)(\BREG)          

  3820. .if \First==1

  3821.     xvmulsp      vs0,   vs26,   vs8  

  3822.     xvmulsp      vs0,   vs28,   vs9      

  3823.     xvmulsp      vs0,   vs30,   vs10  

  3824.     xvmulsp      vs0,   vs32,   vs11     

  3825. .else 

  3826.     xvmaddasp      vs0,   vs26,   vs8  

  3827.     xvmaddasp      vs0,   vs28,   vs9     

  3828.     xvmaddasp      vs0,   vs30,   vs10  

  3829.     xvmaddasp      vs0,   vs32,   vs11  

  3830.  .endif

  3831. .if \IsLast==1   

  3832.     addi        \AREG, \AREG, DISP4(\Index,16)  

  3833.     addi        \BREG, \BREG, DISP16(\Index,64)

  3834. .endif 

  3835. .endm

  3836. 

  3837. .macro SAVE4x1

  3838.   slwi    T10, LDC ,   1 

  3839.   add     T1, CO, LDC  

  3840.   add     T2, CO, T10  

  3841.   add     T3, T1, T10     

  3842.   /*convert alpha_r for multiply*/

  3843.   xscvspdp  vs4,alpha_r

  3844. /* v0 corresponds to vs32, do not forget*/

  3845. #if !defined(TRMMKERNEL)

  3846.   lxssp  v0,0(CO)  

  3847.   lxssp  v2,0(T1) 

  3848.   lxssp  v4,0(T2) 

  3849.   lxssp  v6,0(T3)  

  3850. #endif

  3851.   xscvspdp  vs24, vs0

  3852.   xxspltw   vs25, vs0, 1 

  3853.   xxspltw   vs26, vs0, 2 

  3854.   xxspltw   vs27, vs0, 3  

  3855.   xscvspdp  vs25,vs25

  3856.   xscvspdp  vs26,vs26

  3857.   xscvspdp  vs27,vs27

  3858. 

  3859. #if defined(TRMMKERNEL)

  3860.   xsmuldp  vs32,vs27, vs4 

  3861.   xsmuldp  vs34,vs26, vs4 

  3862.   xsmuldp  vs36,vs25, vs4 

  3863.   xsmuldp  vs38,vs24, vs4  

  3864. #else

  3865.   xsmaddadp  vs32,vs27, vs4 

  3866.   xsmaddadp  vs34,vs26, vs4 

  3867.   xsmaddadp  vs36,vs25, vs4 

  3868.   xsmaddadp  vs38,vs24, vs4   

  3869. #endif  

  3870.   stxssp  v0,0(CO)  

  3871.   stxssp  v2,0(T1) 

  3872.   stxssp  v4,0(T2) 

  3873.   stxssp  v6,0(T3)  

  3874.   addi CO,CO,4

  3875. .endm

  3876. 

  3877. /****************************N=2 section*****************/

  3878. 

  3879. .macro KERNEL2x16_2   OffsetA,OffsetB, Index,IsLast

  3880.   KERNEL2x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  3881. .endm

  3882. 

  3883.  

  3884. .macro Zero2x16

  3885.     xxlxor      vs0,   vs0,   vs0

  3886.     xxlxor      vs1,   vs1,   vs1 

  3887.     xxlxor      vs2,   vs2,   vs2

  3888.     xxlxor      vs3,   vs3,   vs3

  3889.     xxlxor      vs4,   vs4,   vs4

  3890.     xxlxor      vs5,   vs5,   vs5 

  3891.     xxlxor      vs6,   vs6,   vs6

  3892.     xxlxor      vs7,   vs7,   vs7      

  3893. .endm

  3894.  

  3895. .macro KERNEL2x16

  3896.   KERNEL2x16_1 AO,BO, 0, 0,0,0

  3897. .endm

  3898. .macro KERNEL2x16_4 OffsetA,OffsetB, Index,IsLast  

  3899.   KERNEL2x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  3900. .endm

  3901.  

  3902. .macro KERNEL2x16_1 AREG,BREG,First,OffsetA,OffsetB,Index

  3903. 

  3904. 

  3905.     lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)

  3906.     xxspltw   vs8,  vs36, 1 

  3907.     xxspltw   vs9,  vs36, 0 

  3908.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  3909.     lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   

  3910.     lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)

  3911.     lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        

  3912.  

  3913.  

  3914. .if \First==1

  3915.     xvmulsp      vs0,   vs26,   vs8

  3916.     xvmulsp      vs1,   vs27,   vs8 

  3917.     xvmulsp      vs2,   vs28,   vs8

  3918.     xvmulsp      vs3,   vs29,   vs8 

  3919. 

  3920.     xvmulsp      vs4,   vs26,   vs9

  3921.     xvmulsp      vs5,   vs27,   vs9 

  3922.     xvmulsp      vs6,   vs28,   vs9

  3923.     xvmulsp      vs7,   vs29,   vs9     

  3924.      

  3925. .else 

  3926.     xvmaddasp      vs0,   vs26,   vs8

  3927.     xvmaddasp      vs1,   vs27,   vs8 

  3928.     xvmaddasp      vs2,   vs28,   vs8

  3929.     xvmaddasp      vs3,   vs29,   vs8 

  3930. 

  3931.     xvmaddasp      vs4,   vs26,   vs9

  3932.     xvmaddasp      vs5,   vs27,   vs9 

  3933.     xvmaddasp      vs6,   vs28,   vs9

  3934.     xvmaddasp      vs7,   vs29,   vs9

  3935.  

  3936.  .endif

  3937.    

  3938.     addi        \BREG, \BREG, DISP2(\Index,8)

  3939.     addi        \AREG, \AREG, DISP16(\Index,64)  

  3940.  

  3941. .endm

  3942. 

  3943. 

  3944. 

  3945. 

  3946. .macro KERNEL2x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  3947. 

  3948.     lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)

  3949.     lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

  3950. 

  3951.     lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)

  3952.     lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)

  3953.     lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)

  3954.     lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  

  3955. 

  3956.     lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)

  3957.     lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)

  3958.     lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)

  3959.     lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) 

  3960. 

  3961.     lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)

  3962.     lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)

  3963.     lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)

  3964.     lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  

  3965.         

  3966.     lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)

  3967.     lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)

  3968.     lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)

  3969.     lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) 

  3970. 

  3971.     xxspltw   vs8,  vs38, 3  

  3972.     xxspltw   vs9,  vs38, 2 

  3973.     xxspltw   vs10, vs38, 1 

  3974.     xxspltw   vs11, vs38, 0  

  3975. 

  3976.     xxspltw   vs12,  vs39, 3  

  3977.     xxspltw   vs13,  vs39, 2 

  3978.     xxspltw   vs14, vs39, 1 

  3979.     xxspltw   vs15, vs39, 0  

  3980. 

  3981.  

  3982.     xvmaddasp      vs0,   vs26,   vs8

  3983.     xvmaddasp      vs1,   vs27,   vs8 

  3984.     xvmaddasp      vs2,   vs28,   vs8

  3985.     xvmaddasp      vs3,   vs29,   vs8 

  3986. 

  3987.     xvmaddasp      vs4,   vs26,   vs9

  3988.     xvmaddasp      vs5,   vs27,   vs9 

  3989.     xvmaddasp      vs6,   vs28,   vs9

  3990.     xvmaddasp      vs7,   vs29,   vs9 

  3991. 

  3992.     xvmaddasp      vs0,   vs16,   vs10

  3993.     xvmaddasp      vs1,   vs17,   vs10 

  3994.     xvmaddasp      vs2,   vs18,   vs10

  3995.     xvmaddasp      vs3,   vs19,   vs10 

  3996. 

  3997.     xvmaddasp      vs4,   vs16,   vs11

  3998.     xvmaddasp      vs5,   vs17,   vs11 

  3999.     xvmaddasp      vs6,   vs18,   vs11

  4000.     xvmaddasp      vs7,   vs19,   vs11  

  4001. 

  4002.     xvmaddasp      vs0,   vs30,   vs12

  4003.     xvmaddasp      vs1,   vs31,   vs12 

  4004.     xvmaddasp      vs2,   vs32,   vs12

  4005.     xvmaddasp      vs3,   vs33,   vs12 

  4006. 

  4007.     xvmaddasp      vs4,   vs30,   vs13

  4008.     xvmaddasp      vs5,   vs31,   vs13 

  4009.     xvmaddasp      vs6,   vs32,   vs13

  4010.     xvmaddasp      vs7,   vs33,   vs13 

  4011. 

  4012.     xvmaddasp      vs0,   vs34,   vs14

  4013.     xvmaddasp      vs1,   vs35,   vs14 

  4014.     xvmaddasp      vs2,   vs36,   vs14

  4015.     xvmaddasp      vs3,   vs37,   vs14 

  4016. 

  4017.     xvmaddasp      vs4,   vs34,   vs15

  4018.     xvmaddasp      vs5,   vs35,   vs15 

  4019.     xvmaddasp      vs6,   vs36,   vs15

  4020.     xvmaddasp      vs7,   vs37,   vs15    

  4021.  

  4022.  

  4023. .if \IsLast==1   

  4024.     addi        \BREG, \BREG, DISP8(\Index,32)  

  4025.     addi        \AREG, \AREG, DISP64(\Index,256)

  4026. .endif 

  4027.   

  4028. .endm

  4029. 

  4030. .macro KERNEL2x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4031. 

  4032.     lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)

  4033.     xxspltw   vs8,  vs36, 3  

  4034.     xxspltw   vs9,  vs36, 2 

  4035.     xxspltw   vs10, vs36, 1 

  4036.     xxspltw   vs11, vs36, 0    

  4037.     lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)

  4038.     lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)

  4039.     lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)

  4040.     lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      

  4041.     lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)

  4042.     lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)

  4043.     lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)

  4044.     lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) 

  4045.  

  4046.  

  4047.     xvmaddasp      vs0,   vs26,   vs8

  4048.     xvmaddasp      vs1,   vs27,   vs8 

  4049.     xvmaddasp      vs2,   vs28,   vs8

  4050.     xvmaddasp      vs3,   vs29,   vs8 

  4051. 

  4052.     xvmaddasp      vs4,   vs26,   vs9

  4053.     xvmaddasp      vs5,   vs27,   vs9 

  4054.     xvmaddasp      vs6,   vs28,   vs9

  4055.     xvmaddasp      vs7,   vs29,   vs9 

  4056. 

  4057.     xvmaddasp      vs0,   vs16,   vs10

  4058.     xvmaddasp      vs1,   vs17,   vs10 

  4059.     xvmaddasp      vs2,   vs18,   vs10

  4060.     xvmaddasp      vs3,   vs19,   vs10 

  4061. 

  4062.     xvmaddasp      vs4,   vs16,   vs11

  4063.     xvmaddasp      vs5,   vs17,   vs11 

  4064.     xvmaddasp      vs6,   vs18,   vs11

  4065.     xvmaddasp      vs7,   vs19,   vs11   

  4066.  

  4067. .if \IsLast==1   

  4068.     addi        \BREG, \BREG, DISP4(\Index,16)  

  4069.     addi        \AREG, \AREG, DISP32(\Index,128)

  4070. .endif 

  4071.   

  4072. .endm

  4073. 

  4074. 

  4075. .macro SAVE2x16

  4076. 

  4077. #ifndef TRMMKERNEL    

  4078.     lxv        vs16, 0(CO)

  4079.     lxv        vs17, 16(CO) 

  4080.     lxv        vs18, 32(CO)  

  4081.     lxv        vs19, 48(CO)      

  4082. #endif

  4083.   add     T1, CO, LDC 

  4084. #ifndef TRMMKERNEL    

  4085.     lxv        vs26, 0(T1)

  4086.     lxv        vs27, 16(T1) 

  4087.     lxv        vs28, 32(T1)  

  4088.     lxv        vs29, 48(T1)      

  4089. #endif

  4090. 

  4091. #if defined(TRMMKERNEL)

  4092.   xvmulsp        vs16, vs0, alpha_r

  4093.   xvmulsp        vs17, vs1, alpha_r 

  4094.   xvmulsp        vs18, vs2, alpha_r

  4095.   xvmulsp        vs19, vs3, alpha_r   

  4096.   xvmulsp        vs26, vs4, alpha_r

  4097.   xvmulsp        vs27, vs5, alpha_r 

  4098.   xvmulsp        vs28, vs6, alpha_r

  4099.   xvmulsp        vs29, vs7, alpha_r

  4100. #else

  4101.   xvmaddasp        vs16, vs0, alpha_r

  4102.   xvmaddasp        vs17, vs1, alpha_r 

  4103.   xvmaddasp        vs18, vs2, alpha_r

  4104.   xvmaddasp        vs19, vs3, alpha_r   

  4105.   xvmaddasp        vs26, vs4, alpha_r

  4106.   xvmaddasp        vs27, vs5, alpha_r 

  4107.   xvmaddasp        vs28, vs6, alpha_r

  4108.   xvmaddasp        vs29, vs7, alpha_r

  4109. #endif

  4110.     stxv        vs16, 0(CO)

  4111.     stxv        vs17, 16(CO) 

  4112.     stxv        vs18, 32(CO)  

  4113.     stxv        vs19, 48(CO)      

  4114.     

  4115.     stxv        vs26, 0(T1)

  4116.     stxv        vs27, 16(T1) 

  4117.     stxv        vs28, 32(T1)  

  4118.     stxv        vs29, 48(T1) 

  4119.  

  4120.   addi CO,CO,64

  4121. 

  4122. .endm

  4123. 

  4124. /*       M=8 N=2 */

  4125. 

  4126. .macro KERNEL2x8_2   OffsetA,OffsetB, Index,IsLast

  4127.   KERNEL2x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4128. .endm

  4129. 

  4130.  

  4131. .macro Zero2x8

  4132.     xxlxor      vs0,   vs0,   vs0

  4133.     xxlxor      vs1,   vs1,   vs1 

  4134.  

  4135.     xxlxor      vs4,   vs4,   vs4

  4136.     xxlxor      vs5,   vs5,   vs5 

  4137.      

  4138. .endm

  4139.  

  4140. .macro KERNEL2x8

  4141.   KERNEL2x8_1 AO,BO, 0, 0,0,0

  4142. .endm

  4143. .macro KERNEL2x8_4 OffsetA,OffsetB, Index,IsLast  

  4144.   KERNEL2x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4145. .endm

  4146.  

  4147. .macro KERNEL2x8_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4148. 

  4149. 

  4150.     lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)

  4151.     xxspltw   vs8,  vs36, 1 

  4152.     xxspltw   vs9,  vs36, 0 

  4153.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)

  4154.     lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)          

  4155.  

  4156.  

  4157. .if \First==1

  4158.     xvmulsp      vs0,   vs26,   vs8

  4159.     xvmulsp      vs1,   vs27,   vs8  

  4160. 

  4161.     xvmulsp      vs4,   vs26,   vs9

  4162.     xvmulsp      vs5,   vs27,   vs9      

  4163.      

  4164. .else 

  4165.     xvmaddasp      vs0,   vs26,   vs8

  4166.     xvmaddasp      vs1,   vs27,   vs8   

  4167. 

  4168.     xvmaddasp      vs4,   vs26,   vs9

  4169.     xvmaddasp      vs5,   vs27,   vs9  

  4170.  

  4171.  .endif

  4172.    

  4173.     addi        \BREG, \BREG, DISP2(\Index,8)

  4174.     addi        \AREG, \AREG, DISP8(\Index,32)  

  4175.  

  4176. .endm

  4177. 

  4178. 

  4179. 

  4180. 

  4181. .macro KERNEL2x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4182. 

  4183.     lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)

  4184.     lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

  4185. 

  4186.     lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)

  4187.     lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) 

  4188. 

  4189.     lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)

  4190.     lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) 

  4191. 

  4192.     lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)

  4193.     lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)  

  4194.         

  4195.     lxv vs34,   DISP32(\Index, 96+ 0+\OffsetA)(\AREG)

  4196.     lxv vs35,   DISP32(\Index, 96+ 16+\OffsetA)(\AREG) 

  4197. 

  4198.     xxspltw   vs8,  vs38, 3  

  4199.     xxspltw   vs9,  vs38, 2 

  4200.     xxspltw   vs10, vs38, 1 

  4201.     xxspltw   vs11, vs38, 0  

  4202. 

  4203.     xxspltw   vs12,  vs39, 3  

  4204.     xxspltw   vs13,  vs39, 2 

  4205.     xxspltw   vs14, vs39, 1 

  4206.     xxspltw   vs15, vs39, 0  

  4207. 

  4208.  

  4209.  

  4210.     xvmaddasp      vs0,   vs26,   vs8

  4211.     xvmaddasp      vs1,   vs27,   vs8 

  4212.     xvmaddasp      vs4,   vs26,   vs9

  4213.     xvmaddasp      vs5,   vs27,   vs9 

  4214.  

  4215. 

  4216.     xvmaddasp      vs0,   vs16,   vs10

  4217.     xvmaddasp      vs1,   vs17,   vs10 

  4218.     xvmaddasp      vs4,   vs16,   vs11

  4219.     xvmaddasp      vs5,   vs17,   vs11 

  4220.  

  4221. 

  4222.     xvmaddasp      vs0,   vs30,   vs12

  4223.     xvmaddasp      vs1,   vs31,   vs12 

  4224.     xvmaddasp      vs4,   vs30,   vs13

  4225.     xvmaddasp      vs5,   vs31,   vs13 

  4226. 

  4227.     xvmaddasp      vs0,   vs34,   vs14

  4228.     xvmaddasp      vs1,   vs35,   vs14 

  4229.     xvmaddasp      vs4,   vs34,   vs15

  4230.     xvmaddasp      vs5,   vs35,   vs15 

  4231.    

  4232.  

  4233.  

  4234. .if \IsLast==1   

  4235.     addi        \BREG, \BREG, DISP8(\Index,32)  

  4236.     addi        \AREG, \AREG, DISP32(\Index,128)

  4237. .endif 

  4238.   

  4239. .endm

  4240. 

  4241. .macro KERNEL2x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4242. 

  4243.     lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)

  4244.     xxspltw   vs8,  vs36, 3  

  4245.     xxspltw   vs9,  vs36, 2 

  4246.     xxspltw   vs10, vs36, 1 

  4247.     xxspltw   vs11, vs36, 0    

  4248.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  4249.     lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)

  4250.     lxv vs16,   DISP16(\Index,32+\OffsetA)(\AREG)

  4251.     lxv vs17,   DISP16(\Index,48+\OffsetA)(\AREG)      

  4252.  

  4253.     xvmaddasp      vs0,   vs26,   vs8

  4254.     xvmaddasp      vs1,   vs27,   vs8  

  4255. 

  4256.     xvmaddasp      vs4,   vs26,   vs9

  4257.     xvmaddasp      vs5,   vs27,   vs9  

  4258. 

  4259.     xvmaddasp      vs0,   vs16,   vs10

  4260.     xvmaddasp      vs1,   vs17,   vs10   

  4261. 

  4262.     xvmaddasp      vs4,   vs16,   vs11

  4263.     xvmaddasp      vs5,   vs17,   vs11     

  4264.  

  4265. .if \IsLast==1   

  4266.     addi        \BREG, \BREG, DISP4(\Index,16)  

  4267.     addi        \AREG, \AREG, DISP16(\Index,64)

  4268. .endif 

  4269.   

  4270. .endm

  4271. 

  4272. 

  4273. .macro SAVE2x8

  4274. 

  4275. #ifndef TRMMKERNEL    

  4276.     lxv        vs16, 0(CO)

  4277.     lxv        vs17, 16(CO)     

  4278. #endif

  4279.   add     T1, CO, LDC 

  4280. #ifndef TRMMKERNEL    

  4281.     lxv        vs26, 0(T1)

  4282.     lxv        vs27, 16(T1) 

  4283.     

  4284. #endif

  4285. 

  4286. #if defined(TRMMKERNEL)

  4287.   xvmulsp        vs16, vs0, alpha_r

  4288.   xvmulsp        vs17, vs1, alpha_r  

  4289.   xvmulsp        vs26, vs4, alpha_r

  4290.   xvmulsp        vs27, vs5, alpha_r 

  4291. #else

  4292.   xvmaddasp        vs16, vs0, alpha_r

  4293.   xvmaddasp        vs17, vs1, alpha_r  

  4294.   xvmaddasp        vs26, vs4, alpha_r

  4295.   xvmaddasp        vs27, vs5, alpha_r 

  4296. #endif

  4297. 

  4298.     stxv        vs16, 0(CO)

  4299.     stxv        vs17, 16(CO) 

  4300.      

  4301.     

  4302.     stxv        vs26, 0(T1)

  4303.     stxv        vs27, 16(T1) 

  4304. 

  4305.   addi CO,CO,32

  4306. 

  4307. .endm

  4308. 

  4309. 

  4310. /*M=4*/

  4311. 

  4312. 

  4313. .macro KERNEL2x4_2   OffsetA,OffsetB, Index,IsLast

  4314.   KERNEL2x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4315. .endm

  4316. 

  4317.  /* we will aggregate on save vs0 +vs4 vs11+vs5 */

  4318. .macro Zero2x4

  4319.     xxlxor      vs0,   vs0,   vs0

  4320.     xxlxor      vs1,   vs1,   vs1 

  4321.  

  4322.     xxlxor      vs4,   vs4,   vs4

  4323.     xxlxor      vs5,   vs5,   vs5 

  4324.     

  4325. .endm

  4326.  

  4327. .macro KERNEL2x4

  4328.   KERNEL2x4_1 AO,BO, 0, 0,0,0

  4329. .endm

  4330. .macro KERNEL2x4_4 OffsetA,OffsetB, Index,IsLast  

  4331.   KERNEL2x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4332. .endm

  4333.  

  4334. .macro KERNEL2x4_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4335. 

  4336. 

  4337.     lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)

  4338.     xxspltw   vs8,  vs36, 1 

  4339.     xxspltw   vs9,  vs36, 0 

  4340.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)        

  4341.  

  4342.  

  4343. .if \First==1

  4344.     xvmulsp      vs0,   vs26,   vs8 

  4345.     xvmulsp      vs1,   vs26,   vs9     

  4346.      

  4347. .else 

  4348.     xvmaddasp      vs0,   vs26,   vs8 

  4349.     xvmaddasp      vs1,   vs26,   vs9 

  4350.  .endif

  4351.    

  4352.     addi        \BREG, \BREG, DISP2(\Index,8)

  4353.     addi        \AREG, \AREG, DISP4(\Index,16)  

  4354.  

  4355. .endm

  4356. 

  4357. 

  4358. 

  4359. 

  4360. .macro KERNEL2x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4361. 

  4362.     lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)

  4363.     lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

  4364. 

  4365.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  4366.     lxv vs16,   DISP16(\Index,16+\OffsetA)(\AREG) 

  4367. 

  4368.     lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)

  4369.     lxv vs34,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) 

  4370.  

  4371. 

  4372.     xxspltw   vs8,  vs38, 3  

  4373.     xxspltw   vs9,  vs38, 2 

  4374.     xxspltw   vs10, vs38, 1 

  4375.     xxspltw   vs11, vs38, 0  

  4376. 

  4377.     xxspltw   vs12,  vs39, 3  

  4378.     xxspltw   vs13,  vs39, 2 

  4379.     xxspltw   vs14, vs39, 1 

  4380.     xxspltw   vs15, vs39, 0  

  4381. 

  4382.  

  4383.     xvmaddasp      vs0,   vs26,   vs8

  4384.     xvmaddasp      vs1,   vs26,   vs9 

  4385.     xvmaddasp      vs4,   vs16,   vs10

  4386.     xvmaddasp      vs5,   vs16,   vs11 

  4387.  

  4388. 

  4389.     xvmaddasp      vs0,   vs30,   vs12

  4390.     xvmaddasp      vs1,   vs30,   vs13 

  4391.     xvmaddasp      vs4,   vs34,   vs14

  4392.     xvmaddasp      vs5,   vs34,   vs15 

  4393.  

  4394.    

  4395.  

  4396.  

  4397. .if \IsLast==1   

  4398.     addi        \BREG, \BREG, DISP8(\Index,32)  

  4399.     addi        \AREG, \AREG, DISP16(\Index,64)

  4400. .endif 

  4401.   

  4402. .endm

  4403. 

  4404. .macro KERNEL2x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4405. 

  4406.     lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)

  4407.     xxspltw   vs8,  vs36, 3  

  4408.     xxspltw   vs9,  vs36, 2 

  4409.     xxspltw   vs10, vs36, 1 

  4410.     xxspltw   vs11, vs36, 0    

  4411.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)

  4412.     lxv vs16,   DISP8(\Index, 16+\OffsetA)(\AREG)      

  4413.  

  4414.     xvmaddasp      vs0,   vs26,   vs8

  4415.     xvmaddasp      vs1,   vs26,   vs9 

  4416.     xvmaddasp      vs4,   vs16,   vs10

  4417.     xvmaddasp      vs5,   vs16,   vs11     

  4418.  

  4419. .if \IsLast==1   

  4420.     addi        \BREG, \BREG, DISP4(\Index,16)  

  4421.     addi        \AREG, \AREG, DISP8(\Index,32)

  4422. .endif 

  4423.   

  4424. .endm

  4425. 

  4426. 

  4427. .macro SAVE2x4

  4428. 

  4429. #ifndef TRMMKERNEL    

  4430.     lxv        vs16, 0(CO)     

  4431. #endif

  4432.   add     T1, CO, LDC 

  4433. #ifndef TRMMKERNEL    

  4434.     lxv        vs26, 0(T1) 

  4435.     

  4436. #endif

  4437.     /*aggregate vectors*/

  4438.   xvaddsp         vs0,vs0,vs4

  4439.   xvaddsp         vs1,vs1,vs5 

  4440. #if defined(TRMMKERNEL)

  4441.   xvmulsp        vs16, vs0, alpha_r 

  4442.   xvmulsp        vs26, vs1, alpha_r 

  4443. #else

  4444.   xvmaddasp        vs16, vs0, alpha_r 

  4445.   xvmaddasp        vs26, vs1, alpha_r 

  4446. #endif

  4447. 

  4448.   stxv        vs16, 0(CO) 

  4449.   stxv        vs26, 0(T1)  

  4450. 

  4451.   addi CO,CO,16

  4452. 

  4453. .endm

  4454. 

  4455. 

  4456. /* M=2 N=2 we will have inner pemrute action before permute was revrsing 3,2,1,0 not iw 2ill inner reverse 1,0,3,2  */

  4457. .macro SWITCH_PERMUTE_INNER

  4458.     xxpermdi	permute_mask,	permute_mask,	permute_mask,2

  4459. .endm

  4460. 

  4461. .macro Zero2x2

  4462.     xxlxor      vs0,   vs0,   vs0

  4463.     xxlxor      vs1,   vs1,   vs1 

  4464.     SWITCH_PERMUTE_INNER

  4465. .endm

  4466.  

  4467. .macro KERNEL2x2

  4468.   KERNEL2x2_1 AO,BO, 0, 0,0,0

  4469. .endm

  4470. .macro KERNEL2x2_4 OffsetA,OffsetB, Index,IsLast  

  4471.   KERNEL2x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4472. .endm

  4473. 

  4474. .macro KERNEL2x2_2 OffsetA,OffsetB, Index,IsLast  

  4475.   KERNEL2x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4476. .endm

  4477.  

  4478. .macro KERNEL2x2_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4479. 

  4480. 

  4481.     lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)

  4482.     xxperm   vs9,  vs36, permute_mask 

  4483.     lxsd v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        

  4484.  

  4485.  

  4486. .if \First==1

  4487.     xvmulsp      vs0,   vs37,   vs36 

  4488.     xvmulsp      vs1,   vs37,   vs9     

  4489.      

  4490. .else 

  4491.     xvmaddasp      vs0,   vs37,   vs36 

  4492.     xvmaddasp      vs1,   vs37,   vs9 

  4493.  .endif

  4494.    

  4495.     addi        \BREG, \BREG, DISP2(\Index,8)

  4496.     addi        \AREG, \AREG, DISP2(\Index,8)  

  4497.  

  4498. .endm

  4499. 

  4500. 

  4501. 

  4502. 

  4503. .macro KERNEL2x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4504. 

  4505.     lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)

  4506.     lxv vs10,    DISP8(\Index, 16+\OffsetB)(\BREG) 

  4507. 

  4508.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)

  4509.     lxv vs16,   DISP8(\Index,16+\OffsetA)(\AREG) 

  4510. 

  4511.  

  4512.     xxperm   vs9,  vs8, permute_mask   

  4513.     xxperm   vs11, vs10, permute_mask  

  4514. 

  4515. 

  4516.  

  4517.     xvmaddasp      vs0,   vs26,   vs8

  4518.     xvmaddasp      vs1,   vs26,   vs9 

  4519.     xvmaddasp      vs0,   vs16,   vs10

  4520.     xvmaddasp      vs1,   vs16,   vs11 

  4521.  

  4522.  

  4523.  

  4524. .if \IsLast==1   

  4525.     addi        \BREG, \BREG, DISP8(\Index,32)  

  4526.     addi        \AREG, \AREG, DISP8(\Index,32)

  4527. .endif 

  4528.   

  4529. .endm

  4530. 

  4531. .macro KERNEL2x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4532. 

  4533.     lxv vs8,    DISP4(\Index, 0+\OffsetB)(\BREG)  

  4534.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG) 

  4535. 

  4536.  

  4537.     xxperm   vs9,  vs8, permute_mask    

  4538. 

  4539.  

  4540.     xvmaddasp      vs0,   vs26,   vs8

  4541.     xvmaddasp      vs1,   vs26,   vs9  

  4542.  

  4543. .if \IsLast==1   

  4544.     addi        \BREG, \BREG, DISP4(\Index,16)  

  4545.     addi        \AREG, \AREG, DISP4(\Index,16)

  4546. .endif 

  4547. .endm

  4548. 

  4549. 

  4550. .macro SAVE2x2

  4551. 

  4552. #ifndef TRMMKERNEL    

  4553.     lxsd v4   , 0(CO)     

  4554. #endif

  4555.   add     T1, CO, LDC 

  4556. #ifndef TRMMKERNEL    

  4557.     lxsd v5   , 0(T1) 

  4558.     

  4559. #endif

  4560.     /*aggregate vectors*/

  4561.   xxpermdi         vs4,vs0,vs0,2

  4562.   xxpermdi         vs5,vs1,vs1,2  

  4563.   xvaddsp          vs0,vs0,vs4

  4564.   xvaddsp         vs1,vs1,vs5 

  4565.   /*   */

  4566.   /* lets correct the order to 00 10 and 10 ,11 from {00,11} {01,10}  */

  4567.   xxperm    vs1,vs1, permute_mask

  4568. 

  4569. 

  4570.   xxmrghw   vs2 ,vs1,vs0

  4571.   xxpermdi         vs2,vs2,vs2,2  

  4572.   xxmrghw   vs3 ,vs0,vs1  

  4573. #if defined(TRMMKERNEL)

  4574.   xvmulsp        vs36, vs2, alpha_r 

  4575.   xvmulsp        vs37, vs3, alpha_r 

  4576. #else

  4577.   xvmaddasp        vs36, vs2, alpha_r 

  4578.   xvmaddasp        vs37, vs3, alpha_r 

  4579. #endif

  4580.   /**** store last two words*/

  4581. 

  4582. 

  4583.   stxsd       v4, 0(CO) 

  4584.   stxsd        v5, 0(T1)  

  4585. 

  4586.   addi CO,CO,8

  4587. 

  4588. .endm

  4589. 

  4590. /*--------------------------- M=1 N=2 */

  4591. .macro Zero2x1

  4592.     xxlxor      vs0,   vs0,   vs0

  4593.     xxlxor      vs1,   vs1,   vs1 

  4594.     xxlxor    vs2,vs2,vs2 

  4595.     xxlxor    vs3,vs3,vs3     

  4596. .endm

  4597.  

  4598. .macro KERNEL2x1

  4599.   KERNEL2x1_1 AO,BO, 0, 0,0,0

  4600. .endm

  4601. .macro KERNEL2x1_4 OffsetA,OffsetB, Index,IsLast  

  4602.   KERNEL2x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4603. .endm

  4604. 

  4605. .macro KERNEL2x1_2 OffsetA,OffsetB, Index,IsLast  

  4606.   KERNEL2x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4607. .endm

  4608.  /*

  4609.    we will calculate 1 alone then will add it to batched ones

  4610.  */

  4611. .macro KERNEL2x1_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4612. 

  4613. 

  4614.     lxssp v3,   DISP2(\Index, 0+\OffsetB)(\BREG)

  4615.     lxssp v4,   DISP2(\Index, 4+\OffsetB)(\BREG) 

  4616.     lxssp v5,   DISP1(\Index, 0+\OffsetA)(\AREG)        

  4617.  

  4618.  

  4619. .if \First==1

  4620.     xvmulsp      vs2,   vs37,   vs35 

  4621.     xvmulsp      vs3,   vs37,   vs36     

  4622.      

  4623. .else 

  4624.     xsmaddadp     vs2,   vs37,   vs35

  4625.     xsmaddadp      vs3,   vs37,   vs36

  4626.  .endif

  4627.    

  4628.     addi        \BREG, \BREG, DISP2(\Index,8)

  4629.     addi        \AREG, \AREG, DISP1(\Index,4)  

  4630.  

  4631. .endm

  4632. 

  4633. 

  4634. 

  4635. 

  4636. .macro KERNEL2x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4637. 

  4638.     lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)

  4639.     lxv vs10,   DISP8(\Index, 16+\OffsetB)(\BREG) 

  4640. 

  4641.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)

  4642.    

  4643.     xxmrglw   vs5, vs26,vs26

  4644.     xxmrghw   vs6, vs26,vs26 

  4645.  

  4646.     xvmaddasp      vs0,   vs8,   vs5

  4647.     xvmaddasp      vs1,   vs10,   vs6 

  4648.  

  4649.  

  4650. .if \IsLast==1   

  4651.     addi        \BREG, \BREG, DISP8(\Index,32)  

  4652.     addi        \AREG, \AREG, DISP4(\Index,16)

  4653. .endif 

  4654.   

  4655. .endm

  4656. 

  4657. .macro KERNEL2x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4658. 

  4659.     lxssp v3,   DISP4(\Index, 0+\OffsetB)(\BREG)

  4660.     lxssp v4,   DISP4(\Index, 4+\OffsetB)(\BREG) 

  4661.     lxssp v7,   DISP4(\Index, 8+\OffsetB)(\BREG)

  4662.     lxssp v8,   DISP4(\Index, 12+\OffsetB)(\BREG)    

  4663.     lxssp v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        

  4664.     lxssp v6,   DISP2(\Index, 4+\OffsetA)(\AREG)  

  4665.  

  4666.  

  4667.     xsmaddadp      vs2,   vs37,   vs35

  4668.     xsmaddadp      vs3,   vs37,   vs36

  4669. 

  4670.     xsmaddadp      vs2,   vs38,   vs39 

  4671.     xsmaddadp      vs3,   vs38,   vs40      

  4672.  

  4673.    

  4674.     addi        \BREG, \BREG, DISP4(\Index,16)

  4675.     addi        \AREG, \AREG, DISP2(\Index,8) 

  4676. .endm

  4677. 

  4678. 

  4679. .macro SAVE2x1

  4680. 

  4681. #ifndef TRMMKERNEL    

  4682.     lxssp v4   , 0(CO)     

  4683. #endif

  4684.   add     T1, CO, LDC 

  4685. #ifndef TRMMKERNEL    

  4686.     lxssp v5   , 0(T1) 

  4687.     

  4688. #endif

  4689. 

  4690.   /*convert alpha_r for multiply*/

  4691.   xscvspdp  vs16,alpha_r

  4692. 

  4693.  /*aggregate vectors 2x2_4   */ 

  4694.       xxpermdi         vs4,vs0,vs0,2

  4695.       xxpermdi         vs5,vs1,vs1,2  

  4696.       xvaddsp          vs0,vs0,vs4

  4697.       xvaddsp         vs1,vs1,vs5 

  4698.       xvaddsp         vs0,vs0,vs1 

  4699. /*aggregate vectors 2x1_2 and 2x1_1 into 2x2_4*/

  4700.   xscvspdp  vs5, vs0

  4701.   xxspltw   vs6, vs0, 1  

  4702.   xscvspdp  vs6,vs6 

  4703.   xsadddp  vs2,vs2,vs6

  4704.   xsadddp  vs3,vs3,vs5  

  4705. 

  4706.   /**** store last two words*/

  4707. #if defined(TRMMKERNEL) 

  4708.   xsmuldp  vs36,vs2, vs16 

  4709.   xsmuldp  vs37,vs3, vs16  

  4710.  

  4711. #else

  4712.   xsmaddadp  vs36,vs2, vs16 

  4713.   xsmaddadp  vs37,vs3, vs16 

  4714. #endif  

  4715. 

  4716.   stxssp       v4, 0(CO) 

  4717.   stxssp        v5, 0(T1)  

  4718. 

  4719.   addi CO,CO,4

  4720. 

  4721. .endm

  4722. 

  4723. 

  4724. 

  4725. /****************************N=1 section*****************/

  4726. 

  4727. .macro KERNEL1x16_2   OffsetA,OffsetB, Index,IsLast

  4728.   KERNEL1x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4729. .endm

  4730. 

  4731.  

  4732. .macro Zero1x16

  4733.     xxlxor      vs0,   vs0,   vs0

  4734.     xxlxor      vs1,   vs1,   vs1 

  4735.     xxlxor      vs2,   vs2,   vs2

  4736.     xxlxor      vs3,   vs3,   vs3       

  4737. .endm

  4738.  

  4739. .macro KERNEL1x16

  4740.   KERNEL1x16_1 AO,BO, 0, 0,0,0

  4741. .endm

  4742. .macro KERNEL1x16_4 OffsetA,OffsetB, Index,IsLast  

  4743.   KERNEL1x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4744. .endm

  4745.  

  4746. .macro KERNEL1x16_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4747. 

  4748. 

  4749.     lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)

  4750.     xscvdpspn   vs36,vs36

  4751.     xxspltw     vs8,  vs36, 0

  4752.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  4753.     lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   

  4754.     lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)

  4755.     lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        

  4756.  

  4757.  

  4758. .if \First==1

  4759.     xvmulsp      vs0,   vs26,   vs8

  4760.     xvmulsp      vs1,   vs27,   vs8 

  4761.     xvmulsp      vs2,   vs28,   vs8

  4762.     xvmulsp      vs3,   vs29,   vs8 

  4763.   

  4764.      

  4765. .else 

  4766.     xvmaddasp      vs0,   vs26,   vs8

  4767.     xvmaddasp      vs1,   vs27,   vs8 

  4768.     xvmaddasp      vs2,   vs28,   vs8

  4769.     xvmaddasp      vs3,   vs29,   vs8 

  4770.  

  4771.  .endif

  4772.    

  4773.     addi        \BREG, \BREG, DISP1(\Index,4)

  4774.     addi        \AREG, \AREG, DISP16(\Index,64)  

  4775.  

  4776. .endm

  4777. 

  4778. 

  4779. 

  4780. 

  4781. .macro KERNEL1x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4782. 

  4783.     lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

  4784. 

  4785.     lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)

  4786.     lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)

  4787.     lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)

  4788.     lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  

  4789. 

  4790.     lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)

  4791.     lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)

  4792.     lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)

  4793.     lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) 

  4794. 

  4795.     xxspltw   vs8,  vs38, 3  

  4796.     xxspltw   vs9,  vs38, 2 

  4797. 

  4798.     lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)

  4799.     lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)

  4800.     lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)

  4801.     lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  

  4802.         

  4803.     lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)

  4804.     lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)

  4805.     lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)

  4806.     lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) 

  4807. 

  4808.     xxspltw   vs10, vs38, 1 

  4809.     xxspltw   vs11, vs38, 0    

  4810. 

  4811.  

  4812.     xvmaddasp      vs0,   vs26,   vs8

  4813.     xvmaddasp      vs1,   vs27,   vs8 

  4814.     xvmaddasp      vs2,   vs28,   vs8

  4815.     xvmaddasp      vs3,   vs29,   vs8 

  4816.  

  4817. 

  4818.     xvmaddasp      vs0,   vs16,   vs9

  4819.     xvmaddasp      vs1,   vs17,   vs9 

  4820.     xvmaddasp      vs2,   vs18,   vs9

  4821.     xvmaddasp      vs3,   vs19,   vs9 

  4822.  

  4823. 

  4824.     xvmaddasp      vs0,   vs30,   vs10

  4825.     xvmaddasp      vs1,   vs31,   vs10 

  4826.     xvmaddasp      vs2,   vs32,   vs10

  4827.     xvmaddasp      vs3,   vs33,   vs10 

  4828.  

  4829. 

  4830.     xvmaddasp      vs0,   vs34,   vs11

  4831.     xvmaddasp      vs1,   vs35,   vs11 

  4832.     xvmaddasp      vs2,   vs36,   vs11

  4833.     xvmaddasp      vs3,   vs37,   vs11 

  4834. 

  4835.  

  4836.  

  4837.  

  4838. .if \IsLast==1   

  4839.     addi        \BREG, \BREG, DISP4(\Index,16)  

  4840.     addi        \AREG, \AREG, DISP64(\Index,256)

  4841. .endif 

  4842.   

  4843. .endm

  4844. 

  4845. .macro KERNEL1x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4846. 

  4847.     lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)

  4848.     xxspltw   vs8,  vs36, 1  

  4849.     xxspltw   vs9,  vs36, 0      

  4850.     lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)

  4851.     lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)

  4852.     lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)

  4853.     lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      

  4854.     lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)

  4855.     lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)

  4856.     lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)

  4857.     lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) 

  4858.  

  4859.  

  4860.     xvmaddasp      vs0,   vs26,   vs8

  4861.     xvmaddasp      vs1,   vs27,   vs8 

  4862.     xvmaddasp      vs2,   vs28,   vs8

  4863.     xvmaddasp      vs3,   vs29,   vs8 

  4864.  

  4865. 

  4866.     xvmaddasp      vs0,   vs16,   vs9

  4867.     xvmaddasp      vs1,   vs17,   vs9 

  4868.     xvmaddasp      vs2,   vs18,   vs9

  4869.     xvmaddasp      vs3,   vs19,   vs9 

  4870.   

  4871.  

  4872. .if \IsLast==1   

  4873.     addi        \BREG, \BREG, DISP2(\Index,8)  

  4874.     addi        \AREG, \AREG, DISP32(\Index,128)

  4875. .endif 

  4876.   

  4877. .endm

  4878. 

  4879. 

  4880. .macro SAVE1x16

  4881. 

  4882. #ifndef TRMMKERNEL    

  4883.     lxv        vs16, 0(CO)

  4884.     lxv        vs17, 16(CO) 

  4885.     lxv        vs18, 32(CO)  

  4886.     lxv        vs19, 48(CO)      

  4887. #endif

  4888.  

  4889. 

  4890. #if defined(TRMMKERNEL)

  4891.   xvmulsp        vs16, vs0, alpha_r

  4892.   xvmulsp        vs17, vs1, alpha_r 

  4893.   xvmulsp        vs18, vs2, alpha_r

  4894.   xvmulsp        vs19, vs3, alpha_r   

  4895. #else

  4896.   xvmaddasp        vs16, vs0, alpha_r

  4897.   xvmaddasp        vs17, vs1, alpha_r 

  4898.   xvmaddasp        vs18, vs2, alpha_r

  4899.   xvmaddasp        vs19, vs3, alpha_r   

  4900. #endif

  4901.     stxv        vs16, 0(CO)

  4902.     stxv        vs17, 16(CO) 

  4903.     stxv        vs18, 32(CO)  

  4904.     stxv        vs19, 48(CO)      

  4905.     

  4906.   addi CO,CO,64

  4907. 

  4908. .endm

  4909. 

  4910. /*       M=8 N=1 */

  4911. 

  4912. .macro KERNEL1x8_2   OffsetA,OffsetB, Index,IsLast

  4913.   KERNEL1x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4914. .endm

  4915. 

  4916.  

  4917. .macro Zero1x8

  4918.     xxlxor      vs0,   vs0,   vs0

  4919.     xxlxor      vs1,   vs1,   vs1  

  4920.     xxlxor      vs2,   vs2,   vs2

  4921.     xxlxor      vs3,   vs3,   vs3          

  4922. .endm

  4923.  

  4924. .macro KERNEL1x8

  4925.   KERNEL1x8_1 AO,BO, 0, 0,0,0

  4926. .endm

  4927. .macro KERNEL1x8_4 OffsetA,OffsetB, Index,IsLast  

  4928.   KERNEL1x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  4929. .endm

  4930.  

  4931. .macro KERNEL1x8_1 AREG,BREG,First,OffsetA,OffsetB,Index

  4932. 

  4933. 

  4934.     lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)

  4935.     xscvdpspn   vs36,vs36

  4936.     xxspltw     vs8,  vs36, 0

  4937.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)

  4938.     lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)         

  4939.  

  4940.  

  4941. .if \First==1

  4942.     xvmulsp      vs0,   vs26,   vs8

  4943.     xvmulsp      vs1,   vs27,   vs8  

  4944.   

  4945.      

  4946. .else 

  4947.     xvmaddasp      vs0,   vs26,   vs8

  4948.     xvmaddasp      vs1,   vs27,   vs8  

  4949.  

  4950.  .endif

  4951.    

  4952.     addi        \BREG, \BREG, DISP1(\Index,4)

  4953.     addi        \AREG, \AREG, DISP8(\Index,32)  

  4954.  

  4955. .endm

  4956. 

  4957. 

  4958. 

  4959. 

  4960. .macro KERNEL1x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  4961. 

  4962.     lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

  4963. 

  4964.     lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)

  4965.     lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) 

  4966. 

  4967.     lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)

  4968.     lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) 

  4969. 

  4970.     xxspltw   vs8,  vs38, 3  

  4971.     xxspltw   vs9,  vs38, 2 

  4972. 

  4973.     lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)

  4974.     lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)   

  4975.         

  4976.     lxv vs34,   DISP32(\Index,64+ 32+ 0+\OffsetA)(\AREG)

  4977.     lxv vs35,   DISP32(\Index,64+ 32+ 16+\OffsetA)(\AREG)  

  4978. 

  4979.     xxspltw   vs10, vs38, 1 

  4980.     xxspltw   vs11, vs38, 0    

  4981. 

  4982.  

  4983.     xvmaddasp      vs0,   vs26,   vs8

  4984.     xvmaddasp      vs1,   vs27,   vs8  

  4985.  

  4986. 

  4987.     xvmaddasp      vs2,   vs16,   vs9

  4988.     xvmaddasp      vs3,   vs17,   vs9  

  4989.  

  4990. 

  4991.     xvmaddasp      vs0,   vs30,   vs10

  4992.     xvmaddasp      vs1,   vs31,   vs10  

  4993.  

  4994. 

  4995.     xvmaddasp      vs2,   vs34,   vs11

  4996.     xvmaddasp      vs3,   vs35,   vs11  

  4997. 

  4998.  

  4999.  

  5000.  

  5001. .if \IsLast==1   

  5002.     addi        \BREG, \BREG, DISP4(\Index,16)  

  5003.     addi        \AREG, \AREG, DISP32(\Index,128)

  5004. .endif 

  5005.   

  5006. .endm

  5007. 

  5008. .macro KERNEL1x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5009. 

  5010.     lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)

  5011.     xxspltw   vs8,  vs36, 1  

  5012.     xxspltw   vs9,  vs36, 0      

  5013.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  5014.     lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)     

  5015.     lxv vs16,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)

  5016.     lxv vs17,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) 

  5017.  

  5018.  

  5019.     xvmaddasp      vs0,   vs26,   vs8

  5020.     xvmaddasp      vs1,   vs27,   vs8  

  5021.  

  5022. 

  5023.     xvmaddasp      vs2,   vs16,   vs9

  5024.     xvmaddasp      vs3,   vs17,   vs9   

  5025.   

  5026.  

  5027. .if \IsLast==1   

  5028.     addi        \BREG, \BREG, DISP2(\Index,8)  

  5029.     addi        \AREG, \AREG, DISP16(\Index,64)

  5030. .endif 

  5031.   

  5032. .endm

  5033. 

  5034. 

  5035. .macro SAVE1x8

  5036. 

  5037. #ifndef TRMMKERNEL    

  5038.     lxv        vs16, 0(CO)

  5039.     lxv        vs17, 16(CO)       

  5040. #endif

  5041.    /* aggregate vs0 vs2 and vs1 vs3*/

  5042.   xvaddsp vs0,vs0,vs2

  5043.   xvaddsp  vs1,vs1,vs3

  5044. #if defined(TRMMKERNEL)

  5045.   xvmulsp        vs16, vs0, alpha_r

  5046.   xvmulsp        vs17, vs1, alpha_r     

  5047. #else

  5048.   xvmaddasp        vs16, vs0, alpha_r

  5049.   xvmaddasp        vs17, vs1, alpha_r  

  5050. #endif

  5051.     stxv        vs16, 0(CO)

  5052.     stxv        vs17, 16(CO)      

  5053.     

  5054.   addi CO,CO,32

  5055. 

  5056. .endm

  5057. /*M=4*/

  5058. 

  5059. .macro KERNEL1x4_2   OffsetA,OffsetB, Index,IsLast

  5060.   KERNEL1x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5061. .endm

  5062. 

  5063.  

  5064. .macro Zero1x4

  5065.     xxlxor      vs0,   vs0,   vs0

  5066.     xxlxor      vs1,   vs1,   vs1  

  5067.     xxlxor      vs2,   vs2,   vs2

  5068.     xxlxor      vs3,   vs3,   vs3          

  5069. .endm

  5070.  

  5071. .macro KERNEL1x4

  5072.   KERNEL1x4_1 AO,BO, 0, 0,0,0

  5073. .endm

  5074. .macro KERNEL1x4_4 OffsetA,OffsetB, Index,IsLast  

  5075.   KERNEL1x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5076. .endm

  5077.  

  5078. .macro KERNEL1x4_1 AREG,BREG,First,OffsetA,OffsetB,Index

  5079. 

  5080. 

  5081.     lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)

  5082.     xscvdpspn   vs36,vs36

  5083.     xxspltw     vs8,  vs36, 0

  5084.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)         

  5085.  

  5086.  

  5087. .if \First==1

  5088.     xvmulsp      vs0,   vs26,   vs8 

  5089. .else 

  5090.     xvmaddasp      vs0,   vs26,   vs8 

  5091.  

  5092.  .endif

  5093.    

  5094.     addi        \BREG, \BREG, DISP1(\Index,4)

  5095.     addi        \AREG, \AREG, DISP4(\Index,16)  

  5096.  

  5097. .endm

  5098. 

  5099. 

  5100. 

  5101. 

  5102. .macro KERNEL1x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5103. 

  5104.     lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

  5105. 

  5106.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)

  5107.     lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG) 

  5108.  

  5109. 

  5110.     xxspltw   vs8,  vs38, 3  

  5111.     xxspltw   vs9,  vs38, 2 

  5112. 

  5113.     lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)

  5114.     lxv vs31,   DISP16(\Index,32+ 16+\OffsetA)(\AREG)   

  5115.           

  5116. 

  5117.     xxspltw   vs10, vs38, 1 

  5118.     xxspltw   vs11, vs38, 0    

  5119. 

  5120.  

  5121.     xvmaddasp      vs0,   vs26,   vs8 

  5122. 

  5123.     xvmaddasp      vs1,   vs27,   vs9 

  5124. 

  5125.     xvmaddasp      vs2,   vs30,   vs10   

  5126.  

  5127. 

  5128.     xvmaddasp      vs3,   vs31,   vs11   

  5129. 

  5130.  

  5131.  

  5132.  

  5133. .if \IsLast==1   

  5134.     addi        \BREG, \BREG, DISP4(\Index,16)  

  5135.     addi        \AREG, \AREG, DISP16(\Index,64)

  5136. .endif 

  5137.   

  5138. .endm

  5139. 

  5140. .macro KERNEL1x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5141. 

  5142.     lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)

  5143.     xxspltw   vs8,  vs36, 1  

  5144.     xxspltw   vs9,  vs36, 0      

  5145.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)

  5146.     lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)      

  5147.  

  5148.  

  5149.     xvmaddasp      vs0,   vs26,   vs8

  5150.     xvmaddasp      vs1,   vs27,   vs9

  5151.   

  5152.  

  5153. .if \IsLast==1   

  5154.     addi        \BREG, \BREG, DISP2(\Index,8)  

  5155.     addi        \AREG, \AREG, DISP8(\Index,32)

  5156. .endif 

  5157.   

  5158. .endm

  5159. 

  5160. 

  5161. .macro SAVE1x4

  5162. 

  5163. #ifndef TRMMKERNEL    

  5164.     lxv        vs16, 0(CO)       

  5165. #endif

  5166.    /* aggregate */

  5167.   xvaddsp vs0,vs0,vs2

  5168.   xvaddsp  vs1,vs1,vs3

  5169.   xvaddsp  vs0,vs1,vs0

  5170. #if defined(TRMMKERNEL)

  5171.   xvmulsp        vs16, vs0, alpha_r     

  5172. #else

  5173.   xvmaddasp        vs16, vs0, alpha_r  

  5174. #endif

  5175.     stxv        vs16, 0(CO)      

  5176.     

  5177.   addi CO,CO,16

  5178. 

  5179. .endm

  5180. 

  5181. /* M=2 N=1*/ 

  5182. .macro Zero1x2

  5183.     xxlxor      vs0,   vs0,   vs0

  5184.     xxlxor      vs1,   vs1,   vs1 

  5185.     xxlxor    vs2,vs2,vs2 

  5186.     xxlxor    vs3,vs3,vs3     

  5187. .endm

  5188.  

  5189. .macro KERNEL1x2

  5190.   KERNEL1x2_1 AO,BO, 0, 0,0,0

  5191. .endm

  5192. .macro KERNEL1x2_4 OffsetA,OffsetB, Index,IsLast  

  5193.   KERNEL1x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5194. .endm

  5195. 

  5196. .macro KERNEL1x2_2 OffsetA,OffsetB, Index,IsLast  

  5197.   KERNEL1x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5198. .endm

  5199.  /*

  5200.    we will calculate 1 alone then will add it to batched ones

  5201.  */

  5202. .macro KERNEL1x2_1 AREG,BREG,First,OffsetA,OffsetB,Index

  5203. 

  5204. 

  5205.     lxssp v3,   DISP2(\Index, 0+\OffsetB)(\AREG)

  5206.     lxssp v4,   DISP2(\Index, 4+\OffsetB)(\AREG) 

  5207.     lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        

  5208.  

  5209.  

  5210. .if \First==1

  5211.     xvmuldp      vs2,   vs37,   vs35 

  5212.     xvmuldp      vs3,   vs37,   vs36     

  5213.      

  5214. .else 

  5215.     xsmaddadp     vs2,   vs37,   vs35

  5216.     xsmaddadp      vs3,   vs37,   vs36

  5217.  .endif

  5218.    

  5219.     addi        \AREG, \AREG,  DISP2(\Index,8) 

  5220.     addi        \BREG, \BREG, DISP1(\Index,4) 

  5221.  

  5222. .endm

  5223. 

  5224. 

  5225. 

  5226. 

  5227. .macro KERNEL1x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5228. 

  5229.     lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG)

  5230.     lxv vs10,   DISP8(\Index, 16+\OffsetB)(\AREG) 

  5231. 

  5232.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG)

  5233.    

  5234.     xxmrglw   vs5, vs26,vs26

  5235.     xxmrghw   vs6, vs26,vs26 

  5236.  

  5237.     xvmaddasp      vs0,   vs8,   vs5

  5238.     xvmaddasp      vs1,   vs10,   vs6 

  5239.  

  5240.  

  5241. .if \IsLast==1   

  5242.     addi        \AREG, \AREG, DISP8(\Index,32)

  5243.     addi        \BREG, \BREG,  DISP4(\Index,16)  

  5244. .endif 

  5245.   

  5246. .endm

  5247. 

  5248. .macro KERNEL1x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5249. 

  5250.     lxssp v3,   DISP4(\Index, 0+\OffsetB)(\AREG)

  5251.     lxssp v4,   DISP4(\Index, 4+\OffsetB)(\AREG) 

  5252.     lxssp v7,   DISP4(\Index, 8+\OffsetB)(\AREG)

  5253.     lxssp v8,   DISP4(\Index, 12+\OffsetB)(\AREG)    

  5254.     lxssp v5,   DISP2(\Index, 0+\OffsetA)(\BREG)        

  5255.     lxssp v6,   DISP2(\Index, 4+\OffsetA)(\BREG)  

  5256.  

  5257.  

  5258.     xsmaddadp      vs2,   vs37,   vs35

  5259.     xsmaddadp      vs3,   vs37,   vs36

  5260. 

  5261.     xsmaddadp      vs2,   vs38,   vs39 

  5262.     xsmaddadp      vs3,   vs38,   vs40      

  5263.  

  5264.    

  5265.     addi        \AREG, \AREG, DISP4(\Index,16)

  5266.     addi        \BREG, \BREG, DISP2(\Index,8) 

  5267. .endm

  5268. 

  5269. 

  5270. .macro SAVE1x2

  5271. 

  5272. #ifndef TRMMKERNEL    

  5273.     lxssp v4   , 0(CO)      

  5274.     lxssp v5   , 4(CO) 

  5275.     

  5276. #endif

  5277. 

  5278.   /*convert alpha_r for multiply*/

  5279.   xscvspdp  vs16,alpha_r

  5280. 

  5281.  /*aggregate vectors 1x2_4   */ 

  5282.       xxpermdi         vs4,vs0,vs0,2

  5283.       xxpermdi         vs5,vs1,vs1,2  

  5284.       xvaddsp          vs0,vs0,vs4

  5285.       xvaddsp         vs1,vs1,vs5 

  5286.       xvaddsp         vs0,vs0,vs1 

  5287. /*aggregate vectors 1x1_2 and 1x1_1 into 1x2_4*/

  5288.   xscvspdp  vs5, vs0

  5289.   xxspltw   vs6, vs0, 1  

  5290.   xscvspdp  vs6,vs6 

  5291.   xsadddp  vs2,vs2,vs6

  5292.   xsadddp  vs3,vs3,vs5  

  5293. 

  5294.   /**** store last two words*/

  5295. #if defined(TRMMKERNEL) 

  5296.   xsmuldp  vs36,vs2, vs16 

  5297.   xsmuldp  vs37,vs3, vs16  

  5298.  

  5299. #else

  5300.   xsmaddadp  vs36,vs2, vs16 

  5301.   xsmaddadp  vs37,vs3, vs16 

  5302. #endif  

  5303. 

  5304.   stxssp       v4, 0(CO) 

  5305.   stxssp        v5, 4(CO)  

  5306. 

  5307.   addi CO,CO,8

  5308. 

  5309. .endm

  5310. /*///////////////// N=1 M=1 //////////////////*/

  5311. .macro Zero1x1

  5312.     xxlxor      vs0,   vs0,   vs0

  5313.     xxlxor      vs1,   vs1,   vs1 

  5314.     xxlxor      vs2, vs2,vs2 

  5315.     xxlxor      vs3,vs3,vs3 

  5316.     xxlxor      vs4,vs4,vs4       

  5317. .endm

  5318.  

  5319. .macro KERNEL1x1

  5320.   KERNEL1x1_1 AO,BO, 1, 0,0,0

  5321. .endm

  5322. 

  5323. .macro KERNEL1x1_16 OffsetA,OffsetB, Index,IsLast  

  5324.   KERNEL1x1_I_16 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5325. .endm

  5326. 

  5327. .macro KERNEL1x1_8 OffsetA,OffsetB, Index,IsLast  

  5328.   KERNEL1x1_I_8 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5329. .endm

  5330. 

  5331. .macro KERNEL1x1_4 OffsetA,OffsetB, Index,IsLast  

  5332.   KERNEL1x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5333. .endm

  5334. 

  5335. .macro KERNEL1x1_2 OffsetA,OffsetB, Index,IsLast  

  5336.   KERNEL1x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast

  5337. .endm

  5338.  /*

  5339.    we will calculate 1 alone ( FIRST==1 to zero vs4) 

  5340.  */

  5341. .macro KERNEL1x1_1 AREG,BREG,First,OffsetA,OffsetB,Index

  5342. 

  5343. 

  5344.     lxssp v3,   DISP1(\Index, 0+\OffsetB)(\AREG) 

  5345.     lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        

  5346.  

  5347.  

  5348. .if \First==1

  5349.     xvmuldp      vs4,   vs37,   vs35       

  5350.      

  5351. .else 

  5352.     xsmaddadp     vs4,   vs37,   vs35 

  5353.  .endif

  5354.    

  5355.     addi        \AREG, \AREG,  DISP1(\Index,4) 

  5356.     addi        \BREG, \BREG, DISP1(\Index,4) 

  5357.  

  5358. .endm

  5359. 

  5360. 

  5361. .macro KERNEL1x1_I_16 AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5362. 

  5363.     lxv vs8,    DISP16(\Index, 0+\OffsetB)(\AREG) 

  5364.     lxv vs9,    DISP16(\Index, 16+\OffsetB)(\AREG) 

  5365.     lxv vs10,   DISP16(\Index, 32+0+\OffsetB)(\AREG) 

  5366.     lxv vs11,   DISP16(\Index, 32+ 16+\OffsetB)(\AREG)        

  5367.     lxv vs26,   DISP16(\Index, 0+\OffsetA)(\BREG) 

  5368.     lxv vs16,   DISP16(\Index, 16+\OffsetA)(\BREG) 

  5369.     lxv vs17,   DISP16(\Index, 32+0+\OffsetA)(\BREG) 

  5370.     lxv vs18,   DISP16(\Index, 32+16+\OffsetA)(\BREG)     

  5371.     xvmaddasp      vs0,   vs8,   vs26 

  5372.     xvmaddasp      vs1,   vs9,   vs16  

  5373.     xvmaddasp      vs2,   vs10,  vs17 

  5374.     xvmaddasp      vs3,   vs11,  vs18

  5375. .if \IsLast==1   

  5376.     addi        \AREG, \AREG, DISP16(\Index,64)

  5377.     addi        \BREG, \BREG,  DISP16(\Index,64)  

  5378. .endif 

  5379.   

  5380. .endm

  5381. 

  5382. .macro KERNEL1x1_I_8  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5383. 

  5384.     lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG) 

  5385.     lxv vs9,    DISP8(\Index, 16+\OffsetB)(\AREG)     

  5386.     lxv vs26,   DISP8(\Index, 0+\OffsetA)(\BREG) 

  5387.     lxv vs16,   DISP8(\Index, 16+\OffsetA)(\BREG) 

  5388.     xvmaddasp      vs0,   vs8,   vs26 

  5389.     xvmaddasp      vs1,   vs9,   vs16 

  5390.  

  5391. .if \IsLast==1   

  5392.     addi        \AREG, \AREG, DISP8(\Index,32)

  5393.     addi        \BREG, \BREG,  DISP8(\Index,32)  

  5394. .endif 

  5395.   

  5396. .endm

  5397. 

  5398. 

  5399. .macro KERNEL1x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5400. 

  5401.     lxv vs8,    DISP4(\Index, 0+\OffsetB)(\AREG) 

  5402.     lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG) 

  5403.  

  5404.     xvmaddasp      vs0,   vs8,   vs26 

  5405.  

  5406.  

  5407. .if \IsLast==1   

  5408.     addi        \AREG, \AREG, DISP4(\Index,16)

  5409.     addi        \BREG, \BREG,  DISP4(\Index,16)  

  5410. .endif 

  5411.   

  5412. .endm

  5413. 

  5414. .macro KERNEL1x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

  5415. 

  5416.     lxsd v4,    DISP2(\Index, 0+\OffsetB)(\AREG) 

  5417.     lxsd v5,   DISP2(\Index, 0+\OffsetA)(\BREG) 

  5418.  

  5419.     xvmaddasp      vs0,   vs36,   vs37 

  5420.  

  5421.     addi        \AREG, \AREG, DISP2(\Index,8)

  5422.     addi        \BREG, \BREG, DISP2(\Index,8) 

  5423. .endm

  5424. 

  5425. 

  5426. .macro SAVE1x1

  5427. 

  5428. #ifndef TRMMKERNEL    

  5429.     lxssp v4   , 0(CO)    

  5430.     

  5431. #endif

  5432. 

  5433.   /*convert alpha_r for multiply*/

  5434.   xscvspdp  vs16,alpha_r

  5435. 

  5436.  /*aggregate vectors   */ 

  5437.       xvaddsp          vs0,vs0,vs1

  5438.       xvaddsp          vs2,vs2,vs3

  5439.       xvaddsp          vs0,vs0,vs2

  5440. 

  5441.       xxpermdi         vs7,vs0,vs0,2   

  5442.       xvaddsp          vs0,vs0,vs7 

  5443. /*aggregate vectors 1x1_2 and 1x1_1 into 1x1_4*/

  5444.   xscvspdp  vs5, vs0

  5445.   xxspltw   vs6, vs0, 1  

  5446.   xscvspdp  vs6,vs6 

  5447.   xsadddp  vs7,vs5,vs6

  5448.   xsadddp  vs4,vs4,vs7  

  5449. 

  5450.   /**** store last two words*/

  5451. #if defined(TRMMKERNEL) 

  5452.   xsmuldp  vs36,vs4, vs16   

  5453.  

  5454. #else

  5455.   xsmaddadp  vs36,vs4, vs16   

  5456. #endif  

  5457. 

  5458.   stxssp       v4, 0(CO)    

  5459. 

  5460.   addi CO,CO,4

  5461. 

  5462. .endm

  5463. 

  5464. 

  5465. 

  5466. 

  5467. /****************************TRMM POINTER REFRESH MACROSES*************************/

  5468. 

  5469. .macro SHIFT_REG  REG1,REG2,SHIFT_VAL

  5470. 		.if \SHIFT_VAL==16 

  5471. 			slwi		\REG1,	\REG2,	6			

  5472. 		.elseif \SHIFT_VAL==8  

  5473. 			slwi		\REG1,	\REG2,	5			 

  5474. 		.elseif \SHIFT_VAL==4

  5475. 			slwi		\REG1,	\REG2,	4			  

  5476. 		.elseif \SHIFT_VAL==2

  5477. 			slwi		\REG1,	\REG2,	3			 

  5478. 		.elseif \SHIFT_VAL==1

  5479. 			slwi		\REG1,	\REG2,	2			 

  5480. 		.endif

  5481. .endm

  5482. 

  5483. /*

  5484. //#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  5485. // 		ptrbb = bb;

  5486. // #else

  5487. // 		ptrba += off*16;

  5488. // 		ptrbb = bb + off*2;

  5489. // #endif

  5490. */

  5491. .macro REFRESH_POINTERS  PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B

  5492.     #if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))

  5493.         /* ptrbb = bb;*/

  5494.         mr \PTR_B,\B_VAL     /* refresh BPOINT */

  5495. 

  5496.     #else

  5497. 		    /*

  5498.         // ptrba  =ptrba+ off*C_A;

  5499.         // ptrbb = bb + off*C_B; 

  5500. 				*/

  5501. 		SHIFT_REG T4,\OFF_VAL,\C_B		/* Number of values in B shifted  */

  5502. 		SHIFT_REG T2,\OFF_VAL,\C_A		/* Number of values in A shifted  */

  5503. 		add		\PTR_B,	\B_VAL ,	T4				/* Add values to BO */

  5504. 		add		\PTR_A,	\PTR_A,	T2				/* Add values to AO  */

  5505.     #endif 

  5506. .endm

  5507. 

  5508. 

  5509. /*

  5510. // #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))

  5511. // 		temp = bk-off;

  5512. // #elif defined(LEFT)

  5513. // 		temp = off+16;	// number of values in A

  5514. // #else

  5515. // 		temp = off+2;	// number of values in B

  5516. // #endif

  5517. */

  5518. .macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B

  5519.     #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))

  5520.                             /* temp = bk-off;*/

  5521.            sub \TEMP_BK,\BK_VAL,\OFF_VAL

  5522. 

  5523.     #elif defined(LEFT)

  5524.                             /* temp = off+INCR_A;	// number of values in A */

  5525.            addi \TEMP_BK, \OFF_VAL, \INCR_A

  5526.     #else

  5527.                             /* temp = off+INCR_B	// number of values in B*/

  5528.            addi \TEMP_BK,\OFF_VAL, \INCR_B

  5529.     #endif

  5530. 

  5531. .endm

  5532. /*

  5533. // #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  5534. // 		temp = bk - off;

  5535. // #ifdef LEFT

  5536. // 		temp -= 16; // number of values in A

  5537. // #else

  5538. // 		temp -= 2; // number of values in B

  5539. // #endif

  5540. // 		ptrba += temp*16;

  5541. // 		ptrbb += temp*2;

  5542. // #endif

  5543. 

  5544. // #ifdef LEFT

  5545. // 		off += 16; // number of values in A

  5546. // #endif

  5547. */

  5548.  

  5549. 

  5550. .macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B

  5551. 

  5552.     #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))

  5553.                     /*temp = bk - off;*/

  5554.                 sub \TEMP_BK,\BK_VAL,\OFF_VAL

  5555.     #ifdef LEFT

  5556.                     /*temp -= 8; // number of values in A*/

  5557.                 addi \TEMP_BK,\TEMP_BK,-\C_A

  5558.     #else

  5559.                     /*temp -= 4; // number of values in B*/

  5560.                 addi \TEMP_BK,\TEMP_BK,-\C_B 

  5561.     #endif

  5562.                     /*ptrba += temp*C_A;

  5563.                     ptrbb += temp*C_B;*/ 

  5564.                 SHIFT_REG T4,\TEMP_BK,\C_A

  5565. 								SHIFT_REG T2,\TEMP_BK,\C_B

  5566.                 add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/ 

  5567. 								add \PTR_B, \PTR_B,T2 

  5568. 

  5569.     #endif

  5570. 

  5571.     #ifdef LEFT

  5572.                     /*off += 8; // number of values in A*/

  5573.                  addi \OFF_VAL,\OFF_VAL,\C_A

  5574.     #endif

  5575. .endm