OpenBLAS/kernel/power/sgemm_macros_power9.S

/***************************************************************************
Copyright (c) 2013-2019, The OpenBLAS Project
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. Neither the name of the OpenBLAS project nor the names of
its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*****************************************************************************/
 
#define unit_size 4
#define DISP64(ind,disp) (ind*unit_size*64+disp)
#define DISP32(ind,disp) (ind*unit_size*32+disp)
#define DISP16(ind,disp) (ind*unit_size*16+disp)
#define DISP8(ind,disp) (ind*unit_size*8+disp)
#define DISP4(ind,disp) (ind*unit_size*4+disp)
#define DISP2(ind,disp) (ind*unit_size*2+disp)
#define DISP1(ind,disp) (ind*unit_size+disp)

/**********************************************************************************************
* Macros for N=8 and M=16
**********************************************************************************************/

 

.macro KERNEL8x16_L1_L4  Index,IsLast
  KERNEL8x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL8x16_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
 
.macro KERNEL8x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro Zero8X16
    xxlxor		vs32,	vs32,	vs32
    xxlxor		vs33,	vs33,	vs33
	xxlxor		vs34,	vs34,	vs34
	xxlxor		vs35,	vs35,	vs35
	xxlxor		vs36,	vs36,	vs36
	xxlxor		vs37,	vs37,	vs37
	xxlxor		vs38,	vs38,	vs38
	xxlxor		vs39,	vs39,	vs39
	xxlxor		vs40,	vs40,	vs40
	xxlxor		vs41,	vs41,	vs41
	xxlxor		vs42,	vs42,	vs42
	xxlxor		vs43,	vs43,	vs43
	xxlxor		vs44,	vs44,	vs44
	xxlxor		vs45,	vs45,	vs45
	xxlxor		vs46,	vs46,	vs46
	xxlxor		vs47,	vs47,	vs47
	xxlxor		vs48,	vs48,	vs48
	xxlxor		vs49,	vs49,	vs49
	xxlxor		vs50,	vs50,	vs50
	xxlxor		vs51,	vs51,	vs51 
	xxlxor		vs52,	vs52,	vs52
	xxlxor		vs53,	vs53,	vs53
	xxlxor		vs54,	vs54,	vs54
	xxlxor		vs55,	vs55,	vs55 
	xxlxor		vs56,	vs56,	vs56
	xxlxor		vs57,	vs57,	vs57
	xxlxor		vs58,	vs58,	vs58
	xxlxor		vs59,	vs59,	vs59 
	xxlxor		vs60,	vs60,	vs60
	xxlxor		vs61,	vs61,	vs61
	xxlxor		vs62,	vs62,	vs62
	xxlxor		vs63,	vs63,	vs63	
.endm

.macro LOAD8x16  OffsetA,OffsetB

	lxv	vs24,	(\OffsetB+0)(BO)
	lxv	vs28,	(\OffsetB+16)(BO)
	xxperm  	vs26,	vs24,		permute_mask
	xxperm  	vs30,	vs28,		permute_mask	  
	lxv	vs0,	(\OffsetA+0)(AO)
	lxv	vs1,	(\OffsetA+16)(AO)
	xxpermdi	vs25,	vs24,	vs24,2	   
	xxpermdi	vs29,	vs28,	vs28,2	  
	lxv	vs2,	(\OffsetA+32)(AO)
	lxv	vs3,	(\OffsetA+48)(AO) 
	xxpermdi	vs27,	vs26,	vs26,2	
	xxpermdi	vs31,	vs30,	vs30,2	 	

.endm

.macro END8x16_NORMAL
  END8x16 0, AO, BO, 64,32 
.endm

.macro END8x16_WITHOUT_ADD
	END8x16 0, AO,BO,0,0
.endm

.macro END8x16 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24
    xvmulsp     vs34, vs2,vs24  
    xvmulsp     vs35, vs3,vs24  

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25
    xvmulsp     vs38, vs2,vs25  
    xvmulsp     vs39, vs3,vs25

    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26
    xvmulsp     vs42, vs2,vs26  
    xvmulsp     vs43, vs3,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27
    xvmulsp     vs46, vs2,vs27  
    xvmulsp     vs47, vs3,vs27

    xvmulsp     vs48, vs0,vs28
    xvmulsp     vs49, vs1,vs28
    xvmulsp     vs50, vs2,vs28  
    xvmulsp     vs51, vs3,vs28  

    xvmulsp     vs52, vs0,vs29
    xvmulsp     vs53, vs1,vs29
    xvmulsp     vs54, vs2,vs29  
    xvmulsp     vs55, vs3,vs29

    xvmulsp     vs56, vs0,vs30
    xvmulsp     vs57, vs1,vs30
    xvmulsp     vs58, vs2,vs30  
    xvmulsp     vs59, vs3,vs30

    xvmulsp     vs60, vs0,vs31
    xvmulsp     vs61, vs1,vs31
    xvmulsp     vs62, vs2,vs31  
    xvmulsp     vs63, vs3,vs31

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24
    xvmaddasp       vs34, vs2,vs24  
    xvmaddasp       vs35, vs3,vs24  

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25
    xvmaddasp       vs38, vs2,vs25  
    xvmaddasp       vs39, vs3,vs25 
    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26
    xvmaddasp       vs42, vs2,vs26  
    xvmaddasp       vs43, vs3,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27
    xvmaddasp       vs46, vs2,vs27  
    xvmaddasp       vs47, vs3,vs27

    xvmaddasp       vs48, vs0,vs28
    xvmaddasp       vs49, vs1,vs28
    xvmaddasp       vs50, vs2,vs28  
    xvmaddasp       vs51, vs3,vs28  

    xvmaddasp       vs52, vs0,vs29
    xvmaddasp       vs53, vs1,vs29
    xvmaddasp       vs54, vs2,vs29  
    xvmaddasp       vs55, vs3,vs29

    xvmaddasp       vs56, vs0,vs30
    xvmaddasp       vs57, vs1,vs30
    xvmaddasp       vs58, vs2,vs30  
    xvmaddasp       vs59, vs3,vs30

    xvmaddasp       vs60, vs0,vs31
    xvmaddasp       vs61, vs1,vs31
    xvmaddasp       vs62, vs2,vs31  
    xvmaddasp       vs63, vs3,vs31 

.endif
.endm  

.macro KERNEL8x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

KERNEL8x16_2  \AREG,\BREG, \OffsetA,\OffsetB, (\Index*2),0 ,0
KERNEL8x16_2  \AREG,\BREG,\OffsetA,\OffsetB, (\Index*2+1),\IsLast ,\Complete

.endm

.macro KERNEL8x16 First

  LOAD8x16 0,0
  END8x16 \First, AO, BO, 64,32 
.endm

.macro LOAD8x16_2
    LOAD8x16_2O AO,BO, 0,0
.endm	

.macro LOAD8x16_2O  AREG,BREG, OffsetA,OffsetB
  lxv	vs8,	(\OffsetB)(\BREG)
  lxv	vs12,	(16+\OffsetB)(\BREG)
  lxv	vs24,	(32+\OffsetB)(\BREG)
  lxv	vs28,	(32+16+\OffsetB)(\BREG)
  lxv	vs4,	(0+\OffsetA)(\AREG)
  lxv	vs5,	(16+\OffsetA)(\AREG)
  xxperm  	vs10,	vs8,		permute_mask
  xxperm  	vs14,	vs12,		permute_mask	
  lxv	vs6,	(32+\OffsetA)(\AREG)
  lxv	vs7,	(48+\OffsetA)(\AREG) 
  xxpermdi	vs9,	vs8,	 vs8,2	 
  xxpermdi	vs13,	vs12,	vs12,2	 
  lxv	vs0,	(64+\OffsetA)(\AREG)
  lxv	vs1,	(64+16+\OffsetA)(\AREG) 
  xxpermdi	vs11,	vs10,	vs10,2	
  xxpermdi	vs15,	vs14,	vs14,2	
  lxv	vs2,	(64+32+\OffsetA)(\AREG)
  lxv	vs3,	(64+48+\OffsetA)(\AREG)

  xxperm  	vs26,	vs24,	permute_mask
  xxperm  	vs30,	vs28,	permute_mask	
  xxpermdi	vs25,	vs24,	vs24,2 
  xxpermdi	vs29,	vs28,	vs28,2	      
  xxpermdi	vs27,	vs26,	vs26,2	
  xxpermdi	vs31,	vs30,	vs30,2	 
.endm

.macro END8x16_2	  
  /*for load2 offset will be 128 and 64*/
   KERNEL8x16_2	AO,BO,	128,64,0 ,1,1 
.endm
 


.macro KERNEL8x16_E2	OffsetA,OffsetB, Index,IsLast 
  KERNEL8x16_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,1 
.endm


.macro KERNEL8x16_L2	OffsetA,OffsetB, Index,IsLast
  KERNEL8x16_2	AO,BO,	\OffsetA,\OffsetB, \Index,\IsLast ,0 
.endm


.macro KERNEL8x16_2  AREG,BREG,  OffsetA,OffsetB, Index,IsLast ,Complete
  xvmaddasp		vs32, vs4,vs8
  xvmaddasp		vs33, vs5,vs8
  xvmaddasp		vs48, vs4,vs12
  xvmaddasp		vs49, vs5,vs12

  xvmaddasp		vs40, vs4,vs10
  xvmaddasp		vs41, vs5,vs10
  xvmaddasp		vs56, vs4,vs14
  xvmaddasp		vs57, vs5,vs14

  xvmaddasp		vs36, vs4,vs9
  xvmaddasp		vs37, vs5,vs9
  xvmaddasp		vs52, vs4,vs13
  xvmaddasp		vs53, vs5,vs13

  xvmaddasp		vs44, vs4,vs11
  xvmaddasp		vs45, vs5,vs11
  xvmaddasp		vs60, vs4,vs15
  xvmaddasp		vs61, vs5,vs15

.if \Complete==0	
   lxv	vs4,	DISP32(\Index,0+\OffsetA)(\AREG)
   lxv	vs5,	DISP32(\Index,16+\OffsetA)(\AREG)
.endif

  xvmaddasp		vs34, vs6,vs8	
  xvmaddasp		vs35, vs7,vs8	
  xvmaddasp		vs50, vs6,vs12
  xvmaddasp		vs51, vs7,vs12
.if \Complete==0  
  lxv vs8,  DISP16(\Index,\OffsetB)(\BREG)
  lxv vs12, DISP16(\Index,16+\OffsetB)(\BREG)
.endif    
  xvmaddasp		vs42, vs6,vs10
  xvmaddasp		vs43, vs7,vs10
  xvmaddasp		vs58, vs6,vs14
  xvmaddasp		vs59, vs7,vs14
.if \Complete==0  
  xxperm    vs10, vs8,    permute_mask
  xxperm    vs14, vs12,   permute_mask    
.endif    
  xvmaddasp		vs38, vs6,vs9	
  xvmaddasp		vs39, vs7,vs9	
  xvmaddasp   vs54, vs6,vs13
  xvmaddasp   vs55, vs7,vs13
.if \Complete==0
  xxpermdi  vs9,  vs8,   vs8,2   
  xxpermdi  vs13, vs12, vs12,2   
.endif    
  xvmaddasp		vs46, vs6,vs11
  xvmaddasp		vs47, vs7,vs11
  xvmaddasp		vs62, vs6,vs15
  xvmaddasp		vs63, vs7,vs15
.if \Complete==0
  xxpermdi  vs11, vs10, vs10,2  
  xxpermdi  vs15, vs14, vs14,2  
.endif  

.if \Complete==0
   lxv	vs6,	DISP32(\Index,32+\OffsetA)(\AREG)
   lxv	vs7,	DISP32(\Index,48+\OffsetA)(\AREG) 
.endif 

  xvmaddasp		vs32, vs0,vs24
  xvmaddasp		vs33, vs1,vs24
  xvmaddasp		vs48, vs0,vs28
  xvmaddasp		vs49, vs1,vs28
  xvmaddasp		vs40, vs0,vs26
  xvmaddasp		vs41, vs1,vs26
  xvmaddasp		vs56, vs0,vs30
  xvmaddasp		vs57, vs1,vs30
  xvmaddasp		vs36, vs0,vs25
  xvmaddasp		vs37, vs1,vs25
  xvmaddasp		vs52, vs0,vs29
  xvmaddasp		vs53, vs1,vs29
  xvmaddasp		vs44, vs0,vs27
  xvmaddasp		vs45, vs1,vs27
  xvmaddasp		vs60, vs0,vs31
  xvmaddasp		vs61, vs1,vs31 
.if \Complete==0
  lxv	vs0,	DISP32(\Index,64+\OffsetA)(\AREG)
  lxv	vs1,	DISP32(\Index,64+16+\OffsetA)(\AREG) 
.endif

  xvmaddasp		vs34, vs2,vs24
  xvmaddasp		vs35, vs3,vs24	  
  xvmaddasp		vs50, vs2,vs28
  xvmaddasp		vs51, vs3,vs28
.if \Complete==0
  lxv vs24, DISP16(\Index,32+\OffsetB)(\BREG)
  lxv vs28, DISP16(\Index,32+16+\OffsetB)(\BREG)
.endif  
  xvmaddasp		vs42, vs2,vs26
  xvmaddasp		vs43, vs3,vs26
  xvmaddasp		vs58, vs2,vs30
  xvmaddasp		vs59, vs3,vs30
.if \Complete==0
  xxperm    vs26, vs24, permute_mask
  xxperm    vs30, vs28, permute_mask  
.endif  
  xvmaddasp		vs38, vs2,vs25
  xvmaddasp		vs39, vs3,vs25
  xvmaddasp		vs54, vs2,vs29
  xvmaddasp		vs55, vs3,vs29
.if \Complete==0
  xxpermdi  vs25, vs24, vs24,2 
  xxpermdi  vs29, vs28, vs28,2    
.endif  
  xvmaddasp		vs46, vs2,vs27
  xvmaddasp		vs47, vs3,vs27
  xvmaddasp		vs62, vs2,vs31	
  xvmaddasp		vs63, vs3,vs31
.if \Complete==0
  xxpermdi  vs27, vs26, vs26,2  
  xxpermdi  vs31, vs30, vs30,2   
.endif
.if \Complete==0
  lxv	vs2,	DISP32(\Index,64+32+\OffsetA)(\AREG)
  lxv	vs3,	DISP32(\Index,64+48+\OffsetA)(\AREG)
.endif


.if \IsLast==1	
.if \Complete==1
	addi		\BREG, \BREG,  DISP16(\Index,\OffsetB)
  addi    \AREG, \AREG, DISP32(\Index,\OffsetA)  

.else
	addi		\BREG, \BREG,  DISP16(\Index,64)
  addi    \AREG, \AREG, DISP32(\Index,128)  

.endif
.endif   


.endm

 
.macro SAVE8x16

  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 

  add     T2, CO, T10  
  add     T3, T1, T10  

  add     T4, T2, T10  
  add     T5, T3, T10 

  add     T6, T4, T10 
  add     T7, T5, T10 



   /* permute to restore butterfly rank 1 updateto normal promoted one */  
    /* permute 16 vs8 MEM(CO) vs9 MEM(CO+LDC) vs10 MEM(CO+2*LDC)  vs11 MEM(CO+3*LDC) */
    /* permute 16 vs12 MEM(16+CO) vs13 MEM(16+CO+LDC) vs14 MEM(16+CO+2*LDC)  vs15 MEM(16+CO+3*LDC) */
    /* permute 16 vs16 MEM(32+CO) vs17 MEM(32+CO+LDC) vs18 MEM(32+CO+2*LDC)  vs19 MEM(32+CO+3*LDC) */
    /* permute 16 vs24 MEM(32+CO) vs25 MEM(32+CO+LDC) vs26 MEM(32+CO+2*LDC)  vs27 MEM(32+CO+3*LDC) */

    xxmrglw     vs8,    vs32,   vs44
    xxmrglw     vs10,   vs36,   vs40  

    xxmrghw     vs1,    vs32,   vs44
    xxmrghw     vs0,    vs36,   vs40

    xxmrglw     vs12,   vs33,   vs45
    xxmrglw     vs14,   vs37,   vs41  

    xxmrghw     vs2,    vs37,   vs41
    xxmrghw     vs3,    vs33,   vs45
#ifndef TRMMKERNEL    
    lxv        vs32, 0(CO)
    lxv        vs33, 16(CO) 
#endif 
    xxmrglw     vs16,   vs34,   vs46
    xxmrglw     vs18,   vs38,   vs42   

    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10 

    xxmrghw     vs4,    vs38,   vs42
    xxmrghw     vs5,    vs34,   vs46

    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14

    xxmrglw     vs24,   vs35,   vs47
    xxmrglw     vs26,   vs39,   vs43  

    xxlor      vs17,    vs16,   vs16
    xxlor      vs19,    vs18,   vs18

    xxmrghw     vs30,   vs39,   vs43 
    xxmrghw     vs31,   vs35,   vs47
#ifndef TRMMKERNEL       
    lxv        vs34, 32(CO)  
    lxv        vs35, 48(CO)      
#endif
    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
#ifndef TRMMKERNEL    
    lxv        vs36, 0(T1)
    lxv        vs37, 16(T1) 
#endif
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      

#ifndef TRMMKERNEL      
    lxv        vs38, 32(T1)  
    lxv        vs39, 48(T1)     
#endif

    xxlor      vs25,    vs24,   vs24
    xxlor      vs27,    vs26,   vs26 



#ifndef TRMMKERNEL       
    lxv        vs40, 0(T2)
    lxv        vs41, 16(T2) 
#endif

    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1
#ifndef TRMMKERNEL     
    lxv        vs42, 32(T2)  
    lxv        vs43, 48(T2)     
#endif  
       
    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2  
#ifndef TRMMKERNEL    
    lxv        vs44, 0(T3)
    lxv        vs45, 16(T3)
#endif
    xxperm     vs16,    vs4,    save_permute_1
    xxperm     vs18,    vs5,    save_permute_1
#ifndef TRMMKERNEL      
    lxv        vs46, 32(T3)  
    lxv        vs47, 48(T3)                 
#endif  

    


      
    xxperm     vs17,    vs4,    save_permute_2   
    xxperm     vs19,    vs5,    save_permute_2      
#ifdef TRMMKERNEL
    xvmulsp     vs32,   vs8,    alpha_r 
    xvmulsp     vs33,   vs12,   alpha_r                 
#else 
    xvmaddasp   vs32,   vs8,    alpha_r 
    xvmaddasp   vs33,   vs12,   alpha_r            
#endif 
    xxperm     vs24,    vs30,   save_permute_1
    xxperm     vs26,    vs31,   save_permute_1 

 
    stxv        vs32, 0(CO)
    stxv        vs33, 16(CO)     
#ifdef TRMMKERNEL   
    xvmulsp     vs34,   vs16,   alpha_r 
    xvmulsp     vs35,   vs24,   alpha_r                 
#else    
    xvmaddasp   vs34,   vs16,   alpha_r 
    xvmaddasp   vs35,   vs24,   alpha_r           
#endif 
         
    xxperm     vs25,    vs30,   save_permute_2   
    xxperm     vs27,    vs31,   save_permute_2  


    stxv        vs34, 32(CO)  
    stxv        vs35, 48(CO)  
#ifdef TRMMKERNEL  
    xvmulsp     vs36,   vs9,    alpha_r 
    xvmulsp     vs37,   vs13,   alpha_r                
#else   
    xvmaddasp   vs36,   vs9,    alpha_r 
    xvmaddasp   vs37,   vs13,   alpha_r           
#endif 
    stxv        vs36, 0(T1)
    stxv        vs37, 16(T1)
#ifdef TRMMKERNEL  
    xvmulsp     vs38,   vs17,   alpha_r 
    xvmulsp     vs39,   vs25,   alpha_r               
#else   
    xvmaddasp   vs38,   vs17,   alpha_r 
    xvmaddasp   vs39,   vs25,   alpha_r         
#endif 
    stxv        vs38, 32(T1)  
    stxv        vs39, 48(T1)

#ifdef TRMMKERNEL
    xvmulsp     vs40,   vs10,   alpha_r 
    xvmulsp     vs41,   vs14,   alpha_r                    
#else 
    xvmaddasp   vs40,   vs10,   alpha_r 
    xvmaddasp   vs41,   vs14,   alpha_r   
#endif   

    stxv        vs40, 0(T2)
    stxv        vs41, 16(T2)  
#ifdef TRMMKERNEL 
    xvmulsp     vs42,   vs18,   alpha_r 
    xvmulsp     vs43,   vs26,   alpha_r                     
#else   
    xvmaddasp   vs42,   vs18,   alpha_r 
    xvmaddasp   vs43,   vs26,   alpha_r
#endif      
    stxv        vs42, 32(T2)  
    stxv        vs43, 48(T2)  
#ifdef TRMMKERNEL  
    xvmulsp     vs44,   vs11,   alpha_r 
    xvmulsp     vs45,   vs15,   alpha_r                    
#else
    xvmaddasp   vs44,   vs11,   alpha_r 
    xvmaddasp   vs45,   vs15,   alpha_r    
#endif      
    stxv        vs44, 0(T3)
    stxv        vs45, 16(T3) 
#ifdef TRMMKERNEL 
    xvmulsp     vs46,   vs19,   alpha_r 
    xvmulsp     vs47,   vs27,   alpha_r                   
#else 
    xvmaddasp   vs46,   vs19,   alpha_r 
    xvmaddasp   vs47,   vs27,   alpha_r 
#endif      
    stxv        vs46, 32(T3)  
    stxv        vs47, 48(T3)
  
 /*****the same with the second 8X8 ****/
 #ifndef TRMMKERNEL 
    lxv        vs32, 0(T4)
    lxv        vs33, 16(T4) 
#endif  
    xxmrglw     vs8,    vs48,   vs60
    xxmrglw     vs10,   vs52,   vs56  
#ifndef TRMMKERNEL    
    lxv        vs34, 32(T4)  
    lxv        vs35, 48(T4)  
#endif  
    xxmrghw     vs1,    vs48,   vs60
    xxmrghw     vs0,    vs52,   vs56
#ifndef TRMMKERNEL        
    lxv        vs36, 0(T5)
    lxv        vs37, 16(T5) 
#endif  
    xxmrglw     vs12,   vs49,   vs61
    xxmrglw     vs14,   vs53,   vs57  
#ifndef TRMMKERNEL    
    lxv        vs38,32(T5)  
    lxv        vs39, 48(T5)     
#endif   
 
    xxmrghw     vs2,    vs53,   vs57
    xxmrghw     vs3,    vs49,   vs61
#ifndef TRMMKERNEL   
    lxv        vs40, 0(T6)
    lxv        vs41, 16(T6)
#endif  
    xxmrglw     vs16,   vs50,   vs62
    xxmrglw     vs18,   vs54,   vs58   
#ifndef TRMMKERNEL      
    lxv        vs42, 32(T6)  
    lxv        vs43, 48(T6) 
#endif  
    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10 
    xxmrghw     vs4,    vs54,   vs58
    xxmrghw     vs5,    vs50,   vs62
#ifndef TRMMKERNEL              
    lxv        vs44, 0(T7)
    lxv        vs45, 16(T7) 
#endif  
    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14
 
    xxmrglw     vs24,   vs51,   vs63
    xxmrglw     vs26,   vs55,   vs59 
#ifndef TRMMKERNEL    
    lxv        vs46, 32(T7)  
    lxv        vs47, 48(T7)     
#endif  
    xxlor      vs17,    vs16,   vs16
    xxlor      vs19,    vs18,   vs18
    xxmrghw     vs30,   vs55,   vs59 
    xxmrghw     vs31,   vs51,   vs63 

 

    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
     
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      

    xxlor      vs25,    vs24,   vs24
    xxlor      vs27,    vs26,   vs26 
    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1

    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2      
 #ifdef TRMMKERNEL
    xvmulsp     vs32,   vs8,    alpha_r 
    xvmulsp     vs33,   vs12,   alpha_r                 
#else 
    xvmaddasp   vs32,   vs8,    alpha_r 
    xvmaddasp   vs33,   vs12,   alpha_r            
#endif  
    xxperm     vs16,    vs4,    save_permute_1
    xxperm     vs18,    vs5,    save_permute_1
    stxv        vs32, 0(T4)
    stxv        vs33, 16(T4) 
    xxperm     vs17,    vs4,    save_permute_2   
    xxperm     vs19,    vs5,    save_permute_2      
    xxperm     vs24,    vs30,   save_permute_1
    xxperm     vs26,    vs31,   save_permute_1 
    xxperm     vs25,    vs30,   save_permute_2   
    xxperm     vs27,    vs31,   save_permute_2      

#ifdef TRMMKERNEL   
    xvmulsp     vs34,   vs16,   alpha_r 
    xvmulsp     vs35,   vs24,   alpha_r                 
#else    
    xvmaddasp   vs34,   vs16,   alpha_r 
    xvmaddasp   vs35,   vs24,   alpha_r           
#endif 
    stxv        vs34, 32(T4)  
    stxv        vs35, 48(T4)  

#ifdef TRMMKERNEL  
    xvmulsp     vs36,   vs9,    alpha_r 
    xvmulsp     vs37,   vs13,   alpha_r                
#else   
    xvmaddasp   vs36,   vs9,    alpha_r 
    xvmaddasp   vs37,   vs13,   alpha_r           
#endif 
    stxv        vs36, 0(T5)
    stxv        vs37, 16(T5) 

#ifdef TRMMKERNEL  
    xvmulsp     vs38,   vs17,   alpha_r 
    xvmulsp     vs39,   vs25,   alpha_r               
#else  
    xvmaddasp   vs38,   vs17,   alpha_r 
    xvmaddasp   vs39,   vs25,   alpha_r         
#endif 



 
    stxv        vs38, 32(T5)  
    stxv        vs39, 48(T5)


#ifdef TRMMKERNEL
    xvmulsp     vs40,   vs10,   alpha_r 
    xvmulsp     vs41,   vs14,   alpha_r                    
#else 
    xvmaddasp   vs40,   vs10,   alpha_r 
    xvmaddasp   vs41,   vs14,   alpha_r   
#endif  
    stxv        vs40, 0(T6)
    stxv        vs41, 16(T6) 
#ifdef TRMMKERNEL 
    xvmulsp     vs42,   vs18,   alpha_r 
    xvmulsp     vs43,   vs26,   alpha_r                     
#else   
    xvmaddasp   vs42,   vs18,   alpha_r 
    xvmaddasp   vs43,   vs26,   alpha_r
#endif  
    stxv        vs42, 32(T6)  
    stxv        vs43, 48(T6)  
#ifdef TRMMKERNEL  
    xvmulsp     vs44,   vs11,   alpha_r 
    xvmulsp     vs45,   vs15,   alpha_r                    
#else
    xvmaddasp   vs44,   vs11,   alpha_r 
    xvmaddasp   vs45,   vs15,   alpha_r    
#endif  

    stxv        vs44, 0(T7)
    stxv        vs45, 16(T7) 
#ifdef TRMMKERNEL 
    xvmulsp     vs46,   vs19,   alpha_r 
    xvmulsp     vs47,   vs27,   alpha_r                   
#else 
    xvmaddasp   vs46,   vs19,   alpha_r 
    xvmaddasp   vs47,   vs27,   alpha_r 
#endif  
 
    stxv        vs46, 32(T7)  
    stxv        vs47, 48(T7)
  

    addi CO,CO,64


.endm



/**********************************************************************************************
* Macros for N=8 and M=8
**********************************************************************************************/

.macro LOAD8x8_1
   LOAD8x8 1
.endm

.macro LOAD8x8_0
   LOAD8x8 0
.endm

.macro KERNEL8x8_L1_L4  Index,IsLast
  KERNEL8x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL8x8_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL8x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL8x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL8x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast
   KERNEL8x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro KERNEL8x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro END8x8_NORMAL
  END8x8 0, AO, BO, 32,32 
.endm

.macro Zero8X8
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33
 
    xxlxor      vs36,   vs36,   vs36
    xxlxor      vs37,   vs37,   vs37
 
    xxlxor      vs40,   vs40,   vs40
    xxlxor      vs41,   vs41,   vs41
 
    xxlxor      vs44,   vs44,   vs44
    xxlxor      vs45,   vs45,   vs45
 
    xxlxor      vs48,   vs48,   vs48
    xxlxor      vs49,   vs49,   vs49
 
    xxlxor      vs52,   vs52,   vs52
    xxlxor      vs53,   vs53,   vs53
 
    xxlxor      vs56,   vs56,   vs56
    xxlxor      vs57,   vs57,   vs57
  
    xxlxor      vs60,   vs60,   vs60
    xxlxor      vs61,   vs61,   vs61
    
.endm

.macro LOAD8x8  Zero

    lxv vs24,   0(BO)
    lxv vs28,   16(BO)
    lxv vs0,     0(AO)
    lxv vs1,    16(AO)

    xxperm      vs26,   vs24,       permute_mask
    xxperm      vs30,   vs28,       permute_mask    
    xxpermdi    vs25,   vs24,   vs24,2     
    xxpermdi    vs29,   vs28,   vs28,2    

    xxpermdi    vs27,   vs26,   vs26,2  
    xxpermdi    vs31,   vs30,   vs30,2      

.if \Zero==1 
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs36,   vs36,   vs36
    xxlxor      vs37,   vs37,   vs37
    xxlxor      vs40,   vs40,   vs40
    xxlxor      vs41,   vs41,   vs41 
    xxlxor      vs44,   vs44,   vs44
    xxlxor      vs45,   vs45,   vs45 
    xxlxor      vs48,   vs48,   vs48
    xxlxor      vs49,   vs49,   vs49 
    xxlxor      vs52,   vs52,   vs52
    xxlxor      vs53,   vs53,   vs53 
    xxlxor      vs56,   vs56,   vs56
    xxlxor      vs57,   vs57,   vs57  
    xxlxor      vs60,   vs60,   vs60
    xxlxor      vs61,   vs61,   vs61  
.endif
.endm


.macro END8x8 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25

    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27

    xvmulsp     vs48, vs0,vs28
    xvmulsp     vs49, vs1,vs28

    xvmulsp     vs52, vs0,vs29
    xvmulsp     vs53, vs1,vs29

    xvmulsp     vs56, vs0,vs30
    xvmulsp     vs57, vs1,vs30

    xvmulsp     vs60, vs0,vs31
    xvmulsp     vs61, vs1,vs31

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

    xvmaddasp       vs48, vs0,vs28
    xvmaddasp       vs49, vs1,vs28

    xvmaddasp       vs52, vs0,vs29
    xvmaddasp       vs53, vs1,vs29

    xvmaddasp       vs56, vs0,vs30
    xvmaddasp       vs57, vs1,vs30

    xvmaddasp       vs60, vs0,vs31
    xvmaddasp       vs61, vs1,vs31

.endif
.endm  

.macro KERNEL8x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs8,    DISP32(\Index, 0+\OffsetB)(\BREG)
    lxv vs12,   DISP32(\Index,16+\OffsetB)(\BREG)

    lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)

    xxperm      vs10,   vs8,        permute_mask
    xxperm      vs14,   vs12,       permute_mask    

    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xxpermdi    vs9,    vs8,    vs8,2    
    xxpermdi    vs13,   vs12,   vs12,2   


    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xxpermdi    vs11,   vs10,   vs10,2  
    xxpermdi    vs15,   vs14,   vs14,2  

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

    xvmaddasp       vs48, vs0,vs28
    xvmaddasp       vs49, vs1,vs28

    xvmaddasp       vs52, vs0,vs29
    xvmaddasp       vs53, vs1,vs29
    lxv vs24,   DISP32(\Index,32+\OffsetB)(\BREG)
    lxv vs28,   DISP32(\Index,32+16+\OffsetB)(\BREG)
    xvmaddasp       vs56, vs0,vs30
    xvmaddasp       vs57, vs1,vs30

    xvmaddasp       vs60, vs0,vs31
    xvmaddasp       vs61, vs1,vs31

    xxperm      vs26,   vs24,       permute_mask
    xxperm      vs30,   vs28,       permute_mask    

    lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)
    lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)


    xxpermdi    vs25,   vs24,   vs24,2     
    xxpermdi    vs29,   vs28,   vs28,2    

    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

    xxpermdi    vs27,   vs26,   vs26,2  
    xxpermdi    vs31,   vs30,   vs30,2      

    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11

    xvmaddasp       vs48, vs4,vs12
    xvmaddasp       vs49, vs5,vs12

    xvmaddasp       vs52, vs4,vs13
    xvmaddasp       vs53, vs5,vs13
    lxv vs8,    DISP32(\Index,64+\OffsetB)(\BREG)
    lxv vs12,   DISP32(\Index,64+16+\OffsetB)(\BREG)
    xvmaddasp       vs56, vs4,vs14
    xvmaddasp       vs57, vs5,vs14

    xvmaddasp       vs60, vs4,vs15
    xvmaddasp       vs61, vs5,vs15

    xxperm      vs10,   vs8,        permute_mask
    xxperm      vs14,   vs12,       permute_mask   
 

    lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)
    lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)

 
    xxpermdi    vs9,    vs8,    vs8,2    
    xxpermdi    vs13,   vs12,   vs12,2  

    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xxpermdi    vs11,   vs10,   vs10,2  
    xxpermdi    vs15,   vs14,   vs14,2  

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

    xvmaddasp       vs48, vs0,vs28
    xvmaddasp       vs49, vs1,vs28

    xvmaddasp       vs52, vs0,vs29
    xvmaddasp       vs53, vs1,vs29
.if \Complete==0
    lxv vs24,   DISP32(\Index,96+\OffsetB)(\BREG)
    lxv vs28,   DISP32(\Index,96+16+\OffsetB)(\BREG)
.endif 
    xvmaddasp       vs56, vs0,vs30
    xvmaddasp       vs57, vs1,vs30
.if \Complete==0
    xxperm      vs26,   vs24,   permute_mask
    xxperm      vs30,   vs28,   permute_mask   
.endif 
    xvmaddasp       vs60, vs0,vs31
    xvmaddasp       vs61, vs1,vs31


.if \Complete==0
    lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)
    lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) 
.endif 

.if \Complete==0     
    xxpermdi    vs25,   vs24,   vs24,2 
    xxpermdi    vs29,   vs28,   vs28,2      

.endif 
.if \IsLast==1  
.if \Complete==1
  
    addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)
    addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)
.else
  
    addi        \BREG, \BREG,  DISP32(\Index,128)
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif
.endif   
 
    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

.if \Complete==0        
    xxpermdi    vs27,   vs26,   vs26,2  
    xxpermdi    vs31,   vs30,   vs30,2  
    
.endif
 
    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11

    xvmaddasp       vs48, vs4,vs12
    xvmaddasp       vs49, vs5,vs12

    xvmaddasp       vs52, vs4,vs13
    xvmaddasp       vs53, vs5,vs13

    xvmaddasp       vs56, vs4,vs14
    xvmaddasp       vs57, vs5,vs14

    xvmaddasp       vs60, vs4,vs15
    xvmaddasp       vs61, vs5,vs15

.endm

.macro KERNEL8x8 First

  LOAD8x8 0
  END8x8 \First, AO, BO, 32,32  
.endm

.macro KERNEL8x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete
    
    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG)
    lxv vs12,   DISP16(\Index,16+\OffsetB)(\BREG)

    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)

    xxperm      vs10,   vs8,        permute_mask
    xxperm      vs14,   vs12,       permute_mask    
    xxpermdi    vs9,    vs8,    vs8,2    
    xxpermdi    vs13,   vs12,   vs12,2   
.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

.endif

    xxpermdi    vs11,   vs10,   vs10,2  
    xxpermdi    vs15,   vs14,   vs14,2  
 
.if \First==1  
    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27

    xvmulsp     vs48, vs0,vs28
    xvmulsp     vs49, vs1,vs28

    xvmulsp     vs52, vs0,vs29
    xvmulsp     vs53, vs1,vs29

    xvmulsp     vs56, vs0,vs30
    xvmulsp     vs57, vs1,vs30

    xvmulsp     vs60, vs0,vs31
    xvmulsp     vs61, vs1,vs31

.else 
    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

    xvmaddasp       vs48, vs0,vs28
    xvmaddasp       vs49, vs1,vs28

    xvmaddasp       vs52, vs0,vs29
    xvmaddasp       vs53, vs1,vs29

    xvmaddasp       vs56, vs0,vs30
    xvmaddasp       vs57, vs1,vs30

    xvmaddasp       vs60, vs0,vs31
    xvmaddasp       vs61, vs1,vs31

.endif
.if \Complete==0
    lxv vs24,   DISP16(\Index,32+\OffsetB)(\BREG)
    lxv vs28,   DISP16(\Index,32+16+\OffsetB)(\BREG)

    lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)
    lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)

    xxperm      vs26,   vs24,   permute_mask
    xxperm      vs30,   vs28,   permute_mask    
    xxpermdi    vs25,   vs24,   vs24,2   
    xxpermdi    vs29,   vs28,   vs28,2  
.endif    
.if \IsLast==1  
.if \Complete==1
    addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB) 
    addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)

.else
    addi        \BREG, \BREG,  DISP16(\Index,64)
    addi        \AREG, \AREG,  DISP16(\Index,64) 
.endif
.endif

.if \First==1
    xvmulsp     vs32, vs4,vs8
    xvmulsp     vs33, vs5,vs8

    xvmulsp     vs36, vs4,vs9
    xvmulsp     vs37, vs5,vs9

.else
    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

.endif 
 
.if \Complete==0        
    xxpermdi    vs27,   vs26,   vs26,2  
    xxpermdi    vs31,   vs30,   vs30,2  
 
.endif
.if \First==1  
    xvmulsp     vs40, vs4,vs10
    xvmulsp     vs41, vs5,vs10

    xvmulsp     vs44, vs4,vs11
    xvmulsp     vs45, vs5,vs11

    xvmulsp     vs48, vs4,vs12
    xvmulsp     vs49, vs5,vs12

    xvmulsp     vs52, vs4,vs13
    xvmulsp     vs53, vs5,vs13

    xvmulsp     vs56, vs4,vs14
    xvmulsp     vs57, vs5,vs14

    xvmulsp     vs60, vs4,vs15
    xvmulsp     vs61, vs5,vs15

.else 
    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11

    xvmaddasp       vs48, vs4,vs12
    xvmaddasp       vs49, vs5,vs12

    xvmaddasp       vs52, vs4,vs13
    xvmaddasp       vs53, vs5,vs13

    xvmaddasp       vs56, vs4,vs14
    xvmaddasp       vs57, vs5,vs14

    xvmaddasp       vs60, vs4,vs15
    xvmaddasp       vs61, vs5,vs15

.endif

.endm


.macro SAVE8x8 
 
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 

  add     T2, CO, T10  
  add     T3, T1, T10  

  add     T4, T2, T10  
  add     T5, T3, T10 

  add     T6, T4, T10 
  add     T7, T5, T10 

#ifndef TRMMKERNEL    
    lxv        vs34, 0(CO)
    lxv        vs35, 16(CO)      
    lxv        vs38, 0(T1)
    lxv        vs39, 16(T1)  
    lxv        vs42, 0(T2)
    lxv        vs43, 16(T2)     
    lxv        vs46, 0(T3)
    lxv        vs47, 16(T3)  

    lxv        vs50, 0(T4)
    lxv        vs51, 16(T4)      
    lxv        vs54, 0(T5)
    lxv        vs55, 16(T5)  
    lxv        vs58, 0(T6)
    lxv        vs59, 16(T6)     
    lxv        vs62, 0(T7)
    lxv        vs63, 16(T7) 
#endif  

    xxmrglw     vs8,    vs32,   vs44
    xxmrglw     vs10,   vs36,   vs40  

    xxmrghw     vs1,    vs32,   vs44
    xxmrghw     vs0,    vs36,   vs40

    xxmrglw     vs12,   vs33,   vs45
    xxmrglw     vs14,   vs37,   vs41  

    xxmrghw     vs2,    vs37,   vs41
    xxmrghw     vs3,    vs33,   vs45

    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10 
 
    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14

    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      

    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1
      
    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2      


    /* multiply add normal way */
 
#ifdef TRMMKERNEL
    xvmulsp     vs34,   vs8,    alpha_r 
    xvmulsp     vs35,   vs12,   alpha_r 
    xvmulsp     vs38,   vs9,    alpha_r 
    xvmulsp     vs39,   vs13,   alpha_r 
    xvmulsp     vs42,   vs10,   alpha_r 
    xvmulsp     vs43,   vs14,   alpha_r 
    xvmulsp     vs46,   vs11,   alpha_r 
    xvmulsp     vs47,   vs15,   alpha_r                    
#else 
    xvmaddasp   vs34,   vs8,    alpha_r 
    xvmaddasp   vs35,   vs12,   alpha_r 
    xvmaddasp   vs38,   vs9,    alpha_r 
    xvmaddasp   vs39,   vs13,   alpha_r  
    xvmaddasp   vs42,   vs10,   alpha_r 
    xvmaddasp   vs43,   vs14,   alpha_r   
    xvmaddasp   vs46,   vs11,   alpha_r 
    xvmaddasp   vs47,   vs15,   alpha_r                     
#endif     
 
   
    xxmrglw     vs8,    vs48,   vs60
    xxmrglw     vs10,   vs52,   vs56  

    xxmrghw     vs1,    vs48,   vs60
    xxmrghw     vs0,    vs52,   vs56
    stxv        vs34, 0(CO)
    stxv        vs35, 16(CO) 
    xxmrglw     vs12,   vs49,   vs61
    xxmrglw     vs14,   vs53,   vs57  
    stxv        vs38, 0(T1)
    stxv        vs39, 16(T1) 
    xxmrghw     vs2,    vs53,   vs57
    xxmrghw     vs3,    vs49,   vs61
    stxv        vs42, 0(T2)
    stxv        vs43, 16(T2)   
    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10  
    stxv        vs46, 0(T3)
    stxv        vs47, 16(T3)  
    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14
   
    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
    
 
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      
 
    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1
    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2      
    
 #ifdef TRMMKERNEL
    xvmulsp     vs50,   vs8,    alpha_r 
    xvmulsp     vs51,   vs12,   alpha_r 
    xvmulsp     vs54,   vs9,    alpha_r 
    xvmulsp     vs55,   vs13,   alpha_r 
    xvmulsp     vs58,   vs10,   alpha_r 
    xvmulsp     vs59,   vs14,   alpha_r 
    xvmulsp     vs62,   vs11,   alpha_r 
    xvmulsp     vs63,   vs15,   alpha_r                    
#else 
    xvmaddasp     vs50,   vs8,    alpha_r 
    xvmaddasp     vs51,   vs12,   alpha_r 
    xvmaddasp     vs54,   vs9,    alpha_r 
    xvmaddasp     vs55,   vs13,   alpha_r 
    xvmaddasp     vs58,   vs10,   alpha_r 
    xvmaddasp     vs59,   vs14,   alpha_r 
    xvmaddasp     vs62,   vs11,   alpha_r 
    xvmaddasp     vs63,   vs15,   alpha_r                     
#endif  

    stxv        vs50, 0(T4)
    stxv        vs51, 16(T4)      
    stxv        vs54, 0(T5)
    stxv        vs55, 16(T5)  
    stxv        vs58, 0(T6)
    stxv        vs59, 16(T6)     
    stxv        vs62, 0(T7)
    stxv        vs63, 16(T7)   

    addi CO,CO,32

.endm


/**********************************************************************************************
* Macros for N=8 and M=4
**********************************************************************************************/

.macro LOAD8x4_1
   LOAD8x4 1
.endm

.macro LOAD8x4_0
   LOAD8x4 0
.endm

.macro KERNEL8x4_L1_L4  Index,IsLast
  KERNEL8x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL8x4_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL8x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL8x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL8x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast
   KERNEL8x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro KERNEL8x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL8x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL8x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro Zero8X4
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs34,   vs34,   vs34
    xxlxor      vs35,   vs35,   vs35
    
    xxlxor      vs48,   vs48,   vs48
    xxlxor      vs49,   vs49,   vs49
    xxlxor      vs50,   vs50,   vs50
    xxlxor      vs51,   vs51,   vs51  
    
.endm

.macro LOAD8x4  Zero

    lxv vs0,     0(AO)
    lxv vs24,   0(BO)
    lxv vs25,   16(BO)



    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2        

.if \Zero==1 
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs34,   vs34,   vs34
    xxlxor      vs35,   vs35,   vs35

    xxlxor      vs48,   vs48,   vs48
    xxlxor      vs49,   vs49,   vs49
    xxlxor      vs50,   vs50,   vs50
    xxlxor      vs51,   vs51,   vs51  
.endif
.endm

.macro END8x4_NORMAL
  END8x4 0, AO, BO, 16,32 
.endm

.macro END8x4 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp      vs32,   vs24,   vs0
    xvmulsp      vs33,   vs24,   vs1 
    xvmulsp      vs34,   vs24,   vs2
    xvmulsp      vs35,   vs24,   vs3

    xvmulsp      vs48,   vs25,   vs0
    xvmulsp      vs49,   vs25,   vs1
    xvmulsp      vs50,   vs25,   vs2
    xvmulsp      vs51,   vs25,   vs3  
.else
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3

    xvmaddasp      vs48,   vs25,   vs0
    xvmaddasp      vs49,   vs25,   vs1
    xvmaddasp      vs50,   vs25,   vs2
    xvmaddasp      vs51,   vs25,   vs3 

.endif
.endm  

.macro KERNEL8x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)
    lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3

    xvmaddasp      vs48,   vs25,   vs0
    xvmaddasp      vs49,   vs25,   vs1
    xvmaddasp      vs50,   vs25,   vs2
    xvmaddasp      vs51,   vs25,   vs3 

    lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)
    lxv vs24,   DISP32(\Index, 32+\OffsetB)(\BREG)
    lxv vs25,   DISP32(\Index, 48+\OffsetB)(\BREG) 

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   

    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7

    xvmaddasp      vs48,   vs27,   vs4
    xvmaddasp      vs49,   vs27,   vs5
    xvmaddasp      vs50,   vs27,   vs6
    xvmaddasp      vs51,   vs27,   vs7
 

    lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)
    lxv vs26,   DISP32(\Index, 64+\OffsetB)(\BREG)
    lxv vs27,   DISP32(\Index, 80+\OffsetB)(\BREG)

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3

    xvmaddasp      vs48,   vs25,   vs0
    xvmaddasp      vs49,   vs25,   vs1
    xvmaddasp      vs50,   vs25,   vs2
    xvmaddasp      vs51,   vs25,   vs3 

.if \Complete==0 

    lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)
    lxv vs24,   DISP32(\Index, 96+\OffsetB)(\BREG)
    lxv vs25,   DISP32(\Index, 96+16+\OffsetB)(\BREG) 

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   
.endif
    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7

    xvmaddasp      vs48,   vs27,   vs4
    xvmaddasp      vs49,   vs27,   vs5
    xvmaddasp      vs50,   vs27,   vs6
    xvmaddasp      vs51,   vs27,   vs7

 
 
.if \IsLast==1  
.if \Complete==1
    addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  
    addi        \BREG, \BREG,  DISP32(\Index,32*3+\OffsetB)

.else
    addi        \AREG, \AREG, DISP16(\Index,64)  
    addi        \BREG, \BREG,  DISP32(\Index,128)

.endif
.endif   
 
 
.endm

.macro KERNEL8x4 First
    LOAD8x4 0
    END8x4 \First, AO, BO, 16,32  
.endm

.macro KERNEL8x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)
    lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
.if \First==1
    xvmulsp      vs32,   vs24,   vs0
    xvmulsp      vs33,   vs24,   vs1 
    xvmulsp      vs34,   vs24,   vs2
    xvmulsp      vs35,   vs24,   vs3

    xvmulsp      vs48,   vs25,   vs0
    xvmulsp      vs49,   vs25,   vs1
    xvmulsp      vs50,   vs25,   vs2
    xvmulsp      vs51,   vs25,   vs3  
.else 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3

    xvmaddasp      vs48,   vs25,   vs0
    xvmaddasp      vs49,   vs25,   vs1
    xvmaddasp      vs50,   vs25,   vs2
    xvmaddasp      vs51,   vs25,   vs3 
.endif

.if \Complete==0 

    lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)
    lxv vs24,   DISP16(\Index, 32+\OffsetB)(\BREG)
    lxv vs25,   DISP16(\Index, 48+\OffsetB)(\BREG) 

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   
.endif

.if \First==1
    xvmulsp      vs32,   vs26,   vs4
    xvmulsp      vs33,   vs26,   vs5 
    xvmulsp      vs34,   vs26,   vs6
    xvmulsp      vs35,   vs26,   vs7

    xvmulsp      vs48,   vs27,   vs4
    xvmulsp      vs49,   vs27,   vs5
    xvmulsp      vs50,   vs27,   vs6
    xvmulsp      vs51,   vs27,   vs7


.else
    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7

    xvmaddasp      vs48,   vs27,   vs4
    xvmaddasp      vs49,   vs27,   vs5
    xvmaddasp      vs50,   vs27,   vs6
    xvmaddasp      vs51,   vs27,   vs7
.endif
 
 
.if \IsLast==1  
.if \Complete==1
    addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  
    addi        \BREG, \BREG,  DISP16(\Index,32+\OffsetB)

.else
    addi        \AREG, \AREG, DISP8(\Index,32)  
    addi        \BREG, \BREG,  DISP16(\Index,64)

.endif
.endif   
     
  
.endm


.macro SAVE8x4
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 
#if !defined(TRMMKERNEL)  
  lxv        vs36, 0(CO)
  lxv        vs37, 0(T1)
#endif  
  add     T2, CO, T10  
  add     T3, T1, T10 
#if !defined(TRMMKERNEL)    
  lxv        vs38, 0(T2)
  lxv        vs39, 0(T3)   
#endif   
  add     T4, T2, T10 
  add     T5, T3, T10
#if !defined(TRMMKERNEL)    
  lxv        vs40, 0(T4)
  lxv        vs41, 0(T5)
#endif  
  add     T6, T4, T10 
  add     T7, T5, T10
#if !defined(TRMMKERNEL)    
  lxv        vs42, 0(T6)
  lxv        vs43, 0(T7)
#endif
  xxmrglw  vs0, vs35,vs32
  xxmrglw  vs1, vs34,vs33 
  xxmrglw  vs4, vs32,vs35
  xxmrglw  vs5, vs33,vs34 


  xxmrghw  vs2, vs35,vs32
  xxmrghw  vs3, vs34,vs33 
  xxmrghw  vs6, vs32,vs35
  xxmrghw  vs7, vs33,vs34  

  xxmrgld  vs24, vs1, vs0  
  xxmrghd  vs25,vs5,vs4 

  xxmrgld  vs26, vs2, vs3  
  xxmrghd  vs27,vs6,vs7


  xxmrglw  vs0, vs51,vs48
  xxmrglw  vs1, vs50,vs49  
  xxmrglw  vs4, vs48,vs51
  xxmrglw  vs5, vs49,vs50 

  xxmrghw  vs2, vs51,vs48
  xxmrghw  vs3, vs50,vs49  
  xxmrghw  vs6, vs48,vs51
  xxmrghw  vs7, vs49,vs50   

  xxmrgld  vs28, vs1, vs0  
  xxmrghd  vs29,vs5,vs4

  xxmrgld  vs30, vs2, vs3   
  xxmrghd  vs31,vs6,vs7
#if defined(TRMMKERNEL)

  xvmulsp        vs36, vs24, alpha_r
  xvmulsp        vs37, vs25, alpha_r 
  xvmulsp        vs38, vs26, alpha_r
  xvmulsp        vs39, vs27, alpha_r   
  xvmulsp        vs40, vs28, alpha_r
  xvmulsp        vs41, vs29, alpha_r 
  xvmulsp        vs42, vs30, alpha_r
  xvmulsp        vs43, vs31, alpha_r
#else
  xvmaddasp        vs36, vs24, alpha_r
  xvmaddasp        vs37, vs25, alpha_r 
  xvmaddasp        vs38, vs26, alpha_r
  xvmaddasp        vs39, vs27, alpha_r   
  xvmaddasp        vs40, vs28, alpha_r
  xvmaddasp        vs41, vs29, alpha_r 
  xvmaddasp        vs42, vs30, alpha_r
  xvmaddasp        vs43, vs31, alpha_r
#endif

  stxv        vs36, 0(CO)
  stxv        vs37, 0(T1) 
  stxv        vs38, 0(T2)
  stxv        vs39, 0(T3)   
  stxv        vs40, 0(T4)
  stxv        vs41, 0(T5) 
  stxv        vs42, 0(T6)
  stxv        vs43, 0(T7)


  addi CO,CO,16
.endm


/**********************************************************************************************
* Macros for N=8 and M=2
**********************************************************************************************/

 
.macro KERNEL8x2_2   OffsetA,OffsetB, Index,IsLast
  KERNEL8x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast
.endm

 

.macro Zero8x2
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor      vs2,   vs2,   vs2
    xxlxor      vs3,   vs3,   vs3
       
.endm
 
.macro KERNEL8x2
  KERNEL8x2_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL8x2_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs27,   DISP8(\Index,16+\OffsetB)(\BREG)      
    xxspltw   vs8,  vs36, 0 
    xxspltw   vs9,  vs36, 1  
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8 
    xvmulsp      vs2,   vs26,   vs9
    xvmulsp      vs3,   vs27,   vs9 
     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs26,   vs9
    xvmaddasp      vs3,   vs27,   vs9
 
 .endif
   
    addi        \AREG, \AREG, DISP2(\Index,8)  
    addi        \BREG, \BREG, DISP8(\Index,32)
 
.endm

.macro KERNEL8x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG)
    lxv vs27,   DISP16(\Index,16+\OffsetB)(\BREG)
    lxv vs28,   DISP16(\Index,32+\OffsetB)(\BREG)
    lxv vs29,   DISP16(\Index,48+\OffsetB)(\BREG)      
    xxspltw   vs8,  vs4, 2  
    xxspltw   vs9,  vs4, 3 
    xxspltw   vs10, vs4, 0 
    xxspltw   vs11, vs4, 1
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8 
    xvmulsp      vs2,   vs26,   vs9
    xvmulsp      vs3,   vs27,   vs9 

    xvmulsp      vs0,   vs28,   vs10
    xvmulsp      vs1,   vs29,   vs10 
    xvmulsp      vs2,   vs28,   vs11
    xvmulsp      vs3,   vs29,   vs11     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs26,   vs9
    xvmaddasp      vs3,   vs27,   vs9

    xvmaddasp      vs0,   vs28,   vs10
    xvmaddasp      vs1,   vs29,   vs10 
    xvmaddasp      vs2,   vs28,   vs11
    xvmaddasp      vs3,   vs29,   vs11  
 .endif

 
.if \IsLast==1   
    addi        \AREG, \AREG, DISP4(\Index,16)  
    addi        \BREG, \BREG, DISP16(\Index,64)
.endif 
  
.endm


.macro SAVE8x2
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC  
  add     T2, CO, T10  
  add     T3, T1, T10     
  add     T4, T2, T10 
  add     T5, T3, T10 
  add     T6, T4, T10 
  add     T7, T5, T10 
  /*convert alpha_r for multiply*/
  xscvspdp  vs4,alpha_r
/* v0 corresponds to vs32, do not forget*/
#if !defined(TRMMKERNEL)
  lxssp  v0,0(CO) 
  lxssp  v1,4(CO) 

  lxssp  v2,0(T1)
  lxssp  v3,4(T1)

  lxssp  v4,0(T2)
  lxssp  v5,4(T2)

  lxssp  v6,0(T3)
  lxssp  v7,4(T3)

  lxssp  v8,0(T4)
  lxssp  v9,4(T4)

  lxssp  v10,0(T5)
  lxssp  v11,4(T5)

  lxssp  v12,0(T6)
  lxssp  v13,4(T6)

  lxssp  v14,0(T7)
  lxssp  v15,4(T7)
#endif
  xscvspdp  vs5, vs2
  xxspltw   vs6, vs2, 1 
  xxspltw   vs7, vs2, 2 
  xxspltw   vs8, vs2, 3  
  xscvspdp  vs6,vs6
  xscvspdp  vs7,vs7
  xscvspdp  vs8,vs8

  xscvspdp  vs24, vs0
  xxspltw   vs25, vs0, 1 
  xxspltw   vs26, vs0, 2 
  xxspltw   vs27, vs0, 3  
  xscvspdp  vs25,vs25
  xscvspdp  vs26,vs26
  xscvspdp  vs27,vs27

  xscvspdp  vs9, vs3
  xxspltw   vs10, vs3, 1 
  xxspltw   vs11, vs3, 2 
  xxspltw   vs12, vs3, 3  
  xscvspdp  vs10,vs10
  xscvspdp  vs11,vs11
  xscvspdp  vs12,vs12

  xscvspdp  vs28, vs1
  xxspltw   vs29, vs1, 1 
  xxspltw   vs30, vs1, 2 
  xxspltw   vs31, vs1, 3  
  xscvspdp  vs29,vs29
  xscvspdp  vs30,vs30
  xscvspdp  vs31,vs31




#if defined(TRMMKERNEL)
  xsmuldp  vs32,vs8, vs4 
  xsmuldp  vs33,vs27, vs4 

  xsmuldp  vs34,vs7, vs4 
  xsmuldp  vs35,vs26, vs4 

  xsmuldp  vs36,vs6, vs4 
  xsmuldp  vs37,vs25, vs4  

  xsmuldp  vs38,vs5, vs4 
  xsmuldp  vs39,vs24, vs4  

  xsmuldp  vs40,vs12, vs4 
  xsmuldp  vs41,vs31, vs4

  xsmuldp  vs42,vs11, vs4 
  xsmuldp  vs43,vs30, vs4  

  xsmuldp  vs44,vs10, vs4 
  xsmuldp  vs45,vs29, vs4 

  xsmuldp  vs46,vs9, vs4 
  xsmuldp  vs47,vs28, vs4      
#else
  xsmaddadp  vs32,vs8, vs4 
  xsmaddadp  vs33,vs27, vs4 

  xsmaddadp  vs34,vs7, vs4 
  xsmaddadp  vs35,vs26, vs4 

  xsmaddadp  vs36,vs6, vs4 
  xsmaddadp  vs37,vs25, vs4  

  xsmaddadp  vs38,vs5, vs4 
  xsmaddadp  vs39,vs24, vs4  

  xsmaddadp  vs40,vs12, vs4 
  xsmaddadp  vs41,vs31, vs4

  xsmaddadp  vs42,vs11, vs4 
  xsmaddadp  vs43,vs30, vs4  

  xsmaddadp  vs44,vs10, vs4 
  xsmaddadp  vs45,vs29, vs4 

  xsmaddadp  vs46,vs9, vs4 
  xsmaddadp  vs47,vs28, vs4     
#endif  

  stxssp  v0,0(CO) 
  stxssp  v1,4(CO) 

  stxssp  v2,0(T1)
  stxssp  v3,4(T1)

  stxssp  v4,0(T2)
  stxssp  v5,4(T2)

  stxssp  v6,0(T3)
  stxssp  v7,4(T3)

  stxssp  v8,0(T4)
  stxssp  v9,4(T4)

  stxssp  v10,0(T5)
  stxssp  v11,4(T5)

  stxssp  v12,0(T6)
  stxssp  v13,4(T6)

  stxssp  v14,0(T7)
  stxssp  v15,4(T7)
 

  addi CO,CO,8
.endm


/**********************************************************************************************
* Macros for N=8 and M=1
**********************************************************************************************/
.macro KERNEL8x1_4   OffsetA,OffsetB, Index,IsLast
  KERNEL8x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro Zero8x1
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1  
.endm

.macro KERNEL8x1
  KERNEL8x1_1 AO,BO, 0 
.endm

.macro KERNEL8x1_2
  KERNEL8x1_2_1 AO,BO, 0 
.endm

.macro KERNEL8x1_1 AREG,BREG,First 
    lxvwsx vs8,  0, \AREG
    lxv vs26,   0(\BREG)
    lxv vs27,   16(\BREG)      
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8  
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
 .endif
    addi        \AREG, \AREG,  4  
    addi        \BREG, \BREG,  32
.endm

.macro KERNEL8x1_2_1 AREG,BREG,First 
    lxsd v4,    0(\AREG)
    lxv vs26,   0(\BREG)
    lxv vs27,  16(\BREG)      
    lxv vs28,  32(\BREG)
    lxv vs29,  48(\BREG) 
    xxspltw   vs8,  vs36, 1 
    xxspltw   vs9,  vs36, 0  
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8  
    xvmulsp      vs0,   vs28,   vs9
    xvmulsp      vs1,   vs29,   vs9     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
    xvmaddasp      vs0,   vs28,   vs9
    xvmaddasp      vs1,   vs29,   vs9 
 .endif
    addi        \AREG, \AREG,  8 
    addi        \BREG, \BREG,  64
.endm

.macro KERNEL8x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  
    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)
    xxspltw   vs8,  vs4, 3 
    xxspltw   vs9,  vs4, 2 
    xxspltw   vs10, vs4, 1 
    xxspltw   vs11, vs4, 0
    lxv vs26,   DISP32(\Index, 0+\OffsetB)(\BREG)
    lxv vs27,   DISP32(\Index,16+\OffsetB)(\BREG)
    lxv vs28,   DISP32(\Index,32+\OffsetB)(\BREG)
    lxv vs29,   DISP32(\Index,48+\OffsetB)(\BREG) 
    lxv vs30,   DISP32(\Index,64+ 0+\OffsetB)(\BREG)
    lxv vs31,   DISP32(\Index,64+16+\OffsetB)(\BREG)
    lxv vs32,   DISP32(\Index,64+32+\OffsetB)(\BREG)
    lxv vs33,   DISP32(\Index,64+48+\OffsetB)(\BREG)         
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8  
    xvmulsp      vs0,   vs28,   vs9
    xvmulsp      vs1,   vs29,   vs9     
    xvmulsp      vs0,   vs30,   vs10
    xvmulsp      vs1,   vs31,   vs10  
    xvmulsp      vs0,   vs32,   vs11
    xvmulsp      vs1,   vs33,   vs11     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
    xvmaddasp      vs0,   vs28,   vs9
    xvmaddasp      vs1,   vs29,   vs9     
    xvmaddasp      vs0,   vs30,   vs10
    xvmaddasp      vs1,   vs31,   vs10  
    xvmaddasp      vs0,   vs32,   vs11
    xvmaddasp      vs1,   vs33,   vs11  
 .endif
.if \IsLast==1   
    addi        \AREG, \AREG, DISP4(\Index,16)  
    addi        \BREG, \BREG, DISP32(\Index,128)
.endif 
.endm

.macro SAVE8x1
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC  
  add     T2, CO, T10  
  add     T3, T1, T10     
  add     T4, T2, T10 
  add     T5, T3, T10 
  add     T6, T4, T10 
  add     T7, T5, T10 
  /*convert alpha_r for multiply*/
  xscvspdp  vs4,alpha_r
/* v0 corresponds to vs32, do not forget*/
#if !defined(TRMMKERNEL)
  lxssp  v0,0(CO)  
  lxssp  v2,0(T1) 
  lxssp  v4,0(T2) 
  lxssp  v6,0(T3) 
  lxssp  v8,0(T4) 
  lxssp  v10,0(T5) 
  lxssp  v12,0(T6) 
  lxssp  v14,0(T7)
#endif
  xscvspdp  vs24, vs0
  xxspltw   vs25, vs0, 1 
  xxspltw   vs26, vs0, 2 
  xxspltw   vs27, vs0, 3  
  xscvspdp  vs25,vs25
  xscvspdp  vs26,vs26
  xscvspdp  vs27,vs27
  xscvspdp  vs28, vs1
  xxspltw   vs29, vs1, 1 
  xxspltw   vs30, vs1, 2 
  xxspltw   vs31, vs1, 3  
  xscvspdp  vs29,vs29
  xscvspdp  vs30,vs30
  xscvspdp  vs31,vs31
#if defined(TRMMKERNEL)
  xsmuldp  vs32,vs27, vs4 
  xsmuldp  vs34,vs26, vs4 
  xsmuldp  vs36,vs25, vs4 
  xsmuldp  vs38,vs24, vs4 
  xsmuldp  vs40,vs31, vs4 
  xsmuldp  vs42,vs30, vs4 
  xsmuldp  vs44,vs29, vs4 
  xsmuldp  vs46,vs28, vs4 
#else
  xsmaddadp  vs32,vs27, vs4 
  xsmaddadp  vs34,vs26, vs4 
  xsmaddadp  vs36,vs25, vs4 
  xsmaddadp  vs38,vs24, vs4 
  xsmaddadp  vs40,vs31, vs4 
  xsmaddadp  vs42,vs30, vs4 
  xsmaddadp  vs44,vs29, vs4 
  xsmaddadp  vs46,vs28, vs4  
#endif  
  stxssp  v0,0(CO)  
  stxssp  v2,0(T1) 
  stxssp  v4,0(T2) 
  stxssp  v6,0(T3) 
  stxssp  v8,0(T4) 
  stxssp  v10,0(T5) 
  stxssp  v12,0(T6) 
  stxssp  v14,0(T7) 
  addi CO,CO,4
.endm



/**********************************************************************************************
* Macros for N=4 and M=16
**********************************************************************************************/

.macro LOAD4x16_1
   LOAD4x16 1
.endm

.macro LOAD4x16_0
   LOAD4x16 0
.endm

.macro KERNEL4x16_L1_L4  Index,IsLast
  KERNEL4x16_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL4x16_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL4x16_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x16_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x16_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL4x16_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x16_I1_L2_3  OffsetA,OffsetB, Index,IsLast
   KERNEL4x16_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro KERNEL4x16_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x16_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x16_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x16_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro Zero4X16
    xxlxor		vs32,	vs32,	vs32
    xxlxor		vs33,	vs33,	vs33
	xxlxor		vs34,	vs34,	vs34
	xxlxor		vs35,	vs35,	vs35
	xxlxor		vs36,	vs36,	vs36
	xxlxor		vs37,	vs37,	vs37
	xxlxor		vs38,	vs38,	vs38
	xxlxor		vs39,	vs39,	vs39
	xxlxor		vs40,	vs40,	vs40
	xxlxor		vs41,	vs41,	vs41
	xxlxor		vs42,	vs42,	vs42
	xxlxor		vs43,	vs43,	vs43
	xxlxor		vs44,	vs44,	vs44
	xxlxor		vs45,	vs45,	vs45
	xxlxor		vs46,	vs46,	vs46
	xxlxor		vs47,	vs47,	vs47	
.endm

.macro LOAD4x16  Zero

	lxv	vs24,	0(BO) 
	lxv	vs0,	 0(AO)
	lxv	vs1,	16(AO)
	lxv	vs2,	32(AO)
	lxv	vs3,	48(AO)
	xxperm  	vs26,	vs24,		permute_mask 	
	xxpermdi	vs25,	vs24,	vs24,2 
	xxpermdi	vs27,	vs26,	vs26,2	 	

.if \Zero==1 
    xxlxor		vs32,	vs32,	vs32
    xxlxor		vs33,	vs33,	vs33
	xxlxor		vs34,	vs34,	vs34
	xxlxor		vs35,	vs35,	vs35
	xxlxor		vs36,	vs36,	vs36
	xxlxor		vs37,	vs37,	vs37
	xxlxor		vs38,	vs38,	vs38
	xxlxor		vs39,	vs39,	vs39
	xxlxor		vs40,	vs40,	vs40
	xxlxor		vs41,	vs41,	vs41
	xxlxor		vs42,	vs42,	vs42
	xxlxor		vs43,	vs43,	vs43
	xxlxor		vs44,	vs44,	vs44
	xxlxor		vs45,	vs45,	vs45
	xxlxor		vs46,	vs46,	vs46
	xxlxor		vs47,	vs47,	vs47
 
.endif
.endm

.macro END4x16_NORMAL
  END4x16 0, AO, BO, 64,16 
.endm

.macro END4x16 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24
    xvmulsp     vs34, vs2,vs24  
    xvmulsp     vs35, vs3,vs24  

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25
    xvmulsp     vs38, vs2,vs25  
    xvmulsp     vs39, vs3,vs25

    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26
    xvmulsp     vs42, vs2,vs26  
    xvmulsp     vs43, vs3,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27
    xvmulsp     vs46, vs2,vs27  
    xvmulsp     vs47, vs3,vs27

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24
    xvmaddasp       vs34, vs2,vs24  
    xvmaddasp       vs35, vs3,vs24  

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25
    xvmaddasp       vs38, vs2,vs25  
    xvmaddasp       vs39, vs3,vs25 
    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26
    xvmaddasp       vs42, vs2,vs26  
    xvmaddasp       vs43, vs3,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27
    xvmaddasp       vs46, vs2,vs27  
    xvmaddasp       vs47, vs3,vs27

.endif
.endm  

.macro KERNEL4x16_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

	lxv	vs8,	DISP16(\Index, 0+\OffsetB)(\BREG) 

 	lxv	vs4,	DISP64(\Index, 0+\OffsetA)(\AREG)
	lxv	vs5,	DISP64(\Index,16+\OffsetA)(\AREG)
	lxv	vs6,	DISP64(\Index,32+\OffsetA)(\AREG)
	lxv	vs7,	DISP64(\Index,48+\OffsetA)(\AREG) 

	xxperm  	vs10,	vs8,		permute_mask 
	xxpermdi	vs9,	vs8,	vs8,2	  

    xvmaddasp		vs32, vs0,vs24
	xvmaddasp		vs33, vs1,vs24
	xvmaddasp		vs34, vs2,vs24	
	xvmaddasp		vs35, vs3,vs24	 

    xvmaddasp		vs36, vs0,vs25
	xvmaddasp		vs37, vs1,vs25
	xvmaddasp		vs38, vs2,vs25	
	xvmaddasp		vs39, vs3,vs25 

 	xxpermdi	vs11,	vs10,	vs10,2	 

    xvmaddasp		vs40, vs0,vs26
	xvmaddasp		vs41, vs1,vs26
	xvmaddasp		vs42, vs2,vs26	
	xvmaddasp		vs43, vs3,vs26

    xvmaddasp		vs44, vs0,vs27
	xvmaddasp		vs45, vs1,vs27
	xvmaddasp		vs46, vs2,vs27	
	xvmaddasp		vs47, vs3,vs27



	lxv	vs24,	DISP16(\Index,16+\OffsetB)(\BREG) 

	lxv	vs0,	DISP64(\Index,64+\OffsetA)(\AREG)
	lxv	vs1,	DISP64(\Index,64+16+\OffsetA)(\AREG)
	lxv	vs2,	DISP64(\Index,64+32+\OffsetA)(\AREG)
	lxv	vs3,	DISP64(\Index,64+48+\OffsetA)(\AREG)

	xxperm  	vs26,	vs24,		permute_mask 
	xxpermdi	vs25,	vs24,	vs24,2	    
 

    xvmaddasp		vs32, vs4,vs8
	xvmaddasp		vs33, vs5,vs8
	xvmaddasp		vs34, vs6,vs8	
	xvmaddasp		vs35, vs7,vs8	
 
    xvmaddasp		vs36, vs4,vs9
	xvmaddasp		vs37, vs5,vs9
	xvmaddasp		vs38, vs6,vs9	
	xvmaddasp		vs39, vs7,vs9
         
	xxpermdi	vs27,	vs26,	vs26,2	 	

    xvmaddasp		vs40, vs4,vs10
	xvmaddasp		vs41, vs5,vs10
	xvmaddasp		vs42, vs6,vs10	
	xvmaddasp		vs43, vs7,vs10

    xvmaddasp		vs44, vs4,vs11
	xvmaddasp		vs45, vs5,vs11
	xvmaddasp		vs46, vs6,vs11	
	xvmaddasp		vs47, vs7,vs11
 

	lxv	vs8,	DISP16(\Index,32+\OffsetB)(\BREG) 

 	lxv	vs4,	DISP64(\Index,128+0+\OffsetA)(\AREG)
	lxv	vs5,	DISP64(\Index,128+16+\OffsetA)(\AREG)
	lxv	vs6,	DISP64(\Index,128+32+\OffsetA)(\AREG)
	lxv	vs7,	DISP64(\Index,128+48+\OffsetA)(\AREG) 

	xxperm  	vs10,	vs8,		permute_mask 
	xxpermdi	vs9,	vs8,	vs8,2	  

    xvmaddasp		vs32, vs0,vs24
	xvmaddasp		vs33, vs1,vs24
	xvmaddasp		vs34, vs2,vs24	
	xvmaddasp		vs35, vs3,vs24	 

    xvmaddasp		vs36, vs0,vs25
	xvmaddasp		vs37, vs1,vs25
	xvmaddasp		vs38, vs2,vs25	
	xvmaddasp		vs39, vs3,vs25

 	xxpermdi	vs11,	vs10,	vs10,2	 

    xvmaddasp		vs40, vs0,vs26
	xvmaddasp		vs41, vs1,vs26
	xvmaddasp		vs42, vs2,vs26	
	xvmaddasp		vs43, vs3,vs26

    xvmaddasp		vs44, vs0,vs27
	xvmaddasp		vs45, vs1,vs27
	xvmaddasp		vs46, vs2,vs27	
	xvmaddasp		vs47, vs3,vs27

 
 
.if \Complete==0
	lxv	vs24,	DISP16(\Index,48+\OffsetB)(\BREG) 

	lxv	vs0,	DISP64(\Index,192+\OffsetA)(\AREG)
	lxv	vs1,	DISP64(\Index,192+16+\OffsetA)(\AREG) 
	lxv	vs2,	DISP64(\Index,192+32+\OffsetA)(\AREG)
	lxv	vs3,	DISP64(\Index,192+48+\OffsetA)(\AREG)

	xxperm  	vs26,	vs24,	permute_mask 	
	xxpermdi	vs25,	vs24,	vs24,2  	

.endif 
.if \IsLast==1	
.if \Complete==1
  
	addi		\BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)
	addi		\AREG, \AREG, DISP64(\Index,64*3+\OffsetA)
.else
  
	addi		\BREG, \BREG,  DISP16(\Index,64)
	addi		\AREG, \AREG, DISP64(\Index,256)
.endif
.endif   
 
    xvmaddasp		vs32, vs4,vs8
	xvmaddasp		vs33, vs5,vs8
	xvmaddasp		vs34, vs6,vs8	
	xvmaddasp		vs35, vs7,vs8	 
 
    xvmaddasp		vs36, vs4,vs9
	xvmaddasp		vs37, vs5,vs9
	xvmaddasp		vs38, vs6,vs9	
	xvmaddasp		vs39, vs7,vs9
  
.if \Complete==0        
	xxpermdi	vs27,	vs26,	vs26,2	 
 	
.endif
 
    xvmaddasp		vs40, vs4,vs10
	xvmaddasp		vs41, vs5,vs10
	xvmaddasp		vs42, vs6,vs10	
	xvmaddasp		vs43, vs7,vs10

    xvmaddasp		vs44, vs4,vs11
	xvmaddasp		vs45, vs5,vs11
	xvmaddasp		vs46, vs6,vs11	
	xvmaddasp		vs47, vs7,vs11

 

.endm

.macro KERNEL4x16 First

  LOAD4x16 0
  END4x16 \First, AO, BO, 64,16 
.endm

.macro KERNEL4x16_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete
	
	lxv	vs8,	DISP8(\Index, 0+\OffsetB)(\BREG) 
 	lxv	vs4,	DISP32(\Index, 0+\OffsetA)(\AREG)
	lxv	vs5,	DISP32(\Index,16+\OffsetA)(\AREG)
	lxv	vs6,	DISP32(\Index,32+\OffsetA)(\AREG)
	lxv	vs7,	DISP32(\Index,48+\OffsetA)(\AREG) 

	xxperm  	vs10,	vs8,		permute_mask 
	xxpermdi	vs9,	vs8,	vs8,2	  
.if \First==1
    xvmulsp		vs32, vs0,vs24
	xvmulsp		vs33, vs1,vs24
	xvmulsp		vs34, vs2,vs24	
	xvmulsp		vs35, vs3,vs24	

    xvmulsp		vs36, vs0,vs25
	xvmulsp		vs37, vs1,vs25
	xvmulsp		vs38, vs2,vs25	
	xvmulsp		vs39, vs3,vs25	
.else
    xvmaddasp		vs32, vs0,vs24
	xvmaddasp		vs33, vs1,vs24
	xvmaddasp		vs34, vs2,vs24	
	xvmaddasp		vs35, vs3,vs24

    xvmaddasp		vs36, vs0,vs25
	xvmaddasp		vs37, vs1,vs25
	xvmaddasp		vs38, vs2,vs25	
	xvmaddasp		vs39, vs3,vs25		
.endif

 	xxpermdi	vs11,	vs10,	vs10,2	 	
 
.if \First==1  
    xvmulsp		vs40, vs0,vs26
	xvmulsp		vs41, vs1,vs26
	xvmulsp		vs42, vs2,vs26	
	xvmulsp		vs43, vs3,vs26

    xvmulsp		vs44, vs0,vs27
	xvmulsp		vs45, vs1,vs27
	xvmulsp		vs46, vs2,vs27	
	xvmulsp		vs47, vs3,vs27

  
.else 
    xvmaddasp		vs40, vs0,vs26
	xvmaddasp		vs41, vs1,vs26
	xvmaddasp		vs42, vs2,vs26	
	xvmaddasp		vs43, vs3,vs26

    xvmaddasp		vs44, vs0,vs27
	xvmaddasp		vs45, vs1,vs27
	xvmaddasp		vs46, vs2,vs27	
	xvmaddasp		vs47, vs3,vs27
 

.endif
.if \Complete==0
	lxv	vs24,	DISP8(\Index,16+\OffsetB)(\BREG) 
	lxv	vs0,	DISP32(\Index,64+\OffsetA)(\AREG)
	lxv	vs1,	DISP32(\Index,64+16+\OffsetA)(\AREG)
	lxv	vs2,	DISP32(\Index,64+32+\OffsetA)(\AREG)
	lxv	vs3,	DISP32(\Index,64+48+\OffsetA)(\AREG)

	xxperm  	vs26,	vs24,	permute_mask 
	xxpermdi	vs25,	vs24,	vs24,2	  
.endif    
.if \IsLast==1	
.if \Complete==1
 	addi		\BREG, \BREG,  DISP8(\Index,16+\OffsetB) 
	addi		\AREG, \AREG, DISP32(\Index,64+\OffsetA)

.else
  	addi		\BREG, \BREG,  DISP8(\Index,32)
	addi		\AREG, \AREG, DISP32(\Index,128) 
.endif
.endif

.if \First==1
    xvmulsp		vs32, vs4,vs8
	xvmulsp		vs33, vs5,vs8
	xvmulsp		vs34, vs6,vs8	
	xvmulsp		vs35, vs7,vs8

    xvmulsp		vs36, vs4,vs9
	xvmulsp		vs37, vs5,vs9
	xvmulsp		vs38, vs6,vs9	
	xvmulsp		vs39, vs7,vs9
.else
    xvmaddasp		vs32, vs4,vs8
	xvmaddasp		vs33, vs5,vs8
	xvmaddasp		vs34, vs6,vs8	
	xvmaddasp		vs35, vs7,vs8	

    xvmaddasp		vs36, vs4,vs9
	xvmaddasp		vs37, vs5,vs9
	xvmaddasp		vs38, vs6,vs9	
	xvmaddasp		vs39, vs7,vs9
.endif 
 
.if \Complete==0        
	xxpermdi	vs27,	vs26,	vs26,2	 
 
.endif
.if \First==1  
    xvmulsp		vs40, vs4,vs10
	xvmulsp		vs41, vs5,vs10
	xvmulsp		vs42, vs6,vs10	
	xvmulsp		vs43, vs7,vs10

    xvmulsp		vs44, vs4,vs11
	xvmulsp		vs45, vs5,vs11
	xvmulsp		vs46, vs6,vs11	
	xvmulsp		vs47, vs7,vs11

 

.else 
    xvmaddasp		vs40, vs4,vs10
	xvmaddasp		vs41, vs5,vs10
	xvmaddasp		vs42, vs6,vs10	
	xvmaddasp		vs43, vs7,vs10

    xvmaddasp		vs44, vs4,vs11
	xvmaddasp		vs45, vs5,vs11
	xvmaddasp		vs46, vs6,vs11	
	xvmaddasp		vs47, vs7,vs11

 

.endif

.endm

 
.macro SAVE4x16

  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 

  add     T2, CO, T10  
  add     T3, T1, T10  

  
 
    xxmrglw     vs8,    vs32,   vs44
    xxmrglw     vs10,   vs36,   vs40  

    xxmrghw     vs1,    vs32,   vs44
    xxmrghw     vs0,    vs36,   vs40

    xxmrglw     vs12,   vs33,   vs45
    xxmrglw     vs14,   vs37,   vs41  

    xxmrghw     vs2,    vs37,   vs41
    xxmrghw     vs3,    vs33,   vs45

    xxmrglw     vs16,   vs34,   vs46
    xxmrglw     vs18,   vs38,   vs42   

    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10 

    xxmrghw     vs4,    vs38,   vs42
    xxmrghw     vs5,    vs34,   vs46

    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14

    xxmrglw     vs24,   vs35,   vs47
    xxmrglw     vs26,   vs39,   vs43  

    xxlor      vs17,    vs16,   vs16
    xxlor      vs19,    vs18,   vs18

    xxmrghw     vs30,   vs39,   vs43 
    xxmrghw     vs31,   vs35,   vs47

    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      

#ifndef TRMMKERNEL    
    lxv        vs32, 0(CO)
    lxv        vs33, 16(CO) 
    lxv        vs34, 32(CO)  
    lxv        vs35, 48(CO)      
#endif
    xxlor      vs25,    vs24,   vs24
    xxlor      vs27,    vs26,   vs26 

#ifndef TRMMKERNEL    
    lxv        vs36, 0(T1)
    lxv        vs37, 16(T1) 
    lxv        vs38, 32(T1)  
    lxv        vs39, 48(T1)     
#endif
#ifndef TRMMKERNEL       
    lxv        vs40, 0(T2)
    lxv        vs41, 16(T2) 
    lxv        vs42, 32(T2)  
    lxv        vs43, 48(T2)     
#endif  
#ifndef TRMMKERNEL    
    lxv        vs44, 0(T3)
    lxv        vs45, 16(T3) 
    lxv        vs46, 32(T3)  
    lxv        vs47, 48(T3)                 
#endif  

    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1
       
    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2      

    xxperm     vs16,    vs4,    save_permute_1
    xxperm     vs18,    vs5,    save_permute_1
      
    xxperm     vs17,    vs4,    save_permute_2   
    xxperm     vs19,    vs5,    save_permute_2      

    xxperm     vs24,    vs30,   save_permute_1
    xxperm     vs26,    vs31,   save_permute_1 
         
    xxperm     vs25,    vs30,   save_permute_2   
    xxperm     vs27,    vs31,   save_permute_2  


    /* multiply add normal way */
 
#ifdef TRMMKERNEL
    xvmulsp     vs32,   vs8,    alpha_r 
    xvmulsp     vs33,   vs12,   alpha_r   
    xvmulsp     vs34,   vs16,   alpha_r 
    xvmulsp     vs35,   vs24,   alpha_r  
    xvmulsp     vs36,   vs9,    alpha_r 
    xvmulsp     vs37,   vs13,   alpha_r  
    xvmulsp     vs38,   vs17,   alpha_r 
    xvmulsp     vs39,   vs25,   alpha_r               
#else 
    xvmaddasp   vs32,   vs8,    alpha_r 
    xvmaddasp   vs33,   vs12,   alpha_r   
    xvmaddasp   vs34,   vs16,   alpha_r 
    xvmaddasp   vs35,   vs24,   alpha_r  
    xvmaddasp   vs36,   vs9,    alpha_r 
    xvmaddasp   vs37,   vs13,   alpha_r   
    xvmaddasp   vs38,   vs17,   alpha_r 
    xvmaddasp   vs39,   vs25,   alpha_r         
#endif 



#ifdef TRMMKERNEL
    xvmulsp     vs40,   vs10,   alpha_r 
    xvmulsp     vs41,   vs14,   alpha_r 
    xvmulsp     vs42,   vs18,   alpha_r 
    xvmulsp     vs43,   vs26,   alpha_r  
    xvmulsp     vs44,   vs11,   alpha_r 
    xvmulsp     vs45,   vs15,   alpha_r  
    xvmulsp     vs46,   vs19,   alpha_r 
    xvmulsp     vs47,   vs27,   alpha_r                   
#else

    xvmaddasp   vs40,   vs10,   alpha_r 
    xvmaddasp   vs41,   vs14,   alpha_r   
    xvmaddasp   vs42,   vs18,   alpha_r 
    xvmaddasp   vs43,   vs26,   alpha_r  
    xvmaddasp   vs44,   vs11,   alpha_r 
    xvmaddasp   vs45,   vs15,   alpha_r 
    xvmaddasp   vs46,   vs19,   alpha_r 
    xvmaddasp   vs47,   vs27,   alpha_r  
        
#endif  

    stxv        vs32, 0(CO)
    stxv        vs33, 16(CO) 
    stxv        vs34, 32(CO)  
    stxv        vs35, 48(CO)  

    stxv        vs36, 0(T1)
    stxv        vs37, 16(T1)  
    stxv        vs38, 32(T1)  
    stxv        vs39, 48(T1)

    stxv        vs40, 0(T2)
    stxv        vs41, 16(T2)  
    stxv        vs42, 32(T2)  
    stxv        vs43, 48(T2)  
    stxv        vs44, 0(T3)
    stxv        vs45, 16(T3) 
    stxv        vs46, 32(T3)  
    stxv        vs47, 48(T3)
   
    addi CO,CO,64


.endm



/**********************************************************************************************
* Macros for N=4 and M=8
**********************************************************************************************/

.macro LOAD4x8_1
   LOAD4x8 1
.endm

.macro LOAD4x8_0
   LOAD4x8 0
.endm

.macro KERNEL4x8_L1_L4  Index,IsLast
  KERNEL4x8_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL4x8_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL4x8_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x8_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x8_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL4x8_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x8_I1_L2_3  OffsetA,OffsetB, Index,IsLast
   KERNEL4x8_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro KERNEL4x8_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x8_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x8_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x8_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro END4x8_NORMAL
  END4x8 0, AO, BO, 32,16 
.endm

.macro Zero4X8
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33
 
    xxlxor      vs36,   vs36,   vs36
    xxlxor      vs37,   vs37,   vs37
 
    xxlxor      vs40,   vs40,   vs40
    xxlxor      vs41,   vs41,   vs41
 
    xxlxor      vs44,   vs44,   vs44
    xxlxor      vs45,   vs45,   vs45
    
.endm

.macro LOAD4x8  Zero

    lxv vs24,   0(BO) 
    lxv vs0,     0(AO)
    lxv vs1,    16(AO)

    xxperm      vs26,   vs24,       permute_mask    
    xxpermdi    vs25,   vs24,   vs24,2      

    xxpermdi    vs27,   vs26,   vs26,2      

.if \Zero==1 
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs36,   vs36,   vs36
    xxlxor      vs37,   vs37,   vs37
    xxlxor      vs40,   vs40,   vs40
    xxlxor      vs41,   vs41,   vs41 
    xxlxor      vs44,   vs44,   vs44
    xxlxor      vs45,   vs45,   vs45 
 
.endif
.endm


.macro END4x8 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25

    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27
 

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27
 

.endif
.endm  

.macro KERNEL4x8_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\BREG) 

    lxv vs4,    DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs5,    DISP32(\Index,16+\OffsetA)(\AREG)

    xxperm      vs10,   vs8,        permute_mask    
    xxpermdi    vs9,    vs8,    vs8,2     

    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xxpermdi    vs11,   vs10,   vs10,2   

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

 

    lxv vs24,   DISP16(\Index,16+\OffsetB)(\BREG) 

    lxv vs0,    DISP32(\Index,32+\OffsetA)(\AREG)
    lxv vs1,    DISP32(\Index,32+16+\OffsetA)(\AREG)

    xxperm      vs26,   vs24,       permute_mask   
    xxpermdi    vs25,   vs24,   vs24,2      

    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

    xxpermdi    vs27,   vs26,   vs26,2       

    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11

 

    lxv vs8,    DISP16(\Index,32+\OffsetB)(\BREG) 

    lxv vs4,    DISP32(\Index,64+0+\OffsetA)(\AREG)
    lxv vs5,    DISP32(\Index,64+16+\OffsetA)(\AREG)

    xxperm      vs10,   vs8,        permute_mask     
    xxpermdi    vs9,    vs8,    vs8,2     

    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

    xxpermdi    vs11,   vs10,   vs10,2   

    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27

 

.if \Complete==0
    lxv vs24,   DISP16(\Index,48+\OffsetB)(\BREG) 

    lxv vs0,    DISP32(\Index,96+\OffsetA)(\AREG)
    lxv vs1,    DISP32(\Index,96+16+\OffsetA)(\AREG) 

    xxperm      vs26,   vs24,   permute_mask     
    xxpermdi    vs25,   vs24,   vs24,2      

.endif 
.if \IsLast==1  
.if \Complete==1
  
    addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)
    addi        \AREG, \AREG, DISP32(\Index,32*3+\OffsetA)
.else
  
    addi        \BREG, \BREG,  DISP16(\Index,64)
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif
.endif   
 
    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

.if \Complete==0        
    xxpermdi    vs27,   vs26,   vs26,2    
    
.endif
 
    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11

 

.endm

.macro KERNEL4x8 First

  LOAD4x8 0
  END4x8 \First, AO, BO, 32,16  
.endm

.macro KERNEL4x8_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete
    
    lxv vs8,     DISP8(\Index, 0+\OffsetB)(\BREG) 
    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs5,    DISP16(\Index,16+\OffsetA)(\AREG)

    xxperm      vs10,   vs8,        permute_mask  
    xxpermdi    vs9,    vs8,    vs8,2     
.if \First==1
    xvmulsp     vs32, vs0,vs24
    xvmulsp     vs33, vs1,vs24

    xvmulsp     vs36, vs0,vs25
    xvmulsp     vs37, vs1,vs25

.else
    xvmaddasp       vs32, vs0,vs24
    xvmaddasp       vs33, vs1,vs24

    xvmaddasp       vs36, vs0,vs25
    xvmaddasp       vs37, vs1,vs25

.endif

    xxpermdi    vs11,   vs10,   vs10,2    
 
.if \First==1  
    xvmulsp     vs40, vs0,vs26
    xvmulsp     vs41, vs1,vs26

    xvmulsp     vs44, vs0,vs27
    xvmulsp     vs45, vs1,vs27
 

.else 
    xvmaddasp       vs40, vs0,vs26
    xvmaddasp       vs41, vs1,vs26

    xvmaddasp       vs44, vs0,vs27
    xvmaddasp       vs45, vs1,vs27
 

.endif
.if \Complete==0
    lxv vs24,    DISP8(\Index,16+\OffsetB)(\BREG) 

    lxv vs0,    DISP16(\Index,32+\OffsetA)(\AREG)
    lxv vs1,    DISP16(\Index,32+16+\OffsetA)(\AREG)

    xxperm      vs26,   vs24,   permute_mask   
    xxpermdi    vs25,   vs24,   vs24,2    
.endif    
.if \IsLast==1  
.if \Complete==1
    addi        \BREG, \BREG,   DISP8(\Index,16+\OffsetB) 
    addi        \AREG, \AREG,  DISP16(\Index,32+\OffsetA)

.else
    addi        \BREG, \BREG,   DISP8(\Index,32)
    addi        \AREG, \AREG,  DISP16(\Index,64) 
.endif
.endif

.if \First==1
    xvmulsp     vs32, vs4,vs8
    xvmulsp     vs33, vs5,vs8

    xvmulsp     vs36, vs4,vs9
    xvmulsp     vs37, vs5,vs9

.else
    xvmaddasp       vs32, vs4,vs8
    xvmaddasp       vs33, vs5,vs8

    xvmaddasp       vs36, vs4,vs9
    xvmaddasp       vs37, vs5,vs9

.endif 
 
.if \Complete==0        
    xxpermdi    vs27,   vs26,   vs26,2   
 
.endif
.if \First==1  
    xvmulsp     vs40, vs4,vs10
    xvmulsp     vs41, vs5,vs10

    xvmulsp     vs44, vs4,vs11
    xvmulsp     vs45, vs5,vs11
 
.else 
    xvmaddasp       vs40, vs4,vs10
    xvmaddasp       vs41, vs5,vs10

    xvmaddasp       vs44, vs4,vs11
    xvmaddasp       vs45, vs5,vs11 

.endif

.endm


.macro SAVE4x8 
 
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 

  add     T2, CO, T10  
  add     T3, T1, T10  

 

#ifndef TRMMKERNEL    
    lxv        vs34, 0(CO)
    lxv        vs35, 16(CO)      
    lxv        vs38, 0(T1)
    lxv        vs39, 16(T1)  
    lxv        vs42, 0(T2)
    lxv        vs43, 16(T2)     
    lxv        vs46, 0(T3)
    lxv        vs47, 16(T3)  

 
#endif  

    xxmrglw     vs8,    vs32,   vs44
    xxmrglw     vs10,   vs36,   vs40  

    xxmrghw     vs1,    vs32,   vs44
    xxmrghw     vs0,    vs36,   vs40

    xxmrglw     vs12,   vs33,   vs45
    xxmrglw     vs14,   vs37,   vs41  

    xxmrghw     vs2,    vs37,   vs41
    xxmrghw     vs3,    vs33,   vs45

    xxlor      vs9, vs8,    vs8
    xxlor      vs11,    vs10,   vs10 
 
    xxlor      vs13,    vs12,   vs12
    xxlor      vs15,    vs14,   vs14

    xxperm      vs8,    vs0,    save_permute_1
    xxperm      vs10,   vs1,    save_permute_1
    xxperm      vs9,    vs0,    save_permute_2  
    xxperm      vs11,   vs1,    save_permute_2      

    xxperm     vs12,    vs2,    save_permute_1
    xxperm     vs14,    vs3,    save_permute_1
      
    xxperm     vs13,    vs2,    save_permute_2   
    xxperm     vs15,    vs3,    save_permute_2      


    /* multiply add normal way */
 
#ifdef TRMMKERNEL
    xvmulsp     vs34,   vs8,    alpha_r 
    xvmulsp     vs35,   vs12,   alpha_r 
    xvmulsp     vs38,   vs9,    alpha_r 
    xvmulsp     vs39,   vs13,   alpha_r 
    xvmulsp     vs42,   vs10,   alpha_r 
    xvmulsp     vs43,   vs14,   alpha_r 
    xvmulsp     vs46,   vs11,   alpha_r 
    xvmulsp     vs47,   vs15,   alpha_r                    
#else 
    xvmaddasp   vs34,   vs8,    alpha_r 
    xvmaddasp   vs35,   vs12,   alpha_r 
    xvmaddasp   vs38,   vs9,    alpha_r 
    xvmaddasp   vs39,   vs13,   alpha_r  
    xvmaddasp   vs42,   vs10,   alpha_r 
    xvmaddasp   vs43,   vs14,   alpha_r   
    xvmaddasp   vs46,   vs11,   alpha_r 
    xvmaddasp   vs47,   vs15,   alpha_r                     
#endif     
 
    
    stxv        vs34, 0(CO)
    stxv        vs35, 16(CO)  
    stxv        vs38, 0(T1)
    stxv        vs39, 16(T1)  
    stxv        vs42, 0(T2)
    stxv        vs43, 16(T2)     
    stxv        vs46, 0(T3)
    stxv        vs47, 16(T3)  
  

    addi CO,CO,32

.endm


/**********************************************************************************************
* Macros for N=4 and M=4
**********************************************************************************************/

.macro LOAD4x4_1
   LOAD4x4 1
.endm

.macro LOAD4x4_0
   LOAD4x4 0
.endm

.macro KERNEL4x4_L1_L4  Index,IsLast
  KERNEL4x4_L1_L4_I AO,BO, 0,0, \Index,\IsLast,0
.endm

.macro KERNEL4x4_I1_L4  OffsetA,OffsetB, Index,IsLast
  KERNEL4x4_L1_L4_I  AO,BO, \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x4_I1_L4_2  OffsetA,OffsetB, Index,IsLast
  KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x4_I1_L4_3  OffsetA,OffsetB, Index,IsLast
  KERNEL4x4_L1_L4_I  AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm
.macro KERNEL4x4_I1_L2_3  OffsetA,OffsetB, Index,IsLast
   KERNEL4x4_L1_L2_I AO,BO,0,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro KERNEL4x4_I2_L4_2  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x4_L1_L4_I  \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,0
.endm

.macro KERNEL4x4_I2_L4_3  AREG,BREG,OffsetA,OffsetB, Index,IsLast
  KERNEL4x4_L1_L4_I \AREG,\BREG,  \OffsetA,\OffsetB,\Index,\IsLast,1
.endm

.macro Zero4X4
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs34,   vs34,   vs34
    xxlxor      vs35,   vs35,   vs35
 
.endm

.macro LOAD4x4  Zero

    lxv vs0,     0(AO)
    lxv vs24,   0(BO) 



    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2        

.if \Zero==1 
    xxlxor      vs32,   vs32,   vs32
    xxlxor      vs33,   vs33,   vs33 
    xxlxor      vs34,   vs34,   vs34
    xxlxor      vs35,   vs35,   vs35
 
.endif
.endm

.macro END4x4_NORMAL
  END4x4 0, AO, BO, 16,16 
.endm

.macro END4x4 First, AREG, BREG, OffsetA, OffsetB

.if \OffsetB != 0 
    addi        \BREG, \BREG, \OffsetB 
.endif
.if \OffsetA != 0 
    addi        \AREG, \AREG, \OffsetA 
.endif  

.if \First==1
    xvmulsp      vs32,   vs24,   vs0
    xvmulsp      vs33,   vs24,   vs1 
    xvmulsp      vs34,   vs24,   vs2
    xvmulsp      vs35,   vs24,   vs3  
.else
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3
 

.endif
.endm  

.macro KERNEL4x4_L1_L4_I  AREG,BREG,   OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs4,    DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) 

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3
 

    lxv vs0,    DISP16(\Index, 16+\OffsetA)(\AREG)
    lxv vs24,   DISP16(\Index, 16+\OffsetB)(\BREG)  

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   

    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7
 
 

    lxv vs4,    DISP16(\Index, 32+\OffsetA)(\AREG)
    lxv vs26,   DISP16(\Index, 32+\OffsetB)(\BREG) 

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3
 

.if \Complete==0 

    lxv vs0,    DISP16(\Index, 48+\OffsetA)(\AREG)
    lxv vs24,   DISP16(\Index, 48+\OffsetB)(\BREG) 

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   
.endif
    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7
 

 
 
.if \IsLast==1  
.if \Complete==1
    addi        \AREG, \AREG, DISP16(\Index,16*3+\OffsetA)  
    addi        \BREG, \BREG,  DISP16(\Index,16*3+\OffsetB)

.else
    addi        \AREG, \AREG, DISP16(\Index,64)  
    addi        \BREG, \BREG,  DISP16(\Index,64)

.endif
.endif   
 
 
.endm

.macro KERNEL4x4 First
    LOAD4x4 0
    END4x4 \First, AO, BO, 16,16  
.endm

.macro KERNEL4x4_L1_L2_I  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast ,Complete

    lxv vs4,    DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) 

    xxperm      vs6,   vs4,       permute_mask  
    xxpermdi    vs5,   vs4,   vs4,2      
    xxpermdi    vs7,   vs6,   vs6,2 
.if \First==1
    xvmulsp      vs32,   vs24,   vs0
    xvmulsp      vs33,   vs24,   vs1 
    xvmulsp      vs34,   vs24,   vs2
    xvmulsp      vs35,   vs24,   vs3
 
.else 
    xvmaddasp      vs32,   vs24,   vs0
    xvmaddasp      vs33,   vs24,   vs1 
    xvmaddasp      vs34,   vs24,   vs2
    xvmaddasp      vs35,   vs24,   vs3
 
.endif

.if \Complete==0 

    lxv vs0,    DISP8(\Index, 16+\OffsetA)(\AREG)
    lxv vs24,   DISP8(\Index, 16+\OffsetB)(\BREG) 

    xxperm      vs2,   vs0,       permute_mask  
    xxpermdi    vs1,   vs0,   vs0,2      
    xxpermdi    vs3,   vs2,   vs2,2   
.endif

.if \First==1
    xvmulsp      vs32,   vs26,   vs4
    xvmulsp      vs33,   vs26,   vs5 
    xvmulsp      vs34,   vs26,   vs6
    xvmulsp      vs35,   vs26,   vs7 


.else
    xvmaddasp      vs32,   vs26,   vs4
    xvmaddasp      vs33,   vs26,   vs5 
    xvmaddasp      vs34,   vs26,   vs6
    xvmaddasp      vs35,   vs26,   vs7
 
.endif
 
 
.if \IsLast==1  
.if \Complete==1
    addi        \AREG, \AREG, DISP8(\Index,16+\OffsetA)  
    addi        \BREG, \BREG,  DISP8(\Index,16+\OffsetB)

.else
    addi        \AREG, \AREG, DISP8(\Index,32)  
    addi        \BREG, \BREG,  DISP8(\Index,32)

.endif
.endif   
     
  
.endm


.macro SAVE4x4
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC 
#if !defined(TRMMKERNEL)  
  lxv        vs36, 0(CO)
  lxv        vs37, 0(T1)
#endif
  add     T2, CO, T10  
  add     T3, T1, T10 
#if !defined(TRMMKERNEL)   
  lxv        vs38, 0(T2)
  lxv        vs39, 0(T3)    
#endif   

  xxmrglw  vs0, vs35,vs32
  xxmrglw  vs1, vs34,vs33 
  xxmrglw  vs4, vs32,vs35
  xxmrglw  vs5, vs33,vs34 


  xxmrghw  vs2, vs35,vs32
  xxmrghw  vs3, vs34,vs33 
  xxmrghw  vs6, vs32,vs35
  xxmrghw  vs7, vs33,vs34  

  xxmrgld  vs24, vs1, vs0  
  xxmrghd  vs25,vs5,vs4 

  xxmrgld  vs26, vs2, vs3  
  xxmrghd  vs27,vs6,vs7

 #if defined(TRMMKERNEL)
  xvmulsp        vs36, vs24, alpha_r
  xvmulsp        vs37, vs25, alpha_r 
  xvmulsp        vs38, vs26, alpha_r
  xvmulsp        vs39, vs27, alpha_r 
#else
  xvmaddasp        vs36, vs24, alpha_r
  xvmaddasp        vs37, vs25, alpha_r 
  xvmaddasp        vs38, vs26, alpha_r
  xvmaddasp        vs39, vs27, alpha_r   
 #endif
  stxv        vs36, 0(CO)
  stxv        vs37, 0(T1) 
  stxv        vs38, 0(T2)
  stxv        vs39, 0(T3)   
 


  addi CO,CO,16
.endm


/**********************************************************************************************
* Macros for N=4 and M=2
**********************************************************************************************/

 
.macro KERNEL4x2_2   OffsetA,OffsetB, Index,IsLast
  KERNEL4x2_I_2 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast
.endm

 

.macro Zero4x2
    xxlxor      vs0,   vs0,   vs0 
    xxlxor      vs2,   vs2,   vs2 
       
.endm
 
.macro KERNEL4x2
  KERNEL4x2_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL4x2_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG)      
    xxspltw   vs8,  vs36, 0 
    xxspltw   vs9,  vs36, 1  
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8 
    xvmulsp      vs2,   vs26,   vs9 
     
.else 
    xvmaddasp      vs0,   vs26,   vs8  
    xvmaddasp      vs2,   vs26,   vs9 
 
 .endif
   
    addi        \AREG, \AREG, DISP2(\Index,8)  
    addi        \BREG, \BREG, DISP4(\Index,16)
 
.endm

.macro KERNEL4x2_I_2  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  

    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)
    lxv vs26,   DISP8(\Index, 0+\OffsetB)(\BREG) 
    lxv vs28,   DISP8(\Index,16+\OffsetB)(\BREG)       
    xxspltw   vs8,  vs4, 2  
    xxspltw   vs9,  vs4, 3 
    xxspltw   vs10, vs4, 0 
    xxspltw   vs11, vs4, 1
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8 
    xvmulsp      vs2,   vs26,   vs9  

    xvmulsp      vs0,   vs28,   vs10 
    xvmulsp      vs2,   vs28,   vs11     
.else 
    xvmaddasp      vs0,   vs26,   vs8 
    xvmaddasp      vs2,   vs26,   vs9 

    xvmaddasp      vs0,   vs28,   vs10 
    xvmaddasp      vs2,   vs28,   vs11   
 .endif

 
.if \IsLast==1   
    addi        \AREG, \AREG, DISP4(\Index,16)  
    addi        \BREG, \BREG, DISP8(\Index,32)
.endif 
  
.endm


.macro SAVE4x2
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC  
  add     T2, CO, T10  
  add     T3, T1, T10     
  /*convert alpha_r for multiply*/
  xscvspdp  vs4,alpha_r
/* v0 corresponds to vs32, do not forget*/
#if !defined(TRMMKERNEL)
  lxssp  v0,0(CO) 
  lxssp  v1,4(CO) 

  lxssp  v2,0(T1)
  lxssp  v3,4(T1)

  lxssp  v4,0(T2)
  lxssp  v5,4(T2)

  lxssp  v6,0(T3)
  lxssp  v7,4(T3)

   
#endif
  xscvspdp  vs5, vs2
  xxspltw   vs6, vs2, 1 
  xxspltw   vs7, vs2, 2 
  xxspltw   vs8, vs2, 3  
  xscvspdp  vs6,vs6
  xscvspdp  vs7,vs7
  xscvspdp  vs8,vs8

  xscvspdp  vs24, vs0
  xxspltw   vs25, vs0, 1 
  xxspltw   vs26, vs0, 2 
  xxspltw   vs27, vs0, 3  
  xscvspdp  vs25,vs25
  xscvspdp  vs26,vs26
  xscvspdp  vs27,vs27
 

#if defined(TRMMKERNEL)
  xsmuldp  vs32,vs8, vs4 
  xsmuldp  vs33,vs27, vs4 

  xsmuldp  vs34,vs7, vs4 
  xsmuldp  vs35,vs26, vs4 

  xsmuldp  vs36,vs6, vs4 
  xsmuldp  vs37,vs25, vs4  

  xsmuldp  vs38,vs5, vs4 
  xsmuldp  vs39,vs24, vs4  

      
#else
  xsmaddadp  vs32,vs8, vs4 
  xsmaddadp  vs33,vs27, vs4 

  xsmaddadp  vs34,vs7, vs4 
  xsmaddadp  vs35,vs26, vs4 

  xsmaddadp  vs36,vs6, vs4 
  xsmaddadp  vs37,vs25, vs4  

  xsmaddadp  vs38,vs5, vs4 
  xsmaddadp  vs39,vs24, vs4  

    
#endif  

  stxssp  v0,0(CO) 
  stxssp  v1,4(CO) 

  stxssp  v2,0(T1)
  stxssp  v3,4(T1)

  stxssp  v4,0(T2)
  stxssp  v5,4(T2)

  stxssp  v6,0(T3)
  stxssp  v7,4(T3)

 
 

  addi CO,CO,8
.endm


/**********************************************************************************************
* Macros for N=4 and M=1
**********************************************************************************************/
.macro KERNEL4x1_4   OffsetA,OffsetB, Index,IsLast
  KERNEL4x1_I_4 AO,BO, 0, \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro Zero4x1
    xxlxor      vs0,   vs0,   vs0 
.endm

.macro KERNEL4x1
  KERNEL4x1_1 AO,BO, 0 
.endm

.macro KERNEL4x1_2
  KERNEL4x1_2_1 AO,BO, 0 
.endm

.macro KERNEL4x1_1 AREG,BREG,First 
    lxvwsx vs8,  0, \AREG
    lxv vs26,   0(\BREG)       
.if \First==1
    xvmulsp      vs0,   vs26,   vs8  
.else 
    xvmaddasp      vs0,   vs26,   vs8  
 .endif
    addi        \AREG, \AREG,  4  
    addi        \BREG, \BREG,  16
.endm

.macro KERNEL4x1_2_1 AREG,BREG,First 
    lxsd v4,    0(\AREG)
    lxv vs26,   0(\BREG)      
    lxv vs28,  16(\BREG) 
    xxspltw   vs8,  vs36, 1 
    xxspltw   vs9,  vs36, 0  
.if \First==1
    xvmulsp      vs0,   vs26,   vs8 
    xvmulsp      vs0,   vs28,   vs9     
.else 
    xvmaddasp      vs0,   vs26,   vs8  
    xvmaddasp      vs0,   vs28,   vs9  
 .endif
    addi        \AREG, \AREG,  8 
    addi        \BREG, \BREG,  32
.endm

.macro KERNEL4x1_I_4  AREG,BREG,First,OffsetA,OffsetB, Index,IsLast  
    lxv vs4,    DISP4(\Index, 0+\OffsetA)(\AREG)
    xxspltw   vs8,  vs4, 3 
    xxspltw   vs9,  vs4, 2 
    xxspltw   vs10, vs4, 1 
    xxspltw   vs11, vs4, 0
    lxv vs26,   DISP16(\Index, 0+\OffsetB)(\BREG) 
    lxv vs28,   DISP16(\Index,16+\OffsetB)(\BREG)  
    lxv vs30,   DISP16(\Index,32+\OffsetB)(\BREG) 
    lxv vs32,   DISP16(\Index,48+\OffsetB)(\BREG)          
.if \First==1
    xvmulsp      vs0,   vs26,   vs8  
    xvmulsp      vs0,   vs28,   vs9      
    xvmulsp      vs0,   vs30,   vs10  
    xvmulsp      vs0,   vs32,   vs11     
.else 
    xvmaddasp      vs0,   vs26,   vs8  
    xvmaddasp      vs0,   vs28,   vs9     
    xvmaddasp      vs0,   vs30,   vs10  
    xvmaddasp      vs0,   vs32,   vs11  
 .endif
.if \IsLast==1   
    addi        \AREG, \AREG, DISP4(\Index,16)  
    addi        \BREG, \BREG, DISP16(\Index,64)
.endif 
.endm

.macro SAVE4x1
  slwi    T10, LDC ,   1 
  add     T1, CO, LDC  
  add     T2, CO, T10  
  add     T3, T1, T10     
  /*convert alpha_r for multiply*/
  xscvspdp  vs4,alpha_r
/* v0 corresponds to vs32, do not forget*/
#if !defined(TRMMKERNEL)
  lxssp  v0,0(CO)  
  lxssp  v2,0(T1) 
  lxssp  v4,0(T2) 
  lxssp  v6,0(T3)  
#endif
  xscvspdp  vs24, vs0
  xxspltw   vs25, vs0, 1 
  xxspltw   vs26, vs0, 2 
  xxspltw   vs27, vs0, 3  
  xscvspdp  vs25,vs25
  xscvspdp  vs26,vs26
  xscvspdp  vs27,vs27

#if defined(TRMMKERNEL)
  xsmuldp  vs32,vs27, vs4 
  xsmuldp  vs34,vs26, vs4 
  xsmuldp  vs36,vs25, vs4 
  xsmuldp  vs38,vs24, vs4  
#else
  xsmaddadp  vs32,vs27, vs4 
  xsmaddadp  vs34,vs26, vs4 
  xsmaddadp  vs36,vs25, vs4 
  xsmaddadp  vs38,vs24, vs4   
#endif  
  stxssp  v0,0(CO)  
  stxssp  v2,0(T1) 
  stxssp  v4,0(T2) 
  stxssp  v6,0(T3)  
  addi CO,CO,4
.endm

/****************************N=2 section*****************/

.macro KERNEL2x16_2   OffsetA,OffsetB, Index,IsLast
  KERNEL2x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 
.macro Zero2x16
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor      vs2,   vs2,   vs2
    xxlxor      vs3,   vs3,   vs3
    xxlxor      vs4,   vs4,   vs4
    xxlxor      vs5,   vs5,   vs5 
    xxlxor      vs6,   vs6,   vs6
    xxlxor      vs7,   vs7,   vs7      
.endm
 
.macro KERNEL2x16
  KERNEL2x16_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL2x16_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL2x16_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1 
    xxspltw   vs9,  vs36, 0 
    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   
    lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)
    lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8 
    xvmulsp      vs2,   vs28,   vs8
    xvmulsp      vs3,   vs29,   vs8 

    xvmulsp      vs4,   vs26,   vs9
    xvmulsp      vs5,   vs27,   vs9 
    xvmulsp      vs6,   vs28,   vs9
    xvmulsp      vs7,   vs29,   vs9     
     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 

    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9 
    xvmaddasp      vs6,   vs28,   vs9
    xvmaddasp      vs7,   vs29,   vs9
 
 .endif
   
    addi        \BREG, \BREG, DISP2(\Index,8)
    addi        \AREG, \AREG, DISP16(\Index,64)  
 
.endm




.macro KERNEL2x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

    lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)
    lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)
    lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  

    lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)
    lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)
    lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) 

    lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)
    lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)
    lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)
    lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  
        
    lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)
    lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)
    lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)
    lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 
    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0  

    xxspltw   vs12,  vs39, 3  
    xxspltw   vs13,  vs39, 2 
    xxspltw   vs14, vs39, 1 
    xxspltw   vs15, vs39, 0  

 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 

    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9 
    xvmaddasp      vs6,   vs28,   vs9
    xvmaddasp      vs7,   vs29,   vs9 

    xvmaddasp      vs0,   vs16,   vs10
    xvmaddasp      vs1,   vs17,   vs10 
    xvmaddasp      vs2,   vs18,   vs10
    xvmaddasp      vs3,   vs19,   vs10 

    xvmaddasp      vs4,   vs16,   vs11
    xvmaddasp      vs5,   vs17,   vs11 
    xvmaddasp      vs6,   vs18,   vs11
    xvmaddasp      vs7,   vs19,   vs11  

    xvmaddasp      vs0,   vs30,   vs12
    xvmaddasp      vs1,   vs31,   vs12 
    xvmaddasp      vs2,   vs32,   vs12
    xvmaddasp      vs3,   vs33,   vs12 

    xvmaddasp      vs4,   vs30,   vs13
    xvmaddasp      vs5,   vs31,   vs13 
    xvmaddasp      vs6,   vs32,   vs13
    xvmaddasp      vs7,   vs33,   vs13 

    xvmaddasp      vs0,   vs34,   vs14
    xvmaddasp      vs1,   vs35,   vs14 
    xvmaddasp      vs2,   vs36,   vs14
    xvmaddasp      vs3,   vs37,   vs14 

    xvmaddasp      vs4,   vs34,   vs15
    xvmaddasp      vs5,   vs35,   vs15 
    xvmaddasp      vs6,   vs36,   vs15
    xvmaddasp      vs7,   vs37,   vs15    
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP8(\Index,32)  
    addi        \AREG, \AREG, DISP64(\Index,256)
.endif 
  
.endm

.macro KERNEL2x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 3  
    xxspltw   vs9,  vs36, 2 
    xxspltw   vs10, vs36, 1 
    xxspltw   vs11, vs36, 0    
    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)
    lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)
    lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      
    lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)
    lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)
    lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) 
 
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 

    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9 
    xvmaddasp      vs6,   vs28,   vs9
    xvmaddasp      vs7,   vs29,   vs9 

    xvmaddasp      vs0,   vs16,   vs10
    xvmaddasp      vs1,   vs17,   vs10 
    xvmaddasp      vs2,   vs18,   vs10
    xvmaddasp      vs3,   vs19,   vs10 

    xvmaddasp      vs4,   vs16,   vs11
    xvmaddasp      vs5,   vs17,   vs11 
    xvmaddasp      vs6,   vs18,   vs11
    xvmaddasp      vs7,   vs19,   vs11   
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif 
  
.endm


.macro SAVE2x16

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)
    lxv        vs17, 16(CO) 
    lxv        vs18, 32(CO)  
    lxv        vs19, 48(CO)      
#endif
  add     T1, CO, LDC 
#ifndef TRMMKERNEL    
    lxv        vs26, 0(T1)
    lxv        vs27, 16(T1) 
    lxv        vs28, 32(T1)  
    lxv        vs29, 48(T1)      
#endif

#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r
  xvmulsp        vs17, vs1, alpha_r 
  xvmulsp        vs18, vs2, alpha_r
  xvmulsp        vs19, vs3, alpha_r   
  xvmulsp        vs26, vs4, alpha_r
  xvmulsp        vs27, vs5, alpha_r 
  xvmulsp        vs28, vs6, alpha_r
  xvmulsp        vs29, vs7, alpha_r
#else
  xvmaddasp        vs16, vs0, alpha_r
  xvmaddasp        vs17, vs1, alpha_r 
  xvmaddasp        vs18, vs2, alpha_r
  xvmaddasp        vs19, vs3, alpha_r   
  xvmaddasp        vs26, vs4, alpha_r
  xvmaddasp        vs27, vs5, alpha_r 
  xvmaddasp        vs28, vs6, alpha_r
  xvmaddasp        vs29, vs7, alpha_r
#endif
    stxv        vs16, 0(CO)
    stxv        vs17, 16(CO) 
    stxv        vs18, 32(CO)  
    stxv        vs19, 48(CO)      
    
    stxv        vs26, 0(T1)
    stxv        vs27, 16(T1) 
    stxv        vs28, 32(T1)  
    stxv        vs29, 48(T1) 
 
  addi CO,CO,64

.endm

/*       M=8 N=2 */

.macro KERNEL2x8_2   OffsetA,OffsetB, Index,IsLast
  KERNEL2x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 
.macro Zero2x8
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
 
    xxlxor      vs4,   vs4,   vs4
    xxlxor      vs5,   vs5,   vs5 
     
.endm
 
.macro KERNEL2x8
  KERNEL2x8_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL2x8_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL2x8_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1 
    xxspltw   vs9,  vs36, 0 
    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)          
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8  

    xvmulsp      vs4,   vs26,   vs9
    xvmulsp      vs5,   vs27,   vs9      
     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8   

    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9  
 
 .endif
   
    addi        \BREG, \BREG, DISP2(\Index,8)
    addi        \AREG, \AREG, DISP8(\Index,32)  
 
.endm




.macro KERNEL2x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) 

    lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) 

    lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)  
        
    lxv vs34,   DISP32(\Index, 96+ 0+\OffsetA)(\AREG)
    lxv vs35,   DISP32(\Index, 96+ 16+\OffsetA)(\AREG) 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 
    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0  

    xxspltw   vs12,  vs39, 3  
    xxspltw   vs13,  vs39, 2 
    xxspltw   vs14, vs39, 1 
    xxspltw   vs15, vs39, 0  

 
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9 
 

    xvmaddasp      vs0,   vs16,   vs10
    xvmaddasp      vs1,   vs17,   vs10 
    xvmaddasp      vs4,   vs16,   vs11
    xvmaddasp      vs5,   vs17,   vs11 
 

    xvmaddasp      vs0,   vs30,   vs12
    xvmaddasp      vs1,   vs31,   vs12 
    xvmaddasp      vs4,   vs30,   vs13
    xvmaddasp      vs5,   vs31,   vs13 

    xvmaddasp      vs0,   vs34,   vs14
    xvmaddasp      vs1,   vs35,   vs14 
    xvmaddasp      vs4,   vs34,   vs15
    xvmaddasp      vs5,   vs35,   vs15 
   
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP8(\Index,32)  
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif 
  
.endm

.macro KERNEL2x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 3  
    xxspltw   vs9,  vs36, 2 
    xxspltw   vs10, vs36, 1 
    xxspltw   vs11, vs36, 0    
    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)
    lxv vs16,   DISP16(\Index,32+\OffsetA)(\AREG)
    lxv vs17,   DISP16(\Index,48+\OffsetA)(\AREG)      
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  

    xvmaddasp      vs4,   vs26,   vs9
    xvmaddasp      vs5,   vs27,   vs9  

    xvmaddasp      vs0,   vs16,   vs10
    xvmaddasp      vs1,   vs17,   vs10   

    xvmaddasp      vs4,   vs16,   vs11
    xvmaddasp      vs5,   vs17,   vs11     
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP16(\Index,64)
.endif 
  
.endm


.macro SAVE2x8

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)
    lxv        vs17, 16(CO)     
#endif
  add     T1, CO, LDC 
#ifndef TRMMKERNEL    
    lxv        vs26, 0(T1)
    lxv        vs27, 16(T1) 
    
#endif

#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r
  xvmulsp        vs17, vs1, alpha_r  
  xvmulsp        vs26, vs4, alpha_r
  xvmulsp        vs27, vs5, alpha_r 
#else
  xvmaddasp        vs16, vs0, alpha_r
  xvmaddasp        vs17, vs1, alpha_r  
  xvmaddasp        vs26, vs4, alpha_r
  xvmaddasp        vs27, vs5, alpha_r 
#endif

    stxv        vs16, 0(CO)
    stxv        vs17, 16(CO) 
     
    
    stxv        vs26, 0(T1)
    stxv        vs27, 16(T1) 

  addi CO,CO,32

.endm


/*M=4*/


.macro KERNEL2x4_2   OffsetA,OffsetB, Index,IsLast
  KERNEL2x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 /* we will aggregate on save vs0 +vs4 vs11+vs5 */
.macro Zero2x4
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
 
    xxlxor      vs4,   vs4,   vs4
    xxlxor      vs5,   vs5,   vs5 
    
.endm
 
.macro KERNEL2x4
  KERNEL2x4_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL2x4_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL2x4_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1 
    xxspltw   vs9,  vs36, 0 
    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)        
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8 
    xvmulsp      vs1,   vs26,   vs9     
     
.else 
    xvmaddasp      vs0,   vs26,   vs8 
    xvmaddasp      vs1,   vs26,   vs9 
 .endif
   
    addi        \BREG, \BREG, DISP2(\Index,8)
    addi        \AREG, \AREG, DISP4(\Index,16)  
 
.endm




.macro KERNEL2x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs39,    DISP8(\Index, 16+\OffsetB)(\BREG) 

    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs16,   DISP16(\Index,16+\OffsetA)(\AREG) 

    lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)
    lxv vs34,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) 
 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 
    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0  

    xxspltw   vs12,  vs39, 3  
    xxspltw   vs13,  vs39, 2 
    xxspltw   vs14, vs39, 1 
    xxspltw   vs15, vs39, 0  

 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs26,   vs9 
    xvmaddasp      vs4,   vs16,   vs10
    xvmaddasp      vs5,   vs16,   vs11 
 

    xvmaddasp      vs0,   vs30,   vs12
    xvmaddasp      vs1,   vs30,   vs13 
    xvmaddasp      vs4,   vs34,   vs14
    xvmaddasp      vs5,   vs34,   vs15 
 
   
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP8(\Index,32)  
    addi        \AREG, \AREG, DISP16(\Index,64)
.endif 
  
.endm

.macro KERNEL2x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs36,    DISP4(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 3  
    xxspltw   vs9,  vs36, 2 
    xxspltw   vs10, vs36, 1 
    xxspltw   vs11, vs36, 0    
    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs16,   DISP8(\Index, 16+\OffsetA)(\AREG)      
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs26,   vs9 
    xvmaddasp      vs4,   vs16,   vs10
    xvmaddasp      vs5,   vs16,   vs11     
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP8(\Index,32)
.endif 
  
.endm


.macro SAVE2x4

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)     
#endif
  add     T1, CO, LDC 
#ifndef TRMMKERNEL    
    lxv        vs26, 0(T1) 
    
#endif
    /*aggregate vectors*/
  xvaddsp         vs0,vs0,vs4
  xvaddsp         vs1,vs1,vs5 
#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r 
  xvmulsp        vs26, vs1, alpha_r 
#else
  xvmaddasp        vs16, vs0, alpha_r 
  xvmaddasp        vs26, vs1, alpha_r 
#endif

  stxv        vs16, 0(CO) 
  stxv        vs26, 0(T1)  

  addi CO,CO,16

.endm


/* M=2 N=2 we will have inner pemrute action before permute was revrsing 3,2,1,0 not iw 2ill inner reverse 1,0,3,2  */
.macro SWITCH_PERMUTE_INNER
    xxpermdi	permute_mask,	permute_mask,	permute_mask,2
.endm

.macro Zero2x2
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    SWITCH_PERMUTE_INNER
.endm
 
.macro KERNEL2x2
  KERNEL2x2_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL2x2_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL2x2_2 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL2x2_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxsd v4,   DISP2(\Index, 0+\OffsetB)(\BREG)
    xxperm   vs9,  vs36, permute_mask 
    lxsd v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        
 
 
.if \First==1
    xvmulsp      vs0,   vs37,   vs36 
    xvmulsp      vs1,   vs37,   vs9     
     
.else 
    xvmaddasp      vs0,   vs37,   vs36 
    xvmaddasp      vs1,   vs37,   vs9 
 .endif
   
    addi        \BREG, \BREG, DISP2(\Index,8)
    addi        \AREG, \AREG, DISP2(\Index,8)  
 
.endm




.macro KERNEL2x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs10,    DISP8(\Index, 16+\OffsetB)(\BREG) 

    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs16,   DISP8(\Index,16+\OffsetA)(\AREG) 

 
    xxperm   vs9,  vs8, permute_mask   
    xxperm   vs11, vs10, permute_mask  


 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs26,   vs9 
    xvmaddasp      vs0,   vs16,   vs10
    xvmaddasp      vs1,   vs16,   vs11 
 
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP8(\Index,32)  
    addi        \AREG, \AREG, DISP8(\Index,32)
.endif 
  
.endm

.macro KERNEL2x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP4(\Index, 0+\OffsetB)(\BREG)  
    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG) 

 
    xxperm   vs9,  vs8, permute_mask    

 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs26,   vs9  
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP4(\Index,16)
.endif 
.endm


.macro SAVE2x2

#ifndef TRMMKERNEL    
    lxsd v4   , 0(CO)     
#endif
  add     T1, CO, LDC 
#ifndef TRMMKERNEL    
    lxsd v5   , 0(T1) 
    
#endif
    /*aggregate vectors*/
  xxpermdi         vs4,vs0,vs0,2
  xxpermdi         vs5,vs1,vs1,2  
  xvaddsp          vs0,vs0,vs4
  xvaddsp         vs1,vs1,vs5 
  /*   */
  /* lets correct the order to 00 10 and 10 ,11 from {00,11} {01,10}  */
  xxperm    vs1,vs1, permute_mask


  xxmrghw   vs2 ,vs1,vs0
  xxpermdi         vs2,vs2,vs2,2  
  xxmrghw   vs3 ,vs0,vs1  
#if defined(TRMMKERNEL)
  xvmulsp        vs36, vs2, alpha_r 
  xvmulsp        vs37, vs3, alpha_r 
#else
  xvmaddasp        vs36, vs2, alpha_r 
  xvmaddasp        vs37, vs3, alpha_r 
#endif
  /**** store last two words*/


  stxsd       v4, 0(CO) 
  stxsd        v5, 0(T1)  

  addi CO,CO,8

.endm

/*--------------------------- M=1 N=2 */
.macro Zero2x1
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor    vs2,vs2,vs2 
    xxlxor    vs3,vs3,vs3     
.endm
 
.macro KERNEL2x1
  KERNEL2x1_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL2x1_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL2x1_2 OffsetA,OffsetB, Index,IsLast  
  KERNEL2x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 /*
   we will calculate 1 alone then will add it to batched ones
 */
.macro KERNEL2x1_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v3,   DISP2(\Index, 0+\OffsetB)(\BREG)
    lxssp v4,   DISP2(\Index, 4+\OffsetB)(\BREG) 
    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\AREG)        
 
 
.if \First==1
    xvmulsp      vs2,   vs37,   vs35 
    xvmulsp      vs3,   vs37,   vs36     
     
.else 
    xsmaddadp     vs2,   vs37,   vs35
    xsmaddadp      vs3,   vs37,   vs36
 .endif
   
    addi        \BREG, \BREG, DISP2(\Index,8)
    addi        \AREG, \AREG, DISP1(\Index,4)  
 
.endm




.macro KERNEL2x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\BREG)
    lxv vs10,   DISP8(\Index, 16+\OffsetB)(\BREG) 

    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)
   
    xxmrglw   vs5, vs26,vs26
    xxmrghw   vs6, vs26,vs26 
 
    xvmaddasp      vs0,   vs8,   vs5
    xvmaddasp      vs1,   vs10,   vs6 
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP8(\Index,32)  
    addi        \AREG, \AREG, DISP4(\Index,16)
.endif 
  
.endm

.macro KERNEL2x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxssp v3,   DISP4(\Index, 0+\OffsetB)(\BREG)
    lxssp v4,   DISP4(\Index, 4+\OffsetB)(\BREG) 
    lxssp v7,   DISP4(\Index, 8+\OffsetB)(\BREG)
    lxssp v8,   DISP4(\Index, 12+\OffsetB)(\BREG)    
    lxssp v5,   DISP2(\Index, 0+\OffsetA)(\AREG)        
    lxssp v6,   DISP2(\Index, 4+\OffsetA)(\AREG)  
 
 
    xsmaddadp      vs2,   vs37,   vs35
    xsmaddadp      vs3,   vs37,   vs36

    xsmaddadp      vs2,   vs38,   vs39 
    xsmaddadp      vs3,   vs38,   vs40      
 
   
    addi        \BREG, \BREG, DISP4(\Index,16)
    addi        \AREG, \AREG, DISP2(\Index,8) 
.endm


.macro SAVE2x1

#ifndef TRMMKERNEL    
    lxssp v4   , 0(CO)     
#endif
  add     T1, CO, LDC 
#ifndef TRMMKERNEL    
    lxssp v5   , 0(T1) 
    
#endif

  /*convert alpha_r for multiply*/
  xscvspdp  vs16,alpha_r

 /*aggregate vectors 2x2_4   */ 
      xxpermdi         vs4,vs0,vs0,2
      xxpermdi         vs5,vs1,vs1,2  
      xvaddsp          vs0,vs0,vs4
      xvaddsp         vs1,vs1,vs5 
      xvaddsp         vs0,vs0,vs1 
/*aggregate vectors 2x1_2 and 2x1_1 into 2x2_4*/
  xscvspdp  vs5, vs0
  xxspltw   vs6, vs0, 1  
  xscvspdp  vs6,vs6 
  xsadddp  vs2,vs2,vs6
  xsadddp  vs3,vs3,vs5  

  /**** store last two words*/
#if defined(TRMMKERNEL) 
  xsmuldp  vs36,vs2, vs16 
  xsmuldp  vs37,vs3, vs16  
 
#else
  xsmaddadp  vs36,vs2, vs16 
  xsmaddadp  vs37,vs3, vs16 
#endif  

  stxssp       v4, 0(CO) 
  stxssp        v5, 0(T1)  

  addi CO,CO,4

.endm



/****************************N=1 section*****************/

.macro KERNEL1x16_2   OffsetA,OffsetB, Index,IsLast
  KERNEL1x16_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 
.macro Zero1x16
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor      vs2,   vs2,   vs2
    xxlxor      vs3,   vs3,   vs3       
.endm
 
.macro KERNEL1x16
  KERNEL1x16_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL1x16_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x16_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL1x16_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)
    xscvdpspn   vs36,vs36
    xxspltw     vs8,  vs36, 0
    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)   
    lxv vs28,   DISP16(\Index, 32+\OffsetA)(\AREG)
    lxv vs29,   DISP16(\Index,48+\OffsetA)(\AREG)        
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8 
    xvmulsp      vs2,   vs28,   vs8
    xvmulsp      vs3,   vs29,   vs8 
  
     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 
 
 .endif
   
    addi        \BREG, \BREG, DISP1(\Index,4)
    addi        \AREG, \AREG, DISP16(\Index,64)  
 
.endm




.macro KERNEL1x16_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

    lxv vs26,   DISP64(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP64(\Index,16+\OffsetA)(\AREG)
    lxv vs28,   DISP64(\Index,32+\OffsetA)(\AREG)
    lxv vs29,   DISP64(\Index,48+\OffsetA)(\AREG)  

    lxv vs16,   DISP64(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP64(\Index,64+ 16+\OffsetA)(\AREG)
    lxv vs18,   DISP64(\Index,64+ 32+\OffsetA)(\AREG)
    lxv vs19,   DISP64(\Index,64+ 48+\OffsetA)(\AREG) 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 

    lxv vs30,   DISP64(\Index,128+ 0+\OffsetA)(\AREG)
    lxv vs31,   DISP64(\Index,128+ 16+\OffsetA)(\AREG)
    lxv vs32,   DISP64(\Index,128+ 32+\OffsetA)(\AREG)
    lxv vs33,   DISP64(\Index,128+ 48+\OffsetA)(\AREG)  
        
    lxv vs34,   DISP64(\Index,128+ 64+ 0+\OffsetA)(\AREG)
    lxv vs35,   DISP64(\Index,128+ 64+ 16+\OffsetA)(\AREG)
    lxv vs36,   DISP64(\Index,128+ 64+ 32+\OffsetA)(\AREG)
    lxv vs37,   DISP64(\Index,128+ 64+ 48+\OffsetA)(\AREG) 

    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0    

 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 
 

    xvmaddasp      vs0,   vs16,   vs9
    xvmaddasp      vs1,   vs17,   vs9 
    xvmaddasp      vs2,   vs18,   vs9
    xvmaddasp      vs3,   vs19,   vs9 
 

    xvmaddasp      vs0,   vs30,   vs10
    xvmaddasp      vs1,   vs31,   vs10 
    xvmaddasp      vs2,   vs32,   vs10
    xvmaddasp      vs3,   vs33,   vs10 
 

    xvmaddasp      vs0,   vs34,   vs11
    xvmaddasp      vs1,   vs35,   vs11 
    xvmaddasp      vs2,   vs36,   vs11
    xvmaddasp      vs3,   vs37,   vs11 

 
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP64(\Index,256)
.endif 
  
.endm

.macro KERNEL1x16_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1  
    xxspltw   vs9,  vs36, 0      
    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG)
    lxv vs28,   DISP32(\Index,32+\OffsetA)(\AREG)
    lxv vs29,   DISP32(\Index,48+\OffsetA)(\AREG)      
    lxv vs16,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)
    lxv vs18,   DISP32(\Index,64+ 32+\OffsetA)(\AREG)
    lxv vs19,   DISP32(\Index,64+ 48+\OffsetA)(\AREG) 
 
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8 
    xvmaddasp      vs2,   vs28,   vs8
    xvmaddasp      vs3,   vs29,   vs8 
 

    xvmaddasp      vs0,   vs16,   vs9
    xvmaddasp      vs1,   vs17,   vs9 
    xvmaddasp      vs2,   vs18,   vs9
    xvmaddasp      vs3,   vs19,   vs9 
  
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP2(\Index,8)  
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif 
  
.endm


.macro SAVE1x16

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)
    lxv        vs17, 16(CO) 
    lxv        vs18, 32(CO)  
    lxv        vs19, 48(CO)      
#endif
 

#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r
  xvmulsp        vs17, vs1, alpha_r 
  xvmulsp        vs18, vs2, alpha_r
  xvmulsp        vs19, vs3, alpha_r   
#else
  xvmaddasp        vs16, vs0, alpha_r
  xvmaddasp        vs17, vs1, alpha_r 
  xvmaddasp        vs18, vs2, alpha_r
  xvmaddasp        vs19, vs3, alpha_r   
#endif
    stxv        vs16, 0(CO)
    stxv        vs17, 16(CO) 
    stxv        vs18, 32(CO)  
    stxv        vs19, 48(CO)      
    
  addi CO,CO,64

.endm

/*       M=8 N=1 */

.macro KERNEL1x8_2   OffsetA,OffsetB, Index,IsLast
  KERNEL1x8_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 
.macro Zero1x8
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1  
    xxlxor      vs2,   vs2,   vs2
    xxlxor      vs3,   vs3,   vs3          
.endm
 
.macro KERNEL1x8
  KERNEL1x8_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL1x8_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x8_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL1x8_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)
    xscvdpspn   vs36,vs36
    xxspltw     vs8,  vs36, 0
    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)         
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8
    xvmulsp      vs1,   vs27,   vs8  
  
     
.else 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
 
 .endif
   
    addi        \BREG, \BREG, DISP1(\Index,4)
    addi        \AREG, \AREG, DISP8(\Index,32)  
 
.endm




.macro KERNEL1x8_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

    lxv vs26,   DISP32(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP32(\Index,16+\OffsetA)(\AREG) 

    lxv vs16,   DISP32(\Index,32+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP32(\Index,32+ 16+\OffsetA)(\AREG) 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 

    lxv vs30,   DISP32(\Index,64+ 0+\OffsetA)(\AREG)
    lxv vs31,   DISP32(\Index,64+ 16+\OffsetA)(\AREG)   
        
    lxv vs34,   DISP32(\Index,64+ 32+ 0+\OffsetA)(\AREG)
    lxv vs35,   DISP32(\Index,64+ 32+ 16+\OffsetA)(\AREG)  

    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0    

 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
 

    xvmaddasp      vs2,   vs16,   vs9
    xvmaddasp      vs3,   vs17,   vs9  
 

    xvmaddasp      vs0,   vs30,   vs10
    xvmaddasp      vs1,   vs31,   vs10  
 

    xvmaddasp      vs2,   vs34,   vs11
    xvmaddasp      vs3,   vs35,   vs11  

 
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP32(\Index,128)
.endif 
  
.endm

.macro KERNEL1x8_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1  
    xxspltw   vs9,  vs36, 0      
    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG)     
    lxv vs16,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)
    lxv vs17,   DISP16(\Index,32+ 16+\OffsetA)(\AREG) 
 
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs8  
 

    xvmaddasp      vs2,   vs16,   vs9
    xvmaddasp      vs3,   vs17,   vs9   
  
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP2(\Index,8)  
    addi        \AREG, \AREG, DISP16(\Index,64)
.endif 
  
.endm


.macro SAVE1x8

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)
    lxv        vs17, 16(CO)       
#endif
   /* aggregate vs0 vs2 and vs1 vs3*/
  xvaddsp vs0,vs0,vs2
  xvaddsp  vs1,vs1,vs3
#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r
  xvmulsp        vs17, vs1, alpha_r     
#else
  xvmaddasp        vs16, vs0, alpha_r
  xvmaddasp        vs17, vs1, alpha_r  
#endif
    stxv        vs16, 0(CO)
    stxv        vs17, 16(CO)      
    
  addi CO,CO,32

.endm
/*M=4*/

.macro KERNEL1x4_2   OffsetA,OffsetB, Index,IsLast
  KERNEL1x4_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

 
.macro Zero1x4
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1  
    xxlxor      vs2,   vs2,   vs2
    xxlxor      vs3,   vs3,   vs3          
.endm
 
.macro KERNEL1x4
  KERNEL1x4_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL1x4_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x4_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 
.macro KERNEL1x4_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v4,   DISP1(\Index, 0+\OffsetB)(\BREG)
    xscvdpspn   vs36,vs36
    xxspltw     vs8,  vs36, 0
    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\AREG)         
 
 
.if \First==1
    xvmulsp      vs0,   vs26,   vs8 
.else 
    xvmaddasp      vs0,   vs26,   vs8 
 
 .endif
   
    addi        \BREG, \BREG, DISP1(\Index,4)
    addi        \AREG, \AREG, DISP4(\Index,16)  
 
.endm




.macro KERNEL1x4_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs38,    DISP4(\Index, 0+\OffsetB)(\BREG) 

    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP16(\Index,16+\OffsetA)(\AREG) 
 

    xxspltw   vs8,  vs38, 3  
    xxspltw   vs9,  vs38, 2 

    lxv vs30,   DISP16(\Index,32+ 0+\OffsetA)(\AREG)
    lxv vs31,   DISP16(\Index,32+ 16+\OffsetA)(\AREG)   
          

    xxspltw   vs10, vs38, 1 
    xxspltw   vs11, vs38, 0    

 
    xvmaddasp      vs0,   vs26,   vs8 

    xvmaddasp      vs1,   vs27,   vs9 

    xvmaddasp      vs2,   vs30,   vs10   
 

    xvmaddasp      vs3,   vs31,   vs11   

 
 
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP4(\Index,16)  
    addi        \AREG, \AREG, DISP16(\Index,64)
.endif 
  
.endm

.macro KERNEL1x4_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\BREG)
    xxspltw   vs8,  vs36, 1  
    xxspltw   vs9,  vs36, 0      
    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\AREG)
    lxv vs27,   DISP8(\Index,16+\OffsetA)(\AREG)      
 
 
    xvmaddasp      vs0,   vs26,   vs8
    xvmaddasp      vs1,   vs27,   vs9
  
 
.if \IsLast==1   
    addi        \BREG, \BREG, DISP2(\Index,8)  
    addi        \AREG, \AREG, DISP8(\Index,32)
.endif 
  
.endm


.macro SAVE1x4

#ifndef TRMMKERNEL    
    lxv        vs16, 0(CO)       
#endif
   /* aggregate */
  xvaddsp vs0,vs0,vs2
  xvaddsp  vs1,vs1,vs3
  xvaddsp  vs0,vs1,vs0
#if defined(TRMMKERNEL)
  xvmulsp        vs16, vs0, alpha_r     
#else
  xvmaddasp        vs16, vs0, alpha_r  
#endif
    stxv        vs16, 0(CO)      
    
  addi CO,CO,16

.endm

/* M=2 N=1*/ 
.macro Zero1x2
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor    vs2,vs2,vs2 
    xxlxor    vs3,vs3,vs3     
.endm
 
.macro KERNEL1x2
  KERNEL1x2_1 AO,BO, 0, 0,0,0
.endm
.macro KERNEL1x2_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x2_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL1x2_2 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x2_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 /*
   we will calculate 1 alone then will add it to batched ones
 */
.macro KERNEL1x2_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v3,   DISP2(\Index, 0+\OffsetB)(\AREG)
    lxssp v4,   DISP2(\Index, 4+\OffsetB)(\AREG) 
    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        
 
 
.if \First==1
    xvmuldp      vs2,   vs37,   vs35 
    xvmuldp      vs3,   vs37,   vs36     
     
.else 
    xsmaddadp     vs2,   vs37,   vs35
    xsmaddadp      vs3,   vs37,   vs36
 .endif
   
    addi        \AREG, \AREG,  DISP2(\Index,8) 
    addi        \BREG, \BREG, DISP1(\Index,4) 
 
.endm




.macro KERNEL1x2_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG)
    lxv vs10,   DISP8(\Index, 16+\OffsetB)(\AREG) 

    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG)
   
    xxmrglw   vs5, vs26,vs26
    xxmrghw   vs6, vs26,vs26 
 
    xvmaddasp      vs0,   vs8,   vs5
    xvmaddasp      vs1,   vs10,   vs6 
 
 
.if \IsLast==1   
    addi        \AREG, \AREG, DISP8(\Index,32)
    addi        \BREG, \BREG,  DISP4(\Index,16)  
.endif 
  
.endm

.macro KERNEL1x2_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxssp v3,   DISP4(\Index, 0+\OffsetB)(\AREG)
    lxssp v4,   DISP4(\Index, 4+\OffsetB)(\AREG) 
    lxssp v7,   DISP4(\Index, 8+\OffsetB)(\AREG)
    lxssp v8,   DISP4(\Index, 12+\OffsetB)(\AREG)    
    lxssp v5,   DISP2(\Index, 0+\OffsetA)(\BREG)        
    lxssp v6,   DISP2(\Index, 4+\OffsetA)(\BREG)  
 
 
    xsmaddadp      vs2,   vs37,   vs35
    xsmaddadp      vs3,   vs37,   vs36

    xsmaddadp      vs2,   vs38,   vs39 
    xsmaddadp      vs3,   vs38,   vs40      
 
   
    addi        \AREG, \AREG, DISP4(\Index,16)
    addi        \BREG, \BREG, DISP2(\Index,8) 
.endm


.macro SAVE1x2

#ifndef TRMMKERNEL    
    lxssp v4   , 0(CO)      
    lxssp v5   , 4(CO) 
    
#endif

  /*convert alpha_r for multiply*/
  xscvspdp  vs16,alpha_r

 /*aggregate vectors 1x2_4   */ 
      xxpermdi         vs4,vs0,vs0,2
      xxpermdi         vs5,vs1,vs1,2  
      xvaddsp          vs0,vs0,vs4
      xvaddsp         vs1,vs1,vs5 
      xvaddsp         vs0,vs0,vs1 
/*aggregate vectors 1x1_2 and 1x1_1 into 1x2_4*/
  xscvspdp  vs5, vs0
  xxspltw   vs6, vs0, 1  
  xscvspdp  vs6,vs6 
  xsadddp  vs2,vs2,vs6
  xsadddp  vs3,vs3,vs5  

  /**** store last two words*/
#if defined(TRMMKERNEL) 
  xsmuldp  vs36,vs2, vs16 
  xsmuldp  vs37,vs3, vs16  
 
#else
  xsmaddadp  vs36,vs2, vs16 
  xsmaddadp  vs37,vs3, vs16 
#endif  

  stxssp       v4, 0(CO) 
  stxssp        v5, 4(CO)  

  addi CO,CO,8

.endm
/*///////////////// N=1 M=1 //////////////////*/
.macro Zero1x1
    xxlxor      vs0,   vs0,   vs0
    xxlxor      vs1,   vs1,   vs1 
    xxlxor      vs2, vs2,vs2 
    xxlxor      vs3,vs3,vs3 
    xxlxor      vs4,vs4,vs4       
.endm
 
.macro KERNEL1x1
  KERNEL1x1_1 AO,BO, 1, 0,0,0
.endm

.macro KERNEL1x1_16 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x1_I_16 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL1x1_8 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x1_I_8 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL1x1_4 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x1_I_4 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm

.macro KERNEL1x1_2 OffsetA,OffsetB, Index,IsLast  
  KERNEL1x1_I_2 AO,BO,  \OffsetA,\OffsetB,\Index,\IsLast
.endm
 /*
   we will calculate 1 alone ( FIRST==1 to zero vs4) 
 */
.macro KERNEL1x1_1 AREG,BREG,First,OffsetA,OffsetB,Index


    lxssp v3,   DISP1(\Index, 0+\OffsetB)(\AREG) 
    lxssp v5,   DISP1(\Index, 0+\OffsetA)(\BREG)        
 
 
.if \First==1
    xvmuldp      vs4,   vs37,   vs35       
     
.else 
    xsmaddadp     vs4,   vs37,   vs35 
 .endif
   
    addi        \AREG, \AREG,  DISP1(\Index,4) 
    addi        \BREG, \BREG, DISP1(\Index,4) 
 
.endm


.macro KERNEL1x1_I_16 AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP16(\Index, 0+\OffsetB)(\AREG) 
    lxv vs9,    DISP16(\Index, 16+\OffsetB)(\AREG) 
    lxv vs10,   DISP16(\Index, 32+0+\OffsetB)(\AREG) 
    lxv vs11,   DISP16(\Index, 32+ 16+\OffsetB)(\AREG)        
    lxv vs26,   DISP16(\Index, 0+\OffsetA)(\BREG) 
    lxv vs16,   DISP16(\Index, 16+\OffsetA)(\BREG) 
    lxv vs17,   DISP16(\Index, 32+0+\OffsetA)(\BREG) 
    lxv vs18,   DISP16(\Index, 32+16+\OffsetA)(\BREG)     
    xvmaddasp      vs0,   vs8,   vs26 
    xvmaddasp      vs1,   vs9,   vs16  
    xvmaddasp      vs2,   vs10,  vs17 
    xvmaddasp      vs3,   vs11,  vs18
.if \IsLast==1   
    addi        \AREG, \AREG, DISP16(\Index,64)
    addi        \BREG, \BREG,  DISP16(\Index,64)  
.endif 
  
.endm

.macro KERNEL1x1_I_8  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP8(\Index, 0+\OffsetB)(\AREG) 
    lxv vs9,    DISP8(\Index, 16+\OffsetB)(\AREG)     
    lxv vs26,   DISP8(\Index, 0+\OffsetA)(\BREG) 
    lxv vs16,   DISP8(\Index, 16+\OffsetA)(\BREG) 
    xvmaddasp      vs0,   vs8,   vs26 
    xvmaddasp      vs1,   vs9,   vs16 
 
.if \IsLast==1   
    addi        \AREG, \AREG, DISP8(\Index,32)
    addi        \BREG, \BREG,  DISP8(\Index,32)  
.endif 
  
.endm


.macro KERNEL1x1_I_4  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxv vs8,    DISP4(\Index, 0+\OffsetB)(\AREG) 
    lxv vs26,   DISP4(\Index, 0+\OffsetA)(\BREG) 
 
    xvmaddasp      vs0,   vs8,   vs26 
 
 
.if \IsLast==1   
    addi        \AREG, \AREG, DISP4(\Index,16)
    addi        \BREG, \BREG,  DISP4(\Index,16)  
.endif 
  
.endm

.macro KERNEL1x1_I_2  AREG,BREG, OffsetA,OffsetB, Index,IsLast  

    lxsd v4,    DISP2(\Index, 0+\OffsetB)(\AREG) 
    lxsd v5,   DISP2(\Index, 0+\OffsetA)(\BREG) 
 
    xvmaddasp      vs0,   vs36,   vs37 
 
    addi        \AREG, \AREG, DISP2(\Index,8)
    addi        \BREG, \BREG, DISP2(\Index,8) 
.endm


.macro SAVE1x1

#ifndef TRMMKERNEL    
    lxssp v4   , 0(CO)    
    
#endif

  /*convert alpha_r for multiply*/
  xscvspdp  vs16,alpha_r

 /*aggregate vectors   */ 
      xvaddsp          vs0,vs0,vs1
      xvaddsp          vs2,vs2,vs3
      xvaddsp          vs0,vs0,vs2

      xxpermdi         vs7,vs0,vs0,2   
      xvaddsp          vs0,vs0,vs7 
/*aggregate vectors 1x1_2 and 1x1_1 into 1x1_4*/
  xscvspdp  vs5, vs0
  xxspltw   vs6, vs0, 1  
  xscvspdp  vs6,vs6 
  xsadddp  vs7,vs5,vs6
  xsadddp  vs4,vs4,vs7  

  /**** store last two words*/
#if defined(TRMMKERNEL) 
  xsmuldp  vs36,vs4, vs16   
 
#else
  xsmaddadp  vs36,vs4, vs16   
#endif  

  stxssp       v4, 0(CO)    

  addi CO,CO,4

.endm




/****************************TRMM POINTER REFRESH MACROSES*************************/

.macro SHIFT_REG  REG1,REG2,SHIFT_VAL
		.if \SHIFT_VAL==16 
			slwi		\REG1,	\REG2,	6			
		.elseif \SHIFT_VAL==8  
			slwi		\REG1,	\REG2,	5			 
		.elseif \SHIFT_VAL==4
			slwi		\REG1,	\REG2,	4			  
		.elseif \SHIFT_VAL==2
			slwi		\REG1,	\REG2,	3			 
		.elseif \SHIFT_VAL==1
			slwi		\REG1,	\REG2,	2			 
		.endif
.endm

/*
//#if (defined(LEFT) &&  defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// 		ptrbb = bb;
// #else
// 		ptrba += off*16;
// 		ptrbb = bb + off*2;
// #endif
*/
.macro REFRESH_POINTERS  PTR_A,PTR_B,OFF_VAL,B_VAL,C_A,C_B
    #if (defined(LEFT) &&  defined(TRANSA)) ||  (!defined(LEFT) && !defined(TRANSA))
        /* ptrbb = bb;*/
        mr \PTR_B,\B_VAL     /* refresh BPOINT */

    #else
		    /*
        // ptrba  =ptrba+ off*C_A;
        // ptrbb = bb + off*C_B; 
				*/
		SHIFT_REG T4,\OFF_VAL,\C_B		/* Number of values in B shifted  */
		SHIFT_REG T2,\OFF_VAL,\C_A		/* Number of values in A shifted  */
		add		\PTR_B,	\B_VAL ,	T4				/* Add values to BO */
		add		\PTR_A,	\PTR_A,	T2				/* Add values to AO  */
    #endif 
.endm


/*
// #if (defined(LEFT) && !defined(TRANSA)) || (!defined(LEFT) && defined(TRANSA))
// 		temp = bk-off;
// #elif defined(LEFT)
// 		temp = off+16;	// number of values in A
// #else
// 		temp = off+2;	// number of values in B
// #endif
*/
.macro REFRESH_TEMP_BK TEMP_BK,BK_VAL,OFF_VAL,INCR_A,INCR_B
    #if (defined(LEFT) && !defined(TRANSA)) ||  (!defined(LEFT) && defined(TRANSA))
                            /* temp = bk-off;*/
           sub \TEMP_BK,\BK_VAL,\OFF_VAL

    #elif defined(LEFT)
                            /* temp = off+INCR_A;	// number of values in A */
           addi \TEMP_BK, \OFF_VAL, \INCR_A
    #else
                            /* temp = off+INCR_B	// number of values in B*/
           addi \TEMP_BK,\OFF_VAL, \INCR_B
    #endif

.endm
/*
// #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
// 		temp = bk - off;
// #ifdef LEFT
// 		temp -= 16; // number of values in A
// #else
// 		temp -= 2; // number of values in B
// #endif
// 		ptrba += temp*16;
// 		ptrbb += temp*2;
// #endif

// #ifdef LEFT
// 		off += 16; // number of values in A
// #endif
*/
 

.macro REFRESH_AFTER_SAVE TEMP_BK,BK_VAL,OFF_VAL,PTR_B,PTR_A,C_A,C_B

    #if ( defined(LEFT) && defined(TRANSA)) || (!defined(LEFT) && !defined(TRANSA))
                    /*temp = bk - off;*/
                sub \TEMP_BK,\BK_VAL,\OFF_VAL
    #ifdef LEFT
                    /*temp -= 8; // number of values in A*/
                addi \TEMP_BK,\TEMP_BK,-\C_A
    #else
                    /*temp -= 4; // number of values in B*/
                addi \TEMP_BK,\TEMP_BK,-\C_B 
    #endif
                    /*ptrba += temp*C_A;
                    ptrbb += temp*C_B;*/ 
                SHIFT_REG T4,\TEMP_BK,\C_A
								SHIFT_REG T2,\TEMP_BK,\C_B
                add \PTR_A, \PTR_A,T4/*ptrba+temp*C_A*/ 
								add \PTR_B, \PTR_B,T2 

    #endif

    #ifdef LEFT
                    /*off += 8; // number of values in A*/
                 addi \OFF_VAL,\OFF_VAL,\C_A
    #endif
.endm