OpenBLAS/kernel/power/trsm_kernel_hummer_LN.S

/*********************************************************************/
/* Copyright 2009, 2010 The University of Texas at Austin.           */
/* All rights reserved.                                              */
/*                                                                   */
/* Redistribution and use in source and binary forms, with or        */
/* without modification, are permitted provided that the following   */
/* conditions are met:                                               */
/*                                                                   */
/*   1. Redistributions of source code must retain the above         */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer.                                                  */
/*                                                                   */
/*   2. Redistributions in binary form must reproduce the above      */
/*      copyright notice, this list of conditions and the following  */
/*      disclaimer in the documentation and/or other materials       */
/*      provided with the distribution.                              */
/*                                                                   */
/*    THIS  SOFTWARE IS PROVIDED  BY THE  UNIVERSITY OF  TEXAS AT    */
/*    AUSTIN  ``AS IS''  AND ANY  EXPRESS OR  IMPLIED WARRANTIES,    */
/*    INCLUDING, BUT  NOT LIMITED  TO, THE IMPLIED  WARRANTIES OF    */
/*    MERCHANTABILITY  AND FITNESS FOR  A PARTICULAR  PURPOSE ARE    */
/*    DISCLAIMED.  IN  NO EVENT SHALL THE UNIVERSITY  OF TEXAS AT    */
/*    AUSTIN OR CONTRIBUTORS BE  LIABLE FOR ANY DIRECT, INDIRECT,    */
/*    INCIDENTAL,  SPECIAL, EXEMPLARY,  OR  CONSEQUENTIAL DAMAGES    */
/*    (INCLUDING, BUT  NOT LIMITED TO,  PROCUREMENT OF SUBSTITUTE    */
/*    GOODS  OR  SERVICES; LOSS  OF  USE,  DATA,  OR PROFITS;  OR    */
/*    BUSINESS INTERRUPTION) HOWEVER CAUSED  AND ON ANY THEORY OF    */
/*    LIABILITY, WHETHER  IN CONTRACT, STRICT  LIABILITY, OR TORT    */
/*    (INCLUDING NEGLIGENCE OR OTHERWISE)  ARISING IN ANY WAY OUT    */
/*    OF  THE  USE OF  THIS  SOFTWARE,  EVEN  IF ADVISED  OF  THE    */
/*    POSSIBILITY OF SUCH DAMAGE.                                    */
/*                                                                   */
/* The views and conclusions contained in the software and           */
/* documentation are those of the authors and should not be          */
/* interpreted as representing official policies, either expressed   */
/* or implied, of The University of Texas at Austin.                 */
/*********************************************************************/

#define ASSEMBLER
#include "common.h"

#define ALPHA    0
#define FZERO	 8

#define	M	r3
#define	N	r4
#define	K	r5

#ifdef linux
#define A	r6
#define	B	r7
#define	C	r8
#define	LDC	r9
#define OFFSET	r10
#endif

#define TEMP	r11
#define AORIG	r12
#define KK	r14
#define INCM1	r15
#define INCM4	r16
#define INCM2	r17
#define INC2	r19
#define INC	r20
#define INC4	r21

#define	I	r22
#define J	r23
#define AO	r24
#define BO	r25
#define AO2	r26
#define	BO2	r27

#define	CO1	r28
#define CO2	r29
#define	CO3	r30
#define	CO4	r31

#ifndef NEEDPARAM

#define A1	f16
#define A2	f17
#define A3	f18
#define A4	f19
#define A5	f20
#define A6	f21
#define A7	f22
#define A8	f23
#define A9	f24
#define A10	f25

#define B1	f26
#define B2	f27
#define B3	f28
#define B4	f29
#define B5	f30
#define B6	f31

#define AP	B6


	PROLOGUE
	PROFCODE

	li	r0, -16

	stfpdux	f14, SP, r0
	stfpdux	f15, SP, r0
	stfpdux	f16, SP, r0
	stfpdux	f17, SP, r0
	stfpdux	f18, SP, r0
	stfpdux	f19, SP, r0
	stfpdux	f20, SP, r0
	stfpdux	f21, SP, r0
	stfpdux	f22, SP, r0
	stfpdux	f23, SP, r0
	stfpdux	f24, SP, r0
	stfpdux	f25, SP, r0
	stfpdux	f26, SP, r0
	stfpdux	f27, SP, r0
	stfpdux	f28, SP, r0
	stfpdux	f29, SP, r0
	stfpdux	f30, SP, r0
	stfpdux	f31, SP, r0

	stwu	r31,  -4(SP)
	stwu	r30,  -4(SP)
	stwu	r29,  -4(SP)
	stwu	r28,  -4(SP)

	stwu	r27,  -4(SP)
	stwu	r26,  -4(SP)
	stwu	r25,  -4(SP)
	stwu	r24,  -4(SP)

	stwu	r23,  -4(SP)
	stwu	r22,  -4(SP)
	stwu	r21,  -4(SP)
	stwu	r20,  -4(SP)

	stwu	r19,  -4(SP)
	stwu	r18,  -4(SP)
	stwu	r17,  -4(SP)
	stwu	r16,  -4(SP)

	stwu	r15,  -4(SP)
	stwu	r14,  -4(SP)	# dummy

	li	r0,   0

	stwu	r0,   -4(SP)
	stwu	r0,   -4(SP)
	stfdu	f1,   -8(SP)

	slwi	LDC, LDC, BASE_SHIFT

	cmpwi	cr0, M, 0
	ble	.L999
	cmpwi	cr0, N, 0
	ble	.L999
	cmpwi	cr0, K, 0
	ble	.L999

	li	INC,    1 * SIZE
	li	INC2,   2 * SIZE
	li	INC4,   4 * SIZE

	li	INCM1, -1 * SIZE
	li	INCM2, -2 * SIZE
	li	INCM4, -4 * SIZE

	addi	C, C, - 1 * SIZE

#ifdef LN
	mullw	r0, M, K
	slwi	r0, r0, BASE_SHIFT
	add	A, A, r0

	slwi	r0, M, BASE_SHIFT
	add	C, C, r0
#endif

#ifdef RN
	neg	KK, OFFSET
#endif

#ifdef RT
	mullw	r0, N, K
	slwi	r0, r0, BASE_SHIFT
	add	B, B, r0

	mullw	r0, N, LDC
	add	C, C, r0

	sub	KK, N, OFFSET
#endif

	srawi.	J, N,  2
	ble	.L50
	.align 4

.L10:
#ifdef RT
	slwi	r0, K, 2 + BASE_SHIFT
	sub	B, B, r0

	slwi	r0, LDC, 2
	sub	C, C, r0
#endif

	mr	CO1, C
	add	CO2, C,   LDC
	add	CO3, CO2, LDC
	add	CO4, CO3, LDC

#ifdef LN
	add	KK, M, OFFSET
#endif

#ifdef LT
	mr	KK, OFFSET
#endif

#if defined(LN) || defined(RT)
	addi	AORIG, A, -4 * SIZE
#else
	addi	AO, A, -4 * SIZE
#endif
#ifndef RT
	add	C,  CO4, LDC
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0

	andi.	I, M,  1
	beq	.L20

#if defined(LT) || defined(RN)
	addi	AO2, AO,   2 * SIZE
	fpmr	f1,  f0
	addi	BO,  B,  - 4 * SIZE
	fpmr	f2,  f0
	addi	BO2, B,  - 2 * SIZE
	fpmr	f3,  f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L44
#else

#ifdef LN
	slwi	r0,   K,  0 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 0 + BASE_SHIFT
	slwi	TEMP, KK, 2 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	AO2, AO,   2 * SIZE
	fpmr	f1,  f0
	addi	BO,  BO, - 4 * SIZE
	fpmr	f2,  f0
	addi	BO2, BO,   2 * SIZE
	fpmr	f3,  f0

	srawi.	r0,  TEMP,  3
	mtspr	CTR, r0
	ble	.L44
#endif

	LFPDUX	A1,  AO,  INC4
	LFPDUX	B1,  BO,  INC4
	LFPDUX	B2,  BO2, INC4
	LFPDUX	A2, AO2,  INC4
	LFPDUX	B3,  BO,  INC4
	LFPDUX	B4,  BO2, INC4

	LFPDUX	A3,  AO,  INC4
	LFPDUX	A5,  BO,  INC4
	LFPDUX	A6,  BO2, INC4
	LFPDUX	A4, AO2,  INC4
	LFPDUX	A7,  BO,  INC4
	LFPDUX	A8,  BO2, INC4
	bdz-	.L43
	.align 4

.L42:
	fxcpmadd	f0,  A1, B1, f0
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f1,  A1, B2, f1
	LFPDUX	B2,  BO2, INC4
	fxcsmadd	f2,  A1, B3, f2
	LFPDUX	B3,  BO,  INC4
	fxcsmadd	f3,  A1, B4, f3
	LFPDUX	B4,  BO2, INC4
	LFPDUX	A1,  AO,  INC4

	fxcpmadd	f0,  A2, A5, f0
	LFPDUX	A5,  BO,  INC4
	fxcpmadd	f1,  A2, A6, f1
	LFPDUX	A6,  BO2, INC4
	fxcsmadd	f2,  A2, A7, f2
	LFPDUX	A7,  BO,  INC4
	fxcsmadd	f3,  A2, A8, f3
	LFPDUX	A8,  BO2, INC4
	LFPDUX	A2, AO2,  INC4

	fxcpmadd	f0,  A3, B1, f0
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f1,  A3, B2, f1
	LFPDUX	B2,  BO2, INC4
	fxcsmadd	f2,  A3, B3, f2
	LFPDUX	B3,  BO,  INC4
	fxcsmadd	f3,  A3, B4, f3
	LFPDUX	B4,  BO2, INC4
	LFPDUX	A3,  AO,  INC4

	fxcpmadd	f0,  A4, A5, f0
	LFPDUX	A5,  BO,  INC4
	fxcpmadd	f1,  A4, A6, f1
	LFPDUX	A6,  BO2, INC4
	fxcsmadd	f2,  A4, A7, f2
	LFPDUX	A7,  BO,  INC4
	fxcsmadd	f3,  A4, A8, f3
	LFPDUX	A8,  BO2, INC4
	LFPDUX	A4, AO2,  INC4
	bdnz+	.L42
	.align 4

.L43:
	fxcpmadd	f0,  A1, B1, f0
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f1,  A1, B2, f1
	LFPDUX	B2,  BO2, INC4
	fxcsmadd	f2,  A1, B3, f2
	LFPDUX	B3,  BO,  INC4
	fxcsmadd	f3,  A1, B4, f3
	LFPDUX	B4,  BO2, INC4

	fxcpmadd	f0,  A2, A5, f0
	LFPDUX	A5,  BO,  INC4
	fxcpmadd	f1,  A2, A6, f1
	LFPDUX	A6,  BO2, INC4
	fxcsmadd	f2,  A2, A7, f2
	LFPDUX	A7,  BO,  INC4
	fxcsmadd	f3,  A2, A8, f3
	LFPDUX	A8,  BO2, INC4

	fxcpmadd	f0,  A3, B1, f0
	fxcpmadd	f1,  A3, B2, f1
	fxcsmadd	f2,  A3, B3, f2
	fxcsmadd	f3,  A3, B4, f3

	fxcpmadd	f0,  A4, A5, f0
	fxcpmadd	f1,  A4, A6, f1
	fxcsmadd	f2,  A4, A7, f2
	fxcsmadd	f3,  A4, A8, f3
	.align 4

.L44:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L48
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L48
#endif

	LFDX	A1,  AO,  INC4
	LFPDUX	B1,  BO,  INC4
	LFPDUX	B2,  BO2, INC4
	add	AO, AO, INC
	bdz-	.L47
	.align 4

.L46:
	fxcpmadd	f0,  A1, B1, f0
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f1,  A1, B2, f1
	LFDX	A1,  AO,  INC4
	LFPDUX	B2,  BO2, INC4
	add	AO, AO, INC
	bdnz+	.L46
	.align 4

.L47:
	fxcpmadd	f0,  A1, B1, f0
	fxcpmadd	f1,  A1, B2, f1
	addi	AO2, AO,   2 * SIZE
	.align 4

.L48:
	fpadd	f0, f0, f2
	fpadd	f1, f1, f3

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 1
#else
	subi	r0, KK, 4
#endif
	slwi	TEMP, r0, 0 + BASE_SHIFT
	slwi	r0,   r0, 2 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO, B,     r0
	addi	AO2, AO,   2 * SIZE
	addi	BO,  BO, - 4 * SIZE
	addi	BO2, BO,   2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFPDX	f16, BO,  INC4
	LFPDX	f17, BO2, INC4

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#else
	LFPDX	f16, AO,  INC4
	LFPDX	f17, AO2, INC4

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#endif

#if defined(LN) || defined(LT)
	LFPDX	A1,  AO, INC4

	fxpmul	f0,  A1, f0
	fxpmul	f1,  A1, f1
#endif

#ifdef RN
	LFD	A1,  (4 +  0) * SIZE(BO)
	LFD	A2,  (4 +  1) * SIZE(BO)
	LFD	A3,  (4 +  2) * SIZE(BO)
	LFD	A4,  (4 +  3) * SIZE(BO)

	LFD	A5,  (4 +  5) * SIZE(BO)
	LFD	A6,  (4 +  6) * SIZE(BO)
	LFD	A7,  (4 +  7) * SIZE(BO)
	LFD	A8,  (4 + 10) * SIZE(BO)

	LFD	A9,  (4 + 11) * SIZE(BO)
	LFD	A10, (4 + 15) * SIZE(BO)

	fsmtp	     f2, f0
	fsmtp	     f3, f1

	fmul	     f0,  A1, f0
	fnmsub	     f2,  A2, f0, f2
	fnmsub	     f1,  A3, f0, f1
	fnmsub	     f3,  A4, f0, f3

	fmul	     f2,  A5, f2
	fnmsub	     f1,  A6, f2, f1
	fnmsub	     f3,  A7, f2, f3

	fmul	     f1,  A8, f1
	fnmsub	     f3,  A9, f1, f3

	fmul	     f3,  A10, f3

	fsmfp	     f0, f2
	fsmfp	     f1, f3
#endif

#ifdef RT
	LFD	A1,  (4 + 15) * SIZE(BO)
	LFD	A2,  (4 + 14) * SIZE(BO)
	LFD	A3,  (4 + 13) * SIZE(BO)
	LFD	A4,  (4 + 12) * SIZE(BO)

	LFD	A5,  (4 + 10) * SIZE(BO)
	LFD	A6,  (4 +  9) * SIZE(BO)
	LFD	A7,  (4 +  8) * SIZE(BO)
	LFD	A8,  (4 +  5) * SIZE(BO)

	LFD	A9,  (4 +  4) * SIZE(BO)
	LFD	A10, (4 +  0) * SIZE(BO)

	fsmtp	     f2, f0
	fsmtp	     f3, f1

	fmul	     f3,  A1, f3
	fnmsub	     f1,  A2, f3, f1
	fnmsub	     f2,  A3, f3, f2
	fnmsub	     f0,  A4, f3, f0

	fmul	     f1,  A5, f1
	fnmsub	     f2,  A6, f1, f2
	fnmsub	     f0,  A7, f1, f0

	fmul	     f2,  A8, f2
	fnmsub	     f0,  A9, f2, f0

	fmul	     f0,  A10, f0

	fsmfp	     f0, f2
	fsmfp	     f1, f3
#endif

#if defined(LN) || defined(LT)
	STFPDX	f0,  BO,  INC4
	STFPDX	f1,  BO2, INC4
#else
	STFPDX	f0,  AO,  INC4
	STFPDX	f1,  AO2, INC4
#endif

#ifdef LN
	subi	CO1, CO1, 1 * SIZE
	subi	CO2, CO2, 1 * SIZE
	subi	CO3, CO3, 1 * SIZE
	subi	CO4, CO4, 1 * SIZE
#endif

	STFDX	f0,  CO1, INC
	STFSDX	f0,  CO2, INC
	STFDX	f1,  CO3, INC
	STFSDX	f1,  CO4, INC

#ifdef RT
	slwi	r0, K, 0 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 0 + BASE_SHIFT
	slwi	TEMP, TEMP, 2 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 1
#endif

#ifdef LN
	subi	KK, KK, 1
#endif
	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L20:
	andi.	I, M,  2
	beq	.L30

#if defined(LT) || defined(RN)
	addi	AO2, AO,   2 * SIZE
	fpmr	f4,  f0
	addi	BO,  B,  - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, B,  - 2 * SIZE
	fpmr	f12, f0

	srawi.	r0,  KK,  2
	mtspr	CTR, r0
	ble	.L34
#else

#ifdef LN
	slwi	r0,   K,  1 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 1 + BASE_SHIFT
	slwi	TEMP, KK, 2 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	AO2, AO,    2 * SIZE
	fpmr	f4,  f0
	addi	BO,  BO,  - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, BO,    2 * SIZE
	fpmr	f12, f0

	srawi.	r0,  TEMP,  2
	mtspr	CTR, r0
	ble	.L34
#endif

	LFPDUX	A1,  AO, INC4
	LFPDUX	B1,  BO, INC4
	LFPDUX	B2, BO2, INC4
	LFPDUX	A2, AO2, INC4
	LFPDUX	B3,  BO, INC4
	LFPDUX	B4, BO2, INC4

	LFPDUX	A3,  AO, INC4
	LFPDUX	A5,  BO, INC4
	LFPDUX	A6, BO2, INC4
	LFPDUX	A4, AO2, INC4
	LFPDUX	A7,  BO, INC4
	LFPDUX	A8, BO2, INC4
	bdz-	.L33
	.align 4

.L32:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	B1,  BO, INC4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12
	LFPDUX	B2, BO2, INC4
	LFPDUX	A1,  AO, INC4

	fxcpmadd	f0,  B3, A2, f0
	fxcsmadd	f4,  B3, A2, f4
	LFPDUX	B3,  BO, INC4
	fxcpmadd	f8,  B4, A2, f8
	fxcsmadd	f12, B4, A2, f12
	LFPDUX	B4, BO2, INC4
	LFPDUX	A2, AO2, INC4

	fxcpmadd	f0,  A5, A3, f0
	fxcsmadd	f4,  A5, A3, f4
	LFPDUX	A5,  BO, INC4
	fxcpmadd	f8,  A6, A3, f8
	fxcsmadd	f12, A6, A3, f12
	LFPDUX	A6, BO2, INC4
	LFPDUX	A3,  AO, INC4

	fxcpmadd	f0,  A7, A4, f0
	fxcsmadd	f4,  A7, A4, f4
	LFPDUX	A7,  BO, INC4
	fxcpmadd	f8,  A8, A4, f8
	fxcsmadd	f12, A8, A4, f12
	LFPDUX	A8, BO2, INC4
	LFPDUX	A4, AO2, INC4
	bdnz+	.L32
	.align 4

.L33:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12

	fxcpmadd	f0,  B3, A2, f0
	fxcsmadd	f4,  B3, A2, f4
	fxcpmadd	f8,  B4, A2, f8
	fxcsmadd	f12, B4, A2, f12

	fxcpmadd	f0,  A5, A3, f0
	fxcsmadd	f4,  A5, A3, f4
	fxcpmadd	f8,  A6, A3, f8
	fxcsmadd	f12, A6, A3, f12

	fxcpmadd	f0,  A7, A4, f0
	fxcsmadd	f4,  A7, A4, f4
	fxcpmadd	f8,  A8, A4, f8
	fxcsmadd	f12, A8, A4, f12
	.align 4

.L34:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L38
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L38
#endif

	LFPDX	A1,  AO,  INC4
	LFPDUX	B1,  BO,  INC4
	LFPDUX	B2,  BO2, INC4
	add	AO, AO, INC2
	bdz-	.L37
	.align 4

.L36:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12
	LFPDX	A1,  AO,  INC4
	LFPDUX	B2,  BO2, INC4
	add	AO, AO, INC2
	bdnz+	.L36
	.align 4

.L37:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12
	.align 4

.L38:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 2
#else
	subi	r0, KK, 4
#endif
	slwi	TEMP, r0, 1 + BASE_SHIFT
	slwi	r0,   r0, 2 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO, B,     r0
	addi	AO2, AO,   2 * SIZE
	addi	BO,  BO, - 4 * SIZE
	addi	BO2, BO,   2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	fpmr	f28, f8

	fsmfp	f0,  f4
	fsmfp	f8,  f12
	fsmtp	f4,  f24
	fsmtp	f12, f28

	LFPDUX	f16, BO,  INC4
	LFPDUX	f17, BO2, INC4
	LFPDUX	f18, BO,  INC4
	LFPDUX	f19, BO2, INC4

	subi	BO,  BO,   8 * SIZE
	subi	BO2, BO2,  8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f8,  f17,  f8
	fpsub	f4,  f18,  f4
	fpsub	f12, f19,  f12
#else
	LFPDUX	f16, AO,  INC4
	LFPDUX	f17, AO2, INC4
	LFPDUX	f18, AO,  INC4
	LFPDUX	f19, AO2, INC4

	subi	AO,  AO,   8 * SIZE
	subi	AO2, AO2,  8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f4,  f17,  f4
	fpsub	f8,  f18,  f8
	fpsub	f12, f19,  f12
#endif

#ifdef LN
	addi	AO,  AO,   8 * SIZE
	addi	AO2, AO2,  8 * SIZE

	LFPDUX	A1,  AO2, INCM4
	LFPDUX	A2,  AO,  INCM4

	addi	AO,  AO,  -4 * SIZE
	addi	AO2, AO2, -4 * SIZE

	fxsmul	f4,  A1, f4
	fxsmul	f12, A1, f12

	fxcpnmsub  f0,  A1, f4,  f0
	fxcpnmsub  f8,  A1, f12, f8

	fxpmul	f0,  A2, f0
	fxpmul	f8,  A2, f8
#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC4
	LFPDUX	A2,  AO2, INC4

	subi	AO,  AO,   4 * SIZE
	subi	AO2, AO2,  4 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f8,  A1,  f8

	fxcsnmsub  f4,  A1, f0, f4
	fxcsnmsub  f12, A1, f8, f12

	fxsmul	f4,  A2,  f4
	fxsmul	f12, A2,  f12
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC4
	LFPDUX	A2,  BO2, INC4
	LFPDUX	A3,  BO,  INC4
	LFPDUX	A4,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A5,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A6,  BO2, INC4

	subi	BO,  BO,  16 * SIZE
	subi	BO2, BO2, 16 * SIZE

	fxpmul	     f0,  A1,  f0
	fxcsnmsub    f4,  A1, f0, f4
	fxcpnmsub    f8,  A2, f0, f8
	fxcsnmsub    f12, A2, f0, f12

	fxsmul	     f4,  A3,  f4
	fxcpnmsub    f8,  A4, f4, f8
	fxcsnmsub    f12, A4, f4, f12

	fxpmul	     f8,  A5,  f8
	fxcsnmsub    f12, A5, f8,  f12
	fxsmul	     f12, A6,  f12
#endif

#ifdef RT
	addi	BO,  BO,  20 * SIZE
	addi	BO2, BO2, 20 * SIZE

	LFPDUX	A1,  BO2, INCM4
	LFPDUX	A2,  BO,  INCM4

	LFPDUX	A3,  BO2, INCM4
	LFPDUX	A4,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A5,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A6,  BO,  INCM4
	subi	BO,  BO,  4 * SIZE
	subi	BO2, BO2, 4 * SIZE

	fxsmul	     f12, A1,  f12
	fxcpnmsub    f8,  A1, f12, f8
	fxcsnmsub    f4,  A2, f12, f4
	fxcpnmsub    f0,  A2, f12, f0

	fxpmul	     f8,  A3,  f8
	fxcsnmsub    f4,  A4, f8,  f4
	fxcpnmsub    f0,  A4, f8,  f0

	fxsmul	     f4,  A5,  f4
	fxcpnmsub    f0,  A5, f4,  f0
	fxpmul	     f0,  A6,  f0
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
	subi	CO2, CO2, 2 * SIZE
	subi	CO3, CO3, 2 * SIZE
	subi	CO4, CO4, 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC4
	STFPDUX	f8,  BO2, INC4
	STFPDUX	f4,  BO,  INC4
	STFPDUX	f12, BO2, INC4

	subi	BO,  BO,   8 * SIZE
	subi	BO2, BO2,  8 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f4,  CO1, INC
	STFSDUX	f0,  CO2, INC
	STFSDUX	f4,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFDUX	f12, CO3, INC
	STFSDUX	f8,  CO4, INC
	STFSDUX	f12, CO4, INC

#else
	STFPDUX	f0,  AO,  INC4
	STFPDUX	f4,  AO2, INC4
	STFPDUX	f8,  AO,  INC4
	STFPDUX	f12, AO2, INC4

	subi	AO,  AO,   8 * SIZE
	subi	AO2, AO2,  8 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f4,  CO2, INC
	STFSDUX	f4,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFSDUX	f8,  CO3, INC
	STFDUX	f12, CO4, INC
	STFSDUX	f12, CO4, INC
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
	subi	CO2, CO2, 2 * SIZE
	subi	CO3, CO3, 2 * SIZE
	subi	CO4, CO4, 2 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 1 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 1 + BASE_SHIFT
	slwi	TEMP, TEMP, 2 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 2
#endif

#ifdef LN
	subi	KK, KK, 2
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L30:
	andi.	I, M,  4
	beq	.L40

#if defined(LT) || defined(RN)
	addi	AO2, AO,   2 * SIZE
	fpmr	f4,  f0
	addi	BO,  B,  - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, B,  - 2 * SIZE
	fpmr	f12, f0

	srawi.	r0,  KK,  2
 	fpmr	f1,  f0
	fpmr	f5,  f0
	fpmr	f9,  f0
	mtspr	CTR, r0
	fpmr	f13, f0
	ble	.L24
#else

#ifdef LN
	slwi	r0,   K,  2 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 2 + BASE_SHIFT
	slwi	TEMP, KK, 2 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	AO2, AO,   2 * SIZE
	fpmr	f4,  f0
	addi	BO,  BO,  - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, BO,    2 * SIZE
	fpmr	f12, f0

	srawi.	r0,  TEMP,  2
 	fpmr	f1,  f0
	fpmr	f5,  f0
	fpmr	f9,  f0
	mtspr	CTR, r0
	fpmr	f13, f0
	ble	.L24
#endif

	LFPDUX	A1,   AO, INC4
	LFPDUX	B1,   BO, INC4
	LFPDUX	A2,  AO2, INC4
	LFPDUX	B2,  BO2, INC4
	LFPDUX	A3,   AO, INC4
	LFPDUX	B3,   BO, INC4
	LFPDUX	A4,  AO2, INC4
	LFPDUX	B4,  BO2, INC4

	LFPDUX	A5,   AO, INC4
	LFPDUX	B5,   BO, INC4
	LFPDUX	A6,  AO2, INC4
	LFPDUX	B6,  BO2, INC4
	LFPDUX	A7,   AO, INC4
	LFPDUX	A9,   BO, INC4
	LFPDUX	A10, BO2, INC4
	bdz-	.L23
	.align 4

.L22:
	fxcpmadd	f0,  B1, A1, f0
	nop
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	A8,  AO2, INC4
	fxcpmadd	f8,  B2, A1, f8
	nop
	fxcsmadd	f12, B2, A1, f12
	LFPDUX	A1,   AO, INC4

	fxcpmadd	f1,  B1, A2, f1
	nop
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	B1,   BO, INC4
	fxcpmadd	f9,  B2, A2, f9
	nop
	fxcsmadd	f13, B2, A2, f13
	LFPDUX	B2,  BO2, INC4

	fxcpmadd	f0,  B3, A3, f0
	nop
	fxcsmadd	f4,  B3, A3, f4
	LFPDUX	A2,  AO2, INC4
	fxcpmadd	f8,  B4, A3, f8
	nop
	fxcsmadd	f12, B4, A3, f12
	LFPDUX	A3,   AO, INC4

	fxcpmadd	f1,  B3, A4, f1
	nop
	fxcsmadd	f5,  B3, A4, f5
	LFPDUX	B3,   BO, INC4
	fxcpmadd	f9,  B4, A4, f9
	nop
	fxcsmadd	f13, B4, A4, f13
	LFPDUX	B4,  BO2, INC4

	fxcpmadd	f0,  B5, A5, f0
	nop
	fxcsmadd	f4,  B5, A5, f4
	LFPDUX	A4,  AO2, INC4
	fxcpmadd	f8,  B6, A5, f8
	nop
	fxcsmadd	f12, B6, A5, f12
	LFPDUX	A5,   AO, INC4

	fxcpmadd	f1,  B5, A6, f1
	nop
	fxcsmadd	f5,  B5, A6, f5
	LFPDUX	B5,   BO, INC4
	fxcpmadd	f9,  B6, A6, f9
	nop
	fxcsmadd	f13, B6, A6, f13
	LFPDUX	B6,  BO2, INC4

	fxcpmadd	f0,  A9,  A7, f0
	nop
	fxcsmadd	f4,  A9,  A7, f4
	LFPDUX	A6,  AO2, INC4
	fxcpmadd	f8,  A10, A7, f8
	nop
	fxcsmadd	f12, A10, A7, f12
	LFPDUX	A7,   AO, INC4

	fxcpmadd	f1,  A9,  A8, f1
	nop
	fxcsmadd	f5,  A9,  A8, f5
	LFPDUX	A9,   BO, INC4
	fxcpmadd	f9,  A10, A8, f9
	nop
	fxcsmadd	f13, A10, A8, f13
	LFPDUX	A10, BO2, INC4
	bdnz+	.L22
	.align 4

.L23:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	A8,  AO2, INC4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12

	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f5,  B1, A2, f5
	fxcpmadd	f9,  B2, A2, f9
	fxcsmadd	f13, B2, A2, f13

	fxcpmadd	f0,  B3, A3, f0
	fxcsmadd	f4,  B3, A3, f4
	fxcpmadd	f8,  B4, A3, f8
	fxcsmadd	f12, B4, A3, f12

	fxcpmadd	f1,  B3, A4, f1
	fxcsmadd	f5,  B3, A4, f5
	fxcpmadd	f9,  B4, A4, f9
	fxcsmadd	f13, B4, A4, f13

	fxcpmadd	f0,  B5, A5, f0
	fxcsmadd	f4,  B5, A5, f4
	fxcpmadd	f8,  B6, A5, f8
	fxcsmadd	f12, B6, A5, f12

	fxcpmadd	f1,  B5, A6, f1
	fxcsmadd	f5,  B5, A6, f5
	fxcpmadd	f9,  B6, A6, f9
	fxcsmadd	f13, B6, A6, f13

	fxcpmadd	f0,  A9, A7, f0
	fxcsmadd	f4,  A9, A7, f4
	fxcpmadd	f8,  A10, A7, f8
	fxcsmadd	f12, A10, A7, f12

	fxcpmadd	f1,  A9, A8, f1
	fxcsmadd	f5,  A9, A8, f5
	fxcpmadd	f9,  A10, A8, f9
	fxcsmadd	f13, A10, A8, f13
	.align 4

.L24:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L28
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L28
#endif

	LFPDUX	A1,  AO,  INC4
	LFPDUX	A2,  AO2, INC4
	LFPDUX	B1,  BO,  INC4
	LFPDUX	B2,  BO2, INC4
	bdz-	.L27
	.align 4

.L26:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12
	LFPDUX	A1,  AO,  INC4

	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	B1,  BO,  INC4
	fxcpmadd	f9,  B2, A2, f9
	fxcsmadd	f13, B2, A2, f13
	LFPDUX	A2,  AO2, INC4
	LFPDUX	B2,  BO2, INC4
	bdnz+	.L26
	.align 4

.L27:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	fxcpmadd	f8,  B2, A1, f8
	fxcsmadd	f12, B2, A1, f12

	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f5,  B1, A2, f5
	fxcpmadd	f9,  B2, A2, f9
	fxcsmadd	f13, B2, A2, f13
	.align 4

.L28:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 4
#else
	subi	r0, KK, 4
#endif
	slwi	TEMP, r0, 2 + BASE_SHIFT
	slwi	r0,   r0, 2 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO, B,     r0
	addi	AO2, AO,   2 * SIZE
	addi	BO,  BO, - 4 * SIZE
	addi	BO2, BO,   2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	fpmr	f25, f1
	fpmr	f28, f8
	fpmr	f29, f9

	fsmfp	f0,  f4
	fsmfp	f1,  f5
	fsmfp	f8,  f12
	fsmfp	f9,  f13

	fsmtp	f4,  f24
	fsmtp	f5,  f25
	fsmtp	f12, f28
	fsmtp	f13, f29

	LFPDUX	f16, BO,  INC4
	LFPDUX	f17, BO2, INC4
	LFPDUX	f18, BO,  INC4
	LFPDUX	f19, BO2, INC4

 	LFPDUX	f20, BO,  INC4
	LFPDUX	f21, BO2, INC4
	LFPDUX	f22, BO,  INC4
	LFPDUX	f23, BO2, INC4

	subi	BO,  BO,  16 * SIZE
	subi	BO2, BO2, 16 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f8,  f17,  f8
	fpsub	f4,  f18,  f4
	fpsub	f12, f19,  f12

	fpsub	f1,  f20,  f1
	fpsub	f9,  f21,  f9
	fpsub	f5,  f22,  f5
	fpsub	f13, f23,  f13
#else
	LFPDUX	f16, AO,  INC4
	LFPDUX	f17, AO2, INC4
	LFPDUX	f18, AO,  INC4
	LFPDUX	f19, AO2, INC4
	LFPDUX	f20, AO,  INC4
	LFPDUX	f21, AO2, INC4
	LFPDUX	f22, AO,  INC4
	LFPDUX	f23, AO2, INC4

	subi	AO,  AO,  16 * SIZE
	subi	AO2, AO2, 16 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
	fpsub	f4,  f18,  f4
	fpsub	f5,  f19,  f5

	fpsub	f8,  f20,  f8
	fpsub	f9,  f21,  f9
	fpsub	f12, f22,  f12
	fpsub	f13, f23,  f13
#endif

#ifdef LN
       addi	AO,  AO,  20 * SIZE
       addi	AO2, AO2, 20 * SIZE

	LFPDUX	A1,  AO2, INCM4
	LFPDUX	A2,  AO,  INCM4
	LFPDUX	A3,  AO2, INCM4
	LFPDUX	A4,  AO,  INCM4

	add	AO2, AO2, INCM4
	LFPDUX	A5,  AO,  INCM4
	add	AO2, AO2, INCM4
	LFPDUX	A6,  AO,  INCM4

	addi	AO,  AO,  -4 * SIZE
	addi	AO2, AO2, -4 * SIZE

	fxsmul	f5,  A1, f5
	fxsmul	f13, A1, f13

	fxcpnmsub  f1,  A1, f5,  f1
	fxcpnmsub  f9,  A1, f13, f9

	fxcsnmsub  f4,  A2, f5,  f4
	fxcsnmsub  f12, A2, f13, f12

	fxcpnmsub  f0,  A2, f5,  f0
	fxcpnmsub  f8,  A2, f13, f8

	fxpmul	f1,  A3, f1
	fxpmul	f9,  A3, f9

	fxcsnmsub  f4,  A4, f1,  f4
	fxcsnmsub  f12, A4, f9,  f12

	fxcpnmsub  f0,  A4, f1,  f0
	fxcpnmsub  f8,  A4, f9,  f8

	fxsmul	f4,  A5, f4
	fxsmul	f12, A5, f12

	fxcpnmsub  f0,  A5, f4,  f0
	fxcpnmsub  f8,  A5, f12, f8

	fxpmul	f0,  A6, f0
	fxpmul	f8,  A6, f8
#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC4
	LFPDUX	A2,  AO2, INC4
	LFPDUX	A3,  AO,  INC4
	LFPDUX	A4,  AO2, INC4

	add	AO,  AO,  INC4
	LFPDUX	A5,  AO2, INC4
	add	AO,  AO,  INC4
	LFPDUX	A6,  AO2, INC4

	subi	AO,  AO,  16 * SIZE
	subi	AO2, AO2, 16 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f8,  A1,  f8

	fxcsnmsub  f4,  A1, f0, f4
	fxcsnmsub  f12, A1, f8, f12

	fxcpnmsub  f1,  A2, f0, f1
	fxcpnmsub  f9,  A2, f8, f9

	fxcsnmsub  f5,  A2, f0, f5
	fxcsnmsub  f13, A2, f8, f13

	fxsmul	f4,  A3,  f4
	fxsmul	f12, A3,  f12

	fxcpnmsub  f1,  A4, f4,  f1
	fxcpnmsub  f9,  A4, f12, f9

	fxcsnmsub  f5,  A4, f4,  f5
	fxcsnmsub  f13, A4, f12, f13

	fxpmul	f1,  A5,  f1
	fxpmul	f9,  A5,  f9

	fxcsnmsub  f5,  A5, f1, f5
	fxcsnmsub  f13, A5, f9, f13

	fxsmul	f5,  A6,  f5
	fxsmul	f13, A6,  f13
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC4
	LFPDUX	A2,  BO2, INC4
	LFPDUX	A3,  BO,  INC4
	LFPDUX	A4,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A5,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A6,  BO2, INC4

	subi	BO,  BO,  16 * SIZE
	subi	BO2, BO2, 16 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
	fxcsnmsub    f4,  A1, f0, f4
	fxcsnmsub    f5,  A1, f1, f5

	fxcpnmsub    f8,  A2, f0, f8
	fxcpnmsub    f9,  A2, f1, f9
	fxcsnmsub    f12, A2, f0, f12
	fxcsnmsub    f13, A2, f1, f13

	fxsmul	f4,  A3,  f4
	fxsmul	f5,  A3,  f5
	fxcpnmsub    f8,  A4, f4, f8
	fxcpnmsub    f9,  A4, f5, f9

	fxcsnmsub    f12, A4, f4, f12
	fxcsnmsub    f13, A4, f5, f13

	fxpmul	f8,  A5,  f8
	fxpmul	f9,  A5,  f9
	fxcsnmsub    f12, A5, f8,  f12
	fxcsnmsub    f13, A5, f9,  f13

	fxsmul	f12,  A6,  f12
	fxsmul	f13,  A6,  f13
#endif

#ifdef RT
	addi	BO,  BO,  20 * SIZE
	addi	BO2, BO2, 20 * SIZE

	LFPDUX	A1,  BO2, INCM4
	LFPDUX	A2,  BO,  INCM4

	LFPDUX	A3,  BO2, INCM4
	LFPDUX	A4,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A5,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A6,  BO,  INCM4
	subi	BO,  BO,  4 * SIZE
	subi	BO2, BO2, 4 * SIZE

	fxsmul	f12, A1,  f12
	fxsmul	f13, A1,  f13
	fxcpnmsub    f8,  A1, f12, f8
	fxcpnmsub    f9,  A1, f13, f9

	fxcsnmsub    f4,  A2, f12, f4
	fxcsnmsub    f5,  A2, f13, f5
	fxcpnmsub    f0,  A2, f12, f0
	fxcpnmsub    f1,  A2, f13, f1

	fxpmul	f8,  A3,  f8
	fxpmul	f9,  A3,  f9
	fxcsnmsub    f4,  A4, f8,  f4
	fxcsnmsub    f5,  A4, f9,  f5

	fxcpnmsub    f0,  A4, f8,  f0
	fxcpnmsub    f1,  A4, f9,  f1

	fxsmul	f4,  A5,  f4
	fxsmul	f5,  A5,  f5
	fxcpnmsub    f0,  A5, f4,  f0
	fxcpnmsub    f1,  A5, f5,  f1

	fxpmul	f0,  A6,  f0
	fxpmul	f1,  A6,  f1
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
	subi	CO2, CO2, 4 * SIZE
	subi	CO3, CO3, 4 * SIZE
	subi	CO4, CO4, 4 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC4
	STFPDUX	f8,  BO2, INC4
	STFPDUX	f4,  BO,  INC4
	STFPDUX	f12, BO2, INC4
	STFPDUX	f1,  BO,  INC4
	STFPDUX	f9,  BO2, INC4
	STFPDUX	f5,  BO,  INC4
	STFPDUX	f13, BO2, INC4

	subi	BO,  BO,  16 * SIZE
	subi	BO2, BO2, 16 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f4,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFDUX	f5,  CO1, INC

	STFSDUX	f0,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFSDUX	f1,  CO2, INC
	STFSDUX	f5,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFDUX	f12, CO3, INC
	STFDUX	f9,  CO3, INC
	STFDUX	f13, CO3, INC

	STFSDUX	f8,  CO4, INC
	STFSDUX	f12, CO4, INC
	STFSDUX	f9,  CO4, INC
	STFSDUX	f13, CO4, INC
#else
	STFPDUX	f0,  AO,  INC4
	STFPDUX	f1,  AO2, INC4
	STFPDUX	f4,  AO,  INC4
	STFPDUX	f5,  AO2, INC4
	STFPDUX	f8,  AO,  INC4
	STFPDUX	f9,  AO2, INC4
	STFPDUX	f12, AO,  INC4
	STFPDUX	f13, AO2, INC4

	subi	AO,  AO,  16 * SIZE
	subi	AO2, AO2, 16 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
	STFDUX	f4,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFDUX	f5,  CO2, INC
	STFSDUX	f5,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFSDUX	f8,  CO3, INC
	STFDUX	f9,  CO3, INC
	STFSDUX	f9,  CO3, INC
	STFDUX	f12, CO4, INC
	STFSDUX	f12, CO4, INC
	STFDUX	f13, CO4, INC
	STFSDUX	f13, CO4, INC
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
	subi	CO2, CO2, 4 * SIZE
	subi	CO3, CO3, 4 * SIZE
	subi	CO4, CO4, 4 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 2 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 2 + BASE_SHIFT
	slwi	TEMP, TEMP, 2 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 4
#endif

#ifdef LN
	subi	KK, KK, 4
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L40:
	srawi.	I, M,  3
	ble	.L49
	.align 4

.L11:
#if defined(LT) || defined(RN)
	addi	AO2, AO,   2 * SIZE
	fpmr	f4,  f0
	addi	BO,  B,  - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, B,  - 2 * SIZE
	fpmr	f12, f0

	fpmr	f5,  f0
	fpmr	f9,  f0
	fpmr	f13, f0
	fpmr	f2,  f0

	fpmr	f6,  f0
	fpmr	f10, f0
	fpmr	f14, f0
	fpmr	f3,  f0

	fpmr	f7,  f0
	fpmr	f11, f0
	fpmr	f15, f0
	nop

	srawi.	r0,  KK,  2
 	fpmr	f1,  f0
	mtspr	CTR, r0
	ble	.L14
#else

#ifdef LN
	slwi	r0,   K,  3 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 3 + BASE_SHIFT
	slwi	TEMP, KK, 2 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	AO2, AO,   2 * SIZE
	fpmr	f4,  f0
	addi	BO,  BO, - 4 * SIZE
	fpmr	f8,  f0
	addi	BO2, BO,   2 * SIZE
	fpmr	f12, f0

	fpmr	f5,  f0
	fpmr	f9,  f0
	fpmr	f13, f0
	fpmr	f2,  f0

	fpmr	f6,  f0
	fpmr	f10, f0
	fpmr	f14, f0
	fpmr	f3,  f0

	fpmr	f7,  f0
	fpmr	f11, f0
	fpmr	f15, f0
	nop

	srawi.	r0,  TEMP,  2
 	fpmr	f1,  f0
	mtspr	CTR, r0
	ble	.L14
#endif

	LFPDUX	A1,  AO, INC4
	fpmr	f5,  f0
	LFPDUX	A3,  AO, INC4
	fpmr	f9,  f0
	LFPDUX	B1,  BO, INC4
	fpmr	f13, f0

	LFPDUX	A5,  AO, INC4
	fpmr	f2,  f0
	LFPDUX	A6,  AO, INC4
	fpmr	f6,  f0
	LFPDUX	B3,  BO, INC4
	fpmr	f10, f0
	LFPDUX	A7,  AO, INC4
	fpmr	f14, f0

	LFPDUX	A8,  AO, INC4
	fpmr	f3,  f0
	LFPDUX	B5,  BO, INC4
	fpmr	f7,  f0
	LFPDUX	A9,  AO, INC4
	fpmr	f11, f0
	LFPDUX	A2, AO2, INC4
	fpmr	f15, f0
	LFPDUX	B2, BO2, INC4
	bdz-	.L13
	.align 4

.L12:

## 1 ##
	fxcpmadd	f0,  B1, A1, f0
	nop
	fxcsmadd	f4,  B1, A1, f4
	nop
	fxcpmadd	f8,  B2, A1, f8
	LFPDUX	B4, BO2, INC4
	fxcsmadd	f12, B2, A1, f12
	LFPDUX	B6,  BO, INC4

	fxcpmadd	f1,  B1, A2, f1
	nop
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B2, A2, f9
	LFPDUX	A10, AO, INC4
	fxcsmadd	f13, B2, A2, f13
	nop

	fxcpmadd	f2,  B1, A3, f2
	nop
	fxcsmadd	f6,  B1, A3, f6
	nop
	fxcpmadd	f10, B2, A3, f10
	nop
	fxcsmadd	f14, B2, A3, f14
	nop

	fxcpmadd	f3,  B1, A4, f3
	nop
	fxcsmadd	f7,  B1, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B2, A4, f11
	LFPDUX	A1,  AO, INC4
	fxcsmadd	f15, B2, A4, f15
	nop

## 2 ##

	fxcpmadd	f0,  B3, A5, f0
	nop
	fxcsmadd	f4,  B3, A5, f4
	nop
	fxcpmadd	f8,  B4, A5, f8
	LFPDUX	B2, BO2, INC4
	fxcsmadd	f12, B4, A5, f12
	LFPDUX	B1,  BO, INC4

	fxcpmadd	f1,  B3, A2, f1
	nop
	fxcsmadd	f5,  B3, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B4, A2, f9
	LFPDUX	A3,  AO, INC4
	fxcsmadd	f13, B4, A2, f13
	nop

	fxcpmadd	f2,  B3, A6, f2
	nop
	fxcsmadd	f6,  B3, A6, f6
	nop
	fxcpmadd	f10, B4, A6, f10
	nop
	fxcsmadd	f14, B4, A6, f14
	nop

	fxcpmadd	f3,  B3, A4, f3
	nop
	fxcsmadd	f7,  B3, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B4, A4, f11
	LFPDUX	A5,  AO, INC4
	fxcsmadd	f15, B4, A4, f15
	nop

## 3 ##

	fxcpmadd	f0,  B5, A7, f0
	nop
	fxcsmadd	f4,  B5, A7, f4
	nop
	fxcpmadd	f8,  B2, A7, f8
	LFPDUX	B4, BO2, INC4
	fxcsmadd	f12, B2, A7, f12
	LFPDUX	B3,  BO, INC4

	fxcpmadd	f1,  B5, A2, f1
	nop
	fxcsmadd	f5,  B5, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B2, A2, f9
	LFPDUX	A6,  AO, INC4
	fxcsmadd	f13, B2, A2, f13
	nop

	fxcpmadd	f2,  B5, A8, f2
	nop
	fxcsmadd	f6,  B5, A8, f6
	nop
	fxcpmadd	f10, B2, A8, f10
	nop
	fxcsmadd	f14, B2, A8, f14
	nop

	fxcpmadd	f3,  B5, A4, f3
	nop
	fxcsmadd	f7,  B5, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B2, A4, f11
	LFPDUX	A7,  AO, INC4
	fxcsmadd	f15, B2, A4, f15
	nop

## 4 ##
	fxcpmadd	f0,  B6, A9, f0
	nop
	fxcsmadd	f4,  B6, A9, f4
	nop
	fxcpmadd	f8,  B4, A9, f8
	LFPDUX	B2, BO2, INC4
	fxcsmadd	f12, B4, A9, f12
	LFPDUX	B5,  BO, INC4

	fxcpmadd	f1,  B6, A2, f1
	nop
	fxcsmadd	f5,  B6, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B4, A2, f9
	LFPDUX	A8,  AO, INC4
	fxcsmadd	f13, B4, A2, f13
	nop

	fxcpmadd	f2,  B6, A10, f2
	nop
	fxcsmadd	f6,  B6, A10, f6
	nop
	fxcpmadd	f10, B4, A10, f10
	nop
	fxcsmadd	f14, B4, A10, f14
	nop

	fxcpmadd	f3,  B6, A4, f3
	LFPDUX	A2, AO2, INC4
	fxcsmadd	f7,  B6, A4, f7
	LFPDUX	A9,  AO, INC4
	fxcpmadd	f11, B4, A4, f11
	nop
	fxcsmadd	f15, B4, A4, f15
	bdnz+	.L12
	.align 4

.L13:
## 1 ##

	fxcpmadd	f0,  B1, A1, f0
	nop
	fxcsmadd	f4,  B1, A1, f4
	nop
	fxcpmadd	f8,  B2, A1, f8
	LFPDUX	B4, BO2, INC4
	fxcsmadd	f12, B2, A1, f12
	LFPDUX	B6,  BO, INC4

	fxcpmadd	f1,  B1, A2, f1
	nop
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B2, A2, f9
	LFPDUX	A10, AO, INC4
	fxcsmadd	f13, B2, A2, f13
	nop

	fxcpmadd	f2,  B1, A3, f2
	nop
	fxcsmadd	f6,  B1, A3, f6
	nop
	fxcpmadd	f10, B2, A3, f10
	nop
	fxcsmadd	f14, B2, A3, f14
	nop

	fxcpmadd	f3,  B1, A4, f3
	nop
	fxcsmadd	f7,  B1, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B2, A4, f11
	nop
	fxcsmadd	f15, B2, A4, f15
	nop

## 2 ##

	fxcpmadd	f0,  B3, A5, f0
	nop
	fxcsmadd	f4,  B3, A5, f4
	nop
	fxcpmadd	f8,  B4, A5, f8
	LFPDUX	B2, BO2, INC4
	fxcsmadd	f12, B4, A5, f12
	nop

	fxcpmadd	f1,  B3, A2, f1
	nop
	fxcsmadd	f5,  B3, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B4, A2, f9
	nop
	fxcsmadd	f13, B4, A2, f13
	nop

	fxcpmadd	f2,  B3, A6, f2
	nop
	fxcsmadd	f6,  B3, A6, f6
	nop
	fxcpmadd	f10, B4, A6, f10
	nop
	fxcsmadd	f14, B4, A6, f14
	nop

	fxcpmadd	f3,  B3, A4, f3
	nop
	fxcsmadd	f7,  B3, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B4, A4, f11
	nop
	fxcsmadd	f15, B4, A4, f15
	nop

## 3 ##

	fxcpmadd	f0,  B5, A7, f0
	nop
	fxcsmadd	f4,  B5, A7, f4
	nop
	fxcpmadd	f8,  B2, A7, f8
	LFPDUX	B4, BO2, INC4
	fxcsmadd	f12, B2, A7, f12
	nop

	fxcpmadd	f1,  B5, A2, f1
	nop
	fxcsmadd	f5,  B5, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B2, A2, f9
	nop

	fxcsmadd	f13, B2, A2, f13

	fxcpmadd	f2,  B5, A8, f2
	nop
	fxcsmadd	f6,  B5, A8, f6
	nop
	fxcpmadd	f10, B2, A8, f10
	nop
	fxcsmadd	f14, B2, A8, f14
	nop

	fxcpmadd	f3,  B5, A4, f3
	nop
	fxcsmadd	f7,  B5, A4, f7
	LFPDUX	A2, AO2, INC4
	fxcpmadd	f11, B2, A4, f11
	nop
	fxcsmadd	f15, B2, A4, f15
	nop

## 4 ##

	fxcpmadd	f0,  B6, A9, f0
	nop
	fxcsmadd	f4,  B6, A9, f4
	nop
	fxcpmadd	f8,  B4, A9, f8
	nop
	fxcsmadd	f12, B4, A9, f12
	nop

	fxcpmadd	f1,  B6, A2, f1
	nop
	fxcsmadd	f5,  B6, A2, f5
	LFPDUX	A4, AO2, INC4
	fxcpmadd	f9,  B4, A2, f9
	nop
	fxcsmadd	f13, B4, A2, f13
	nop

	fxcpmadd	f2,  B6, A10, f2
	nop
	fxcsmadd	f6,  B6, A10, f6
	nop
	fxcpmadd	f10, B4, A10, f10
	nop
	fxcsmadd	f14, B4, A10, f14
	nop

	fxcpmadd	f3,  B6, A4, f3
	nop
	fxcsmadd	f7,  B6, A4, f7
	nop
	fxcpmadd	f11, B4, A4, f11
	nop
	fxcsmadd	f15, B4, A4, f15
	nop
	.align 4

.L14:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L18
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L18
#endif
	.align 4

.L15:
	LFPDUX	A2,  AO,  INC4
	LFPDUX	A4,  AO2, INC4
	LFPDUX	A10, BO,  INC4
	LFPDUX	B4,  BO2, INC4
	bdz-	.L17
	.align 4

.L16:
	fxcpmadd	f0,  A10, A2, f0
	fxcsmadd	f4,  A10, A2, f4
	fxcpmadd	f8,  B4, A2, f8
	fxcsmadd	f12, B4, A2, f12
	LFPDUX	A2, AO,  INC4

	fxcpmadd	f1,  A10, A4, f1
	fxcsmadd	f5,  A10, A4, f5
	fxcpmadd	f9,  B4, A4, f9
	fxcsmadd	f13, B4, A4, f13
	LFPDUX	A4, AO2, INC4

	fxcpmadd	f2,  A10, A2, f2
	fxcsmadd	f6,  A10, A2, f6
	fxcpmadd	f10, B4, A2, f10
	fxcsmadd	f14, B4, A2, f14
	LFPDUX	A2, AO,  INC4

	fxcpmadd	f3,  A10, A4, f3
	fxcsmadd	f7,  A10, A4, f7
	LFPDUX	A10, BO,  INC4
	fxcpmadd	f11, B4, A4, f11
	fxcsmadd	f15, B4, A4, f15
	LFPDUX	A4, AO2, INC4
	LFPDUX	B4, BO2, INC4
	bdnz+	.L16
	.align 4

.L17:
	fxcpmadd	f0,  A10, A2, f0
	fxcsmadd	f4,  A10, A2, f4
	fxcpmadd	f8,  B4, A2, f8
	fxcsmadd	f12, B4, A2, f12
	LFPDUX	A2, AO,  INC4

	fxcpmadd	f1,  A10, A4, f1
	fxcsmadd	f5,  A10, A4, f5
	fxcpmadd	f9,  B4, A4, f9
	fxcsmadd	f13, B4, A4, f13
	LFPDUX	A4, AO2, INC4

	fxcpmadd	f2,  A10, A2, f2
	fxcsmadd	f6,  A10, A2, f6
	fxcpmadd	f10, B4, A2, f10
	fxcsmadd	f14, B4, A2, f14

	fxcpmadd	f3,  A10, A4, f3
	fxcsmadd	f7,  A10, A4, f7
	fxcpmadd	f11, B4, A4, f11
	fxcsmadd	f15, B4, A4, f15
	.align 4

.L18:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 8
#else
	subi	r0, KK, 4
#endif
	slwi	TEMP, r0, 3 + BASE_SHIFT
	slwi	r0,   r0, 2 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO, B,     r0
	addi	AO2, AO,   2 * SIZE
	addi	BO,  BO, - 4 * SIZE
	addi	BO2, BO,   2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	LFPDUX	f16, BO,  INC4
	fpmr	f25, f1
	nop
	fpmr	f26, f2
	LFPDUX	f17, BO2, INC4
	fpmr	f27, f3
	nop

	fpmr	f28, f8
	LFPDUX	f18, BO,  INC4
	fpmr	f29, f9
	nop
	fpmr	f30, f10
	LFPDUX	f19, BO2, INC4
	fpmr	f31, f11
	nop

	fsmfp	f0,  f4
 	LFPDUX	f20, BO,  INC4
	fsmfp	f1,  f5
	nop
	fsmfp	f2,  f6
	LFPDUX	f21, BO2, INC4
	fsmfp	f3,  f7
	nop

	fsmfp	f8,  f12
	LFPDUX	f22, BO,  INC4
	fsmfp	f9,  f13
	nop
	fsmfp	f10, f14
	LFPDUX	f23, BO2, INC4
	fsmfp	f11, f15
	nop

	fsmtp	f4,  f24
	LFPDUX	f24, BO,  INC4
	fsmtp	f5,  f25
	nop
	fsmtp	f6,  f26
	LFPDUX	f25, BO2, INC4
	fsmtp	f7,  f27
	nop

	fsmtp	f12, f28
	LFPDUX	f26, BO,  INC4
	fsmtp	f13, f29
	nop
	fsmtp	f14, f30
	LFPDUX	f27, BO2, INC4
	fsmtp	f15, f31
	nop

	fpsub	f0,  f16,  f0
	LFPDUX	f28, BO,  INC4
	fpsub	f8,  f17,  f8
	nop
	fpsub	f4,  f18,  f4
	LFPDUX	f29, BO2, INC4
	fpsub	f12, f19,  f12
	nop

	fpsub	f1,  f20,  f1
	LFPDUX	f30, BO,  INC4
	fpsub	f9,  f21,  f9
	subi	BO,  BO,  32 * SIZE
	fpsub	f5,  f22,  f5
	LFPDUX	f31, BO2, INC4
	fpsub	f13, f23,  f13
	subi	BO2, BO2, 32 * SIZE

	fpsub	f2,  f24,  f2
	fpsub	f10, f25,  f10
	fpsub	f6,  f26,  f6
	fpsub	f14, f27,  f14
	fpsub	f3,  f28,  f3
	fpsub	f11, f29,  f11
	fpsub	f7,  f30,  f7
	fpsub	f15, f31,  f15

#else
	LFPDUX	f16, AO,  INC4
	LFPDUX	f17, AO2, INC4
	LFPDUX	f18, AO,  INC4
	LFPDUX	f19, AO2, INC4
	LFPDUX	f20, AO,  INC4
	LFPDUX	f21, AO2, INC4
	LFPDUX	f22, AO,  INC4
	LFPDUX	f23, AO2, INC4

	fpsub	f0,  f16,  f0
	LFPDUX	f24, AO,  INC4
	fpsub	f1,  f17,  f1
	LFPDUX	f25, AO2, INC4
	fpsub	f2,  f18,  f2
	LFPDUX	f26, AO,  INC4
	fpsub	f3,  f19,  f3
	LFPDUX	f27, AO2, INC4
	fpsub	f4,  f20,  f4
	LFPDUX	f28, AO,  INC4
	fpsub	f5,  f21,  f5
	LFPDUX	f29, AO2, INC4
	fpsub	f6,  f22,  f6
	LFPDUX	f30, AO,  INC4
	fpsub	f7,  f23,  f7
	LFPDUX	f31, AO2, INC4

	fpsub	f8,  f24,  f8
	subi	AO,  AO,  32 * SIZE
	fpsub	f9,  f25,  f9
	subi	AO2, AO2, 32 * SIZE
	fpsub	f10, f26,  f10
	fpsub	f11, f27,  f11
	fpsub	f12, f28,  f12
	fpsub	f13, f29,  f13
	fpsub	f14, f30,  f14
	fpsub	f15, f31,  f15
#endif

#ifdef LN
       addi	AO,  AO,  68 * SIZE
       addi	AO2, AO2, 68 * SIZE

	LFPDUX	A1,  AO2, INCM4
	LFPDUX	A2,  AO,  INCM4
	LFPDUX	A3,  AO2, INCM4
	LFPDUX	A4,  AO,  INCM4
	LFPDUX	A5,  AO2, INCM4
	LFPDUX	A6,  AO,  INCM4
	LFPDUX	A7,  AO2, INCM4
	LFPDUX	A8,  AO,  INCM4

	fxsmul	f7,  A1, f7
	fxsmul	f15, A1, f15

	fxcpnmsub  f3,  A1, f7,  f3
	fxcpnmsub  f11, A1, f15, f11

	fxcsnmsub  f6,  A2, f7,  f6
	fxcsnmsub  f14, A2, f15, f14

	fxcpnmsub  f2,  A2, f7,  f2
	fxcpnmsub  f10, A2, f15, f10

	fxcsnmsub  f5,  A3, f7,  f5
	fxcsnmsub  f13, A3, f15, f13

	fxcpnmsub  f1,  A3, f7,  f1
	fxcpnmsub  f9,  A3, f15, f9

	fxcsnmsub  f4,  A4, f7,  f4
	fxcsnmsub  f12, A4, f15, f12

	fxcpnmsub  f0,  A4, f7,  f0
	fxcpnmsub  f8,  A4, f15, f8

	fxpmul	f3,  A5, f3
	fxpmul	f11, A5, f11

	fxcsnmsub  f6,  A6, f3,  f6
	fxcsnmsub  f14, A6, f11, f14

	fxcpnmsub  f2,  A6, f3,  f2
	fxcpnmsub  f10, A6, f11, f10

	fxcsnmsub  f5,  A7, f3,  f5
	fxcsnmsub  f13, A7, f11, f13

	fxcpnmsub  f1,  A7, f3,  f1
	fxcpnmsub  f9,  A7, f11, f9

	fxcsnmsub  f4,  A8, f3,  f4
	fxcsnmsub  f12, A8, f11, f12

	fxcpnmsub  f0,  A8, f3,  f0
	fxcpnmsub  f8,  A8, f11, f8

	add	AO2, AO2, INCM4
	LFPDUX	A1,  AO,  INCM4
	LFPDUX	A2,  AO2, INCM4
	LFPDUX	A3,  AO,  INCM4

	add	AO2, AO2, INCM4
	LFPDUX	A4,  AO,  INCM4
	LFPDUX	A5,  AO2, INCM4
	LFPDUX	A6,  AO,  INCM4

	add	AO2, AO2, INCM4
	add	AO,  AO,  INCM4
	LFPDUX	A7,  AO2, INCM4
	LFPDUX	A8,  AO,  INCM4


	fxsmul	f6,  A1, f6
	fxsmul	f14, A1, f14

	fxcpnmsub  f2,  A1, f6,  f2
	fxcpnmsub  f10, A1, f14, f10

	fxcsnmsub  f5,  A2, f6,  f5
	fxcsnmsub  f13, A2, f14, f13

	fxcpnmsub  f1,  A2, f6,  f1
	fxcpnmsub  f9,  A2, f14, f9

	fxcsnmsub  f4,  A3, f6,  f4
	fxcsnmsub  f12, A3, f14, f12

	fxcpnmsub  f0,  A3, f6,  f0
	fxcpnmsub  f8,  A3, f14, f8

	fxpmul	f2,  A4, f2
	fxpmul	f10, A4, f10

	fxcsnmsub  f5,  A5, f2,  f5
	fxcsnmsub  f13, A5, f10, f13

	fxcpnmsub  f1,  A5, f2,  f1
	fxcpnmsub  f9,  A5, f10, f9

	fxcsnmsub  f4,  A6, f2,  f4
	fxcsnmsub  f12, A6, f10, f12

	fxcpnmsub  f0,  A6, f2,  f0
	fxcpnmsub  f8,  A6, f10, f8

	fxsmul	f5,  A7, f5
	fxsmul	f13, A7, f13

	fxcpnmsub  f1,  A7, f5,  f1
	fxcpnmsub  f9,  A7, f13, f9

	fxcsnmsub  f4,  A8, f5,  f4
	fxcsnmsub  f12, A8, f13, f12

	fxcpnmsub  f0,  A8, f5,  f0
	fxcpnmsub  f8,  A8, f13, f8

	add	AO2, AO2, INCM4
	add	AO,  AO,  INCM4
	LFPDUX	A1,  AO2, INCM4
	LFPDUX	A2,  AO,  INCM4

	subi	AO2, AO2, 8 * SIZE
	add	AO,  AO,  INCM4
	LFPDUX	A3,  AO,  INCM4

	subi	AO2, AO2, 8 * SIZE
	add	AO,  AO,  INCM4
	LFPDUX	A4,  AO,  INCM4

	addi	AO,  AO,  -4 * SIZE
	addi	AO2, AO2, -4 * SIZE

	fxpmul	f1,  A1, f1
	fxpmul	f9,  A1, f9

	fxcsnmsub  f4,  A2, f1,  f4
	fxcsnmsub  f12, A2, f9,  f12

	fxcpnmsub  f0,  A2, f1,  f0
	fxcpnmsub  f8,  A2, f9,  f8

	fxsmul	f4,  A3, f4
	fxsmul	f12, A3, f12

	fxcpnmsub  f0,  A3, f4,  f0
	fxcpnmsub  f8,  A3, f12, f8

	fxpmul	f0,  A4, f0
	fxpmul	f8,  A4, f8

#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC4
	LFPDUX	A2,  AO2, INC4
	LFPDUX	A3,  AO,  INC4
	LFPDUX	A4,  AO2, INC4

	LFPDUX	A5,  AO,  INC4
	LFPDUX	A6,  AO2, INC4
	LFPDUX	A7,  AO,  INC4
	LFPDUX	A8,  AO2, INC4

	fxpmul	f0,  A1,  f0
	fxpmul	f8,  A1,  f8

	fxcsnmsub  f4,  A1, f0, f4
	fxcsnmsub  f12, A1, f8, f12

	fxcpnmsub  f1,  A2, f0, f1
	fxcpnmsub  f9,  A2, f8, f9

	fxcsnmsub  f5,  A2, f0, f5
	fxcsnmsub  f13, A2, f8, f13

	fxcpnmsub  f2,  A3, f0, f2
	fxcpnmsub  f10, A3, f8, f10

	fxcsnmsub  f6,  A3, f0, f6
	fxcsnmsub  f14, A3, f8, f14

	fxcpnmsub  f3,  A4, f0, f3
	fxcpnmsub  f11, A4, f8, f11

	fxcsnmsub  f7,  A4, f0, f7
	fxcsnmsub  f15, A4, f8, f15

	fxsmul	f4,  A5,  f4
	fxsmul	f12, A5,  f12

	fxcpnmsub  f1,  A6, f4,  f1
	fxcpnmsub  f9,  A6, f12, f9

	fxcsnmsub  f5,  A6, f4,  f5
	fxcsnmsub  f13, A6, f12, f13

	fxcpnmsub  f2,  A7, f4,  f2
	fxcpnmsub  f10, A7, f12, f10

	fxcsnmsub  f6,  A7, f4,  f6
	fxcsnmsub  f14, A7, f12, f14

	fxcpnmsub  f3,  A8, f4,  f3
	fxcpnmsub  f11, A8, f12, f11

	fxcsnmsub  f7,  A8, f4,  f7
	fxcsnmsub  f15, A8, f12, f15

	add	AO,  AO,  INC4
	LFPDUX	A1,  AO2, INC4
	LFPDUX	A2,  AO,  INC4
	LFPDUX	A3,  AO2, INC4

	add	AO,  AO,  INC4
	LFPDUX	A4,  AO2, INC4
	LFPDUX	A5,  AO,  INC4
	LFPDUX	A6,  AO2, INC4

	add	AO,  AO,  INC4
	add	AO2, AO2, INC4
	LFPDUX	A7,  AO,  INC4
	LFPDUX	A8,  AO2, INC4

	fxpmul	f1,  A1,  f1
	fxpmul	f9,  A1,  f9

	fxcsnmsub  f5,  A1, f1, f5
	fxcsnmsub  f13, A1, f9, f13

	fxcpnmsub  f2,  A2, f1, f2
	fxcpnmsub  f10, A2, f9, f10

	fxcsnmsub  f6,  A2, f1, f6
	fxcsnmsub  f14, A2, f9, f14

	fxcpnmsub  f3,  A3, f1, f3
	fxcpnmsub  f11, A3, f9, f11

	fxcsnmsub  f7,  A3, f1, f7
	fxcsnmsub  f15, A3, f9, f15

	fxsmul	f5,  A4,  f5
	fxsmul	f13, A4,  f13

	fxcpnmsub  f2,  A5, f5,  f2
	fxcpnmsub  f10, A5, f13, f10

	fxcsnmsub  f6,  A5, f5,  f6
	fxcsnmsub  f14, A5, f13, f14

	fxcpnmsub  f3,  A6, f5,  f3
	fxcpnmsub  f11, A6, f13, f11

	fxcsnmsub  f7,  A6, f5,  f7
	fxcsnmsub  f15, A6, f13, f15

	fxpmul	f2,  A7,  f2
	fxpmul	f10, A7,  f10

	fxcsnmsub  f6,  A7, f2,  f6
	fxcsnmsub  f14, A7, f10, f14

	fxcpnmsub  f3,  A8, f2,  f3
	fxcpnmsub  f11, A8, f10, f11

	fxcsnmsub  f7,  A8, f2,  f7
	fxcsnmsub  f15, A8, f10, f15

	add	AO,  AO,  INC4
	add	AO2, AO2, INC4
	LFPDUX	A1,  AO,  INC4
	LFPDUX	A2,  AO2, INC4

	addi	AO,  AO,  8 * SIZE
	addi	AO2, AO2, 4 * SIZE
	LFPDUX	A3,  AO2, INC4

	addi	AO,  AO,  8 * SIZE
	addi	AO2, AO2, 4 * SIZE
	LFPDUX	A4,  AO2, INC4

	subi	AO,  AO,  64 * SIZE
	subi	AO2, AO2, 64 * SIZE

	fxsmul	f6,  A1,  f6
	fxsmul	f14, A1,  f14

	fxcpnmsub  f3,  A2, f6,  f3
	fxcpnmsub  f11, A2, f14, f11

	fxcsnmsub  f7,  A2, f6,  f7
	fxcsnmsub  f15, A2, f14, f15

	fxpmul	f3,  A3,  f3
	fxpmul	f11, A3,  f11

	fxcsnmsub  f7,  A3, f3,  f7
	fxcsnmsub  f15, A3, f11, f15

	fxsmul	f7,  A4,  f7
	fxsmul	f15, A4,  f15
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC4
	LFPDUX	A2,  BO2, INC4
	LFPDUX	A3,  BO,  INC4
	LFPDUX	A4,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A5,  BO2, INC4

	add	BO,  BO,  INC4
	LFPDUX	A6,  BO2, INC4
	subi	BO,  BO,  16 * SIZE
	subi	BO2, BO2, 16 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
	fxpmul	f2,  A1,  f2
	fxpmul	f3,  A1,  f3

	fxcsnmsub    f4,  A1, f0, f4
	fxcsnmsub    f5,  A1, f1, f5
	fxcsnmsub    f6,  A1, f2, f6
	fxcsnmsub    f7,  A1, f3, f7

	fxcpnmsub    f8,  A2, f0, f8
	fxcpnmsub    f9,  A2, f1, f9
	fxcpnmsub    f10, A2, f2, f10
	fxcpnmsub    f11, A2, f3, f11

	fxcsnmsub    f12, A2, f0, f12
	fxcsnmsub    f13, A2, f1, f13
	fxcsnmsub    f14, A2, f2, f14
	fxcsnmsub    f15, A2, f3, f15

	fxsmul	f4,  A3,  f4
	fxsmul	f5,  A3,  f5
	fxsmul	f6,  A3,  f6
	fxsmul	f7,  A3,  f7

	fxcpnmsub    f8,  A4, f4, f8
	fxcpnmsub    f9,  A4, f5, f9
	fxcpnmsub    f10, A4, f6, f10
	fxcpnmsub    f11, A4, f7, f11

	fxcsnmsub    f12, A4, f4, f12
	fxcsnmsub    f13, A4, f5, f13
	fxcsnmsub    f14, A4, f6, f14
	fxcsnmsub    f15, A4, f7, f15

	fxpmul	f8,  A5,  f8
	fxpmul	f9,  A5,  f9
	fxpmul	f10, A5,  f10
	fxpmul	f11, A5,  f11

	fxcsnmsub    f12, A5, f8,  f12
	fxcsnmsub    f13, A5, f9,  f13
	fxcsnmsub    f14, A5, f10, f14
	fxcsnmsub    f15, A5, f11, f15

	fxsmul	f12,  A6,  f12
	fxsmul	f13,  A6,  f13
	fxsmul	f14,  A6,  f14
	fxsmul	f15,  A6,  f15

#endif

#ifdef RT
	addi	BO,  BO,  20 * SIZE
	addi	BO2, BO2, 20 * SIZE

	LFPDUX	A1,  BO2, INCM4
	LFPDUX	A2,  BO,  INCM4

	LFPDUX	A3,  BO2, INCM4
	LFPDUX	A4,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A5,  BO,  INCM4

	add	BO2, BO2, INCM4
	LFPDUX	A6,  BO,  INCM4
	subi	BO,  BO,  4 * SIZE
	subi	BO2, BO2, 4 * SIZE

	fxsmul	f12, A1,  f12
	fxsmul	f13, A1,  f13
	fxsmul	f14, A1,  f14
	fxsmul	f15, A1,  f15

	fxcpnmsub    f8,  A1, f12, f8
	fxcpnmsub    f9,  A1, f13, f9
	fxcpnmsub    f10, A1, f14, f10
	fxcpnmsub    f11, A1, f15, f11

	fxcsnmsub    f4,  A2, f12, f4
	fxcsnmsub    f5,  A2, f13, f5
	fxcsnmsub    f6,  A2, f14, f6
	fxcsnmsub    f7,  A2, f15, f7

	fxcpnmsub    f0,  A2, f12, f0
	fxcpnmsub    f1,  A2, f13, f1
	fxcpnmsub    f2,  A2, f14, f2
	fxcpnmsub    f3,  A2, f15, f3

	fxpmul	f8,  A3,  f8
	fxpmul	f9,  A3,  f9
	fxpmul	f10, A3,  f10
	fxpmul	f11, A3,  f11

	fxcsnmsub    f4,  A4, f8,  f4
	fxcsnmsub    f5,  A4, f9,  f5
	fxcsnmsub    f6,  A4, f10, f6
	fxcsnmsub    f7,  A4, f11, f7

	fxcpnmsub    f0,  A4, f8,  f0
	fxcpnmsub    f1,  A4, f9,  f1
	fxcpnmsub    f2,  A4, f10, f2
	fxcpnmsub    f3,  A4, f11, f3

	fxsmul	f4,  A5,  f4
	fxsmul	f5,  A5,  f5
	fxsmul	f6,  A5,  f6
	fxsmul	f7,  A5,  f7

	fxcpnmsub    f0,  A5, f4,  f0
	fxcpnmsub    f1,  A5, f5,  f1
	fxcpnmsub    f2,  A5, f6,  f2
	fxcpnmsub    f3,  A5, f7,  f3

	fxpmul	f0,  A6,  f0
	fxpmul	f1,  A6,  f1
	fxpmul	f2,  A6,  f2
	fxpmul	f3,  A6,  f3

#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
	subi	CO2, CO2, 8 * SIZE
	subi	CO3, CO3, 8 * SIZE
	subi	CO4, CO4, 8 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC4
	STFPDUX	f8,  BO2, INC4
	STFPDUX	f4,  BO,  INC4
	STFPDUX	f12, BO2, INC4
	STFPDUX	f1,  BO,  INC4
	STFPDUX	f9,  BO2, INC4
	STFPDUX	f5,  BO,  INC4
	STFPDUX	f13, BO2, INC4
	STFPDUX	f2,  BO,  INC4
	STFPDUX	f10, BO2, INC4
	STFPDUX	f6,  BO,  INC4
	STFPDUX	f14, BO2, INC4
	STFPDUX	f3,  BO,  INC4
	STFPDUX	f11, BO2, INC4
	STFPDUX	f7,  BO,  INC4
	STFPDUX	f15, BO2, INC4

	subi	BO,  BO,  32 * SIZE
	subi	BO2, BO2, 32 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f4,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFDUX	f5,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFDUX	f6,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFDUX	f7,  CO1, INC

	STFSDUX	f0,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFSDUX	f1,  CO2, INC
	STFSDUX	f5,  CO2, INC
	STFSDUX	f2,  CO2, INC
	STFSDUX	f6,  CO2, INC
	STFSDUX	f3,  CO2, INC
	STFSDUX	f7,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFDUX	f12, CO3, INC
	STFDUX	f9,  CO3, INC
	STFDUX	f13, CO3, INC
	STFDUX	f10, CO3, INC
	STFDUX	f14, CO3, INC
	STFDUX	f11, CO3, INC
	STFDUX	f15, CO3, INC

	STFSDUX	f8,  CO4, INC
	STFSDUX	f12, CO4, INC
	STFSDUX	f9,  CO4, INC
	STFSDUX	f13, CO4, INC
	STFSDUX	f10, CO4, INC
	STFSDUX	f14, CO4, INC
	STFSDUX	f11, CO4, INC
	STFSDUX	f15, CO4, INC

#else
	STFPDUX	f0,  AO,  INC4
	STFPDUX	f1,  AO2, INC4
	STFPDUX	f2,  AO,  INC4
	STFPDUX	f3,  AO2, INC4
	STFPDUX	f4,  AO,  INC4
	STFPDUX	f5,  AO2, INC4
	STFPDUX	f6,  AO,  INC4
	STFPDUX	f7,  AO2, INC4
	STFPDUX	f8,  AO,  INC4
	STFPDUX	f9,  AO2, INC4
	STFPDUX	f10, AO,  INC4
	STFPDUX	f11, AO2, INC4
	STFPDUX	f12, AO,  INC4
	STFPDUX	f13, AO2, INC4
	STFPDUX	f14, AO,  INC4
	STFPDUX	f15, AO2, INC4

	subi	AO,  AO,  32 * SIZE
	subi	AO2, AO2, 32 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFSDUX	f2,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFSDUX	f3,  CO1, INC

	STFDUX	f4,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFDUX	f5,  CO2, INC
	STFSDUX	f5,  CO2, INC
	STFDUX	f6,  CO2, INC
	STFSDUX	f6,  CO2, INC
	STFDUX	f7,  CO2, INC
	STFSDUX	f7,  CO2, INC

	STFDUX	f8,  CO3, INC
	STFSDUX	f8,  CO3, INC
	STFDUX	f9,  CO3, INC
	STFSDUX	f9,  CO3, INC
	STFDUX	f10, CO3, INC
	STFSDUX	f10, CO3, INC
	STFDUX	f11, CO3, INC
	STFSDUX	f11, CO3, INC

	STFDUX	f12, CO4, INC
	STFSDUX	f12, CO4, INC
	STFDUX	f13, CO4, INC
	STFSDUX	f13, CO4, INC
	STFDUX	f14, CO4, INC
	STFSDUX	f14, CO4, INC
	STFDUX	f15, CO4, INC
	STFSDUX	f15, CO4, INC
#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
	subi	CO2, CO2, 8 * SIZE
	subi	CO3, CO3, 8 * SIZE
	subi	CO4, CO4, 8 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 3 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 3 + BASE_SHIFT
	slwi	TEMP, TEMP, 2 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 8
#endif

#ifdef LN
	subi	KK, KK, 8
#endif

	addic.	I, I, -1
	li	r0, FZERO

	lfpsx	f0, SP, r0
	bgt+	.L11
	.align 4

.L49:
#ifdef LN
	slwi	r0, K, 2 + BASE_SHIFT
	add	B, B, r0
#endif

#if defined(LT) || defined(RN)
	addi	B,  BO, 4 * SIZE
#endif

#ifdef RN
	addi	KK, KK, 4
#endif

#ifdef RT
	subi	KK, KK, 4
#endif

	addic.	J, J, -1
	bgt+	.L10
	.align 4

.L50:
	andi.	J, N,  2
	beq	.L90

#ifdef RT
	slwi	r0, K, 1 + BASE_SHIFT
	sub	B, B, r0

	slwi	r0, LDC, 1
	sub	C, C, r0
#endif

	mr	CO1, C
	add	CO2, C,   LDC

#ifdef LN
	add	KK, M, OFFSET
#endif

#ifdef LT
	mr	KK, OFFSET
#endif


#if defined(LN) || defined(RT)
	addi	AORIG, A, -2 * SIZE
#else
	addi	AO, A, -2 * SIZE
#endif
#ifndef RT
	add	C,  CO2, LDC
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0

	andi.	I, M,  1
	beq	.L60

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L84
#else

#ifdef LN
	slwi	r0,   K,  0 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 0 + BASE_SHIFT
	slwi	TEMP, KK, 1 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	BO,  BO,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  TEMP,  3
	mtspr	CTR, r0
	ble	.L84

#endif

	LFPDUX	B1,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2

	LFPDUX	B2,  BO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2

	LFPDUX	B3,  BO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdz-	.L83
	.align 4

.L82:
	fxcpmadd	f0,  A1, B1, f0
	LFPDUX	B1,  BO,  INC2
	fxcsmadd	f1,  A1, B2, f1
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f2,  A2, B3, f2
	LFPDUX	B3,  BO,  INC2
	fxcsmadd	f3,  A2, B4, f3
	LFPDUX	B4,  BO,  INC2
	LFPDUX	A2,  AO,  INC2

	fxcpmadd	f0,  A3, B1, f0
	LFPDUX	B1,  BO,  INC2
	fxcsmadd	f1,  A3, B2, f1
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f2,  A4, B3, f2
	LFPDUX	B3,  BO,  INC2
	fxcsmadd	f3,  A4, B4, f3
	LFPDUX	B4,  BO,  INC2
	LFPDUX	A4,  AO,  INC2
	bdnz+	.L82
	.align 4

.L83:
	fxcpmadd	f0,  A1, B1, f0
	LFPDUX	B1,  BO,  INC2
	fxcsmadd	f1,  A1, B2, f1
	LFPDUX	B2,  BO,  INC2
	fxcpmadd	f2,  A2, B3, f2
	LFPDUX	B3,  BO,  INC2
	fxcsmadd	f3,  A2, B4, f3
	LFPDUX	B4,  BO,  INC2

	fxcpmadd	f0,  A3, B1, f0
	fxcsmadd	f1,  A3, B2, f1
	fxcpmadd	f2,  A4, B3, f2
	fxcsmadd	f3,  A4, B4, f3
	.align 4

.L84:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L88
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L88
#endif

	LFDX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	add	AO, AO, INC
	bdz-	.L87
	.align 4

.L86:
	fxcpmadd	f0,  A1, B1, f0
	LFDX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	add	AO, AO, INC
	bdnz+	.L86
	.align 4

.L87:
	fxcpmadd	f0,  A1, B1, f0
	.align 4

.L88:
	fpadd	f0, f0, f1
	fpadd	f2, f2, f3
	fpadd	f0, f0, f2

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 1
#else
	subi	r0, KK, 2
#endif
	slwi	TEMP, r0, 0 + BASE_SHIFT
	slwi	r0,   r0, 1 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFPDX	f16, BO,  INC2

	fpsub	f0,  f16,  f0
#else
	LFPDX	f16, AO,  INC2

	fpsub	f0,  f16,  f0
#endif

#ifdef LN
	LFPDX	A1,  AO,  INC2

	fxpmul	   f0,  A1, f0
#endif

#ifdef LT
	LFPDX	A1,  AO,  INC2

	fxpmul	   f0,  A1, f0
#endif

#ifdef RN
	LFD	A1,  (2 + 0) * SIZE(BO)
	LFD	A2,  (2 + 1) * SIZE(BO)
	LFD	A3,  (2 + 3) * SIZE(BO)

	fsmtp	     f1, f0

	fmul	     f0,  A1, f0
	fnmsub	     f1,  A2, f0, f1

	fmul	     f1,  A3, f1
	fsmfp	     f0, f1
#endif

#ifdef RT
	LFD	A1,  (2 + 3) * SIZE(BO)
	LFD	A2,  (2 + 2) * SIZE(BO)
	LFD	A3,  (2 + 0) * SIZE(BO)

	fsmtp	     f1, f0

	fmul	     f1,  A1, f1
	fnmsub	     f0,  A2, f1, f0

	fmul	     f0,  A3, f0
	fsmfp	     f0, f1
#endif

#ifdef LN
	subi	CO1, CO1, 1 * SIZE
	subi	CO2, CO2, 1 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDX	f0,  BO,  INC2

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO2, INC
#else
	STFPDX	f0,  AO,  INC2

	STFDUX	f0,  CO1, INC
	STFDUX	f1,  CO2, INC
#endif

#ifdef LN
	subi	CO1, CO1, 1 * SIZE
	subi	CO2, CO2, 1 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 0 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 0 + BASE_SHIFT
	slwi	TEMP, TEMP, 1 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 1
#endif

#ifdef LN
	subi	KK, KK, 1
#endif
	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L60:
	andi.	I, M,  2
	beq	.L70

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
	fpmr	f1,  f0

	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L74
#else

#ifdef LN
	slwi	r0,   K,  1 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 1 + BASE_SHIFT
	slwi	TEMP, KK, 1 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	BO,  BO,  - 2 * SIZE
	fpmr	f1,  f0

	fpmr	f2,  f0
	fpmr	f3, f0

	srawi.	r0,  TEMP,  3
	mtspr	CTR, r0
	ble	.L74
#endif

	LFPDUX	A1,  AO, INC2
	LFPDUX	B1,  BO, INC2
	LFPDUX	A2,  AO, INC2
	LFPDUX	B2,  BO, INC2
	LFPDUX	A3,  AO, INC2
	LFPDUX	B3,  BO, INC2
	LFPDUX	A4,  AO, INC2
	LFPDUX	B4,  BO, INC2

	LFPDUX	A5,  AO, INC2
	LFPDUX	B5,  BO, INC2
	LFPDUX	A6,  AO, INC2
	LFPDUX	B6,  BO, INC2
	LFPDUX	A7,  AO, INC2
	LFPDUX	A9,  BO, INC2
	LFPDUX	A8,  AO, INC2
	LFPDUX	A10, BO, INC2
	bdz-	.L73
	.align 4

.L72:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f1,  B1, A1, f1
	LFPDUX	A1,  AO, INC2
	LFPDUX	B1,  BO, INC2
	fxcpmadd	f2,  B2, A2, f2
	fxcsmadd	f3,  B2, A2, f3
	LFPDUX	A2,  AO, INC2
	LFPDUX	B2,  BO, INC2

	fxcpmadd	f0,  B3, A3, f0
	fxcsmadd	f1,  B3, A3, f1
	LFPDUX	A3,  AO, INC2
	LFPDUX	B3,  BO, INC2
	fxcpmadd	f2,  B4, A4, f2
	fxcsmadd	f3,  B4, A4, f3
	LFPDUX	A4,  AO, INC2
	LFPDUX	B4,  BO, INC2

	fxcpmadd	f0,  B5, A5, f0
	fxcsmadd	f1,  B5, A5, f1
	LFPDUX	A5,  AO, INC2
	LFPDUX	B5,  BO, INC2
	fxcpmadd	f2,  B6, A6, f2
	fxcsmadd	f3,  B6, A6, f3
	LFPDUX	A6,  AO, INC2
	LFPDUX	B6,  BO, INC2

	fxcpmadd	f0,  A9,  A7, f0
	fxcsmadd	f1,  A9,  A7, f1
	LFPDUX	A7,  AO, INC2
	LFPDUX	A9,  BO, INC2
	fxcpmadd	f2,  A10, A8, f2
	fxcsmadd	f3,  A10, A8, f3
	LFPDUX	A8,  AO, INC2
	LFPDUX	A10, BO, INC2
	bdnz+	.L72
	.align 4

.L73:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f1,  B1, A1, f1
	fxcpmadd	f2,  B2, A2, f2
	fxcsmadd	f3,  B2, A2, f3

	fxcpmadd	f0,  B3, A3, f0
	fxcsmadd	f1,  B3, A3, f1
	fxcpmadd	f2,  B4, A4, f2
	fxcsmadd	f3,  B4, A4, f3

	fxcpmadd	f0,  B5, A5, f0
	fxcsmadd	f1,  B5, A5, f1
	fxcpmadd	f2,  B6, A6, f2
	fxcsmadd	f3,  B6, A6, f3

	fxcpmadd	f0,  A9,  A7, f0
	fxcsmadd	f1,  A9,  A7, f1
	fxcpmadd	f2,  A10, A8, f2
	fxcsmadd	f3,  A10, A8, f3
	.align 4

.L74:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L78
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L78
#endif

	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	bdz-	.L77
	.align 4

.L76:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f1,  B1, A1, f1
	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	bdnz+	.L76
	.align 4

.L77:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f1,  B1, A1, f1
	.align 4

.L78:
	fpadd	f0, f0, f2
	fpadd	f1, f1, f3

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 2
#else
	subi	r0, KK, 2
#endif
	slwi	TEMP, r0, 1 + BASE_SHIFT
	slwi	r0,   r0, 1 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	fsmfp	f0,  f1
	fsmtp	f1,  f24

	LFPDUX	f16, BO,  INC2
	LFPDUX	f17, BO,  INC2

	subi	BO,  BO,   4 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#else
	LFPDUX	f16, AO,  INC2
	LFPDUX	f17, AO,  INC2

	subi	AO,  AO,   4 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#endif

#ifdef LN
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2

	addi	AO,  AO,  -4 * SIZE

	fxsmul	   f1,  A2, f1
	fxcpnmsub  f0,  A2, f1,  f0
	fxpmul	   f0,  A1, f0
#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2

	addi	AO,  AO,  -4 * SIZE

	fxpmul	   f0,  A1, f0
	fxcsnmsub  f1,  A1, f0, f1

	fxsmul	   f1,  A2,  f1
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC2
	LFPDUX	A2,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxpmul	     f0,  A1, f0
	fxcsnmsub    f1,  A1, f0, f1

	fxsmul	     f1,  A2,  f1
#endif

#ifdef RT
	LFPDUX	A2,  BO,  INC2
	LFPDUX	A1,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxsmul	     f1,  A1, f1
	fxcpnmsub    f0,  A1, f1,  f0
	fxpmul	     f0,  A2,  f0
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
	subi	CO2, CO2, 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC2
	STFPDUX	f1,  BO,  INC2

	subi	BO,  BO,   4 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC

	STFSDUX	f0,  CO2, INC
	STFSDUX	f1,  CO2, INC
#else
	STFPDUX	f0,  AO,  INC2
	STFPDUX	f1,  AO,  INC2

	subi	AO,  AO,   4 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC

	STFDUX	f1,  CO2, INC
	STFSDUX	f1,  CO2, INC
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
	subi	CO2, CO2, 2 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 1 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 1 + BASE_SHIFT
	slwi	TEMP, TEMP, 1 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 2
#endif

#ifdef LN
	subi	KK, KK, 2
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L70:
	andi.	I, M,  4
	beq	.L80

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
 	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  2
	mtspr	CTR, r0
	ble	.L64
#else

#ifdef LN
	slwi	r0,   K,  2 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 2 + BASE_SHIFT
	slwi	TEMP, KK, 1 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

 	fpmr	f1,  f0
	addi	BO,  BO,  - 2 * SIZE
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  TEMP,  2
	mtspr	CTR, r0
	ble	.L64
#endif

	LFPDUX	B1,  BO, INC2
	LFPDUX	A1,  AO, INC2
	LFPDUX	A2,  AO, INC2
	LFPDUX	B2,  BO, INC2
	LFPDUX	A3,  AO, INC2
	LFPDUX	A4,  AO, INC2

	LFPDUX	B3,  BO, INC2
	LFPDUX	A5,  AO, INC2
	LFPDUX	A6,  AO, INC2
	LFPDUX	B4,  BO, INC2
	LFPDUX	A7,  AO, INC2
	LFPDUX	A8,  AO, INC2
	bdz-	.L63
	.align 4

.L62:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f2,  B1, A1, f2
	LFPDUX	A1,  AO, INC2
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f3,  B1, A2, f3
	LFPDUX	A2,  AO, INC2
	LFPDUX	B1,  BO, INC2

	fxcpmadd	f0,  B2, A3, f0
	fxcsmadd	f2,  B2, A3, f2
	LFPDUX	A3,  AO, INC2
	fxcpmadd	f1,  B2, A4, f1
	fxcsmadd	f3,  B2, A4, f3
	LFPDUX	A4,  AO, INC2
	LFPDUX	B2,  BO, INC2

	fxcpmadd	f0,  B3, A5, f0
	fxcsmadd	f2,  B3, A5, f2
	LFPDUX	A5,  AO, INC2
	fxcpmadd	f1,  B3, A6, f1
	fxcsmadd	f3,  B3, A6, f3
	LFPDUX	A6,  AO, INC2
	LFPDUX	B3,  BO, INC2

	fxcpmadd	f0,  B4, A7, f0
	fxcsmadd	f2,  B4, A7, f2
	LFPDUX	A7,  AO, INC2
	fxcpmadd	f1,  B4, A8, f1
	fxcsmadd	f3,  B4, A8, f3
	LFPDUX	A8,  AO, INC2
	LFPDUX	B4,  BO, INC2
	bdnz+	.L62
	.align 4

.L63:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f2,  B1, A1, f2
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f3,  B1, A2, f3

	fxcpmadd	f0,  B2, A3, f0
	fxcsmadd	f2,  B2, A3, f2
	fxcpmadd	f1,  B2, A4, f1
	fxcsmadd	f3,  B2, A4, f3

	fxcpmadd	f0,  B3, A5, f0
	fxcsmadd	f2,  B3, A5, f2
	fxcpmadd	f1,  B3, A6, f1
	fxcsmadd	f3,  B3, A6, f3

	fxcpmadd	f0,  B4, A7, f0
	fxcsmadd	f2,  B4, A7, f2
	fxcpmadd	f1,  B4, A8, f1
	fxcsmadd	f3,  B4, A8, f3
	.align 4

.L64:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L68
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L68
#endif

	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	bdz-	.L67
	.align 4

.L66:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f2,  B1, A1, f2
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f3,  B1, A2, f3
	LFPDUX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	bdnz+	.L66
	.align 4

.L67:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f2,  B1, A1, f2
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f3,  B1, A2, f3
	.align 4

.L68:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 4
#else
	subi	r0, KK, 2
#endif
	slwi	TEMP, r0, 2 + BASE_SHIFT
	slwi	r0,   r0, 1 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	fpmr	f25, f1

	fsmfp	f0,  f2
	fsmfp	f1,  f3
	fsmtp	f2,  f24
	fsmtp	f3,  f25

	LFPDUX	f16, BO,  INC2
	LFPDUX	f17, BO,  INC2
	LFPDUX	f18, BO,  INC2
	LFPDUX	f19, BO,  INC2

	subi	BO,  BO,   8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f2,  f17,  f2
	fpsub	f1,  f18,  f1
	fpsub	f3,  f19,  f3
#else
	LFPDUX	f16, AO,  INC2
	LFPDUX	f17, AO,  INC2
	LFPDUX	f18, AO,  INC2
	LFPDUX	f19, AO,  INC2

	subi	AO,  AO,   8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
	fpsub	f2,  f18,  f2
	fpsub	f3,  f19,  f3
#endif

#ifdef LN
	addi	AO,  AO,  18 * SIZE

	LFPDUX	A1,  AO,  INCM2
	LFPDUX	A2,  AO,  INCM2
	LFPDUX	A3,  AO,  INCM2
	LFPDUX	A4,  AO,  INCM2
	add	AO,  AO,  INCM2
	LFPDUX	A5,  AO,  INCM2
	add	AO,  AO,  INCM2
	LFPDUX	A6,  AO,  INCM2

	subi	AO,  AO,  2 * SIZE

	fxsmul	   f3,  A1, f3
	fxcpnmsub  f1,  A1, f3,  f1
	fxcsnmsub  f2,  A2, f3,  f2
	fxcpnmsub  f0,  A2, f3,  f0

	fxpmul	   f1,  A3, f1
	fxcsnmsub  f2,  A4, f1,  f2
	fxcpnmsub  f0,  A4, f1,  f0

	fxsmul	   f2,  A5, f2
	fxcpnmsub  f0,  A5, f2,  f0

	fxpmul	   f0,  A6, f0
#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2

	add	AO,  AO,  INC2
	LFPDUX	A5,  AO,  INC2
	add	AO,  AO,  INC2
	LFPDUX	A6,  AO,  INC2

	subi	AO,  AO,  16 * SIZE

	fxpmul	   f0,  A1,  f0
	fxcsnmsub  f2,  A1, f0, f2
	fxcpnmsub  f1,  A2, f0, f1
	fxcsnmsub  f3,  A2, f0, f3

	fxsmul	   f2,  A3,  f2
	fxcpnmsub  f1,  A4,  f2,  f1
	fxcsnmsub  f3,  A4,  f2,  f3

	fxpmul	   f1,  A5,  f1
	fxcsnmsub  f3,  A5,  f1, f3

	fxsmul	   f3,  A6,  f3
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC2
	LFPDUX	A2,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1

	fxcsnmsub    f2,  A1, f0, f2
	fxcsnmsub    f3,  A1, f1, f3

	fxsmul	f2,  A2,  f2
	fxsmul	f3,  A2,  f3
#endif

#ifdef RT
	LFPDUX	A2,  BO,  INC2
	LFPDUX	A1,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxsmul	f2,  A1,  f2
	fxsmul	f3,  A1,  f3

	fxcpnmsub    f0,  A1, f2,  f0
	fxcpnmsub    f1,  A1, f3,  f1

	fxpmul	f0,  A2,  f0
	fxpmul	f1,  A2,  f1
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
	subi	CO2, CO2, 4 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC2
	STFPDUX	f2,  BO,  INC2
	STFPDUX	f1,  BO,  INC2
	STFPDUX	f3,  BO,  INC2

	subi	BO,  BO,   8 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFDUX	f3,  CO1, INC

	STFSDUX	f0,  CO2, INC
	STFSDUX	f2,  CO2, INC
	STFSDUX	f1,  CO2, INC
	STFSDUX	f3,  CO2, INC
#else
	STFPDUX	f0,  AO,  INC2
	STFPDUX	f1,  AO,  INC2
	STFPDUX	f2,  AO,  INC2
	STFPDUX	f3,  AO,  INC2

	subi	AO,  AO,   8 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC

	STFDUX	f2,  CO2, INC
	STFSDUX	f2,  CO2, INC
	STFDUX	f3,  CO2, INC
	STFSDUX	f3,  CO2, INC
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
	subi	CO2, CO2, 4 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 2 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 2 + BASE_SHIFT
	slwi	TEMP, TEMP, 1 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 4
#endif

#ifdef LN
	subi	KK, KK, 4
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L80:
	srawi.	I, M,  3
	ble	.L89
	.align 4

.L51:
#if defined(LT) || defined(RN)
	fpmr	f4,  f0
	addi	BO,   B,  - 2 * SIZE
 	fpmr	f1,  f0
	fpmr	f5,  f0
	fpmr	f2,  f0
	fpmr	f6,  f0

	srawi.	r0,  KK,  2
	fpmr	f3,  f0
	mtspr	CTR, r0
	fpmr	f7,  f0
	ble	.L54
#else

#ifdef LN
	slwi	r0,   K,  3 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 3 + BASE_SHIFT
	slwi	TEMP, KK, 1 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	fpmr	f4,  f0
	addi	BO,  BO,  - 2 * SIZE
 	fpmr	f1,  f0
	fpmr	f5,  f0
	fpmr	f2,  f0
	fpmr	f6,  f0

	srawi.	r0,  TEMP,  2
	fpmr	f3,  f0
	mtspr	CTR, r0
	fpmr	f7,  f0
	ble	.L54
#endif

	LFPDUX	B1,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2

	LFPDUX	B3,  BO,  INC2
	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2
	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2
	bdz-	.L53
	.align 4

.L52:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	B4,  BO,  INC2
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	nop
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	A2,  AO,  INC2

	fxcpmadd	f2,  B1, A3, f2
	nop
	fxcsmadd	f6,  B1, A3, f6
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	nop
	fxcsmadd	f7,  B1, A4, f7
	LFPDUX	A4,  AO,  INC2

	fxcpmadd	f0,  B2, A5, f0
	LFPDUX	B1,  BO,  INC2
	fxcsmadd	f4,  B2, A5, f4
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B2, A6, f1
	nop
	fxcsmadd	f5,  B2, A6, f5
	LFPDUX	A6,  AO,  INC2

	fxcpmadd	f2,  B2, A7, f2
	nop
	fxcsmadd	f6,  B2, A7, f6
	LFPDUX	A7,  AO,  INC2
	fxcpmadd	f3,  B2, A8, f3
	nop
	fxcsmadd	f7,  B2, A8, f7
	LFPDUX	A8,  AO,  INC2

	fxcpmadd	f0,  B3, A1, f0
	LFPDUX	B2,  BO,  INC2
	fxcsmadd	f4,  B3, A1, f4
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B3, A2, f1
	nop
	fxcsmadd	f5,  B3, A2, f5
	LFPDUX	A2,  AO,  INC2

	fxcpmadd	f2,  B3, A3, f2
	nop
	fxcsmadd	f6,  B3, A3, f6
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B3, A4, f3
	nop
	fxcsmadd	f7,  B3, A4, f7
	LFPDUX	A4,  AO,  INC2

	fxcpmadd	f0,  B4, A5, f0
	LFPDUX	B3,  BO,  INC2
	fxcsmadd	f4,  B4, A5, f4
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B4, A6, f1
	nop
	fxcsmadd	f5,  B4, A6, f5
	LFPDUX	A6,  AO,  INC2

	fxcpmadd	f2,  B4, A7, f2
	nop
	fxcsmadd	f6,  B4, A7, f6
	LFPDUX	A7,  AO,  INC2
	fxcpmadd	f3,  B4, A8, f3
	nop
	fxcsmadd	f7,  B4, A8, f7
	LFPDUX	A8,  AO,  INC2
	bdnz+	.L52
	.align 4

.L53:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	B4,  BO,  INC2
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	nop
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	A2,  AO,  INC2

	fxcpmadd	f2,  B1, A3, f2
	nop
	fxcsmadd	f6,  B1, A3, f6
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	nop
	fxcsmadd	f7,  B1, A4, f7
	LFPDUX	A4,  AO,  INC2

	fxcpmadd	f0,  B2, A5, f0
	nop
	fxcsmadd	f4,  B2, A5, f4
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B2, A6, f1
	nop
	fxcsmadd	f5,  B2, A6, f5
	LFPDUX	A6,  AO,  INC2

	fxcpmadd	f2,  B2, A7, f2
	nop
	fxcsmadd	f6,  B2, A7, f6
	LFPDUX	A7,  AO,  INC2
	fxcpmadd	f3,  B2, A8, f3
	nop
	fxcsmadd	f7,  B2, A8, f7
	LFPDUX	A8,  AO,  INC2

	fxcpmadd	f0,  B3, A1, f0
	fxcsmadd	f4,  B3, A1, f4
	fxcpmadd	f1,  B3, A2, f1
	fxcsmadd	f5,  B3, A2, f5

	fxcpmadd	f2,  B3, A3, f2
	fxcsmadd	f6,  B3, A3, f6
	fxcpmadd	f3,  B3, A4, f3
	fxcsmadd	f7,  B3, A4, f7

	fxcpmadd	f0,  B4, A5, f0
	fxcsmadd	f4,  B4, A5, f4
	fxcpmadd	f1,  B4, A6, f1
	fxcsmadd	f5,  B4, A6, f5

	fxcpmadd	f2,  B4, A7, f2
	fxcsmadd	f6,  B4, A7, f6
	fxcpmadd	f3,  B4, A8, f3
	fxcsmadd	f7,  B4, A8, f7
	.align 4

.L54:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L58
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L58
#endif

	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	bdz-	.L57
	.align 4

.L56:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f5,  B1, A2, f5
	LFPDUX	A2,  AO,  INC2

	fxcpmadd	f2,  B1, A3, f2
	fxcsmadd	f6,  B1, A3, f6
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	fxcsmadd	f7,  B1, A4, f7
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	bdnz+	.L56
	.align 4

.L57:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f4,  B1, A1, f4
	fxcpmadd	f1,  B1, A2, f1
	fxcsmadd	f5,  B1, A2, f5

	fxcpmadd	f2,  B1, A3, f2
	fxcsmadd	f6,  B1, A3, f6
	fxcpmadd	f3,  B1, A4, f3
	fxcsmadd	f7,  B1, A4, f7
	.align 4

.L58:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 8
#else
	subi	r0, KK, 2
#endif
	slwi	TEMP, r0, 3 + BASE_SHIFT
	slwi	r0,   r0, 1 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	fpmr	f24, f0
	fpmr	f25, f1
	fpmr	f26, f2
	fpmr	f27, f3

	fsmfp	f0,  f4
	fsmfp	f1,  f5
	fsmfp	f2,  f6
	fsmfp	f3,  f7

	fsmtp	f4,  f24
	fsmtp	f5,  f25
	fsmtp	f6,  f26
	fsmtp	f7,  f27

	LFPDUX	f16, BO,  INC2
	LFPDUX	f17, BO,  INC2
	LFPDUX	f18, BO,  INC2
	LFPDUX	f19, BO,  INC2

 	LFPDUX	f20, BO,  INC2
	LFPDUX	f21, BO,  INC2
	LFPDUX	f22, BO,  INC2
	LFPDUX	f23, BO,  INC2

	subi	BO,  BO,  16 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f4,  f17,  f4
	fpsub	f1,  f18,  f1
	fpsub	f5,  f19,  f5

	fpsub	f2,  f20,  f2
	fpsub	f6,  f21,  f6
	fpsub	f3,  f22,  f3
	fpsub	f7,  f23,  f7

#else
	LFPDUX	f16, AO,  INC2
	LFPDUX	f17, AO,  INC2
	LFPDUX	f18, AO,  INC2
	LFPDUX	f19, AO,  INC2

	LFPDUX	f20, AO,  INC2
	LFPDUX	f21, AO,  INC2
	LFPDUX	f22, AO,  INC2
	LFPDUX	f23, AO,  INC2

	subi	AO,  AO,  16 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
	fpsub	f2,  f18,  f2
	fpsub	f3,  f19,  f3
	fpsub	f4,  f20,  f4
	fpsub	f5,  f21,  f5
	fpsub	f6,  f22,  f6
	fpsub	f7,  f23,  f7
#endif

#ifdef LN
       addi	AO,  AO,  66 * SIZE

	LFPDUX	A1,  AO,  INCM2
	LFPDUX	A2,  AO,  INCM2
	LFPDUX	A3,  AO,  INCM2
	LFPDUX	A4,  AO,  INCM2
	LFPDUX	A5,  AO,  INCM2
	LFPDUX	A6,  AO,  INCM2
	LFPDUX	A7,  AO,  INCM2
	LFPDUX	A8,  AO,  INCM2

	fxsmul	   f7,  A1, f7
	fxcpnmsub  f3,  A1, f7,  f3
	fxcsnmsub  f6,  A2, f7,  f6
	fxcpnmsub  f2,  A2, f7,  f2

	fxcsnmsub  f5,  A3, f7,  f5
	fxcpnmsub  f1,  A3, f7,  f1
	fxcsnmsub  f4,  A4, f7,  f4
	fxcpnmsub  f0,  A4, f7,  f0

	fxpmul	   f3,  A5, f3
	fxcsnmsub  f6,  A6, f3,  f6
	fxcpnmsub  f2,  A6, f3,  f2

	fxcsnmsub  f5,  A7, f3,  f5
	fxcpnmsub  f1,  A7, f3,  f1
	fxcsnmsub  f4,  A8, f3,  f4
	fxcpnmsub  f0,  A8, f3,  f0

	add	AO,  AO,  INCM2
	LFPDUX	A1,  AO,  INCM2
	LFPDUX	A2,  AO,  INCM2
	LFPDUX	A3,  AO,  INCM2

	add	AO,  AO,  INCM2
	LFPDUX	A4,  AO,  INCM2
	LFPDUX	A5,  AO,  INCM2
	LFPDUX	A6,  AO,  INCM2

	add	AO,  AO,  INCM2
	add	AO,  AO,  INCM2
	LFPDUX	A7,  AO,  INCM2
	LFPDUX	A8,  AO,  INCM2

	fxsmul	   f6,  A1, f6
	fxcpnmsub  f2,  A1, f6,  f2
	fxcsnmsub  f5,  A2, f6,  f5
	fxcpnmsub  f1,  A2, f6,  f1
	fxcsnmsub  f4,  A3, f6,  f4
	fxcpnmsub  f0,  A3, f6,  f0

	fxpmul	   f2,  A4, f2
	fxcsnmsub  f5,  A5, f2,  f5
	fxcpnmsub  f1,  A5, f2,  f1
	fxcsnmsub  f4,  A6, f2,  f4
	fxcpnmsub  f0,  A6, f2,  f0

	fxsmul	   f5,  A7, f5
	fxcpnmsub  f1,  A7, f5,  f1
	fxcsnmsub  f4,  A8, f5,  f4
	fxcpnmsub  f0,  A8, f5,  f0

	add	AO,  AO,  INCM2
	add	AO,  AO,  INCM2
	LFPDUX	A1,  AO,  INCM2
	LFPDUX	A2,  AO,  INCM2

	subi	AO,  AO, 6 * SIZE
	LFPDUX	A3,  AO,  INCM2
	subi	AO,  AO, 6 * SIZE
	LFPDUX	A4,  AO,  INCM2

	addi	AO,  AO,  -2 * SIZE

	fxpmul	   f1,  A1, f1
	fxcsnmsub  f4,  A2, f1,  f4
	fxcpnmsub  f0,  A2, f1,  f0

	fxsmul	   f4,  A3, f4
	fxcpnmsub  f0,  A3, f4,  f0

	fxpmul	f0,  A4, f0
#endif

#ifdef LT
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2

	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2
	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2

	fxpmul	   f0,  A1,  f0
	fxcsnmsub  f4,  A1, f0, f4
	fxcpnmsub  f1,  A2, f0, f1
	fxcsnmsub  f5,  A2, f0, f5
	fxcpnmsub  f2,  A3, f0, f2
	fxcsnmsub  f6,  A3, f0, f6
	fxcpnmsub  f3,  A4, f0, f3
	fxcsnmsub  f7,  A4, f0, f7

	fxsmul	   f4,  A5,  f4
	fxcpnmsub  f1,  A6, f4,  f1
	fxcsnmsub  f5,  A6, f4,  f5
	fxcpnmsub  f2,  A7, f4,  f2
	fxcsnmsub  f6,  A7, f4,  f6
	fxcpnmsub  f3,  A8, f4,  f3
	fxcsnmsub  f7,  A8, f4,  f7

	add	AO,  AO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2

	add	AO,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2

	add	AO,  AO,  INC2
	add	AO,  AO,  INC2
	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2

	fxpmul	   f1,  A1,  f1
	fxcsnmsub  f5,  A1, f1, f5
	fxcpnmsub  f2,  A2, f1, f2
	fxcsnmsub  f6,  A2, f1, f6
	fxcpnmsub  f3,  A3, f1, f3
	fxcsnmsub  f7,  A3, f1, f7

	fxsmul	   f5,  A4,  f5
	fxcpnmsub  f2,  A5, f5,  f2
	fxcsnmsub  f6,  A5, f5,  f6
	fxcpnmsub  f3,  A6, f5,  f3
	fxcsnmsub  f7,  A6, f5,  f7

	fxpmul	   f2,  A7,  f2
	fxcsnmsub  f6,  A7, f2,  f6
	fxcpnmsub  f3,  A8, f2,  f3
	fxcsnmsub  f7,  A8, f2,  f7

	add	AO,  AO,  INC2
	add	AO,  AO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2

	addi	AO,  AO,  6 * SIZE
	LFPDUX	A3,  AO,  INC2
	addi	AO,  AO,  6 * SIZE
	LFPDUX	A4,  AO,  INC2

	subi	AO,  AO,  64 * SIZE

	fxsmul	   f6,  A1,  f6
	fxcpnmsub  f3,  A2, f6,  f3
	fxcsnmsub  f7,  A2, f6,  f7

	fxpmul	   f3,  A3,  f3
	fxcsnmsub  f7,  A3, f3,  f7

	fxsmul	   f7,  A4,  f7
#endif

#ifdef RN
	LFPDUX	A1,  BO,  INC2
	LFPDUX	A2,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
	fxpmul	f2,  A1,  f2
	fxpmul	f3,  A1,  f3

	fxcsnmsub    f4,  A1, f0, f4
	fxcsnmsub    f5,  A1, f1, f5
	fxcsnmsub    f6,  A1, f2, f6
	fxcsnmsub    f7,  A1, f3, f7

	fxsmul	f4,  A2,  f4
	fxsmul	f5,  A2,  f5
	fxsmul	f6,  A2,  f6
	fxsmul	f7,  A2,  f7
#endif

#ifdef RT
	LFPDUX	A2,  BO,  INC2
	LFPDUX	A1,  BO,  INC2

	subi	BO, BO, 4 * SIZE

	fxsmul	f4,  A1,  f4
	fxsmul	f5,  A1,  f5
	fxsmul	f6,  A1,  f6
	fxsmul	f7,  A1,  f7

	fxcpnmsub    f0,  A1, f4,  f0
	fxcpnmsub    f1,  A1, f5,  f1
	fxcpnmsub    f2,  A1, f6,  f2
	fxcpnmsub    f3,  A1, f7,  f3

	fxpmul	f0,  A2,  f0
	fxpmul	f1,  A2,  f1
	fxpmul	f2,  A2,  f2
	fxpmul	f3,  A2,  f3

#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
	subi	CO2, CO2, 8 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC2
	STFPDUX	f4,  BO,  INC2
	STFPDUX	f1,  BO,  INC2
	STFPDUX	f5,  BO,  INC2
	STFPDUX	f2,  BO,  INC2
	STFPDUX	f6,  BO,  INC2
	STFPDUX	f3,  BO,  INC2
	STFPDUX	f7,  BO,  INC2

	subi	BO,  BO,  16 * SIZE

	STFDUX	f0,  CO1, INC
	STFDUX	f4,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFDUX	f5,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFDUX	f6,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFDUX	f7,  CO1, INC

	STFSDUX	f0,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFSDUX	f1,  CO2, INC
	STFSDUX	f5,  CO2, INC
	STFSDUX	f2,  CO2, INC
	STFSDUX	f6,  CO2, INC
	STFSDUX	f3,  CO2, INC
	STFSDUX	f7,  CO2, INC
#else
	STFPDUX	f0,  AO,  INC2
	STFPDUX	f1,  AO,  INC2
	STFPDUX	f2,  AO,  INC2
	STFPDUX	f3,  AO,  INC2
	STFPDUX	f4,  AO,  INC2
	STFPDUX	f5,  AO,  INC2
	STFPDUX	f6,  AO,  INC2
	STFPDUX	f7,  AO,  INC2

	subi	AO,  AO,  16 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFSDUX	f2,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFSDUX	f3,  CO1, INC

	STFDUX	f4,  CO2, INC
	STFSDUX	f4,  CO2, INC
	STFDUX	f5,  CO2, INC
	STFSDUX	f5,  CO2, INC
	STFDUX	f6,  CO2, INC
	STFSDUX	f6,  CO2, INC
	STFDUX	f7,  CO2, INC
	STFSDUX	f7,  CO2, INC
#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
	subi	CO2, CO2, 8 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 3 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 3 + BASE_SHIFT
	slwi	TEMP, TEMP, 1 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 8
#endif

#ifdef LN
	subi	KK, KK, 8
#endif

	addic.	I, I, -1
	li	r0, FZERO

	lfpsx	f0, SP, r0
	bgt+	.L51
	.align 4

.L89:
#ifdef LN
	slwi	r0, K, 1 + BASE_SHIFT
	add	B, B, r0
#endif

#if defined(LT) || defined(RN)
	addi	B,  BO, 2 * SIZE
#endif

#ifdef RN
	addi	KK, KK, 2
#endif

#ifdef RT
	subi	KK, KK, 2
#endif
	.align 4

.L90:
	andi.	J, N,  1
	beq	.L999

#ifdef RT
	slwi	r0, K, 0 + BASE_SHIFT
	sub	B, B, r0

	sub	C, C, LDC
#endif

	mr	CO1, C

#ifdef LN
	add	KK, M, OFFSET
#endif

#ifdef LT
	mr	KK, OFFSET
#endif

#if defined(LN) || defined(RT)
	addi	AORIG, A, -2 * SIZE
#else
	addi	AO, A, -2 * SIZE
#endif
#ifndef RT
	add	C,  CO1, LDC
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0

	andi.	I, M,  1
	beq	.L100

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L124
#else

#ifdef LN
	slwi	r0,   K,  0 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 0 + BASE_SHIFT
	slwi	TEMP, KK, 0 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	BO,  BO,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  TEMP,  3
	mtspr	CTR, r0
	ble	.L124
#endif

	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	B3,  BO,  INC2
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdz-	.L123
	.align 4

.L122:
	fpmadd	f0,  A1, B1, f0
	LFPDUX	A1,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	fpmadd	f1,  A2, B2, f1
	LFPDUX	A2,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	fpmadd	f2,  A3, B3, f2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	B3,  BO,  INC2
	fpmadd	f3,  A4, B4, f3
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdnz+	.L122
	.align 4

.L123:
	fpmadd	f0,  A1, B1, f0
	fpmadd	f1,  A2, B2, f1
	fpmadd	f2,  A3, B3, f2
	fpmadd	f3,  A4, B4, f3
	.align 4

.L124:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L128
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L128
#endif

	LFDX	A1,  AO,  INC2
	LFDX	B1,  BO,  INC2
	add	AO, AO, INC
	add	BO, BO, INC
	bdz-	.L127
	.align 4

.L126:
	fmadd	f0,  A1, B1, f0
	LFDX	A1,  AO,  INC2
	LFDX	B1,  BO,  INC2
	add	AO, AO, INC
	add	BO, BO, INC
	bdnz+	.L126
	.align 4

.L127:
	fmadd	f0,  A1, B1, f0
	.align 4

.L128:
	fpadd	f0, f0, f1
	fpadd	f2, f2, f3
	fpadd	f0, f0, f2
	fsmtp	f1, f0

	fadd	f0, f0, f1

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 1
#else
	subi	r0, KK, 1
#endif
	slwi	TEMP, r0, 0 + BASE_SHIFT
	slwi	r0,   r0, 0 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFDX	f16, BO,  INC2

	fsub	f0,  f16,  f0
#else
	LFDX	f16, AO,  INC2

	fsub	f0,  f16,  f0
#endif

#ifdef LN
	LFD	A1, (2 +  0) * SIZE(AO)

	fmul	f0, A1, f0
#endif

#ifdef LT
	LFD	A1, (2 +  0) * SIZE(AO)

	fmul	f0, A1, f0
#endif

#ifdef RN
	LFDX	A1,  BO,  INC2

	fmul	f0,  A1,  f0
#endif

#ifdef RT
	LFDX	A1,  BO,  INC2

	fmul	f0,  A1,  f0
#endif

#ifdef LN
	subi	CO1, CO1, 1 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFDX	f0,  BO,  INC2

	STFDUX	f0,  CO1, INC
#else
	STFDX	f0,  AO,  INC2

	STFDUX	f0,  CO1, INC
#endif

#ifdef LN
	subi	CO1, CO1, 1 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 0 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 0 + BASE_SHIFT
	slwi	TEMP, TEMP, 0 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 1
#endif

#ifdef LN
	subi	KK, KK, 1
#endif
	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L100:
	andi.	I, M,  2
	beq	.L110

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L114
#else

#ifdef LN
	slwi	r0,   K,  1 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 1 + BASE_SHIFT
	slwi	TEMP, KK, 0 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	BO,  BO,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0, TEMP,  3
	mtspr	CTR, r0
	ble	.L114
#endif

	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	B1,  BO,  INC2

	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B2,  BO,  INC2

	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2
	LFPDUX	B3,  BO,  INC2

	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdz-	.L113
	.align 4

.L112:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcsmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	LFPDUX	B1,  BO,  INC2
	fxcpmadd	f2,  B2, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcsmadd	f3,  B2, A4, f3
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	fxcpmadd	f0,  B3, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcsmadd	f1,  B3, A6, f1
	LFPDUX	A6,  AO,  INC2
	LFPDUX	B3,  BO,  INC2
	fxcpmadd	f2,  B4, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B4, A8, f3
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdnz+	.L112
	.align 4

.L113:
	fxcpmadd	f0,  B1, A1, f0
	fxcsmadd	f1,  B1, A2, f1
	fxcpmadd	f2,  B2, A3, f2
	fxcsmadd	f3,  B2, A4, f3
	fxcpmadd	f0,  B3, A5, f0
	fxcsmadd	f1,  B3, A6, f1
	fxcpmadd	f2,  B4, A7, f2
	fxcsmadd	f3,  B4, A8, f3
	.align 4

.L114:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L118
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L118
#endif

	LFPDUX	A1,  AO,  INC2
	LFDX	B1,  BO,  INC2
	add	BO, BO, INC
	bdz-	.L117
	.align 4

.L116:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	LFDX	B1,  BO,  INC2
	add	BO, BO, INC
	bdnz+	.L116
	.align 4

.L117:
	fxcpmadd	f0,  B1, A1, f0
	.align 4

.L118:
	fpadd	f0, f0, f1
	fpadd	f2, f3, f2
	fpadd	f0, f0, f2

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 2
#else
	subi	r0, KK, 1
#endif
	slwi	TEMP, r0, 1 + BASE_SHIFT
	slwi	r0,   r0, 0 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFPDX	f16, BO,  INC2

	fpsub	f0,  f16,  f0
#else
	LFPDX	f16, AO,  INC2

	fpsub	f0,  f16,  f0
#endif

#ifdef LN
	fsmtp	f4, f0

	LFD	A1, (2 +  3) * SIZE(AO)
	LFD	A2, (2 +  2) * SIZE(AO)
	LFD	A3, (2 +  0) * SIZE(AO)

	fmul	f4, A1, f4
	fnmsub	f0, A2, f4, f0
	fmul	f0, A3, f0
	fsmfp	f0, f4
#endif

#ifdef LT
	fsmtp	f4, f0

	LFD	A1, (2 +  0) * SIZE(AO)
	LFD	A2, (2 +  1) * SIZE(AO)
	LFD	A3, (2 +  3) * SIZE(AO)

	fmul	f0, A1, f0
	fnmsub	f4, A2, f0, f4
	fmul	f4, A3, f4

	fsmfp	f0, f4
#endif

#ifdef RN
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
#endif

#ifdef RT
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDX	f0,  BO,  INC2

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
#else
	STFPDX	f0,  AO,  INC2

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
#endif

#ifdef LN
	subi	CO1, CO1, 2 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 1 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 1 + BASE_SHIFT
	slwi	TEMP, TEMP, 0 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 2
#endif

#ifdef LN
	subi	KK, KK, 2
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L110:
	andi.	I, M,  4
	beq	.L120

#if defined(LT) || defined(RN)
	addi	BO,  B,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3, f0

	srawi.	r0,  KK,  3
	mtspr	CTR, r0
	ble	.L104
#else

#ifdef LN
	slwi	r0,   K,  2 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 2 + BASE_SHIFT
	slwi	TEMP, KK, 0 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

	addi	BO,  BO,  - 2 * SIZE
	fpmr	f1,  f0
	fpmr	f2,  f0
	fpmr	f3, f0

	srawi.	r0,  TEMP,  3
	mtspr	CTR, r0
	ble	.L104
#endif

	LFPDUX	B1,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2
	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B3,  BO,  INC2
	LFPDUX	B4,  BO,  INC2

	bdz-	.L103
	.align 4

.L102:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcsmadd	f2,  B1, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcsmadd	f3,  B1, A4, f3
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B1,  BO,  INC2

	fxcpmadd	f0,  B2, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B2, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B2, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B2, A8, f3
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B2,  BO,  INC2

	fxcpmadd	f0,  B3, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B3, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcsmadd	f2,  B3, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcsmadd	f3,  B3, A4, f3
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B3,  BO,  INC2

	fxcpmadd	f0,  B4, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B4, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B4, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B4, A8, f3
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B4,  BO,  INC2
	bdnz+	.L102
	.align 4

.L103:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcsmadd	f2,  B1, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcsmadd	f3,  B1, A4, f3
	LFPDUX	A4,  AO,  INC2

	fxcpmadd	f0,  B2, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcpmadd	f1,  B2, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B2, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B2, A8, f3
	LFPDUX	A8,  AO,  INC2

	fxcpmadd	f0,  B3, A1, f0
	fxcpmadd	f1,  B3, A2, f1
	fxcsmadd	f2,  B3, A3, f2
	fxcsmadd	f3,  B3, A4, f3

	fxcpmadd	f0,  B4, A5, f0
	fxcpmadd	f1,  B4, A6, f1
	fxcsmadd	f2,  B4, A7, f2
	fxcsmadd	f3,  B4, A8, f3
	.align 4

.L104:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  7
	mtspr	CTR, r0
	ble+	.L108
#else
	andi.	r0, TEMP, 7
	mtspr	CTR, r0
	ble+	.L108
#endif

	LFPDUX	A1,  AO,  INC2
	LFDX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	add	BO, BO, INC
	bdz-	.L107
	.align 4

.L106:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFDX	B1,  BO,  INC2
	LFPDUX	A2,  AO,  INC2
	add	BO, BO, INC
	bdnz+	.L106
	.align 4

.L107:
	fxcpmadd	f0,  B1, A1, f0
	fxcpmadd	f1,  B1, A2, f1
	.align 4

.L108:
	fpadd	f0, f0, f2
	fpadd	f1, f1, f3

#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 4
#else
	subi	r0, KK, 1
#endif
	slwi	TEMP, r0, 2 + BASE_SHIFT
	slwi	r0,   r0, 0 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFPDUX	f16, BO,  INC2
	LFPDUX	f17, BO,  INC2

	subi	BO,  BO,   4 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#else
	LFPDUX	f16, AO,  INC2
	LFPDUX	f17, AO,  INC2

	subi	AO,  AO,   4 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
#endif

#ifdef LN
	fsmtp	f4, f0
	fsmtp	f5, f1

	LFD	A1, (2 + 15) * SIZE(AO)
	LFD	A2, (2 + 14) * SIZE(AO)
	LFD	A3, (2 + 13) * SIZE(AO)
	LFD	A4, (2 + 12) * SIZE(AO)

	fmul	f5, A1, f5
	fnmsub	f1, A2, f5, f1
	fnmsub	f4, A3, f5, f4
	fnmsub	f0, A4, f5, f0

	LFD	A1, (2 + 10) * SIZE(AO)
	LFD	A2, (2 +  9) * SIZE(AO)
	LFD	A3, (2 +  8) * SIZE(AO)

	fmul	f1, A1, f1
	fnmsub	f4, A2, f1, f4
	fnmsub	f0, A3, f1, f0

	LFD	A1, (2 +  5) * SIZE(AO)
	LFD	A2, (2 +  4) * SIZE(AO)

	fmul	f4, A1, f4
	fnmsub	f0, A2, f4, f0

	LFD	A1, (2 +  0) * SIZE(AO)

	fmul	f0, A1, f0

	fsmfp	f0, f4
	fsmfp	f1, f5
#endif

#ifdef LT
	fsmtp	f4, f0
	fsmtp	f5, f1

	LFD	A1, (2 +  0) * SIZE(AO)
	LFD	A2, (2 +  1) * SIZE(AO)
	LFD	A3, (2 +  2) * SIZE(AO)
	LFD	A4, (2 +  3) * SIZE(AO)

	fmul	f0, A1, f0
	fnmsub	f4, A2, f0, f4
	fnmsub	f1, A3, f0, f1
	fnmsub	f5, A4, f0, f5

	LFD	A1, (2 +  5) * SIZE(AO)
	LFD	A2, (2 +  6) * SIZE(AO)
	LFD	A3, (2 +  7) * SIZE(AO)

	fmul	f4, A1, f4
	fnmsub	f1, A2, f4, f1
	fnmsub	f5, A3, f4, f5

	LFD	A1, (2 + 10) * SIZE(AO)
	LFD	A2, (2 + 11) * SIZE(AO)

	fmul	f1, A1, f1
	fnmsub	f5, A2, f1, f5

	LFD	A1, (2 + 15) * SIZE(AO)

	fmul	f5, A1, f5

	fsmfp	f0, f4
	fsmfp	f1, f5
#endif

#ifdef RN
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
#endif

#ifdef RT
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC2
	STFPDUX	f1,  BO,  INC2

	subi	BO,  BO,   4 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
#else
	STFPDUX	f0,  AO,  INC2
	STFPDUX	f1,  AO,  INC2

	subi	AO,  AO,   4 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
#endif

#ifdef LN
	subi	CO1, CO1, 4 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 2 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 2 + BASE_SHIFT
	slwi	TEMP, TEMP, 0 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 4
#endif

#ifdef LN
	subi	KK, KK, 4
#endif

	li	r0, FZERO
	lfpsx	f0, SP, r0
	.align 4

.L120:
	srawi.	I, M,  3
	ble	.L129
	.align 4

.L91:
#if defined(LT) || defined(RN)
 	fpmr	f1,  f0
	addi	BO,  B,  - 2 * SIZE
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  KK,  2
	mtspr	CTR, r0
	ble	.L94
#else

#ifdef LN
	slwi	r0,   K,  3 + BASE_SHIFT
	sub	AORIG, AORIG, r0
#endif

	slwi	r0  , KK, 3 + BASE_SHIFT
	slwi	TEMP, KK, 0 + BASE_SHIFT
	add	AO, AORIG, r0
	add	BO, B,     TEMP

	sub	TEMP, K, KK

 	fpmr	f1,  f0
	addi	BO,  BO,  - 2 * SIZE
	fpmr	f2,  f0
	fpmr	f3,  f0

	srawi.	r0,  TEMP,  2
	mtspr	CTR, r0
	ble	.L94
#endif

	LFPDUX	B1,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	LFPDUX	A5,  AO,  INC2
	LFPDUX	A6,  AO,  INC2
	LFPDUX	A7,  AO,  INC2
	LFPDUX	A8,  AO,  INC2
	bdz-	.L93
	.align 4

.L92:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcpmadd	f2,  B1, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	LFPDUX	A4,  AO,  INC2

	fxcsmadd	f0,  B1, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcsmadd	f1,  B1, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B1, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B1, A8, f3
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B1,  BO,  INC2

	fxcpmadd	f0,  B2, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B2, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcpmadd	f2,  B2, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B2, A4, f3
	LFPDUX	A4,  AO,  INC2

	fxcsmadd	f0,  B2, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcsmadd	f1,  B2, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B2, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B2, A8, f3
	LFPDUX	A8,  AO,  INC2
	LFPDUX	B2,  BO,  INC2
	bdnz+	.L92
	.align 4

.L93:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcpmadd	f2,  B1, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	LFPDUX	A4,  AO,  INC2

	fxcsmadd	f0,  B1, A5, f0
	LFPDUX	A5,  AO,  INC2
	fxcsmadd	f1,  B1, A6, f1
	LFPDUX	A6,  AO,  INC2
	fxcsmadd	f2,  B1, A7, f2
	LFPDUX	A7,  AO,  INC2
	fxcsmadd	f3,  B1, A8, f3
	LFPDUX	A8,  AO,  INC2

	fxcpmadd	f0,  B2, A1, f0
	fxcpmadd	f1,  B2, A2, f1
	fxcpmadd	f2,  B2, A3, f2
	fxcpmadd	f3,  B2, A4, f3

	fxcsmadd	f0,  B2, A5, f0
	fxcsmadd	f1,  B2, A6, f1
	fxcsmadd	f2,  B2, A7, f2
	fxcsmadd	f3,  B2, A8, f3
	.align 4

.L94:
#if defined(LT) || defined(RN)
	andi.	r0,  KK,  3
	mtspr	CTR, r0
	ble+	.L98
#else
	andi.	r0, TEMP, 3
	mtspr	CTR, r0
	ble+	.L98
#endif

	LFDX	B1,  BO,  INC2
	LFPDUX	A1,  AO,  INC2
	LFPDUX	A2,  AO,  INC2
	LFPDUX	A3,  AO,  INC2
	LFPDUX	A4,  AO,  INC2
	add	BO, BO, INC
	bdz-	.L97
	.align 4

.L96:
	fxcpmadd	f0,  B1, A1, f0
	LFPDUX	A1,  AO,  INC2
	fxcpmadd	f1,  B1, A2, f1
	LFPDUX	A2,  AO,  INC2
	fxcpmadd	f2,  B1, A3, f2
	LFPDUX	A3,  AO,  INC2
	fxcpmadd	f3,  B1, A4, f3
	LFDX	B1,  BO,  INC2
	LFPDUX	A4,  AO,  INC2
	add	BO, BO, INC
	bdnz+	.L96
	.align 4

.L97:
	fxcpmadd	f0,  B1, A1, f0
	fxcpmadd	f1,  B1, A2, f1
	fxcpmadd	f2,  B1, A3, f2
	fxcpmadd	f3,  B1, A4, f3
	.align 4

.L98:
#if defined(LN) || defined(RT)
#ifdef LN
	subi	r0, KK, 8
#else
	subi	r0, KK, 1
#endif
	slwi	TEMP, r0, 3 + BASE_SHIFT
	slwi	r0,   r0, 0 + BASE_SHIFT
	add	AO, AORIG, TEMP
	add	BO,  B,     r0
	addi	BO,  BO, - 2 * SIZE
#endif

#if defined(LN) || defined(LT)
	LFPDUX	f16, BO,  INC2
	LFPDUX	f17, BO,  INC2
	LFPDUX	f18, BO,  INC2
	LFPDUX	f19, BO,  INC2

	subi	BO,  BO,   8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
	fpsub	f2,  f18,  f2
	fpsub	f3,  f19,  f3
#else
	LFPDUX	f16, AO,  INC2
	LFPDUX	f17, AO,  INC2
	LFPDUX	f18, AO,  INC2
	LFPDUX	f19, AO,  INC2

	subi	AO,  AO,   8 * SIZE

	fpsub	f0,  f16,  f0
	fpsub	f1,  f17,  f1
	fpsub	f2,  f18,  f2
	fpsub	f3,  f19,  f3
#endif

#ifdef LN
	fsmtp	f4, f0
	fsmtp	f5, f1
	fsmtp	f6, f2
	fsmtp	f7, f3

	LFD	A1, (2 + 63) * SIZE(AO)
	LFD	A2, (2 + 62) * SIZE(AO)
	LFD	A3, (2 + 61) * SIZE(AO)
	LFD	A4, (2 + 60) * SIZE(AO)
	LFD	A5, (2 + 59) * SIZE(AO)
	LFD	A6, (2 + 58) * SIZE(AO)
	LFD	A7, (2 + 57) * SIZE(AO)
	LFD	A8, (2 + 56) * SIZE(AO)

	fmul	f7, A1, f7
	fnmsub	f3, A2, f7, f3
	fnmsub	f6, A3, f7, f6
	fnmsub	f2, A4, f7, f2
	fnmsub	f5, A5, f7, f5
	fnmsub	f1, A6, f7, f1
	fnmsub	f4, A7, f7, f4
	fnmsub	f0, A8, f7, f0

	LFD	A1, (2 + 54) * SIZE(AO)
	LFD	A2, (2 + 53) * SIZE(AO)
	LFD	A3, (2 + 52) * SIZE(AO)
	LFD	A4, (2 + 51) * SIZE(AO)
	LFD	A5, (2 + 50) * SIZE(AO)
	LFD	A6, (2 + 49) * SIZE(AO)
	LFD	A7, (2 + 48) * SIZE(AO)

	fmul	f3, A1, f3
	fnmsub	f6, A2, f3, f6
	fnmsub	f2, A3, f3, f2
	fnmsub	f5, A4, f3, f5
	fnmsub	f1, A5, f3, f1
	fnmsub	f4, A6, f3, f4
	fnmsub	f0, A7, f3, f0

	LFD	A1, (2 + 45) * SIZE(AO)
	LFD	A2, (2 + 44) * SIZE(AO)
	LFD	A3, (2 + 43) * SIZE(AO)
	LFD	A4, (2 + 42) * SIZE(AO)
	LFD	A5, (2 + 41) * SIZE(AO)
	LFD	A6, (2 + 40) * SIZE(AO)

	fmul	f6, A1, f6
	fnmsub	f2, A2, f6, f2
	fnmsub	f5, A3, f6, f5
	fnmsub	f1, A4, f6, f1
	fnmsub	f4, A5, f6, f4
	fnmsub	f0, A6, f6, f0

	LFD	A1, (2 + 36) * SIZE(AO)
	LFD	A2, (2 + 35) * SIZE(AO)
	LFD	A3, (2 + 34) * SIZE(AO)
	LFD	A4, (2 + 33) * SIZE(AO)
	LFD	A5, (2 + 32) * SIZE(AO)

	fmul	f2, A1, f2
	fnmsub	f5, A2, f2, f5
	fnmsub	f1, A3, f2, f1
	fnmsub	f4, A4, f2, f4
	fnmsub	f0, A5, f2, f0

	LFD	A1, (2 + 27) * SIZE(AO)
	LFD	A2, (2 + 26) * SIZE(AO)
	LFD	A3, (2 + 25) * SIZE(AO)
	LFD	A4, (2 + 24) * SIZE(AO)

	fmul	f5, A1, f5
	fnmsub	f1, A2, f5, f1
	fnmsub	f4, A3, f5, f4
	fnmsub	f0, A4, f5, f0

	LFD	A1, (2 + 18) * SIZE(AO)
	LFD	A2, (2 + 17) * SIZE(AO)
	LFD	A3, (2 + 16) * SIZE(AO)

	fmul	f1, A1, f1
	fnmsub	f4, A2, f1, f4
	fnmsub	f0, A3, f1, f0

	LFD	A1, (2 +  9) * SIZE(AO)
	LFD	A2, (2 +  8) * SIZE(AO)

	fmul	f4, A1, f4
	fnmsub	f0, A2, f4, f0

	LFD	A1, (2 +  0) * SIZE(AO)

	fmul	f0, A1, f0

	fsmfp	f0, f4
	fsmfp	f1, f5
	fsmfp	f2, f6
	fsmfp	f3, f7
#endif

#ifdef LT
	fsmtp	f4, f0
	fsmtp	f5, f1
	fsmtp	f6, f2
	fsmtp	f7, f3

	LFD	A1, (2 +  0) * SIZE(AO)
	LFD	A2, (2 +  1) * SIZE(AO)
	LFD	A3, (2 +  2) * SIZE(AO)
	LFD	A4, (2 +  3) * SIZE(AO)
	LFD	A5, (2 +  4) * SIZE(AO)
	LFD	A6, (2 +  5) * SIZE(AO)
	LFD	A7, (2 +  6) * SIZE(AO)
	LFD	A8, (2 +  7) * SIZE(AO)

	fmul	f0, A1, f0
	fnmsub	f4, A2, f0, f4
	fnmsub	f1, A3, f0, f1
	fnmsub	f5, A4, f0, f5
	fnmsub	f2, A5, f0, f2
	fnmsub	f6, A6, f0, f6
	fnmsub	f3, A7, f0, f3
	fnmsub	f7, A8, f0, f7

	LFD	A1, (2 +  9) * SIZE(AO)
	LFD	A2, (2 + 10) * SIZE(AO)
	LFD	A3, (2 + 11) * SIZE(AO)
	LFD	A4, (2 + 12) * SIZE(AO)
	LFD	A5, (2 + 13) * SIZE(AO)
	LFD	A6, (2 + 14) * SIZE(AO)
	LFD	A7, (2 + 15) * SIZE(AO)

	fmul	f4, A1, f4
	fnmsub	f1, A2, f4, f1
	fnmsub	f5, A3, f4, f5
	fnmsub	f2, A4, f4, f2
	fnmsub	f6, A5, f4, f6
	fnmsub	f3, A6, f4, f3
	fnmsub	f7, A7, f4, f7

	LFD	A1, (2 + 18) * SIZE(AO)
	LFD	A2, (2 + 19) * SIZE(AO)
	LFD	A3, (2 + 20) * SIZE(AO)
	LFD	A4, (2 + 21) * SIZE(AO)
	LFD	A5, (2 + 22) * SIZE(AO)
	LFD	A6, (2 + 23) * SIZE(AO)

	fmul	f1, A1, f1
	fnmsub	f5, A2, f1, f5
	fnmsub	f2, A3, f1, f2
	fnmsub	f6, A4, f1, f6
	fnmsub	f3, A5, f1, f3
	fnmsub	f7, A6, f1, f7

	LFD	A1, (2 + 27) * SIZE(AO)
	LFD	A2, (2 + 28) * SIZE(AO)
	LFD	A3, (2 + 29) * SIZE(AO)
	LFD	A4, (2 + 30) * SIZE(AO)
	LFD	A5, (2 + 31) * SIZE(AO)

	fmul	f5, A1, f5
	fnmsub	f2, A2, f5, f2
	fnmsub	f6, A3, f5, f6
	fnmsub	f3, A4, f5, f3
	fnmsub	f7, A5, f5, f7

	LFD	A1, (2 + 36) * SIZE(AO)
	LFD	A2, (2 + 37) * SIZE(AO)
	LFD	A3, (2 + 38) * SIZE(AO)
	LFD	A4, (2 + 39) * SIZE(AO)

	fmul	f2, A1, f2
	fnmsub	f6, A2, f2, f6
	fnmsub	f3, A3, f2, f3
	fnmsub	f7, A4, f2, f7

	LFD	A1, (2 + 45) * SIZE(AO)
	LFD	A2, (2 + 46) * SIZE(AO)
	LFD	A3, (2 + 47) * SIZE(AO)

	fmul	f6, A1, f6
	fnmsub	f3, A2, f6, f3
	fnmsub	f7, A3, f6, f7

	LFD	A1, (2 + 54) * SIZE(AO)
	LFD	A2, (2 + 55) * SIZE(AO)

	fmul	f3, A1, f3
	fnmsub	f7, A2, f3, f7

	LFD	A1, (2 + 63) * SIZE(AO)

	fmul	f7, A1, f7

	fsmfp	f0, f4
	fsmfp	f1, f5
	fsmfp	f2, f6
	fsmfp	f3, f7
#endif

#ifdef RN
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
	fxpmul	f2,  A1,  f2
	fxpmul	f3,  A1,  f3
#endif

#ifdef RT
	LFPDX	A1,  BO,  INC2

	fxpmul	f0,  A1,  f0
	fxpmul	f1,  A1,  f1
	fxpmul	f2,  A1,  f2
	fxpmul	f3,  A1,  f3

#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
#endif

#if defined(LN) || defined(LT)
	STFPDUX	f0,  BO,  INC2
	STFPDUX	f1,  BO,  INC2
	STFPDUX	f2,  BO,  INC2
	STFPDUX	f3,  BO,  INC2

	subi	BO,  BO,   8 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFSDUX	f2,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFSDUX	f3,  CO1, INC
#else
	STFPDUX	f0,  AO,  INC2
	STFPDUX	f1,  AO,  INC2
	STFPDUX	f2,  AO,  INC2
	STFPDUX	f3,  AO,  INC2

	subi	AO,  AO,   8 * SIZE

	STFDUX	f0,  CO1, INC
	STFSDUX	f0,  CO1, INC
	STFDUX	f1,  CO1, INC
	STFSDUX	f1,  CO1, INC
	STFDUX	f2,  CO1, INC
	STFSDUX	f2,  CO1, INC
	STFDUX	f3,  CO1, INC
	STFSDUX	f3,  CO1, INC
#endif

#ifdef LN
	subi	CO1, CO1, 8 * SIZE
#endif

#ifdef RT
	slwi	r0, K, 3 + BASE_SHIFT
	add	AORIG, AORIG, r0
#endif

#if defined(LT) || defined(RN)
	sub	TEMP, K, KK
	slwi	r0,   TEMP, 3 + BASE_SHIFT
	slwi	TEMP, TEMP, 0 + BASE_SHIFT
	add	AO, AO, r0
	add	BO, BO, TEMP
#endif

#ifdef LT
	addi	KK, KK, 8
#endif

#ifdef LN
	subi	KK, KK, 8
#endif

	addic.	I, I, -1
	li	r0, FZERO

	lfpsx	f0, SP, r0
	bgt+	.L91
	.align 4

.L129:
#ifdef LN
	slwi	r0, K, 0 + BASE_SHIFT
	add	B, B, r0
#endif

#if defined(LT) || defined(RN)
	addi	B,  BO, 2 * SIZE
#endif

#ifdef RN
	addi	KK, KK, 1
#endif

#ifdef RT
	subi	KK, KK, 1
#endif
	.align 4


.L999:
	addi	SP, SP, 12

	lwzu	r14,   4(SP)
	lwzu	r15,   4(SP)

	lwzu	r16,   4(SP)
	lwzu	r17,   4(SP)
	lwzu	r18,   4(SP)
	lwzu	r19,   4(SP)

	lwzu	r20,   4(SP)
	lwzu	r21,   4(SP)
	lwzu	r22,   4(SP)
	lwzu	r23,   4(SP)

	lwzu	r24,   4(SP)
	lwzu	r25,   4(SP)
	lwzu	r26,   4(SP)
	lwzu	r27,   4(SP)

	lwzu	r28,   4(SP)
	lwzu	r29,   4(SP)
	lwzu	r30,   4(SP)
	lwzu	r31,   4(SP)

	subi	SP, SP, 12
	li	r0, 16

	lfpdux	f31, SP, r0
	lfpdux	f30, SP, r0
	lfpdux	f29, SP, r0
	lfpdux	f28, SP, r0
	lfpdux	f27, SP, r0
	lfpdux	f26, SP, r0
	lfpdux	f25, SP, r0
	lfpdux	f24, SP, r0
	lfpdux	f23, SP, r0
	lfpdux	f22, SP, r0
	lfpdux	f21, SP, r0
	lfpdux	f20, SP, r0
	lfpdux	f19, SP, r0
	lfpdux	f18, SP, r0
	lfpdux	f17, SP, r0
	lfpdux	f16, SP, r0
	lfpdux	f15, SP, r0
	lfpdux	f14, SP, r0
	addi	SP, SP, 16
	blr


	EPILOGUE
#endif