OSchip
/
OpenBLAS
Not watched
1
0
Fork 0
Code
Releases 66 Wiki evaluate Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain HPC
You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.
Tree: c43ec53bdd
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'c43ec53bdd'
${ noResults }
OpenBLAS/kernel/power/sgemm_macros_16x8_power8.S

7327 lines
97 kB
Raw Blame History 

  1. /***************************************************************************

  2. Copyright (c) 2013-2016, The OpenBLAS Project

  3. All rights reserved.

  4. Redistribution and use in source and binary forms, with or without

  5. modification, are permitted provided that the following conditions are

  6. met:

  7. 1. Redistributions of source code must retain the above copyright

  8. notice, this list of conditions and the following disclaimer.

  9. 2. Redistributions in binary form must reproduce the above copyright

  10. notice, this list of conditions and the following disclaimer in

  11. the documentation and/or other materials provided with the

  12. distribution.

  13. 3. Neither the name of the OpenBLAS project nor the names of

  14. its contributors may be used to endorse or promote products

  15. derived from this software without specific prior written permission.

  16. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"

  17. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

  18. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

  19. ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE

  20. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

  21. DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR

  22. SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

  23. CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,

  24. OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

  25. USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

  26. *****************************************************************************/

  27. 

  28. /**************************************************************************************

  29. * 2016/04/21 Werner Saar (wernsaar@googlemail.com)

  30. * 	 BLASTEST 		: OK

  31. * 	 CTEST			: OK

  32. * 	 TEST			: OK

  33. *	 LAPACK-TEST		: OK

  34. **************************************************************************************/

  35. 

  36. 

  37. /**********************************************************************************************

  38. * Macros for N=8 and M=16

  39. **********************************************************************************************/

  40. 

  41. #if defined(_AIX)

  42. define(`LOAD8x16_1', `

  43. #else

  44. .macro LOAD8x16_1

  45. #endif

  46. 

  47. 	lxvw4x		vs0,	o0,	AO

  48. 	lxvw4x		vs1,	o16,	AO

  49. 	lxvw4x		vs2,	o32,	AO

  50. 	lxvw4x		vs3,	o48,	AO

  51. 

  52. 	addi		AO,	AO,	64

  53. 

  54. 	mr		T1,	BO

  55. 

  56. 	lxvw4x		vs8,	o0,	T1

  57. 	lxvw4x		vs9,	o16,	T1

  58. 	lxvw4x		vs10,	o32,	T1

  59. 	lxvw4x		vs11,	o48,	T1

  60. 

  61. 	addi		T1,	T1,	64

  62. 

  63. 	lxvw4x		vs12,	o0,	T1

  64. 	lxvw4x		vs13,	o16,	T1

  65. 	lxvw4x		vs14,	o32,	T1

  66. 	lxvw4x		vs15,	o48,	T1

  67. 

  68. 	addi		BO,	BO,	128

  69. 

  70. #if defined(_AIX)

  71. ')

  72. #else

  73. .endm

  74. #endif

  75. 

  76. #if defined(_AIX)

  77. define(`KERNEL8x16_I1', `

  78. #else

  79. .macro KERNEL8x16_I1

  80. #endif

  81. 

  82. 

  83. 	lxvw4x		vs4,	o0,	AO

  84. 	lxvw4x		vs5,	o16,	AO

  85. 	lxvw4x		vs6,	o32,	AO

  86. 	lxvw4x		vs7,	o48,	AO

  87. 

  88. 	addi		AO,	AO,	64

  89. 

  90. 	mr		T1,	BO

  91. 

  92. 	lxvw4x		vs16,	o0,	T1

  93. 	lxvw4x		vs17,	o16,	T1

  94. 	lxvw4x		vs18,	o32,	T1

  95. 	lxvw4x		vs19,	o48,	T1

  96. 

  97. 	addi		T1,	T1,	64

  98. 

  99. 	lxvw4x		vs20,	o0,	T1

  100. 	lxvw4x		vs21,	o16,	T1

  101. 	lxvw4x		vs22,	o32,	T1

  102. 	lxvw4x		vs23,	o48,	T1

  103. 

  104. 	addi		BO,	BO,	128

  105. 

  106. 

  107. 	xvmulsp		vs32,	vs0,	vs8

  108. 	xvmulsp		vs33,	vs1,	vs8

  109. 	xvmulsp		vs34,	vs2,	vs8

  110. 	xvmulsp		vs35,	vs3,	vs8

  111. 

  112. 	xvmulsp		vs36,	vs0,	vs9

  113. 	xvmulsp		vs37,	vs1,	vs9

  114. 	xvmulsp		vs38,	vs2,	vs9

  115. 	xvmulsp		vs39,	vs3,	vs9

  116. 

  117. 	xvmulsp		vs40,	vs0,	vs10

  118. 	xvmulsp		vs41,	vs1,	vs10

  119. 	xvmulsp		vs42,	vs2,	vs10

  120. 	xvmulsp		vs43,	vs3,	vs10

  121. 

  122. 	xvmulsp		vs44,	vs0,	vs11

  123. 	xvmulsp		vs45,	vs1,	vs11

  124. 	xvmulsp		vs46,	vs2,	vs11

  125. 	xvmulsp		vs47,	vs3,	vs11

  126. 

  127. 	xvmulsp		vs48,	vs0,	vs12

  128. 	xvmulsp		vs49,	vs1,	vs12

  129. 	xvmulsp		vs50,	vs2,	vs12

  130. 	xvmulsp		vs51,	vs3,	vs12

  131. 

  132. 	xvmulsp		vs52,	vs0,	vs13

  133. 	xvmulsp		vs53,	vs1,	vs13

  134. 	xvmulsp		vs54,	vs2,	vs13

  135. 	xvmulsp		vs55,	vs3,	vs13

  136. 

  137. 	xvmulsp		vs56,	vs0,	vs14

  138. 	xvmulsp		vs57,	vs1,	vs14

  139. 	xvmulsp		vs58,	vs2,	vs14

  140. 	xvmulsp		vs59,	vs3,	vs14

  141. 

  142. 	xvmulsp		vs60,	vs0,	vs15

  143. 	xvmulsp		vs61,	vs1,	vs15

  144. 	xvmulsp		vs62,	vs2,	vs15

  145. 	xvmulsp		vs63,	vs3,	vs15

  146. 

  147. 

  148. #if defined(_AIX)

  149. ')

  150. #else

  151. .endm

  152. #endif

  153. 

  154. #if defined(_AIX)

  155. define(`KERNEL8x16_1', `

  156. #else

  157. .macro KERNEL8x16_1

  158. #endif

  159. 

  160. 

  161. 	lxvw4x		vs4,	o0,	AO

  162. 	lxvw4x		vs5,	o16,	AO

  163. 	lxvw4x		vs6,	o32,	AO

  164. 	lxvw4x		vs7,	o48,	AO

  165. 

  166. 	addi		AO,	AO,	64

  167. 

  168. 	mr		T1,	BO

  169. 

  170. 	lxvw4x		vs16,	o0,	T1

  171. 	lxvw4x		vs17,	o16,	T1

  172. 	lxvw4x		vs18,	o32,	T1

  173. 	lxvw4x		vs19,	o48,	T1

  174. 

  175. 	addi		T1,	T1,	64

  176. 

  177. 	lxvw4x		vs20,	o0,	T1

  178. 	lxvw4x		vs21,	o16,	T1

  179. 	lxvw4x		vs22,	o32,	T1

  180. 	lxvw4x		vs23,	o48,	T1

  181. 

  182. 	addi		BO,	BO,	128

  183. 

  184. 

  185. 	xvmaddasp	vs32,	vs0,	vs8

  186. 	xvmaddasp	vs33,	vs1,	vs8

  187. 	xvmaddasp	vs34,	vs2,	vs8

  188. 	xvmaddasp	vs35,	vs3,	vs8

  189. 

  190. 	xvmaddasp	vs36,	vs0,	vs9

  191. 	xvmaddasp	vs37,	vs1,	vs9

  192. 	xvmaddasp	vs38,	vs2,	vs9

  193. 	xvmaddasp	vs39,	vs3,	vs9

  194. 

  195. 	xvmaddasp	vs40,	vs0,	vs10

  196. 	xvmaddasp	vs41,	vs1,	vs10

  197. 	xvmaddasp	vs42,	vs2,	vs10

  198. 	xvmaddasp	vs43,	vs3,	vs10

  199. 

  200. 	xvmaddasp	vs44,	vs0,	vs11

  201. 	xvmaddasp	vs45,	vs1,	vs11

  202. 	xvmaddasp	vs46,	vs2,	vs11

  203. 	xvmaddasp	vs47,	vs3,	vs11

  204. 

  205. 	xvmaddasp	vs48,	vs0,	vs12

  206. 	xvmaddasp	vs49,	vs1,	vs12

  207. 	xvmaddasp	vs50,	vs2,	vs12

  208. 	xvmaddasp	vs51,	vs3,	vs12

  209. 

  210. 	xvmaddasp	vs52,	vs0,	vs13

  211. 	xvmaddasp	vs53,	vs1,	vs13

  212. 	xvmaddasp	vs54,	vs2,	vs13

  213. 	xvmaddasp	vs55,	vs3,	vs13

  214. 

  215. 	xvmaddasp	vs56,	vs0,	vs14

  216. 	xvmaddasp	vs57,	vs1,	vs14

  217. 	xvmaddasp	vs58,	vs2,	vs14

  218. 	xvmaddasp	vs59,	vs3,	vs14

  219. 

  220. 	xvmaddasp	vs60,	vs0,	vs15

  221. 	xvmaddasp	vs61,	vs1,	vs15

  222. 	xvmaddasp	vs62,	vs2,	vs15

  223. 	xvmaddasp	vs63,	vs3,	vs15

  224. 

  225. 

  226. #if defined(_AIX)

  227. ')

  228. #else

  229. .endm

  230. #endif

  231. 

  232. #if defined(_AIX)

  233. define(`KERNEL8x16_2', `

  234. #else

  235. .macro KERNEL8x16_2

  236. #endif

  237. 

  238. 

  239. 	lxvw4x		vs0,	o0,	AO

  240. 	lxvw4x		vs1,	o16,	AO

  241. 	lxvw4x		vs2,	o32,	AO

  242. 	lxvw4x		vs3,	o48,	AO

  243. 

  244. 	addi		AO,	AO,	64

  245. 

  246. 	mr		T1,	BO

  247. 

  248. 	lxvw4x		vs8,	o0,	T1

  249. 	lxvw4x		vs9,	o16,	T1

  250. 	lxvw4x		vs10,	o32,	T1

  251. 	lxvw4x		vs11,	o48,	T1

  252. 

  253. 	addi		T1,	T1,	64

  254. 

  255. 	lxvw4x		vs12,	o0,	T1

  256. 	lxvw4x		vs13,	o16,	T1

  257. 	lxvw4x		vs14,	o32,	T1

  258. 	lxvw4x		vs15,	o48,	T1

  259. 

  260. 	addi		BO,	BO,	128

  261. 

  262. 

  263. 	xvmaddasp	vs32,	vs4,	vs16

  264. 	xvmaddasp	vs33,	vs5,	vs16

  265. 	xvmaddasp	vs34,	vs6,	vs16

  266. 	xvmaddasp	vs35,	vs7,	vs16

  267. 

  268. 	xvmaddasp	vs36,	vs4,	vs17

  269. 	xvmaddasp	vs37,	vs5,	vs17

  270. 	xvmaddasp	vs38,	vs6,	vs17

  271. 	xvmaddasp	vs39,	vs7,	vs17

  272. 

  273. 	xvmaddasp	vs40,	vs4,	vs18

  274. 	xvmaddasp	vs41,	vs5,	vs18

  275. 	xvmaddasp	vs42,	vs6,	vs18

  276. 	xvmaddasp	vs43,	vs7,	vs18

  277. 

  278. 	xvmaddasp	vs44,	vs4,	vs19

  279. 	xvmaddasp	vs45,	vs5,	vs19

  280. 	xvmaddasp	vs46,	vs6,	vs19

  281. 	xvmaddasp	vs47,	vs7,	vs19

  282. 

  283. 	xvmaddasp	vs48,	vs4,	vs20

  284. 	xvmaddasp	vs49,	vs5,	vs20

  285. 	xvmaddasp	vs50,	vs6,	vs20

  286. 	xvmaddasp	vs51,	vs7,	vs20

  287. 

  288. 	xvmaddasp	vs52,	vs4,	vs21

  289. 	xvmaddasp	vs53,	vs5,	vs21

  290. 	xvmaddasp	vs54,	vs6,	vs21

  291. 	xvmaddasp	vs55,	vs7,	vs21

  292. 

  293. 	xvmaddasp	vs56,	vs4,	vs22

  294. 	xvmaddasp	vs57,	vs5,	vs22

  295. 	xvmaddasp	vs58,	vs6,	vs22

  296. 	xvmaddasp	vs59,	vs7,	vs22

  297. 

  298. 	xvmaddasp	vs60,	vs4,	vs23

  299. 	xvmaddasp	vs61,	vs5,	vs23

  300. 	xvmaddasp	vs62,	vs6,	vs23

  301. 	xvmaddasp	vs63,	vs7,	vs23

  302. 

  303. 

  304. #if defined(_AIX)

  305. ')

  306. #else

  307. .endm

  308. #endif

  309. 

  310. #if defined(_AIX)

  311. define(`KERNEL8x16_E2', `

  312. #else

  313. .macro KERNEL8x16_E2

  314. #endif

  315. 

  316. 

  317. 	xvmaddasp	vs32,	vs4,	vs16

  318. 	xvmaddasp	vs33,	vs5,	vs16

  319. 	xvmaddasp	vs34,	vs6,	vs16

  320. 	xvmaddasp	vs35,	vs7,	vs16

  321. 

  322. 	xvmaddasp	vs36,	vs4,	vs17

  323. 	xvmaddasp	vs37,	vs5,	vs17

  324. 	xvmaddasp	vs38,	vs6,	vs17

  325. 	xvmaddasp	vs39,	vs7,	vs17

  326. 

  327. 	xvmaddasp	vs40,	vs4,	vs18

  328. 	xvmaddasp	vs41,	vs5,	vs18

  329. 	xvmaddasp	vs42,	vs6,	vs18

  330. 	xvmaddasp	vs43,	vs7,	vs18

  331. 

  332. 	xvmaddasp	vs44,	vs4,	vs19

  333. 	xvmaddasp	vs45,	vs5,	vs19

  334. 	xvmaddasp	vs46,	vs6,	vs19

  335. 	xvmaddasp	vs47,	vs7,	vs19

  336. 

  337. 	xvmaddasp	vs48,	vs4,	vs20

  338. 	xvmaddasp	vs49,	vs5,	vs20

  339. 	xvmaddasp	vs50,	vs6,	vs20

  340. 	xvmaddasp	vs51,	vs7,	vs20

  341. 

  342. 	xvmaddasp	vs52,	vs4,	vs21

  343. 	xvmaddasp	vs53,	vs5,	vs21

  344. 	xvmaddasp	vs54,	vs6,	vs21

  345. 	xvmaddasp	vs55,	vs7,	vs21

  346. 

  347. 	xvmaddasp	vs56,	vs4,	vs22

  348. 	xvmaddasp	vs57,	vs5,	vs22

  349. 	xvmaddasp	vs58,	vs6,	vs22

  350. 	xvmaddasp	vs59,	vs7,	vs22

  351. 

  352. 	xvmaddasp	vs60,	vs4,	vs23

  353. 	xvmaddasp	vs61,	vs5,	vs23

  354. 	xvmaddasp	vs62,	vs6,	vs23

  355. 	xvmaddasp	vs63,	vs7,	vs23

  356. 

  357. 

  358. #if defined(_AIX)

  359. ')

  360. #else

  361. .endm

  362. #endif

  363. 

  364. #if defined(_AIX)

  365. define(`KERNEL8x16_SUBI1', `

  366. #else

  367. .macro KERNEL8x16_SUBI1

  368. #endif

  369. 

  370. 

  371. 	lxvw4x		vs0,	o0,	AO

  372. 	lxvw4x		vs1,	o16,	AO

  373. 	lxvw4x		vs2,	o32,	AO

  374. 	lxvw4x		vs3,	o48,	AO

  375. 

  376. 	addi		AO,	AO,	64

  377. 

  378. 	mr		T1,	BO

  379. 

  380. 	lxvw4x		vs8,	o0,	T1

  381. 	lxvw4x		vs9,	o16,	T1

  382. 	lxvw4x		vs10,	o32,	T1

  383. 	lxvw4x		vs11,	o48,	T1

  384. 

  385. 	addi		T1,	T1,	64

  386. 

  387. 	lxvw4x		vs12,	o0,	T1

  388. 	lxvw4x		vs13,	o16,	T1

  389. 	lxvw4x		vs14,	o32,	T1

  390. 	lxvw4x		vs15,	o48,	T1

  391. 

  392. 	addi		BO,	BO,	128

  393. 

  394. 

  395. 	xvmulsp		vs32,	vs0,	vs8

  396. 	xvmulsp		vs33,	vs1,	vs8

  397. 	xvmulsp		vs34,	vs2,	vs8

  398. 	xvmulsp		vs35,	vs3,	vs8

  399. 

  400. 	xvmulsp		vs36,	vs0,	vs9

  401. 	xvmulsp		vs37,	vs1,	vs9

  402. 	xvmulsp		vs38,	vs2,	vs9

  403. 	xvmulsp		vs39,	vs3,	vs9

  404. 

  405. 	xvmulsp		vs40,	vs0,	vs10

  406. 	xvmulsp		vs41,	vs1,	vs10

  407. 	xvmulsp		vs42,	vs2,	vs10

  408. 	xvmulsp		vs43,	vs3,	vs10

  409. 

  410. 	xvmulsp		vs44,	vs0,	vs11

  411. 	xvmulsp		vs45,	vs1,	vs11

  412. 	xvmulsp		vs46,	vs2,	vs11

  413. 	xvmulsp		vs47,	vs3,	vs11

  414. 

  415. 	xvmulsp		vs48,	vs0,	vs12

  416. 	xvmulsp		vs49,	vs1,	vs12

  417. 	xvmulsp		vs50,	vs2,	vs12

  418. 	xvmulsp		vs51,	vs3,	vs12

  419. 

  420. 	xvmulsp		vs52,	vs0,	vs13

  421. 	xvmulsp		vs53,	vs1,	vs13

  422. 	xvmulsp		vs54,	vs2,	vs13

  423. 	xvmulsp		vs55,	vs3,	vs13

  424. 

  425. 	xvmulsp		vs56,	vs0,	vs14

  426. 	xvmulsp		vs57,	vs1,	vs14

  427. 	xvmulsp		vs58,	vs2,	vs14

  428. 	xvmulsp		vs59,	vs3,	vs14

  429. 

  430. 	xvmulsp		vs60,	vs0,	vs15

  431. 	xvmulsp		vs61,	vs1,	vs15

  432. 	xvmulsp		vs62,	vs2,	vs15

  433. 	xvmulsp		vs63,	vs3,	vs15

  434. 

  435. 

  436. #if defined(_AIX)

  437. ')

  438. #else

  439. .endm

  440. #endif

  441. 

  442. #if defined(_AIX)

  443. define(`KERNEL8x16_SUB1', `

  444. #else

  445. .macro KERNEL8x16_SUB1

  446. #endif

  447. 

  448. 

  449. 	lxvw4x		vs0,	o0,	AO

  450. 	lxvw4x		vs1,	o16,	AO

  451. 	lxvw4x		vs2,	o32,	AO

  452. 	lxvw4x		vs3,	o48,	AO

  453. 

  454. 	addi		AO,	AO,	64

  455. 

  456. 	mr		T1,	BO

  457. 

  458. 	lxvw4x		vs8,	o0,	T1

  459. 	lxvw4x		vs9,	o16,	T1

  460. 	lxvw4x		vs10,	o32,	T1

  461. 	lxvw4x		vs11,	o48,	T1

  462. 

  463. 	addi		T1,	T1,	64

  464. 

  465. 	lxvw4x		vs12,	o0,	T1

  466. 	lxvw4x		vs13,	o16,	T1

  467. 	lxvw4x		vs14,	o32,	T1

  468. 	lxvw4x		vs15,	o48,	T1

  469. 

  470. 	addi		BO,	BO,	128

  471. 

  472. 

  473. 	xvmaddasp	vs32,	vs0,	vs8

  474. 	xvmaddasp	vs33,	vs1,	vs8

  475. 	xvmaddasp	vs34,	vs2,	vs8

  476. 	xvmaddasp	vs35,	vs3,	vs8

  477. 

  478. 	xvmaddasp	vs36,	vs0,	vs9

  479. 	xvmaddasp	vs37,	vs1,	vs9

  480. 	xvmaddasp	vs38,	vs2,	vs9

  481. 	xvmaddasp	vs39,	vs3,	vs9

  482. 

  483. 	xvmaddasp	vs40,	vs0,	vs10

  484. 	xvmaddasp	vs41,	vs1,	vs10

  485. 	xvmaddasp	vs42,	vs2,	vs10

  486. 	xvmaddasp	vs43,	vs3,	vs10

  487. 

  488. 	xvmaddasp	vs44,	vs0,	vs11

  489. 	xvmaddasp	vs45,	vs1,	vs11

  490. 	xvmaddasp	vs46,	vs2,	vs11

  491. 	xvmaddasp	vs47,	vs3,	vs11

  492. 

  493. 	xvmaddasp	vs48,	vs0,	vs12

  494. 	xvmaddasp	vs49,	vs1,	vs12

  495. 	xvmaddasp	vs50,	vs2,	vs12

  496. 	xvmaddasp	vs51,	vs3,	vs12

  497. 

  498. 	xvmaddasp	vs52,	vs0,	vs13

  499. 	xvmaddasp	vs53,	vs1,	vs13

  500. 	xvmaddasp	vs54,	vs2,	vs13

  501. 	xvmaddasp	vs55,	vs3,	vs13

  502. 

  503. 	xvmaddasp	vs56,	vs0,	vs14

  504. 	xvmaddasp	vs57,	vs1,	vs14

  505. 	xvmaddasp	vs58,	vs2,	vs14

  506. 	xvmaddasp	vs59,	vs3,	vs14

  507. 

  508. 	xvmaddasp	vs60,	vs0,	vs15

  509. 	xvmaddasp	vs61,	vs1,	vs15

  510. 	xvmaddasp	vs62,	vs2,	vs15

  511. 	xvmaddasp	vs63,	vs3,	vs15

  512. 

  513. 

  514. #if defined(_AIX)

  515. ')

  516. #else

  517. .endm

  518. #endif

  519. 

  520. #if defined(_AIX)

  521. define(`SAVE8x16', `

  522. #else

  523. .macro SAVE8x16

  524. #endif

  525. 

  526. 	mr		T1,	CO

  527. 

  528. #ifndef TRMMKERNEL

  529. 

  530. 	lxvw4x		vs0,	o0,	T1

  531. 	lxvw4x		vs1,	o16,	T1

  532. 	lxvw4x		vs2,	o32,	T1

  533. 	lxvw4x		vs3,	o48,	T1

  534. 

  535. #endif

  536. 

  537. #ifdef TRMMKERNEL

  538. 	xvmulsp		vs0,	vs32,	alpha_vr

  539. 	xvmulsp		vs1,	vs33,	alpha_vr

  540. 	xvmulsp		vs2,	vs34,	alpha_vr

  541. 	xvmulsp		vs3,	vs35,	alpha_vr

  542. #else

  543. 	xvmaddasp	vs0,	vs32,	alpha_vr

  544. 	xvmaddasp	vs1,	vs33,	alpha_vr

  545. 	xvmaddasp	vs2,	vs34,	alpha_vr

  546. 	xvmaddasp	vs3,	vs35,	alpha_vr

  547. #endif

  548. 

  549. 	stxvw4x		vs0,	o0,	T1

  550. 	stxvw4x		vs1,	o16,	T1

  551. 	stxvw4x		vs2,	o32,	T1

  552. 	stxvw4x		vs3,	o48,	T1

  553. 

  554. 	add		T1,	T1,	LDC

  555. 

  556. 

  557. #ifndef TRMMKERNEL

  558. 

  559. 	lxvw4x		vs0,	o0,	T1

  560. 	lxvw4x		vs1,	o16,	T1

  561. 	lxvw4x		vs2,	o32,	T1

  562. 	lxvw4x		vs3,	o48,	T1

  563. 

  564. #endif

  565. 

  566. #ifdef TRMMKERNEL

  567. 	xvmulsp		vs0,	vs36,	alpha_vr

  568. 	xvmulsp		vs1,	vs37,	alpha_vr

  569. 	xvmulsp		vs2,	vs38,	alpha_vr

  570. 	xvmulsp		vs3,	vs39,	alpha_vr

  571. #else

  572. 	xvmaddasp	vs0,	vs36,	alpha_vr

  573. 	xvmaddasp	vs1,	vs37,	alpha_vr

  574. 	xvmaddasp	vs2,	vs38,	alpha_vr

  575. 	xvmaddasp	vs3,	vs39,	alpha_vr

  576. #endif

  577. 

  578. 	stxvw4x		vs0,	o0,	T1

  579. 	stxvw4x		vs1,	o16,	T1

  580. 	stxvw4x		vs2,	o32,	T1

  581. 	stxvw4x		vs3,	o48,	T1

  582. 

  583. 	add		T1,	T1,	LDC

  584. 

  585. 

  586. #ifndef TRMMKERNEL

  587. 

  588. 	lxvw4x		vs0,	o0,	T1

  589. 	lxvw4x		vs1,	o16,	T1

  590. 	lxvw4x		vs2,	o32,	T1

  591. 	lxvw4x		vs3,	o48,	T1

  592. 

  593. #endif

  594. 

  595. #ifdef TRMMKERNEL

  596. 	xvmulsp		vs0,	vs40,	alpha_vr

  597. 	xvmulsp		vs1,	vs41,	alpha_vr

  598. 	xvmulsp		vs2,	vs42,	alpha_vr

  599. 	xvmulsp		vs3,	vs43,	alpha_vr

  600. #else

  601. 	xvmaddasp	vs0,	vs40,	alpha_vr

  602. 	xvmaddasp	vs1,	vs41,	alpha_vr

  603. 	xvmaddasp	vs2,	vs42,	alpha_vr

  604. 	xvmaddasp	vs3,	vs43,	alpha_vr

  605. #endif

  606. 

  607. 	stxvw4x		vs0,	o0,	T1

  608. 	stxvw4x		vs1,	o16,	T1

  609. 	stxvw4x		vs2,	o32,	T1

  610. 	stxvw4x		vs3,	o48,	T1

  611. 

  612. 	add		T1,	T1,	LDC

  613. 

  614. 

  615. #ifndef TRMMKERNEL

  616. 

  617. 	lxvw4x		vs0,	o0,	T1

  618. 	lxvw4x		vs1,	o16,	T1

  619. 	lxvw4x		vs2,	o32,	T1

  620. 	lxvw4x		vs3,	o48,	T1

  621. 

  622. #endif

  623. 

  624. #ifdef TRMMKERNEL

  625. 	xvmulsp		vs0,	vs44,	alpha_vr

  626. 	xvmulsp		vs1,	vs45,	alpha_vr

  627. 	xvmulsp		vs2,	vs46,	alpha_vr

  628. 	xvmulsp		vs3,	vs47,	alpha_vr

  629. #else

  630. 	xvmaddasp	vs0,	vs44,	alpha_vr

  631. 	xvmaddasp	vs1,	vs45,	alpha_vr

  632. 	xvmaddasp	vs2,	vs46,	alpha_vr

  633. 	xvmaddasp	vs3,	vs47,	alpha_vr

  634. #endif

  635. 

  636. 	stxvw4x		vs0,	o0,	T1

  637. 	stxvw4x		vs1,	o16,	T1

  638. 	stxvw4x		vs2,	o32,	T1

  639. 	stxvw4x		vs3,	o48,	T1

  640. 

  641. 	add		T1,	T1,	LDC

  642. 

  643. 

  644. #ifndef TRMMKERNEL

  645. 

  646. 	lxvw4x		vs0,	o0,	T1

  647. 	lxvw4x		vs1,	o16,	T1

  648. 	lxvw4x		vs2,	o32,	T1

  649. 	lxvw4x		vs3,	o48,	T1

  650. 

  651. #endif

  652. 

  653. #ifdef TRMMKERNEL

  654. 	xvmulsp		vs0,	vs48,	alpha_vr

  655. 	xvmulsp		vs1,	vs49,	alpha_vr

  656. 	xvmulsp		vs2,	vs50,	alpha_vr

  657. 	xvmulsp		vs3,	vs51,	alpha_vr

  658. #else

  659. 	xvmaddasp	vs0,	vs48,	alpha_vr

  660. 	xvmaddasp	vs1,	vs49,	alpha_vr

  661. 	xvmaddasp	vs2,	vs50,	alpha_vr

  662. 	xvmaddasp	vs3,	vs51,	alpha_vr

  663. #endif

  664. 

  665. 	stxvw4x		vs0,	o0,	T1

  666. 	stxvw4x		vs1,	o16,	T1

  667. 	stxvw4x		vs2,	o32,	T1

  668. 	stxvw4x		vs3,	o48,	T1

  669. 

  670. 	add		T1,	T1,	LDC

  671. 

  672. 

  673. #ifndef TRMMKERNEL

  674. 

  675. 	lxvw4x		vs0,	o0,	T1

  676. 	lxvw4x		vs1,	o16,	T1

  677. 	lxvw4x		vs2,	o32,	T1

  678. 	lxvw4x		vs3,	o48,	T1

  679. 

  680. #endif

  681. 

  682. #ifdef TRMMKERNEL

  683. 	xvmulsp		vs0,	vs52,	alpha_vr

  684. 	xvmulsp		vs1,	vs53,	alpha_vr

  685. 	xvmulsp		vs2,	vs54,	alpha_vr

  686. 	xvmulsp		vs3,	vs55,	alpha_vr

  687. #else

  688. 	xvmaddasp	vs0,	vs52,	alpha_vr

  689. 	xvmaddasp	vs1,	vs53,	alpha_vr

  690. 	xvmaddasp	vs2,	vs54,	alpha_vr

  691. 	xvmaddasp	vs3,	vs55,	alpha_vr

  692. #endif

  693. 

  694. 	stxvw4x		vs0,	o0,	T1

  695. 	stxvw4x		vs1,	o16,	T1

  696. 	stxvw4x		vs2,	o32,	T1

  697. 	stxvw4x		vs3,	o48,	T1

  698. 

  699. 	add		T1,	T1,	LDC

  700. 

  701. 

  702. #ifndef TRMMKERNEL

  703. 

  704. 	lxvw4x		vs0,	o0,	T1

  705. 	lxvw4x		vs1,	o16,	T1

  706. 	lxvw4x		vs2,	o32,	T1

  707. 	lxvw4x		vs3,	o48,	T1

  708. 

  709. #endif

  710. 

  711. #ifdef TRMMKERNEL

  712. 	xvmulsp		vs0,	vs56,	alpha_vr

  713. 	xvmulsp		vs1,	vs57,	alpha_vr

  714. 	xvmulsp		vs2,	vs58,	alpha_vr

  715. 	xvmulsp		vs3,	vs59,	alpha_vr

  716. #else

  717. 	xvmaddasp	vs0,	vs56,	alpha_vr

  718. 	xvmaddasp	vs1,	vs57,	alpha_vr

  719. 	xvmaddasp	vs2,	vs58,	alpha_vr

  720. 	xvmaddasp	vs3,	vs59,	alpha_vr

  721. #endif

  722. 

  723. 	stxvw4x		vs0,	o0,	T1

  724. 	stxvw4x		vs1,	o16,	T1

  725. 	stxvw4x		vs2,	o32,	T1

  726. 	stxvw4x		vs3,	o48,	T1

  727. 

  728. 	add		T1,	T1,	LDC

  729. 

  730. 

  731. #ifndef TRMMKERNEL

  732. 

  733. 	lxvw4x		vs0,	o0,	T1

  734. 	lxvw4x		vs1,	o16,	T1

  735. 	lxvw4x		vs2,	o32,	T1

  736. 	lxvw4x		vs3,	o48,	T1

  737. 

  738. #endif

  739. 

  740. #ifdef TRMMKERNEL

  741. 	xvmulsp		vs0,	vs60,	alpha_vr

  742. 	xvmulsp		vs1,	vs61,	alpha_vr

  743. 	xvmulsp		vs2,	vs62,	alpha_vr

  744. 	xvmulsp		vs3,	vs63,	alpha_vr

  745. #else

  746. 	xvmaddasp	vs0,	vs60,	alpha_vr

  747. 	xvmaddasp	vs1,	vs61,	alpha_vr

  748. 	xvmaddasp	vs2,	vs62,	alpha_vr

  749. 	xvmaddasp	vs3,	vs63,	alpha_vr

  750. #endif

  751. 

  752. 	stxvw4x		vs0,	o0,	T1

  753. 	stxvw4x		vs1,	o16,	T1

  754. 	stxvw4x		vs2,	o32,	T1

  755. 	stxvw4x		vs3,	o48,	T1

  756. 

  757. 	add		T1,	T1,	LDC

  758. 

  759. 	addi		CO,	CO,	64

  760. 

  761. #if defined(_AIX)

  762. ')

  763. #else

  764. .endm

  765. #endif

  766. 

  767. 

  768. /**********************************************************************************************

  769. * Macros for N=8 and M=8

  770. **********************************************************************************************/

  771. 

  772. #if defined(_AIX)

  773. define(`LOAD8x8_1', `

  774. #else

  775. .macro LOAD8x8_1

  776. #endif

  777. 

  778. 	lxvw4x		vs0,	o0,	AO

  779. 	lxvw4x		vs1,	o16,	AO

  780. 

  781. 	addi		AO,	AO,	32

  782. 

  783. 	mr		T1,	BO

  784. 

  785. 	lxvw4x		vs8,	o0,	T1

  786. 	lxvw4x		vs9,	o16,	T1

  787. 	lxvw4x		vs10,	o32,	T1

  788. 	lxvw4x		vs11,	o48,	T1

  789. 

  790. 	addi		T1,	T1,	64

  791. 

  792. 	lxvw4x		vs12,	o0,	T1

  793. 	lxvw4x		vs13,	o16,	T1

  794. 	lxvw4x		vs14,	o32,	T1

  795. 	lxvw4x		vs15,	o48,	T1

  796. 

  797. 	addi		BO,	BO,	128

  798. 

  799. #if defined(_AIX)

  800. ')

  801. #else

  802. .endm

  803. #endif

  804. 

  805. #if defined(_AIX)

  806. define(`KERNEL8x8_I1', `

  807. #else

  808. .macro KERNEL8x8_I1

  809. #endif

  810. 

  811. 

  812. 	lxvw4x		vs4,	o0,	AO

  813. 	lxvw4x		vs5,	o16,	AO

  814. 

  815. 	addi		AO,	AO,	32

  816. 

  817. 	mr		T1,	BO

  818. 

  819. 	lxvw4x		vs16,	o0,	T1

  820. 	lxvw4x		vs17,	o16,	T1

  821. 	lxvw4x		vs18,	o32,	T1

  822. 	lxvw4x		vs19,	o48,	T1

  823. 

  824. 	addi		T1,	T1,	64

  825. 

  826. 	lxvw4x		vs20,	o0,	T1

  827. 	lxvw4x		vs21,	o16,	T1

  828. 	lxvw4x		vs22,	o32,	T1

  829. 	lxvw4x		vs23,	o48,	T1

  830. 

  831. 	addi		BO,	BO,	128

  832. 

  833. 

  834. 	xvmulsp		vs32,	vs0,	vs8

  835. 	xvmulsp		vs33,	vs1,	vs8

  836. 

  837. 	xvmulsp		vs34,	vs0,	vs9

  838. 	xvmulsp		vs35,	vs1,	vs9

  839. 

  840. 	xvmulsp		vs36,	vs0,	vs10

  841. 	xvmulsp		vs37,	vs1,	vs10

  842. 

  843. 	xvmulsp		vs38,	vs0,	vs11

  844. 	xvmulsp		vs39,	vs1,	vs11

  845. 

  846. 	xvmulsp		vs40,	vs0,	vs12

  847. 	xvmulsp		vs41,	vs1,	vs12

  848. 

  849. 	xvmulsp		vs42,	vs0,	vs13

  850. 	xvmulsp		vs43,	vs1,	vs13

  851. 

  852. 	xvmulsp		vs44,	vs0,	vs14

  853. 	xvmulsp		vs45,	vs1,	vs14

  854. 

  855. 	xvmulsp		vs46,	vs0,	vs15

  856. 	xvmulsp		vs47,	vs1,	vs15

  857. 

  858. 

  859. #if defined(_AIX)

  860. ')

  861. #else

  862. .endm

  863. #endif

  864. 

  865. #if defined(_AIX)

  866. define(`KERNEL8x8_1', `

  867. #else

  868. .macro KERNEL8x8_1

  869. #endif

  870. 

  871. 

  872. 	lxvw4x		vs4,	o0,	AO

  873. 	lxvw4x		vs5,	o16,	AO

  874. 

  875. 	addi		AO,	AO,	32

  876. 

  877. 	mr		T1,	BO

  878. 

  879. 	lxvw4x		vs16,	o0,	T1

  880. 	lxvw4x		vs17,	o16,	T1

  881. 	lxvw4x		vs18,	o32,	T1

  882. 	lxvw4x		vs19,	o48,	T1

  883. 

  884. 	addi		T1,	T1,	64

  885. 

  886. 	lxvw4x		vs20,	o0,	T1

  887. 	lxvw4x		vs21,	o16,	T1

  888. 	lxvw4x		vs22,	o32,	T1

  889. 	lxvw4x		vs23,	o48,	T1

  890. 

  891. 	addi		BO,	BO,	128

  892. 

  893. 

  894. 	xvmaddasp	vs32,	vs0,	vs8

  895. 	xvmaddasp	vs33,	vs1,	vs8

  896. 

  897. 	xvmaddasp	vs34,	vs0,	vs9

  898. 	xvmaddasp	vs35,	vs1,	vs9

  899. 

  900. 	xvmaddasp	vs36,	vs0,	vs10

  901. 	xvmaddasp	vs37,	vs1,	vs10

  902. 

  903. 	xvmaddasp	vs38,	vs0,	vs11

  904. 	xvmaddasp	vs39,	vs1,	vs11

  905. 

  906. 	xvmaddasp	vs40,	vs0,	vs12

  907. 	xvmaddasp	vs41,	vs1,	vs12

  908. 

  909. 	xvmaddasp	vs42,	vs0,	vs13

  910. 	xvmaddasp	vs43,	vs1,	vs13

  911. 

  912. 	xvmaddasp	vs44,	vs0,	vs14

  913. 	xvmaddasp	vs45,	vs1,	vs14

  914. 

  915. 	xvmaddasp	vs46,	vs0,	vs15

  916. 	xvmaddasp	vs47,	vs1,	vs15

  917. 

  918. 

  919. #if defined(_AIX)

  920. ')

  921. #else

  922. .endm

  923. #endif

  924. 

  925. #if defined(_AIX)

  926. define(`KERNEL8x8_2', `

  927. #else

  928. .macro KERNEL8x8_2

  929. #endif

  930. 

  931. 

  932. 	lxvw4x		vs0,	o0,	AO

  933. 	lxvw4x		vs1,	o16,	AO

  934. 

  935. 	addi		AO,	AO,	32

  936. 

  937. 	mr		T1,	BO

  938. 

  939. 	lxvw4x		vs8,	o0,	T1

  940. 	lxvw4x		vs9,	o16,	T1

  941. 	lxvw4x		vs10,	o32,	T1

  942. 	lxvw4x		vs11,	o48,	T1

  943. 

  944. 	addi		T1,	T1,	64

  945. 

  946. 	lxvw4x		vs12,	o0,	T1

  947. 	lxvw4x		vs13,	o16,	T1

  948. 	lxvw4x		vs14,	o32,	T1

  949. 	lxvw4x		vs15,	o48,	T1

  950. 

  951. 	addi		BO,	BO,	128

  952. 

  953. 

  954. 	xvmaddasp	vs32,	vs4,	vs16

  955. 	xvmaddasp	vs33,	vs5,	vs16

  956. 

  957. 	xvmaddasp	vs34,	vs4,	vs17

  958. 	xvmaddasp	vs35,	vs5,	vs17

  959. 

  960. 	xvmaddasp	vs36,	vs4,	vs18

  961. 	xvmaddasp	vs37,	vs5,	vs18

  962. 

  963. 	xvmaddasp	vs38,	vs4,	vs19

  964. 	xvmaddasp	vs39,	vs5,	vs19

  965. 

  966. 	xvmaddasp	vs40,	vs4,	vs20

  967. 	xvmaddasp	vs41,	vs5,	vs20

  968. 

  969. 	xvmaddasp	vs42,	vs4,	vs21

  970. 	xvmaddasp	vs43,	vs5,	vs21

  971. 

  972. 	xvmaddasp	vs44,	vs4,	vs22

  973. 	xvmaddasp	vs45,	vs5,	vs22

  974. 

  975. 	xvmaddasp	vs46,	vs4,	vs23

  976. 	xvmaddasp	vs47,	vs5,	vs23

  977. 

  978. 

  979. #if defined(_AIX)

  980. ')

  981. #else

  982. .endm

  983. #endif

  984. 

  985. #if defined(_AIX)

  986. define(`KERNEL8x8_E2', `

  987. #else

  988. .macro KERNEL8x8_E2

  989. #endif

  990. 

  991. 

  992. 	xvmaddasp	vs32,	vs4,	vs16

  993. 	xvmaddasp	vs33,	vs5,	vs16

  994. 

  995. 	xvmaddasp	vs34,	vs4,	vs17

  996. 	xvmaddasp	vs35,	vs5,	vs17

  997. 

  998. 	xvmaddasp	vs36,	vs4,	vs18

  999. 	xvmaddasp	vs37,	vs5,	vs18

  1000. 

  1001. 	xvmaddasp	vs38,	vs4,	vs19

  1002. 	xvmaddasp	vs39,	vs5,	vs19

  1003. 

  1004. 	xvmaddasp	vs40,	vs4,	vs20

  1005. 	xvmaddasp	vs41,	vs5,	vs20

  1006. 

  1007. 	xvmaddasp	vs42,	vs4,	vs21

  1008. 	xvmaddasp	vs43,	vs5,	vs21

  1009. 

  1010. 	xvmaddasp	vs44,	vs4,	vs22

  1011. 	xvmaddasp	vs45,	vs5,	vs22

  1012. 

  1013. 	xvmaddasp	vs46,	vs4,	vs23

  1014. 	xvmaddasp	vs47,	vs5,	vs23

  1015. 

  1016. 

  1017. #if defined(_AIX)

  1018. ')

  1019. #else

  1020. .endm

  1021. #endif

  1022. 

  1023. #if defined(_AIX)

  1024. define(`KERNEL8x8_SUBI1', `

  1025. #else

  1026. .macro KERNEL8x8_SUBI1

  1027. #endif

  1028. 

  1029. 

  1030. 	lxvw4x		vs0,	o0,	AO

  1031. 	lxvw4x		vs1,	o16,	AO

  1032. 

  1033. 	addi		AO,	AO,	32

  1034. 

  1035. 	mr		T1,	BO

  1036. 

  1037. 	lxvw4x		vs8,	o0,	T1

  1038. 	lxvw4x		vs9,	o16,	T1

  1039. 	lxvw4x		vs10,	o32,	T1

  1040. 	lxvw4x		vs11,	o48,	T1

  1041. 

  1042. 	addi		T1,	T1,	64

  1043. 

  1044. 	lxvw4x		vs12,	o0,	T1

  1045. 	lxvw4x		vs13,	o16,	T1

  1046. 	lxvw4x		vs14,	o32,	T1

  1047. 	lxvw4x		vs15,	o48,	T1

  1048. 

  1049. 	addi		BO,	BO,	128

  1050. 

  1051. 

  1052. 	xvmulsp		vs32,	vs0,	vs8

  1053. 	xvmulsp		vs33,	vs1,	vs8

  1054. 

  1055. 	xvmulsp		vs34,	vs0,	vs9

  1056. 	xvmulsp		vs35,	vs1,	vs9

  1057. 

  1058. 	xvmulsp		vs36,	vs0,	vs10

  1059. 	xvmulsp		vs37,	vs1,	vs10

  1060. 

  1061. 	xvmulsp		vs38,	vs0,	vs11

  1062. 	xvmulsp		vs39,	vs1,	vs11

  1063. 

  1064. 	xvmulsp		vs40,	vs0,	vs12

  1065. 	xvmulsp		vs41,	vs1,	vs12

  1066. 

  1067. 	xvmulsp		vs42,	vs0,	vs13

  1068. 	xvmulsp		vs43,	vs1,	vs13

  1069. 

  1070. 	xvmulsp		vs44,	vs0,	vs14

  1071. 	xvmulsp		vs45,	vs1,	vs14

  1072. 

  1073. 	xvmulsp		vs46,	vs0,	vs15

  1074. 	xvmulsp		vs47,	vs1,	vs15

  1075. 

  1076. 

  1077. #if defined(_AIX)

  1078. ')

  1079. #else

  1080. .endm

  1081. #endif

  1082. 

  1083. #if defined(_AIX)

  1084. define(`KERNEL8x8_SUB1', `

  1085. #else

  1086. .macro KERNEL8x8_SUB1

  1087. #endif

  1088. 

  1089. 

  1090. 	lxvw4x		vs0,	o0,	AO

  1091. 	lxvw4x		vs1,	o16,	AO

  1092. 

  1093. 	addi		AO,	AO,	32

  1094. 

  1095. 	mr		T1,	BO

  1096. 

  1097. 	lxvw4x		vs8,	o0,	T1

  1098. 	lxvw4x		vs9,	o16,	T1

  1099. 	lxvw4x		vs10,	o32,	T1

  1100. 	lxvw4x		vs11,	o48,	T1

  1101. 

  1102. 	addi		T1,	T1,	64

  1103. 

  1104. 	lxvw4x		vs12,	o0,	T1

  1105. 	lxvw4x		vs13,	o16,	T1

  1106. 	lxvw4x		vs14,	o32,	T1

  1107. 	lxvw4x		vs15,	o48,	T1

  1108. 

  1109. 	addi		BO,	BO,	128

  1110. 

  1111. 

  1112. 	xvmaddasp	vs32,	vs0,	vs8

  1113. 	xvmaddasp	vs33,	vs1,	vs8

  1114. 

  1115. 	xvmaddasp	vs34,	vs0,	vs9

  1116. 	xvmaddasp	vs35,	vs1,	vs9

  1117. 

  1118. 	xvmaddasp	vs36,	vs0,	vs10

  1119. 	xvmaddasp	vs37,	vs1,	vs10

  1120. 

  1121. 	xvmaddasp	vs38,	vs0,	vs11

  1122. 	xvmaddasp	vs39,	vs1,	vs11

  1123. 

  1124. 	xvmaddasp	vs40,	vs0,	vs12

  1125. 	xvmaddasp	vs41,	vs1,	vs12

  1126. 

  1127. 	xvmaddasp	vs42,	vs0,	vs13

  1128. 	xvmaddasp	vs43,	vs1,	vs13

  1129. 

  1130. 	xvmaddasp	vs44,	vs0,	vs14

  1131. 	xvmaddasp	vs45,	vs1,	vs14

  1132. 

  1133. 	xvmaddasp	vs46,	vs0,	vs15

  1134. 	xvmaddasp	vs47,	vs1,	vs15

  1135. 

  1136. 

  1137. #if defined(_AIX)

  1138. ')

  1139. #else

  1140. .endm

  1141. #endif

  1142. 

  1143. #if defined(_AIX)

  1144. define(`SAVE8x8', `

  1145. #else

  1146. .macro SAVE8x8

  1147. #endif

  1148. 

  1149. 	mr		T1,	CO

  1150. 

  1151. #ifndef TRMMKERNEL

  1152. 

  1153. 	lxvw4x		vs0,	o0,	T1

  1154. 	lxvw4x		vs1,	o16,	T1

  1155. 

  1156. #endif

  1157. 

  1158. #ifdef TRMMKERNEL

  1159. 	xvmulsp		vs0,	vs32,	alpha_vr

  1160. 	xvmulsp		vs1,	vs33,	alpha_vr

  1161. #else

  1162. 	xvmaddasp	vs0,	vs32,	alpha_vr

  1163. 	xvmaddasp	vs1,	vs33,	alpha_vr

  1164. #endif

  1165. 

  1166. 	stxvw4x		vs0,	o0,	T1

  1167. 	stxvw4x		vs1,	o16,	T1

  1168. 

  1169. 	add		T1,	T1,	LDC

  1170. 

  1171. 

  1172. #ifndef TRMMKERNEL

  1173. 

  1174. 	lxvw4x		vs0,	o0,	T1

  1175. 	lxvw4x		vs1,	o16,	T1

  1176. 

  1177. #endif

  1178. 

  1179. #ifdef TRMMKERNEL

  1180. 	xvmulsp		vs0,	vs34,	alpha_vr

  1181. 	xvmulsp		vs1,	vs35,	alpha_vr

  1182. #else

  1183. 	xvmaddasp	vs0,	vs34,	alpha_vr

  1184. 	xvmaddasp	vs1,	vs35,	alpha_vr

  1185. #endif

  1186. 

  1187. 	stxvw4x		vs0,	o0,	T1

  1188. 	stxvw4x		vs1,	o16,	T1

  1189. 

  1190. 	add		T1,	T1,	LDC

  1191. 

  1192. 

  1193. #ifndef TRMMKERNEL

  1194. 

  1195. 	lxvw4x		vs0,	o0,	T1

  1196. 	lxvw4x		vs1,	o16,	T1

  1197. 

  1198. #endif

  1199. 

  1200. #ifdef TRMMKERNEL

  1201. 	xvmulsp		vs0,	vs36,	alpha_vr

  1202. 	xvmulsp		vs1,	vs37,	alpha_vr

  1203. #else

  1204. 	xvmaddasp	vs0,	vs36,	alpha_vr

  1205. 	xvmaddasp	vs1,	vs37,	alpha_vr

  1206. #endif

  1207. 

  1208. 	stxvw4x		vs0,	o0,	T1

  1209. 	stxvw4x		vs1,	o16,	T1

  1210. 

  1211. 	add		T1,	T1,	LDC

  1212. 

  1213. 

  1214. #ifndef TRMMKERNEL

  1215. 

  1216. 	lxvw4x		vs0,	o0,	T1

  1217. 	lxvw4x		vs1,	o16,	T1

  1218. 

  1219. #endif

  1220. 

  1221. #ifdef TRMMKERNEL

  1222. 	xvmulsp		vs0,	vs38,	alpha_vr

  1223. 	xvmulsp		vs1,	vs39,	alpha_vr

  1224. #else

  1225. 	xvmaddasp	vs0,	vs38,	alpha_vr

  1226. 	xvmaddasp	vs1,	vs39,	alpha_vr

  1227. #endif

  1228. 

  1229. 	stxvw4x		vs0,	o0,	T1

  1230. 	stxvw4x		vs1,	o16,	T1

  1231. 

  1232. 	add		T1,	T1,	LDC

  1233. 

  1234. 

  1235. #ifndef TRMMKERNEL

  1236. 

  1237. 	lxvw4x		vs0,	o0,	T1

  1238. 	lxvw4x		vs1,	o16,	T1

  1239. 

  1240. #endif

  1241. 

  1242. #ifdef TRMMKERNEL

  1243. 	xvmulsp		vs0,	vs40,	alpha_vr

  1244. 	xvmulsp		vs1,	vs41,	alpha_vr

  1245. #else

  1246. 	xvmaddasp	vs0,	vs40,	alpha_vr

  1247. 	xvmaddasp	vs1,	vs41,	alpha_vr

  1248. #endif

  1249. 

  1250. 	stxvw4x		vs0,	o0,	T1

  1251. 	stxvw4x		vs1,	o16,	T1

  1252. 

  1253. 	add		T1,	T1,	LDC

  1254. 

  1255. 

  1256. #ifndef TRMMKERNEL

  1257. 

  1258. 	lxvw4x		vs0,	o0,	T1

  1259. 	lxvw4x		vs1,	o16,	T1

  1260. 

  1261. #endif

  1262. 

  1263. #ifdef TRMMKERNEL

  1264. 	xvmulsp		vs0,	vs42,	alpha_vr

  1265. 	xvmulsp		vs1,	vs43,	alpha_vr

  1266. #else

  1267. 	xvmaddasp	vs0,	vs42,	alpha_vr

  1268. 	xvmaddasp	vs1,	vs43,	alpha_vr

  1269. #endif

  1270. 

  1271. 	stxvw4x		vs0,	o0,	T1

  1272. 	stxvw4x		vs1,	o16,	T1

  1273. 

  1274. 	add		T1,	T1,	LDC

  1275. 

  1276. 

  1277. #ifndef TRMMKERNEL

  1278. 

  1279. 	lxvw4x		vs0,	o0,	T1

  1280. 	lxvw4x		vs1,	o16,	T1

  1281. 

  1282. #endif

  1283. 

  1284. #ifdef TRMMKERNEL

  1285. 	xvmulsp		vs0,	vs44,	alpha_vr

  1286. 	xvmulsp		vs1,	vs45,	alpha_vr

  1287. #else

  1288. 	xvmaddasp	vs0,	vs44,	alpha_vr

  1289. 	xvmaddasp	vs1,	vs45,	alpha_vr

  1290. #endif

  1291. 

  1292. 	stxvw4x		vs0,	o0,	T1

  1293. 	stxvw4x		vs1,	o16,	T1

  1294. 

  1295. 	add		T1,	T1,	LDC

  1296. 

  1297. 

  1298. #ifndef TRMMKERNEL

  1299. 

  1300. 	lxvw4x		vs0,	o0,	T1

  1301. 	lxvw4x		vs1,	o16,	T1

  1302. 

  1303. #endif

  1304. 

  1305. #ifdef TRMMKERNEL

  1306. 	xvmulsp		vs0,	vs46,	alpha_vr

  1307. 	xvmulsp		vs1,	vs47,	alpha_vr

  1308. #else

  1309. 	xvmaddasp	vs0,	vs46,	alpha_vr

  1310. 	xvmaddasp	vs1,	vs47,	alpha_vr

  1311. #endif

  1312. 

  1313. 	stxvw4x		vs0,	o0,	T1

  1314. 	stxvw4x		vs1,	o16,	T1

  1315. 

  1316. 	add		T1,	T1,	LDC

  1317. 

  1318. 	addi		CO,	CO,	32

  1319. 

  1320. #if defined(_AIX)

  1321. ')

  1322. #else

  1323. .endm

  1324. #endif

  1325. 

  1326. 

  1327. /**********************************************************************************************

  1328. * Macros for N=8 and M=4

  1329. **********************************************************************************************/

  1330. 

  1331. #if defined(_AIX)

  1332. define(`LOAD8x4_1', `

  1333. #else

  1334. .macro LOAD8x4_1

  1335. #endif

  1336. 

  1337. 	lxvw4x		vs0,	o0,	AO

  1338. 

  1339. 	addi		AO,	AO,	16

  1340. 

  1341. 	mr		T1,	BO

  1342. 

  1343. 	lxvw4x		vs8,	o0,	T1

  1344. 	lxvw4x		vs9,	o16,	T1

  1345. 	lxvw4x		vs10,	o32,	T1

  1346. 	lxvw4x		vs11,	o48,	T1

  1347. 

  1348. 	addi		T1,	T1,	64

  1349. 

  1350. 	lxvw4x		vs12,	o0,	T1

  1351. 	lxvw4x		vs13,	o16,	T1

  1352. 	lxvw4x		vs14,	o32,	T1

  1353. 	lxvw4x		vs15,	o48,	T1

  1354. 

  1355. 	addi		BO,	BO,	128

  1356. 

  1357. #if defined(_AIX)

  1358. ')

  1359. #else

  1360. .endm

  1361. #endif

  1362. 

  1363. #if defined(_AIX)

  1364. define(`KERNEL8x4_I1', `

  1365. #else

  1366. .macro KERNEL8x4_I1

  1367. #endif

  1368. 

  1369. 

  1370. 	lxvw4x		vs4,	o0,	AO

  1371. 

  1372. 	addi		AO,	AO,	16

  1373. 

  1374. 	mr		T1,	BO

  1375. 

  1376. 	lxvw4x		vs16,	o0,	T1

  1377. 	lxvw4x		vs17,	o16,	T1

  1378. 	lxvw4x		vs18,	o32,	T1

  1379. 	lxvw4x		vs19,	o48,	T1

  1380. 

  1381. 	addi		T1,	T1,	64

  1382. 

  1383. 	lxvw4x		vs20,	o0,	T1

  1384. 	lxvw4x		vs21,	o16,	T1

  1385. 	lxvw4x		vs22,	o32,	T1

  1386. 	lxvw4x		vs23,	o48,	T1

  1387. 

  1388. 	addi		BO,	BO,	128

  1389. 

  1390. 

  1391. 	xvmulsp		vs32,	vs0,	vs8

  1392. 

  1393. 	xvmulsp		vs33,	vs0,	vs9

  1394. 

  1395. 	xvmulsp		vs34,	vs0,	vs10

  1396. 

  1397. 	xvmulsp		vs35,	vs0,	vs11

  1398. 

  1399. 	xvmulsp		vs36,	vs0,	vs12

  1400. 

  1401. 	xvmulsp		vs37,	vs0,	vs13

  1402. 

  1403. 	xvmulsp		vs38,	vs0,	vs14

  1404. 

  1405. 	xvmulsp		vs39,	vs0,	vs15

  1406. 

  1407. 

  1408. #if defined(_AIX)

  1409. ')

  1410. #else

  1411. .endm

  1412. #endif

  1413. 

  1414. #if defined(_AIX)

  1415. define(`KERNEL8x4_1', `

  1416. #else

  1417. .macro KERNEL8x4_1

  1418. #endif

  1419. 

  1420. 

  1421. 	lxvw4x		vs4,	o0,	AO

  1422. 

  1423. 	addi		AO,	AO,	16

  1424. 

  1425. 	mr		T1,	BO

  1426. 

  1427. 	lxvw4x		vs16,	o0,	T1

  1428. 	lxvw4x		vs17,	o16,	T1

  1429. 	lxvw4x		vs18,	o32,	T1

  1430. 	lxvw4x		vs19,	o48,	T1

  1431. 

  1432. 	addi		T1,	T1,	64

  1433. 

  1434. 	lxvw4x		vs20,	o0,	T1

  1435. 	lxvw4x		vs21,	o16,	T1

  1436. 	lxvw4x		vs22,	o32,	T1

  1437. 	lxvw4x		vs23,	o48,	T1

  1438. 

  1439. 	addi		BO,	BO,	128

  1440. 

  1441. 

  1442. 	xvmaddasp	vs32,	vs0,	vs8

  1443. 

  1444. 	xvmaddasp	vs33,	vs0,	vs9

  1445. 

  1446. 	xvmaddasp	vs34,	vs0,	vs10

  1447. 

  1448. 	xvmaddasp	vs35,	vs0,	vs11

  1449. 

  1450. 	xvmaddasp	vs36,	vs0,	vs12

  1451. 

  1452. 	xvmaddasp	vs37,	vs0,	vs13

  1453. 

  1454. 	xvmaddasp	vs38,	vs0,	vs14

  1455. 

  1456. 	xvmaddasp	vs39,	vs0,	vs15

  1457. 

  1458. 

  1459. #if defined(_AIX)

  1460. ')

  1461. #else

  1462. .endm

  1463. #endif

  1464. 

  1465. #if defined(_AIX)

  1466. define(`KERNEL8x4_2', `

  1467. #else

  1468. .macro KERNEL8x4_2

  1469. #endif

  1470. 

  1471. 

  1472. 	lxvw4x		vs0,	o0,	AO

  1473. 

  1474. 	addi		AO,	AO,	16

  1475. 

  1476. 	mr		T1,	BO

  1477. 

  1478. 	lxvw4x		vs8,	o0,	T1

  1479. 	lxvw4x		vs9,	o16,	T1

  1480. 	lxvw4x		vs10,	o32,	T1

  1481. 	lxvw4x		vs11,	o48,	T1

  1482. 

  1483. 	addi		T1,	T1,	64

  1484. 

  1485. 	lxvw4x		vs12,	o0,	T1

  1486. 	lxvw4x		vs13,	o16,	T1

  1487. 	lxvw4x		vs14,	o32,	T1

  1488. 	lxvw4x		vs15,	o48,	T1

  1489. 

  1490. 	addi		BO,	BO,	128

  1491. 

  1492. 

  1493. 	xvmaddasp	vs32,	vs4,	vs16

  1494. 

  1495. 	xvmaddasp	vs33,	vs4,	vs17

  1496. 

  1497. 	xvmaddasp	vs34,	vs4,	vs18

  1498. 

  1499. 	xvmaddasp	vs35,	vs4,	vs19

  1500. 

  1501. 	xvmaddasp	vs36,	vs4,	vs20

  1502. 

  1503. 	xvmaddasp	vs37,	vs4,	vs21

  1504. 

  1505. 	xvmaddasp	vs38,	vs4,	vs22

  1506. 

  1507. 	xvmaddasp	vs39,	vs4,	vs23

  1508. 

  1509. 

  1510. #if defined(_AIX)

  1511. ')

  1512. #else

  1513. .endm

  1514. #endif

  1515. 

  1516. #if defined(_AIX)

  1517. define(`KERNEL8x4_E2', `

  1518. #else

  1519. .macro KERNEL8x4_E2

  1520. #endif

  1521. 

  1522. 

  1523. 	xvmaddasp	vs32,	vs4,	vs16

  1524. 

  1525. 	xvmaddasp	vs33,	vs4,	vs17

  1526. 

  1527. 	xvmaddasp	vs34,	vs4,	vs18

  1528. 

  1529. 	xvmaddasp	vs35,	vs4,	vs19

  1530. 

  1531. 	xvmaddasp	vs36,	vs4,	vs20

  1532. 

  1533. 	xvmaddasp	vs37,	vs4,	vs21

  1534. 

  1535. 	xvmaddasp	vs38,	vs4,	vs22

  1536. 

  1537. 	xvmaddasp	vs39,	vs4,	vs23

  1538. 

  1539. 

  1540. #if defined(_AIX)

  1541. ')

  1542. #else

  1543. .endm

  1544. #endif

  1545. 

  1546. #if defined(_AIX)

  1547. define(`KERNEL8x4_SUBI1', `

  1548. #else

  1549. .macro KERNEL8x4_SUBI1

  1550. #endif

  1551. 

  1552. 

  1553. 	lxvw4x		vs0,	o0,	AO

  1554. 

  1555. 	addi		AO,	AO,	16

  1556. 

  1557. 	mr		T1,	BO

  1558. 

  1559. 	lxvw4x		vs8,	o0,	T1

  1560. 	lxvw4x		vs9,	o16,	T1

  1561. 	lxvw4x		vs10,	o32,	T1

  1562. 	lxvw4x		vs11,	o48,	T1

  1563. 

  1564. 	addi		T1,	T1,	64

  1565. 

  1566. 	lxvw4x		vs12,	o0,	T1

  1567. 	lxvw4x		vs13,	o16,	T1

  1568. 	lxvw4x		vs14,	o32,	T1

  1569. 	lxvw4x		vs15,	o48,	T1

  1570. 

  1571. 	addi		BO,	BO,	128

  1572. 

  1573. 

  1574. 	xvmulsp		vs32,	vs0,	vs8

  1575. 

  1576. 	xvmulsp		vs33,	vs0,	vs9

  1577. 

  1578. 	xvmulsp		vs34,	vs0,	vs10

  1579. 

  1580. 	xvmulsp		vs35,	vs0,	vs11

  1581. 

  1582. 	xvmulsp		vs36,	vs0,	vs12

  1583. 

  1584. 	xvmulsp		vs37,	vs0,	vs13

  1585. 

  1586. 	xvmulsp		vs38,	vs0,	vs14

  1587. 

  1588. 	xvmulsp		vs39,	vs0,	vs15

  1589. 

  1590. 

  1591. #if defined(_AIX)

  1592. ')

  1593. #else

  1594. .endm

  1595. #endif

  1596. 

  1597. #if defined(_AIX)

  1598. define(`KERNEL8x4_SUB1', `

  1599. #else

  1600. .macro KERNEL8x4_SUB1

  1601. #endif

  1602. 

  1603. 

  1604. 	lxvw4x		vs0,	o0,	AO

  1605. 

  1606. 	addi		AO,	AO,	16

  1607. 

  1608. 	mr		T1,	BO

  1609. 

  1610. 	lxvw4x		vs8,	o0,	T1

  1611. 	lxvw4x		vs9,	o16,	T1

  1612. 	lxvw4x		vs10,	o32,	T1

  1613. 	lxvw4x		vs11,	o48,	T1

  1614. 

  1615. 	addi		T1,	T1,	64

  1616. 

  1617. 	lxvw4x		vs12,	o0,	T1

  1618. 	lxvw4x		vs13,	o16,	T1

  1619. 	lxvw4x		vs14,	o32,	T1

  1620. 	lxvw4x		vs15,	o48,	T1

  1621. 

  1622. 	addi		BO,	BO,	128

  1623. 

  1624. 

  1625. 	xvmaddasp	vs32,	vs0,	vs8

  1626. 

  1627. 	xvmaddasp	vs33,	vs0,	vs9

  1628. 

  1629. 	xvmaddasp	vs34,	vs0,	vs10

  1630. 

  1631. 	xvmaddasp	vs35,	vs0,	vs11

  1632. 

  1633. 	xvmaddasp	vs36,	vs0,	vs12

  1634. 

  1635. 	xvmaddasp	vs37,	vs0,	vs13

  1636. 

  1637. 	xvmaddasp	vs38,	vs0,	vs14

  1638. 

  1639. 	xvmaddasp	vs39,	vs0,	vs15

  1640. 

  1641. 

  1642. #if defined(_AIX)

  1643. ')

  1644. #else

  1645. .endm

  1646. #endif

  1647. 

  1648. #if defined(_AIX)

  1649. define(`SAVE8x4', `

  1650. #else

  1651. .macro SAVE8x4

  1652. #endif

  1653. 

  1654. 	mr		T1,	CO

  1655. 

  1656. #ifndef TRMMKERNEL

  1657. 

  1658. 	lxvw4x		vs0,	o0,	T1

  1659. 

  1660. #endif

  1661. 

  1662. #ifdef TRMMKERNEL

  1663. 	xvmulsp		vs0,	vs32,	alpha_vr

  1664. #else

  1665. 	xvmaddasp	vs0,	vs32,	alpha_vr

  1666. #endif

  1667. 

  1668. 	stxvw4x		vs0,	o0,	T1

  1669. 

  1670. 	add		T1,	T1,	LDC

  1671. 

  1672. 

  1673. #ifndef TRMMKERNEL

  1674. 

  1675. 	lxvw4x		vs0,	o0,	T1

  1676. 

  1677. #endif

  1678. 

  1679. #ifdef TRMMKERNEL

  1680. 	xvmulsp		vs0,	vs33,	alpha_vr

  1681. #else

  1682. 	xvmaddasp	vs0,	vs33,	alpha_vr

  1683. #endif

  1684. 

  1685. 	stxvw4x		vs0,	o0,	T1

  1686. 

  1687. 	add		T1,	T1,	LDC

  1688. 

  1689. 

  1690. #ifndef TRMMKERNEL

  1691. 

  1692. 	lxvw4x		vs0,	o0,	T1

  1693. 

  1694. #endif

  1695. 

  1696. #ifdef TRMMKERNEL

  1697. 	xvmulsp		vs0,	vs34,	alpha_vr

  1698. #else

  1699. 	xvmaddasp	vs0,	vs34,	alpha_vr

  1700. #endif

  1701. 

  1702. 	stxvw4x		vs0,	o0,	T1

  1703. 

  1704. 	add		T1,	T1,	LDC

  1705. 

  1706. 

  1707. #ifndef TRMMKERNEL

  1708. 

  1709. 	lxvw4x		vs0,	o0,	T1

  1710. 

  1711. #endif

  1712. 

  1713. #ifdef TRMMKERNEL

  1714. 	xvmulsp		vs0,	vs35,	alpha_vr

  1715. #else

  1716. 	xvmaddasp	vs0,	vs35,	alpha_vr

  1717. #endif

  1718. 

  1719. 	stxvw4x		vs0,	o0,	T1

  1720. 

  1721. 	add		T1,	T1,	LDC

  1722. 

  1723. 

  1724. #ifndef TRMMKERNEL

  1725. 

  1726. 	lxvw4x		vs0,	o0,	T1

  1727. 

  1728. #endif

  1729. 

  1730. #ifdef TRMMKERNEL

  1731. 	xvmulsp		vs0,	vs36,	alpha_vr

  1732. #else

  1733. 	xvmaddasp	vs0,	vs36,	alpha_vr

  1734. #endif

  1735. 

  1736. 	stxvw4x		vs0,	o0,	T1

  1737. 

  1738. 	add		T1,	T1,	LDC

  1739. 

  1740. 

  1741. #ifndef TRMMKERNEL

  1742. 

  1743. 	lxvw4x		vs0,	o0,	T1

  1744. 

  1745. #endif

  1746. 

  1747. #ifdef TRMMKERNEL

  1748. 	xvmulsp		vs0,	vs37,	alpha_vr

  1749. #else

  1750. 	xvmaddasp	vs0,	vs37,	alpha_vr

  1751. #endif

  1752. 

  1753. 	stxvw4x		vs0,	o0,	T1

  1754. 

  1755. 	add		T1,	T1,	LDC

  1756. 

  1757. 

  1758. #ifndef TRMMKERNEL

  1759. 

  1760. 	lxvw4x		vs0,	o0,	T1

  1761. 

  1762. #endif

  1763. 

  1764. #ifdef TRMMKERNEL

  1765. 	xvmulsp		vs0,	vs38,	alpha_vr

  1766. #else

  1767. 	xvmaddasp	vs0,	vs38,	alpha_vr

  1768. #endif

  1769. 

  1770. 	stxvw4x		vs0,	o0,	T1

  1771. 

  1772. 	add		T1,	T1,	LDC

  1773. 

  1774. 

  1775. #ifndef TRMMKERNEL

  1776. 

  1777. 	lxvw4x		vs0,	o0,	T1

  1778. 

  1779. #endif

  1780. 

  1781. #ifdef TRMMKERNEL

  1782. 	xvmulsp		vs0,	vs39,	alpha_vr

  1783. #else

  1784. 	xvmaddasp	vs0,	vs39,	alpha_vr

  1785. #endif

  1786. 

  1787. 	stxvw4x		vs0,	o0,	T1

  1788. 

  1789. 	add		T1,	T1,	LDC

  1790. 

  1791. 	addi		CO,	CO,	16

  1792. 

  1793. #if defined(_AIX)

  1794. ')

  1795. #else

  1796. .endm

  1797. #endif

  1798. 

  1799. 

  1800. /**********************************************************************************************

  1801. * Macros for N=8 and M=2

  1802. **********************************************************************************************/

  1803. 

  1804. #if defined(_AIX)

  1805. define(`LOAD8x2_1', `

  1806. #else

  1807. .macro LOAD8x2_1

  1808. #endif

  1809. 

  1810. 	lxsspx		vs0,	o0,	AO

  1811. 	lxsspx		vs1,	o4,	AO

  1812. 

  1813. 	addi		AO,	AO,	8

  1814. 

  1815. 	mr		T1,	BO

  1816. 

  1817. 	lxsspx		vs8,	o0,	T1

  1818. 	lxsspx		vs9,	o16,	T1

  1819. 	lxsspx		vs10,	o32,	T1

  1820. 	lxsspx		vs11,	o48,	T1

  1821. 

  1822. 	addi		T1,	T1,	64

  1823. 

  1824. 	lxsspx		vs12,	o0,	T1

  1825. 	lxsspx		vs13,	o16,	T1

  1826. 	lxsspx		vs14,	o32,	T1

  1827. 	lxsspx		vs15,	o48,	T1

  1828. 

  1829. 

  1830. 	addi		BO,	BO,	128

  1831. 

  1832. #if defined(_AIX)

  1833. ')

  1834. #else

  1835. .endm

  1836. #endif

  1837. 

  1838. #if defined(_AIX)

  1839. define(`KERNEL8x2_I1', `

  1840. #else

  1841. .macro KERNEL8x2_I1

  1842. #endif

  1843. 

  1844. 

  1845. 	lxsspx		vs4,	o0,	AO

  1846. 	lxsspx		vs5,	o4,	AO

  1847. 

  1848. 	addi		AO,	AO,	8

  1849. 

  1850. 	mr		T1,	BO

  1851. 

  1852. 	lxsspx		vs16,	o0,	T1

  1853. 	lxsspx		vs17,	o16,	T1

  1854. 	lxsspx		vs18,	o32,	T1

  1855. 	lxsspx		vs19,	o48,	T1

  1856. 

  1857. 	addi		T1,	T1,	64

  1858. 

  1859. 	lxsspx		vs20,	o0,	T1

  1860. 	lxsspx		vs21,	o16,	T1

  1861. 	lxsspx		vs22,	o32,	T1

  1862. 	lxsspx		vs23,	o48,	T1

  1863. 

  1864. 

  1865. 	addi		BO,	BO,	128

  1866. 

  1867. 

  1868. 	xsmuldp		vs32,	vs0,	vs8

  1869. 	xsmuldp		vs33,	vs1,	vs8

  1870. 

  1871. 	xsmuldp		vs34,	vs0,	vs9

  1872. 	xsmuldp		vs35,	vs1,	vs9

  1873. 

  1874. 	xsmuldp		vs36,	vs0,	vs10

  1875. 	xsmuldp		vs37,	vs1,	vs10

  1876. 

  1877. 	xsmuldp		vs38,	vs0,	vs11

  1878. 	xsmuldp		vs39,	vs1,	vs11

  1879. 

  1880. 	xsmuldp		vs40,	vs0,	vs12

  1881. 	xsmuldp		vs41,	vs1,	vs12

  1882. 

  1883. 	xsmuldp		vs42,	vs0,	vs13

  1884. 	xsmuldp		vs43,	vs1,	vs13

  1885. 

  1886. 	xsmuldp		vs44,	vs0,	vs14

  1887. 	xsmuldp		vs45,	vs1,	vs14

  1888. 

  1889. 	xsmuldp		vs46,	vs0,	vs15

  1890. 	xsmuldp		vs47,	vs1,	vs15

  1891. 

  1892. 

  1893. #if defined(_AIX)

  1894. ')

  1895. #else

  1896. .endm

  1897. #endif

  1898. 

  1899. #if defined(_AIX)

  1900. define(`KERNEL8x2_1', `

  1901. #else

  1902. .macro KERNEL8x2_1

  1903. #endif

  1904. 

  1905. 

  1906. 	lxsspx		vs4,	o0,	AO

  1907. 	lxsspx		vs5,	o4,	AO

  1908. 

  1909. 	addi		AO,	AO,	8

  1910. 

  1911. 	mr		T1,	BO

  1912. 

  1913. 	lxsspx		vs16,	o0,	T1

  1914. 	lxsspx		vs17,	o16,	T1

  1915. 	lxsspx		vs18,	o32,	T1

  1916. 	lxsspx		vs19,	o48,	T1

  1917. 

  1918. 	addi		T1,	T1,	64

  1919. 

  1920. 	lxsspx		vs20,	o0,	T1

  1921. 	lxsspx		vs21,	o16,	T1

  1922. 	lxsspx		vs22,	o32,	T1

  1923. 	lxsspx		vs23,	o48,	T1

  1924. 

  1925. 

  1926. 	addi		BO,	BO,	128

  1927. 

  1928. 

  1929. 	xsmaddadp	vs32,	vs0,	vs8

  1930. 	xsmaddadp	vs33,	vs1,	vs8

  1931. 

  1932. 	xsmaddadp	vs34,	vs0,	vs9

  1933. 	xsmaddadp	vs35,	vs1,	vs9

  1934. 

  1935. 	xsmaddadp	vs36,	vs0,	vs10

  1936. 	xsmaddadp	vs37,	vs1,	vs10

  1937. 

  1938. 	xsmaddadp	vs38,	vs0,	vs11

  1939. 	xsmaddadp	vs39,	vs1,	vs11

  1940. 

  1941. 	xsmaddadp	vs40,	vs0,	vs12

  1942. 	xsmaddadp	vs41,	vs1,	vs12

  1943. 

  1944. 	xsmaddadp	vs42,	vs0,	vs13

  1945. 	xsmaddadp	vs43,	vs1,	vs13

  1946. 

  1947. 	xsmaddadp	vs44,	vs0,	vs14

  1948. 	xsmaddadp	vs45,	vs1,	vs14

  1949. 

  1950. 	xsmaddadp	vs46,	vs0,	vs15

  1951. 	xsmaddadp	vs47,	vs1,	vs15

  1952. 

  1953. 

  1954. #if defined(_AIX)

  1955. ')

  1956. #else

  1957. .endm

  1958. #endif

  1959. 

  1960. #if defined(_AIX)

  1961. define(`KERNEL8x2_2', `

  1962. #else

  1963. .macro KERNEL8x2_2

  1964. #endif

  1965. 

  1966. 

  1967. 	lxsspx		vs0,	o0,	AO

  1968. 	lxsspx		vs1,	o4,	AO

  1969. 

  1970. 	addi		AO,	AO,	8

  1971. 

  1972. 	mr		T1,	BO

  1973. 

  1974. 	lxsspx		vs8,	o0,	T1

  1975. 	lxsspx		vs9,	o16,	T1

  1976. 	lxsspx		vs10,	o32,	T1

  1977. 	lxsspx		vs11,	o48,	T1

  1978. 

  1979. 	addi		T1,	T1,	64

  1980. 

  1981. 	lxsspx		vs12,	o0,	T1

  1982. 	lxsspx		vs13,	o16,	T1

  1983. 	lxsspx		vs14,	o32,	T1

  1984. 	lxsspx		vs15,	o48,	T1

  1985. 

  1986. 

  1987. 	addi		BO,	BO,	128

  1988. 

  1989. 

  1990. 	xsmaddadp	vs32,	vs4,	vs16

  1991. 	xsmaddadp	vs33,	vs5,	vs16

  1992. 

  1993. 	xsmaddadp	vs34,	vs4,	vs17

  1994. 	xsmaddadp	vs35,	vs5,	vs17

  1995. 

  1996. 	xsmaddadp	vs36,	vs4,	vs18

  1997. 	xsmaddadp	vs37,	vs5,	vs18

  1998. 

  1999. 	xsmaddadp	vs38,	vs4,	vs19

  2000. 	xsmaddadp	vs39,	vs5,	vs19

  2001. 

  2002. 	xsmaddadp	vs40,	vs4,	vs20

  2003. 	xsmaddadp	vs41,	vs5,	vs20

  2004. 

  2005. 	xsmaddadp	vs42,	vs4,	vs21

  2006. 	xsmaddadp	vs43,	vs5,	vs21

  2007. 

  2008. 	xsmaddadp	vs44,	vs4,	vs22

  2009. 	xsmaddadp	vs45,	vs5,	vs22

  2010. 

  2011. 	xsmaddadp	vs46,	vs4,	vs23

  2012. 	xsmaddadp	vs47,	vs5,	vs23

  2013. 

  2014. 

  2015. #if defined(_AIX)

  2016. ')

  2017. #else

  2018. .endm

  2019. #endif

  2020. 

  2021. #if defined(_AIX)

  2022. define(`KERNEL8x2_E2', `

  2023. #else

  2024. .macro KERNEL8x2_E2

  2025. #endif

  2026. 

  2027. 

  2028. 	xsmaddadp	vs32,	vs4,	vs16

  2029. 	xsmaddadp	vs33,	vs5,	vs16

  2030. 

  2031. 	xsmaddadp	vs34,	vs4,	vs17

  2032. 	xsmaddadp	vs35,	vs5,	vs17

  2033. 

  2034. 	xsmaddadp	vs36,	vs4,	vs18

  2035. 	xsmaddadp	vs37,	vs5,	vs18

  2036. 

  2037. 	xsmaddadp	vs38,	vs4,	vs19

  2038. 	xsmaddadp	vs39,	vs5,	vs19

  2039. 

  2040. 	xsmaddadp	vs40,	vs4,	vs20

  2041. 	xsmaddadp	vs41,	vs5,	vs20

  2042. 

  2043. 	xsmaddadp	vs42,	vs4,	vs21

  2044. 	xsmaddadp	vs43,	vs5,	vs21

  2045. 

  2046. 	xsmaddadp	vs44,	vs4,	vs22

  2047. 	xsmaddadp	vs45,	vs5,	vs22

  2048. 

  2049. 	xsmaddadp	vs46,	vs4,	vs23

  2050. 	xsmaddadp	vs47,	vs5,	vs23

  2051. 

  2052. 

  2053. #if defined(_AIX)

  2054. ')

  2055. #else

  2056. .endm

  2057. #endif

  2058. 

  2059. #if defined(_AIX)

  2060. define(`KERNEL8x2_SUBI1', `

  2061. #else

  2062. .macro KERNEL8x2_SUBI1

  2063. #endif

  2064. 

  2065. 

  2066. 	lxsspx		vs0,	o0,	AO

  2067. 	lxsspx		vs1,	o4,	AO

  2068. 

  2069. 	addi		AO,	AO,	8

  2070. 

  2071. 	mr		T1,	BO

  2072. 

  2073. 	lxsspx		vs8,	o0,	T1

  2074. 	lxsspx		vs9,	o16,	T1

  2075. 	lxsspx		vs10,	o32,	T1

  2076. 	lxsspx		vs11,	o48,	T1

  2077. 

  2078. 	addi		T1,	T1,	64

  2079. 

  2080. 	lxsspx		vs12,	o0,	T1

  2081. 	lxsspx		vs13,	o16,	T1

  2082. 	lxsspx		vs14,	o32,	T1

  2083. 	lxsspx		vs15,	o48,	T1

  2084. 

  2085. 

  2086. 	addi		BO,	BO,	128

  2087. 

  2088. 

  2089. 	xsmuldp		vs32,	vs0,	vs8

  2090. 	xsmuldp		vs33,	vs1,	vs8

  2091. 

  2092. 	xsmuldp		vs34,	vs0,	vs9

  2093. 	xsmuldp		vs35,	vs1,	vs9

  2094. 

  2095. 	xsmuldp		vs36,	vs0,	vs10

  2096. 	xsmuldp		vs37,	vs1,	vs10

  2097. 

  2098. 	xsmuldp		vs38,	vs0,	vs11

  2099. 	xsmuldp		vs39,	vs1,	vs11

  2100. 

  2101. 	xsmuldp		vs40,	vs0,	vs12

  2102. 	xsmuldp		vs41,	vs1,	vs12

  2103. 

  2104. 	xsmuldp		vs42,	vs0,	vs13

  2105. 	xsmuldp		vs43,	vs1,	vs13

  2106. 

  2107. 	xsmuldp		vs44,	vs0,	vs14

  2108. 	xsmuldp		vs45,	vs1,	vs14

  2109. 

  2110. 	xsmuldp		vs46,	vs0,	vs15

  2111. 	xsmuldp		vs47,	vs1,	vs15

  2112. 

  2113. 

  2114. #if defined(_AIX)

  2115. ')

  2116. #else

  2117. .endm

  2118. #endif

  2119. 

  2120. #if defined(_AIX)

  2121. define(`KERNEL8x2_SUB1', `

  2122. #else

  2123. .macro KERNEL8x2_SUB1

  2124. #endif

  2125. 

  2126. 

  2127. 	lxsspx		vs0,	o0,	AO

  2128. 	lxsspx		vs1,	o4,	AO

  2129. 

  2130. 	addi		AO,	AO,	8

  2131. 

  2132. 	mr		T1,	BO

  2133. 

  2134. 	lxsspx		vs8,	o0,	T1

  2135. 	lxsspx		vs9,	o16,	T1

  2136. 	lxsspx		vs10,	o32,	T1

  2137. 	lxsspx		vs11,	o48,	T1

  2138. 

  2139. 	addi		T1,	T1,	64

  2140. 

  2141. 	lxsspx		vs12,	o0,	T1

  2142. 	lxsspx		vs13,	o16,	T1

  2143. 	lxsspx		vs14,	o32,	T1

  2144. 	lxsspx		vs15,	o48,	T1

  2145. 

  2146. 

  2147. 	addi		BO,	BO,	128

  2148. 

  2149. 

  2150. 	xsmaddadp	vs32,	vs0,	vs8

  2151. 	xsmaddadp	vs33,	vs1,	vs8

  2152. 

  2153. 	xsmaddadp	vs34,	vs0,	vs9

  2154. 	xsmaddadp	vs35,	vs1,	vs9

  2155. 

  2156. 	xsmaddadp	vs36,	vs0,	vs10

  2157. 	xsmaddadp	vs37,	vs1,	vs10

  2158. 

  2159. 	xsmaddadp	vs38,	vs0,	vs11

  2160. 	xsmaddadp	vs39,	vs1,	vs11

  2161. 

  2162. 	xsmaddadp	vs40,	vs0,	vs12

  2163. 	xsmaddadp	vs41,	vs1,	vs12

  2164. 

  2165. 	xsmaddadp	vs42,	vs0,	vs13

  2166. 	xsmaddadp	vs43,	vs1,	vs13

  2167. 

  2168. 	xsmaddadp	vs44,	vs0,	vs14

  2169. 	xsmaddadp	vs45,	vs1,	vs14

  2170. 

  2171. 	xsmaddadp	vs46,	vs0,	vs15

  2172. 	xsmaddadp	vs47,	vs1,	vs15

  2173. 

  2174. 

  2175. #if defined(_AIX)

  2176. ')

  2177. #else

  2178. .endm

  2179. #endif

  2180. 

  2181. #if defined(_AIX)

  2182. define(`SAVE8x2', `

  2183. #else

  2184. .macro SAVE8x2

  2185. #endif

  2186. 

  2187. 	mr		T1,	CO

  2188. 

  2189. #ifndef TRMMKERNEL

  2190. 

  2191. 	lxsspx		vs0,	o0,	T1

  2192. 	lxsspx		vs1,	o4,	T1

  2193. 

  2194. #endif

  2195. 

  2196. #ifdef TRMMKERNEL

  2197. 	xsmuldp		vs0,	vs32,	alpha_r

  2198. 	xsmuldp		vs1,	vs33,	alpha_r

  2199. #else

  2200. 	xsmaddadp	vs0,	vs32,	alpha_r

  2201. 	xsmaddadp	vs1,	vs33,	alpha_r

  2202. #endif

  2203. 

  2204. 	stxsspx		vs0,	o0,	T1

  2205. 	stxsspx		vs1,	o4,	T1

  2206. 

  2207. 	add		T1,	T1,	LDC

  2208. 

  2209. 

  2210. #ifndef TRMMKERNEL

  2211. 

  2212. 	lxsspx		vs0,	o0,	T1

  2213. 	lxsspx		vs1,	o4,	T1

  2214. 

  2215. #endif

  2216. 

  2217. #ifdef TRMMKERNEL

  2218. 	xsmuldp		vs0,	vs34,	alpha_r

  2219. 	xsmuldp		vs1,	vs35,	alpha_r

  2220. #else

  2221. 	xsmaddadp	vs0,	vs34,	alpha_r

  2222. 	xsmaddadp	vs1,	vs35,	alpha_r

  2223. #endif

  2224. 

  2225. 	stxsspx		vs0,	o0,	T1

  2226. 	stxsspx		vs1,	o4,	T1

  2227. 

  2228. 	add		T1,	T1,	LDC

  2229. 

  2230. 

  2231. #ifndef TRMMKERNEL

  2232. 

  2233. 	lxsspx		vs0,	o0,	T1

  2234. 	lxsspx		vs1,	o4,	T1

  2235. 

  2236. #endif

  2237. 

  2238. #ifdef TRMMKERNEL

  2239. 	xsmuldp		vs0,	vs36,	alpha_r

  2240. 	xsmuldp		vs1,	vs37,	alpha_r

  2241. #else

  2242. 	xsmaddadp	vs0,	vs36,	alpha_r

  2243. 	xsmaddadp	vs1,	vs37,	alpha_r

  2244. #endif

  2245. 

  2246. 	stxsspx		vs0,	o0,	T1

  2247. 	stxsspx		vs1,	o4,	T1

  2248. 

  2249. 	add		T1,	T1,	LDC

  2250. 

  2251. 

  2252. #ifndef TRMMKERNEL

  2253. 

  2254. 	lxsspx		vs0,	o0,	T1

  2255. 	lxsspx		vs1,	o4,	T1

  2256. 

  2257. #endif

  2258. 

  2259. #ifdef TRMMKERNEL

  2260. 	xsmuldp		vs0,	vs38,	alpha_r

  2261. 	xsmuldp		vs1,	vs39,	alpha_r

  2262. #else

  2263. 	xsmaddadp	vs0,	vs38,	alpha_r

  2264. 	xsmaddadp	vs1,	vs39,	alpha_r

  2265. #endif

  2266. 

  2267. 	stxsspx		vs0,	o0,	T1

  2268. 	stxsspx		vs1,	o4,	T1

  2269. 

  2270. 	add		T1,	T1,	LDC

  2271. 

  2272. 

  2273. #ifndef TRMMKERNEL

  2274. 

  2275. 	lxsspx		vs0,	o0,	T1

  2276. 	lxsspx		vs1,	o4,	T1

  2277. 

  2278. #endif

  2279. 

  2280. #ifdef TRMMKERNEL

  2281. 	xsmuldp		vs0,	vs40,	alpha_r

  2282. 	xsmuldp		vs1,	vs41,	alpha_r

  2283. #else

  2284. 	xsmaddadp	vs0,	vs40,	alpha_r

  2285. 	xsmaddadp	vs1,	vs41,	alpha_r

  2286. #endif

  2287. 

  2288. 	stxsspx		vs0,	o0,	T1

  2289. 	stxsspx		vs1,	o4,	T1

  2290. 

  2291. 	add		T1,	T1,	LDC

  2292. 

  2293. 

  2294. #ifndef TRMMKERNEL

  2295. 

  2296. 	lxsspx		vs0,	o0,	T1

  2297. 	lxsspx		vs1,	o4,	T1

  2298. 

  2299. #endif

  2300. 

  2301. #ifdef TRMMKERNEL

  2302. 	xsmuldp		vs0,	vs42,	alpha_r

  2303. 	xsmuldp		vs1,	vs43,	alpha_r

  2304. #else

  2305. 	xsmaddadp	vs0,	vs42,	alpha_r

  2306. 	xsmaddadp	vs1,	vs43,	alpha_r

  2307. #endif

  2308. 

  2309. 	stxsspx		vs0,	o0,	T1

  2310. 	stxsspx		vs1,	o4,	T1

  2311. 

  2312. 	add		T1,	T1,	LDC

  2313. 

  2314. 

  2315. #ifndef TRMMKERNEL

  2316. 

  2317. 	lxsspx		vs0,	o0,	T1

  2318. 	lxsspx		vs1,	o4,	T1

  2319. 

  2320. #endif

  2321. 

  2322. #ifdef TRMMKERNEL

  2323. 	xsmuldp		vs0,	vs44,	alpha_r

  2324. 	xsmuldp		vs1,	vs45,	alpha_r

  2325. #else

  2326. 	xsmaddadp	vs0,	vs44,	alpha_r

  2327. 	xsmaddadp	vs1,	vs45,	alpha_r

  2328. #endif

  2329. 

  2330. 	stxsspx		vs0,	o0,	T1

  2331. 	stxsspx		vs1,	o4,	T1

  2332. 

  2333. 	add		T1,	T1,	LDC

  2334. 

  2335. 

  2336. #ifndef TRMMKERNEL

  2337. 

  2338. 	lxsspx		vs0,	o0,	T1

  2339. 	lxsspx		vs1,	o4,	T1

  2340. 

  2341. #endif

  2342. 

  2343. #ifdef TRMMKERNEL

  2344. 	xsmuldp		vs0,	vs46,	alpha_r

  2345. 	xsmuldp		vs1,	vs47,	alpha_r

  2346. #else

  2347. 	xsmaddadp	vs0,	vs46,	alpha_r

  2348. 	xsmaddadp	vs1,	vs47,	alpha_r

  2349. #endif

  2350. 

  2351. 	stxsspx		vs0,	o0,	T1

  2352. 	stxsspx		vs1,	o4,	T1

  2353. 

  2354. 	add		T1,	T1,	LDC

  2355. 

  2356. 	addi		CO,	CO,	8

  2357. 

  2358. #if defined(_AIX)

  2359. ')

  2360. #else

  2361. .endm

  2362. #endif

  2363. 

  2364. 

  2365. /**********************************************************************************************

  2366. * Macros for N=8 and M=1

  2367. **********************************************************************************************/

  2368. 

  2369. #if defined(_AIX)

  2370. define(`LOAD8x1_1', `

  2371. #else

  2372. .macro LOAD8x1_1

  2373. #endif

  2374. 

  2375. 	lxsspx		vs0,	o0,	AO

  2376. 

  2377. 	addi		AO,	AO,	4

  2378. 

  2379. 	mr		T1,	BO

  2380. 

  2381. 	lxsspx		vs8,	o0,	T1

  2382. 	lxsspx		vs9,	o16,	T1

  2383. 	lxsspx		vs10,	o32,	T1

  2384. 	lxsspx		vs11,	o48,	T1

  2385. 

  2386. 	addi		T1,	T1,	64

  2387. 

  2388. 	lxsspx		vs12,	o0,	T1

  2389. 	lxsspx		vs13,	o16,	T1

  2390. 	lxsspx		vs14,	o32,	T1

  2391. 	lxsspx		vs15,	o48,	T1

  2392. 

  2393. 

  2394. 	addi		BO,	BO,	128

  2395. 

  2396. #if defined(_AIX)

  2397. ')

  2398. #else

  2399. .endm

  2400. #endif

  2401. 

  2402. #if defined(_AIX)

  2403. define(`KERNEL8x1_I1', `

  2404. #else

  2405. .macro KERNEL8x1_I1

  2406. #endif

  2407. 

  2408. 

  2409. 	lxsspx		vs4,	o0,	AO

  2410. 

  2411. 	addi		AO,	AO,	4

  2412. 

  2413. 	mr		T1,	BO

  2414. 

  2415. 	lxsspx		vs16,	o0,	T1

  2416. 	lxsspx		vs17,	o16,	T1

  2417. 	lxsspx		vs18,	o32,	T1

  2418. 	lxsspx		vs19,	o48,	T1

  2419. 

  2420. 	addi		T1,	T1,	64

  2421. 

  2422. 	lxsspx		vs20,	o0,	T1

  2423. 	lxsspx		vs21,	o16,	T1

  2424. 	lxsspx		vs22,	o32,	T1

  2425. 	lxsspx		vs23,	o48,	T1

  2426. 

  2427. 

  2428. 	addi		BO,	BO,	128

  2429. 

  2430. 

  2431. 	xsmuldp		vs32,	vs0,	vs8

  2432. 

  2433. 	xsmuldp		vs33,	vs0,	vs9

  2434. 

  2435. 	xsmuldp		vs34,	vs0,	vs10

  2436. 

  2437. 	xsmuldp		vs35,	vs0,	vs11

  2438. 

  2439. 	xsmuldp		vs36,	vs0,	vs12

  2440. 

  2441. 	xsmuldp		vs37,	vs0,	vs13

  2442. 

  2443. 	xsmuldp		vs38,	vs0,	vs14

  2444. 

  2445. 	xsmuldp		vs39,	vs0,	vs15

  2446. 

  2447. 

  2448. #if defined(_AIX)

  2449. ')

  2450. #else

  2451. .endm

  2452. #endif

  2453. 

  2454. #if defined(_AIX)

  2455. define(`KERNEL8x1_1', `

  2456. #else

  2457. .macro KERNEL8x1_1

  2458. #endif

  2459. 

  2460. 

  2461. 	lxsspx		vs4,	o0,	AO

  2462. 

  2463. 	addi		AO,	AO,	4

  2464. 

  2465. 	mr		T1,	BO

  2466. 

  2467. 	lxsspx		vs16,	o0,	T1

  2468. 	lxsspx		vs17,	o16,	T1

  2469. 	lxsspx		vs18,	o32,	T1

  2470. 	lxsspx		vs19,	o48,	T1

  2471. 

  2472. 	addi		T1,	T1,	64

  2473. 

  2474. 	lxsspx		vs20,	o0,	T1

  2475. 	lxsspx		vs21,	o16,	T1

  2476. 	lxsspx		vs22,	o32,	T1

  2477. 	lxsspx		vs23,	o48,	T1

  2478. 

  2479. 

  2480. 	addi		BO,	BO,	128

  2481. 

  2482. 

  2483. 	xsmaddadp	vs32,	vs0,	vs8

  2484. 

  2485. 	xsmaddadp	vs33,	vs0,	vs9

  2486. 

  2487. 	xsmaddadp	vs34,	vs0,	vs10

  2488. 

  2489. 	xsmaddadp	vs35,	vs0,	vs11

  2490. 

  2491. 	xsmaddadp	vs36,	vs0,	vs12

  2492. 

  2493. 	xsmaddadp	vs37,	vs0,	vs13

  2494. 

  2495. 	xsmaddadp	vs38,	vs0,	vs14

  2496. 

  2497. 	xsmaddadp	vs39,	vs0,	vs15

  2498. 

  2499. 

  2500. #if defined(_AIX)

  2501. ')

  2502. #else

  2503. .endm

  2504. #endif

  2505. 

  2506. #if defined(_AIX)

  2507. define(`KERNEL8x1_2', `

  2508. #else

  2509. .macro KERNEL8x1_2

  2510. #endif

  2511. 

  2512. 

  2513. 	lxsspx		vs0,	o0,	AO

  2514. 

  2515. 	addi		AO,	AO,	4

  2516. 

  2517. 	mr		T1,	BO

  2518. 

  2519. 	lxsspx		vs8,	o0,	T1

  2520. 	lxsspx		vs9,	o16,	T1

  2521. 	lxsspx		vs10,	o32,	T1

  2522. 	lxsspx		vs11,	o48,	T1

  2523. 

  2524. 	addi		T1,	T1,	64

  2525. 

  2526. 	lxsspx		vs12,	o0,	T1

  2527. 	lxsspx		vs13,	o16,	T1

  2528. 	lxsspx		vs14,	o32,	T1

  2529. 	lxsspx		vs15,	o48,	T1

  2530. 

  2531. 

  2532. 	addi		BO,	BO,	128

  2533. 

  2534. 

  2535. 	xsmaddadp	vs32,	vs4,	vs16

  2536. 

  2537. 	xsmaddadp	vs33,	vs4,	vs17

  2538. 

  2539. 	xsmaddadp	vs34,	vs4,	vs18

  2540. 

  2541. 	xsmaddadp	vs35,	vs4,	vs19

  2542. 

  2543. 	xsmaddadp	vs36,	vs4,	vs20

  2544. 

  2545. 	xsmaddadp	vs37,	vs4,	vs21

  2546. 

  2547. 	xsmaddadp	vs38,	vs4,	vs22

  2548. 

  2549. 	xsmaddadp	vs39,	vs4,	vs23

  2550. 

  2551. 

  2552. #if defined(_AIX)

  2553. ')

  2554. #else

  2555. .endm

  2556. #endif

  2557. 

  2558. #if defined(_AIX)

  2559. define(`KERNEL8x1_E2', `

  2560. #else

  2561. .macro KERNEL8x1_E2

  2562. #endif

  2563. 

  2564. 

  2565. 	xsmaddadp	vs32,	vs4,	vs16

  2566. 

  2567. 	xsmaddadp	vs33,	vs4,	vs17

  2568. 

  2569. 	xsmaddadp	vs34,	vs4,	vs18

  2570. 

  2571. 	xsmaddadp	vs35,	vs4,	vs19

  2572. 

  2573. 	xsmaddadp	vs36,	vs4,	vs20

  2574. 

  2575. 	xsmaddadp	vs37,	vs4,	vs21

  2576. 

  2577. 	xsmaddadp	vs38,	vs4,	vs22

  2578. 

  2579. 	xsmaddadp	vs39,	vs4,	vs23

  2580. 

  2581. 

  2582. #if defined(_AIX)

  2583. ')

  2584. #else

  2585. .endm

  2586. #endif

  2587. 

  2588. #if defined(_AIX)

  2589. define(`KERNEL8x1_SUBI1', `

  2590. #else

  2591. .macro KERNEL8x1_SUBI1

  2592. #endif

  2593. 

  2594. 

  2595. 	lxsspx		vs0,	o0,	AO

  2596. 

  2597. 	addi		AO,	AO,	4

  2598. 

  2599. 	mr		T1,	BO

  2600. 

  2601. 	lxsspx		vs8,	o0,	T1

  2602. 	lxsspx		vs9,	o16,	T1

  2603. 	lxsspx		vs10,	o32,	T1

  2604. 	lxsspx		vs11,	o48,	T1

  2605. 

  2606. 	addi		T1,	T1,	64

  2607. 

  2608. 	lxsspx		vs12,	o0,	T1

  2609. 	lxsspx		vs13,	o16,	T1

  2610. 	lxsspx		vs14,	o32,	T1

  2611. 	lxsspx		vs15,	o48,	T1

  2612. 

  2613. 

  2614. 	addi		BO,	BO,	128

  2615. 

  2616. 

  2617. 	xsmuldp		vs32,	vs0,	vs8

  2618. 

  2619. 	xsmuldp		vs33,	vs0,	vs9

  2620. 

  2621. 	xsmuldp		vs34,	vs0,	vs10

  2622. 

  2623. 	xsmuldp		vs35,	vs0,	vs11

  2624. 

  2625. 	xsmuldp		vs36,	vs0,	vs12

  2626. 

  2627. 	xsmuldp		vs37,	vs0,	vs13

  2628. 

  2629. 	xsmuldp		vs38,	vs0,	vs14

  2630. 

  2631. 	xsmuldp		vs39,	vs0,	vs15

  2632. 

  2633. 

  2634. #if defined(_AIX)

  2635. ')

  2636. #else

  2637. .endm

  2638. #endif

  2639. 

  2640. #if defined(_AIX)

  2641. define(`KERNEL8x1_SUB1', `

  2642. #else

  2643. .macro KERNEL8x1_SUB1

  2644. #endif

  2645. 

  2646. 

  2647. 	lxsspx		vs0,	o0,	AO

  2648. 

  2649. 	addi		AO,	AO,	4

  2650. 

  2651. 	mr		T1,	BO

  2652. 

  2653. 	lxsspx		vs8,	o0,	T1

  2654. 	lxsspx		vs9,	o16,	T1

  2655. 	lxsspx		vs10,	o32,	T1

  2656. 	lxsspx		vs11,	o48,	T1

  2657. 

  2658. 	addi		T1,	T1,	64

  2659. 

  2660. 	lxsspx		vs12,	o0,	T1

  2661. 	lxsspx		vs13,	o16,	T1

  2662. 	lxsspx		vs14,	o32,	T1

  2663. 	lxsspx		vs15,	o48,	T1

  2664. 

  2665. 

  2666. 	addi		BO,	BO,	128

  2667. 

  2668. 

  2669. 	xsmaddadp	vs32,	vs0,	vs8

  2670. 

  2671. 	xsmaddadp	vs33,	vs0,	vs9

  2672. 

  2673. 	xsmaddadp	vs34,	vs0,	vs10

  2674. 

  2675. 	xsmaddadp	vs35,	vs0,	vs11

  2676. 

  2677. 	xsmaddadp	vs36,	vs0,	vs12

  2678. 

  2679. 	xsmaddadp	vs37,	vs0,	vs13

  2680. 

  2681. 	xsmaddadp	vs38,	vs0,	vs14

  2682. 

  2683. 	xsmaddadp	vs39,	vs0,	vs15

  2684. 

  2685. 

  2686. #if defined(_AIX)

  2687. ')

  2688. #else

  2689. .endm

  2690. #endif

  2691. 

  2692. #if defined(_AIX)

  2693. define(`SAVE8x1', `

  2694. #else

  2695. .macro SAVE8x1

  2696. #endif

  2697. 

  2698. 	mr		T1,	CO

  2699. 

  2700. #ifndef TRMMKERNEL

  2701. 

  2702. 	lxsspx		vs0,	o0,	T1

  2703. 

  2704. #endif

  2705. 

  2706. #ifdef TRMMKERNEL

  2707. 	xsmuldp		vs0,	vs32,	alpha_r

  2708. #else

  2709. 	xsmaddadp	vs0,	vs32,	alpha_r

  2710. #endif

  2711. 

  2712. 	stxsspx		vs0,	o0,	T1

  2713. 

  2714. 	add		T1,	T1,	LDC

  2715. 

  2716. 

  2717. #ifndef TRMMKERNEL

  2718. 

  2719. 	lxsspx		vs0,	o0,	T1

  2720. 

  2721. #endif

  2722. 

  2723. #ifdef TRMMKERNEL

  2724. 	xsmuldp		vs0,	vs33,	alpha_r

  2725. #else

  2726. 	xsmaddadp	vs0,	vs33,	alpha_r

  2727. #endif

  2728. 

  2729. 	stxsspx		vs0,	o0,	T1

  2730. 

  2731. 	add		T1,	T1,	LDC

  2732. 

  2733. 

  2734. #ifndef TRMMKERNEL

  2735. 

  2736. 	lxsspx		vs0,	o0,	T1

  2737. 

  2738. #endif

  2739. 

  2740. #ifdef TRMMKERNEL

  2741. 	xsmuldp		vs0,	vs34,	alpha_r

  2742. #else

  2743. 	xsmaddadp	vs0,	vs34,	alpha_r

  2744. #endif

  2745. 

  2746. 	stxsspx		vs0,	o0,	T1

  2747. 

  2748. 	add		T1,	T1,	LDC

  2749. 

  2750. 

  2751. #ifndef TRMMKERNEL

  2752. 

  2753. 	lxsspx		vs0,	o0,	T1

  2754. 

  2755. #endif

  2756. 

  2757. #ifdef TRMMKERNEL

  2758. 	xsmuldp		vs0,	vs35,	alpha_r

  2759. #else

  2760. 	xsmaddadp	vs0,	vs35,	alpha_r

  2761. #endif

  2762. 

  2763. 	stxsspx		vs0,	o0,	T1

  2764. 

  2765. 	add		T1,	T1,	LDC

  2766. 

  2767. 

  2768. #ifndef TRMMKERNEL

  2769. 

  2770. 	lxsspx		vs0,	o0,	T1

  2771. 

  2772. #endif

  2773. 

  2774. #ifdef TRMMKERNEL

  2775. 	xsmuldp		vs0,	vs36,	alpha_r

  2776. #else

  2777. 	xsmaddadp	vs0,	vs36,	alpha_r

  2778. #endif

  2779. 

  2780. 	stxsspx		vs0,	o0,	T1

  2781. 

  2782. 	add		T1,	T1,	LDC

  2783. 

  2784. 

  2785. #ifndef TRMMKERNEL

  2786. 

  2787. 	lxsspx		vs0,	o0,	T1

  2788. 

  2789. #endif

  2790. 

  2791. #ifdef TRMMKERNEL

  2792. 	xsmuldp		vs0,	vs37,	alpha_r

  2793. #else

  2794. 	xsmaddadp	vs0,	vs37,	alpha_r

  2795. #endif

  2796. 

  2797. 	stxsspx		vs0,	o0,	T1

  2798. 

  2799. 	add		T1,	T1,	LDC

  2800. 

  2801. 

  2802. #ifndef TRMMKERNEL

  2803. 

  2804. 	lxsspx		vs0,	o0,	T1

  2805. 

  2806. #endif

  2807. 

  2808. #ifdef TRMMKERNEL

  2809. 	xsmuldp		vs0,	vs38,	alpha_r

  2810. #else

  2811. 	xsmaddadp	vs0,	vs38,	alpha_r

  2812. #endif

  2813. 

  2814. 	stxsspx		vs0,	o0,	T1

  2815. 

  2816. 	add		T1,	T1,	LDC

  2817. 

  2818. 

  2819. #ifndef TRMMKERNEL

  2820. 

  2821. 	lxsspx		vs0,	o0,	T1

  2822. 

  2823. #endif

  2824. 

  2825. #ifdef TRMMKERNEL

  2826. 	xsmuldp		vs0,	vs39,	alpha_r

  2827. #else

  2828. 	xsmaddadp	vs0,	vs39,	alpha_r

  2829. #endif

  2830. 

  2831. 	stxsspx		vs0,	o0,	T1

  2832. 

  2833. 	add		T1,	T1,	LDC

  2834. 

  2835. 	addi		CO,	CO,	4

  2836. 

  2837. #if defined(_AIX)

  2838. ')

  2839. #else

  2840. .endm

  2841. #endif

  2842. 

  2843. 

  2844. /**********************************************************************************************

  2845. * Macros for N=4 and M=16

  2846. **********************************************************************************************/

  2847. 

  2848. #if defined(_AIX)

  2849. define(`LOAD4x16_1', `

  2850. #else

  2851. .macro LOAD4x16_1

  2852. #endif

  2853. 

  2854. 	lxvw4x		vs0,	o0,	AO

  2855. 	lxvw4x		vs1,	o16,	AO

  2856. 	lxvw4x		vs2,	o32,	AO

  2857. 	lxvw4x		vs3,	o48,	AO

  2858. 

  2859. 	addi		AO,	AO,	64

  2860. 

  2861. 	mr		T1,	BO

  2862. 

  2863. 	lxvw4x		vs8,	o0,	T1

  2864. 	lxvw4x		vs9,	o16,	T1

  2865. 	lxvw4x		vs10,	o32,	T1

  2866. 	lxvw4x		vs11,	o48,	T1

  2867. 

  2868. 	addi		BO,	BO,	64

  2869. 

  2870. #if defined(_AIX)

  2871. ')

  2872. #else

  2873. .endm

  2874. #endif

  2875. 

  2876. #if defined(_AIX)

  2877. define(`KERNEL4x16_I1', `

  2878. #else

  2879. .macro KERNEL4x16_I1

  2880. #endif

  2881. 

  2882. 

  2883. 	lxvw4x		vs4,	o0,	AO

  2884. 	lxvw4x		vs5,	o16,	AO

  2885. 	lxvw4x		vs6,	o32,	AO

  2886. 	lxvw4x		vs7,	o48,	AO

  2887. 

  2888. 	addi		AO,	AO,	64

  2889. 

  2890. 	mr		T1,	BO

  2891. 

  2892. 	lxvw4x		vs16,	o0,	T1

  2893. 	lxvw4x		vs17,	o16,	T1

  2894. 	lxvw4x		vs18,	o32,	T1

  2895. 	lxvw4x		vs19,	o48,	T1

  2896. 

  2897. 	addi		BO,	BO,	64

  2898. 

  2899. 

  2900. 	xvmulsp		vs32,	vs0,	vs8

  2901. 	xvmulsp		vs33,	vs1,	vs8

  2902. 	xvmulsp		vs34,	vs2,	vs8

  2903. 	xvmulsp		vs35,	vs3,	vs8

  2904. 

  2905. 	xvmulsp		vs36,	vs0,	vs9

  2906. 	xvmulsp		vs37,	vs1,	vs9

  2907. 	xvmulsp		vs38,	vs2,	vs9

  2908. 	xvmulsp		vs39,	vs3,	vs9

  2909. 

  2910. 	xvmulsp		vs40,	vs0,	vs10

  2911. 	xvmulsp		vs41,	vs1,	vs10

  2912. 	xvmulsp		vs42,	vs2,	vs10

  2913. 	xvmulsp		vs43,	vs3,	vs10

  2914. 

  2915. 	xvmulsp		vs44,	vs0,	vs11

  2916. 	xvmulsp		vs45,	vs1,	vs11

  2917. 	xvmulsp		vs46,	vs2,	vs11

  2918. 	xvmulsp		vs47,	vs3,	vs11

  2919. 

  2920. 

  2921. #if defined(_AIX)

  2922. ')

  2923. #else

  2924. .endm

  2925. #endif

  2926. 

  2927. #if defined(_AIX)

  2928. define(`KERNEL4x16_1', `

  2929. #else

  2930. .macro KERNEL4x16_1

  2931. #endif

  2932. 

  2933. 

  2934. 	lxvw4x		vs4,	o0,	AO

  2935. 	lxvw4x		vs5,	o16,	AO

  2936. 	lxvw4x		vs6,	o32,	AO

  2937. 	lxvw4x		vs7,	o48,	AO

  2938. 

  2939. 	addi		AO,	AO,	64

  2940. 

  2941. 	mr		T1,	BO

  2942. 

  2943. 	lxvw4x		vs16,	o0,	T1

  2944. 	lxvw4x		vs17,	o16,	T1

  2945. 	lxvw4x		vs18,	o32,	T1

  2946. 	lxvw4x		vs19,	o48,	T1

  2947. 

  2948. 	addi		BO,	BO,	64

  2949. 

  2950. 

  2951. 	xvmaddasp	vs32,	vs0,	vs8

  2952. 	xvmaddasp	vs33,	vs1,	vs8

  2953. 	xvmaddasp	vs34,	vs2,	vs8

  2954. 	xvmaddasp	vs35,	vs3,	vs8

  2955. 

  2956. 	xvmaddasp	vs36,	vs0,	vs9

  2957. 	xvmaddasp	vs37,	vs1,	vs9

  2958. 	xvmaddasp	vs38,	vs2,	vs9

  2959. 	xvmaddasp	vs39,	vs3,	vs9

  2960. 

  2961. 	xvmaddasp	vs40,	vs0,	vs10

  2962. 	xvmaddasp	vs41,	vs1,	vs10

  2963. 	xvmaddasp	vs42,	vs2,	vs10

  2964. 	xvmaddasp	vs43,	vs3,	vs10

  2965. 

  2966. 	xvmaddasp	vs44,	vs0,	vs11

  2967. 	xvmaddasp	vs45,	vs1,	vs11

  2968. 	xvmaddasp	vs46,	vs2,	vs11

  2969. 	xvmaddasp	vs47,	vs3,	vs11

  2970. 

  2971. 

  2972. #if defined(_AIX)

  2973. ')

  2974. #else

  2975. .endm

  2976. #endif

  2977. 

  2978. #if defined(_AIX)

  2979. define(`KERNEL4x16_2', `

  2980. #else

  2981. .macro KERNEL4x16_2

  2982. #endif

  2983. 

  2984. 

  2985. 	lxvw4x		vs0,	o0,	AO

  2986. 	lxvw4x		vs1,	o16,	AO

  2987. 	lxvw4x		vs2,	o32,	AO

  2988. 	lxvw4x		vs3,	o48,	AO

  2989. 

  2990. 	addi		AO,	AO,	64

  2991. 

  2992. 	mr		T1,	BO

  2993. 

  2994. 	lxvw4x		vs8,	o0,	T1

  2995. 	lxvw4x		vs9,	o16,	T1

  2996. 	lxvw4x		vs10,	o32,	T1

  2997. 	lxvw4x		vs11,	o48,	T1

  2998. 

  2999. 	addi		BO,	BO,	64

  3000. 

  3001. 

  3002. 	xvmaddasp	vs32,	vs4,	vs16

  3003. 	xvmaddasp	vs33,	vs5,	vs16

  3004. 	xvmaddasp	vs34,	vs6,	vs16

  3005. 	xvmaddasp	vs35,	vs7,	vs16

  3006. 

  3007. 	xvmaddasp	vs36,	vs4,	vs17

  3008. 	xvmaddasp	vs37,	vs5,	vs17

  3009. 	xvmaddasp	vs38,	vs6,	vs17

  3010. 	xvmaddasp	vs39,	vs7,	vs17

  3011. 

  3012. 	xvmaddasp	vs40,	vs4,	vs18

  3013. 	xvmaddasp	vs41,	vs5,	vs18

  3014. 	xvmaddasp	vs42,	vs6,	vs18

  3015. 	xvmaddasp	vs43,	vs7,	vs18

  3016. 

  3017. 	xvmaddasp	vs44,	vs4,	vs19

  3018. 	xvmaddasp	vs45,	vs5,	vs19

  3019. 	xvmaddasp	vs46,	vs6,	vs19

  3020. 	xvmaddasp	vs47,	vs7,	vs19

  3021. 

  3022. 

  3023. #if defined(_AIX)

  3024. ')

  3025. #else

  3026. .endm

  3027. #endif

  3028. 

  3029. #if defined(_AIX)

  3030. define(`KERNEL4x16_E2', `

  3031. #else

  3032. .macro KERNEL4x16_E2

  3033. #endif

  3034. 

  3035. 

  3036. 	xvmaddasp	vs32,	vs4,	vs16

  3037. 	xvmaddasp	vs33,	vs5,	vs16

  3038. 	xvmaddasp	vs34,	vs6,	vs16

  3039. 	xvmaddasp	vs35,	vs7,	vs16

  3040. 

  3041. 	xvmaddasp	vs36,	vs4,	vs17

  3042. 	xvmaddasp	vs37,	vs5,	vs17

  3043. 	xvmaddasp	vs38,	vs6,	vs17

  3044. 	xvmaddasp	vs39,	vs7,	vs17

  3045. 

  3046. 	xvmaddasp	vs40,	vs4,	vs18

  3047. 	xvmaddasp	vs41,	vs5,	vs18

  3048. 	xvmaddasp	vs42,	vs6,	vs18

  3049. 	xvmaddasp	vs43,	vs7,	vs18

  3050. 

  3051. 	xvmaddasp	vs44,	vs4,	vs19

  3052. 	xvmaddasp	vs45,	vs5,	vs19

  3053. 	xvmaddasp	vs46,	vs6,	vs19

  3054. 	xvmaddasp	vs47,	vs7,	vs19

  3055. 

  3056. 

  3057. #if defined(_AIX)

  3058. ')

  3059. #else

  3060. .endm

  3061. #endif

  3062. 

  3063. #if defined(_AIX)

  3064. define(`KERNEL4x16_SUBI1', `

  3065. #else

  3066. .macro KERNEL4x16_SUBI1

  3067. #endif

  3068. 

  3069. 

  3070. 	lxvw4x		vs0,	o0,	AO

  3071. 	lxvw4x		vs1,	o16,	AO

  3072. 	lxvw4x		vs2,	o32,	AO

  3073. 	lxvw4x		vs3,	o48,	AO

  3074. 

  3075. 	addi		AO,	AO,	64

  3076. 

  3077. 	mr		T1,	BO

  3078. 

  3079. 	lxvw4x		vs8,	o0,	T1

  3080. 	lxvw4x		vs9,	o16,	T1

  3081. 	lxvw4x		vs10,	o32,	T1

  3082. 	lxvw4x		vs11,	o48,	T1

  3083. 

  3084. 	addi		BO,	BO,	64

  3085. 

  3086. 

  3087. 	xvmulsp		vs32,	vs0,	vs8

  3088. 	xvmulsp		vs33,	vs1,	vs8

  3089. 	xvmulsp		vs34,	vs2,	vs8

  3090. 	xvmulsp		vs35,	vs3,	vs8

  3091. 

  3092. 	xvmulsp		vs36,	vs0,	vs9

  3093. 	xvmulsp		vs37,	vs1,	vs9

  3094. 	xvmulsp		vs38,	vs2,	vs9

  3095. 	xvmulsp		vs39,	vs3,	vs9

  3096. 

  3097. 	xvmulsp		vs40,	vs0,	vs10

  3098. 	xvmulsp		vs41,	vs1,	vs10

  3099. 	xvmulsp		vs42,	vs2,	vs10

  3100. 	xvmulsp		vs43,	vs3,	vs10

  3101. 

  3102. 	xvmulsp		vs44,	vs0,	vs11

  3103. 	xvmulsp		vs45,	vs1,	vs11

  3104. 	xvmulsp		vs46,	vs2,	vs11

  3105. 	xvmulsp		vs47,	vs3,	vs11

  3106. 

  3107. 

  3108. #if defined(_AIX)

  3109. ')

  3110. #else

  3111. .endm

  3112. #endif

  3113. 

  3114. #if defined(_AIX)

  3115. define(`KERNEL4x16_SUB1', `

  3116. #else

  3117. .macro KERNEL4x16_SUB1

  3118. #endif

  3119. 

  3120. 

  3121. 	lxvw4x		vs0,	o0,	AO

  3122. 	lxvw4x		vs1,	o16,	AO

  3123. 	lxvw4x		vs2,	o32,	AO

  3124. 	lxvw4x		vs3,	o48,	AO

  3125. 

  3126. 	addi		AO,	AO,	64

  3127. 

  3128. 	mr		T1,	BO

  3129. 

  3130. 	lxvw4x		vs8,	o0,	T1

  3131. 	lxvw4x		vs9,	o16,	T1

  3132. 	lxvw4x		vs10,	o32,	T1

  3133. 	lxvw4x		vs11,	o48,	T1

  3134. 

  3135. 	addi		BO,	BO,	64

  3136. 

  3137. 

  3138. 	xvmaddasp	vs32,	vs0,	vs8

  3139. 	xvmaddasp	vs33,	vs1,	vs8

  3140. 	xvmaddasp	vs34,	vs2,	vs8

  3141. 	xvmaddasp	vs35,	vs3,	vs8

  3142. 

  3143. 	xvmaddasp	vs36,	vs0,	vs9

  3144. 	xvmaddasp	vs37,	vs1,	vs9

  3145. 	xvmaddasp	vs38,	vs2,	vs9

  3146. 	xvmaddasp	vs39,	vs3,	vs9

  3147. 

  3148. 	xvmaddasp	vs40,	vs0,	vs10

  3149. 	xvmaddasp	vs41,	vs1,	vs10

  3150. 	xvmaddasp	vs42,	vs2,	vs10

  3151. 	xvmaddasp	vs43,	vs3,	vs10

  3152. 

  3153. 	xvmaddasp	vs44,	vs0,	vs11

  3154. 	xvmaddasp	vs45,	vs1,	vs11

  3155. 	xvmaddasp	vs46,	vs2,	vs11

  3156. 	xvmaddasp	vs47,	vs3,	vs11

  3157. 

  3158. 

  3159. #if defined(_AIX)

  3160. ')

  3161. #else

  3162. .endm

  3163. #endif

  3164. 

  3165. #if defined(_AIX)

  3166. define(`SAVE4x16', `

  3167. #else

  3168. .macro SAVE4x16

  3169. #endif

  3170. 

  3171. 	mr		T1,	CO

  3172. 

  3173. #ifndef TRMMKERNEL

  3174. 

  3175. 	lxvw4x		vs0,	o0,	T1

  3176. 	lxvw4x		vs1,	o16,	T1

  3177. 	lxvw4x		vs2,	o32,	T1

  3178. 	lxvw4x		vs3,	o48,	T1

  3179. 

  3180. #endif

  3181. 

  3182. #ifdef TRMMKERNEL

  3183. 	xvmulsp		vs0,	vs32,	alpha_vr

  3184. 	xvmulsp		vs1,	vs33,	alpha_vr

  3185. 	xvmulsp		vs2,	vs34,	alpha_vr

  3186. 	xvmulsp		vs3,	vs35,	alpha_vr

  3187. #else

  3188. 	xvmaddasp	vs0,	vs32,	alpha_vr

  3189. 	xvmaddasp	vs1,	vs33,	alpha_vr

  3190. 	xvmaddasp	vs2,	vs34,	alpha_vr

  3191. 	xvmaddasp	vs3,	vs35,	alpha_vr

  3192. #endif

  3193. 

  3194. 	stxvw4x		vs0,	o0,	T1

  3195. 	stxvw4x		vs1,	o16,	T1

  3196. 	stxvw4x		vs2,	o32,	T1

  3197. 	stxvw4x		vs3,	o48,	T1

  3198. 

  3199. 	add		T1,	T1,	LDC

  3200. 

  3201. 

  3202. #ifndef TRMMKERNEL

  3203. 

  3204. 	lxvw4x		vs0,	o0,	T1

  3205. 	lxvw4x		vs1,	o16,	T1

  3206. 	lxvw4x		vs2,	o32,	T1

  3207. 	lxvw4x		vs3,	o48,	T1

  3208. 

  3209. #endif

  3210. 

  3211. #ifdef TRMMKERNEL

  3212. 	xvmulsp		vs0,	vs36,	alpha_vr

  3213. 	xvmulsp		vs1,	vs37,	alpha_vr

  3214. 	xvmulsp		vs2,	vs38,	alpha_vr

  3215. 	xvmulsp		vs3,	vs39,	alpha_vr

  3216. #else

  3217. 	xvmaddasp	vs0,	vs36,	alpha_vr

  3218. 	xvmaddasp	vs1,	vs37,	alpha_vr

  3219. 	xvmaddasp	vs2,	vs38,	alpha_vr

  3220. 	xvmaddasp	vs3,	vs39,	alpha_vr

  3221. #endif

  3222. 

  3223. 	stxvw4x		vs0,	o0,	T1

  3224. 	stxvw4x		vs1,	o16,	T1

  3225. 	stxvw4x		vs2,	o32,	T1

  3226. 	stxvw4x		vs3,	o48,	T1

  3227. 

  3228. 	add		T1,	T1,	LDC

  3229. 

  3230. 

  3231. #ifndef TRMMKERNEL

  3232. 

  3233. 	lxvw4x		vs0,	o0,	T1

  3234. 	lxvw4x		vs1,	o16,	T1

  3235. 	lxvw4x		vs2,	o32,	T1

  3236. 	lxvw4x		vs3,	o48,	T1

  3237. 

  3238. #endif

  3239. 

  3240. #ifdef TRMMKERNEL

  3241. 	xvmulsp		vs0,	vs40,	alpha_vr

  3242. 	xvmulsp		vs1,	vs41,	alpha_vr

  3243. 	xvmulsp		vs2,	vs42,	alpha_vr

  3244. 	xvmulsp		vs3,	vs43,	alpha_vr

  3245. #else

  3246. 	xvmaddasp	vs0,	vs40,	alpha_vr

  3247. 	xvmaddasp	vs1,	vs41,	alpha_vr

  3248. 	xvmaddasp	vs2,	vs42,	alpha_vr

  3249. 	xvmaddasp	vs3,	vs43,	alpha_vr

  3250. #endif

  3251. 

  3252. 	stxvw4x		vs0,	o0,	T1

  3253. 	stxvw4x		vs1,	o16,	T1

  3254. 	stxvw4x		vs2,	o32,	T1

  3255. 	stxvw4x		vs3,	o48,	T1

  3256. 

  3257. 	add		T1,	T1,	LDC

  3258. 

  3259. 

  3260. #ifndef TRMMKERNEL

  3261. 

  3262. 	lxvw4x		vs0,	o0,	T1

  3263. 	lxvw4x		vs1,	o16,	T1

  3264. 	lxvw4x		vs2,	o32,	T1

  3265. 	lxvw4x		vs3,	o48,	T1

  3266. 

  3267. #endif

  3268. 

  3269. #ifdef TRMMKERNEL

  3270. 	xvmulsp		vs0,	vs44,	alpha_vr

  3271. 	xvmulsp		vs1,	vs45,	alpha_vr

  3272. 	xvmulsp		vs2,	vs46,	alpha_vr

  3273. 	xvmulsp		vs3,	vs47,	alpha_vr

  3274. #else

  3275. 	xvmaddasp	vs0,	vs44,	alpha_vr

  3276. 	xvmaddasp	vs1,	vs45,	alpha_vr

  3277. 	xvmaddasp	vs2,	vs46,	alpha_vr

  3278. 	xvmaddasp	vs3,	vs47,	alpha_vr

  3279. #endif

  3280. 

  3281. 	stxvw4x		vs0,	o0,	T1

  3282. 	stxvw4x		vs1,	o16,	T1

  3283. 	stxvw4x		vs2,	o32,	T1

  3284. 	stxvw4x		vs3,	o48,	T1

  3285. 

  3286. 	add		T1,	T1,	LDC

  3287. 

  3288. 	addi		CO,	CO,	64

  3289. 

  3290. #if defined(_AIX)

  3291. ')

  3292. #else

  3293. .endm

  3294. #endif

  3295. 

  3296. 

  3297. /**********************************************************************************************

  3298. * Macros for N=4 and M=8

  3299. **********************************************************************************************/

  3300. 

  3301. #if defined(_AIX)

  3302. define(`LOAD4x8_1', `

  3303. #else

  3304. .macro LOAD4x8_1

  3305. #endif

  3306. 

  3307. 	lxvw4x		vs0,	o0,	AO

  3308. 	lxvw4x		vs1,	o16,	AO

  3309. 

  3310. 	addi		AO,	AO,	32

  3311. 

  3312. 	mr		T1,	BO

  3313. 

  3314. 	lxvw4x		vs8,	o0,	T1

  3315. 	lxvw4x		vs9,	o16,	T1

  3316. 	lxvw4x		vs10,	o32,	T1

  3317. 	lxvw4x		vs11,	o48,	T1

  3318. 

  3319. 	addi		BO,	BO,	64

  3320. 

  3321. #if defined(_AIX)

  3322. ')

  3323. #else

  3324. .endm

  3325. #endif

  3326. 

  3327. #if defined(_AIX)

  3328. define(`KERNEL4x8_I1', `

  3329. #else

  3330. .macro KERNEL4x8_I1

  3331. #endif

  3332. 

  3333. 

  3334. 	lxvw4x		vs4,	o0,	AO

  3335. 	lxvw4x		vs5,	o16,	AO

  3336. 

  3337. 	addi		AO,	AO,	32

  3338. 

  3339. 	mr		T1,	BO

  3340. 

  3341. 	lxvw4x		vs16,	o0,	T1

  3342. 	lxvw4x		vs17,	o16,	T1

  3343. 	lxvw4x		vs18,	o32,	T1

  3344. 	lxvw4x		vs19,	o48,	T1

  3345. 

  3346. 	addi		BO,	BO,	64

  3347. 

  3348. 

  3349. 	xvmulsp		vs32,	vs0,	vs8

  3350. 	xvmulsp		vs33,	vs1,	vs8

  3351. 

  3352. 	xvmulsp		vs34,	vs0,	vs9

  3353. 	xvmulsp		vs35,	vs1,	vs9

  3354. 

  3355. 	xvmulsp		vs36,	vs0,	vs10

  3356. 	xvmulsp		vs37,	vs1,	vs10

  3357. 

  3358. 	xvmulsp		vs38,	vs0,	vs11

  3359. 	xvmulsp		vs39,	vs1,	vs11

  3360. 

  3361. 

  3362. #if defined(_AIX)

  3363. ')

  3364. #else

  3365. .endm

  3366. #endif

  3367. 

  3368. #if defined(_AIX)

  3369. define(`KERNEL4x8_1', `

  3370. #else

  3371. .macro KERNEL4x8_1

  3372. #endif

  3373. 

  3374. 

  3375. 	lxvw4x		vs4,	o0,	AO

  3376. 	lxvw4x		vs5,	o16,	AO

  3377. 

  3378. 	addi		AO,	AO,	32

  3379. 

  3380. 	mr		T1,	BO

  3381. 

  3382. 	lxvw4x		vs16,	o0,	T1

  3383. 	lxvw4x		vs17,	o16,	T1

  3384. 	lxvw4x		vs18,	o32,	T1

  3385. 	lxvw4x		vs19,	o48,	T1

  3386. 

  3387. 	addi		BO,	BO,	64

  3388. 

  3389. 

  3390. 	xvmaddasp	vs32,	vs0,	vs8

  3391. 	xvmaddasp	vs33,	vs1,	vs8

  3392. 

  3393. 	xvmaddasp	vs34,	vs0,	vs9

  3394. 	xvmaddasp	vs35,	vs1,	vs9

  3395. 

  3396. 	xvmaddasp	vs36,	vs0,	vs10

  3397. 	xvmaddasp	vs37,	vs1,	vs10

  3398. 

  3399. 	xvmaddasp	vs38,	vs0,	vs11

  3400. 	xvmaddasp	vs39,	vs1,	vs11

  3401. 

  3402. 

  3403. #if defined(_AIX)

  3404. ')

  3405. #else

  3406. .endm

  3407. #endif

  3408. 

  3409. #if defined(_AIX)

  3410. define(`KERNEL4x8_2', `

  3411. #else

  3412. .macro KERNEL4x8_2

  3413. #endif

  3414. 

  3415. 

  3416. 	lxvw4x		vs0,	o0,	AO

  3417. 	lxvw4x		vs1,	o16,	AO

  3418. 

  3419. 	addi		AO,	AO,	32

  3420. 

  3421. 	mr		T1,	BO

  3422. 

  3423. 	lxvw4x		vs8,	o0,	T1

  3424. 	lxvw4x		vs9,	o16,	T1

  3425. 	lxvw4x		vs10,	o32,	T1

  3426. 	lxvw4x		vs11,	o48,	T1

  3427. 

  3428. 	addi		BO,	BO,	64

  3429. 

  3430. 

  3431. 	xvmaddasp	vs32,	vs4,	vs16

  3432. 	xvmaddasp	vs33,	vs5,	vs16

  3433. 

  3434. 	xvmaddasp	vs34,	vs4,	vs17

  3435. 	xvmaddasp	vs35,	vs5,	vs17

  3436. 

  3437. 	xvmaddasp	vs36,	vs4,	vs18

  3438. 	xvmaddasp	vs37,	vs5,	vs18

  3439. 

  3440. 	xvmaddasp	vs38,	vs4,	vs19

  3441. 	xvmaddasp	vs39,	vs5,	vs19

  3442. 

  3443. 

  3444. #if defined(_AIX)

  3445. ')

  3446. #else

  3447. .endm

  3448. #endif

  3449. 

  3450. #if defined(_AIX)

  3451. define(`KERNEL4x8_E2', `

  3452. #else

  3453. .macro KERNEL4x8_E2

  3454. #endif

  3455. 

  3456. 

  3457. 	xvmaddasp	vs32,	vs4,	vs16

  3458. 	xvmaddasp	vs33,	vs5,	vs16

  3459. 

  3460. 	xvmaddasp	vs34,	vs4,	vs17

  3461. 	xvmaddasp	vs35,	vs5,	vs17

  3462. 

  3463. 	xvmaddasp	vs36,	vs4,	vs18

  3464. 	xvmaddasp	vs37,	vs5,	vs18

  3465. 

  3466. 	xvmaddasp	vs38,	vs4,	vs19

  3467. 	xvmaddasp	vs39,	vs5,	vs19

  3468. 

  3469. 

  3470. #if defined(_AIX)

  3471. ')

  3472. #else

  3473. .endm

  3474. #endif

  3475. 

  3476. #if defined(_AIX)

  3477. define(`KERNEL4x8_SUBI1', `

  3478. #else

  3479. .macro KERNEL4x8_SUBI1

  3480. #endif

  3481. 

  3482. 

  3483. 	lxvw4x		vs0,	o0,	AO

  3484. 	lxvw4x		vs1,	o16,	AO

  3485. 

  3486. 	addi		AO,	AO,	32

  3487. 

  3488. 	mr		T1,	BO

  3489. 

  3490. 	lxvw4x		vs8,	o0,	T1

  3491. 	lxvw4x		vs9,	o16,	T1

  3492. 	lxvw4x		vs10,	o32,	T1

  3493. 	lxvw4x		vs11,	o48,	T1

  3494. 

  3495. 	addi		BO,	BO,	64

  3496. 

  3497. 

  3498. 	xvmulsp		vs32,	vs0,	vs8

  3499. 	xvmulsp		vs33,	vs1,	vs8

  3500. 

  3501. 	xvmulsp		vs34,	vs0,	vs9

  3502. 	xvmulsp		vs35,	vs1,	vs9

  3503. 

  3504. 	xvmulsp		vs36,	vs0,	vs10

  3505. 	xvmulsp		vs37,	vs1,	vs10

  3506. 

  3507. 	xvmulsp		vs38,	vs0,	vs11

  3508. 	xvmulsp		vs39,	vs1,	vs11

  3509. 

  3510. 

  3511. #if defined(_AIX)

  3512. ')

  3513. #else

  3514. .endm

  3515. #endif

  3516. 

  3517. #if defined(_AIX)

  3518. define(`KERNEL4x8_SUB1', `

  3519. #else

  3520. .macro KERNEL4x8_SUB1

  3521. #endif

  3522. 

  3523. 

  3524. 	lxvw4x		vs0,	o0,	AO

  3525. 	lxvw4x		vs1,	o16,	AO

  3526. 

  3527. 	addi		AO,	AO,	32

  3528. 

  3529. 	mr		T1,	BO

  3530. 

  3531. 	lxvw4x		vs8,	o0,	T1

  3532. 	lxvw4x		vs9,	o16,	T1

  3533. 	lxvw4x		vs10,	o32,	T1

  3534. 	lxvw4x		vs11,	o48,	T1

  3535. 

  3536. 	addi		BO,	BO,	64

  3537. 

  3538. 

  3539. 	xvmaddasp	vs32,	vs0,	vs8

  3540. 	xvmaddasp	vs33,	vs1,	vs8

  3541. 

  3542. 	xvmaddasp	vs34,	vs0,	vs9

  3543. 	xvmaddasp	vs35,	vs1,	vs9

  3544. 

  3545. 	xvmaddasp	vs36,	vs0,	vs10

  3546. 	xvmaddasp	vs37,	vs1,	vs10

  3547. 

  3548. 	xvmaddasp	vs38,	vs0,	vs11

  3549. 	xvmaddasp	vs39,	vs1,	vs11

  3550. 

  3551. 

  3552. #if defined(_AIX)

  3553. ')

  3554. #else

  3555. .endm

  3556. #endif

  3557. 

  3558. #if defined(_AIX)

  3559. define(`SAVE4x8', `

  3560. #else

  3561. .macro SAVE4x8

  3562. #endif

  3563. 

  3564. 	mr		T1,	CO

  3565. 

  3566. #ifndef TRMMKERNEL

  3567. 

  3568. 	lxvw4x		vs0,	o0,	T1

  3569. 	lxvw4x		vs1,	o16,	T1

  3570. 

  3571. #endif

  3572. 

  3573. #ifdef TRMMKERNEL

  3574. 	xvmulsp		vs0,	vs32,	alpha_vr

  3575. 	xvmulsp		vs1,	vs33,	alpha_vr

  3576. #else

  3577. 	xvmaddasp	vs0,	vs32,	alpha_vr

  3578. 	xvmaddasp	vs1,	vs33,	alpha_vr

  3579. #endif

  3580. 

  3581. 	stxvw4x		vs0,	o0,	T1

  3582. 	stxvw4x		vs1,	o16,	T1

  3583. 

  3584. 	add		T1,	T1,	LDC

  3585. 

  3586. 

  3587. #ifndef TRMMKERNEL

  3588. 

  3589. 	lxvw4x		vs0,	o0,	T1

  3590. 	lxvw4x		vs1,	o16,	T1

  3591. 

  3592. #endif

  3593. 

  3594. #ifdef TRMMKERNEL

  3595. 	xvmulsp		vs0,	vs34,	alpha_vr

  3596. 	xvmulsp		vs1,	vs35,	alpha_vr

  3597. #else

  3598. 	xvmaddasp	vs0,	vs34,	alpha_vr

  3599. 	xvmaddasp	vs1,	vs35,	alpha_vr

  3600. #endif

  3601. 

  3602. 	stxvw4x		vs0,	o0,	T1

  3603. 	stxvw4x		vs1,	o16,	T1

  3604. 

  3605. 	add		T1,	T1,	LDC

  3606. 

  3607. 

  3608. #ifndef TRMMKERNEL

  3609. 

  3610. 	lxvw4x		vs0,	o0,	T1

  3611. 	lxvw4x		vs1,	o16,	T1

  3612. 

  3613. #endif

  3614. 

  3615. #ifdef TRMMKERNEL

  3616. 	xvmulsp		vs0,	vs36,	alpha_vr

  3617. 	xvmulsp		vs1,	vs37,	alpha_vr

  3618. #else

  3619. 	xvmaddasp	vs0,	vs36,	alpha_vr

  3620. 	xvmaddasp	vs1,	vs37,	alpha_vr

  3621. #endif

  3622. 

  3623. 	stxvw4x		vs0,	o0,	T1

  3624. 	stxvw4x		vs1,	o16,	T1

  3625. 

  3626. 	add		T1,	T1,	LDC

  3627. 

  3628. 

  3629. #ifndef TRMMKERNEL

  3630. 

  3631. 	lxvw4x		vs0,	o0,	T1

  3632. 	lxvw4x		vs1,	o16,	T1

  3633. 

  3634. #endif

  3635. 

  3636. #ifdef TRMMKERNEL

  3637. 	xvmulsp		vs0,	vs38,	alpha_vr

  3638. 	xvmulsp		vs1,	vs39,	alpha_vr

  3639. #else

  3640. 	xvmaddasp	vs0,	vs38,	alpha_vr

  3641. 	xvmaddasp	vs1,	vs39,	alpha_vr

  3642. #endif

  3643. 

  3644. 	stxvw4x		vs0,	o0,	T1

  3645. 	stxvw4x		vs1,	o16,	T1

  3646. 

  3647. 	add		T1,	T1,	LDC

  3648. 

  3649. 	addi		CO,	CO,	32

  3650. 

  3651. #if defined(_AIX)

  3652. ')

  3653. #else

  3654. .endm

  3655. #endif

  3656. 

  3657. 

  3658. /**********************************************************************************************

  3659. * Macros for N=4 and M=4

  3660. **********************************************************************************************/

  3661. 

  3662. #if defined(_AIX)

  3663. define(`LOAD4x4_1', `

  3664. #else

  3665. .macro LOAD4x4_1

  3666. #endif

  3667. 

  3668. 	lxvw4x		vs0,	o0,	AO

  3669. 

  3670. 	addi		AO,	AO,	16

  3671. 

  3672. 	mr		T1,	BO

  3673. 

  3674. 	lxvw4x		vs8,	o0,	T1

  3675. 	lxvw4x		vs9,	o16,	T1

  3676. 	lxvw4x		vs10,	o32,	T1

  3677. 	lxvw4x		vs11,	o48,	T1

  3678. 

  3679. 	addi		BO,	BO,	64

  3680. 

  3681. #if defined(_AIX)

  3682. ')

  3683. #else

  3684. .endm

  3685. #endif

  3686. 

  3687. #if defined(_AIX)

  3688. define(`KERNEL4x4_I1', `

  3689. #else

  3690. .macro KERNEL4x4_I1

  3691. #endif

  3692. 

  3693. 

  3694. 	lxvw4x		vs4,	o0,	AO

  3695. 

  3696. 	addi		AO,	AO,	16

  3697. 

  3698. 	mr		T1,	BO

  3699. 

  3700. 	lxvw4x		vs16,	o0,	T1

  3701. 	lxvw4x		vs17,	o16,	T1

  3702. 	lxvw4x		vs18,	o32,	T1

  3703. 	lxvw4x		vs19,	o48,	T1

  3704. 

  3705. 	addi		BO,	BO,	64

  3706. 

  3707. 

  3708. 	xvmulsp		vs32,	vs0,	vs8

  3709. 

  3710. 	xvmulsp		vs33,	vs0,	vs9

  3711. 

  3712. 	xvmulsp		vs34,	vs0,	vs10

  3713. 

  3714. 	xvmulsp		vs35,	vs0,	vs11

  3715. 

  3716. 

  3717. #if defined(_AIX)

  3718. ')

  3719. #else

  3720. .endm

  3721. #endif

  3722. 

  3723. #if defined(_AIX)

  3724. define(`KERNEL4x4_1', `

  3725. #else

  3726. .macro KERNEL4x4_1

  3727. #endif

  3728. 

  3729. 

  3730. 	lxvw4x		vs4,	o0,	AO

  3731. 

  3732. 	addi		AO,	AO,	16

  3733. 

  3734. 	mr		T1,	BO

  3735. 

  3736. 	lxvw4x		vs16,	o0,	T1

  3737. 	lxvw4x		vs17,	o16,	T1

  3738. 	lxvw4x		vs18,	o32,	T1

  3739. 	lxvw4x		vs19,	o48,	T1

  3740. 

  3741. 	addi		BO,	BO,	64

  3742. 

  3743. 

  3744. 	xvmaddasp	vs32,	vs0,	vs8

  3745. 

  3746. 	xvmaddasp	vs33,	vs0,	vs9

  3747. 

  3748. 	xvmaddasp	vs34,	vs0,	vs10

  3749. 

  3750. 	xvmaddasp	vs35,	vs0,	vs11

  3751. 

  3752. 

  3753. #if defined(_AIX)

  3754. ')

  3755. #else

  3756. .endm

  3757. #endif

  3758. 

  3759. #if defined(_AIX)

  3760. define(`KERNEL4x4_2', `

  3761. #else

  3762. .macro KERNEL4x4_2

  3763. #endif

  3764. 

  3765. 

  3766. 	lxvw4x		vs0,	o0,	AO

  3767. 

  3768. 	addi		AO,	AO,	16

  3769. 

  3770. 	mr		T1,	BO

  3771. 

  3772. 	lxvw4x		vs8,	o0,	T1

  3773. 	lxvw4x		vs9,	o16,	T1

  3774. 	lxvw4x		vs10,	o32,	T1

  3775. 	lxvw4x		vs11,	o48,	T1

  3776. 

  3777. 	addi		BO,	BO,	64

  3778. 

  3779. 

  3780. 	xvmaddasp	vs32,	vs4,	vs16

  3781. 

  3782. 	xvmaddasp	vs33,	vs4,	vs17

  3783. 

  3784. 	xvmaddasp	vs34,	vs4,	vs18

  3785. 

  3786. 	xvmaddasp	vs35,	vs4,	vs19

  3787. 

  3788. 

  3789. #if defined(_AIX)

  3790. ')

  3791. #else

  3792. .endm

  3793. #endif

  3794. 

  3795. #if defined(_AIX)

  3796. define(`KERNEL4x4_E2', `

  3797. #else

  3798. .macro KERNEL4x4_E2

  3799. #endif

  3800. 

  3801. 

  3802. 	xvmaddasp	vs32,	vs4,	vs16

  3803. 

  3804. 	xvmaddasp	vs33,	vs4,	vs17

  3805. 

  3806. 	xvmaddasp	vs34,	vs4,	vs18

  3807. 

  3808. 	xvmaddasp	vs35,	vs4,	vs19

  3809. 

  3810. 

  3811. #if defined(_AIX)

  3812. ')

  3813. #else

  3814. .endm

  3815. #endif

  3816. 

  3817. #if defined(_AIX)

  3818. define(`KERNEL4x4_SUBI1', `

  3819. #else

  3820. .macro KERNEL4x4_SUBI1

  3821. #endif

  3822. 

  3823. 

  3824. 	lxvw4x		vs0,	o0,	AO

  3825. 

  3826. 	addi		AO,	AO,	16

  3827. 

  3828. 	mr		T1,	BO

  3829. 

  3830. 	lxvw4x		vs8,	o0,	T1

  3831. 	lxvw4x		vs9,	o16,	T1

  3832. 	lxvw4x		vs10,	o32,	T1

  3833. 	lxvw4x		vs11,	o48,	T1

  3834. 

  3835. 	addi		BO,	BO,	64

  3836. 

  3837. 

  3838. 	xvmulsp		vs32,	vs0,	vs8

  3839. 

  3840. 	xvmulsp		vs33,	vs0,	vs9

  3841. 

  3842. 	xvmulsp		vs34,	vs0,	vs10

  3843. 

  3844. 	xvmulsp		vs35,	vs0,	vs11

  3845. 

  3846. 

  3847. #if defined(_AIX)

  3848. ')

  3849. #else

  3850. .endm

  3851. #endif

  3852. 

  3853. #if defined(_AIX)

  3854. define(`KERNEL4x4_SUB1', `

  3855. #else

  3856. .macro KERNEL4x4_SUB1

  3857. #endif

  3858. 

  3859. 

  3860. 	lxvw4x		vs0,	o0,	AO

  3861. 

  3862. 	addi		AO,	AO,	16

  3863. 

  3864. 	mr		T1,	BO

  3865. 

  3866. 	lxvw4x		vs8,	o0,	T1

  3867. 	lxvw4x		vs9,	o16,	T1

  3868. 	lxvw4x		vs10,	o32,	T1

  3869. 	lxvw4x		vs11,	o48,	T1

  3870. 

  3871. 	addi		BO,	BO,	64

  3872. 

  3873. 

  3874. 	xvmaddasp	vs32,	vs0,	vs8

  3875. 

  3876. 	xvmaddasp	vs33,	vs0,	vs9

  3877. 

  3878. 	xvmaddasp	vs34,	vs0,	vs10

  3879. 

  3880. 	xvmaddasp	vs35,	vs0,	vs11

  3881. 

  3882. 

  3883. #if defined(_AIX)

  3884. ')

  3885. #else

  3886. .endm

  3887. #endif

  3888. 

  3889. #if defined(_AIX)

  3890. define(`SAVE4x4', `

  3891. #else

  3892. .macro SAVE4x4

  3893. #endif

  3894. 

  3895. 	mr		T1,	CO

  3896. 

  3897. #ifndef TRMMKERNEL

  3898. 

  3899. 	lxvw4x		vs0,	o0,	T1

  3900. 

  3901. #endif

  3902. 

  3903. #ifdef TRMMKERNEL

  3904. 	xvmulsp		vs0,	vs32,	alpha_vr

  3905. #else

  3906. 	xvmaddasp	vs0,	vs32,	alpha_vr

  3907. #endif

  3908. 

  3909. 	stxvw4x		vs0,	o0,	T1

  3910. 

  3911. 	add		T1,	T1,	LDC

  3912. 

  3913. 

  3914. #ifndef TRMMKERNEL

  3915. 

  3916. 	lxvw4x		vs0,	o0,	T1

  3917. 

  3918. #endif

  3919. 

  3920. #ifdef TRMMKERNEL

  3921. 	xvmulsp		vs0,	vs33,	alpha_vr

  3922. #else

  3923. 	xvmaddasp	vs0,	vs33,	alpha_vr

  3924. #endif

  3925. 

  3926. 	stxvw4x		vs0,	o0,	T1

  3927. 

  3928. 	add		T1,	T1,	LDC

  3929. 

  3930. 

  3931. #ifndef TRMMKERNEL

  3932. 

  3933. 	lxvw4x		vs0,	o0,	T1

  3934. 

  3935. #endif

  3936. 

  3937. #ifdef TRMMKERNEL

  3938. 	xvmulsp		vs0,	vs34,	alpha_vr

  3939. #else

  3940. 	xvmaddasp	vs0,	vs34,	alpha_vr

  3941. #endif

  3942. 

  3943. 	stxvw4x		vs0,	o0,	T1

  3944. 

  3945. 	add		T1,	T1,	LDC

  3946. 

  3947. 

  3948. #ifndef TRMMKERNEL

  3949. 

  3950. 	lxvw4x		vs0,	o0,	T1

  3951. 

  3952. #endif

  3953. 

  3954. #ifdef TRMMKERNEL

  3955. 	xvmulsp		vs0,	vs35,	alpha_vr

  3956. #else

  3957. 	xvmaddasp	vs0,	vs35,	alpha_vr

  3958. #endif

  3959. 

  3960. 	stxvw4x		vs0,	o0,	T1

  3961. 

  3962. 	add		T1,	T1,	LDC

  3963. 

  3964. 	addi		CO,	CO,	16

  3965. 

  3966. #if defined(_AIX)

  3967. ')

  3968. #else

  3969. .endm

  3970. #endif

  3971. 

  3972. 

  3973. /**********************************************************************************************

  3974. * Macros for N=4 and M=2

  3975. **********************************************************************************************/

  3976. 

  3977. #if defined(_AIX)

  3978. define(`LOAD4x2_1', `

  3979. #else

  3980. .macro LOAD4x2_1

  3981. #endif

  3982. 

  3983. 	lxsspx		vs0,	o0,	AO

  3984. 	lxsspx		vs1,	o4,	AO

  3985. 

  3986. 	addi		AO,	AO,	8

  3987. 

  3988. 	mr		T1,	BO

  3989. 

  3990. 	lxsspx		vs8,	o0,	T1

  3991. 	lxsspx		vs9,	o16,	T1

  3992. 	lxsspx		vs10,	o32,	T1

  3993. 	lxsspx		vs11,	o48,	T1

  3994. 

  3995. 

  3996. 	addi		BO,	BO,	64

  3997. 

  3998. #if defined(_AIX)

  3999. ')

  4000. #else

  4001. .endm

  4002. #endif

  4003. 

  4004. #if defined(_AIX)

  4005. define(`KERNEL4x2_I1', `

  4006. #else

  4007. .macro KERNEL4x2_I1

  4008. #endif

  4009. 

  4010. 

  4011. 	lxsspx		vs4,	o0,	AO

  4012. 	lxsspx		vs5,	o4,	AO

  4013. 

  4014. 	addi		AO,	AO,	8

  4015. 

  4016. 	mr		T1,	BO

  4017. 

  4018. 	lxsspx		vs16,	o0,	T1

  4019. 	lxsspx		vs17,	o16,	T1

  4020. 	lxsspx		vs18,	o32,	T1

  4021. 	lxsspx		vs19,	o48,	T1

  4022. 

  4023. 

  4024. 	addi		BO,	BO,	64

  4025. 

  4026. 

  4027. 	xsmuldp		vs32,	vs0,	vs8

  4028. 	xsmuldp		vs33,	vs1,	vs8

  4029. 

  4030. 	xsmuldp		vs34,	vs0,	vs9

  4031. 	xsmuldp		vs35,	vs1,	vs9

  4032. 

  4033. 	xsmuldp		vs36,	vs0,	vs10

  4034. 	xsmuldp		vs37,	vs1,	vs10

  4035. 

  4036. 	xsmuldp		vs38,	vs0,	vs11

  4037. 	xsmuldp		vs39,	vs1,	vs11

  4038. 

  4039. 

  4040. #if defined(_AIX)

  4041. ')

  4042. #else

  4043. .endm

  4044. #endif

  4045. 

  4046. #if defined(_AIX)

  4047. define(`KERNEL4x2_1', `

  4048. #else

  4049. .macro KERNEL4x2_1

  4050. #endif

  4051. 

  4052. 

  4053. 	lxsspx		vs4,	o0,	AO

  4054. 	lxsspx		vs5,	o4,	AO

  4055. 

  4056. 	addi		AO,	AO,	8

  4057. 

  4058. 	mr		T1,	BO

  4059. 

  4060. 	lxsspx		vs16,	o0,	T1

  4061. 	lxsspx		vs17,	o16,	T1

  4062. 	lxsspx		vs18,	o32,	T1

  4063. 	lxsspx		vs19,	o48,	T1

  4064. 

  4065. 

  4066. 	addi		BO,	BO,	64

  4067. 

  4068. 

  4069. 	xsmaddadp	vs32,	vs0,	vs8

  4070. 	xsmaddadp	vs33,	vs1,	vs8

  4071. 

  4072. 	xsmaddadp	vs34,	vs0,	vs9

  4073. 	xsmaddadp	vs35,	vs1,	vs9

  4074. 

  4075. 	xsmaddadp	vs36,	vs0,	vs10

  4076. 	xsmaddadp	vs37,	vs1,	vs10

  4077. 

  4078. 	xsmaddadp	vs38,	vs0,	vs11

  4079. 	xsmaddadp	vs39,	vs1,	vs11

  4080. 

  4081. 

  4082. #if defined(_AIX)

  4083. ')

  4084. #else

  4085. .endm

  4086. #endif

  4087. 

  4088. #if defined(_AIX)

  4089. define(`KERNEL4x2_2', `

  4090. #else

  4091. .macro KERNEL4x2_2

  4092. #endif

  4093. 

  4094. 

  4095. 	lxsspx		vs0,	o0,	AO

  4096. 	lxsspx		vs1,	o4,	AO

  4097. 

  4098. 	addi		AO,	AO,	8

  4099. 

  4100. 	mr		T1,	BO

  4101. 

  4102. 	lxsspx		vs8,	o0,	T1

  4103. 	lxsspx		vs9,	o16,	T1

  4104. 	lxsspx		vs10,	o32,	T1

  4105. 	lxsspx		vs11,	o48,	T1

  4106. 

  4107. 

  4108. 	addi		BO,	BO,	64

  4109. 

  4110. 

  4111. 	xsmaddadp	vs32,	vs4,	vs16

  4112. 	xsmaddadp	vs33,	vs5,	vs16

  4113. 

  4114. 	xsmaddadp	vs34,	vs4,	vs17

  4115. 	xsmaddadp	vs35,	vs5,	vs17

  4116. 

  4117. 	xsmaddadp	vs36,	vs4,	vs18

  4118. 	xsmaddadp	vs37,	vs5,	vs18

  4119. 

  4120. 	xsmaddadp	vs38,	vs4,	vs19

  4121. 	xsmaddadp	vs39,	vs5,	vs19

  4122. 

  4123. 

  4124. #if defined(_AIX)

  4125. ')

  4126. #else

  4127. .endm

  4128. #endif

  4129. 

  4130. #if defined(_AIX)

  4131. define(`KERNEL4x2_E2', `

  4132. #else

  4133. .macro KERNEL4x2_E2

  4134. #endif

  4135. 

  4136. 

  4137. 	xsmaddadp	vs32,	vs4,	vs16

  4138. 	xsmaddadp	vs33,	vs5,	vs16

  4139. 

  4140. 	xsmaddadp	vs34,	vs4,	vs17

  4141. 	xsmaddadp	vs35,	vs5,	vs17

  4142. 

  4143. 	xsmaddadp	vs36,	vs4,	vs18

  4144. 	xsmaddadp	vs37,	vs5,	vs18

  4145. 

  4146. 	xsmaddadp	vs38,	vs4,	vs19

  4147. 	xsmaddadp	vs39,	vs5,	vs19

  4148. 

  4149. 

  4150. #if defined(_AIX)

  4151. ')

  4152. #else

  4153. .endm

  4154. #endif

  4155. 

  4156. #if defined(_AIX)

  4157. define(`KERNEL4x2_SUBI1', `

  4158. #else

  4159. .macro KERNEL4x2_SUBI1

  4160. #endif

  4161. 

  4162. 

  4163. 	lxsspx		vs0,	o0,	AO

  4164. 	lxsspx		vs1,	o4,	AO

  4165. 

  4166. 	addi		AO,	AO,	8

  4167. 

  4168. 	mr		T1,	BO

  4169. 

  4170. 	lxsspx		vs8,	o0,	T1

  4171. 	lxsspx		vs9,	o16,	T1

  4172. 	lxsspx		vs10,	o32,	T1

  4173. 	lxsspx		vs11,	o48,	T1

  4174. 

  4175. 

  4176. 	addi		BO,	BO,	64

  4177. 

  4178. 

  4179. 	xsmuldp		vs32,	vs0,	vs8

  4180. 	xsmuldp		vs33,	vs1,	vs8

  4181. 

  4182. 	xsmuldp		vs34,	vs0,	vs9

  4183. 	xsmuldp		vs35,	vs1,	vs9

  4184. 

  4185. 	xsmuldp		vs36,	vs0,	vs10

  4186. 	xsmuldp		vs37,	vs1,	vs10

  4187. 

  4188. 	xsmuldp		vs38,	vs0,	vs11

  4189. 	xsmuldp		vs39,	vs1,	vs11

  4190. 

  4191. 

  4192. #if defined(_AIX)

  4193. ')

  4194. #else

  4195. .endm

  4196. #endif

  4197. 

  4198. #if defined(_AIX)

  4199. define(`KERNEL4x2_SUB1', `

  4200. #else

  4201. .macro KERNEL4x2_SUB1

  4202. #endif

  4203. 

  4204. 

  4205. 	lxsspx		vs0,	o0,	AO

  4206. 	lxsspx		vs1,	o4,	AO

  4207. 

  4208. 	addi		AO,	AO,	8

  4209. 

  4210. 	mr		T1,	BO

  4211. 

  4212. 	lxsspx		vs8,	o0,	T1

  4213. 	lxsspx		vs9,	o16,	T1

  4214. 	lxsspx		vs10,	o32,	T1

  4215. 	lxsspx		vs11,	o48,	T1

  4216. 

  4217. 

  4218. 	addi		BO,	BO,	64

  4219. 

  4220. 

  4221. 	xsmaddadp	vs32,	vs0,	vs8

  4222. 	xsmaddadp	vs33,	vs1,	vs8

  4223. 

  4224. 	xsmaddadp	vs34,	vs0,	vs9

  4225. 	xsmaddadp	vs35,	vs1,	vs9

  4226. 

  4227. 	xsmaddadp	vs36,	vs0,	vs10

  4228. 	xsmaddadp	vs37,	vs1,	vs10

  4229. 

  4230. 	xsmaddadp	vs38,	vs0,	vs11

  4231. 	xsmaddadp	vs39,	vs1,	vs11

  4232. 

  4233. 

  4234. #if defined(_AIX)

  4235. ')

  4236. #else

  4237. .endm

  4238. #endif

  4239. 

  4240. #if defined(_AIX)

  4241. define(`SAVE4x2', `

  4242. #else

  4243. .macro SAVE4x2

  4244. #endif

  4245. 

  4246. 	mr		T1,	CO

  4247. 

  4248. #ifndef TRMMKERNEL

  4249. 

  4250. 	lxsspx		vs0,	o0,	T1

  4251. 	lxsspx		vs1,	o4,	T1

  4252. 

  4253. #endif

  4254. 

  4255. #ifdef TRMMKERNEL

  4256. 	xsmuldp		vs0,	vs32,	alpha_r

  4257. 	xsmuldp		vs1,	vs33,	alpha_r

  4258. #else

  4259. 	xsmaddadp	vs0,	vs32,	alpha_r

  4260. 	xsmaddadp	vs1,	vs33,	alpha_r

  4261. #endif

  4262. 

  4263. 	stxsspx		vs0,	o0,	T1

  4264. 	stxsspx		vs1,	o4,	T1

  4265. 

  4266. 	add		T1,	T1,	LDC

  4267. 

  4268. 

  4269. #ifndef TRMMKERNEL

  4270. 

  4271. 	lxsspx		vs0,	o0,	T1

  4272. 	lxsspx		vs1,	o4,	T1

  4273. 

  4274. #endif

  4275. 

  4276. #ifdef TRMMKERNEL

  4277. 	xsmuldp		vs0,	vs34,	alpha_r

  4278. 	xsmuldp		vs1,	vs35,	alpha_r

  4279. #else

  4280. 	xsmaddadp	vs0,	vs34,	alpha_r

  4281. 	xsmaddadp	vs1,	vs35,	alpha_r

  4282. #endif

  4283. 

  4284. 	stxsspx		vs0,	o0,	T1

  4285. 	stxsspx		vs1,	o4,	T1

  4286. 

  4287. 	add		T1,	T1,	LDC

  4288. 

  4289. 

  4290. #ifndef TRMMKERNEL

  4291. 

  4292. 	lxsspx		vs0,	o0,	T1

  4293. 	lxsspx		vs1,	o4,	T1

  4294. 

  4295. #endif

  4296. 

  4297. #ifdef TRMMKERNEL

  4298. 	xsmuldp		vs0,	vs36,	alpha_r

  4299. 	xsmuldp		vs1,	vs37,	alpha_r

  4300. #else

  4301. 	xsmaddadp	vs0,	vs36,	alpha_r

  4302. 	xsmaddadp	vs1,	vs37,	alpha_r

  4303. #endif

  4304. 

  4305. 	stxsspx		vs0,	o0,	T1

  4306. 	stxsspx		vs1,	o4,	T1

  4307. 

  4308. 	add		T1,	T1,	LDC

  4309. 

  4310. 

  4311. #ifndef TRMMKERNEL

  4312. 

  4313. 	lxsspx		vs0,	o0,	T1

  4314. 	lxsspx		vs1,	o4,	T1

  4315. 

  4316. #endif

  4317. 

  4318. #ifdef TRMMKERNEL

  4319. 	xsmuldp		vs0,	vs38,	alpha_r

  4320. 	xsmuldp		vs1,	vs39,	alpha_r

  4321. #else

  4322. 	xsmaddadp	vs0,	vs38,	alpha_r

  4323. 	xsmaddadp	vs1,	vs39,	alpha_r

  4324. #endif

  4325. 

  4326. 	stxsspx		vs0,	o0,	T1

  4327. 	stxsspx		vs1,	o4,	T1

  4328. 

  4329. 	add		T1,	T1,	LDC

  4330. 

  4331. 	addi		CO,	CO,	8

  4332. 

  4333. #if defined(_AIX)

  4334. ')

  4335. #else

  4336. .endm

  4337. #endif

  4338. 

  4339. 

  4340. /**********************************************************************************************

  4341. * Macros for N=4 and M=1

  4342. **********************************************************************************************/

  4343. 

  4344. #if defined(_AIX)

  4345. define(`LOAD4x1_1', `

  4346. #else

  4347. .macro LOAD4x1_1

  4348. #endif

  4349. 

  4350. 	lxsspx		vs0,	o0,	AO

  4351. 

  4352. 	addi		AO,	AO,	4

  4353. 

  4354. 	mr		T1,	BO

  4355. 

  4356. 	lxsspx		vs8,	o0,	T1

  4357. 	lxsspx		vs9,	o16,	T1

  4358. 	lxsspx		vs10,	o32,	T1

  4359. 	lxsspx		vs11,	o48,	T1

  4360. 

  4361. 

  4362. 	addi		BO,	BO,	64

  4363. 

  4364. #if defined(_AIX)

  4365. ')

  4366. #else

  4367. .endm

  4368. #endif

  4369. 

  4370. #if defined(_AIX)

  4371. define(`KERNEL4x1_I1', `

  4372. #else

  4373. .macro KERNEL4x1_I1

  4374. #endif

  4375. 

  4376. 

  4377. 	lxsspx		vs4,	o0,	AO

  4378. 

  4379. 	addi		AO,	AO,	4

  4380. 

  4381. 	mr		T1,	BO

  4382. 

  4383. 	lxsspx		vs16,	o0,	T1

  4384. 	lxsspx		vs17,	o16,	T1

  4385. 	lxsspx		vs18,	o32,	T1

  4386. 	lxsspx		vs19,	o48,	T1

  4387. 

  4388. 

  4389. 	addi		BO,	BO,	64

  4390. 

  4391. 

  4392. 	xsmuldp		vs32,	vs0,	vs8

  4393. 

  4394. 	xsmuldp		vs33,	vs0,	vs9

  4395. 

  4396. 	xsmuldp		vs34,	vs0,	vs10

  4397. 

  4398. 	xsmuldp		vs35,	vs0,	vs11

  4399. 

  4400. 

  4401. #if defined(_AIX)

  4402. ')

  4403. #else

  4404. .endm

  4405. #endif

  4406. 

  4407. #if defined(_AIX)

  4408. define(`KERNEL4x1_1', `

  4409. #else

  4410. .macro KERNEL4x1_1

  4411. #endif

  4412. 

  4413. 

  4414. 	lxsspx		vs4,	o0,	AO

  4415. 

  4416. 	addi		AO,	AO,	4

  4417. 

  4418. 	mr		T1,	BO

  4419. 

  4420. 	lxsspx		vs16,	o0,	T1

  4421. 	lxsspx		vs17,	o16,	T1

  4422. 	lxsspx		vs18,	o32,	T1

  4423. 	lxsspx		vs19,	o48,	T1

  4424. 

  4425. 

  4426. 	addi		BO,	BO,	64

  4427. 

  4428. 

  4429. 	xsmaddadp	vs32,	vs0,	vs8

  4430. 

  4431. 	xsmaddadp	vs33,	vs0,	vs9

  4432. 

  4433. 	xsmaddadp	vs34,	vs0,	vs10

  4434. 

  4435. 	xsmaddadp	vs35,	vs0,	vs11

  4436. 

  4437. 

  4438. #if defined(_AIX)

  4439. ')

  4440. #else

  4441. .endm

  4442. #endif

  4443. 

  4444. #if defined(_AIX)

  4445. define(`KERNEL4x1_2', `

  4446. #else

  4447. .macro KERNEL4x1_2

  4448. #endif

  4449. 

  4450. 

  4451. 	lxsspx		vs0,	o0,	AO

  4452. 

  4453. 	addi		AO,	AO,	4

  4454. 

  4455. 	mr		T1,	BO

  4456. 

  4457. 	lxsspx		vs8,	o0,	T1

  4458. 	lxsspx		vs9,	o16,	T1

  4459. 	lxsspx		vs10,	o32,	T1

  4460. 	lxsspx		vs11,	o48,	T1

  4461. 

  4462. 

  4463. 	addi		BO,	BO,	64

  4464. 

  4465. 

  4466. 	xsmaddadp	vs32,	vs4,	vs16

  4467. 

  4468. 	xsmaddadp	vs33,	vs4,	vs17

  4469. 

  4470. 	xsmaddadp	vs34,	vs4,	vs18

  4471. 

  4472. 	xsmaddadp	vs35,	vs4,	vs19

  4473. 

  4474. 

  4475. #if defined(_AIX)

  4476. ')

  4477. #else

  4478. .endm

  4479. #endif

  4480. 

  4481. #if defined(_AIX)

  4482. define(`KERNEL4x1_E2', `

  4483. #else

  4484. .macro KERNEL4x1_E2

  4485. #endif

  4486. 

  4487. 

  4488. 	xsmaddadp	vs32,	vs4,	vs16

  4489. 

  4490. 	xsmaddadp	vs33,	vs4,	vs17

  4491. 

  4492. 	xsmaddadp	vs34,	vs4,	vs18

  4493. 

  4494. 	xsmaddadp	vs35,	vs4,	vs19

  4495. 

  4496. 

  4497. #if defined(_AIX)

  4498. ')

  4499. #else

  4500. .endm

  4501. #endif

  4502. 

  4503. #if defined(_AIX)

  4504. define(`KERNEL4x1_SUBI1', `

  4505. #else

  4506. .macro KERNEL4x1_SUBI1

  4507. #endif

  4508. 

  4509. 

  4510. 	lxsspx		vs0,	o0,	AO

  4511. 

  4512. 	addi		AO,	AO,	4

  4513. 

  4514. 	mr		T1,	BO

  4515. 

  4516. 	lxsspx		vs8,	o0,	T1

  4517. 	lxsspx		vs9,	o16,	T1

  4518. 	lxsspx		vs10,	o32,	T1

  4519. 	lxsspx		vs11,	o48,	T1

  4520. 

  4521. 

  4522. 	addi		BO,	BO,	64

  4523. 

  4524. 

  4525. 	xsmuldp		vs32,	vs0,	vs8

  4526. 

  4527. 	xsmuldp		vs33,	vs0,	vs9

  4528. 

  4529. 	xsmuldp		vs34,	vs0,	vs10

  4530. 

  4531. 	xsmuldp		vs35,	vs0,	vs11

  4532. 

  4533. 

  4534. #if defined(_AIX)

  4535. ')

  4536. #else

  4537. .endm

  4538. #endif

  4539. 

  4540. #if defined(_AIX)

  4541. define(`KERNEL4x1_SUB1', `

  4542. #else

  4543. .macro KERNEL4x1_SUB1

  4544. #endif

  4545. 

  4546. 

  4547. 	lxsspx		vs0,	o0,	AO

  4548. 

  4549. 	addi		AO,	AO,	4

  4550. 

  4551. 	mr		T1,	BO

  4552. 

  4553. 	lxsspx		vs8,	o0,	T1

  4554. 	lxsspx		vs9,	o16,	T1

  4555. 	lxsspx		vs10,	o32,	T1

  4556. 	lxsspx		vs11,	o48,	T1

  4557. 

  4558. 

  4559. 	addi		BO,	BO,	64

  4560. 

  4561. 

  4562. 	xsmaddadp	vs32,	vs0,	vs8

  4563. 

  4564. 	xsmaddadp	vs33,	vs0,	vs9

  4565. 

  4566. 	xsmaddadp	vs34,	vs0,	vs10

  4567. 

  4568. 	xsmaddadp	vs35,	vs0,	vs11

  4569. 

  4570. 

  4571. #if defined(_AIX)

  4572. ')

  4573. #else

  4574. .endm

  4575. #endif

  4576. 

  4577. #if defined(_AIX)

  4578. define(`SAVE4x1', `

  4579. #else

  4580. .macro SAVE4x1

  4581. #endif

  4582. 

  4583. 	mr		T1,	CO

  4584. 

  4585. #ifndef TRMMKERNEL

  4586. 

  4587. 	lxsspx		vs0,	o0,	T1

  4588. 

  4589. #endif

  4590. 

  4591. #ifdef TRMMKERNEL

  4592. 	xsmuldp		vs0,	vs32,	alpha_r

  4593. #else

  4594. 	xsmaddadp	vs0,	vs32,	alpha_r

  4595. #endif

  4596. 

  4597. 	stxsspx		vs0,	o0,	T1

  4598. 

  4599. 	add		T1,	T1,	LDC

  4600. 

  4601. 

  4602. #ifndef TRMMKERNEL

  4603. 

  4604. 	lxsspx		vs0,	o0,	T1

  4605. 

  4606. #endif

  4607. 

  4608. #ifdef TRMMKERNEL

  4609. 	xsmuldp		vs0,	vs33,	alpha_r

  4610. #else

  4611. 	xsmaddadp	vs0,	vs33,	alpha_r

  4612. #endif

  4613. 

  4614. 	stxsspx		vs0,	o0,	T1

  4615. 

  4616. 	add		T1,	T1,	LDC

  4617. 

  4618. 

  4619. #ifndef TRMMKERNEL

  4620. 

  4621. 	lxsspx		vs0,	o0,	T1

  4622. 

  4623. #endif

  4624. 

  4625. #ifdef TRMMKERNEL

  4626. 	xsmuldp		vs0,	vs34,	alpha_r

  4627. #else

  4628. 	xsmaddadp	vs0,	vs34,	alpha_r

  4629. #endif

  4630. 

  4631. 	stxsspx		vs0,	o0,	T1

  4632. 

  4633. 	add		T1,	T1,	LDC

  4634. 

  4635. 

  4636. #ifndef TRMMKERNEL

  4637. 

  4638. 	lxsspx		vs0,	o0,	T1

  4639. 

  4640. #endif

  4641. 

  4642. #ifdef TRMMKERNEL

  4643. 	xsmuldp		vs0,	vs35,	alpha_r

  4644. #else

  4645. 	xsmaddadp	vs0,	vs35,	alpha_r

  4646. #endif

  4647. 

  4648. 	stxsspx		vs0,	o0,	T1

  4649. 

  4650. 	add		T1,	T1,	LDC

  4651. 

  4652. 	addi		CO,	CO,	4

  4653. 

  4654. #if defined(_AIX)

  4655. ')

  4656. #else

  4657. .endm

  4658. #endif

  4659. 

  4660. 

  4661. /**********************************************************************************************

  4662. * Macros for N=2 and M=16

  4663. **********************************************************************************************/

  4664. 

  4665. #if defined(_AIX)

  4666. define(`LOAD2x16_1', `

  4667. #else

  4668. .macro LOAD2x16_1

  4669. #endif

  4670. 

  4671. 	lxvw4x		vs0,	o0,	AO

  4672. 	lxvw4x		vs1,	o16,	AO

  4673. 	lxvw4x		vs2,	o32,	AO

  4674. 	lxvw4x		vs3,	o48,	AO

  4675. 

  4676. 	addi		AO,	AO,	64

  4677. 

  4678. 	mr		T1,	BO

  4679. 

  4680. 	lxvw4x		vs8,	o0,	T1

  4681. 	lxvw4x		vs9,	o16,	T1

  4682. 

  4683. 	addi		BO,	BO,	32

  4684. 

  4685. #if defined(_AIX)

  4686. ')

  4687. #else

  4688. .endm

  4689. #endif

  4690. 

  4691. #if defined(_AIX)

  4692. define(`KERNEL2x16_I1', `

  4693. #else

  4694. .macro KERNEL2x16_I1

  4695. #endif

  4696. 

  4697. 

  4698. 	lxvw4x		vs4,	o0,	AO

  4699. 	lxvw4x		vs5,	o16,	AO

  4700. 	lxvw4x		vs6,	o32,	AO

  4701. 	lxvw4x		vs7,	o48,	AO

  4702. 

  4703. 	addi		AO,	AO,	64

  4704. 

  4705. 	mr		T1,	BO

  4706. 

  4707. 	lxvw4x		vs16,	o0,	T1

  4708. 	lxvw4x		vs17,	o16,	T1

  4709. 

  4710. 	addi		BO,	BO,	32

  4711. 

  4712. 

  4713. 	xvmulsp		vs32,	vs0,	vs8

  4714. 	xvmulsp		vs33,	vs1,	vs8

  4715. 	xvmulsp		vs34,	vs2,	vs8

  4716. 	xvmulsp		vs35,	vs3,	vs8

  4717. 

  4718. 	xvmulsp		vs36,	vs0,	vs9

  4719. 	xvmulsp		vs37,	vs1,	vs9

  4720. 	xvmulsp		vs38,	vs2,	vs9

  4721. 	xvmulsp		vs39,	vs3,	vs9

  4722. 

  4723. 

  4724. #if defined(_AIX)

  4725. ')

  4726. #else

  4727. .endm

  4728. #endif

  4729. 

  4730. #if defined(_AIX)

  4731. define(`KERNEL2x16_1', `

  4732. #else

  4733. .macro KERNEL2x16_1

  4734. #endif

  4735. 

  4736. 

  4737. 	lxvw4x		vs4,	o0,	AO

  4738. 	lxvw4x		vs5,	o16,	AO

  4739. 	lxvw4x		vs6,	o32,	AO

  4740. 	lxvw4x		vs7,	o48,	AO

  4741. 

  4742. 	addi		AO,	AO,	64

  4743. 

  4744. 	mr		T1,	BO

  4745. 

  4746. 	lxvw4x		vs16,	o0,	T1

  4747. 	lxvw4x		vs17,	o16,	T1

  4748. 

  4749. 	addi		BO,	BO,	32

  4750. 

  4751. 

  4752. 	xvmaddasp	vs32,	vs0,	vs8

  4753. 	xvmaddasp	vs33,	vs1,	vs8

  4754. 	xvmaddasp	vs34,	vs2,	vs8

  4755. 	xvmaddasp	vs35,	vs3,	vs8

  4756. 

  4757. 	xvmaddasp	vs36,	vs0,	vs9

  4758. 	xvmaddasp	vs37,	vs1,	vs9

  4759. 	xvmaddasp	vs38,	vs2,	vs9

  4760. 	xvmaddasp	vs39,	vs3,	vs9

  4761. 

  4762. 

  4763. #if defined(_AIX)

  4764. ')

  4765. #else

  4766. .endm

  4767. #endif

  4768. 

  4769. #if defined(_AIX)

  4770. define(`KERNEL2x16_2', `

  4771. #else

  4772. .macro KERNEL2x16_2

  4773. #endif

  4774. 

  4775. 

  4776. 	lxvw4x		vs0,	o0,	AO

  4777. 	lxvw4x		vs1,	o16,	AO

  4778. 	lxvw4x		vs2,	o32,	AO

  4779. 	lxvw4x		vs3,	o48,	AO

  4780. 

  4781. 	addi		AO,	AO,	64

  4782. 

  4783. 	mr		T1,	BO

  4784. 

  4785. 	lxvw4x		vs8,	o0,	T1

  4786. 	lxvw4x		vs9,	o16,	T1

  4787. 

  4788. 	addi		BO,	BO,	32

  4789. 

  4790. 

  4791. 	xvmaddasp	vs32,	vs4,	vs16

  4792. 	xvmaddasp	vs33,	vs5,	vs16

  4793. 	xvmaddasp	vs34,	vs6,	vs16

  4794. 	xvmaddasp	vs35,	vs7,	vs16

  4795. 

  4796. 	xvmaddasp	vs36,	vs4,	vs17

  4797. 	xvmaddasp	vs37,	vs5,	vs17

  4798. 	xvmaddasp	vs38,	vs6,	vs17

  4799. 	xvmaddasp	vs39,	vs7,	vs17

  4800. 

  4801. 

  4802. #if defined(_AIX)

  4803. ')

  4804. #else

  4805. .endm

  4806. #endif

  4807. 

  4808. #if defined(_AIX)

  4809. define(`KERNEL2x16_E2', `

  4810. #else

  4811. .macro KERNEL2x16_E2

  4812. #endif

  4813. 

  4814. 

  4815. 	xvmaddasp	vs32,	vs4,	vs16

  4816. 	xvmaddasp	vs33,	vs5,	vs16

  4817. 	xvmaddasp	vs34,	vs6,	vs16

  4818. 	xvmaddasp	vs35,	vs7,	vs16

  4819. 

  4820. 	xvmaddasp	vs36,	vs4,	vs17

  4821. 	xvmaddasp	vs37,	vs5,	vs17

  4822. 	xvmaddasp	vs38,	vs6,	vs17

  4823. 	xvmaddasp	vs39,	vs7,	vs17

  4824. 

  4825. 

  4826. #if defined(_AIX)

  4827. ')

  4828. #else

  4829. .endm

  4830. #endif

  4831. 

  4832. #if defined(_AIX)

  4833. define(`KERNEL2x16_SUBI1', `

  4834. #else

  4835. .macro KERNEL2x16_SUBI1

  4836. #endif

  4837. 

  4838. 

  4839. 	lxvw4x		vs0,	o0,	AO

  4840. 	lxvw4x		vs1,	o16,	AO

  4841. 	lxvw4x		vs2,	o32,	AO

  4842. 	lxvw4x		vs3,	o48,	AO

  4843. 

  4844. 	addi		AO,	AO,	64

  4845. 

  4846. 	mr		T1,	BO

  4847. 

  4848. 	lxvw4x		vs8,	o0,	T1

  4849. 	lxvw4x		vs9,	o16,	T1

  4850. 

  4851. 	addi		BO,	BO,	32

  4852. 

  4853. 

  4854. 	xvmulsp		vs32,	vs0,	vs8

  4855. 	xvmulsp		vs33,	vs1,	vs8

  4856. 	xvmulsp		vs34,	vs2,	vs8

  4857. 	xvmulsp		vs35,	vs3,	vs8

  4858. 

  4859. 	xvmulsp		vs36,	vs0,	vs9

  4860. 	xvmulsp		vs37,	vs1,	vs9

  4861. 	xvmulsp		vs38,	vs2,	vs9

  4862. 	xvmulsp		vs39,	vs3,	vs9

  4863. 

  4864. 

  4865. #if defined(_AIX)

  4866. ')

  4867. #else

  4868. .endm

  4869. #endif

  4870. 

  4871. #if defined(_AIX)

  4872. define(`KERNEL2x16_SUB1', `

  4873. #else

  4874. .macro KERNEL2x16_SUB1

  4875. #endif

  4876. 

  4877. 

  4878. 	lxvw4x		vs0,	o0,	AO

  4879. 	lxvw4x		vs1,	o16,	AO

  4880. 	lxvw4x		vs2,	o32,	AO

  4881. 	lxvw4x		vs3,	o48,	AO

  4882. 

  4883. 	addi		AO,	AO,	64

  4884. 

  4885. 	mr		T1,	BO

  4886. 

  4887. 	lxvw4x		vs8,	o0,	T1

  4888. 	lxvw4x		vs9,	o16,	T1

  4889. 

  4890. 	addi		BO,	BO,	32

  4891. 

  4892. 

  4893. 	xvmaddasp	vs32,	vs0,	vs8

  4894. 	xvmaddasp	vs33,	vs1,	vs8

  4895. 	xvmaddasp	vs34,	vs2,	vs8

  4896. 	xvmaddasp	vs35,	vs3,	vs8

  4897. 

  4898. 	xvmaddasp	vs36,	vs0,	vs9

  4899. 	xvmaddasp	vs37,	vs1,	vs9

  4900. 	xvmaddasp	vs38,	vs2,	vs9

  4901. 	xvmaddasp	vs39,	vs3,	vs9

  4902. 

  4903. 

  4904. #if defined(_AIX)

  4905. ')

  4906. #else

  4907. .endm

  4908. #endif

  4909. 

  4910. #if defined(_AIX)

  4911. define(`SAVE2x16', `

  4912. #else

  4913. .macro SAVE2x16

  4914. #endif

  4915. 

  4916. 	mr		T1,	CO

  4917. 

  4918. #ifndef TRMMKERNEL

  4919. 

  4920. 	lxvw4x		vs0,	o0,	T1

  4921. 	lxvw4x		vs1,	o16,	T1

  4922. 	lxvw4x		vs2,	o32,	T1

  4923. 	lxvw4x		vs3,	o48,	T1

  4924. 

  4925. #endif

  4926. 

  4927. #ifdef TRMMKERNEL

  4928. 	xvmulsp		vs0,	vs32,	alpha_vr

  4929. 	xvmulsp		vs1,	vs33,	alpha_vr

  4930. 	xvmulsp		vs2,	vs34,	alpha_vr

  4931. 	xvmulsp		vs3,	vs35,	alpha_vr

  4932. #else

  4933. 	xvmaddasp	vs0,	vs32,	alpha_vr

  4934. 	xvmaddasp	vs1,	vs33,	alpha_vr

  4935. 	xvmaddasp	vs2,	vs34,	alpha_vr

  4936. 	xvmaddasp	vs3,	vs35,	alpha_vr

  4937. #endif

  4938. 

  4939. 	stxvw4x		vs0,	o0,	T1

  4940. 	stxvw4x		vs1,	o16,	T1

  4941. 	stxvw4x		vs2,	o32,	T1

  4942. 	stxvw4x		vs3,	o48,	T1

  4943. 

  4944. 	add		T1,	T1,	LDC

  4945. 

  4946. 

  4947. #ifndef TRMMKERNEL

  4948. 

  4949. 	lxvw4x		vs0,	o0,	T1

  4950. 	lxvw4x		vs1,	o16,	T1

  4951. 	lxvw4x		vs2,	o32,	T1

  4952. 	lxvw4x		vs3,	o48,	T1

  4953. 

  4954. #endif

  4955. 

  4956. #ifdef TRMMKERNEL

  4957. 	xvmulsp		vs0,	vs36,	alpha_vr

  4958. 	xvmulsp		vs1,	vs37,	alpha_vr

  4959. 	xvmulsp		vs2,	vs38,	alpha_vr

  4960. 	xvmulsp		vs3,	vs39,	alpha_vr

  4961. #else

  4962. 	xvmaddasp	vs0,	vs36,	alpha_vr

  4963. 	xvmaddasp	vs1,	vs37,	alpha_vr

  4964. 	xvmaddasp	vs2,	vs38,	alpha_vr

  4965. 	xvmaddasp	vs3,	vs39,	alpha_vr

  4966. #endif

  4967. 

  4968. 	stxvw4x		vs0,	o0,	T1

  4969. 	stxvw4x		vs1,	o16,	T1

  4970. 	stxvw4x		vs2,	o32,	T1

  4971. 	stxvw4x		vs3,	o48,	T1

  4972. 

  4973. 	add		T1,	T1,	LDC

  4974. 

  4975. 	addi		CO,	CO,	64

  4976. 

  4977. #if defined(_AIX)

  4978. ')

  4979. #else

  4980. .endm

  4981. #endif

  4982. 

  4983. 

  4984. /**********************************************************************************************

  4985. * Macros for N=2 and M=8

  4986. **********************************************************************************************/

  4987. 

  4988. #if defined(_AIX)

  4989. define(`LOAD2x8_1', `

  4990. #else

  4991. .macro LOAD2x8_1

  4992. #endif

  4993. 

  4994. 	lxvw4x		vs0,	o0,	AO

  4995. 	lxvw4x		vs1,	o16,	AO

  4996. 

  4997. 	addi		AO,	AO,	32

  4998. 

  4999. 	mr		T1,	BO

  5000. 

  5001. 	lxvw4x		vs8,	o0,	T1

  5002. 	lxvw4x		vs9,	o16,	T1

  5003. 

  5004. 	addi		BO,	BO,	32

  5005. 

  5006. #if defined(_AIX)

  5007. ')

  5008. #else

  5009. .endm

  5010. #endif

  5011. 

  5012. #if defined(_AIX)

  5013. define(`KERNEL2x8_I1', `

  5014. #else

  5015. .macro KERNEL2x8_I1

  5016. #endif

  5017. 

  5018. 

  5019. 	lxvw4x		vs4,	o0,	AO

  5020. 	lxvw4x		vs5,	o16,	AO

  5021. 

  5022. 	addi		AO,	AO,	32

  5023. 

  5024. 	mr		T1,	BO

  5025. 

  5026. 	lxvw4x		vs16,	o0,	T1

  5027. 	lxvw4x		vs17,	o16,	T1

  5028. 

  5029. 	addi		BO,	BO,	32

  5030. 

  5031. 

  5032. 	xvmulsp		vs32,	vs0,	vs8

  5033. 	xvmulsp		vs33,	vs1,	vs8

  5034. 

  5035. 	xvmulsp		vs34,	vs0,	vs9

  5036. 	xvmulsp		vs35,	vs1,	vs9

  5037. 

  5038. 

  5039. #if defined(_AIX)

  5040. ')

  5041. #else

  5042. .endm

  5043. #endif

  5044. 

  5045. #if defined(_AIX)

  5046. define(`KERNEL2x8_1', `

  5047. #else

  5048. .macro KERNEL2x8_1

  5049. #endif

  5050. 

  5051. 

  5052. 	lxvw4x		vs4,	o0,	AO

  5053. 	lxvw4x		vs5,	o16,	AO

  5054. 

  5055. 	addi		AO,	AO,	32

  5056. 

  5057. 	mr		T1,	BO

  5058. 

  5059. 	lxvw4x		vs16,	o0,	T1

  5060. 	lxvw4x		vs17,	o16,	T1

  5061. 

  5062. 	addi		BO,	BO,	32

  5063. 

  5064. 

  5065. 	xvmaddasp	vs32,	vs0,	vs8

  5066. 	xvmaddasp	vs33,	vs1,	vs8

  5067. 

  5068. 	xvmaddasp	vs34,	vs0,	vs9

  5069. 	xvmaddasp	vs35,	vs1,	vs9

  5070. 

  5071. 

  5072. #if defined(_AIX)

  5073. ')

  5074. #else

  5075. .endm

  5076. #endif

  5077. 

  5078. #if defined(_AIX)

  5079. define(`KERNEL2x8_2', `

  5080. #else

  5081. .macro KERNEL2x8_2

  5082. #endif

  5083. 

  5084. 

  5085. 	lxvw4x		vs0,	o0,	AO

  5086. 	lxvw4x		vs1,	o16,	AO

  5087. 

  5088. 	addi		AO,	AO,	32

  5089. 

  5090. 	mr		T1,	BO

  5091. 

  5092. 	lxvw4x		vs8,	o0,	T1

  5093. 	lxvw4x		vs9,	o16,	T1

  5094. 

  5095. 	addi		BO,	BO,	32

  5096. 

  5097. 

  5098. 	xvmaddasp	vs32,	vs4,	vs16

  5099. 	xvmaddasp	vs33,	vs5,	vs16

  5100. 

  5101. 	xvmaddasp	vs34,	vs4,	vs17

  5102. 	xvmaddasp	vs35,	vs5,	vs17

  5103. 

  5104. 

  5105. #if defined(_AIX)

  5106. ')

  5107. #else

  5108. .endm

  5109. #endif

  5110. 

  5111. #if defined(_AIX)

  5112. define(`KERNEL2x8_E2', `

  5113. #else

  5114. .macro KERNEL2x8_E2

  5115. #endif

  5116. 

  5117. 

  5118. 	xvmaddasp	vs32,	vs4,	vs16

  5119. 	xvmaddasp	vs33,	vs5,	vs16

  5120. 

  5121. 	xvmaddasp	vs34,	vs4,	vs17

  5122. 	xvmaddasp	vs35,	vs5,	vs17

  5123. 

  5124. 

  5125. #if defined(_AIX)

  5126. ')

  5127. #else

  5128. .endm

  5129. #endif

  5130. 

  5131. #if defined(_AIX)

  5132. define(`KERNEL2x8_SUBI1', `

  5133. #else

  5134. .macro KERNEL2x8_SUBI1

  5135. #endif

  5136. 

  5137. 

  5138. 	lxvw4x		vs0,	o0,	AO

  5139. 	lxvw4x		vs1,	o16,	AO

  5140. 

  5141. 	addi		AO,	AO,	32

  5142. 

  5143. 	mr		T1,	BO

  5144. 

  5145. 	lxvw4x		vs8,	o0,	T1

  5146. 	lxvw4x		vs9,	o16,	T1

  5147. 

  5148. 	addi		BO,	BO,	32

  5149. 

  5150. 

  5151. 	xvmulsp		vs32,	vs0,	vs8

  5152. 	xvmulsp		vs33,	vs1,	vs8

  5153. 

  5154. 	xvmulsp		vs34,	vs0,	vs9

  5155. 	xvmulsp		vs35,	vs1,	vs9

  5156. 

  5157. 

  5158. #if defined(_AIX)

  5159. ')

  5160. #else

  5161. .endm

  5162. #endif

  5163. 

  5164. #if defined(_AIX)

  5165. define(`KERNEL2x8_SUB1', `

  5166. #else

  5167. .macro KERNEL2x8_SUB1

  5168. #endif

  5169. 

  5170. 

  5171. 	lxvw4x		vs0,	o0,	AO

  5172. 	lxvw4x		vs1,	o16,	AO

  5173. 

  5174. 	addi		AO,	AO,	32

  5175. 

  5176. 	mr		T1,	BO

  5177. 

  5178. 	lxvw4x		vs8,	o0,	T1

  5179. 	lxvw4x		vs9,	o16,	T1

  5180. 

  5181. 	addi		BO,	BO,	32

  5182. 

  5183. 

  5184. 	xvmaddasp	vs32,	vs0,	vs8

  5185. 	xvmaddasp	vs33,	vs1,	vs8

  5186. 

  5187. 	xvmaddasp	vs34,	vs0,	vs9

  5188. 	xvmaddasp	vs35,	vs1,	vs9

  5189. 

  5190. 

  5191. #if defined(_AIX)

  5192. ')

  5193. #else

  5194. .endm

  5195. #endif

  5196. 

  5197. #if defined(_AIX)

  5198. define(`SAVE2x8', `

  5199. #else

  5200. .macro SAVE2x8

  5201. #endif

  5202. 

  5203. 	mr		T1,	CO

  5204. 

  5205. #ifndef TRMMKERNEL

  5206. 

  5207. 	lxvw4x		vs0,	o0,	T1

  5208. 	lxvw4x		vs1,	o16,	T1

  5209. 

  5210. #endif

  5211. 

  5212. #ifdef TRMMKERNEL

  5213. 	xvmulsp		vs0,	vs32,	alpha_vr

  5214. 	xvmulsp		vs1,	vs33,	alpha_vr

  5215. #else

  5216. 	xvmaddasp	vs0,	vs32,	alpha_vr

  5217. 	xvmaddasp	vs1,	vs33,	alpha_vr

  5218. #endif

  5219. 

  5220. 	stxvw4x		vs0,	o0,	T1

  5221. 	stxvw4x		vs1,	o16,	T1

  5222. 

  5223. 	add		T1,	T1,	LDC

  5224. 

  5225. 

  5226. #ifndef TRMMKERNEL

  5227. 

  5228. 	lxvw4x		vs0,	o0,	T1

  5229. 	lxvw4x		vs1,	o16,	T1

  5230. 

  5231. #endif

  5232. 

  5233. #ifdef TRMMKERNEL

  5234. 	xvmulsp		vs0,	vs34,	alpha_vr

  5235. 	xvmulsp		vs1,	vs35,	alpha_vr

  5236. #else

  5237. 	xvmaddasp	vs0,	vs34,	alpha_vr

  5238. 	xvmaddasp	vs1,	vs35,	alpha_vr

  5239. #endif

  5240. 

  5241. 	stxvw4x		vs0,	o0,	T1

  5242. 	stxvw4x		vs1,	o16,	T1

  5243. 

  5244. 	add		T1,	T1,	LDC

  5245. 

  5246. 	addi		CO,	CO,	32

  5247. 

  5248. #if defined(_AIX)

  5249. ')

  5250. #else

  5251. .endm

  5252. #endif

  5253. 

  5254. 

  5255. /**********************************************************************************************

  5256. * Macros for N=2 and M=4

  5257. **********************************************************************************************/

  5258. 

  5259. #if defined(_AIX)

  5260. define(`LOAD2x4_1', `

  5261. #else

  5262. .macro LOAD2x4_1

  5263. #endif

  5264. 

  5265. 	lxvw4x		vs0,	o0,	AO

  5266. 

  5267. 	addi		AO,	AO,	16

  5268. 

  5269. 	mr		T1,	BO

  5270. 

  5271. 	lxvw4x		vs8,	o0,	T1

  5272. 	lxvw4x		vs9,	o16,	T1

  5273. 

  5274. 	addi		BO,	BO,	32

  5275. 

  5276. #if defined(_AIX)

  5277. ')

  5278. #else

  5279. .endm

  5280. #endif

  5281. 

  5282. #if defined(_AIX)

  5283. define(`KERNEL2x4_I1', `

  5284. #else

  5285. .macro KERNEL2x4_I1

  5286. #endif

  5287. 

  5288. 

  5289. 	lxvw4x		vs4,	o0,	AO

  5290. 

  5291. 	addi		AO,	AO,	16

  5292. 

  5293. 	mr		T1,	BO

  5294. 

  5295. 	lxvw4x		vs16,	o0,	T1

  5296. 	lxvw4x		vs17,	o16,	T1

  5297. 

  5298. 	addi		BO,	BO,	32

  5299. 

  5300. 

  5301. 	xvmulsp		vs32,	vs0,	vs8

  5302. 

  5303. 	xvmulsp		vs33,	vs0,	vs9

  5304. 

  5305. 

  5306. #if defined(_AIX)

  5307. ')

  5308. #else

  5309. .endm

  5310. #endif

  5311. 

  5312. #if defined(_AIX)

  5313. define(`KERNEL2x4_1', `

  5314. #else

  5315. .macro KERNEL2x4_1

  5316. #endif

  5317. 

  5318. 

  5319. 	lxvw4x		vs4,	o0,	AO

  5320. 

  5321. 	addi		AO,	AO,	16

  5322. 

  5323. 	mr		T1,	BO

  5324. 

  5325. 	lxvw4x		vs16,	o0,	T1

  5326. 	lxvw4x		vs17,	o16,	T1

  5327. 

  5328. 	addi		BO,	BO,	32

  5329. 

  5330. 

  5331. 	xvmaddasp	vs32,	vs0,	vs8

  5332. 

  5333. 	xvmaddasp	vs33,	vs0,	vs9

  5334. 

  5335. 

  5336. #if defined(_AIX)

  5337. ')

  5338. #else

  5339. .endm

  5340. #endif

  5341. 

  5342. #if defined(_AIX)

  5343. define(`KERNEL2x4_2', `

  5344. #else

  5345. .macro KERNEL2x4_2

  5346. #endif

  5347. 

  5348. 

  5349. 	lxvw4x		vs0,	o0,	AO

  5350. 

  5351. 	addi		AO,	AO,	16

  5352. 

  5353. 	mr		T1,	BO

  5354. 

  5355. 	lxvw4x		vs8,	o0,	T1

  5356. 	lxvw4x		vs9,	o16,	T1

  5357. 

  5358. 	addi		BO,	BO,	32

  5359. 

  5360. 

  5361. 	xvmaddasp	vs32,	vs4,	vs16

  5362. 

  5363. 	xvmaddasp	vs33,	vs4,	vs17

  5364. 

  5365. 

  5366. #if defined(_AIX)

  5367. ')

  5368. #else

  5369. .endm

  5370. #endif

  5371. 

  5372. #if defined(_AIX)

  5373. define(`KERNEL2x4_E2', `

  5374. #else

  5375. .macro KERNEL2x4_E2

  5376. #endif

  5377. 

  5378. 

  5379. 	xvmaddasp	vs32,	vs4,	vs16

  5380. 

  5381. 	xvmaddasp	vs33,	vs4,	vs17

  5382. 

  5383. 

  5384. #if defined(_AIX)

  5385. ')

  5386. #else

  5387. .endm

  5388. #endif

  5389. 

  5390. #if defined(_AIX)

  5391. define(`KERNEL2x4_SUBI1', `

  5392. #else

  5393. .macro KERNEL2x4_SUBI1

  5394. #endif

  5395. 

  5396. 

  5397. 	lxvw4x		vs0,	o0,	AO

  5398. 

  5399. 	addi		AO,	AO,	16

  5400. 

  5401. 	mr		T1,	BO

  5402. 

  5403. 	lxvw4x		vs8,	o0,	T1

  5404. 	lxvw4x		vs9,	o16,	T1

  5405. 

  5406. 	addi		BO,	BO,	32

  5407. 

  5408. 

  5409. 	xvmulsp		vs32,	vs0,	vs8

  5410. 

  5411. 	xvmulsp		vs33,	vs0,	vs9

  5412. 

  5413. 

  5414. #if defined(_AIX)

  5415. ')

  5416. #else

  5417. .endm

  5418. #endif

  5419. 

  5420. #if defined(_AIX)

  5421. define(`KERNEL2x4_SUB1', `

  5422. #else

  5423. .macro KERNEL2x4_SUB1

  5424. #endif

  5425. 

  5426. 

  5427. 	lxvw4x		vs0,	o0,	AO

  5428. 

  5429. 	addi		AO,	AO,	16

  5430. 

  5431. 	mr		T1,	BO

  5432. 

  5433. 	lxvw4x		vs8,	o0,	T1

  5434. 	lxvw4x		vs9,	o16,	T1

  5435. 

  5436. 	addi		BO,	BO,	32

  5437. 

  5438. 

  5439. 	xvmaddasp	vs32,	vs0,	vs8

  5440. 

  5441. 	xvmaddasp	vs33,	vs0,	vs9

  5442. 

  5443. 

  5444. #if defined(_AIX)

  5445. ')

  5446. #else

  5447. .endm

  5448. #endif

  5449. 

  5450. #if defined(_AIX)

  5451. define(`SAVE2x4', `

  5452. #else

  5453. .macro SAVE2x4

  5454. #endif

  5455. 

  5456. 	mr		T1,	CO

  5457. 

  5458. #ifndef TRMMKERNEL

  5459. 

  5460. 	lxvw4x		vs0,	o0,	T1

  5461. 

  5462. #endif

  5463. 

  5464. #ifdef TRMMKERNEL

  5465. 	xvmulsp		vs0,	vs32,	alpha_vr

  5466. #else

  5467. 	xvmaddasp	vs0,	vs32,	alpha_vr

  5468. #endif

  5469. 

  5470. 	stxvw4x		vs0,	o0,	T1

  5471. 

  5472. 	add		T1,	T1,	LDC

  5473. 

  5474. 

  5475. #ifndef TRMMKERNEL

  5476. 

  5477. 	lxvw4x		vs0,	o0,	T1

  5478. 

  5479. #endif

  5480. 

  5481. #ifdef TRMMKERNEL

  5482. 	xvmulsp		vs0,	vs33,	alpha_vr

  5483. #else

  5484. 	xvmaddasp	vs0,	vs33,	alpha_vr

  5485. #endif

  5486. 

  5487. 	stxvw4x		vs0,	o0,	T1

  5488. 

  5489. 	add		T1,	T1,	LDC

  5490. 

  5491. 	addi		CO,	CO,	16

  5492. 

  5493. #if defined(_AIX)

  5494. ')

  5495. #else

  5496. .endm

  5497. #endif

  5498. 

  5499. 

  5500. /**********************************************************************************************

  5501. * Macros for N=2 and M=2

  5502. **********************************************************************************************/

  5503. 

  5504. #if defined(_AIX)

  5505. define(`LOAD2x2_1', `

  5506. #else

  5507. .macro LOAD2x2_1

  5508. #endif

  5509. 

  5510. 	lxsspx		vs0,	o0,	AO

  5511. 	lxsspx		vs1,	o4,	AO

  5512. 

  5513. 	addi		AO,	AO,	8

  5514. 

  5515. 	mr		T1,	BO

  5516. 

  5517. 	lxsspx		vs8,	o0,	T1

  5518. 	lxsspx		vs9,	o16,	T1

  5519. 

  5520. 

  5521. 	addi		BO,	BO,	32

  5522. 

  5523. #if defined(_AIX)

  5524. ')

  5525. #else

  5526. .endm

  5527. #endif

  5528. 

  5529. #if defined(_AIX)

  5530. define(`KERNEL2x2_I1', `

  5531. #else

  5532. .macro KERNEL2x2_I1

  5533. #endif

  5534. 

  5535. 

  5536. 	lxsspx		vs4,	o0,	AO

  5537. 	lxsspx		vs5,	o4,	AO

  5538. 

  5539. 	addi		AO,	AO,	8

  5540. 

  5541. 	mr		T1,	BO

  5542. 

  5543. 	lxsspx		vs16,	o0,	T1

  5544. 	lxsspx		vs17,	o16,	T1

  5545. 

  5546. 

  5547. 	addi		BO,	BO,	32

  5548. 

  5549. 

  5550. 	xsmuldp		vs32,	vs0,	vs8

  5551. 	xsmuldp		vs33,	vs1,	vs8

  5552. 

  5553. 	xsmuldp		vs34,	vs0,	vs9

  5554. 	xsmuldp		vs35,	vs1,	vs9

  5555. 

  5556. 

  5557. #if defined(_AIX)

  5558. ')

  5559. #else

  5560. .endm

  5561. #endif

  5562. 

  5563. #if defined(_AIX)

  5564. define(`KERNEL2x2_1', `

  5565. #else

  5566. .macro KERNEL2x2_1

  5567. #endif

  5568. 

  5569. 

  5570. 	lxsspx		vs4,	o0,	AO

  5571. 	lxsspx		vs5,	o4,	AO

  5572. 

  5573. 	addi		AO,	AO,	8

  5574. 

  5575. 	mr		T1,	BO

  5576. 

  5577. 	lxsspx		vs16,	o0,	T1

  5578. 	lxsspx		vs17,	o16,	T1

  5579. 

  5580. 

  5581. 	addi		BO,	BO,	32

  5582. 

  5583. 

  5584. 	xsmaddadp	vs32,	vs0,	vs8

  5585. 	xsmaddadp	vs33,	vs1,	vs8

  5586. 

  5587. 	xsmaddadp	vs34,	vs0,	vs9

  5588. 	xsmaddadp	vs35,	vs1,	vs9

  5589. 

  5590. 

  5591. #if defined(_AIX)

  5592. ')

  5593. #else

  5594. .endm

  5595. #endif

  5596. 

  5597. #if defined(_AIX)

  5598. define(`KERNEL2x2_2', `

  5599. #else

  5600. .macro KERNEL2x2_2

  5601. #endif

  5602. 

  5603. 

  5604. 	lxsspx		vs0,	o0,	AO

  5605. 	lxsspx		vs1,	o4,	AO

  5606. 

  5607. 	addi		AO,	AO,	8

  5608. 

  5609. 	mr		T1,	BO

  5610. 

  5611. 	lxsspx		vs8,	o0,	T1

  5612. 	lxsspx		vs9,	o16,	T1

  5613. 

  5614. 

  5615. 	addi		BO,	BO,	32

  5616. 

  5617. 

  5618. 	xsmaddadp	vs32,	vs4,	vs16

  5619. 	xsmaddadp	vs33,	vs5,	vs16

  5620. 

  5621. 	xsmaddadp	vs34,	vs4,	vs17

  5622. 	xsmaddadp	vs35,	vs5,	vs17

  5623. 

  5624. 

  5625. #if defined(_AIX)

  5626. ')

  5627. #else

  5628. .endm

  5629. #endif

  5630. 

  5631. #if defined(_AIX)

  5632. define(`KERNEL2x2_E2', `

  5633. #else

  5634. .macro KERNEL2x2_E2

  5635. #endif

  5636. 

  5637. 

  5638. 	xsmaddadp	vs32,	vs4,	vs16

  5639. 	xsmaddadp	vs33,	vs5,	vs16

  5640. 

  5641. 	xsmaddadp	vs34,	vs4,	vs17

  5642. 	xsmaddadp	vs35,	vs5,	vs17

  5643. 

  5644. 

  5645. #if defined(_AIX)

  5646. ')

  5647. #else

  5648. .endm

  5649. #endif

  5650. 

  5651. #if defined(_AIX)

  5652. define(`KERNEL2x2_SUBI1', `

  5653. #else

  5654. .macro KERNEL2x2_SUBI1

  5655. #endif

  5656. 

  5657. 

  5658. 	lxsspx		vs0,	o0,	AO

  5659. 	lxsspx		vs1,	o4,	AO

  5660. 

  5661. 	addi		AO,	AO,	8

  5662. 

  5663. 	mr		T1,	BO

  5664. 

  5665. 	lxsspx		vs8,	o0,	T1

  5666. 	lxsspx		vs9,	o16,	T1

  5667. 

  5668. 

  5669. 	addi		BO,	BO,	32

  5670. 

  5671. 

  5672. 	xsmuldp		vs32,	vs0,	vs8

  5673. 	xsmuldp		vs33,	vs1,	vs8

  5674. 

  5675. 	xsmuldp		vs34,	vs0,	vs9

  5676. 	xsmuldp		vs35,	vs1,	vs9

  5677. 

  5678. 

  5679. #if defined(_AIX)

  5680. ')

  5681. #else

  5682. .endm

  5683. #endif

  5684. 

  5685. #if defined(_AIX)

  5686. define(`KERNEL2x2_SUB1', `

  5687. #else

  5688. .macro KERNEL2x2_SUB1

  5689. #endif

  5690. 

  5691. 

  5692. 	lxsspx		vs0,	o0,	AO

  5693. 	lxsspx		vs1,	o4,	AO

  5694. 

  5695. 	addi		AO,	AO,	8

  5696. 

  5697. 	mr		T1,	BO

  5698. 

  5699. 	lxsspx		vs8,	o0,	T1

  5700. 	lxsspx		vs9,	o16,	T1

  5701. 

  5702. 

  5703. 	addi		BO,	BO,	32

  5704. 

  5705. 

  5706. 	xsmaddadp	vs32,	vs0,	vs8

  5707. 	xsmaddadp	vs33,	vs1,	vs8

  5708. 

  5709. 	xsmaddadp	vs34,	vs0,	vs9

  5710. 	xsmaddadp	vs35,	vs1,	vs9

  5711. 

  5712. 

  5713. #if defined(_AIX)

  5714. ')

  5715. #else

  5716. .endm

  5717. #endif

  5718. 

  5719. #if defined(_AIX)

  5720. define(`SAVE2x2', `

  5721. #else

  5722. .macro SAVE2x2

  5723. #endif

  5724. 

  5725. 	mr		T1,	CO

  5726. 

  5727. #ifndef TRMMKERNEL

  5728. 

  5729. 	lxsspx		vs0,	o0,	T1

  5730. 	lxsspx		vs1,	o4,	T1

  5731. 

  5732. #endif

  5733. 

  5734. #ifdef TRMMKERNEL

  5735. 	xsmuldp		vs0,	vs32,	alpha_r

  5736. 	xsmuldp		vs1,	vs33,	alpha_r

  5737. #else

  5738. 	xsmaddadp	vs0,	vs32,	alpha_r

  5739. 	xsmaddadp	vs1,	vs33,	alpha_r

  5740. #endif

  5741. 

  5742. 	stxsspx		vs0,	o0,	T1

  5743. 	stxsspx		vs1,	o4,	T1

  5744. 

  5745. 	add		T1,	T1,	LDC

  5746. 

  5747. 

  5748. #ifndef TRMMKERNEL

  5749. 

  5750. 	lxsspx		vs0,	o0,	T1

  5751. 	lxsspx		vs1,	o4,	T1

  5752. 

  5753. #endif

  5754. 

  5755. #ifdef TRMMKERNEL

  5756. 	xsmuldp		vs0,	vs34,	alpha_r

  5757. 	xsmuldp		vs1,	vs35,	alpha_r

  5758. #else

  5759. 	xsmaddadp	vs0,	vs34,	alpha_r

  5760. 	xsmaddadp	vs1,	vs35,	alpha_r

  5761. #endif

  5762. 

  5763. 	stxsspx		vs0,	o0,	T1

  5764. 	stxsspx		vs1,	o4,	T1

  5765. 

  5766. 	add		T1,	T1,	LDC

  5767. 

  5768. 	addi		CO,	CO,	8

  5769. 

  5770. #if defined(_AIX)

  5771. ')

  5772. #else

  5773. .endm

  5774. #endif

  5775. 

  5776. 

  5777. /**********************************************************************************************

  5778. * Macros for N=2 and M=1

  5779. **********************************************************************************************/

  5780. 

  5781. #if defined(_AIX)

  5782. define(`LOAD2x1_1', `

  5783. #else

  5784. .macro LOAD2x1_1

  5785. #endif

  5786. 

  5787. 	lxsspx		vs0,	o0,	AO

  5788. 

  5789. 	addi		AO,	AO,	4

  5790. 

  5791. 	mr		T1,	BO

  5792. 

  5793. 	lxsspx		vs8,	o0,	T1

  5794. 	lxsspx		vs9,	o16,	T1

  5795. 

  5796. 

  5797. 	addi		BO,	BO,	32

  5798. 

  5799. #if defined(_AIX)

  5800. ')

  5801. #else

  5802. .endm

  5803. #endif

  5804. 

  5805. #if defined(_AIX)

  5806. define(`KERNEL2x1_I1', `

  5807. #else

  5808. .macro KERNEL2x1_I1

  5809. #endif

  5810. 

  5811. 

  5812. 	lxsspx		vs4,	o0,	AO

  5813. 

  5814. 	addi		AO,	AO,	4

  5815. 

  5816. 	mr		T1,	BO

  5817. 

  5818. 	lxsspx		vs16,	o0,	T1

  5819. 	lxsspx		vs17,	o16,	T1

  5820. 

  5821. 

  5822. 	addi		BO,	BO,	32

  5823. 

  5824. 

  5825. 	xsmuldp		vs32,	vs0,	vs8

  5826. 

  5827. 	xsmuldp		vs33,	vs0,	vs9

  5828. 

  5829. 

  5830. #if defined(_AIX)

  5831. ')

  5832. #else

  5833. .endm

  5834. #endif

  5835. 

  5836. #if defined(_AIX)

  5837. define(`KERNEL2x1_1', `

  5838. #else

  5839. .macro KERNEL2x1_1

  5840. #endif

  5841. 

  5842. 

  5843. 	lxsspx		vs4,	o0,	AO

  5844. 

  5845. 	addi		AO,	AO,	4

  5846. 

  5847. 	mr		T1,	BO

  5848. 

  5849. 	lxsspx		vs16,	o0,	T1

  5850. 	lxsspx		vs17,	o16,	T1

  5851. 

  5852. 

  5853. 	addi		BO,	BO,	32

  5854. 

  5855. 

  5856. 	xsmaddadp	vs32,	vs0,	vs8

  5857. 

  5858. 	xsmaddadp	vs33,	vs0,	vs9

  5859. 

  5860. 

  5861. #if defined(_AIX)

  5862. ')

  5863. #else

  5864. .endm

  5865. #endif

  5866. 

  5867. #if defined(_AIX)

  5868. define(`KERNEL2x1_2', `

  5869. #else

  5870. .macro KERNEL2x1_2

  5871. #endif

  5872. 

  5873. 

  5874. 	lxsspx		vs0,	o0,	AO

  5875. 

  5876. 	addi		AO,	AO,	4

  5877. 

  5878. 	mr		T1,	BO

  5879. 

  5880. 	lxsspx		vs8,	o0,	T1

  5881. 	lxsspx		vs9,	o16,	T1

  5882. 

  5883. 

  5884. 	addi		BO,	BO,	32

  5885. 

  5886. 

  5887. 	xsmaddadp	vs32,	vs4,	vs16

  5888. 

  5889. 	xsmaddadp	vs33,	vs4,	vs17

  5890. 

  5891. 

  5892. #if defined(_AIX)

  5893. ')

  5894. #else

  5895. .endm

  5896. #endif

  5897. 

  5898. #if defined(_AIX)

  5899. define(`KERNEL2x1_E2', `

  5900. #else

  5901. .macro KERNEL2x1_E2

  5902. #endif

  5903. 

  5904. 

  5905. 	xsmaddadp	vs32,	vs4,	vs16

  5906. 

  5907. 	xsmaddadp	vs33,	vs4,	vs17

  5908. 

  5909. 

  5910. #if defined(_AIX)

  5911. ')

  5912. #else

  5913. .endm

  5914. #endif

  5915. 

  5916. #if defined(_AIX)

  5917. define(`KERNEL2x1_SUBI1', `

  5918. #else

  5919. .macro KERNEL2x1_SUBI1

  5920. #endif

  5921. 

  5922. 

  5923. 	lxsspx		vs0,	o0,	AO

  5924. 

  5925. 	addi		AO,	AO,	4

  5926. 

  5927. 	mr		T1,	BO

  5928. 

  5929. 	lxsspx		vs8,	o0,	T1

  5930. 	lxsspx		vs9,	o16,	T1

  5931. 

  5932. 

  5933. 	addi		BO,	BO,	32

  5934. 

  5935. 

  5936. 	xsmuldp		vs32,	vs0,	vs8

  5937. 

  5938. 	xsmuldp		vs33,	vs0,	vs9

  5939. 

  5940. 

  5941. #if defined(_AIX)

  5942. ')

  5943. #else

  5944. .endm

  5945. #endif

  5946. 

  5947. #if defined(_AIX)

  5948. define(`KERNEL2x1_SUB1', `

  5949. #else

  5950. .macro KERNEL2x1_SUB1

  5951. #endif

  5952. 

  5953. 

  5954. 	lxsspx		vs0,	o0,	AO

  5955. 

  5956. 	addi		AO,	AO,	4

  5957. 

  5958. 	mr		T1,	BO

  5959. 

  5960. 	lxsspx		vs8,	o0,	T1

  5961. 	lxsspx		vs9,	o16,	T1

  5962. 

  5963. 

  5964. 	addi		BO,	BO,	32

  5965. 

  5966. 

  5967. 	xsmaddadp	vs32,	vs0,	vs8

  5968. 

  5969. 	xsmaddadp	vs33,	vs0,	vs9

  5970. 

  5971. 

  5972. #if defined(_AIX)

  5973. ')

  5974. #else

  5975. .endm

  5976. #endif

  5977. 

  5978. #if defined(_AIX)

  5979. define(`SAVE2x1', `

  5980. #else

  5981. .macro SAVE2x1

  5982. #endif

  5983. 

  5984. 	mr		T1,	CO

  5985. 

  5986. #ifndef TRMMKERNEL

  5987. 

  5988. 	lxsspx		vs0,	o0,	T1

  5989. 

  5990. #endif

  5991. 

  5992. #ifdef TRMMKERNEL

  5993. 	xsmuldp		vs0,	vs32,	alpha_r

  5994. #else

  5995. 	xsmaddadp	vs0,	vs32,	alpha_r

  5996. #endif

  5997. 

  5998. 	stxsspx		vs0,	o0,	T1

  5999. 

  6000. 	add		T1,	T1,	LDC

  6001. 

  6002. 

  6003. #ifndef TRMMKERNEL

  6004. 

  6005. 	lxsspx		vs0,	o0,	T1

  6006. 

  6007. #endif

  6008. 

  6009. #ifdef TRMMKERNEL

  6010. 	xsmuldp		vs0,	vs33,	alpha_r

  6011. #else

  6012. 	xsmaddadp	vs0,	vs33,	alpha_r

  6013. #endif

  6014. 

  6015. 	stxsspx		vs0,	o0,	T1

  6016. 

  6017. 	add		T1,	T1,	LDC

  6018. 

  6019. 	addi		CO,	CO,	4

  6020. 

  6021. #if defined(_AIX)

  6022. ')

  6023. #else

  6024. .endm

  6025. #endif

  6026. 

  6027. 

  6028. /**********************************************************************************************

  6029. * Macros for N=1 and M=16

  6030. **********************************************************************************************/

  6031. 

  6032. #if defined(_AIX)

  6033. define(`LOAD1x16_1', `

  6034. #else

  6035. .macro LOAD1x16_1

  6036. #endif

  6037. 

  6038. 	lxvw4x		vs0,	o0,	AO

  6039. 	lxvw4x		vs1,	o16,	AO

  6040. 	lxvw4x		vs2,	o32,	AO

  6041. 	lxvw4x		vs3,	o48,	AO

  6042. 

  6043. 	addi		AO,	AO,	64

  6044. 

  6045. 	mr		T1,	BO

  6046. 

  6047. 	lxvw4x		vs8,	o0,	T1

  6048. 

  6049. 	addi		BO,	BO,	16

  6050. 

  6051. #if defined(_AIX)

  6052. ')

  6053. #else

  6054. .endm

  6055. #endif

  6056. 

  6057. #if defined(_AIX)

  6058. define(`KERNEL1x16_I1', `

  6059. #else

  6060. .macro KERNEL1x16_I1

  6061. #endif

  6062. 

  6063. 

  6064. 	lxvw4x		vs4,	o0,	AO

  6065. 	lxvw4x		vs5,	o16,	AO

  6066. 	lxvw4x		vs6,	o32,	AO

  6067. 	lxvw4x		vs7,	o48,	AO

  6068. 

  6069. 	addi		AO,	AO,	64

  6070. 

  6071. 	mr		T1,	BO

  6072. 

  6073. 	lxvw4x		vs16,	o0,	T1

  6074. 

  6075. 	addi		BO,	BO,	16

  6076. 

  6077. 

  6078. 	xvmulsp		vs32,	vs0,	vs8

  6079. 	xvmulsp		vs33,	vs1,	vs8

  6080. 	xvmulsp		vs34,	vs2,	vs8

  6081. 	xvmulsp		vs35,	vs3,	vs8

  6082. 

  6083. 

  6084. #if defined(_AIX)

  6085. ')

  6086. #else

  6087. .endm

  6088. #endif

  6089. 

  6090. #if defined(_AIX)

  6091. define(`KERNEL1x16_1', `

  6092. #else

  6093. .macro KERNEL1x16_1

  6094. #endif

  6095. 

  6096. 

  6097. 	lxvw4x		vs4,	o0,	AO

  6098. 	lxvw4x		vs5,	o16,	AO

  6099. 	lxvw4x		vs6,	o32,	AO

  6100. 	lxvw4x		vs7,	o48,	AO

  6101. 

  6102. 	addi		AO,	AO,	64

  6103. 

  6104. 	mr		T1,	BO

  6105. 

  6106. 	lxvw4x		vs16,	o0,	T1

  6107. 

  6108. 	addi		BO,	BO,	16

  6109. 

  6110. 

  6111. 	xvmaddasp	vs32,	vs0,	vs8

  6112. 	xvmaddasp	vs33,	vs1,	vs8

  6113. 	xvmaddasp	vs34,	vs2,	vs8

  6114. 	xvmaddasp	vs35,	vs3,	vs8

  6115. 

  6116. 

  6117. #if defined(_AIX)

  6118. ')

  6119. #else

  6120. .endm

  6121. #endif

  6122. 

  6123. #if defined(_AIX)

  6124. define(`KERNEL1x16_2', `

  6125. #else

  6126. .macro KERNEL1x16_2

  6127. #endif

  6128. 

  6129. 

  6130. 	lxvw4x		vs0,	o0,	AO

  6131. 	lxvw4x		vs1,	o16,	AO

  6132. 	lxvw4x		vs2,	o32,	AO

  6133. 	lxvw4x		vs3,	o48,	AO

  6134. 

  6135. 	addi		AO,	AO,	64

  6136. 

  6137. 	mr		T1,	BO

  6138. 

  6139. 	lxvw4x		vs8,	o0,	T1

  6140. 

  6141. 	addi		BO,	BO,	16

  6142. 

  6143. 

  6144. 	xvmaddasp	vs32,	vs4,	vs16

  6145. 	xvmaddasp	vs33,	vs5,	vs16

  6146. 	xvmaddasp	vs34,	vs6,	vs16

  6147. 	xvmaddasp	vs35,	vs7,	vs16

  6148. 

  6149. 

  6150. #if defined(_AIX)

  6151. ')

  6152. #else

  6153. .endm

  6154. #endif

  6155. 

  6156. #if defined(_AIX)

  6157. define(`KERNEL1x16_E2', `

  6158. #else

  6159. .macro KERNEL1x16_E2

  6160. #endif

  6161. 

  6162. 

  6163. 	xvmaddasp	vs32,	vs4,	vs16

  6164. 	xvmaddasp	vs33,	vs5,	vs16

  6165. 	xvmaddasp	vs34,	vs6,	vs16

  6166. 	xvmaddasp	vs35,	vs7,	vs16

  6167. 

  6168. 

  6169. #if defined(_AIX)

  6170. ')

  6171. #else

  6172. .endm

  6173. #endif

  6174. 

  6175. #if defined(_AIX)

  6176. define(`KERNEL1x16_SUBI1', `

  6177. #else

  6178. .macro KERNEL1x16_SUBI1

  6179. #endif

  6180. 

  6181. 

  6182. 	lxvw4x		vs0,	o0,	AO

  6183. 	lxvw4x		vs1,	o16,	AO

  6184. 	lxvw4x		vs2,	o32,	AO

  6185. 	lxvw4x		vs3,	o48,	AO

  6186. 

  6187. 	addi		AO,	AO,	64

  6188. 

  6189. 	mr		T1,	BO

  6190. 

  6191. 	lxvw4x		vs8,	o0,	T1

  6192. 

  6193. 	addi		BO,	BO,	16

  6194. 

  6195. 

  6196. 	xvmulsp		vs32,	vs0,	vs8

  6197. 	xvmulsp		vs33,	vs1,	vs8

  6198. 	xvmulsp		vs34,	vs2,	vs8

  6199. 	xvmulsp		vs35,	vs3,	vs8

  6200. 

  6201. 

  6202. #if defined(_AIX)

  6203. ')

  6204. #else

  6205. .endm

  6206. #endif

  6207. 

  6208. #if defined(_AIX)

  6209. define(`KERNEL1x16_SUB1', `

  6210. #else

  6211. .macro KERNEL1x16_SUB1

  6212. #endif

  6213. 

  6214. 

  6215. 	lxvw4x		vs0,	o0,	AO

  6216. 	lxvw4x		vs1,	o16,	AO

  6217. 	lxvw4x		vs2,	o32,	AO

  6218. 	lxvw4x		vs3,	o48,	AO

  6219. 

  6220. 	addi		AO,	AO,	64

  6221. 

  6222. 	mr		T1,	BO

  6223. 

  6224. 	lxvw4x		vs8,	o0,	T1

  6225. 

  6226. 	addi		BO,	BO,	16

  6227. 

  6228. 

  6229. 	xvmaddasp	vs32,	vs0,	vs8

  6230. 	xvmaddasp	vs33,	vs1,	vs8

  6231. 	xvmaddasp	vs34,	vs2,	vs8

  6232. 	xvmaddasp	vs35,	vs3,	vs8

  6233. 

  6234. 

  6235. #if defined(_AIX)

  6236. ')

  6237. #else

  6238. .endm

  6239. #endif

  6240. 

  6241. #if defined(_AIX)

  6242. define(`SAVE1x16', `

  6243. #else

  6244. .macro SAVE1x16

  6245. #endif

  6246. 

  6247. 	mr		T1,	CO

  6248. 

  6249. #ifndef TRMMKERNEL

  6250. 

  6251. 	lxvw4x		vs0,	o0,	T1

  6252. 	lxvw4x		vs1,	o16,	T1

  6253. 	lxvw4x		vs2,	o32,	T1

  6254. 	lxvw4x		vs3,	o48,	T1

  6255. 

  6256. #endif

  6257. 

  6258. #ifdef TRMMKERNEL

  6259. 	xvmulsp		vs0,	vs32,	alpha_vr

  6260. 	xvmulsp		vs1,	vs33,	alpha_vr

  6261. 	xvmulsp		vs2,	vs34,	alpha_vr

  6262. 	xvmulsp		vs3,	vs35,	alpha_vr

  6263. #else

  6264. 	xvmaddasp	vs0,	vs32,	alpha_vr

  6265. 	xvmaddasp	vs1,	vs33,	alpha_vr

  6266. 	xvmaddasp	vs2,	vs34,	alpha_vr

  6267. 	xvmaddasp	vs3,	vs35,	alpha_vr

  6268. #endif

  6269. 

  6270. 	stxvw4x		vs0,	o0,	T1

  6271. 	stxvw4x		vs1,	o16,	T1

  6272. 	stxvw4x		vs2,	o32,	T1

  6273. 	stxvw4x		vs3,	o48,	T1

  6274. 

  6275. 	add		T1,	T1,	LDC

  6276. 

  6277. 	addi		CO,	CO,	64

  6278. 

  6279. #if defined(_AIX)

  6280. ')

  6281. #else

  6282. .endm

  6283. #endif

  6284. 

  6285. 

  6286. /**********************************************************************************************

  6287. * Macros for N=1 and M=8

  6288. **********************************************************************************************/

  6289. 

  6290. #if defined(_AIX)

  6291. define(`LOAD1x8_1', `

  6292. #else

  6293. .macro LOAD1x8_1

  6294. #endif

  6295. 

  6296. 	lxvw4x		vs0,	o0,	AO

  6297. 	lxvw4x		vs1,	o16,	AO

  6298. 

  6299. 	addi		AO,	AO,	32

  6300. 

  6301. 	mr		T1,	BO

  6302. 

  6303. 	lxvw4x		vs8,	o0,	T1

  6304. 

  6305. 	addi		BO,	BO,	16

  6306. 

  6307. #if defined(_AIX)

  6308. ')

  6309. #else

  6310. .endm

  6311. #endif

  6312. 

  6313. #if defined(_AIX)

  6314. define(`KERNEL1x8_I1', `

  6315. #else

  6316. .macro KERNEL1x8_I1

  6317. #endif

  6318. 

  6319. 

  6320. 	lxvw4x		vs4,	o0,	AO

  6321. 	lxvw4x		vs5,	o16,	AO

  6322. 

  6323. 	addi		AO,	AO,	32

  6324. 

  6325. 	mr		T1,	BO

  6326. 

  6327. 	lxvw4x		vs16,	o0,	T1

  6328. 

  6329. 	addi		BO,	BO,	16

  6330. 

  6331. 

  6332. 	xvmulsp		vs32,	vs0,	vs8

  6333. 	xvmulsp		vs33,	vs1,	vs8

  6334. 

  6335. 

  6336. #if defined(_AIX)

  6337. ')

  6338. #else

  6339. .endm

  6340. #endif

  6341. 

  6342. #if defined(_AIX)

  6343. define(`KERNEL1x8_1', `

  6344. #else

  6345. .macro KERNEL1x8_1

  6346. #endif

  6347. 

  6348. 

  6349. 	lxvw4x		vs4,	o0,	AO

  6350. 	lxvw4x		vs5,	o16,	AO

  6351. 

  6352. 	addi		AO,	AO,	32

  6353. 

  6354. 	mr		T1,	BO

  6355. 

  6356. 	lxvw4x		vs16,	o0,	T1

  6357. 

  6358. 	addi		BO,	BO,	16

  6359. 

  6360. 

  6361. 	xvmaddasp	vs32,	vs0,	vs8

  6362. 	xvmaddasp	vs33,	vs1,	vs8

  6363. 

  6364. 

  6365. #if defined(_AIX)

  6366. ')

  6367. #else

  6368. .endm

  6369. #endif

  6370. 

  6371. #if defined(_AIX)

  6372. define(`KERNEL1x8_2', `

  6373. #else

  6374. .macro KERNEL1x8_2

  6375. #endif

  6376. 

  6377. 

  6378. 	lxvw4x		vs0,	o0,	AO

  6379. 	lxvw4x		vs1,	o16,	AO

  6380. 

  6381. 	addi		AO,	AO,	32

  6382. 

  6383. 	mr		T1,	BO

  6384. 

  6385. 	lxvw4x		vs8,	o0,	T1

  6386. 

  6387. 	addi		BO,	BO,	16

  6388. 

  6389. 

  6390. 	xvmaddasp	vs32,	vs4,	vs16

  6391. 	xvmaddasp	vs33,	vs5,	vs16

  6392. 

  6393. 

  6394. #if defined(_AIX)

  6395. ')

  6396. #else

  6397. .endm

  6398. #endif

  6399. 

  6400. #if defined(_AIX)

  6401. define(`KERNEL1x8_E2', `

  6402. #else

  6403. .macro KERNEL1x8_E2

  6404. #endif

  6405. 

  6406. 

  6407. 	xvmaddasp	vs32,	vs4,	vs16

  6408. 	xvmaddasp	vs33,	vs5,	vs16

  6409. 

  6410. 

  6411. #if defined(_AIX)

  6412. ')

  6413. #else

  6414. .endm

  6415. #endif

  6416. 

  6417. #if defined(_AIX)

  6418. define(`KERNEL1x8_SUBI1', `

  6419. #else

  6420. .macro KERNEL1x8_SUBI1

  6421. #endif

  6422. 

  6423. 

  6424. 	lxvw4x		vs0,	o0,	AO

  6425. 	lxvw4x		vs1,	o16,	AO

  6426. 

  6427. 	addi		AO,	AO,	32

  6428. 

  6429. 	mr		T1,	BO

  6430. 

  6431. 	lxvw4x		vs8,	o0,	T1

  6432. 

  6433. 	addi		BO,	BO,	16

  6434. 

  6435. 

  6436. 	xvmulsp		vs32,	vs0,	vs8

  6437. 	xvmulsp		vs33,	vs1,	vs8

  6438. 

  6439. 

  6440. #if defined(_AIX)

  6441. ')

  6442. #else

  6443. .endm

  6444. #endif

  6445. 

  6446. #if defined(_AIX)

  6447. define(`KERNEL1x8_SUB1', `

  6448. #else

  6449. .macro KERNEL1x8_SUB1

  6450. #endif

  6451. 

  6452. 

  6453. 	lxvw4x		vs0,	o0,	AO

  6454. 	lxvw4x		vs1,	o16,	AO

  6455. 

  6456. 	addi		AO,	AO,	32

  6457. 

  6458. 	mr		T1,	BO

  6459. 

  6460. 	lxvw4x		vs8,	o0,	T1

  6461. 

  6462. 	addi		BO,	BO,	16

  6463. 

  6464. 

  6465. 	xvmaddasp	vs32,	vs0,	vs8

  6466. 	xvmaddasp	vs33,	vs1,	vs8

  6467. 

  6468. 

  6469. #if defined(_AIX)

  6470. ')

  6471. #else

  6472. .endm

  6473. #endif

  6474. 

  6475. #if defined(_AIX)

  6476. define(`SAVE1x8', `

  6477. #else

  6478. .macro SAVE1x8

  6479. #endif

  6480. 

  6481. 	mr		T1,	CO

  6482. 

  6483. #ifndef TRMMKERNEL

  6484. 

  6485. 	lxvw4x		vs0,	o0,	T1

  6486. 	lxvw4x		vs1,	o16,	T1

  6487. 

  6488. #endif

  6489. 

  6490. #ifdef TRMMKERNEL

  6491. 	xvmulsp		vs0,	vs32,	alpha_vr

  6492. 	xvmulsp		vs1,	vs33,	alpha_vr

  6493. #else

  6494. 	xvmaddasp	vs0,	vs32,	alpha_vr

  6495. 	xvmaddasp	vs1,	vs33,	alpha_vr

  6496. #endif

  6497. 

  6498. 	stxvw4x		vs0,	o0,	T1

  6499. 	stxvw4x		vs1,	o16,	T1

  6500. 

  6501. 	add		T1,	T1,	LDC

  6502. 

  6503. 	addi		CO,	CO,	32

  6504. 

  6505. #if defined(_AIX)

  6506. ')

  6507. #else

  6508. .endm

  6509. #endif

  6510. 

  6511. 

  6512. /**********************************************************************************************

  6513. * Macros for N=1 and M=4

  6514. **********************************************************************************************/

  6515. 

  6516. #if defined(_AIX)

  6517. define(`LOAD1x4_1', `

  6518. #else

  6519. .macro LOAD1x4_1

  6520. #endif

  6521. 

  6522. 	lxvw4x		vs0,	o0,	AO

  6523. 

  6524. 	addi		AO,	AO,	16

  6525. 

  6526. 	mr		T1,	BO

  6527. 

  6528. 	lxvw4x		vs8,	o0,	T1

  6529. 

  6530. 	addi		BO,	BO,	16

  6531. 

  6532. #if defined(_AIX)

  6533. ')

  6534. #else

  6535. .endm

  6536. #endif

  6537. 

  6538. #if defined(_AIX)

  6539. define(`KERNEL1x4_I1', `

  6540. #else

  6541. .macro KERNEL1x4_I1

  6542. #endif

  6543. 

  6544. 

  6545. 	lxvw4x		vs4,	o0,	AO

  6546. 

  6547. 	addi		AO,	AO,	16

  6548. 

  6549. 	mr		T1,	BO

  6550. 

  6551. 	lxvw4x		vs16,	o0,	T1

  6552. 

  6553. 	addi		BO,	BO,	16

  6554. 

  6555. 

  6556. 	xvmulsp		vs32,	vs0,	vs8

  6557. 

  6558. 

  6559. #if defined(_AIX)

  6560. ')

  6561. #else

  6562. .endm

  6563. #endif

  6564. 

  6565. #if defined(_AIX)

  6566. define(`KERNEL1x4_1', `

  6567. #else

  6568. .macro KERNEL1x4_1

  6569. #endif

  6570. 

  6571. 

  6572. 	lxvw4x		vs4,	o0,	AO

  6573. 

  6574. 	addi		AO,	AO,	16

  6575. 

  6576. 	mr		T1,	BO

  6577. 

  6578. 	lxvw4x		vs16,	o0,	T1

  6579. 

  6580. 	addi		BO,	BO,	16

  6581. 

  6582. 

  6583. 	xvmaddasp	vs32,	vs0,	vs8

  6584. 

  6585. 

  6586. #if defined(_AIX)

  6587. ')

  6588. #else

  6589. .endm

  6590. #endif

  6591. 

  6592. #if defined(_AIX)

  6593. define(`KERNEL1x4_2', `

  6594. #else

  6595. .macro KERNEL1x4_2

  6596. #endif

  6597. 

  6598. 

  6599. 	lxvw4x		vs0,	o0,	AO

  6600. 

  6601. 	addi		AO,	AO,	16

  6602. 

  6603. 	mr		T1,	BO

  6604. 

  6605. 	lxvw4x		vs8,	o0,	T1

  6606. 

  6607. 	addi		BO,	BO,	16

  6608. 

  6609. 

  6610. 	xvmaddasp	vs32,	vs4,	vs16

  6611. 

  6612. 

  6613. #if defined(_AIX)

  6614. ')

  6615. #else

  6616. .endm

  6617. #endif

  6618. 

  6619. #if defined(_AIX)

  6620. define(`KERNEL1x4_E2', `

  6621. #else

  6622. .macro KERNEL1x4_E2

  6623. #endif

  6624. 

  6625. 

  6626. 	xvmaddasp	vs32,	vs4,	vs16

  6627. 

  6628. 

  6629. #if defined(_AIX)

  6630. ')

  6631. #else

  6632. .endm

  6633. #endif

  6634. 

  6635. #if defined(_AIX)

  6636. define(`KERNEL1x4_SUBI1', `

  6637. #else

  6638. .macro KERNEL1x4_SUBI1

  6639. #endif

  6640. 

  6641. 

  6642. 	lxvw4x		vs0,	o0,	AO

  6643. 

  6644. 	addi		AO,	AO,	16

  6645. 

  6646. 	mr		T1,	BO

  6647. 

  6648. 	lxvw4x		vs8,	o0,	T1

  6649. 

  6650. 	addi		BO,	BO,	16

  6651. 

  6652. 

  6653. 	xvmulsp		vs32,	vs0,	vs8

  6654. 

  6655. 

  6656. #if defined(_AIX)

  6657. ')

  6658. #else

  6659. .endm

  6660. #endif

  6661. 

  6662. #if defined(_AIX)

  6663. define(`KERNEL1x4_SUB1', `

  6664. #else

  6665. .macro KERNEL1x4_SUB1

  6666. #endif

  6667. 

  6668. 

  6669. 	lxvw4x		vs0,	o0,	AO

  6670. 

  6671. 	addi		AO,	AO,	16

  6672. 

  6673. 	mr		T1,	BO

  6674. 

  6675. 	lxvw4x		vs8,	o0,	T1

  6676. 

  6677. 	addi		BO,	BO,	16

  6678. 

  6679. 

  6680. 	xvmaddasp	vs32,	vs0,	vs8

  6681. 

  6682. 

  6683. #if defined(_AIX)

  6684. ')

  6685. #else

  6686. .endm

  6687. #endif

  6688. 

  6689. #if defined(_AIX)

  6690. define(`SAVE1x4', `

  6691. #else

  6692. .macro SAVE1x4

  6693. #endif

  6694. 

  6695. 	mr		T1,	CO

  6696. 

  6697. #ifndef TRMMKERNEL

  6698. 

  6699. 	lxvw4x		vs0,	o0,	T1

  6700. 

  6701. #endif

  6702. 

  6703. #ifdef TRMMKERNEL

  6704. 	xvmulsp		vs0,	vs32,	alpha_vr

  6705. #else

  6706. 	xvmaddasp	vs0,	vs32,	alpha_vr

  6707. #endif

  6708. 

  6709. 	stxvw4x		vs0,	o0,	T1

  6710. 

  6711. 	add		T1,	T1,	LDC

  6712. 

  6713. 	addi		CO,	CO,	16

  6714. 

  6715. #if defined(_AIX)

  6716. ')

  6717. #else

  6718. .endm

  6719. #endif

  6720. 

  6721. 

  6722. /**********************************************************************************************

  6723. * Macros for N=1 and M=2

  6724. **********************************************************************************************/

  6725. 

  6726. #if defined(_AIX)

  6727. define(`LOAD1x2_1', `

  6728. #else

  6729. .macro LOAD1x2_1

  6730. #endif

  6731. 

  6732. 	lxsspx		vs0,	o0,	AO

  6733. 	lxsspx		vs1,	o4,	AO

  6734. 

  6735. 	addi		AO,	AO,	8

  6736. 

  6737. 	mr		T1,	BO

  6738. 

  6739. 	lxsspx		vs8,	o0,	T1

  6740. 

  6741. 

  6742. 	addi		BO,	BO,	16

  6743. 

  6744. #if defined(_AIX)

  6745. ')

  6746. #else

  6747. .endm

  6748. #endif

  6749. 

  6750. #if defined(_AIX)

  6751. define(`KERNEL1x2_I1', `

  6752. #else

  6753. .macro KERNEL1x2_I1

  6754. #endif

  6755. 

  6756. 

  6757. 	lxsspx		vs4,	o0,	AO

  6758. 	lxsspx		vs5,	o4,	AO

  6759. 

  6760. 	addi		AO,	AO,	8

  6761. 

  6762. 	mr		T1,	BO

  6763. 

  6764. 	lxsspx		vs16,	o0,	T1

  6765. 

  6766. 

  6767. 	addi		BO,	BO,	16

  6768. 

  6769. 

  6770. 	xsmuldp		vs32,	vs0,	vs8

  6771. 	xsmuldp		vs33,	vs1,	vs8

  6772. 

  6773. 

  6774. #if defined(_AIX)

  6775. ')

  6776. #else

  6777. .endm

  6778. #endif

  6779. 

  6780. #if defined(_AIX)

  6781. define(`KERNEL1x2_1', `

  6782. #else

  6783. .macro KERNEL1x2_1

  6784. #endif

  6785. 

  6786. 

  6787. 	lxsspx		vs4,	o0,	AO

  6788. 	lxsspx		vs5,	o4,	AO

  6789. 

  6790. 	addi		AO,	AO,	8

  6791. 

  6792. 	mr		T1,	BO

  6793. 

  6794. 	lxsspx		vs16,	o0,	T1

  6795. 

  6796. 

  6797. 	addi		BO,	BO,	16

  6798. 

  6799. 

  6800. 	xsmaddadp	vs32,	vs0,	vs8

  6801. 	xsmaddadp	vs33,	vs1,	vs8

  6802. 

  6803. 

  6804. #if defined(_AIX)

  6805. ')

  6806. #else

  6807. .endm

  6808. #endif

  6809. 

  6810. #if defined(_AIX)

  6811. define(`KERNEL1x2_2', `

  6812. #else

  6813. .macro KERNEL1x2_2

  6814. #endif

  6815. 

  6816. 

  6817. 	lxsspx		vs0,	o0,	AO

  6818. 	lxsspx		vs1,	o4,	AO

  6819. 

  6820. 	addi		AO,	AO,	8

  6821. 

  6822. 	mr		T1,	BO

  6823. 

  6824. 	lxsspx		vs8,	o0,	T1

  6825. 

  6826. 

  6827. 	addi		BO,	BO,	16

  6828. 

  6829. 

  6830. 	xsmaddadp	vs32,	vs4,	vs16

  6831. 	xsmaddadp	vs33,	vs5,	vs16

  6832. 

  6833. 

  6834. #if defined(_AIX)

  6835. ')

  6836. #else

  6837. .endm

  6838. #endif

  6839. 

  6840. #if defined(_AIX)

  6841. define(`KERNEL1x2_E2', `

  6842. #else

  6843. .macro KERNEL1x2_E2

  6844. #endif

  6845. 

  6846. 

  6847. 	xsmaddadp	vs32,	vs4,	vs16

  6848. 	xsmaddadp	vs33,	vs5,	vs16

  6849. 

  6850. 

  6851. #if defined(_AIX)

  6852. ')

  6853. #else

  6854. .endm

  6855. #endif

  6856. 

  6857. #if defined(_AIX)

  6858. define(`KERNEL1x2_SUBI1', `

  6859. #else

  6860. .macro KERNEL1x2_SUBI1

  6861. #endif

  6862. 

  6863. 

  6864. 	lxsspx		vs0,	o0,	AO

  6865. 	lxsspx		vs1,	o4,	AO

  6866. 

  6867. 	addi		AO,	AO,	8

  6868. 

  6869. 	mr		T1,	BO

  6870. 

  6871. 	lxsspx		vs8,	o0,	T1

  6872. 

  6873. 

  6874. 	addi		BO,	BO,	16

  6875. 

  6876. 

  6877. 	xsmuldp		vs32,	vs0,	vs8

  6878. 	xsmuldp		vs33,	vs1,	vs8

  6879. 

  6880. 

  6881. #if defined(_AIX)

  6882. ')

  6883. #else

  6884. .endm

  6885. #endif

  6886. 

  6887. #if defined(_AIX)

  6888. define(`KERNEL1x2_SUB1', `

  6889. #else

  6890. .macro KERNEL1x2_SUB1

  6891. #endif

  6892. 

  6893. 

  6894. 	lxsspx		vs0,	o0,	AO

  6895. 	lxsspx		vs1,	o4,	AO

  6896. 

  6897. 	addi		AO,	AO,	8

  6898. 

  6899. 	mr		T1,	BO

  6900. 

  6901. 	lxsspx		vs8,	o0,	T1

  6902. 

  6903. 

  6904. 	addi		BO,	BO,	16

  6905. 

  6906. 

  6907. 	xsmaddadp	vs32,	vs0,	vs8

  6908. 	xsmaddadp	vs33,	vs1,	vs8

  6909. 

  6910. 

  6911. #if defined(_AIX)

  6912. ')

  6913. #else

  6914. .endm

  6915. #endif

  6916. 

  6917. #if defined(_AIX)

  6918. define(`SAVE1x2', `

  6919. #else

  6920. .macro SAVE1x2

  6921. #endif

  6922. 

  6923. 	mr		T1,	CO

  6924. 

  6925. #ifndef TRMMKERNEL

  6926. 

  6927. 	lxsspx		vs0,	o0,	T1

  6928. 	lxsspx		vs1,	o4,	T1

  6929. 

  6930. #endif

  6931. 

  6932. #ifdef TRMMKERNEL

  6933. 	xsmuldp		vs0,	vs32,	alpha_r

  6934. 	xsmuldp		vs1,	vs33,	alpha_r

  6935. #else

  6936. 	xsmaddadp	vs0,	vs32,	alpha_r

  6937. 	xsmaddadp	vs1,	vs33,	alpha_r

  6938. #endif

  6939. 

  6940. 	stxsspx		vs0,	o0,	T1

  6941. 	stxsspx		vs1,	o4,	T1

  6942. 

  6943. 	add		T1,	T1,	LDC

  6944. 

  6945. 	addi		CO,	CO,	8

  6946. 

  6947. #if defined(_AIX)

  6948. ')

  6949. #else

  6950. .endm

  6951. #endif

  6952. 

  6953. 

  6954. /**********************************************************************************************

  6955. * Macros for N=1 and M=1

  6956. **********************************************************************************************/

  6957. 

  6958. #if defined(_AIX)

  6959. define(`LOAD1x1_1', `

  6960. #else

  6961. .macro LOAD1x1_1

  6962. #endif

  6963. 

  6964. 	lxsspx		vs0,	o0,	AO

  6965. 

  6966. 	addi		AO,	AO,	4

  6967. 

  6968. 	mr		T1,	BO

  6969. 

  6970. 	lxsspx		vs8,	o0,	T1

  6971. 

  6972. 

  6973. 	addi		BO,	BO,	16

  6974. 

  6975. #if defined(_AIX)

  6976. ')

  6977. #else

  6978. .endm

  6979. #endif

  6980. 

  6981. #if defined(_AIX)

  6982. define(`KERNEL1x1_I1', `

  6983. #else

  6984. .macro KERNEL1x1_I1

  6985. #endif

  6986. 

  6987. 

  6988. 	lxsspx		vs4,	o0,	AO

  6989. 

  6990. 	addi		AO,	AO,	4

  6991. 

  6992. 	mr		T1,	BO

  6993. 

  6994. 	lxsspx		vs16,	o0,	T1

  6995. 

  6996. 

  6997. 	addi		BO,	BO,	16

  6998. 

  6999. 

  7000. 	xsmuldp		vs32,	vs0,	vs8

  7001. 

  7002. 

  7003. #if defined(_AIX)

  7004. ')

  7005. #else

  7006. .endm

  7007. #endif

  7008. 

  7009. #if defined(_AIX)

  7010. define(`KERNEL1x1_1', `

  7011. #else

  7012. .macro KERNEL1x1_1

  7013. #endif

  7014. 

  7015. 

  7016. 	lxsspx		vs4,	o0,	AO

  7017. 

  7018. 	addi		AO,	AO,	4

  7019. 

  7020. 	mr		T1,	BO

  7021. 

  7022. 	lxsspx		vs16,	o0,	T1

  7023. 

  7024. 

  7025. 	addi		BO,	BO,	16

  7026. 

  7027. 

  7028. 	xsmaddadp	vs32,	vs0,	vs8

  7029. 

  7030. 

  7031. #if defined(_AIX)

  7032. ')

  7033. #else

  7034. .endm

  7035. #endif

  7036. 

  7037. #if defined(_AIX)

  7038. define(`KERNEL1x1_2', `

  7039. #else

  7040. .macro KERNEL1x1_2

  7041. #endif

  7042. 

  7043. 

  7044. 	lxsspx		vs0,	o0,	AO

  7045. 

  7046. 	addi		AO,	AO,	4

  7047. 

  7048. 	mr		T1,	BO

  7049. 

  7050. 	lxsspx		vs8,	o0,	T1

  7051. 

  7052. 

  7053. 	addi		BO,	BO,	16

  7054. 

  7055. 

  7056. 	xsmaddadp	vs32,	vs4,	vs16

  7057. 

  7058. 

  7059. #if defined(_AIX)

  7060. ')

  7061. #else

  7062. .endm

  7063. #endif

  7064. 

  7065. #if defined(_AIX)

  7066. define(`KERNEL1x1_E2', `

  7067. #else

  7068. .macro KERNEL1x1_E2

  7069. #endif

  7070. 

  7071. 

  7072. 	xsmaddadp	vs32,	vs4,	vs16

  7073. 

  7074. 

  7075. #if defined(_AIX)

  7076. ')

  7077. #else

  7078. .endm

  7079. #endif

  7080. 

  7081. #if defined(_AIX)

  7082. define(`KERNEL1x1_SUBI1', `

  7083. #else

  7084. .macro KERNEL1x1_SUBI1

  7085. #endif

  7086. 

  7087. 

  7088. 	lxsspx		vs0,	o0,	AO

  7089. 

  7090. 	addi		AO,	AO,	4

  7091. 

  7092. 	mr		T1,	BO

  7093. 

  7094. 	lxsspx		vs8,	o0,	T1

  7095. 

  7096. 

  7097. 	addi		BO,	BO,	16

  7098. 

  7099. 

  7100. 	xsmuldp		vs32,	vs0,	vs8

  7101. 

  7102. 

  7103. #if defined(_AIX)

  7104. ')

  7105. #else

  7106. .endm

  7107. #endif

  7108. 

  7109. #if defined(_AIX)

  7110. define(`KERNEL1x1_SUB1', `

  7111. #else

  7112. .macro KERNEL1x1_SUB1

  7113. #endif

  7114. 

  7115. 

  7116. 	lxsspx		vs0,	o0,	AO

  7117. 

  7118. 	addi		AO,	AO,	4

  7119. 

  7120. 	mr		T1,	BO

  7121. 

  7122. 	lxsspx		vs8,	o0,	T1

  7123. 

  7124. 

  7125. 	addi		BO,	BO,	16

  7126. 

  7127. 

  7128. 	xsmaddadp	vs32,	vs0,	vs8

  7129. 

  7130. 

  7131. #if defined(_AIX)

  7132. ')

  7133. #else

  7134. .endm

  7135. #endif

  7136. 

  7137. #if defined(_AIX)

  7138. define(`SAVE1x1', `

  7139. #else

  7140. .macro SAVE1x1

  7141. #endif

  7142. 

  7143. 	mr		T1,	CO

  7144. 

  7145. #ifndef TRMMKERNEL

  7146. 

  7147. 	lxsspx		vs0,	o0,	T1

  7148. 

  7149. #endif

  7150. 

  7151. #ifdef TRMMKERNEL

  7152. 	xsmuldp		vs0,	vs32,	alpha_r

  7153. #else

  7154. 	xsmaddadp	vs0,	vs32,	alpha_r

  7155. #endif

  7156. 

  7157. 	stxsspx		vs0,	o0,	T1

  7158. 

  7159. 	add		T1,	T1,	LDC

  7160. 

  7161. 	addi		CO,	CO,	4

  7162. 

  7163. #if defined(_AIX)

  7164. ')

  7165. #else

  7166. .endm

  7167. #endif

  7168. 

  7169. 

  7170. 

  7171. 

  7172. 

  7173. #if defined(_AIX)

  7174. define(`COPYB_4x8', `

  7175. #else

  7176. .macro COPYB_4x8

  7177. #endif

  7178. 

  7179. 

  7180. 	lxvw4x		vs5,	o0,	BO

  7181. 	xxspltw		vs6,	vs5,	0

  7182. 	xxspltw		vs7,	vs5,	1

  7183. 	xxspltw		vs8,	vs5,	2

  7184. 	xxspltw		vs9,	vs5,	3

  7185. 

  7186. 	lxvw4x		vs10,	o16,	BO

  7187. 	xxspltw		vs11,	vs10,	0

  7188. 	xxspltw		vs12,	vs10,	1

  7189. 	xxspltw		vs13,	vs10,	2

  7190. 	xxspltw		vs14,	vs10,	3

  7191. 

  7192. 	lxvw4x		vs15,	o32,	BO

  7193. 	xxspltw		vs16,	vs15,	0

  7194. 	xxspltw		vs17,	vs15,	1

  7195. 	xxspltw		vs18,	vs15,	2

  7196. 	xxspltw		vs19,	vs15,	3

  7197. 

  7198. 	lxvw4x		vs20,	o48,	BO

  7199. 	xxspltw		vs21,	vs20,	0

  7200. 	xxspltw		vs22,	vs20,	1

  7201. 	xxspltw		vs23,	vs20,	2

  7202. 	xxspltw		vs24,	vs20,	3

  7203. 

  7204. 	addi		BO,	BO,	64

  7205. 	lxvw4x		vs35,	o0,	BO

  7206. 	xxspltw		vs36,	vs35,	0

  7207. 	xxspltw		vs37,	vs35,	1

  7208. 	xxspltw		vs38,	vs35,	2

  7209. 	xxspltw		vs39,	vs35,	3

  7210. 

  7211. 	lxvw4x		vs40,	o16,	BO

  7212. 	xxspltw		vs41,	vs40,	0

  7213. 	xxspltw		vs42,	vs40,	1

  7214. 	xxspltw		vs43,	vs40,	2

  7215. 	xxspltw		vs44,	vs40,	3

  7216. 

  7217. 	lxvw4x		vs45,	o32,	BO

  7218. 	xxspltw		vs46,	vs45,	0

  7219. 	xxspltw		vs47,	vs45,	1

  7220. 	xxspltw		vs48,	vs45,	2

  7221. 	xxspltw		vs49,	vs45,	3

  7222. 

  7223. 	lxvw4x		vs50,	o48,	BO

  7224. 	xxspltw		vs51,	vs50,	0

  7225. 	xxspltw		vs52,	vs50,	1

  7226. 	xxspltw		vs53,	vs50,	2

  7227. 	xxspltw		vs54,	vs50,	3

  7228. 

  7229. 	addi		BO,	BO,	64

  7230. 

  7231. 

  7232. 	stxvw4x		vs6,	o0,	BBO

  7233. 	stxvw4x		vs7,	o16,	BBO

  7234. 	stxvw4x		vs8,	o32,	BBO

  7235. 	stxvw4x		vs9,	o48,	BBO

  7236. 

  7237. 	addi		BBO,	BBO,	64

  7238. 	stxvw4x		vs11,	o0,	BBO

  7239. 	stxvw4x		vs12,	o16,	BBO

  7240. 	stxvw4x		vs13,	o32,	BBO

  7241. 	stxvw4x		vs14,	o48,	BBO

  7242. 

  7243. 	addi		BBO,	BBO,	64

  7244. 	stxvw4x		vs16,	o0,	BBO

  7245. 	stxvw4x		vs17,	o16,	BBO

  7246. 	stxvw4x		vs18,	o32,	BBO

  7247. 	stxvw4x		vs19,	o48,	BBO

  7248. 

  7249. 	addi		BBO,	BBO,	64

  7250. 	stxvw4x		vs21,	o0,	BBO

  7251. 	stxvw4x		vs22,	o16,	BBO

  7252. 	stxvw4x		vs23,	o32,	BBO

  7253. 	stxvw4x		vs24,	o48,	BBO

  7254. 

  7255. 	addi		BBO,	BBO,	64

  7256. 	stxvw4x		vs36,	o0,	BBO

  7257. 	stxvw4x		vs37,	o16,	BBO

  7258. 	stxvw4x		vs38,	o32,	BBO

  7259. 	stxvw4x		vs39,	o48,	BBO

  7260. 

  7261. 	addi		BBO,	BBO,	64

  7262. 	stxvw4x		vs41,	o0,	BBO

  7263. 	stxvw4x		vs42,	o16,	BBO

  7264. 	stxvw4x		vs43,	o32,	BBO

  7265. 	stxvw4x		vs44,	o48,	BBO

  7266. 

  7267. 	addi		BBO,	BBO,	64

  7268. 	stxvw4x		vs46,	o0,	BBO

  7269. 	stxvw4x		vs47,	o16,	BBO

  7270. 	stxvw4x		vs48,	o32,	BBO

  7271. 	stxvw4x		vs49,	o48,	BBO

  7272. 

  7273. 	addi		BBO,	BBO,	64

  7274. 	stxvw4x		vs51,	o0,	BBO

  7275. 	stxvw4x		vs52,	o16,	BBO

  7276. 	stxvw4x		vs53,	o32,	BBO

  7277. 	stxvw4x		vs54,	o48,	BBO

  7278. 

  7279. 	addi		BBO,	BBO,	64

  7280. #if defined(_AIX)

  7281. ')

  7282. #else

  7283. .endm

  7284. #endif

  7285. 

  7286. 

  7287. #if defined(_AIX)

  7288. define(`COPYB_1x8', `

  7289. #else

  7290. .macro COPYB_1x8

  7291. #endif

  7292. 

  7293. 

  7294. 	lxvw4x		vs5,	o0,	BO

  7295. 	xxspltw		vs6,	vs5,	0

  7296. 	xxspltw		vs7,	vs5,	1

  7297. 	xxspltw		vs8,	vs5,	2

  7298. 	xxspltw		vs9,	vs5,	3

  7299. 

  7300. 	lxvw4x		vs10,	o16,	BO

  7301. 	xxspltw		vs11,	vs10,	0

  7302. 	xxspltw		vs12,	vs10,	1

  7303. 	xxspltw		vs13,	vs10,	2

  7304. 	xxspltw		vs14,	vs10,	3

  7305. 

  7306. 

  7307. 	addi		BO,	BO,	32

  7308. 

  7309. 	stxvw4x		vs6,	o0,	BBO

  7310. 	stxvw4x		vs7,	o16,	BBO

  7311. 	stxvw4x		vs8,	o32,	BBO

  7312. 	stxvw4x		vs9,	o48,	BBO

  7313. 

  7314. 	addi		BBO,	BBO,	64

  7315. 	stxvw4x		vs11,	o0,	BBO

  7316. 	stxvw4x		vs12,	o16,	BBO

  7317. 	stxvw4x		vs13,	o32,	BBO

  7318. 	stxvw4x		vs14,	o48,	BBO

  7319. 

  7320. 	addi		BBO,	BBO,	64

  7321. #if defined(_AIX)

  7322. ')

  7323. #else

  7324. .endm

  7325. #endif