[llvm] [AArch64][SVE] Add more MOVPRFX pseudos for SVE immediate instructions. (PR #192491)
Ricardo Jesus via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 17 09:49:58 PDT 2026
https://github.com/rj-jesus updated https://github.com/llvm/llvm-project/pull/192491
>From 21e846d6c0dec0b65f18e8362d0088f501adcb41 Mon Sep 17 00:00:00 2001
From: Ricardo Jesus <rjj at nvidia.com>
Date: Thu, 2 Apr 2026 05:06:38 -0700
Subject: [PATCH 1/3] [AArch64][SVE] Add more MOVPRFX pseudos for SVE immediate
instructions.
This patchs adds MOVPRFX pseudos for the following instructions:
* AND (immediate)
* ORR (immediate)
* EOR (immediate)
* MUL (immediate)
* SMAX (immediate)
* SMIN (immediate)
* UMAX (immediate)
* UMIN (immediate)
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 8 +
.../lib/Target/AArch64/AArch64InstrFormats.td | 2 +
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 1 +
.../Target/AArch64/AArch64RegisterInfo.cpp | 1 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 14 +-
.../Target/AArch64/AArch64SchedNeoverseV1.td | 2 +-
.../lib/Target/AArch64/AArch64SchedOlympus.td | 8 +-
llvm/lib/Target/AArch64/SVEInstrFormats.td | 65 +-
llvm/test/CodeGen/AArch64/aarch64-smull.ll | 16 +-
llvm/test/CodeGen/AArch64/clmul-scalable.ll | 3041 +++++++++--------
...interleaving-add-mull-scalable-contract.ll | 29 +-
.../CodeGen/AArch64/llvm-ir-to-intrinsic.ll | 19 +-
...ate-sm-changing-call-disable-coalescing.ll | 4 +-
.../AArch64/sve-gather-scatter-dag-combine.ll | 10 +-
llvm/test/CodeGen/AArch64/sve-ldst-sext.ll | 36 +-
llvm/test/CodeGen/AArch64/sve-ldst-zext.ll | 26 +-
.../AArch64/sve-masked-gather-legalize.ll | 7 +-
llvm/test/CodeGen/AArch64/sve-sext-zext.ll | 24 +-
llvm/test/CodeGen/AArch64/sve2-histcnt.ll | 4 +-
19 files changed, 1696 insertions(+), 1621 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index f3aea26b3bbad..86cfffd3f17a3 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -537,6 +537,9 @@ bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
// ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm
std::tie(DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 1, 2);
break;
+ case AArch64::DestructiveBinaryImmUnpred:
+ std::tie(DOPIdx, SrcIdx) = std::make_tuple(1, 2);
+ break;
case AArch64::DestructiveBinaryShImmUnpred:
std::tie(DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3);
break;
@@ -560,6 +563,7 @@ bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
break;
case AArch64::DestructiveUnaryPassthru:
case AArch64::DestructiveBinaryImm:
+ case AArch64::DestructiveBinaryImmUnpred:
case AArch64::DestructiveBinaryShImmUnpred:
case AArch64::Destructive2xRegImmUnpred:
DOPRegIsUnique = true;
@@ -688,6 +692,10 @@ bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
.add(MI.getOperand(SrcIdx))
.add(MI.getOperand(Src2Idx));
break;
+ case AArch64::DestructiveBinaryImmUnpred:
+ DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
+ .add(MI.getOperand(SrcIdx));
+ break;
case AArch64::DestructiveBinaryShImmUnpred:
case AArch64::Destructive2xRegImmUnpred:
DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 406ac2fa62df2..a1ad3196aae83 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -60,6 +60,8 @@ def DestructiveUnaryPassthru : DestructiveInstTypeEnum<10>;
// from the governing predicate.
def DestructivePredicate : DestructiveInstTypeEnum<11>;
+def DestructiveBinaryImmUnpred : DestructiveInstTypeEnum<12>;
+
class FalseLanesEnum<bits<2> val> {
bits<2> Value = val;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 52bd8e3f9580c..60a8182fa163b 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -878,6 +878,7 @@ enum DestructiveInstType {
Destructive2xRegImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
DestructiveUnaryPassthru = TSFLAG_DESTRUCTIVE_INST_TYPE(0xa),
DestructivePredicate = TSFLAG_DESTRUCTIVE_INST_TYPE(0xb),
+ DestructiveBinaryImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0xc),
};
enum FalseLaneType {
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 51ef14f4ee1fa..8a0ce28852b31 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1246,6 +1246,7 @@ bool AArch64RegisterInfo::getRegAllocationHints(
case AArch64::DestructiveUnaryPassthru:
AddHintIfSuitable(R, Def.getOperand(3));
break;
+ case AArch64::DestructiveBinaryImmUnpred:
case AArch64::DestructiveBinaryShImmUnpred:
AddHintIfSuitable(R, Def.getOperand(1));
break;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 436f50fe451be..5eb005691fac7 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4588,9 +4588,9 @@ class PromoteNEONToSVEImmWithShift
let Predicates = [HasSVE_or_SME] in {
foreach VT = [v2i64, v4i32, v8i16, v16i8] in {
// Logical operations
- def : PromoteNEONToSVEImm<and, VT, NEONSplatOfSVELogicalImmPat, AND_ZI, i64>;
- def : PromoteNEONToSVEImm<xor, VT, NEONSplatOfSVELogicalImmPat, EOR_ZI, i64>;
- def : PromoteNEONToSVEImm< or, VT, NEONSplatOfSVELogicalImmPat, ORR_ZI, i64>;
+ def : PromoteNEONToSVEImm<and, VT, NEONSplatOfSVELogicalImmPat, AND_ZI_PSEUDO, i64>;
+ def : PromoteNEONToSVEImm<xor, VT, NEONSplatOfSVELogicalImmPat, EOR_ZI_PSEUDO, i64>;
+ def : PromoteNEONToSVEImm< or, VT, NEONSplatOfSVELogicalImmPat, ORR_ZI_PSEUDO, i64>;
}
// Arith operations: ADD
@@ -4606,10 +4606,10 @@ let Predicates = [HasSVE_or_SME] in {
def : PromoteNEONToSVEImmWithShift<sub, v16i8, NEONSplatOfSVEAddSubImm, SUB_ZI_B_PSEUDO>;
// Arith operations: MUL
- def : PromoteNEONToSVEImm<mul, v2i64, NEONSplatOfSVEAddSubImm, MUL_ZI_D, i32>;
- def : PromoteNEONToSVEImm<mul, v4i32, NEONSplatOfSVEArithSImm, MUL_ZI_S, i32>;
- def : PromoteNEONToSVEImm<mul, v8i16, NEONSplatOfSVEArithSImm, MUL_ZI_H, i32>;
- def : PromoteNEONToSVEImm<mul, v16i8, NEONSplatOfSVEArithSImm, MUL_ZI_B, i32>;
+ def : PromoteNEONToSVEImm<mul, v2i64, NEONSplatOfSVEAddSubImm, MUL_ZI_D_PSEUDO, i32>;
+ def : PromoteNEONToSVEImm<mul, v4i32, NEONSplatOfSVEArithSImm, MUL_ZI_S_PSEUDO, i32>;
+ def : PromoteNEONToSVEImm<mul, v8i16, NEONSplatOfSVEArithSImm, MUL_ZI_H_PSEUDO, i32>;
+ def : PromoteNEONToSVEImm<mul, v16i8, NEONSplatOfSVEArithSImm, MUL_ZI_B_PSEUDO, i32>;
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index d333dfcc50d33..6c40304e7cd6b 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -1506,7 +1506,7 @@ def : InstRW<[V1Write_2c_1V01],
"^[SU](MAX|MIN)_Z(I|P[mZ]Z)_[BHSD]",
"^[SU]Q(ADD|SUB)_Z(I|ZZ)_[BHSD]",
"^SUBR_Z(I|P[mZ]Z)_[BHSD]",
- "^(AND|EOR|ORR)_ZI$",
+ "^(AND|EOR|ORR)_ZI",
"^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZP?ZZ",
"^EOR(BT|TB)_ZZZ_[BHSD]$",
"^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]")>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedOlympus.td b/llvm/lib/Target/AArch64/AArch64SchedOlympus.td
index 7798096c10dc2..9d843a9490618 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedOlympus.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedOlympus.td
@@ -2385,7 +2385,7 @@ def : InstRW<[OlympusWrite_2c_1V], (instregex "^ADR_LSL_ZZZ_[SD]_[0123]",
// AND (vectors, predicated)
def : InstRW<[OlympusWrite_2c_1V], (instregex "^AND_ZP[mZ]Z_[BHSD]")>;
// AND (immediate)
-def : InstRW<[OlympusWrite_2c_1V], (instrs AND_ZI)>;
+def : InstRW<[OlympusWrite_2c_1V], (instregex "^AND_ZI")>;
// AND (vectors, unpredicated)
def : InstRW<[OlympusWrite_2c_1V], (instrs AND_ZZZ)>;
// ASR (immediate, predicated)
@@ -2435,7 +2435,7 @@ def : InstRW<[OlympusWrite_2c_1V], (instrs DUPM_ZI)>;
// EOR (vectors, predicated)
def : InstRW<[OlympusWrite_2c_1V], (instregex "^EOR_ZP[mZ]Z_[BHSD]")>;
// EOR (immediate)
-def : InstRW<[OlympusWrite_2c_1V], (instrs EOR_ZI)>;
+def : InstRW<[OlympusWrite_2c_1V], (instregex "^EOR_ZI")>;
// EOR (vectors, unpredicated)
def : InstRW<[OlympusWrite_2c_1V], (instrs EOR_ZZZ)>;
// EORBT
@@ -2494,7 +2494,7 @@ def : InstRW<[OlympusWrite_2c_1V], (instregex "^NOT_ZPmZ_[BHSD]")>;
// ORR (vectors, predicated)
def : InstRW<[OlympusWrite_2c_1V], (instregex "^ORR_ZP[mZ]Z_[BHSD]")>;
// ORR (immediate)
-def : InstRW<[OlympusWrite_2c_1V], (instrs ORR_ZI)>;
+def : InstRW<[OlympusWrite_2c_1V], (instregex "^ORR_ZI")>;
// ORR (vectors, unpredicated)
def : InstRW<[OlympusWrite_2c_1V], (instrs ORR_ZZZ)>;
// PMUL
@@ -2860,7 +2860,7 @@ def : InstRW<[OlympusWrite_5c_1V0123_2], (instrs SQRDMULH_ZZZ_D)>;
// UMULH (unpredicated)
def : InstRW<[OlympusWrite_5c_1V0123_2], (instrs UMULH_ZZZ_D)>;
// MUL (immediate)
-def : InstRW<[OlympusWrite_5c_1V0123_2], (instrs MUL_ZI_D)>;
+def : InstRW<[OlympusWrite_5c_1V0123_2], (instregex "^MUL_ZI_D")>;
// SQDMULH (indexed)
def : InstRW<[OlympusWrite_5c_1V0123_2], (instrs SQDMULH_ZZZI_D)>;
// SQRDMULH (indexed)
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 72434fa86e04c..f05de6ed9840b 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -2074,7 +2074,7 @@ multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op,
class sve_int_log_imm<bits<2> opc, string asm>
: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),
asm, "\t$Zdn, $_Zdn, $imms13",
- "", []>, Sched<[]> {
+ "", []>, SVEPseudo2Instr<NAME, 1>, Sched<[]> {
bits<5> Zdn;
bits<13> imms13;
let Inst{31-24} = 0b00000101;
@@ -2085,7 +2085,7 @@ class sve_int_log_imm<bits<2> opc, string asm>
let Constraints = "$Zdn = $_Zdn";
let DecoderMethod = "DecodeSVELogicalImmInstruction";
- let DestructiveInstType = DestructiveOther;
+ let DestructiveInstType = DestructiveBinaryImmUnpred;
let ElementSize = ElementSizeNone;
let hasSideEffects = 0;
}
@@ -2093,10 +2093,12 @@ class sve_int_log_imm<bits<2> opc, string asm>
multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> {
def NAME : sve_int_log_imm<opc, asm>;
- def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>;
+ def _PSEUDO : UnpredTwoOpImmPseudo<NAME, ZPR64, logical_imm64>;
+
+ def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>;
@@ -2116,10 +2118,10 @@ multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOpera
}
multiclass sve_int_log_imm_bic<SDPatternOperator op> {
- def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
}
class sve_int_dup_mask_imm<string asm>
@@ -5354,7 +5356,7 @@ class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,
: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
asm, "\t$Zdn, $_Zdn, $imm",
"",
- []>, Sched<[]> {
+ []>, SVEPseudo2Instr<NAME, 1>, Sched<[]> {
bits<5> Zdn;
bits<8> imm;
let Inst{31-24} = 0b00100101;
@@ -5365,7 +5367,7 @@ class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,
let Inst{4-0} = Zdn;
let Constraints = "$Zdn = $_Zdn";
- let DestructiveInstType = DestructiveOther;
+ let DestructiveInstType = DestructiveBinaryImmUnpred;
let ElementSize = ElementSizeNone;
let hasSideEffects = 0;
}
@@ -5376,10 +5378,15 @@ multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8_32b>;
def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8_32b>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;
+ def _B_PSEUDO : UnpredTwoOpImmPseudo<NAME # _B, ZPR8, simm8_32b>;
+ def _H_PSEUDO : UnpredTwoOpImmPseudo<NAME # _H, ZPR16, simm8_32b>;
+ def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, simm8_32b>;
+ def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, simm8_32b>;
+
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Pseudo>(NAME # _D_PSEUDO)>;
}
multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {
@@ -5388,10 +5395,15 @@ multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperato
def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>;
def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>;
+ def _B_PSEUDO : UnpredTwoOpImmPseudo<NAME # _B, ZPR8, imm0_255>;
+ def _H_PSEUDO : UnpredTwoOpImmPseudo<NAME # _H, ZPR16, imm0_255>;
+ def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, imm0_255>;
+ def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, imm0_255>;
+
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Pseudo>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
}
multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {
@@ -5400,10 +5412,15 @@ multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {
def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8_32b>;
def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8_32b>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>;
+ def _B_PSEUDO : UnpredTwoOpImmPseudo<NAME # _B, ZPR8, simm8_32b>;
+ def _H_PSEUDO : UnpredTwoOpImmPseudo<NAME # _H, ZPR16, simm8_32b>;
+ def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, simm8_32b>;
+ def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, simm8_32b>;
+
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D_PSEUDO)>;
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AArch64/aarch64-smull.ll b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
index ca7f40d560afd..5896b408be49c 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-smull.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-smull.ll
@@ -1458,11 +1458,13 @@ define <8 x i32> @amull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {
; CHECK-SVE-LABEL: amull2_i16:
; CHECK-SVE: // %bb.0:
; CHECK-SVE-NEXT: smull v2.4s, v0.4h, v1.4h
-; CHECK-SVE-NEXT: smull2 v1.4s, v0.8h, v1.8h
-; CHECK-SVE-NEXT: and z2.s, z2.s, #0xffff
+; CHECK-SVE-NEXT: smull2 v0.4s, v0.8h, v1.8h
+; CHECK-SVE-NEXT: movprfx z1, z0
; CHECK-SVE-NEXT: and z1.s, z1.s, #0xffff
+; CHECK-SVE-NEXT: movprfx z0, z2
+; CHECK-SVE-NEXT: and z0.s, z0.s, #0xffff
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
-; CHECK-SVE-NEXT: mov v0.16b, v2.16b
; CHECK-SVE-NEXT: ret
;
; CHECK-GI-LABEL: amull2_i16:
@@ -1493,11 +1495,13 @@ define <4 x i64> @amull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {
; CHECK-SVE-LABEL: amull2_i32:
; CHECK-SVE: // %bb.0:
; CHECK-SVE-NEXT: smull v2.2d, v0.2s, v1.2s
-; CHECK-SVE-NEXT: smull2 v1.2d, v0.4s, v1.4s
-; CHECK-SVE-NEXT: and z2.d, z2.d, #0xffffffff
+; CHECK-SVE-NEXT: smull2 v0.2d, v0.4s, v1.4s
+; CHECK-SVE-NEXT: movprfx z1, z0
; CHECK-SVE-NEXT: and z1.d, z1.d, #0xffffffff
+; CHECK-SVE-NEXT: movprfx z0, z2
+; CHECK-SVE-NEXT: and z0.d, z0.d, #0xffffffff
+; CHECK-SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-SVE-NEXT: // kill: def $q1 killed $q1 killed $z1
-; CHECK-SVE-NEXT: mov v0.16b, v2.16b
; CHECK-SVE-NEXT: ret
;
; CHECK-GI-LABEL: amull2_i32:
diff --git a/llvm/test/CodeGen/AArch64/clmul-scalable.ll b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
index d2c7f98854d91..ef903902b9e65 100644
--- a/llvm/test/CodeGen/AArch64/clmul-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-scalable.ll
@@ -9,22 +9,22 @@
define <vscale x 16 x i8> @clmul_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i8> %y) {
; CHECK-SVE-LABEL: clmul_nxv16i8:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: ptrue p0.b
-; CHECK-SVE-NEXT: and z1.b, z1.b, #0x80
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.b, z2.b, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.b, z3.b, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.b, z4.b, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.b, z5.b, #0x8
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.b, z6.b, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.b, z7.b, #0x20
+; CHECK-SVE-NEXT: ptrue p0.b
+; CHECK-SVE-NEXT: movprfx z24, z1
; CHECK-SVE-NEXT: and z24.b, z24.b, #0x40
+; CHECK-SVE-NEXT: and z1.b, z1.b, #0x80
; CHECK-SVE-NEXT: mul z2.b, p0/m, z2.b, z0.b
; CHECK-SVE-NEXT: mul z3.b, p0/m, z3.b, z0.b
; CHECK-SVE-NEXT: mul z4.b, p0/m, z4.b, z0.b
@@ -44,22 +44,22 @@ define <vscale x 16 x i8> @clmul_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i
;
; CHECK-SVE-AES-LABEL: clmul_nxv16i8:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: ptrue p0.b
-; CHECK-SVE-AES-NEXT: and z1.b, z1.b, #0x80
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.b, z2.b, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.b, z3.b, #0x1
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.b, z4.b, #0x4
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.b, z5.b, #0x8
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
; CHECK-SVE-AES-NEXT: and z6.b, z6.b, #0x10
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
; CHECK-SVE-AES-NEXT: and z7.b, z7.b, #0x20
+; CHECK-SVE-AES-NEXT: ptrue p0.b
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
; CHECK-SVE-AES-NEXT: and z24.b, z24.b, #0x40
+; CHECK-SVE-AES-NEXT: and z1.b, z1.b, #0x80
; CHECK-SVE-AES-NEXT: mul z2.b, p0/m, z2.b, z0.b
; CHECK-SVE-AES-NEXT: mul z3.b, p0/m, z3.b, z0.b
; CHECK-SVE-AES-NEXT: mul z4.b, p0/m, z4.b, z0.b
@@ -103,62 +103,62 @@ define <vscale x 16 x i8> @clmul_nxv16i8(<vscale x 16 x i8> %x, <vscale x 16 x i
define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i16> %y) {
; CHECK-SVE-LABEL: clmul_nxv8i16:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: ptrue p0.h
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.h, z4.h, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.h, z5.h, #0x8
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: ptrue p0.h
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.h, z7.h, #0x20
+; CHECK-SVE-NEXT: movprfx z24, z1
; CHECK-SVE-NEXT: and z24.h, z24.h, #0x80
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.h, z25.h, #0x100
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.h, z26.h, #0x800
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.h, z27.h, #0x1000
; CHECK-SVE-NEXT: mul z2.h, p0/m, z2.h, z0.h
; CHECK-SVE-NEXT: mul z3.h, p0/m, z3.h, z0.h
-; CHECK-SVE-NEXT: and z25.h, z25.h, #0x100
; CHECK-SVE-NEXT: mul z4.h, p0/m, z4.h, z0.h
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.h, z28.h, #0x40
; CHECK-SVE-NEXT: mul z5.h, p0/m, z5.h, z0.h
-; CHECK-SVE-NEXT: mov z28.d, z1.d
; CHECK-SVE-NEXT: mul z6.h, p0/m, z6.h, z0.h
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.h, z29.h, #0x200
; CHECK-SVE-NEXT: mul z7.h, p0/m, z7.h, z0.h
-; CHECK-SVE-NEXT: and z26.h, z26.h, #0x800
; CHECK-SVE-NEXT: mul z24.h, p0/m, z24.h, z0.h
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.h, z30.h, #0x2000
; CHECK-SVE-NEXT: mul z25.h, p0/m, z25.h, z0.h
-; CHECK-SVE-NEXT: and z27.h, z27.h, #0x1000
-; CHECK-SVE-NEXT: mov z29.d, z1.d
-; CHECK-SVE-NEXT: mov z30.d, z1.d
-; CHECK-SVE-NEXT: and z28.h, z28.h, #0x40
; CHECK-SVE-NEXT: mul z26.h, p0/m, z26.h, z0.h
-; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-NEXT: mul z27.h, p0/m, z27.h, z0.h
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
+; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
; CHECK-SVE-NEXT: mul z28.h, p0/m, z28.h, z0.h
-; CHECK-SVE-NEXT: and z29.h, z29.h, #0x200
-; CHECK-SVE-NEXT: and z30.h, z30.h, #0x2000
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
+; CHECK-SVE-NEXT: mul z29.h, p0/m, z29.h, z0.h
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.h, z4.h, #0x400
+; CHECK-SVE-NEXT: mul z30.h, p0/m, z30.h, z0.h
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.h, z5.h, #0x4000
; CHECK-SVE-NEXT: eor z6.d, z6.d, z7.d
; CHECK-SVE-NEXT: eor z7.d, z24.d, z25.d
; CHECK-SVE-NEXT: and z1.h, z1.h, #0x8000
-; CHECK-SVE-NEXT: and z4.h, z4.h, #0x400
-; CHECK-SVE-NEXT: and z5.h, z5.h, #0x4000
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mul z29.h, p0/m, z29.h, z0.h
-; CHECK-SVE-NEXT: mul z30.h, p0/m, z30.h, z0.h
; CHECK-SVE-NEXT: eor z24.d, z26.d, z27.d
-; CHECK-SVE-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-NEXT: mul z4.h, p0/m, z4.h, z0.h
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-NEXT: mul z5.h, p0/m, z5.h, z0.h
-; CHECK-SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h
+; CHECK-SVE-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-NEXT: mul z0.h, p0/m, z0.h, z1.h
; CHECK-SVE-NEXT: eor z7.d, z24.d, z30.d
; CHECK-SVE-NEXT: eor z1.d, z2.d, z3.d
; CHECK-SVE-NEXT: eor z2.d, z6.d, z4.d
@@ -170,62 +170,62 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SVE-AES-LABEL: clmul_nxv8i16:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE-AES-NEXT: ptrue p0.h
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z25.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.h, z4.h, #0x4
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.h, z5.h, #0x8
-; CHECK-SVE-AES-NEXT: mov z26.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z27.d, z1.d
+; CHECK-SVE-AES-NEXT: ptrue p0.h
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
; CHECK-SVE-AES-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
; CHECK-SVE-AES-NEXT: and z7.h, z7.h, #0x20
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
; CHECK-SVE-AES-NEXT: and z24.h, z24.h, #0x80
+; CHECK-SVE-AES-NEXT: movprfx z25, z1
+; CHECK-SVE-AES-NEXT: and z25.h, z25.h, #0x100
+; CHECK-SVE-AES-NEXT: movprfx z26, z1
+; CHECK-SVE-AES-NEXT: and z26.h, z26.h, #0x800
+; CHECK-SVE-AES-NEXT: movprfx z27, z1
+; CHECK-SVE-AES-NEXT: and z27.h, z27.h, #0x1000
; CHECK-SVE-AES-NEXT: mul z2.h, p0/m, z2.h, z0.h
; CHECK-SVE-AES-NEXT: mul z3.h, p0/m, z3.h, z0.h
-; CHECK-SVE-AES-NEXT: and z25.h, z25.h, #0x100
; CHECK-SVE-AES-NEXT: mul z4.h, p0/m, z4.h, z0.h
+; CHECK-SVE-AES-NEXT: movprfx z28, z1
+; CHECK-SVE-AES-NEXT: and z28.h, z28.h, #0x40
; CHECK-SVE-AES-NEXT: mul z5.h, p0/m, z5.h, z0.h
-; CHECK-SVE-AES-NEXT: mov z28.d, z1.d
; CHECK-SVE-AES-NEXT: mul z6.h, p0/m, z6.h, z0.h
+; CHECK-SVE-AES-NEXT: movprfx z29, z1
+; CHECK-SVE-AES-NEXT: and z29.h, z29.h, #0x200
; CHECK-SVE-AES-NEXT: mul z7.h, p0/m, z7.h, z0.h
-; CHECK-SVE-AES-NEXT: and z26.h, z26.h, #0x800
; CHECK-SVE-AES-NEXT: mul z24.h, p0/m, z24.h, z0.h
+; CHECK-SVE-AES-NEXT: movprfx z30, z1
+; CHECK-SVE-AES-NEXT: and z30.h, z30.h, #0x2000
; CHECK-SVE-AES-NEXT: mul z25.h, p0/m, z25.h, z0.h
-; CHECK-SVE-AES-NEXT: and z27.h, z27.h, #0x1000
-; CHECK-SVE-AES-NEXT: mov z29.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z30.d, z1.d
-; CHECK-SVE-AES-NEXT: and z28.h, z28.h, #0x40
; CHECK-SVE-AES-NEXT: mul z26.h, p0/m, z26.h, z0.h
-; CHECK-SVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-AES-NEXT: mul z27.h, p0/m, z27.h, z0.h
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z3.d, z2.d
; CHECK-SVE-AES-NEXT: mul z28.h, p0/m, z28.h, z0.h
-; CHECK-SVE-AES-NEXT: and z29.h, z29.h, #0x200
-; CHECK-SVE-AES-NEXT: and z30.h, z30.h, #0x2000
+; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z5.d
+; CHECK-SVE-AES-NEXT: mul z29.h, p0/m, z29.h, z0.h
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
+; CHECK-SVE-AES-NEXT: and z4.h, z4.h, #0x400
+; CHECK-SVE-AES-NEXT: mul z30.h, p0/m, z30.h, z0.h
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
+; CHECK-SVE-AES-NEXT: and z5.h, z5.h, #0x4000
; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z7.d
; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z25.d
; CHECK-SVE-AES-NEXT: and z1.h, z1.h, #0x8000
-; CHECK-SVE-AES-NEXT: and z4.h, z4.h, #0x400
-; CHECK-SVE-AES-NEXT: and z5.h, z5.h, #0x4000
-; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-AES-NEXT: mul z29.h, p0/m, z29.h, z0.h
-; CHECK-SVE-AES-NEXT: mul z30.h, p0/m, z30.h, z0.h
; CHECK-SVE-AES-NEXT: eor z24.d, z26.d, z27.d
-; CHECK-SVE-AES-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-AES-NEXT: mul z4.h, p0/m, z4.h, z0.h
+; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-AES-NEXT: mul z5.h, p0/m, z5.h, z0.h
-; CHECK-SVE-AES-NEXT: mul z0.h, p0/m, z0.h, z1.h
+; CHECK-SVE-AES-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-AES-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-AES-NEXT: mul z0.h, p0/m, z0.h, z1.h
; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z30.d
; CHECK-SVE-AES-NEXT: eor z1.d, z2.d, z3.d
; CHECK-SVE-AES-NEXT: eor z2.d, z6.d, z4.d
@@ -237,57 +237,57 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv8i16:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
+; CHECK-SME-STREAMING-NEXT: movprfx z2, z1
; CHECK-SME-STREAMING-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x20
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x10
; CHECK-SME-STREAMING-NEXT: mul z2.h, z0.h, z2.h
; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z4.h
; CHECK-SME-STREAMING-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z6.h, z0.h, z6.h
; CHECK-SME-STREAMING-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x10
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x20
; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x80
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x40
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x200
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x80
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x40
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x200
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z6.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x800
-; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x400
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x2000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x800
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x400
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x2000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x8000
+; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x4000
; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SME-STREAMING-NEXT: mul z0.h, z0.h, z1.h
@@ -297,57 +297,57 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv8i16:
; CHECK-SME-STREAMING-SSVE-AES: // %bb.0:
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z6.d, z1.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z2, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z4, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x20
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x10
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z2.h, z0.h, z2.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z4.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z6.h, z0.h, z6.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x10
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x20
; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x80
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x40
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x200
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x80
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x40
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x200
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z6.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x800
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x400
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x2000
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x800
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x400
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x2000
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x8000
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z1.h, z1.h, #0x4000
; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z0.h, z0.h, z1.h
@@ -357,57 +357,57 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SVE2-LABEL: clmul_nxv8i16:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
+; CHECK-SVE2-NEXT: movprfx z2, z1
; CHECK-SVE2-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE2-NEXT: movprfx z3, z1
; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE2-NEXT: movprfx z4, z1
; CHECK-SVE2-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x20
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x10
; CHECK-SVE2-NEXT: mul z2.h, z0.h, z2.h
; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
; CHECK-SVE2-NEXT: mul z4.h, z0.h, z4.h
; CHECK-SVE2-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z6.h, z0.h, z6.h
; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x10
+; CHECK-SVE2-NEXT: movprfx z3, z1
+; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x20
; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-NEXT: and z4.h, z4.h, #0x80
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x40
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x200
+; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x80
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x40
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x200
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-NEXT: mul z6.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: and z4.h, z4.h, #0x800
-; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x400
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x2000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x800
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x400
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x2000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x8000
+; CHECK-SVE2-NEXT: and z1.h, z1.h, #0x4000
; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SVE2-NEXT: mul z0.h, z0.h, z1.h
@@ -417,57 +417,57 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
;
; CHECK-SVE2-AES-LABEL: clmul_nxv8i16:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
+; CHECK-SVE2-AES-NEXT: movprfx z2, z1
; CHECK-SVE2-AES-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE2-AES-NEXT: movprfx z3, z1
; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE2-AES-NEXT: movprfx z4, z1
; CHECK-SVE2-AES-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x20
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
+; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x10
; CHECK-SVE2-AES-NEXT: mul z2.h, z0.h, z2.h
; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z4.h
; CHECK-SVE2-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z6.h, z0.h, z6.h
; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x10
+; CHECK-SVE2-AES-NEXT: movprfx z3, z1
+; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x20
; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-AES-NEXT: and z4.h, z4.h, #0x80
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x40
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x200
+; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
+; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x80
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
+; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x40
+; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
+; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x200
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x100
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-AES-NEXT: mul z6.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: and z4.h, z4.h, #0x800
-; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x400
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x2000
+; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
+; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x800
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
+; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x400
+; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
+; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
+; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x2000
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x1000
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z1.h, z1.h, #0x4000
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x8000
+; CHECK-SVE2-AES-NEXT: and z1.h, z1.h, #0x4000
; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
; CHECK-SVE2-AES-NEXT: mul z0.h, z0.h, z1.h
@@ -481,264 +481,282 @@ define <vscale x 8 x i16> @clmul_nxv8i16(<vscale x 8 x i16> %x, <vscale x 8 x i1
define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
; CHECK-SVE-LABEL: clmul_nxv4i32:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: ptrue p0.s
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
+; CHECK-SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE-NEXT: addvl sp, sp, #-1
+; CHECK-SVE-NEXT: str z8, [sp] // 16-byte Folded Spill
+; CHECK-SVE-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-SVE-NEXT: .cfi_offset w29, -16
+; CHECK-SVE-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.s, z2.s, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE-NEXT: mov z26.d, z1.d
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.s, z4.s, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.s, z5.s, #0x8
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.s, z6.s, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.s, z7.s, #0x20
-; CHECK-SVE-NEXT: and z24.s, z24.s, #0x80
-; CHECK-SVE-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.s, z24.s, #0x40
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.s, z25.s, #0x80
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.s, z26.s, #0x100
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.s, z27.s, #0x200
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.s, z29.s, #0x800
; CHECK-SVE-NEXT: mul z2.s, p0/m, z2.s, z0.s
; CHECK-SVE-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.s, z30.s, #0x400000
; CHECK-SVE-NEXT: mul z4.s, p0/m, z4.s, z0.s
; CHECK-SVE-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-NEXT: and z26.s, z26.s, #0x40
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.s, z31.s, #0x800000
; CHECK-SVE-NEXT: mul z6.s, p0/m, z6.s, z0.s
; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-NEXT: mov z28.d, z1.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.s, z28.s, #0x400
; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-NEXT: and z27.s, z27.s, #0x200
+; CHECK-SVE-NEXT: movprfx z8, z1
+; CHECK-SVE-NEXT: and z8.s, z8.s, #0x2000000
; CHECK-SVE-NEXT: mul z26.s, p0/m, z26.s, z0.s
; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z5.d
-; CHECK-SVE-NEXT: mov z29.d, z1.d
-; CHECK-SVE-NEXT: and z28.s, z28.s, #0x8000
; CHECK-SVE-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-NEXT: eor z5.d, z6.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z3.s, z3.s, #0x400
-; CHECK-SVE-NEXT: eor z6.d, z24.d, z25.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mul z28.s, p0/m, z28.s, z0.s
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z5.d, z26.d
-; CHECK-SVE-NEXT: and z7.s, z7.s, #0x800
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-NEXT: eor z5.d, z6.d, z27.d
-; CHECK-SVE-NEXT: and z24.s, z24.s, #0x1000
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: and z25.s, z25.s, #0x800000
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-NEXT: and z26.s, z26.s, #0x40000
-; CHECK-SVE-NEXT: and z29.s, z29.s, #0x100000
-; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-NEXT: eor z3.d, z5.d, z3.d
-; CHECK-SVE-NEXT: and z6.s, z6.s, #0x2000
-; CHECK-SVE-NEXT: and z4.s, z4.s, #0x10000
-; CHECK-SVE-NEXT: and z27.s, z27.s, #0x1000000
-; CHECK-SVE-NEXT: mul z26.s, p0/m, z26.s, z0.s
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.s, z5.s, #0x1000
; CHECK-SVE-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-NEXT: eor z4.d, z6.d, z7.d
+; CHECK-SVE-NEXT: movprfx z6, z1
+; CHECK-SVE-NEXT: and z6.s, z6.s, #0x10000
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.s, z7.s, #0x20000
+; CHECK-SVE-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-NEXT: mul z31.s, p0/m, z31.s, z0.s
+; CHECK-SVE-NEXT: eor z25.d, z25.d, z26.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-NEXT: mul z5.s, p0/m, z5.s, z0.s
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z24.d
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.s, z4.s, #0x2000
; CHECK-SVE-NEXT: mul z6.s, p0/m, z6.s, z0.s
+; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.s, z24.s, #0x40000
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.s, z26.s, #0x1000000
+; CHECK-SVE-NEXT: mul z28.s, p0/m, z28.s, z0.s
+; CHECK-SVE-NEXT: mul z8.s, p0/m, z8.s, z0.s
; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
+; CHECK-SVE-NEXT: eor z3.d, z25.d, z27.d
; CHECK-SVE-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-NEXT: eor z5.d, z7.d, z24.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z3.s, z3.s, #0x20000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: and z24.s, z24.s, #0x400000
-; CHECK-SVE-NEXT: and z7.s, z7.s, #0x4000
-; CHECK-SVE-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-NEXT: and z6.s, z6.s, #0x80000
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.s, z25.s, #0x4000
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.s, z27.s, #0x80000
; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z3.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mul z6.s, p0/m, z6.s, z0.s
-; CHECK-SVE-NEXT: eor z24.d, z24.d, z25.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z7.d
-; CHECK-SVE-NEXT: and z4.s, z4.s, #0x2000000
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: and z25.s, z25.s, #0x4000000
-; CHECK-SVE-NEXT: eor z7.d, z24.d, z27.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: and z26.s, z26.s, #0x200000
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z28.d
-; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-NEXT: and z24.s, z24.s, #0x20000000
-; CHECK-SVE-NEXT: and z6.s, z6.s, #0x8000000
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z29.d
-; CHECK-SVE-NEXT: eor z4.d, z7.d, z4.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
; CHECK-SVE-NEXT: mul z26.s, p0/m, z26.s, z0.s
-; CHECK-SVE-NEXT: and z27.s, z27.s, #0x40000000
-; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-NEXT: and z1.s, z1.s, #0x80000000
-; CHECK-SVE-NEXT: mul z6.s, p0/m, z6.s, z0.s
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z5.d
-; CHECK-SVE-NEXT: and z7.s, z7.s, #0x10000000
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z25.d
+; CHECK-SVE-NEXT: eor z5.d, z29.d, z5.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z7.d
+; CHECK-SVE-NEXT: eor z7.d, z30.d, z31.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.s, z29.s, #0x8000
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.s, z30.s, #0x100000
+; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
; CHECK-SVE-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z26.d
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.s, z31.s, #0x4000000
+; CHECK-SVE-NEXT: eor z4.d, z5.d, z4.d
+; CHECK-SVE-NEXT: eor z5.d, z6.d, z24.d
+; CHECK-SVE-NEXT: eor z3.d, z3.d, z28.d
+; CHECK-SVE-NEXT: eor z6.d, z7.d, z26.d
+; CHECK-SVE-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.s, z7.s, #0x200000
+; CHECK-SVE-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-NEXT: mul z31.s, p0/m, z31.s, z0.s
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.s, z24.s, #0x8000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z25.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z27.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z8.d
; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z6.d
+; CHECK-SVE-NEXT: movprfx z3, z1
+; CHECK-SVE-NEXT: and z3.s, z3.s, #0x10000000
+; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.s, z25.s, #0x20000000
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.s, z26.s, #0x40000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z29.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z30.d
+; CHECK-SVE-NEXT: and z1.s, z1.s, #0x80000000
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z31.d
+; CHECK-SVE-NEXT: mul z3.s, p0/m, z3.s, z0.s
+; CHECK-SVE-NEXT: ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
+; CHECK-SVE-NEXT: mul z26.s, p0/m, z26.s, z0.s
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
+; CHECK-SVE-NEXT: eor z4.d, z5.d, z7.d
; CHECK-SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s
-; CHECK-SVE-NEXT: eor z1.d, z2.d, z3.d
-; CHECK-SVE-NEXT: eor z3.d, z24.d, z27.d
-; CHECK-SVE-NEXT: eor z2.d, z4.d, z7.d
-; CHECK-SVE-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SVE-NEXT: eor z5.d, z6.d, z24.d
+; CHECK-SVE-NEXT: eor z1.d, z2.d, z4.d
+; CHECK-SVE-NEXT: eor z2.d, z5.d, z3.d
+; CHECK-SVE-NEXT: eor z3.d, z25.d, z26.d
; CHECK-SVE-NEXT: eor z1.d, z1.d, z2.d
+; CHECK-SVE-NEXT: eor z0.d, z3.d, z0.d
; CHECK-SVE-NEXT: eor z0.d, z1.d, z0.d
+; CHECK-SVE-NEXT: addvl sp, sp, #1
+; CHECK-SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SVE-NEXT: ret
;
; CHECK-SVE-AES-LABEL: clmul_nxv4i32:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: ptrue p0.s
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z25.d, z1.d
+; CHECK-SVE-AES-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE-AES-NEXT: addvl sp, sp, #-1
+; CHECK-SVE-AES-NEXT: str z8, [sp] // 16-byte Folded Spill
+; CHECK-SVE-AES-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-SVE-AES-NEXT: .cfi_offset w29, -16
+; CHECK-SVE-AES-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.s, z2.s, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.s, z3.s, #0x1
-; CHECK-SVE-AES-NEXT: mov z26.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x4
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.s, z5.s, #0x8
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x10
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x20
-; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x80
-; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-AES-NEXT: ptrue p0.s
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
+; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x40
+; CHECK-SVE-AES-NEXT: movprfx z25, z1
+; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x80
+; CHECK-SVE-AES-NEXT: movprfx z26, z1
+; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x100
+; CHECK-SVE-AES-NEXT: movprfx z27, z1
+; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x200
+; CHECK-SVE-AES-NEXT: movprfx z29, z1
+; CHECK-SVE-AES-NEXT: and z29.s, z29.s, #0x800
; CHECK-SVE-AES-NEXT: mul z2.s, p0/m, z2.s, z0.s
; CHECK-SVE-AES-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-AES-NEXT: mov z27.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z30, z1
+; CHECK-SVE-AES-NEXT: and z30.s, z30.s, #0x400000
; CHECK-SVE-AES-NEXT: mul z4.s, p0/m, z4.s, z0.s
; CHECK-SVE-AES-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x40
+; CHECK-SVE-AES-NEXT: movprfx z31, z1
+; CHECK-SVE-AES-NEXT: and z31.s, z31.s, #0x800000
; CHECK-SVE-AES-NEXT: mul z6.s, p0/m, z6.s, z0.s
; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-AES-NEXT: mov z28.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z28, z1
+; CHECK-SVE-AES-NEXT: and z28.s, z28.s, #0x400
; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x200
+; CHECK-SVE-AES-NEXT: movprfx z8, z1
+; CHECK-SVE-AES-NEXT: and z8.s, z8.s, #0x2000000
; CHECK-SVE-AES-NEXT: mul z26.s, p0/m, z26.s, z0.s
; CHECK-SVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: eor z4.d, z4.d, z5.d
-; CHECK-SVE-AES-NEXT: mov z29.d, z1.d
-; CHECK-SVE-AES-NEXT: and z28.s, z28.s, #0x8000
; CHECK-SVE-AES-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z5.d, z6.d, z7.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: and z3.s, z3.s, #0x400
-; CHECK-SVE-AES-NEXT: eor z6.d, z24.d, z25.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z28.s, p0/m, z28.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z4.d
-; CHECK-SVE-AES-NEXT: mov z25.d, z1.d
-; CHECK-SVE-AES-NEXT: eor z4.d, z5.d, z26.d
-; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x800
-; CHECK-SVE-AES-NEXT: mov z26.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z5.d, z6.d, z27.d
-; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x1000
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x800000
-; CHECK-SVE-AES-NEXT: mov z27.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z4.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x40000
-; CHECK-SVE-AES-NEXT: and z29.s, z29.s, #0x100000
-; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z3.d, z5.d, z3.d
-; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x2000
-; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x10000
-; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x1000000
-; CHECK-SVE-AES-NEXT: mul z26.s, p0/m, z26.s, z0.s
+; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z5.d
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
+; CHECK-SVE-AES-NEXT: and z5.s, z5.s, #0x1000
; CHECK-SVE-AES-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-AES-NEXT: eor z4.d, z6.d, z7.d
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
+; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x10000
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
+; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x20000
+; CHECK-SVE-AES-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-AES-NEXT: mul z31.s, p0/m, z31.s, z0.s
+; CHECK-SVE-AES-NEXT: eor z25.d, z25.d, z26.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-AES-NEXT: mul z5.s, p0/m, z5.s, z0.s
+; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z24.d
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
+; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x2000
; CHECK-SVE-AES-NEXT: mul z6.s, p0/m, z6.s, z0.s
+; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
+; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x40000
+; CHECK-SVE-AES-NEXT: movprfx z26, z1
+; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x1000000
+; CHECK-SVE-AES-NEXT: mul z28.s, p0/m, z28.s, z0.s
+; CHECK-SVE-AES-NEXT: mul z8.s, p0/m, z8.s, z0.s
; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
+; CHECK-SVE-AES-NEXT: eor z3.d, z25.d, z27.d
; CHECK-SVE-AES-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-AES-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z5.d, z7.d, z24.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: and z3.s, z3.s, #0x20000
-; CHECK-SVE-AES-NEXT: eor z5.d, z5.d, z6.d
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x400000
-; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x4000
-; CHECK-SVE-AES-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x80000
+; CHECK-SVE-AES-NEXT: movprfx z25, z1
+; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x4000
+; CHECK-SVE-AES-NEXT: movprfx z27, z1
+; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x80000
; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z3.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z6.s, p0/m, z6.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z24.d, z24.d, z25.d
-; CHECK-SVE-AES-NEXT: mov z25.d, z1.d
-; CHECK-SVE-AES-NEXT: eor z5.d, z5.d, z7.d
-; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x2000000
-; CHECK-SVE-AES-NEXT: eor z3.d, z3.d, z26.d
-; CHECK-SVE-AES-NEXT: mov z26.d, z1.d
-; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x4000000
-; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z27.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z3.d, z3.d, z6.d
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x200000
-; CHECK-SVE-AES-NEXT: mov z27.d, z1.d
-; CHECK-SVE-AES-NEXT: eor z5.d, z5.d, z28.d
-; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x20000000
-; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x8000000
-; CHECK-SVE-AES-NEXT: eor z3.d, z3.d, z29.d
-; CHECK-SVE-AES-NEXT: eor z4.d, z7.d, z4.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
; CHECK-SVE-AES-NEXT: mul z26.s, p0/m, z26.s, z0.s
-; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x40000000
-; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-AES-NEXT: and z1.s, z1.s, #0x80000000
-; CHECK-SVE-AES-NEXT: mul z6.s, p0/m, z6.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z5.d
-; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x10000000
-; CHECK-SVE-AES-NEXT: eor z4.d, z4.d, z25.d
+; CHECK-SVE-AES-NEXT: eor z5.d, z29.d, z5.d
+; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z7.d
+; CHECK-SVE-AES-NEXT: eor z7.d, z30.d, z31.d
+; CHECK-SVE-AES-NEXT: movprfx z29, z1
+; CHECK-SVE-AES-NEXT: and z29.s, z29.s, #0x8000
+; CHECK-SVE-AES-NEXT: movprfx z30, z1
+; CHECK-SVE-AES-NEXT: and z30.s, z30.s, #0x100000
+; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
; CHECK-SVE-AES-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z3.d, z3.d, z26.d
+; CHECK-SVE-AES-NEXT: movprfx z31, z1
+; CHECK-SVE-AES-NEXT: and z31.s, z31.s, #0x4000000
+; CHECK-SVE-AES-NEXT: eor z4.d, z5.d, z4.d
+; CHECK-SVE-AES-NEXT: eor z5.d, z6.d, z24.d
+; CHECK-SVE-AES-NEXT: eor z3.d, z3.d, z28.d
+; CHECK-SVE-AES-NEXT: eor z6.d, z7.d, z26.d
+; CHECK-SVE-AES-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
+; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x200000
+; CHECK-SVE-AES-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-AES-NEXT: mul z31.s, p0/m, z31.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
+; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x8000000
+; CHECK-SVE-AES-NEXT: eor z4.d, z4.d, z25.d
+; CHECK-SVE-AES-NEXT: eor z5.d, z5.d, z27.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z8.d
; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z4.d, z4.d, z6.d
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
+; CHECK-SVE-AES-NEXT: and z3.s, z3.s, #0x10000000
+; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z25, z1
+; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x20000000
+; CHECK-SVE-AES-NEXT: movprfx z26, z1
+; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x40000000
+; CHECK-SVE-AES-NEXT: eor z4.d, z4.d, z29.d
+; CHECK-SVE-AES-NEXT: eor z5.d, z5.d, z30.d
+; CHECK-SVE-AES-NEXT: and z1.s, z1.s, #0x80000000
+; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z31.d
+; CHECK-SVE-AES-NEXT: mul z3.s, p0/m, z3.s, z0.s
+; CHECK-SVE-AES-NEXT: ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
+; CHECK-SVE-AES-NEXT: mul z26.s, p0/m, z26.s, z0.s
+; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z4.d
+; CHECK-SVE-AES-NEXT: eor z4.d, z5.d, z7.d
; CHECK-SVE-AES-NEXT: mul z0.s, p0/m, z0.s, z1.s
-; CHECK-SVE-AES-NEXT: eor z1.d, z2.d, z3.d
-; CHECK-SVE-AES-NEXT: eor z3.d, z24.d, z27.d
-; CHECK-SVE-AES-NEXT: eor z2.d, z4.d, z7.d
-; CHECK-SVE-AES-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SVE-AES-NEXT: eor z5.d, z6.d, z24.d
+; CHECK-SVE-AES-NEXT: eor z1.d, z2.d, z4.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z5.d, z3.d
+; CHECK-SVE-AES-NEXT: eor z3.d, z25.d, z26.d
; CHECK-SVE-AES-NEXT: eor z1.d, z1.d, z2.d
+; CHECK-SVE-AES-NEXT: eor z0.d, z3.d, z0.d
; CHECK-SVE-AES-NEXT: eor z0.d, z1.d, z0.d
+; CHECK-SVE-AES-NEXT: addvl sp, sp, #1
+; CHECK-SVE-AES-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SVE-AES-NEXT: ret
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv4i32:
@@ -775,261 +793,271 @@ define <vscale x 4 x i32> @clmul_nxv4i32(<vscale x 4 x i32> %x, <vscale x 4 x i3
define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
; CHECK-SVE-LABEL: clmul_nxv2i64:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: ptrue p0.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
+; CHECK-SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE-NEXT: addvl sp, sp, #-1
+; CHECK-SVE-NEXT: str z8, [sp] // 16-byte Folded Spill
+; CHECK-SVE-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-SVE-NEXT: .cfi_offset w29, -16
+; CHECK-SVE-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.d, z4.d, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.d, z5.d, #0x8
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.d, z6.d, #0x10
-; CHECK-SVE-NEXT: mov z26.d, z1.d
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.d, z7.d, #0x20
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x40
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x80
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x80
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x100
; CHECK-SVE-NEXT: mul z2.d, p0/m, z2.d, z0.d
; CHECK-SVE-NEXT: mul z3.d, p0/m, z3.d, z0.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40
; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x100
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x200
; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: mov z28.d, z1.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x800
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x20000
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x1000
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
+; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: mov z29.d, z1.d
; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x400000
-; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: eor z4.d, z6.d, z7.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z29.d, z29.d, #0x800000
-; CHECK-SVE-NEXT: mov z30.d, z1.d
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.d, z4.d, #0x400
; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x200
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x800
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z24.d
-; CHECK-SVE-NEXT: eor z4.d, z25.d, z26.d
; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x1000
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: and z30.d, z30.d, #0x800000000
-; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x2000
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x10000
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x400
-; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.d, z5.d, #0x2000
; CHECK-SVE-NEXT: eor z6.d, z6.d, z7.d
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z4.d, z4.d, #0x4000
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z25.d
-; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: eor z25.d, z26.d, z27.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: eor z7.d, z24.d, z25.d
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x10000
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.d, z30.d, #0x80000000
; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x80000
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z24.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x1000000
-; CHECK-SVE-NEXT: eor z5.d, z25.d, z5.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x8000
-; CHECK-SVE-NEXT: eor z4.d, z6.d, z4.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x2000000
-; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: eor z6.d, z28.d, z29.d
-; CHECK-SVE-NEXT: mov z28.d, z1.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x100000
-; CHECK-SVE-NEXT: mov z29.d, z1.d
; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z7.d
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x4000000
-; CHECK-SVE-NEXT: mov z7.d, z1.d
+; CHECK-SVE-NEXT: movprfx z3, z1
+; CHECK-SVE-NEXT: and z3.d, z3.d, #0x4000
+; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z26.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x20000
+; CHECK-SVE-NEXT: eor z7.d, z7.d, z27.d
+; CHECK-SVE-NEXT: eor z24.d, z28.d, z29.d
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x400000
+; CHECK-SVE-NEXT: mul z3.d, p0/m, z3.d, z0.d
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z27.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: and z29.d, z29.d, #0x40000000
-; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z26.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x200000
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x20000000
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z5.d, z25.d
-; CHECK-SVE-NEXT: eor z5.d, z6.d, z24.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x80000
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z6.d
+; CHECK-SVE-NEXT: movprfx z6, z1
+; CHECK-SVE-NEXT: and z6.d, z6.d, #0x800000
+; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
+; CHECK-SVE-NEXT: eor z4.d, z7.d, z4.d
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x40000
+; CHECK-SVE-NEXT: eor z5.d, z24.d, z5.d
; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x4000000000
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x80000000
-; CHECK-SVE-NEXT: eor z2.d, z5.d, z28.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mov z28.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x2000000000
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x40000000
+; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x100000000
-; CHECK-SVE-NEXT: eor z24.d, z27.d, z29.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x8000000000
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x8000000
-; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: mov z29.d, z1.d
-; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z4.d
-; CHECK-SVE-NEXT: eor z6.d, z24.d, z6.d
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x200000000
-; CHECK-SVE-NEXT: mov z24.d, z1.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x8000
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.d, z31.d, #0x100000
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.d, z4.d, #0x1000000
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
-; CHECK-SVE-NEXT: and z29.d, z29.d, #0x10000000
+; CHECK-SVE-NEXT: eor z3.d, z5.d, z3.d
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.d, z5.d, #0x20000000
; CHECK-SVE-NEXT: eor z25.d, z25.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: eor z5.d, z6.d, z5.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x400000000
-; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x20000000000
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x10000000000
-; CHECK-SVE-NEXT: eor z4.d, z25.d, z28.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x2000000
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z27.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
+; CHECK-SVE-NEXT: eor z6.d, z27.d, z6.d
+; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
+; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
+; CHECK-SVE-NEXT: eor z7.d, z25.d, z7.d
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x4000000
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x40000000000
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z29.d
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x400000000000
-; CHECK-SVE-NEXT: mov z28.d, z1.d
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x800000000000
-; CHECK-SVE-NEXT: mov z29.d, z1.d
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x100000000
+; CHECK-SVE-NEXT: mul z31.d, p0/m, z31.d, z0.d
+; CHECK-SVE-NEXT: eor z3.d, z3.d, z28.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x400000000
+; CHECK-SVE-NEXT: movprfx z8, z1
+; CHECK-SVE-NEXT: and z8.d, z8.d, #0x800000000000000
+; CHECK-SVE-NEXT: eor z4.d, z6.d, z4.d
+; CHECK-SVE-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x200000
; CHECK-SVE-NEXT: eor z5.d, z5.d, z24.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x8000000
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x400000000000000
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x1000000000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z30.d
-; CHECK-SVE-NEXT: mov z30.d, z1.d
-; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
-; CHECK-SVE-NEXT: and z29.d, z29.d, #0x200000000000
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x80000000000
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x200000000
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
; CHECK-SVE-NEXT: eor z4.d, z4.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z31.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z30.d
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: and z30.d, z30.d, #0x800000000000000
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x10000000
; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x1000000000000
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z5.d, z24.d
-; CHECK-SVE-NEXT: eor z5.d, z25.d, z27.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.d, z30.d, #0x200000000000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z25.d
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x800000000000
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.d, z31.d, #0x400000000000000
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z27.d
+; CHECK-SVE-NEXT: eor z3.d, z6.d, z7.d
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x2000000000000
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x100000000000000
+; CHECK-SVE-NEXT: movprfx z6, z1
+; CHECK-SVE-NEXT: and z6.d, z6.d, #0x800000000
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x2000000000
+; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z24.d
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x4000000000
+; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z29.d
; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x200000000000000
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x100000000000
+; CHECK-SVE-NEXT: movprfx z3, z1
+; CHECK-SVE-NEXT: and z3.d, z3.d, #0x400000000000
+; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x4000000000000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x100000000000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z26.d
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x8000000000000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z27.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x10000000000000
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x8000000000
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z28.d
+; CHECK-SVE-NEXT: movprfx z27, z0
+; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z3.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x1000000000000
+; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
+; CHECK-SVE-NEXT: mul z31.d, p0/m, z31.d, z0.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.d, z4.d, #0x10000000000
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z6.d
-; CHECK-SVE-NEXT: eor z6.d, z24.d, z25.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z29.d
+; CHECK-SVE-NEXT: eor z3.d, z5.d, z6.d
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.d, z5.d, #0x2000000000000
+; CHECK-SVE-NEXT: eor z6.d, z7.d, z24.d
+; CHECK-SVE-NEXT: eor z24.d, z27.d, z25.d
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x20000000000
+; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x4000000000000
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x1000000000
+; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
+; CHECK-SVE-NEXT: mul z8.d, p0/m, z8.d, z0.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z26.d
+; CHECK-SVE-NEXT: eor z24.d, z24.d, z28.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x20000000000000
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x1000000000000000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z28.d
-; CHECK-SVE-NEXT: mov z28.d, z1.d
-; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40000000000
+; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x8000000000000
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z7.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40000000000000
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x2000000000000000
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z30.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z30.d, z1.d
-; CHECK-SVE-NEXT: and z1.d, z1.d, #0x8000000000000000
+; CHECK-SVE-NEXT: eor z4.d, z6.d, z4.d
+; CHECK-SVE-NEXT: eor z5.d, z24.d, z5.d
+; CHECK-SVE-NEXT: movprfx z6, z1
+; CHECK-SVE-NEXT: and z6.d, z6.d, #0x80000000000
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x10000000000000
; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z7.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z27.d
+; CHECK-SVE-NEXT: eor z7.d, z29.d, z30.d
+; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x100000000000
+; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x20000000000000
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.d, z30.d, #0x1000000000000000
+; CHECK-SVE-NEXT: eor z3.d, z3.d, z25.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z26.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z28.d
+; CHECK-SVE-NEXT: eor z7.d, z7.d, z31.d
+; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x200000000000
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x40000000000000
+; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
+; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.d, z31.d, #0x2000000000000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z6.d
; CHECK-SVE-NEXT: eor z5.d, z5.d, z24.d
+; CHECK-SVE-NEXT: eor z6.d, z7.d, z8.d
+; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.d, z7.d, #0x80000000000000
-; CHECK-SVE-NEXT: and z30.d, z30.d, #0x4000000000000000
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z25.d
+; CHECK-SVE-NEXT: mul z31.d, p0/m, z31.d, z0.d
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x4000000000000000
+; CHECK-SVE-NEXT: and z1.d, z1.d, #0x8000000000000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z27.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z29.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z30.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
-; CHECK-SVE-NEXT: eor z4.d, z5.d, z26.d
-; CHECK-SVE-NEXT: eor z5.d, z6.d, z28.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
+; CHECK-SVE-NEXT: ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
; CHECK-SVE-NEXT: mul z0.d, p0/m, z0.d, z1.d
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z26.d
+; CHECK-SVE-NEXT: eor z4.d, z5.d, z28.d
+; CHECK-SVE-NEXT: eor z5.d, z6.d, z31.d
; CHECK-SVE-NEXT: eor z1.d, z2.d, z3.d
; CHECK-SVE-NEXT: eor z2.d, z4.d, z7.d
-; CHECK-SVE-NEXT: eor z3.d, z5.d, z30.d
+; CHECK-SVE-NEXT: eor z3.d, z5.d, z24.d
; CHECK-SVE-NEXT: eor z1.d, z1.d, z2.d
; CHECK-SVE-NEXT: eor z0.d, z3.d, z0.d
; CHECK-SVE-NEXT: eor z0.d, z1.d, z0.d
+; CHECK-SVE-NEXT: addvl sp, sp, #1
+; CHECK-SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SVE-NEXT: ret
;
; CHECK-SVE-AES-LABEL: clmul_nxv2i64:
@@ -1041,225 +1069,225 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv2i64:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
+; CHECK-SME-STREAMING-NEXT: movprfx z2, z1
; CHECK-SME-STREAMING-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x20
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10
; CHECK-SME-STREAMING-NEXT: mul z2.d, z0.d, z2.d
; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
; CHECK-SME-STREAMING-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x10
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20
; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80000000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40000000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100000000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800000000000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400000000000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400000000000000
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000000000000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000000000000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z1.d, z1.d, #0x4000000000000000
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
+; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000000000000000
+; CHECK-SME-STREAMING-NEXT: and z1.d, z1.d, #0x4000000000000000
; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
; CHECK-SME-STREAMING-NEXT: mul z0.d, z0.d, z1.d
@@ -1276,225 +1304,225 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
;
; CHECK-SVE2-LABEL: clmul_nxv2i64:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
+; CHECK-SVE2-NEXT: movprfx z2, z1
; CHECK-SVE2-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SVE2-NEXT: movprfx z3, z1
; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SVE2-NEXT: movprfx z4, z1
; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x20
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10
; CHECK-SVE2-NEXT: mul z2.d, z0.d, z2.d
; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x10
+; CHECK-SVE2-NEXT: movprfx z3, z1
+; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20
; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80000000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40000000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100000000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800000000000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400000000000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400000000000000
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000000000000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000000000000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: and z1.d, z1.d, #0x4000000000000000
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
+; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000000000000000
+; CHECK-SVE2-NEXT: and z1.d, z1.d, #0x4000000000000000
; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
; CHECK-SVE2-NEXT: mul z0.d, z0.d, z1.d
@@ -1521,15 +1549,15 @@ define <vscale x 2 x i64> @clmul_nxv2i64(<vscale x 2 x i64> %x, <vscale x 2 x i6
define <vscale x 16 x i8> @clmul_nxv16i8_zext(<vscale x 16 x i4> %x, <vscale x 16 x i4> %y) {
; CHECK-SVE-LABEL: clmul_nxv16i8_zext:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
; CHECK-SVE-NEXT: and z0.b, z0.b, #0xf
-; CHECK-SVE-NEXT: and z1.b, z1.b, #0x8
-; CHECK-SVE-NEXT: ptrue p0.b
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.b, z2.b, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.b, z3.b, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.b, z4.b, #0x4
+; CHECK-SVE-NEXT: and z1.b, z1.b, #0x8
+; CHECK-SVE-NEXT: ptrue p0.b
; CHECK-SVE-NEXT: mul z2.b, p0/m, z2.b, z0.b
; CHECK-SVE-NEXT: mul z3.b, p0/m, z3.b, z0.b
; CHECK-SVE-NEXT: mul z4.b, p0/m, z4.b, z0.b
@@ -1541,15 +1569,15 @@ define <vscale x 16 x i8> @clmul_nxv16i8_zext(<vscale x 16 x i4> %x, <vscale x 1
;
; CHECK-SVE-AES-LABEL: clmul_nxv16i8_zext:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
; CHECK-SVE-AES-NEXT: and z0.b, z0.b, #0xf
-; CHECK-SVE-AES-NEXT: and z1.b, z1.b, #0x8
-; CHECK-SVE-AES-NEXT: ptrue p0.b
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.b, z2.b, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.b, z3.b, #0x1
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.b, z4.b, #0x4
+; CHECK-SVE-AES-NEXT: and z1.b, z1.b, #0x8
+; CHECK-SVE-AES-NEXT: ptrue p0.b
; CHECK-SVE-AES-NEXT: mul z2.b, p0/m, z2.b, z0.b
; CHECK-SVE-AES-NEXT: mul z3.b, p0/m, z3.b, z0.b
; CHECK-SVE-AES-NEXT: mul z4.b, p0/m, z4.b, z0.b
@@ -1595,21 +1623,21 @@ define <vscale x 16 x i8> @clmul_nxv16i8_zext(<vscale x 16 x i4> %x, <vscale x 1
define <vscale x 8 x i16> @clmul_nxv8i16_zext(<vscale x 8 x i8> %x, <vscale x 8 x i8> %y) {
; CHECK-SVE-LABEL: clmul_nxv8i16_zext:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
; CHECK-SVE-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: ptrue p0.h
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.h, z4.h, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.h, z5.h, #0x8
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.h, z7.h, #0x20
+; CHECK-SVE-NEXT: ptrue p0.h
+; CHECK-SVE-NEXT: movprfx z24, z1
; CHECK-SVE-NEXT: and z24.h, z24.h, #0x40
; CHECK-SVE-NEXT: and z1.h, z1.h, #0x80
; CHECK-SVE-NEXT: mul z2.h, p0/m, z2.h, z0.h
@@ -1631,21 +1659,21 @@ define <vscale x 8 x i16> @clmul_nxv8i16_zext(<vscale x 8 x i8> %x, <vscale x 8
;
; CHECK-SVE-AES-LABEL: clmul_nxv8i16_zext:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
; CHECK-SVE-AES-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
-; CHECK-SVE-AES-NEXT: ptrue p0.h
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.h, z4.h, #0x4
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.h, z5.h, #0x8
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
; CHECK-SVE-AES-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
; CHECK-SVE-AES-NEXT: and z7.h, z7.h, #0x20
+; CHECK-SVE-AES-NEXT: ptrue p0.h
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
; CHECK-SVE-AES-NEXT: and z24.h, z24.h, #0x40
; CHECK-SVE-AES-NEXT: and z1.h, z1.h, #0x80
; CHECK-SVE-AES-NEXT: mul z2.h, p0/m, z2.h, z0.h
@@ -1667,134 +1695,134 @@ define <vscale x 8 x i16> @clmul_nxv8i16_zext(<vscale x 8 x i8> %x, <vscale x 8
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv8i16_zext:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z2, z0
+; CHECK-SME-STREAMING-NEXT: and z2.h, z2.h, #0xff
+; CHECK-SME-STREAMING-NEXT: movprfx z0, z1
+; CHECK-SME-STREAMING-NEXT: and z0.h, z0.h, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
; CHECK-SME-STREAMING-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x20
-; CHECK-SME-STREAMING-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x10
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x40
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SME-STREAMING-NEXT: mul z0.h, z2.h, z0.h
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z2.h, z4.h
+; CHECK-SME-STREAMING-NEXT: mul z5.h, z2.h, z5.h
+; CHECK-SME-STREAMING-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-NEXT: and z3.h, z3.h, #0x20
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SME-STREAMING-NEXT: mul z4.h, z2.h, z6.h
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z3.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z0.d, z2.d
-; CHECK-SME-STREAMING-NEXT: ret
-;
-; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv8i16_zext:
-; CHECK-SME-STREAMING-SSVE-AES: // %bb.0:
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z2.h, z2.h, #0x2
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x1
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x8
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x20
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x10
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z1.h, z1.h, #0x40
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SME-STREAMING-NEXT: and z1.h, z1.h, #0x40
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.h, z2.h, z5.h
+; CHECK-SME-STREAMING-NEXT: mul z1.h, z2.h, z1.h
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
+; CHECK-SME-STREAMING-NEXT: ret
+;
+; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv8i16_zext:
+; CHECK-SME-STREAMING-SSVE-AES: // %bb.0:
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z2, z0
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z2.h, z2.h, #0xff
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z0, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z0.h, z0.h, #0x2
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z4, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x4
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z0.h, z2.h, z0.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z2.h, z4.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z5.h, z2.h, z5.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z3.h, z3.h, #0x20
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z4.h, z2.h, z6.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z3.d, z4.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SME-STREAMING-SSVE-AES-NEXT: mov z0.d, z2.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: and z1.h, z1.h, #0x40
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z3.h, z2.h, z5.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: mul z1.h, z2.h, z1.h
+; CHECK-SME-STREAMING-SSVE-AES-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
; CHECK-SME-STREAMING-SSVE-AES-NEXT: ret
;
; CHECK-SVE2-LABEL: clmul_nxv8i16_zext:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE2-NEXT: movprfx z2, z0
+; CHECK-SVE2-NEXT: and z2.h, z2.h, #0xff
+; CHECK-SVE2-NEXT: movprfx z0, z1
+; CHECK-SVE2-NEXT: and z0.h, z0.h, #0x2
+; CHECK-SVE2-NEXT: movprfx z3, z1
; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE2-NEXT: movprfx z4, z1
; CHECK-SVE2-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x20
-; CHECK-SVE2-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x10
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-NEXT: and z1.h, z1.h, #0x40
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE2-NEXT: mul z0.h, z2.h, z0.h
+; CHECK-SVE2-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SVE2-NEXT: mul z4.h, z2.h, z4.h
+; CHECK-SVE2-NEXT: mul z5.h, z2.h, z5.h
+; CHECK-SVE2-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SVE2-NEXT: movprfx z3, z1
+; CHECK-SVE2-NEXT: and z3.h, z3.h, #0x20
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SVE2-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SVE2-NEXT: mul z4.h, z2.h, z6.h
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z4.d
-; CHECK-SVE2-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-NEXT: and z1.h, z1.h, #0x40
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.h, z2.h, z5.h
+; CHECK-SVE2-NEXT: mul z1.h, z2.h, z1.h
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv8i16_zext:
; CHECK-SVE2-AES: // %bb.0:
-; CHECK-SVE2-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z0.h, z0.h, #0xff
-; CHECK-SVE2-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z2.h, z2.h, #0x2
+; CHECK-SVE2-AES-NEXT: movprfx z2, z0
+; CHECK-SVE2-AES-NEXT: and z2.h, z2.h, #0xff
+; CHECK-SVE2-AES-NEXT: movprfx z0, z1
+; CHECK-SVE2-AES-NEXT: and z0.h, z0.h, #0x2
+; CHECK-SVE2-AES-NEXT: movprfx z3, z1
; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x1
+; CHECK-SVE2-AES-NEXT: movprfx z4, z1
; CHECK-SVE2-AES-NEXT: and z4.h, z4.h, #0x8
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x4
-; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x20
-; CHECK-SVE2-AES-NEXT: mul z2.h, z0.h, z2.h
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z4.h
-; CHECK-SVE2-AES-NEXT: mul z5.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x10
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-AES-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-AES-NEXT: mul z4.h, z0.h, z6.h
-; CHECK-SVE2-AES-NEXT: and z1.h, z1.h, #0x40
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z3.h
+; CHECK-SVE2-AES-NEXT: movprfx z6, z1
+; CHECK-SVE2-AES-NEXT: and z6.h, z6.h, #0x10
+; CHECK-SVE2-AES-NEXT: mul z0.h, z2.h, z0.h
+; CHECK-SVE2-AES-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SVE2-AES-NEXT: mul z4.h, z2.h, z4.h
+; CHECK-SVE2-AES-NEXT: mul z5.h, z2.h, z5.h
+; CHECK-SVE2-AES-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SVE2-AES-NEXT: movprfx z3, z1
+; CHECK-SVE2-AES-NEXT: and z3.h, z3.h, #0x20
+; CHECK-SVE2-AES-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z2.h, z3.h
+; CHECK-SVE2-AES-NEXT: mul z4.h, z2.h, z6.h
+; CHECK-SVE2-AES-NEXT: movprfx z5, z1
; CHECK-SVE2-AES-NEXT: and z5.h, z5.h, #0x80
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z3.d, z4.d
-; CHECK-SVE2-AES-NEXT: mul z3.h, z0.h, z5.h
-; CHECK-SVE2-AES-NEXT: mul z0.h, z0.h, z1.h
-; CHECK-SVE2-AES-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-AES-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-AES-NEXT: and z1.h, z1.h, #0x40
+; CHECK-SVE2-AES-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-AES-NEXT: mul z3.h, z2.h, z5.h
+; CHECK-SVE2-AES-NEXT: mul z1.h, z2.h, z1.h
+; CHECK-SVE2-AES-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
; CHECK-SVE2-AES-NEXT: ret
%zextx = zext <vscale x 8 x i8> %x to <vscale x 8 x i16>
%zexty = zext <vscale x 8 x i8> %y to <vscale x 8 x i16>
@@ -1805,63 +1833,63 @@ define <vscale x 8 x i16> @clmul_nxv8i16_zext(<vscale x 8 x i8> %x, <vscale x 8
define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4 x i16> %y) {
; CHECK-SVE-LABEL: clmul_nxv4i32_zext:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
; CHECK-SVE-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-SVE-NEXT: ptrue p0.s
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.s, z2.s, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.s, z3.s, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.s, z4.s, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.s, z5.s, #0x8
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: ptrue p0.s
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.s, z6.s, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.s, z7.s, #0x20
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.s, z24.s, #0x80
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.s, z26.s, #0x800
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.s, z27.s, #0x1000
; CHECK-SVE-NEXT: mul z2.s, p0/m, z2.s, z0.s
; CHECK-SVE-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-NEXT: and z24.s, z24.s, #0x80
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.s, z28.s, #0x40
; CHECK-SVE-NEXT: mul z4.s, p0/m, z4.s, z0.s
; CHECK-SVE-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.s, z29.s, #0x200
; CHECK-SVE-NEXT: mul z6.s, p0/m, z6.s, z0.s
; CHECK-SVE-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-NEXT: mov z28.d, z1.d
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.s, z30.s, #0x2000
; CHECK-SVE-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-NEXT: and z26.s, z26.s, #0x800
-; CHECK-SVE-NEXT: and z27.s, z27.s, #0x1000
; CHECK-SVE-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-NEXT: mov z29.d, z1.d
-; CHECK-SVE-NEXT: mov z30.d, z1.d
-; CHECK-SVE-NEXT: and z28.s, z28.s, #0x40
-; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-NEXT: mul z26.s, p0/m, z26.s, z0.s
; CHECK-SVE-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: and z29.s, z29.s, #0x200
-; CHECK-SVE-NEXT: and z30.s, z30.s, #0x2000
-; CHECK-SVE-NEXT: mov z5.d, z1.d
+; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-NEXT: mul z28.s, p0/m, z28.s, z0.s
-; CHECK-SVE-NEXT: eor z6.d, z6.d, z7.d
-; CHECK-SVE-NEXT: eor z7.d, z24.d, z25.d
-; CHECK-SVE-NEXT: and z4.s, z4.s, #0x400
-; CHECK-SVE-NEXT: and z1.s, z1.s, #0x8000
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.s, z4.s, #0x400
; CHECK-SVE-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.s, z5.s, #0x4000
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z7.d
+; CHECK-SVE-NEXT: eor z7.d, z24.d, z25.d
+; CHECK-SVE-NEXT: and z1.s, z1.s, #0x8000
; CHECK-SVE-NEXT: eor z24.d, z26.d, z27.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s
+; CHECK-SVE-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-NEXT: mul z0.s, p0/m, z0.s, z1.s
; CHECK-SVE-NEXT: eor z7.d, z24.d, z30.d
; CHECK-SVE-NEXT: eor z1.d, z2.d, z3.d
; CHECK-SVE-NEXT: eor z2.d, z6.d, z4.d
@@ -1873,63 +1901,63 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
;
; CHECK-SVE-AES-LABEL: clmul_nxv4i32_zext:
; CHECK-SVE-AES: // %bb.0:
-; CHECK-SVE-AES-NEXT: mov z2.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z3.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
; CHECK-SVE-AES-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-SVE-AES-NEXT: ptrue p0.s
-; CHECK-SVE-AES-NEXT: mov z6.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z7.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z24.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z2, z1
; CHECK-SVE-AES-NEXT: and z2.s, z2.s, #0x2
+; CHECK-SVE-AES-NEXT: movprfx z3, z1
; CHECK-SVE-AES-NEXT: and z3.s, z3.s, #0x1
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x4
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.s, z5.s, #0x8
-; CHECK-SVE-AES-NEXT: mov z25.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z26.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z27.d, z1.d
+; CHECK-SVE-AES-NEXT: ptrue p0.s
+; CHECK-SVE-AES-NEXT: movprfx z6, z1
; CHECK-SVE-AES-NEXT: and z6.s, z6.s, #0x10
+; CHECK-SVE-AES-NEXT: movprfx z7, z1
; CHECK-SVE-AES-NEXT: and z7.s, z7.s, #0x20
+; CHECK-SVE-AES-NEXT: movprfx z24, z1
+; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x80
+; CHECK-SVE-AES-NEXT: movprfx z25, z1
+; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-AES-NEXT: movprfx z26, z1
+; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x800
+; CHECK-SVE-AES-NEXT: movprfx z27, z1
+; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x1000
; CHECK-SVE-AES-NEXT: mul z2.s, p0/m, z2.s, z0.s
; CHECK-SVE-AES-NEXT: mul z3.s, p0/m, z3.s, z0.s
-; CHECK-SVE-AES-NEXT: and z24.s, z24.s, #0x80
+; CHECK-SVE-AES-NEXT: movprfx z28, z1
+; CHECK-SVE-AES-NEXT: and z28.s, z28.s, #0x40
; CHECK-SVE-AES-NEXT: mul z4.s, p0/m, z4.s, z0.s
; CHECK-SVE-AES-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-AES-NEXT: and z25.s, z25.s, #0x100
+; CHECK-SVE-AES-NEXT: movprfx z29, z1
+; CHECK-SVE-AES-NEXT: and z29.s, z29.s, #0x200
; CHECK-SVE-AES-NEXT: mul z6.s, p0/m, z6.s, z0.s
; CHECK-SVE-AES-NEXT: mul z7.s, p0/m, z7.s, z0.s
-; CHECK-SVE-AES-NEXT: mov z28.d, z1.d
+; CHECK-SVE-AES-NEXT: movprfx z30, z1
+; CHECK-SVE-AES-NEXT: and z30.s, z30.s, #0x2000
; CHECK-SVE-AES-NEXT: mul z24.s, p0/m, z24.s, z0.s
-; CHECK-SVE-AES-NEXT: and z26.s, z26.s, #0x800
-; CHECK-SVE-AES-NEXT: and z27.s, z27.s, #0x1000
; CHECK-SVE-AES-NEXT: mul z25.s, p0/m, z25.s, z0.s
-; CHECK-SVE-AES-NEXT: mov z29.d, z1.d
-; CHECK-SVE-AES-NEXT: mov z30.d, z1.d
-; CHECK-SVE-AES-NEXT: and z28.s, z28.s, #0x40
-; CHECK-SVE-AES-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-AES-NEXT: mul z26.s, p0/m, z26.s, z0.s
; CHECK-SVE-AES-NEXT: mul z27.s, p0/m, z27.s, z0.s
-; CHECK-SVE-AES-NEXT: mov z4.d, z1.d
-; CHECK-SVE-AES-NEXT: and z29.s, z29.s, #0x200
-; CHECK-SVE-AES-NEXT: and z30.s, z30.s, #0x2000
-; CHECK-SVE-AES-NEXT: mov z5.d, z1.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z3.d, z2.d
+; CHECK-SVE-AES-NEXT: eor z3.d, z4.d, z5.d
; CHECK-SVE-AES-NEXT: mul z28.s, p0/m, z28.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z7.d
-; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z25.d
-; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x400
-; CHECK-SVE-AES-NEXT: and z1.s, z1.s, #0x8000
-; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-AES-NEXT: mul z29.s, p0/m, z29.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z4, z1
+; CHECK-SVE-AES-NEXT: and z4.s, z4.s, #0x400
; CHECK-SVE-AES-NEXT: mul z30.s, p0/m, z30.s, z0.s
+; CHECK-SVE-AES-NEXT: movprfx z5, z1
; CHECK-SVE-AES-NEXT: and z5.s, z5.s, #0x4000
+; CHECK-SVE-AES-NEXT: eor z6.d, z6.d, z7.d
+; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z25.d
+; CHECK-SVE-AES-NEXT: and z1.s, z1.s, #0x8000
; CHECK-SVE-AES-NEXT: eor z24.d, z26.d, z27.d
+; CHECK-SVE-AES-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-AES-NEXT: mul z4.s, p0/m, z4.s, z0.s
-; CHECK-SVE-AES-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-AES-NEXT: mul z5.s, p0/m, z5.s, z0.s
-; CHECK-SVE-AES-NEXT: mul z0.s, p0/m, z0.s, z1.s
+; CHECK-SVE-AES-NEXT: eor z3.d, z6.d, z28.d
; CHECK-SVE-AES-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-AES-NEXT: mul z0.s, p0/m, z0.s, z1.s
; CHECK-SVE-AES-NEXT: eor z7.d, z24.d, z30.d
; CHECK-SVE-AES-NEXT: eor z1.d, z2.d, z3.d
; CHECK-SVE-AES-NEXT: eor z2.d, z6.d, z4.d
@@ -1983,134 +2011,143 @@ define <vscale x 4 x i32> @clmul_nxv4i32_zext(<vscale x 4 x i16> %x, <vscale x 4
define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2 x i32> %y) {
; CHECK-SVE-LABEL: clmul_nxv2i64_zext:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: mov z2.d, z1.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
-; CHECK-SVE-NEXT: mov z4.d, z1.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
+; CHECK-SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-SVE-NEXT: addvl sp, sp, #-1
+; CHECK-SVE-NEXT: str z8, [sp] // 16-byte Folded Spill
+; CHECK-SVE-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
+; CHECK-SVE-NEXT: .cfi_offset w29, -16
+; CHECK-SVE-NEXT: .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
; CHECK-SVE-NEXT: and z0.d, z0.d, #0xffffffff
-; CHECK-SVE-NEXT: ptrue p0.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
+; CHECK-SVE-NEXT: movprfx z2, z1
; CHECK-SVE-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SVE-NEXT: movprfx z3, z1
; CHECK-SVE-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SVE-NEXT: movprfx z4, z1
; CHECK-SVE-NEXT: and z4.d, z4.d, #0x4
+; CHECK-SVE-NEXT: movprfx z5, z1
; CHECK-SVE-NEXT: and z5.d, z5.d, #0x8
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
+; CHECK-SVE-NEXT: movprfx z6, z1
; CHECK-SVE-NEXT: and z6.d, z6.d, #0x10
+; CHECK-SVE-NEXT: movprfx z7, z1
; CHECK-SVE-NEXT: and z7.d, z7.d, #0x20
+; CHECK-SVE-NEXT: ptrue p0.d
+; CHECK-SVE-NEXT: movprfx z24, z1
; CHECK-SVE-NEXT: and z24.d, z24.d, #0x80
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x100
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x200
; CHECK-SVE-NEXT: mul z2.d, p0/m, z2.d, z0.d
; CHECK-SVE-NEXT: mul z3.d, p0/m, z3.d, z0.d
-; CHECK-SVE-NEXT: mov z27.d, z1.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x20000
; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x100
+; CHECK-SVE-NEXT: movprfx z30, z1
+; CHECK-SVE-NEXT: and z30.d, z30.d, #0x40000
; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40
+; CHECK-SVE-NEXT: movprfx z31, z1
+; CHECK-SVE-NEXT: and z31.d, z31.d, #0x1000000
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x200
-; CHECK-SVE-NEXT: mov z28.d, z1.d
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE-NEXT: mov z29.d, z1.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x400
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
+; CHECK-SVE-NEXT: eor z2.d, z3.d, z2.d
; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
+; CHECK-SVE-NEXT: movprfx z5, z1
+; CHECK-SVE-NEXT: and z5.d, z5.d, #0x800
+; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
; CHECK-SVE-NEXT: eor z4.d, z6.d, z7.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z28.d, z28.d, #0x8000
-; CHECK-SVE-NEXT: and z29.d, z29.d, #0x100000
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SVE-NEXT: eor z6.d, z24.d, z25.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
+; CHECK-SVE-NEXT: movprfx z6, z1
+; CHECK-SVE-NEXT: and z6.d, z6.d, #0x1000
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x10000
+; CHECK-SVE-NEXT: eor z24.d, z24.d, z25.d
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x400000
+; CHECK-SVE-NEXT: mul z30.d, p0/m, z30.d, z0.d
; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z26.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x800
-; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
-; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: eor z4.d, z6.d, z27.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x1000
+; CHECK-SVE-NEXT: mul z31.d, p0/m, z31.d, z0.d
+; CHECK-SVE-NEXT: eor z3.d, z4.d, z26.d
+; CHECK-SVE-NEXT: movprfx z4, z1
+; CHECK-SVE-NEXT: and z4.d, z4.d, #0x800000
+; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
+; CHECK-SVE-NEXT: movprfx z26, z1
+; CHECK-SVE-NEXT: and z26.d, z26.d, #0x2000
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x800000
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z4.d, z5.d
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x2000
-; CHECK-SVE-NEXT: mov z5.d, z1.d
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: and z26.d, z26.d, #0x40000
-; CHECK-SVE-NEXT: and z27.d, z27.d, #0x1000000
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
+; CHECK-SVE-NEXT: movprfx z8, z1
+; CHECK-SVE-NEXT: and z8.d, z8.d, #0x2000000
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: mul z4.d, p0/m, z4.d, z0.d
; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
-; CHECK-SVE-NEXT: mov z3.d, z1.d
+; CHECK-SVE-NEXT: eor z3.d, z24.d, z27.d
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
-; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x20000
-; CHECK-SVE-NEXT: eor z4.d, z7.d, z24.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
-; CHECK-SVE-NEXT: and z3.d, z3.d, #0x10000
-; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x400000
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
-; CHECK-SVE-NEXT: mul z3.d, p0/m, z3.d, z0.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x4000
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x4000
+; CHECK-SVE-NEXT: movprfx z27, z1
+; CHECK-SVE-NEXT: and z27.d, z27.d, #0x80000
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z6.d
+; CHECK-SVE-NEXT: eor z6.d, z7.d, z29.d
+; CHECK-SVE-NEXT: movprfx z7, z1
+; CHECK-SVE-NEXT: and z7.d, z7.d, #0x8000
+; CHECK-SVE-NEXT: mul z8.d, p0/m, z8.d, z0.d
+; CHECK-SVE-NEXT: movprfx z29, z1
+; CHECK-SVE-NEXT: and z29.d, z29.d, #0x4000000
+; CHECK-SVE-NEXT: eor z3.d, z3.d, z28.d
+; CHECK-SVE-NEXT: eor z4.d, z25.d, z4.d
; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x80000
+; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
+; CHECK-SVE-NEXT: movprfx z25, z1
+; CHECK-SVE-NEXT: and z25.d, z25.d, #0x100000
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z26.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z30.d
; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z5.d
-; CHECK-SVE-NEXT: mov z5.d, z1.d
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: eor z24.d, z24.d, z25.d
-; CHECK-SVE-NEXT: mov z25.d, z1.d
-; CHECK-SVE-NEXT: and z5.d, z5.d, #0x2000000
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z26.d
-; CHECK-SVE-NEXT: mov z26.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z7.d
-; CHECK-SVE-NEXT: and z25.d, z25.d, #0x4000000
-; CHECK-SVE-NEXT: eor z7.d, z24.d, z27.d
-; CHECK-SVE-NEXT: mov z24.d, z1.d
-; CHECK-SVE-NEXT: mul z5.d, p0/m, z5.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z6.d
-; CHECK-SVE-NEXT: mov z6.d, z1.d
+; CHECK-SVE-NEXT: movprfx z26, z1
; CHECK-SVE-NEXT: and z26.d, z26.d, #0x200000
-; CHECK-SVE-NEXT: mov z27.d, z1.d
-; CHECK-SVE-NEXT: eor z4.d, z4.d, z28.d
+; CHECK-SVE-NEXT: mul z29.d, p0/m, z29.d, z0.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z31.d
+; CHECK-SVE-NEXT: movprfx z28, z1
+; CHECK-SVE-NEXT: and z28.d, z28.d, #0x8000000
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z3.d
; CHECK-SVE-NEXT: mul z25.d, p0/m, z25.d, z0.d
-; CHECK-SVE-NEXT: and z24.d, z24.d, #0x20000000
-; CHECK-SVE-NEXT: and z6.d, z6.d, #0x8000000
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z29.d
-; CHECK-SVE-NEXT: eor z5.d, z7.d, z5.d
-; CHECK-SVE-NEXT: mov z7.d, z1.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z24.d
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z27.d
; CHECK-SVE-NEXT: mul z26.d, p0/m, z26.d, z0.d
+; CHECK-SVE-NEXT: movprfx z3, z1
+; CHECK-SVE-NEXT: and z3.d, z3.d, #0x10000000
+; CHECK-SVE-NEXT: movprfx z24, z1
+; CHECK-SVE-NEXT: and z24.d, z24.d, #0x20000000
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z8.d
+; CHECK-SVE-NEXT: mul z28.d, p0/m, z28.d, z0.d
+; CHECK-SVE-NEXT: movprfx z27, z1
; CHECK-SVE-NEXT: and z27.d, z27.d, #0x40000000
-; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
+; CHECK-SVE-NEXT: eor z5.d, z5.d, z7.d
; CHECK-SVE-NEXT: and z1.d, z1.d, #0x80000000
-; CHECK-SVE-NEXT: mul z6.d, p0/m, z6.d, z0.d
-; CHECK-SVE-NEXT: eor z2.d, z2.d, z4.d
-; CHECK-SVE-NEXT: and z7.d, z7.d, #0x10000000
-; CHECK-SVE-NEXT: eor z5.d, z5.d, z25.d
+; CHECK-SVE-NEXT: ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-SVE-NEXT: eor z6.d, z6.d, z25.d
+; CHECK-SVE-NEXT: mul z3.d, p0/m, z3.d, z0.d
+; CHECK-SVE-NEXT: mul z24.d, p0/m, z24.d, z0.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z29.d
; CHECK-SVE-NEXT: mul z27.d, p0/m, z27.d, z0.d
-; CHECK-SVE-NEXT: eor z3.d, z3.d, z26.d
-; CHECK-SVE-NEXT: mul z7.d, p0/m, z7.d, z0.d
-; CHECK-SVE-NEXT: eor z4.d, z5.d, z6.d
+; CHECK-SVE-NEXT: eor z2.d, z2.d, z5.d
; CHECK-SVE-NEXT: mul z0.d, p0/m, z0.d, z1.d
-; CHECK-SVE-NEXT: eor z1.d, z2.d, z3.d
+; CHECK-SVE-NEXT: eor z5.d, z6.d, z26.d
+; CHECK-SVE-NEXT: eor z4.d, z4.d, z28.d
+; CHECK-SVE-NEXT: eor z1.d, z2.d, z5.d
+; CHECK-SVE-NEXT: eor z2.d, z4.d, z3.d
; CHECK-SVE-NEXT: eor z3.d, z24.d, z27.d
-; CHECK-SVE-NEXT: eor z2.d, z4.d, z7.d
-; CHECK-SVE-NEXT: eor z0.d, z3.d, z0.d
; CHECK-SVE-NEXT: eor z1.d, z1.d, z2.d
+; CHECK-SVE-NEXT: eor z0.d, z3.d, z0.d
; CHECK-SVE-NEXT: eor z0.d, z1.d, z0.d
+; CHECK-SVE-NEXT: addvl sp, sp, #1
+; CHECK-SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
; CHECK-SVE-NEXT: ret
;
; CHECK-SVE-AES-LABEL: clmul_nxv2i64_zext:
@@ -2124,119 +2161,119 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
;
; CHECK-SME-STREAMING-LABEL: clmul_nxv2i64_zext:
; CHECK-SME-STREAMING: // %bb.0:
-; CHECK-SME-STREAMING-NEXT: mov z2.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z0.d, z0.d, #0xffffffff
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z2, z0
+; CHECK-SME-STREAMING-NEXT: and z2.d, z2.d, #0xffffffff
+; CHECK-SME-STREAMING-NEXT: movprfx z0, z1
+; CHECK-SME-STREAMING-NEXT: and z0.d, z0.d, #0x2
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SME-STREAMING-NEXT: movprfx z4, z1
; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x20
-; CHECK-SME-STREAMING-NEXT: mul z2.d, z0.d, z2.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x10
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10
+; CHECK-SME-STREAMING-NEXT: mul z0.d, z2.d, z0.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z4.d
+; CHECK-SME-STREAMING-NEXT: mul z5.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SME-STREAMING-NEXT: movprfx z3, z1
+; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x200000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x40000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x200000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x2000000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x800000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x400000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x2000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z4.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SME-STREAMING-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z3.d, z1.d
-; CHECK-SME-STREAMING-NEXT: mov z6.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z3.d, z3.d, #0x20000000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x8000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
+; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x4000000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
+; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x20000000
+; CHECK-SME-STREAMING-NEXT: movprfx z6, z1
; CHECK-SME-STREAMING-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SME-STREAMING-NEXT: mov z5.d, z1.d
-; CHECK-SME-STREAMING-NEXT: and z1.d, z1.d, #0x40000000
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SME-STREAMING-NEXT: movprfx z5, z1
; CHECK-SME-STREAMING-NEXT: and z5.d, z5.d, #0x80000000
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mul z3.d, z0.d, z5.d
-; CHECK-SME-STREAMING-NEXT: mul z0.d, z0.d, z1.d
-; CHECK-SME-STREAMING-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SME-STREAMING-NEXT: mov z0.d, z2.d
+; CHECK-SME-STREAMING-NEXT: and z1.d, z1.d, #0x40000000
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SME-STREAMING-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SME-STREAMING-NEXT: mul z1.d, z2.d, z1.d
+; CHECK-SME-STREAMING-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
; CHECK-SME-STREAMING-NEXT: ret
;
; CHECK-SME-STREAMING-SSVE-AES-LABEL: clmul_nxv2i64_zext:
@@ -2250,119 +2287,119 @@ define <vscale x 2 x i64> @clmul_nxv2i64_zext(<vscale x 2 x i32> %x, <vscale x 2
;
; CHECK-SVE2-LABEL: clmul_nxv2i64_zext:
; CHECK-SVE2: // %bb.0:
-; CHECK-SVE2-NEXT: mov z2.d, z1.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z0.d, z0.d, #0xffffffff
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z2.d, z2.d, #0x2
+; CHECK-SVE2-NEXT: movprfx z2, z0
+; CHECK-SVE2-NEXT: and z2.d, z2.d, #0xffffffff
+; CHECK-SVE2-NEXT: movprfx z0, z1
+; CHECK-SVE2-NEXT: and z0.d, z0.d, #0x2
+; CHECK-SVE2-NEXT: movprfx z3, z1
; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x1
+; CHECK-SVE2-NEXT: movprfx z4, z1
; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4
-; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x20
-; CHECK-SVE2-NEXT: mul z2.d, z0.d, z2.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: eor z2.d, z3.d, z2.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x10
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z3.d, z6.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10
+; CHECK-SVE2-NEXT: mul z0.d, z2.d, z0.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z3.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z4.d
+; CHECK-SVE2-NEXT: mul z5.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: eor z0.d, z3.d, z0.d
+; CHECK-SVE2-NEXT: movprfx z3, z1
+; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z5.d, z4.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z3.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x80000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x40000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x200000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x40000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x200000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x100000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x800000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x400000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x2000000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x800000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x400000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x2000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x1000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z4.d, z1.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z6.d, z0.d, z6.d
-; CHECK-SVE2-NEXT: and z4.d, z4.d, #0x8000000
-; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x4000000
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z4.d
-; CHECK-SVE2-NEXT: mul z5.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z6.d, z3.d
-; CHECK-SVE2-NEXT: mov z3.d, z1.d
-; CHECK-SVE2-NEXT: mov z6.d, z1.d
-; CHECK-SVE2-NEXT: and z3.d, z3.d, #0x20000000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x8000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
+; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x4000000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
+; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x20000000
+; CHECK-SVE2-NEXT: movprfx z6, z1
; CHECK-SVE2-NEXT: and z6.d, z6.d, #0x10000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z5.d, z4.d
-; CHECK-SVE2-NEXT: mov z5.d, z1.d
-; CHECK-SVE2-NEXT: and z1.d, z1.d, #0x40000000
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mul z4.d, z0.d, z6.d
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z4.d, z2.d, z6.d
+; CHECK-SVE2-NEXT: movprfx z5, z1
; CHECK-SVE2-NEXT: and z5.d, z5.d, #0x80000000
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z4.d, z3.d
-; CHECK-SVE2-NEXT: mul z3.d, z0.d, z5.d
-; CHECK-SVE2-NEXT: mul z0.d, z0.d, z1.d
-; CHECK-SVE2-NEXT: eor3 z2.d, z2.d, z0.d, z3.d
-; CHECK-SVE2-NEXT: mov z0.d, z2.d
+; CHECK-SVE2-NEXT: and z1.d, z1.d, #0x40000000
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z4.d, z3.d
+; CHECK-SVE2-NEXT: mul z3.d, z2.d, z5.d
+; CHECK-SVE2-NEXT: mul z1.d, z2.d, z1.d
+; CHECK-SVE2-NEXT: eor3 z0.d, z0.d, z1.d, z3.d
; CHECK-SVE2-NEXT: ret
;
; CHECK-SVE2-AES-LABEL: clmul_nxv2i64_zext:
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll
index 258eaabee9376..6b164d2531327 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-contract.ll
@@ -202,29 +202,30 @@ entry:
define <vscale x 4 x double> @mul_add_rot_mull(<vscale x 4 x double> %a, <vscale x 4 x double> %b, <vscale x 4 x double> %c, <vscale x 4 x double> %d) {
; CHECK-LABEL: mul_add_rot_mull:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: uzp2 z24.d, z4.d, z5.d
-; CHECK-NEXT: movi v25.2d, #0000000000000000
-; CHECK-NEXT: uzp1 z4.d, z4.d, z5.d
+; CHECK-NEXT: movi v24.2d, #0000000000000000
+; CHECK-NEXT: uzp1 z25.d, z4.d, z5.d
+; CHECK-NEXT: uzp2 z4.d, z4.d, z5.d
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: mov z26.d, z24.d
-; CHECK-NEXT: and z25.d, z25.d, #0x7fffffffffffffff
+; CHECK-NEXT: movprfx z5, z4
+; CHECK-NEXT: and z5.d, z5.d, #0x8000000000000000
+; CHECK-NEXT: movprfx z26, z25
; CHECK-NEXT: and z26.d, z26.d, #0x8000000000000000
-; CHECK-NEXT: orr z5.d, z25.d, z26.d
-; CHECK-NEXT: fadd z5.d, z4.d, z5.d
-; CHECK-NEXT: and z4.d, z4.d, #0x8000000000000000
-; CHECK-NEXT: orr z4.d, z25.d, z4.d
-; CHECK-NEXT: uzp2 z25.d, z0.d, z1.d
+; CHECK-NEXT: and z24.d, z24.d, #0x7fffffffffffffff
+; CHECK-NEXT: orr z5.d, z24.d, z5.d
+; CHECK-NEXT: orr z24.d, z24.d, z26.d
+; CHECK-NEXT: uzp2 z26.d, z0.d, z1.d
; CHECK-NEXT: uzp1 z0.d, z0.d, z1.d
; CHECK-NEXT: uzp2 z1.d, z2.d, z3.d
; CHECK-NEXT: uzp1 z2.d, z2.d, z3.d
-; CHECK-NEXT: fsub z4.d, z4.d, z24.d
+; CHECK-NEXT: fadd z5.d, z25.d, z5.d
+; CHECK-NEXT: fsub z4.d, z24.d, z4.d
; CHECK-NEXT: uzp2 z24.d, z6.d, z7.d
+; CHECK-NEXT: fmul z25.d, z0.d, z1.d
+; CHECK-NEXT: fmul z1.d, z26.d, z1.d
; CHECK-NEXT: uzp1 z6.d, z6.d, z7.d
-; CHECK-NEXT: fmul z26.d, z0.d, z1.d
-; CHECK-NEXT: fmul z1.d, z25.d, z1.d
; CHECK-NEXT: fmul z3.d, z4.d, z24.d
; CHECK-NEXT: fmul z24.d, z5.d, z24.d
-; CHECK-NEXT: fmad z25.d, p0/m, z2.d, z26.d
+; CHECK-NEXT: fmla z25.d, p0/m, z26.d, z2.d
; CHECK-NEXT: fnmsb z0.d, p0/m, z2.d, z1.d
; CHECK-NEXT: fmla z3.d, p0/m, z6.d, z5.d
; CHECK-NEXT: fnmsb z4.d, p0/m, z6.d, z24.d
diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
index b9ffab24d86b0..6b4ebbe2be5ae 100644
--- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
@@ -1082,11 +1082,11 @@ define <vscale x 2 x i64> @fshl_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
; CHECK-LABEL: fshl_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z3.d, #63 // =0x3f
-; CHECK-NEXT: mov z4.d, z2.d
+; CHECK-NEXT: movprfx z4, z2
+; CHECK-NEXT: and z4.d, z4.d, #0x3f
; CHECK-NEXT: lsr z1.d, z1.d, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: bic z2.d, z3.d, z2.d
-; CHECK-NEXT: and z4.d, z4.d, #0x3f
; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d
; CHECK-NEXT: lsr z1.d, p0/m, z1.d, z2.d
; CHECK-NEXT: orr z0.d, z0.d, z1.d
@@ -1099,17 +1099,18 @@ define <vscale x 4 x i64> @fshl_illegal_i64(<vscale x 4 x i64> %a, <vscale x 4 x
; CHECK-LABEL: fshl_illegal_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z6.d, #63 // =0x3f
+; CHECK-NEXT: movprfx z7, z4
+; CHECK-NEXT: and z7.d, z7.d, #0x3f
; CHECK-NEXT: lsr z2.d, z2.d, #1
; CHECK-NEXT: lsr z3.d, z3.d, #1
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: bic z7.d, z6.d, z4.d
-; CHECK-NEXT: and z4.d, z4.d, #0x3f
+; CHECK-NEXT: bic z4.d, z6.d, z4.d
; CHECK-NEXT: bic z6.d, z6.d, z5.d
; CHECK-NEXT: and z5.d, z5.d, #0x3f
-; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z4.d
-; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z7.d
-; CHECK-NEXT: lsr z3.d, p0/m, z3.d, z6.d
+; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z7.d
+; CHECK-NEXT: lsr z2.d, p0/m, z2.d, z4.d
; CHECK-NEXT: lsl z1.d, p0/m, z1.d, z5.d
+; CHECK-NEXT: lsr z3.d, p0/m, z3.d, z6.d
; CHECK-NEXT: orr z0.d, z0.d, z2.d
; CHECK-NEXT: orr z1.d, z1.d, z3.d
; CHECK-NEXT: ret
@@ -1172,11 +1173,11 @@ define <vscale x 2 x i64> @fshr_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b
; CHECK-LABEL: fshr_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: mov z3.d, #63 // =0x3f
-; CHECK-NEXT: mov z4.d, z2.d
+; CHECK-NEXT: movprfx z4, z2
+; CHECK-NEXT: and z4.d, z4.d, #0x3f
; CHECK-NEXT: lsl z0.d, z0.d, #1
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: bic z2.d, z3.d, z2.d
-; CHECK-NEXT: and z4.d, z4.d, #0x3f
; CHECK-NEXT: lsr z1.d, p0/m, z1.d, z4.d
; CHECK-NEXT: lsl z0.d, p0/m, z0.d, z2.d
; CHECK-NEXT: orr z0.d, z0.d, z1.d
diff --git a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
index 3bbcaca7468b1..5e4ccf999da7d 100644
--- a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
+++ b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
@@ -836,12 +836,12 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 {
; CHECK-NEXT: stp x30, x19, [sp, #80] // 16-byte Folded Spill
; CHECK-NEXT: sub sp, sp, #16
; CHECK-NEXT: addvl sp, sp, #-1
-; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: and z1.b, z1.b, #0x1
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: add x8, sp, #16
; CHECK-NEXT: mov x19, x0
; CHECK-NEXT: str d0, [sp, #8] // 8-byte Spill
-; CHECK-NEXT: and z1.b, z1.b, #0x1
; CHECK-NEXT: cmpne p1.b, p0/z, z1.b, #0
; CHECK-NEXT: str p1, [x8, #7, mul vl] // 2-byte Spill
; CHECK-NEXT: smstop sm
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
index 73d043b411696..66068e86ce8a7 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-dag-combine.ll
@@ -7,9 +7,10 @@
define <vscale x 2 x i64> @no_dag_combine_zext_sext(<vscale x 2 x i1> %pg,
; CHECK-LABEL: no_dag_combine_zext_sext:
; CHECK: // %bb.0:
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d, #16]
-; CHECK-NEXT: st1b { z0.d }, p1, [x0]
+; CHECK-NEXT: ld1b { z1.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: movprfx z0, z1
; CHECK-NEXT: and z0.d, z0.d, #0xff
+; CHECK-NEXT: st1b { z1.d }, p1, [x0]
; CHECK-NEXT: ret
<vscale x 2 x i64> %base,
ptr %res_out,
@@ -54,9 +55,10 @@ define <vscale x 2 x i64> @no_dag_combine_sext(<vscale x 2 x i1> %pg,
define <vscale x 2 x i64> @no_dag_combine_zext(<vscale x 2 x i1> %pg,
; CHECK-LABEL: no_dag_combine_zext:
; CHECK: // %bb.0:
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [z0.d, #16]
-; CHECK-NEXT: st1b { z0.d }, p1, [x0]
+; CHECK-NEXT: ld1b { z1.d }, p0/z, [z0.d, #16]
+; CHECK-NEXT: movprfx z0, z1
; CHECK-NEXT: and z0.d, z0.d, #0xff
+; CHECK-NEXT: st1b { z1.d }, p1, [x0]
; CHECK-NEXT: ret
<vscale x 2 x i64> %base,
ptr %res_out,
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
index 71d5bad97f2d7..bce17b8566922 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-sext.ll
@@ -291,10 +291,10 @@ define <vscale x 2 x i64> @load_frozen_before_sext_multiuse2(ptr %src) {
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: sxtb z1.d, p0/m, z0.d
-; CHECK-NEXT: mul z0.d, z0.d, #13
-; CHECK-NEXT: and z0.d, z0.d, #0xff
-; CHECK-NEXT: add z0.d, z1.d, z0.d
+; CHECK-NEXT: mul z1.d, z1.d, #13
+; CHECK-NEXT: sxtb z0.d, p0/m, z0.d
+; CHECK-NEXT: and z1.d, z1.d, #0xff
+; CHECK-NEXT: add z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%load = load <vscale x 2 x i8>, ptr %src, align 1
%load.frozen = freeze <vscale x 2 x i8> %load
@@ -310,16 +310,16 @@ define <vscale x 2 x i64> @load_frozen_before_sext_multiuse3(ptr %src, ptr %dst1
; CHECK-LABEL: load_frozen_before_sext_multiuse3:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0]
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: sxtb z0.d, p0/m, z1.d
+; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
+; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: and z1.d, z1.d, #0xff
-; CHECK-NEXT: mov z2.d, z1.d
-; CHECK-NEXT: mul z1.d, z1.d, #49
-; CHECK-NEXT: mul z2.d, z2.d, #33
-; CHECK-NEXT: add z0.d, z0.d, z1.d
-; CHECK-NEXT: st1w { z2.d }, p0, [x1]
-; CHECK-NEXT: st1h { z1.d }, p0, [x2]
+; CHECK-NEXT: sxtb z0.d, p0/m, z0.d
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: mul z2.d, z2.d, #49
+; CHECK-NEXT: mul z1.d, z1.d, #33
+; CHECK-NEXT: add z0.d, z0.d, z2.d
+; CHECK-NEXT: st1w { z1.d }, p0, [x1]
+; CHECK-NEXT: st1h { z2.d }, p0, [x2]
; CHECK-NEXT: ret
%load = load <vscale x 2 x i8>, ptr %src, align 1
%load.frozen = freeze <vscale x 2 x i8> %load
@@ -346,11 +346,11 @@ define <vscale x 2 x i64> @load_frozen_before_sext_multiuse4(ptr %src1, ptr %src
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
; CHECK-NEXT: ld1b { z2.d }, p0/z, [x1]
; CHECK-NEXT: movprfx z1, z0
-; CHECK-NEXT: sxtb z1.d, p0/m, z0.d
-; CHECK-NEXT: and z0.d, z0.d, #0xff
-; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, #3
-; CHECK-NEXT: mov z0.d, x8
-; CHECK-NEXT: add z0.d, z1.d, z0.d
+; CHECK-NEXT: and z1.d, z1.d, #0xff
+; CHECK-NEXT: sxtb z0.d, p0/m, z0.d
+; CHECK-NEXT: cmpeq p1.d, p0/z, z1.d, #3
+; CHECK-NEXT: mov z1.d, x8
+; CHECK-NEXT: add z0.d, z0.d, z1.d
; CHECK-NEXT: sel z1.d, p1, z2.d, z3.d
; CHECK-NEXT: st1b { z1.d }, p0, [x2]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
index 93d9f974fdbd7..f862479943186 100644
--- a/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-ldst-zext.ll
@@ -272,8 +272,8 @@ define <vscale x 2 x i64> @load_frozen_before_zext_multiuse(ptr %src) {
; CHECK-LABEL: load_frozen_before_zext_multiuse:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
-; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: ld1b { z1.d }, p0/z, [x0]
+; CHECK-NEXT: movprfx z0, z1
; CHECK-NEXT: and z0.d, z0.d, #0xff
; CHECK-NEXT: // fake_use: $z1
; CHECK-NEXT: ret
@@ -290,11 +290,11 @@ define <vscale x 2 x i64> @load_frozen_before_zext_multiuse2(ptr %src) {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0]
-; CHECK-NEXT: mov z1.d, z0.d
-; CHECK-NEXT: mul z0.d, z0.d, #13
-; CHECK-NEXT: and z1.d, z1.d, #0xff
-; CHECK-NEXT: sxtb z0.d, p0/m, z0.d
-; CHECK-NEXT: add z0.d, z1.d, z0.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: mul z1.d, z1.d, #13
+; CHECK-NEXT: and z0.d, z0.d, #0xff
+; CHECK-NEXT: sxtb z1.d, p0/m, z1.d
+; CHECK-NEXT: add z0.d, z0.d, z1.d
; CHECK-NEXT: ret
%load = load <vscale x 2 x i8>, ptr %src, align 1
%load.frozen = freeze <vscale x 2 x i8> %load
@@ -314,12 +314,12 @@ define <vscale x 2 x i64> @load_frozen_before_sext_multiuse3(ptr %src, ptr %dst1
; CHECK-NEXT: movprfx z1, z0
; CHECK-NEXT: sxtb z1.d, p0/m, z0.d
; CHECK-NEXT: and z0.d, z0.d, #0xff
-; CHECK-NEXT: mov z2.d, z1.d
-; CHECK-NEXT: mul z1.d, z1.d, #49
-; CHECK-NEXT: mul z2.d, z2.d, #33
-; CHECK-NEXT: add z0.d, z0.d, z1.d
-; CHECK-NEXT: st1w { z2.d }, p0, [x1]
-; CHECK-NEXT: st1h { z1.d }, p0, [x2]
+; CHECK-NEXT: movprfx z2, z1
+; CHECK-NEXT: mul z2.d, z2.d, #49
+; CHECK-NEXT: mul z1.d, z1.d, #33
+; CHECK-NEXT: add z0.d, z0.d, z2.d
+; CHECK-NEXT: st1w { z1.d }, p0, [x1]
+; CHECK-NEXT: st1h { z2.d }, p0, [x2]
; CHECK-NEXT: ret
%load = load <vscale x 2 x i8>, ptr %src, align 1
%load.frozen = freeze <vscale x 2 x i8> %load
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
index 5d58d8992694a..adc95e9515e2e 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -31,10 +31,11 @@ define <vscale x 2 x i64> @masked_sgather_zext(ptr %base, <vscale x 2 x i64> %of
; CHECK: // %bb.0:
; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, z0.d]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: add z1.d, z0.d, z1.d
+; CHECK-NEXT: movprfx z2, z0
+; CHECK-NEXT: and z2.d, z2.d, #0xff
+; CHECK-NEXT: add z0.d, z0.d, z1.d
; CHECK-NEXT: and z0.d, z0.d, #0xff
-; CHECK-NEXT: and z1.d, z1.d, #0xff
-; CHECK-NEXT: mul z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: mul z0.d, p0/m, z0.d, z2.d
; CHECK-NEXT: ret
%ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets
%data = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)
diff --git a/llvm/test/CodeGen/AArch64/sve-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
index 54dd83eaec507..5f143743fc2f4 100644
--- a/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-sext-zext.ll
@@ -332,10 +332,10 @@ define <vscale x 16 x i64> @zext_b_to_d(<vscale x 16 x i8> %a) {
define <vscale x 4 x i64> @zext_4i8_4i64(<vscale x 4 x i8> %aval) {
; CHECK-LABEL: zext_4i8_4i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z0.s, z0.s, #0xff
-; CHECK-NEXT: uunpklo z2.d, z0.s
-; CHECK-NEXT: uunpkhi z1.d, z0.s
-; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: and z1.s, z1.s, #0xff
+; CHECK-NEXT: uunpklo z0.d, z1.s
+; CHECK-NEXT: uunpkhi z1.d, z1.s
; CHECK-NEXT: ret
%aext = zext <vscale x 4 x i8> %aval to <vscale x 4 x i64>
ret <vscale x 4 x i64> %aext
@@ -344,10 +344,10 @@ define <vscale x 4 x i64> @zext_4i8_4i64(<vscale x 4 x i8> %aval) {
define <vscale x 4 x i64> @zext_4i16_4i64(<vscale x 4 x i16> %aval) {
; CHECK-LABEL: zext_4i16_4i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z0.s, z0.s, #0xffff
-; CHECK-NEXT: uunpklo z2.d, z0.s
-; CHECK-NEXT: uunpkhi z1.d, z0.s
-; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: and z1.s, z1.s, #0xffff
+; CHECK-NEXT: uunpklo z0.d, z1.s
+; CHECK-NEXT: uunpkhi z1.d, z1.s
; CHECK-NEXT: ret
%aext = zext <vscale x 4 x i16> %aval to <vscale x 4 x i64>
ret <vscale x 4 x i64> %aext
@@ -356,10 +356,10 @@ define <vscale x 4 x i64> @zext_4i16_4i64(<vscale x 4 x i16> %aval) {
define <vscale x 8 x i32> @zext_8i8_8i32(<vscale x 8 x i8> %aval) {
; CHECK-LABEL: zext_8i8_8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: and z0.h, z0.h, #0xff
-; CHECK-NEXT: uunpklo z2.s, z0.h
-; CHECK-NEXT: uunpkhi z1.s, z0.h
-; CHECK-NEXT: mov z0.d, z2.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: and z1.h, z1.h, #0xff
+; CHECK-NEXT: uunpklo z0.s, z1.h
+; CHECK-NEXT: uunpkhi z1.s, z1.h
; CHECK-NEXT: ret
%aext = zext <vscale x 8 x i8> %aval to <vscale x 8 x i32>
ret <vscale x 8 x i32> %aext
diff --git a/llvm/test/CodeGen/AArch64/sve2-histcnt.ll b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
index f8e190a29e051..6596abe2f105a 100644
--- a/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-histcnt.ll
@@ -419,11 +419,11 @@ define void @histogram_zext_from_i16_to_i32(ptr %base, <vscale x 4 x i16> %indic
define void @histogram_2_lane_zext(ptr %base, <vscale x 2 x i32> %indices, <vscale x 2 x i1> %mask) #0 {
; CHECK-LABEL: histogram_2_lane_zext:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: and z1.d, z1.d, #0xffffffff
; CHECK-NEXT: mov z3.d, #1 // =0x1
; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: ld1w { z2.d }, p0/z, [x0, z0.d, uxtw #2]
-; CHECK-NEXT: and z1.d, z1.d, #0xffffffff
; CHECK-NEXT: histcnt z1.d, p0/z, z1.d, z1.d
; CHECK-NEXT: mad z1.d, p1/m, z3.d, z2.d
; CHECK-NEXT: st1w { z1.d }, p0, [x0, z0.d, uxtw #2]
>From d74206f743bb9608be0a273a22e6be85c48dd2f0 Mon Sep 17 00:00:00 2001
From: Ricardo Jesus <rjj at nvidia.com>
Date: Fri, 17 Apr 2026 08:55:12 -0700
Subject: [PATCH 2/3] Prefer casting to Instruction.
---
llvm/lib/Target/AArch64/SVEInstrFormats.td | 32 +++++++++++-----------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index f05de6ed9840b..d3f0011cdc71a 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -2095,10 +2095,10 @@ multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOpera
def _PSEUDO : UnpredTwoOpImmPseudo<NAME, ZPR64, logical_imm64>;
- def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
- def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Pseudo>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME # _PSEUDO)>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME # _PSEUDO)>;
def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
(!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>;
@@ -2118,10 +2118,10 @@ multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOpera
}
multiclass sve_int_log_imm_bic<SDPatternOperator op> {
- def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
- def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Pseudo>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>;
+ def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>;
}
class sve_int_dup_mask_imm<string asm>
@@ -5383,10 +5383,10 @@ multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, simm8_32b>;
def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, simm8_32b>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _B_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D_PSEUDO)>;
}
multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {
@@ -5400,10 +5400,10 @@ multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperato
def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, imm0_255>;
def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, imm0_255>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Pseudo>(NAME # _B_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_Arith_Any_Predicate<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D_PSEUDO)>;
}
multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {
>From cfd30752840195da44ebcafb51eecea11e64cc55 Mon Sep 17 00:00:00 2001
From: Ricardo Jesus <rjj at nvidia.com>
Date: Fri, 17 Apr 2026 09:45:14 -0700
Subject: [PATCH 3/3] Update other instances of cast<Pseudo>.
---
llvm/lib/Target/AArch64/SVEInstrFormats.td | 190 ++++++++++-----------
1 file changed, 95 insertions(+), 95 deletions(-)
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index d3f0011cdc71a..8f5d3556cde17 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -2353,15 +2353,15 @@ multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> {
def _S_ZERO : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
def _D_ZERO : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
- def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D_ZERO)>;
}
multiclass sve_fp_2op_p_zds_zeroing_bfloat<SDPatternOperator op> {
def _ZERO : PredTwoOpPseudo<NAME, ZPR16, FalseLanesZero>;
- def : SVE_3_Op_Pat_SelZero<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Pseudo>(NAME # _ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _ZERO)>;
}
class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty>
@@ -3251,9 +3251,9 @@ multiclass sve2_fp_un_pred_zeroing_hsd<SDPatternOperator op> {
def _S_ZERO : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesZero>;
def _D_ZERO : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesZero>;
- def : SVE_1_Op_PassthruZero_Pat<nxv8i16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_1_Op_PassthruZero_Pat<nxv4i32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_1_Op_PassthruZero_Pat<nxv2i64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_1_Op_PassthruZero_Pat<nxv8i16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_1_Op_PassthruZero_Pat<nxv4i32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_1_Op_PassthruZero_Pat<nxv2i64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D_ZERO)>;
}
multiclass sve2_fp_convert_down_odd_rounding<string asm, string op, SDPatternOperator ir_op = null_frag> {
@@ -4268,7 +4268,7 @@ multiclass sve2_int_un_pred_arit_s<bits<2> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
- defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
+ defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
}
multiclass sve2_int_un_pred_arit<bits<2> opc, string asm, SDPatternOperator op> {
@@ -4291,10 +4291,10 @@ multiclass sve2_int_un_pred_arit<bits<2> opc, string asm, SDPatternOperator op>
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve2_int_un_pred_arit_z_S<bits<2> opc, string asm, SDPatternOperator op> {
@@ -4993,10 +4993,10 @@ multiclass sve_int_un_pred_arit<bits<3> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5028,9 +5028,9 @@ multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_h_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5056,8 +5056,8 @@ multiclass sve_int_un_pred_arit_w<bits<3> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_w_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5077,7 +5077,7 @@ multiclass sve_int_un_pred_arit_d<bits<3> opc, string asm,
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_d_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5107,10 +5107,10 @@ multiclass sve_int_un_pred_arit_bitwise<bits<3> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_bitwise_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5148,15 +5148,15 @@ multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, !cast<Pseudo>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, !cast<Instruction>(NAME # _H_UNDEF)>;
}
multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm, SDPatternOperator op> {
@@ -5177,12 +5177,12 @@ multiclass sve_fp_un_pred_arit_hsd<SDPatternOperator op> {
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
multiclass sve_int_un_pred_arit_bhsd<SDPatternOperator op> {
@@ -5191,10 +5191,10 @@ multiclass sve_int_un_pred_arit_bhsd<SDPatternOperator op> {
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
//===----------------------------------------------------------------------===//
@@ -5316,16 +5316,16 @@ multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op,
def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, addsub_imm8_opt_lsl_i32>;
def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, addsub_imm8_opt_lsl_i64>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Pseudo>(NAME # _B_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D_PSEUDO)>;
// Extra patterns for add(x, splat(-ve)) -> sub(x, +ve). There is no i8
// pattern as all i8 constants can be handled by an add.
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubNegImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubNegImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubNegImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubNegImm16Pat, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubNegImm32Pat, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubNegImm64Pat, !cast<Instruction>(NAME # _D_PSEUDO)>;
}
multiclass sve_int_arith_imm0_ssat<bits<3> opc, string asm, SDPatternOperator op,
@@ -5340,15 +5340,15 @@ multiclass sve_int_arith_imm0_ssat<bits<3> opc, string asm, SDPatternOperator op
def _S_PSEUDO : UnpredTwoOpImmPseudo<NAME # _S, ZPR32, addsub_imm8_opt_lsl_i32>;
def _D_PSEUDO : UnpredTwoOpImmPseudo<NAME # _D, ZPR64, addsub_imm8_opt_lsl_i64>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubSSatPosImm8Pat, !cast<Pseudo>(NAME # _B_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubSSatPosImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubSSatPosImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubSSatPosImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubSSatPosImm8Pat, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubSSatPosImm16Pat, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubSSatPosImm32Pat, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubSSatPosImm64Pat, !cast<Instruction>(NAME # _D_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, inv_op, ZPR8, i32, SVEAddSubSSatNegImm8Pat, !cast<Pseudo>(NAME # _B_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubSSatNegImm16Pat, !cast<Pseudo>(NAME # _H_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubSSatNegImm32Pat, !cast<Pseudo>(NAME # _S_PSEUDO)>;
- def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubSSatNegImm64Pat, !cast<Pseudo>(NAME # _D_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, inv_op, ZPR8, i32, SVEAddSubSSatNegImm8Pat, !cast<Instruction>(NAME # _B_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubSSatNegImm16Pat, !cast<Instruction>(NAME # _H_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubSSatNegImm32Pat, !cast<Instruction>(NAME # _S_PSEUDO)>;
+ def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubSSatNegImm64Pat, !cast<Instruction>(NAME # _D_PSEUDO)>;
}
class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,
@@ -6559,10 +6559,10 @@ multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> {
def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftL32, FalseLanesZero>;
def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftL64, FalseLanesZero>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftL8, !cast<Pseudo>(NAME # _B_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftL16, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftL32, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftL64, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftL8, !cast<Instruction>(NAME # _B_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftL16, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftL32, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftL64, !cast<Instruction>(NAME # _D_ZERO)>;
}
multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps,
@@ -6605,10 +6605,10 @@ multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op =
def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>;
def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftR8, !cast<Pseudo>(NAME # _B_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftR16, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftR32, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftR64, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, vecshiftR8, !cast<Instruction>(NAME # _B_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, vecshiftR16, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, vecshiftR32, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, vecshiftR64, !cast<Instruction>(NAME # _D_ZERO)>;
}
class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc,
@@ -6660,10 +6660,10 @@ multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> {
def _S_ZERO : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
def _D_ZERO : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
- def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_ZERO)>;
- def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D_ZERO)>;
}
multiclass sve_int_bin_pred_imm_zeroing_bhsd<SDPatternOperator op,
@@ -6674,10 +6674,10 @@ multiclass sve_int_bin_pred_imm_zeroing_bhsd<SDPatternOperator op,
def _S_ZERO : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesZero>;
def _D_ZERO : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesZero>;
- def : SVE_2_Op_Imm_Pat_Zero<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Pseudo>(NAME # _B_ZERO)>;
- def : SVE_2_Op_Imm_Pat_Zero<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Pseudo>(NAME # _H_ZERO)>;
- def : SVE_2_Op_Imm_Pat_Zero<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Pseudo>(NAME # _S_ZERO)>;
- def : SVE_2_Op_Imm_Pat_Zero<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Pseudo>(NAME # _D_ZERO)>;
+ def : SVE_2_Op_Imm_Pat_Zero<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _B_ZERO)>;
+ def : SVE_2_Op_Imm_Pat_Zero<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _H_ZERO)>;
+ def : SVE_2_Op_Imm_Pat_Zero<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _S_ZERO)>;
+ def : SVE_2_Op_Imm_Pat_Zero<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _D_ZERO)>;
}
multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm,
@@ -9899,21 +9899,21 @@ multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {
def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
// Predicated pseudo floating point (BFloat) two operand instructions.
multiclass sve_fp_bin_pred_bfloat<SDPatternOperator op> {
def _UNDEF : PredTwoOpPseudo<NAME, ZPR16, FalseLanesUndef>;
- def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Pseudo>(NAME # _UNDEF)>;
- def : SVE_3_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, !cast<Pseudo>(NAME # _UNDEF)>;
- def : SVE_3_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, !cast<Pseudo>(NAME # _UNDEF)>;
+ def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _UNDEF)>;
+ def : SVE_3_Op_Pat<nxv4bf16, op, nxv4i1, nxv4bf16, nxv4bf16, !cast<Instruction>(NAME # _UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2bf16, op, nxv2i1, nxv2bf16, nxv2bf16, !cast<Instruction>(NAME # _UNDEF)>;
}
// Predicated pseudo floating point three operand instructions.
@@ -9946,10 +9946,10 @@ multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> {
def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
// As sve_int_bin_pred but when only i32 and i64 vector types are required.
@@ -9957,8 +9957,8 @@ multiclass sve_int_bin_pred_sd<SDPatternOperator op> {
def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
// Predicated pseudo integer two operand instructions. Second operand is an
@@ -9983,10 +9983,10 @@ multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> {
def _S_UNDEF : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
def _D_UNDEF : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
- def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
- def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
- def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
- def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
+ def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B_UNDEF)>;
+ def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H_UNDEF)>;
+ def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S_UNDEF)>;
+ def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D_UNDEF)>;
}
//===----------------------------------------------------------------------===//
More information about the llvm-commits
mailing list