[llvm] [AArch64] Remove copy in SVE/SME predicate spill and fill (PR #81716)
Sam Tebbs via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 27 06:31:48 PST 2024
https://github.com/SamTebbs33 updated https://github.com/llvm/llvm-project/pull/81716
>From fb0a7681df70fb8a40bf8bcbda2e165da685ef18 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Tue, 13 Feb 2024 11:23:34 +0000
Subject: [PATCH 1/4] [AArch64] Remove copy in SVE/SME predicate spill
7dc20ab introduced an extra COPY when spilling a PNR register, which
can't be elided as the input (PNR predicate) and output (PPR predicate)
register classes differ. This patch emits a new ConvertPNRtoPPR
pseudo instruction instead. When this is expanded, it gets erased if the
PNR is a subregister of the PPR, since the conversion is implicit,
otherwise it is lowered to an ORR.
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 17 +++++++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 6 ++--
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 2 ++
.../spillfill-sve-different-predicate.mir | 30 +++++++++++++++++++
llvm/test/CodeGen/AArch64/spillfill-sve.mir | 3 +-
5 files changed, 54 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 352c61d48e2fff..7dfd4ed5b38cff 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1119,6 +1119,23 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
default:
break;
+ case AArch64::ConvertPNRtoPPR: {
+ auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
+ MachineOperand DstMO = MI.getOperand(0);
+ MachineOperand SrcMO = MI.getOperand(1);
+ unsigned SrcReg = SrcMO.getReg();
+ if (!TRI->isSubRegister(DstMO.getReg(), SrcReg)) {
+ unsigned SrcSuperReg = TRI->getMatchingSuperReg(SrcReg, AArch64::psub,
+ &AArch64::PPRRegClass);
+ BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_PPzPP))
+ .add(DstMO)
+ .addReg(SrcSuperReg)
+ .addReg(SrcSuperReg)
+ .addReg(SrcSuperReg);
+ }
+ MI.eraseFromParent();
+ return true;
+ }
case AArch64::BSPv8i8:
case AArch64::BSPv16i8: {
Register DstReg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 656259727c124f..01c3a1c92d07d7 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4789,6 +4789,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
bool Offset = true;
MCRegister PNRReg = MCRegister::NoRegister;
unsigned StackID = TargetStackID::Default;
+ const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
switch (TRI->getSpillSize(*RC)) {
case 1:
if (AArch64::FPR8RegClass.hasSubClassEq(RC))
@@ -4807,8 +4808,9 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
"Unexpected register store without SVE2p1 or SME2");
if (SrcReg.isVirtual()) {
auto NewSrcReg =
- MF.getRegInfo().createVirtualRegister(&AArch64::PPRRegClass);
- BuildMI(MBB, MBBI, DebugLoc(), get(TargetOpcode::COPY), NewSrcReg)
+ MF.getRegInfo().createVirtualRegister(&AArch64::PPR_p8to15RegClass);
+ BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ConvertPNRtoPPR),
+ NewSrcReg)
.addReg(SrcReg);
SrcReg = NewSrcReg;
} else
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index c4d69232c9e30e..92d4c3774908e2 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2308,6 +2308,8 @@ let Predicates = [HasBF16, HasSVEorSME] in {
defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>;
} // End HasBF16, HasSVEorSME
+def ConvertPNRtoPPR : Pseudo<(outs PPRAny:$Pd), (ins PNRAny:$Pm), []>, Sched<[]>;
+
let Predicates = [HasSVEorSME] in {
// InstAliases
def : InstAlias<"mov $Zd, $Zn",
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir b/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
new file mode 100644
index 00000000000000..abbc1d615b66f7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
@@ -0,0 +1,30 @@
+# RUN: llc -mtriple=aarch64-linux-gnu -start-after=virtregrewriter -stop-after=aarch64-expand-pseudo -mattr=+sme2 -verify-machineinstrs -o - %s \
+# RUN: | FileCheck %s
+
+--- |
+ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+ target triple = "aarch64-unknown-linux-gnu"
+
+ define void @test_convert_different_reg() #0 { entry: unreachable }
+
+ attributes #0 = { "target-features"="+sme2" }
+
+---
+name: test_convert_different_reg
+tracksRegLiveness: true
+body: |
+ bb.0.entry:
+ ; CHECK-LABEL: name: test_convert_different_reg
+ ; CHECK: renamable $pn8 = WHILEGE_CXX_B undef $x0, undef $x0, 0, implicit-def dead $nzcv
+ ; CHECK-NEXT: renamable $p9 = ORR_PPzPP $p8, $p8, $p8
+ ; CHECK-NEXT: STR_PXI killed renamable $p9, $sp, 7
+ ; CHECK-NEXT: renamable $p0 = LDR_PXI $sp, 7
+ early-clobber $sp = frame-setup STRXpre killed $fp, $sp, -16
+ frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
+ frame-setup CFI_INSTRUCTION offset $w29, -16
+ renamable $pn8 = WHILEGE_CXX_B undef $x0, undef $x0, 0, implicit-def dead $nzcv
+ renamable $p9 = ConvertPNRtoPPR killed renamable $pn8
+ STR_PXI killed renamable $p9, $sp, 7
+ renamable $p0 = LDR_PXI $sp, 7
+ early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16
+ RET undef $lr
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve.mir b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
index ef7d55a1c2395f..af062d33c5642f 100644
--- a/llvm/test/CodeGen/AArch64/spillfill-sve.mir
+++ b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
@@ -213,8 +213,7 @@ body: |
; EXPAND-LABEL: name: spills_fills_stack_id_virtreg_pnr
; EXPAND: renamable $pn8 = WHILEGE_CXX_B
- ; EXPAND: $p0 = ORR_PPzPP $p8, $p8, killed $p8
- ; EXPAND: STR_PXI killed renamable $p0, $sp, 7
+ ; EXPAND: STR_PXI killed renamable $p8, $sp, 7
;
; EXPAND: renamable $p0 = LDR_PXI $sp, 7
; EXPAND: $p8 = ORR_PPzPP $p0, $p0, killed $p0, implicit-def $pn8
>From d7e5c13562ac4fbbb867479813b2a9f90d044489 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Wed, 21 Feb 2024 16:30:21 +0000
Subject: [PATCH 2/4] fixup: constrain reg class instead of convert
---
.../AArch64/AArch64ExpandPseudoInsts.cpp | 17 -----------
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 10 ++-----
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 2 --
.../spillfill-sve-different-predicate.mir | 30 -------------------
llvm/test/CodeGen/AArch64/spillfill-sve.mir | 5 ++--
5 files changed, 4 insertions(+), 60 deletions(-)
delete mode 100644 llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 7dfd4ed5b38cff..352c61d48e2fff 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1119,23 +1119,6 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
default:
break;
- case AArch64::ConvertPNRtoPPR: {
- auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
- MachineOperand DstMO = MI.getOperand(0);
- MachineOperand SrcMO = MI.getOperand(1);
- unsigned SrcReg = SrcMO.getReg();
- if (!TRI->isSubRegister(DstMO.getReg(), SrcReg)) {
- unsigned SrcSuperReg = TRI->getMatchingSuperReg(SrcReg, AArch64::psub,
- &AArch64::PPRRegClass);
- BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::ORR_PPzPP))
- .add(DstMO)
- .addReg(SrcSuperReg)
- .addReg(SrcSuperReg)
- .addReg(SrcSuperReg);
- }
- MI.eraseFromParent();
- return true;
- }
case AArch64::BSPv8i8:
case AArch64::BSPv16i8: {
Register DstReg = MI.getOperand(0).getReg();
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 01c3a1c92d07d7..e5d201d1be63ec 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4789,7 +4789,6 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
bool Offset = true;
MCRegister PNRReg = MCRegister::NoRegister;
unsigned StackID = TargetStackID::Default;
- const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
switch (TRI->getSpillSize(*RC)) {
case 1:
if (AArch64::FPR8RegClass.hasSubClassEq(RC))
@@ -4807,12 +4806,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
"Unexpected register store without SVE2p1 or SME2");
if (SrcReg.isVirtual()) {
- auto NewSrcReg =
- MF.getRegInfo().createVirtualRegister(&AArch64::PPR_p8to15RegClass);
- BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ConvertPNRtoPPR),
- NewSrcReg)
- .addReg(SrcReg);
- SrcReg = NewSrcReg;
+ MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::PPRRegClass);
} else
SrcReg = (SrcReg - AArch64::PN0) + AArch64::P0;
Opc = AArch64::STR_PXI;
@@ -4992,7 +4986,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
"Unexpected register load without SVE2p1 or SME2");
PNRReg = DestReg;
if (DestReg.isVirtual())
- DestReg = MF.getRegInfo().createVirtualRegister(&AArch64::PPRRegClass);
+ MF.getRegInfo().constrainRegClass(DestReg, &AArch64::PPRRegClass);
else
DestReg = (DestReg - AArch64::PN0) + AArch64::P0;
Opc = AArch64::LDR_PXI;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 92d4c3774908e2..c4d69232c9e30e 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2308,8 +2308,6 @@ let Predicates = [HasBF16, HasSVEorSME] in {
defm BFCVTNT_ZPmZ : sve_bfloat_convert<0b0, "bfcvtnt", int_aarch64_sve_fcvtnt_bf16f32>;
} // End HasBF16, HasSVEorSME
-def ConvertPNRtoPPR : Pseudo<(outs PPRAny:$Pd), (ins PNRAny:$Pm), []>, Sched<[]>;
-
let Predicates = [HasSVEorSME] in {
// InstAliases
def : InstAlias<"mov $Zd, $Zn",
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir b/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
deleted file mode 100644
index abbc1d615b66f7..00000000000000
--- a/llvm/test/CodeGen/AArch64/spillfill-sve-different-predicate.mir
+++ /dev/null
@@ -1,30 +0,0 @@
-# RUN: llc -mtriple=aarch64-linux-gnu -start-after=virtregrewriter -stop-after=aarch64-expand-pseudo -mattr=+sme2 -verify-machineinstrs -o - %s \
-# RUN: | FileCheck %s
-
---- |
- target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
- target triple = "aarch64-unknown-linux-gnu"
-
- define void @test_convert_different_reg() #0 { entry: unreachable }
-
- attributes #0 = { "target-features"="+sme2" }
-
----
-name: test_convert_different_reg
-tracksRegLiveness: true
-body: |
- bb.0.entry:
- ; CHECK-LABEL: name: test_convert_different_reg
- ; CHECK: renamable $pn8 = WHILEGE_CXX_B undef $x0, undef $x0, 0, implicit-def dead $nzcv
- ; CHECK-NEXT: renamable $p9 = ORR_PPzPP $p8, $p8, $p8
- ; CHECK-NEXT: STR_PXI killed renamable $p9, $sp, 7
- ; CHECK-NEXT: renamable $p0 = LDR_PXI $sp, 7
- early-clobber $sp = frame-setup STRXpre killed $fp, $sp, -16
- frame-setup CFI_INSTRUCTION escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22
- frame-setup CFI_INSTRUCTION offset $w29, -16
- renamable $pn8 = WHILEGE_CXX_B undef $x0, undef $x0, 0, implicit-def dead $nzcv
- renamable $p9 = ConvertPNRtoPPR killed renamable $pn8
- STR_PXI killed renamable $p9, $sp, 7
- renamable $p0 = LDR_PXI $sp, 7
- early-clobber $sp, $fp = frame-destroy LDRXpost $sp, 16
- RET undef $lr
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve.mir b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
index af062d33c5642f..fba8af5a7028ae 100644
--- a/llvm/test/CodeGen/AArch64/spillfill-sve.mir
+++ b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
@@ -213,10 +213,9 @@ body: |
; EXPAND-LABEL: name: spills_fills_stack_id_virtreg_pnr
; EXPAND: renamable $pn8 = WHILEGE_CXX_B
- ; EXPAND: STR_PXI killed renamable $p8, $sp, 7
+ ; EXPAND: STR_PXI killed renamable $pn8, $sp, 7
;
- ; EXPAND: renamable $p0 = LDR_PXI $sp, 7
- ; EXPAND: $p8 = ORR_PPzPP $p0, $p0, killed $p0, implicit-def $pn8
+ ; EXPAND: renamable $pn8 = LDR_PXI $sp, 7
; EXPAND: $p0 = PEXT_PCI_B killed renamable $pn8, 0
>From ebb0a345353ca62d5041a2cec406d0c342c47d88 Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Tue, 27 Feb 2024 14:29:47 +0000
Subject: [PATCH 3/4] fixup: let predicate spill and fill take ppr or pnr
---
llvm/lib/Target/AArch64/AArch64RegisterInfo.td | 10 ++++++++++
llvm/lib/Target/AArch64/SVEInstrFormats.td | 6 +++---
llvm/test/CodeGen/AArch64/spillfill-sve.mir | 2 +-
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index b70ab856888478..00aef0da760ec0 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -996,6 +996,16 @@ let Namespace = "AArch64" in {
def psub1 : SubRegIndex<16, -1>;
}
+class PPRorPNRClass : RegisterClass<
+ "AArch64",
+ [ nxv16i1, nxv8i1, nxv4i1, nxv2i1, nxv1i1 ], 16,
+ (add PPR, PNR)> {
+ let Size = 16;
+}
+def PPRorPNR : PPRorPNRClass;
+def PPRorPNRAsmOpAny : PPRAsmOperand<"PPRorPNRAny", "PPRorPNR", 0>;
+def PPRorPNRAny : PPRRegOp<"", PPRorPNRAsmOpAny, ElementSizeNone, PPRorPNR>;
+
// Pairs of SVE predicate vector registers.
def PSeqPairs : RegisterTuples<[psub0, psub1], [(rotl PPR, 0), (rotl PPR, 1)]>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 789ec817d3d8b8..e829788cdc3940 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -6680,7 +6680,7 @@ multiclass sve_mem_z_spill<string asm> {
}
class sve_mem_p_spill<string asm>
-: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9),
+: I<(outs), (ins PPRorPNRAny:$Pt, GPR64sp:$Rn, simm9:$imm9),
asm, "\t$Pt, [$Rn, $imm9, mul vl]",
"",
[]>, Sched<[]> {
@@ -6703,7 +6703,7 @@ multiclass sve_mem_p_spill<string asm> {
def NAME : sve_mem_p_spill<asm>;
def : InstAlias<asm # "\t$Pt, [$Rn]",
- (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>;
+ (!cast<Instruction>(NAME) PPRorPNRAny:$Pt, GPR64sp:$Rn, 0), 1>;
}
//===----------------------------------------------------------------------===//
@@ -7915,7 +7915,7 @@ multiclass sve_mem_z_fill<string asm> {
}
class sve_mem_p_fill<string asm>
-: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9),
+: I<(outs PPRorPNRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9),
asm, "\t$Pt, [$Rn, $imm9, mul vl]",
"",
[]>, Sched<[]> {
diff --git a/llvm/test/CodeGen/AArch64/spillfill-sve.mir b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
index fba8af5a7028ae..e30853c8d087b1 100644
--- a/llvm/test/CodeGen/AArch64/spillfill-sve.mir
+++ b/llvm/test/CodeGen/AArch64/spillfill-sve.mir
@@ -1,5 +1,5 @@
# RUN: llc -mtriple=aarch64-linux-gnu -run-pass=greedy %s -o - | FileCheck %s
-# RUN: llc -mtriple=aarch64-linux-gnu -start-before=greedy -stop-after=aarch64-expand-pseudo %s -o - | FileCheck %s --check-prefix=EXPAND
+# RUN: llc -mtriple=aarch64-linux-gnu -start-before=greedy -stop-after=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=EXPAND
--- |
; ModuleID = '<stdin>'
source_filename = "<stdin>"
>From f736dc290f04a4de930806cfd37861be2cc22def Mon Sep 17 00:00:00 2001
From: Samuel Tebbs <samuel.tebbs at arm.com>
Date: Tue, 27 Feb 2024 14:29:58 +0000
Subject: [PATCH 4/4] fixup: remove braces
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index e5d201d1be63ec..6cda87f420b190 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -4805,9 +4805,9 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
} else if (AArch64::PNRRegClass.hasSubClassEq(RC)) {
assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
"Unexpected register store without SVE2p1 or SME2");
- if (SrcReg.isVirtual()) {
+ if (SrcReg.isVirtual())
MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::PPRRegClass);
- } else
+ else
SrcReg = (SrcReg - AArch64::PN0) + AArch64::P0;
Opc = AArch64::STR_PXI;
StackID = TargetStackID::ScalableVector;
More information about the llvm-commits
mailing list