[llvm] [GISel][CombinerHelper] Push freeze through non-poison-producing operands (PR #90618)
Dhruv Chawla via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 30 20:28:56 PDT 2024
https://github.com/dc03-work updated https://github.com/llvm/llvm-project/pull/90618
>From daff084d559d424f6b5727c6d4a440f826f50d01 Mon Sep 17 00:00:00 2001
From: Dhruv Chawla <dhruvc at nvidia.com>
Date: Tue, 30 Apr 2024 17:04:00 +0530
Subject: [PATCH 1/3] [GISel][CombinerHelper] Push freeze through
non-poison-producing operands
This combine matches the existing fold in InstCombine, i.e.
InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
It tries to push freeze through an operand if the operand has only one
maybe-poison operand and all other operands are guaranteed non-poison.
This is beneficial because it can potentially enable other optimizations
to occur that would otherwise be blocked because of the freeze.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 2 +
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 29 +++++++++
.../include/llvm/Target/GlobalISel/Combine.td | 9 ++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 59 +++++++++++++++++++
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 17 +++++-
llvm/lib/Target/AArch64/AArch64Combine.td | 3 +-
.../AArch64/GlobalISel/combine-select.mir | 42 ++++++-------
llvm/test/CodeGen/AArch64/pr58431.ll | 4 +-
8 files changed, 139 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 76e8d1166ae0cd..bd59aab03a6103 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -856,6 +856,8 @@ class CombinerHelper {
/// Combine insert vector element OOB.
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
+ bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo);
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 25e47114e4a39a..90a806df86226a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -780,6 +780,35 @@ class GFreeze : public GenericMachineInstr {
}
};
+/// Represents cast instructions.
+class GCastOp : public GenericMachineInstr {
+public:
+ Register getSourceReg() const { return getOperand(1).getReg(); }
+
+ static bool classof(const MachineInstr *MI) {
+ switch (MI->getOpcode()) {
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::G_ANYEXT:
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_FPTOSI:
+ case TargetOpcode::G_FPTOUI:
+ case TargetOpcode::G_UITOFP:
+ case TargetOpcode::G_SITOFP:
+ case TargetOpcode::G_FPTRUNC:
+ case TargetOpcode::G_FPEXT:
+ case TargetOpcode::G_INTTOPTR:
+ case TargetOpcode::G_PTRTOINT:
+ case TargetOpcode::G_BITCAST:
+ case TargetOpcode::G_ADDRSPACE_CAST:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+};
+
} // namespace llvm
#endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index dbbb3abaa83047..7185cd2bbcb136 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -218,6 +218,13 @@ def idempotent_prop : GICombineRule<
(match (idempotent_prop_frags $dst, $src)),
(apply (GIReplaceReg $dst, $src))>;
+// Convert freeze(Op(Op0, NonPoisonOps...)) to Op(freeze(Op0), NonPoisonOps...)
+// when Op0 is not guaranteed non-poison
+def push_freeze_to_prevent_poison_propagation : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FREEZE):$root,
+ [{ return Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
def extending_loads : GICombineRule<
(defs root:$root, extending_load_matchdata:$matchinfo),
@@ -1662,7 +1669,7 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
sub_add_reg, select_to_minmax, redundant_binop_in_equality,
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
combine_concat_vector, double_icmp_zero_and_or_combine, match_addos,
- combine_shuffle_concat]>;
+ combine_shuffle_concat, push_freeze_to_prevent_poison_propagation]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 653e7689b57743..0a79702bf7080d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -223,6 +223,64 @@ void CombinerHelper::applyCombineCopy(MachineInstr &MI) {
replaceRegWith(MRI, DstReg, SrcReg);
}
+bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
+ MachineInstr &MI, BuildFnTy &MatchInfo) {
+ // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating
+ Register DstOp = MI.getOperand(0).getReg();
+ Register OrigOp = MI.getOperand(1).getReg();
+
+ if (OrigOp.isPhysical() || !MRI.hasOneNonDBGUse(OrigOp))
+ return false;
+
+ MachineInstr *OrigDef = MRI.getUniqueVRegDef(OrigOp);
+ // Avoid trying to fold G_PHI, G_UNMERGE_VALUES, G_FREEZE (the latter is
+ // handled by idempotent_prop)
+ if (!OrigDef || OrigDef->isPHI() || isa<GUnmerge>(OrigDef) ||
+ isa<GFreeze>(OrigDef))
+ return false;
+
+ if (canCreateUndefOrPoison(OrigOp, MRI))
+ return false;
+
+ std::optional<MachineOperand> MaybePoisonOperand = std::nullopt;
+ for (MachineOperand &Operand : OrigDef->uses()) {
+ // Avoid working on non-register operands or physical registers.
+ if (!Operand.isReg() || Operand.getReg().isPhysical())
+ return false;
+
+ if (isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI))
+ continue;
+
+ if (!MaybePoisonOperand)
+ MaybePoisonOperand = Operand;
+ // We have more than one maybe-poison operand. Moving the freeze is unsafe.
+ else
+ return false;
+ }
+
+ // Eliminate freeze if all operands are guaranteed non-poison
+ if (!MaybePoisonOperand) {
+ MatchInfo = [=](MachineIRBuilder &B) { MRI.replaceRegWith(DstOp, OrigOp); };
+ return true;
+ }
+
+ if (!MaybePoisonOperand->isReg())
+ return false;
+
+ Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
+ LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);
+
+ MatchInfo = [=](MachineIRBuilder &B) mutable {
+ auto Reg = MRI.createGenericVirtualRegister(MaybePoisonOperandRegTy);
+ B.setInsertPt(*OrigDef->getParent(), OrigDef->getIterator());
+ B.buildFreeze(Reg, MaybePoisonOperandReg);
+ replaceRegOpWith(
+ MRI, *OrigDef->findRegisterUseOperand(MaybePoisonOperandReg, TRI), Reg);
+ replaceRegWith(MRI, DstOp, OrigOp);
+ };
+ return true;
+}
+
bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI,
SmallVector<Register> &Ops) {
assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
@@ -3060,6 +3118,7 @@ bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI);
if (!LeftHandInst || !RightHandInst)
return false;
+
unsigned HandOpcode = LeftHandInst->getOpcode();
if (HandOpcode != RightHandInst->getOpcode())
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 4e3781cb4e9d59..3f9497fcb6855a 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1737,6 +1737,8 @@ static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
case TargetOpcode::G_FREEZE:
return false;
default:
+ if (isa<GCastOp>(RegDef) || isa<GBinOp>(RegDef))
+ return false;
return true;
}
}
@@ -1750,13 +1752,26 @@ static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
MachineInstr *RegDef = MRI.getVRegDef(Reg);
+ auto OpCheck = [&](MachineOperand &Operand) {
+ if (!Operand.isReg())
+ return true;
+
+ return isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI, Depth + 1,
+ Kind);
+ };
+
switch (RegDef->getOpcode()) {
case TargetOpcode::G_FREEZE:
return true;
case TargetOpcode::G_IMPLICIT_DEF:
return !includesUndef(Kind);
default:
- return false;
+ GenericMachineInstr *Opr = dyn_cast<GBinOp>(RegDef);
+ if (!Opr)
+ Opr = dyn_cast<GCastOp>(RegDef);
+
+ return Opr && !::llvm::canCreateUndefOrPoison(Reg, MRI) &&
+ all_of(Opr->operands(), OpCheck);
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 10cad6d1924407..7229d4fe52261f 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -295,5 +295,6 @@ def AArch64PostLegalizerCombiner
ptr_add_immed_chain, overlapping_and,
split_store_zero_128, undef_combines,
select_to_minmax, or_to_bsp, combine_concat_vector,
- commute_constant_to_rhs]> {
+ commute_constant_to_rhs,
+ push_freeze_to_prevent_poison_propagation]> {
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
index 2bf7e84a379ba0..90d0896d168930 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
@@ -117,9 +117,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -144,9 +144,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -172,9 +172,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d2
; CHECK-NEXT: %c:_(<2 x s1>) = G_TRUNC [[COPY]](<2 x s32>)
- ; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[COPY1]](<2 x s32>)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s1>) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s32>) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[FREEZE]](<2 x s32>)
+ ; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, %f
; CHECK-NEXT: %ext:_(<2 x s32>) = G_ANYEXT %sel(<2 x s1>)
; CHECK-NEXT: $d0 = COPY %ext(<2 x s32>)
%0:_(<2 x s32>) = COPY $d0
@@ -201,9 +201,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -229,9 +229,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -257,11 +257,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
; CHECK-NEXT: %one:_(s1) = G_CONSTANT i1 true
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, %one
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], [[FREEZE]]
+ ; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -287,11 +287,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, [[C]]
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], [[FREEZE]]
+ ; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
diff --git a/llvm/test/CodeGen/AArch64/pr58431.ll b/llvm/test/CodeGen/AArch64/pr58431.ll
index dcd97597ae4093..e87d8f7874d62e 100644
--- a/llvm/test/CodeGen/AArch64/pr58431.ll
+++ b/llvm/test/CodeGen/AArch64/pr58431.ll
@@ -4,8 +4,8 @@
define i32 @f(i64 %0) {
; CHECK-LABEL: f:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #10
-; CHECK-NEXT: mov w9, w0
+; CHECK-NEXT: mov w8, #10 // =0xa
+; CHECK-NEXT: and x9, x0, #0xffffffff
; CHECK-NEXT: udiv x10, x9, x8
; CHECK-NEXT: msub x0, x10, x8, x9
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
>From 5975d59eb124f0434c89381a6212f20a15087dcd Mon Sep 17 00:00:00 2001
From: Dhruv Chawla <dhruvc at nvidia.com>
Date: Tue, 30 Apr 2024 20:32:39 +0530
Subject: [PATCH 2/3] Address reviewer comments
---
.../include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 3 ++-
llvm/include/llvm/Target/GlobalISel/Combine.td | 4 ++--
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 12 ++++--------
3 files changed, 8 insertions(+), 11 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index bd59aab03a6103..f1cc92b7cb40b4 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -856,7 +856,8 @@ class CombinerHelper {
/// Combine insert vector element OOB.
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
- bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo);
+ bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI,
+ BuildFnTy &MatchInfo);
private:
/// Checks for legality of an indexed variant of \p LdSt.
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 7185cd2bbcb136..0a3567440aea78 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -222,8 +222,8 @@ def idempotent_prop : GICombineRule<
// when Op0 is not guaranteed non-poison
def push_freeze_to_prevent_poison_propagation : GICombineRule<
(defs root:$root, build_fn_matchinfo:$matchinfo),
- (match (wip_match_opcode G_FREEZE):$root,
- [{ return Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
+ (match (G_FREEZE $dst, $src):$root,
+ [{ return !isGuaranteedNotToBePoison(${src}.getReg(), MRI) && Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
def extending_loads : GICombineRule<
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 0a79702bf7080d..fe32e1dbc796e9 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -242,10 +242,10 @@ bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
if (canCreateUndefOrPoison(OrigOp, MRI))
return false;
- std::optional<MachineOperand> MaybePoisonOperand = std::nullopt;
+ std::optional<MachineOperand> MaybePoisonOperand;
for (MachineOperand &Operand : OrigDef->uses()) {
- // Avoid working on non-register operands or physical registers.
- if (!Operand.isReg() || Operand.getReg().isPhysical())
+ // Avoid working on non-register operands.
+ if (!Operand.isReg())
return false;
if (isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI))
@@ -258,15 +258,12 @@ bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
return false;
}
- // Eliminate freeze if all operands are guaranteed non-poison
+ // Eliminate freeze if all operands are guaranteed non-poison.
if (!MaybePoisonOperand) {
MatchInfo = [=](MachineIRBuilder &B) { MRI.replaceRegWith(DstOp, OrigOp); };
return true;
}
- if (!MaybePoisonOperand->isReg())
- return false;
-
Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);
@@ -3118,7 +3115,6 @@ bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI);
if (!LeftHandInst || !RightHandInst)
return false;
-
unsigned HandOpcode = LeftHandInst->getOpcode();
if (HandOpcode != RightHandInst->getOpcode())
return false;
>From a49eea31db1d5c9e415999c6d3b4f6277ceaa497 Mon Sep 17 00:00:00 2001
From: Dhruv Chawla <dhruvc at nvidia.com>
Date: Wed, 1 May 2024 08:57:32 +0530
Subject: [PATCH 3/3] Add support for checking and dropping poison-generating
flags
---
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 77 +++++++++++++++++++
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 7 +-
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 9 +++
3 files changed, 91 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 90a806df86226a..74789cad54fd41 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -34,6 +34,83 @@ class GenericMachineInstr : public MachineInstr {
static bool classof(const MachineInstr *MI) {
return isPreISelGenericOpcode(MI->getOpcode());
}
+
+ bool hasPoisonGeneratingFlags() const {
+ switch (getOpcode()) {
+ case TargetOpcode::G_ADD:
+ case TargetOpcode::G_SUB:
+ case TargetOpcode::G_MUL:
+ case TargetOpcode::G_SHL:
+ case TargetOpcode::G_TRUNC:
+ return getFlag(NoUWrap) || getFlag(NoSWrap);
+
+ case TargetOpcode::G_UDIV:
+ case TargetOpcode::G_SDIV:
+ case TargetOpcode::G_ASHR:
+ case TargetOpcode::G_LSHR:
+ return getFlag(IsExact);
+
+ case TargetOpcode::G_OR:
+ return getFlag(Disjoint);
+
+ case TargetOpcode::G_UITOFP:
+ case TargetOpcode::G_ZEXT:
+ return getFlag(NonNeg);
+
+ case TargetOpcode::G_FNEG:
+ case TargetOpcode::G_FADD:
+ case TargetOpcode::G_FSUB:
+ case TargetOpcode::G_FMUL:
+ case TargetOpcode::G_FDIV:
+ case TargetOpcode::G_FREM:
+ case TargetOpcode::G_FCMP:
+ return getFlag(FmNoNans) || getFlag(FmNoInfs);
+
+ default:
+ return false;
+ }
+ }
+
+ void dropPoisonGeneratingFlags() {
+ switch (getOpcode()) {
+ case TargetOpcode::G_ADD:
+ case TargetOpcode::G_SUB:
+ case TargetOpcode::G_MUL:
+ case TargetOpcode::G_SHL:
+ case TargetOpcode::G_TRUNC:
+ clearFlag(NoUWrap);
+ clearFlag(NoSWrap);
+ break;
+
+ case TargetOpcode::G_UDIV:
+ case TargetOpcode::G_SDIV:
+ case TargetOpcode::G_ASHR:
+ case TargetOpcode::G_LSHR:
+ clearFlag(IsExact);
+ break;
+
+ case TargetOpcode::G_OR:
+ clearFlag(Disjoint);
+ break;
+
+ case TargetOpcode::G_UITOFP:
+ case TargetOpcode::G_ZEXT:
+ clearFlag(NonNeg);
+ break;
+
+ case TargetOpcode::G_FNEG:
+ case TargetOpcode::G_FADD:
+ case TargetOpcode::G_FSUB:
+ case TargetOpcode::G_FMUL:
+ case TargetOpcode::G_FDIV:
+ case TargetOpcode::G_FREM:
+ case TargetOpcode::G_FCMP:
+ clearFlag(FmNoNans);
+ clearFlag(FmNoInfs);
+ break;
+ }
+ assert(!hasPoisonGeneratingFlags());
+ }
};
/// Provides common memory operand functionality.
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index fe32e1dbc796e9..804f72b0846630 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -234,12 +234,13 @@ bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
MachineInstr *OrigDef = MRI.getUniqueVRegDef(OrigOp);
// Avoid trying to fold G_PHI, G_UNMERGE_VALUES, G_FREEZE (the latter is
- // handled by idempotent_prop)
+ // handled by idempotent_prop).
if (!OrigDef || OrigDef->isPHI() || isa<GUnmerge>(OrigDef) ||
isa<GFreeze>(OrigDef))
return false;
- if (canCreateUndefOrPoison(OrigOp, MRI))
+ if (canCreateUndefOrPoison(OrigOp, MRI,
+ /*ConsiderFlagsAndMetadata*/ false))
return false;
std::optional<MachineOperand> MaybePoisonOperand;
@@ -258,6 +259,8 @@ bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
return false;
}
+ cast<GenericMachineInstr>(OrigDef)->dropPoisonGeneratingFlags();
+
// Eliminate freeze if all operands are guaranteed non-poison.
if (!MaybePoisonOperand) {
MatchInfo = [=](MachineIRBuilder &B) { MRI.replaceRegWith(DstOp, OrigOp); };
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 3f9497fcb6855a..7cd971aff0dc09 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1733,6 +1733,15 @@ static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
UndefPoisonKind Kind) {
MachineInstr *RegDef = MRI.getVRegDef(Reg);
+ if (auto *GMI = dyn_cast<GenericMachineInstr>(RegDef)) {
+ if (ConsiderFlagsAndMetadata && includesPoison(Kind) &&
+ GMI->hasPoisonGeneratingFlags())
+ return true;
+ }
+ // Conservatively return true.
+ else
+ return true;
+
switch (RegDef->getOpcode()) {
case TargetOpcode::G_FREEZE:
return false;
More information about the llvm-commits
mailing list