[llvm] [GISel][CombinerHelper] Push freeze through non-poison-producing operands (PR #90618)
Dhruv Chawla via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 30 07:47:15 PDT 2024
https://github.com/dc03-work created https://github.com/llvm/llvm-project/pull/90618
This combine matches the existing fold in InstCombine, i.e. InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
It tries to push freeze through an operand if the operand has only one maybe-poison operand and all other operands are guaranteed non-poison.
This is beneficial because it can potentially enable other optimizations to occur that would otherwise be blocked because of the freeze.
>From daff084d559d424f6b5727c6d4a440f826f50d01 Mon Sep 17 00:00:00 2001
From: Dhruv Chawla <dhruvc at nvidia.com>
Date: Tue, 30 Apr 2024 17:04:00 +0530
Subject: [PATCH] [GISel][CombinerHelper] Push freeze through
non-poison-producing operands
This combine matches the existing fold in InstCombine, i.e.
InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating.
It tries to push freeze through an operand if the operand has only one
maybe-poison operand and all other operands are guaranteed non-poison.
This is beneficial because it can potentially enable other optimizations
to occur that would otherwise be blocked because of the freeze.
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 2 +
.../CodeGen/GlobalISel/GenericMachineInstrs.h | 29 +++++++++
.../include/llvm/Target/GlobalISel/Combine.td | 9 ++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 59 +++++++++++++++++++
llvm/lib/CodeGen/GlobalISel/Utils.cpp | 17 +++++-
llvm/lib/Target/AArch64/AArch64Combine.td | 3 +-
.../AArch64/GlobalISel/combine-select.mir | 42 ++++++-------
llvm/test/CodeGen/AArch64/pr58431.ll | 4 +-
8 files changed, 139 insertions(+), 26 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 76e8d1166ae0cd..bd59aab03a6103 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -856,6 +856,8 @@ class CombinerHelper {
/// Combine insert vector element OOB.
bool matchInsertVectorElementOOB(MachineInstr &MI, BuildFnTy &MatchInfo);
+ bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo);
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 25e47114e4a39a..90a806df86226a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -780,6 +780,35 @@ class GFreeze : public GenericMachineInstr {
}
};
+/// Represents cast instructions.
+class GCastOp : public GenericMachineInstr {
+public:
+ Register getSourceReg() const { return getOperand(1).getReg(); }
+
+ static bool classof(const MachineInstr *MI) {
+ switch (MI->getOpcode()) {
+ case TargetOpcode::G_TRUNC:
+ case TargetOpcode::G_ANYEXT:
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_FPTOSI:
+ case TargetOpcode::G_FPTOUI:
+ case TargetOpcode::G_UITOFP:
+ case TargetOpcode::G_SITOFP:
+ case TargetOpcode::G_FPTRUNC:
+ case TargetOpcode::G_FPEXT:
+ case TargetOpcode::G_INTTOPTR:
+ case TargetOpcode::G_PTRTOINT:
+ case TargetOpcode::G_BITCAST:
+ case TargetOpcode::G_ADDRSPACE_CAST:
+ return true;
+
+ default:
+ return false;
+ }
+ }
+};
+
} // namespace llvm
#endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index dbbb3abaa83047..7185cd2bbcb136 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -218,6 +218,13 @@ def idempotent_prop : GICombineRule<
(match (idempotent_prop_frags $dst, $src)),
(apply (GIReplaceReg $dst, $src))>;
+// Convert freeze(Op(Op0, NonPoisonOps...)) to Op(freeze(Op0), NonPoisonOps...)
+// when Op0 is not guaranteed non-poison
+def push_freeze_to_prevent_poison_propagation : GICombineRule<
+ (defs root:$root, build_fn_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_FREEZE):$root,
+ [{ return Helper.matchFreezeOfSingleMaybePoisonOperand(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
def extending_loads : GICombineRule<
(defs root:$root, extending_load_matchdata:$matchinfo),
@@ -1662,7 +1669,7 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
sub_add_reg, select_to_minmax, redundant_binop_in_equality,
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
combine_concat_vector, double_icmp_zero_and_or_combine, match_addos,
- combine_shuffle_concat]>;
+ combine_shuffle_concat, push_freeze_to_prevent_poison_propagation]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 653e7689b57743..0a79702bf7080d 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -223,6 +223,64 @@ void CombinerHelper::applyCombineCopy(MachineInstr &MI) {
replaceRegWith(MRI, DstReg, SrcReg);
}
+bool CombinerHelper::matchFreezeOfSingleMaybePoisonOperand(
+ MachineInstr &MI, BuildFnTy &MatchInfo) {
+ // Ported from InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating
+ Register DstOp = MI.getOperand(0).getReg();
+ Register OrigOp = MI.getOperand(1).getReg();
+
+ if (OrigOp.isPhysical() || !MRI.hasOneNonDBGUse(OrigOp))
+ return false;
+
+ MachineInstr *OrigDef = MRI.getUniqueVRegDef(OrigOp);
+ // Avoid trying to fold G_PHI, G_UNMERGE_VALUES, G_FREEZE (the latter is
+ // handled by idempotent_prop)
+ if (!OrigDef || OrigDef->isPHI() || isa<GUnmerge>(OrigDef) ||
+ isa<GFreeze>(OrigDef))
+ return false;
+
+ if (canCreateUndefOrPoison(OrigOp, MRI))
+ return false;
+
+ std::optional<MachineOperand> MaybePoisonOperand = std::nullopt;
+ for (MachineOperand &Operand : OrigDef->uses()) {
+ // Avoid working on non-register operands or physical registers.
+ if (!Operand.isReg() || Operand.getReg().isPhysical())
+ return false;
+
+ if (isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI))
+ continue;
+
+ if (!MaybePoisonOperand)
+ MaybePoisonOperand = Operand;
+ // We have more than one maybe-poison operand. Moving the freeze is unsafe.
+ else
+ return false;
+ }
+
+ // Eliminate freeze if all operands are guaranteed non-poison
+ if (!MaybePoisonOperand) {
+ MatchInfo = [=](MachineIRBuilder &B) { MRI.replaceRegWith(DstOp, OrigOp); };
+ return true;
+ }
+
+ if (!MaybePoisonOperand->isReg())
+ return false;
+
+ Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
+ LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);
+
+ MatchInfo = [=](MachineIRBuilder &B) mutable {
+ auto Reg = MRI.createGenericVirtualRegister(MaybePoisonOperandRegTy);
+ B.setInsertPt(*OrigDef->getParent(), OrigDef->getIterator());
+ B.buildFreeze(Reg, MaybePoisonOperandReg);
+ replaceRegOpWith(
+ MRI, *OrigDef->findRegisterUseOperand(MaybePoisonOperandReg, TRI), Reg);
+ replaceRegWith(MRI, DstOp, OrigOp);
+ };
+ return true;
+}
+
bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI,
SmallVector<Register> &Ops) {
assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
@@ -3060,6 +3118,7 @@ bool CombinerHelper::matchHoistLogicOpWithSameOpcodeHands(
MachineInstr *RightHandInst = getDefIgnoringCopies(RHSReg, MRI);
if (!LeftHandInst || !RightHandInst)
return false;
+
unsigned HandOpcode = LeftHandInst->getOpcode();
if (HandOpcode != RightHandInst->getOpcode())
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 4e3781cb4e9d59..3f9497fcb6855a 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1737,6 +1737,8 @@ static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
case TargetOpcode::G_FREEZE:
return false;
default:
+ if (isa<GCastOp>(RegDef) || isa<GBinOp>(RegDef))
+ return false;
return true;
}
}
@@ -1750,13 +1752,26 @@ static bool isGuaranteedNotToBeUndefOrPoison(Register Reg,
MachineInstr *RegDef = MRI.getVRegDef(Reg);
+ auto OpCheck = [&](MachineOperand &Operand) {
+ if (!Operand.isReg())
+ return true;
+
+ return isGuaranteedNotToBeUndefOrPoison(Operand.getReg(), MRI, Depth + 1,
+ Kind);
+ };
+
switch (RegDef->getOpcode()) {
case TargetOpcode::G_FREEZE:
return true;
case TargetOpcode::G_IMPLICIT_DEF:
return !includesUndef(Kind);
default:
- return false;
+ GenericMachineInstr *Opr = dyn_cast<GBinOp>(RegDef);
+ if (!Opr)
+ Opr = dyn_cast<GCastOp>(RegDef);
+
+ return Opr && !::llvm::canCreateUndefOrPoison(Reg, MRI) &&
+ all_of(Opr->operands(), OpCheck);
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 10cad6d1924407..7229d4fe52261f 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -295,5 +295,6 @@ def AArch64PostLegalizerCombiner
ptr_add_immed_chain, overlapping_and,
split_store_zero_128, undef_combines,
select_to_minmax, or_to_bsp, combine_concat_vector,
- commute_constant_to_rhs]> {
+ commute_constant_to_rhs,
+ push_freeze_to_prevent_poison_propagation]> {
}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
index 2bf7e84a379ba0..90d0896d168930 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-select.mir
@@ -117,9 +117,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -144,9 +144,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_OR %c, %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -172,9 +172,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d2
; CHECK-NEXT: %c:_(<2 x s1>) = G_TRUNC [[COPY]](<2 x s32>)
- ; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[COPY1]](<2 x s32>)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s1>) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<2 x s32>) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(<2 x s1>) = G_TRUNC [[FREEZE]](<2 x s32>)
+ ; CHECK-NEXT: %sel:_(<2 x s1>) = G_OR %c, %f
; CHECK-NEXT: %ext:_(<2 x s32>) = G_ANYEXT %sel(<2 x s1>)
; CHECK-NEXT: $d0 = COPY %ext(<2 x s32>)
%0:_(<2 x s32>) = COPY $d0
@@ -201,9 +201,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -229,9 +229,9 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_AND %c, [[FREEZE]]
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
+ ; CHECK-NEXT: %sel:_(s1) = G_AND %c, %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -257,11 +257,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %t:_(s1) = G_TRUNC [[FREEZE]](s64)
; CHECK-NEXT: %one:_(s1) = G_CONSTANT i1 true
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, %one
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %t
- ; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], [[FREEZE]]
+ ; CHECK-NEXT: %sel:_(s1) = G_OR [[XOR]], %t
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
@@ -287,11 +287,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x2
; CHECK-NEXT: %c:_(s1) = G_TRUNC [[COPY]](s64)
- ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[COPY1]](s64)
+ ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s64) = G_FREEZE [[COPY1]]
+ ; CHECK-NEXT: %f:_(s1) = G_TRUNC [[FREEZE]](s64)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR %c, [[C]]
- ; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE %f
- ; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], [[FREEZE]]
+ ; CHECK-NEXT: %sel:_(s1) = G_AND [[XOR]], %f
; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %sel(s1)
; CHECK-NEXT: $w0 = COPY %ext(s32)
%0:_(s64) = COPY $x0
diff --git a/llvm/test/CodeGen/AArch64/pr58431.ll b/llvm/test/CodeGen/AArch64/pr58431.ll
index dcd97597ae4093..e87d8f7874d62e 100644
--- a/llvm/test/CodeGen/AArch64/pr58431.ll
+++ b/llvm/test/CodeGen/AArch64/pr58431.ll
@@ -4,8 +4,8 @@
define i32 @f(i64 %0) {
; CHECK-LABEL: f:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #10
-; CHECK-NEXT: mov w9, w0
+; CHECK-NEXT: mov w8, #10 // =0xa
+; CHECK-NEXT: and x9, x0, #0xffffffff
; CHECK-NEXT: udiv x10, x9, x8
; CHECK-NEXT: msub x0, x10, x8, x9
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
More information about the llvm-commits
mailing list