[llvm] [IndVars] Recompute flags if needed in widenIVUse of IV increment. (PR #82352)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 20 04:32:00 PST 2024
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/82352
widenIVUse may hoist a wide induction increment and introduce new uses, but does not recompute the wrap flags. In some cases this can make the new uses of the wide IV inc more poisonous.
Update the code to recompute flags if needed when hoisting an IV. If both the narrow and wide IV increment's flags match and we can re-use the flags from the increments, there's no need to recompute the flags, as the replacement won't make the new uses of the wide IV's increment more poisonous.
Note that this also updates a stale comment which claimed that the widen increment is only used if it dominates the new use.
Happy to move out introducing canReuseFlagsFromOriginalIVInc to a separate patch if desired and once we agree on the API.
The helper should also be used to guard the code added in da437330be, which I am planning on doing separately once the helper lands.
Fixes https://github.com/llvm/llvm-project/issues/82243.
>From 741a29c53b0b15c3c2f05a6ed42aa2cb21671fc7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 20 Feb 2024 11:48:50 +0000
Subject: [PATCH] [IndVars] Recompute flags if needed in widenIVUse of IV
increment.
widenIVUse may hoist a wide induction increment and introduce new uses,
but does not recompute the wrap flags. In some cases this can make the
new uses of the wide IV inc more poisonous.
Update the code to recompute flags if needed when hoisting an IV. If
both the narrow and wide IV increment's flags match and we can re-use
the flags from the increments, there's no need to recompute the flags,
as the replacement won't make the new uses of the wide IV's increment
more poisonous.
Note that this also updates a stale comment which claimed that the widen
increment is only used if it dominates the new use.
Happy to move out introducing canReuseFlagsFromOriginalIVInc to a
separate patch if desired and once we agree on the API.
The helper should also be used to guard the code added in da437330be,
which I am planning on doing separately once the helper lands.
Fixes https://github.com/llvm/llvm-project/issues/82243.
---
.../Utils/ScalarEvolutionExpander.h | 8 +++++
.../Utils/ScalarEvolutionExpander.cpp | 9 ++++++
llvm/lib/Transforms/Utils/SimplifyIndVar.cpp | 30 ++++++++++++-------
...wide-inc-for-narrow-use-recompute-flags.ll | 4 +--
4 files changed, 38 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
index 035705b7f4b78b..fa10443f14bb74 100644
--- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h
@@ -258,6 +258,14 @@ class SCEVExpander : public SCEVVisitor<SCEVExpander, Value *> {
bool hoistIVInc(Instruction *IncV, Instruction *InsertPos,
bool RecomputePoisonFlags = false);
+ /// Return true if both increments directly increment the corresponding IV PHI
+ /// nodes and have the same opcode. It is not safe to re-use the flags from
+ /// the original increment, if it is more complex and SCEV expansion may have
+ /// yielded a more simplified wider increment.
+ static bool canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi, PHINode *WidePhi,
+ Instruction *OrigInc,
+ Instruction *WideInc);
+
/// replace congruent phis with their most canonical representative. Return
/// the number of phis eliminated.
unsigned replaceCongruentIVs(Loop *L, const DominatorTree *DT,
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 3a28909473d95f..fbe1dba5b8d4e4 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -771,6 +771,15 @@ bool SCEVExpander::hoistIVInc(Instruction *IncV, Instruction *InsertPos,
return true;
}
+bool SCEVExpander::canReuseFlagsFromOriginalIVInc(PHINode *OrigPhi,
+ PHINode *WidePhi,
+ Instruction *OrigInc,
+ Instruction *WideInc) {
+ return match(OrigInc, m_c_BinOp(m_Specific(OrigPhi), m_Value())) &&
+ match(WideInc, m_c_BinOp(m_Specific(WidePhi), m_Value())) &&
+ OrigInc->getOpcode() == WideInc->getOpcode();
+}
+
/// Determine if this cyclic phi is in a form that would have been generated by
/// LSR. We don't care if the phi was actually expanded in this pass, as long
/// as it is in a low-cost form, for example, no implied multiplication. This
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 5aa6df49e7f676..66bba1ca2f1d7c 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1131,7 +1131,8 @@ class WidenIV {
const SCEV *getSCEVByOpCode(const SCEV *LHS, const SCEV *RHS,
unsigned OpCode) const;
- Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter);
+ Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter,
+ PHINode *OrigPhi, PHINode *WidePhi);
bool widenLoopCompare(NarrowIVDefUse DU);
bool widenWithVariantUse(NarrowIVDefUse DU);
@@ -1731,7 +1732,9 @@ bool WidenIV::widenWithVariantUse(WidenIV::NarrowIVDefUse DU) {
/// Determine whether an individual user of the narrow IV can be widened. If so,
/// return the wide clone of the user.
-Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewriter) {
+Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
+ SCEVExpander &Rewriter, PHINode *OrigPhi,
+ PHINode *WidePhi) {
assert(ExtendKindMap.count(DU.NarrowDef) &&
"Should already know the kind of extension used to widen NarrowDef");
@@ -1825,11 +1828,18 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewri
if (!WideAddRec.first)
return nullptr;
- // Reuse the IV increment that SCEVExpander created as long as it dominates
- // NarrowUse.
+ // Reuse the IV increment that SCEVExpander created. Recompute flags, unless
+ // the flags for both increments agree and it is safe to use the ones from
+ // the original inc. In that case, the new use of the wide increment won't
+ // be more poisonous.
+ bool NeedToRecomputeFlags =
+ !SCEVExpander::canReuseFlagsFromOriginalIVInc(OrigPhi, WidePhi,
+ DU.NarrowUse, WideInc) ||
+ DU.NarrowUse->hasNoUnsignedWrap() != WideInc->hasNoUnsignedWrap() ||
+ DU.NarrowUse->hasNoSignedWrap() != WideInc->hasNoSignedWrap();
Instruction *WideUse = nullptr;
if (WideAddRec.first == WideIncExpr &&
- Rewriter.hoistIVInc(WideInc, DU.NarrowUse))
+ Rewriter.hoistIVInc(WideInc, DU.NarrowUse, NeedToRecomputeFlags))
WideUse = WideInc;
else {
WideUse = cloneIVUser(DU, WideAddRec.first);
@@ -1996,11 +2006,9 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) {
// the same opcode. It is not safe to re-use the flags from the original
// increment, if it is more complex and SCEV expansion may have yielded a
// more simplified wider increment.
- bool MatchingOps =
- match(OrigInc, m_c_BinOp(m_Specific(OrigPhi), m_Value())) &&
- match(WideInc, m_c_BinOp(m_Specific(WidePhi), m_Value())) &&
- OrigInc->getOpcode() == WideInc->getOpcode();
- if (MatchingOps && isa<OverflowingBinaryOperator>(OrigInc) &&
+ if (SCEVExpander::canReuseFlagsFromOriginalIVInc(OrigPhi, WidePhi,
+ OrigInc, WideInc) &&
+ isa<OverflowingBinaryOperator>(OrigInc) &&
isa<OverflowingBinaryOperator>(WideInc)) {
WideInc->setHasNoUnsignedWrap(WideInc->hasNoUnsignedWrap() ||
OrigInc->hasNoUnsignedWrap());
@@ -2024,7 +2032,7 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) {
// Process a def-use edge. This may replace the use, so don't hold a
// use_iterator across it.
- Instruction *WideUse = widenIVUse(DU, Rewriter);
+ Instruction *WideUse = widenIVUse(DU, Rewriter, OrigPhi, WidePhi);
// Follow all def-use edges from the previous narrow use.
if (WideUse)
diff --git a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
index ddb3890f3bb360..cc99ee312ccb7f 100644
--- a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
+++ b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
@@ -4,7 +4,7 @@
target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
; Test for https://github.com/llvm/llvm-project/issues/82243.
-; FIXME: Currently no-wrap flags on hoisted wide IV are not dropped properly.
+; Check that NUW flag on hoisted wide IV is dropped properly.
define void @test_pr82243(ptr %f) {
; CHECK-LABEL: define void @test_pr82243(
; CHECK-SAME: ptr [[F:%.*]]) {
@@ -14,7 +14,7 @@ define void @test_pr82243(ptr %f) {
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ], [ 1, [[ENTRY:%.*]] ]
; CHECK-NEXT: [[GEP_IV_EXT:%.*]] = getelementptr i32, ptr [[F]], i64 [[INDVARS_IV]]
; CHECK-NEXT: store i32 1, ptr [[GEP_IV_EXT]], align 4
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], -1
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
; CHECK-NEXT: [[SHL:%.*]] = shl i32 123, [[TMP0]]
; CHECK-NEXT: [[GEP_SHL:%.*]] = getelementptr i32, ptr [[F]], i32 [[SHL]]
More information about the llvm-commits
mailing list