[llvm] [InstCombine] Evaluate expressions in a different type iteratively (PR #217879)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 21 08:46:52 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Maksim Shelegov (mshelego)
<details>
<summary>Changes</summary>
canEvaluateTruncated, canEvaluateZExtd, canEvaluateSExtd and
EvaluateInDifferentType walked the expression by recursing on operands, so
the traversal depth was bounded by the size of the native stack and deep
expressions could overflow it.
Rewrite both walks to use an explicit stack. The canEvaluate* predicates are
split into a "pre" step, which decides whether a value is admissible on its
own and reports the operands to walk into, and a "post" step, which combines
the verdicts of those operands. Only the zext predicate needs the latter, to
propagate BitsToClear. EvaluateInDifferentType is split the same way, into
collectOperandsToEvaluate and createInDifferentType.
The Visited map now distinguishes in-progress values from finished ones,
which preserves the existing conservative rejection of use chain loops. Note
that a negative verdict is never recorded anymore, because the first
rejection aborts the whole traversal.
Other than no longer being bounded by the native stack, this is intended to
be NFC.
---
Patch is 49.16 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217879.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp (+452-198)
- (added) llvm/test/Transforms/InstCombine/evaluate-in-different-type.ll (+370)
``````````diff
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 2db6396b9d661..2bf5de920989d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -12,6 +12,7 @@
#include "InstCombineInternal.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/STLFunctionalExtras.h"
@@ -36,20 +37,69 @@ using namespace PatternMatch;
using EvaluatedMap = SmallDenseMap<Value *, Value *, 8>;
-static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
- InstCombinerImpl &IC,
- EvaluatedMap &Processed) {
- // Since we cover transformation of instructions with multiple users, we might
- // come to the same node via multiple paths. We should not create a
- // replacement for every single one of them though.
- if (Value *Result = Processed.lookup(V))
- return Result;
-
- if (Constant *C = dyn_cast<Constant>(V))
- return ConstantFoldIntegerCast(C, Ty, isSigned, IC.getDataLayout());
+/// Collect the operands of \p I that have to be rebuilt in the destination type
+/// before \p I itself can be rebuilt. \p OpTy receives the type to rebuild them
+/// in, which differs from \p Ty only for shuffles, whose operands keep their
+/// own element count. This has to agree with createInDifferentType below, which
+/// consumes the results positionally.
+static void collectOperandsToEvaluate(Instruction *I, Type *Ty, Type *&OpTy,
+ SmallVectorImpl<Value *> &Ops) {
+ OpTy = Ty;
+ switch (I->getOpcode()) {
+ case Instruction::Add:
+ case Instruction::Sub:
+ case Instruction::Mul:
+ case Instruction::And:
+ case Instruction::Or:
+ case Instruction::Xor:
+ case Instruction::AShr:
+ case Instruction::LShr:
+ case Instruction::Shl:
+ case Instruction::UDiv:
+ case Instruction::URem:
+ Ops.append({I->getOperand(0), I->getOperand(1)});
+ return;
+ case Instruction::Select:
+ Ops.append({I->getOperand(1), I->getOperand(2)});
+ return;
+ case Instruction::PHI:
+ for (Value *IncValue : cast<PHINode>(I)->incoming_values())
+ Ops.push_back(IncValue);
+ return;
+ case Instruction::Call:
+ if (const auto *II = dyn_cast<IntrinsicInst>(I)) {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::umin:
+ case Intrinsic::umax:
+ case Intrinsic::smin:
+ case Intrinsic::smax:
+ Ops.append({II->getArgOperand(0), II->getArgOperand(1)});
+ return;
+ case Intrinsic::abs:
+ Ops.push_back(II->getArgOperand(0));
+ return;
+ default:
+ return;
+ }
+ }
+ return;
+ case Instruction::ShuffleVector: {
+ auto *ScalarTy = cast<VectorType>(Ty)->getElementType();
+ auto *VTy = cast<VectorType>(I->getOperand(0)->getType());
+ OpTy = VectorType::get(ScalarTy, VTy->getElementCount());
+ Ops.append({I->getOperand(0), I->getOperand(1)});
+ return;
+ }
+ default:
+ return;
+ }
+}
- // Otherwise, it must be an instruction.
- Instruction *I = cast<Instruction>(V);
+/// Rebuild \p I in type \p Ty, taking the already rebuilt operands reported by
+/// collectOperandsToEvaluate from \p NewOps.
+static Value *createInDifferentType(Instruction *I, Type *Ty, bool isSigned,
+ ArrayRef<Value *> NewOps,
+ InstCombinerImpl &IC) {
Instruction *Res = nullptr;
unsigned Opc = I->getOpcode();
switch (Opc) {
@@ -64,11 +114,8 @@ static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
case Instruction::Shl:
case Instruction::UDiv:
case Instruction::URem: {
- Value *LHS = EvaluateInDifferentTypeImpl(I->getOperand(0), Ty, isSigned, IC,
- Processed);
- Value *RHS = EvaluateInDifferentTypeImpl(I->getOperand(1), Ty, isSigned, IC,
- Processed);
- Res = BinaryOperator::Create((Instruction::BinaryOps)Opc, LHS, RHS);
+ Res = BinaryOperator::Create((Instruction::BinaryOps)Opc, NewOps[0],
+ NewOps[1]);
if (Opc == Instruction::LShr || Opc == Instruction::AShr)
Res->setIsExact(I->isExact());
break;
@@ -100,21 +147,14 @@ static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
}
break;
case Instruction::Select: {
- Value *True = EvaluateInDifferentTypeImpl(I->getOperand(1), Ty, isSigned,
- IC, Processed);
- Value *False = EvaluateInDifferentTypeImpl(I->getOperand(2), Ty, isSigned,
- IC, Processed);
- Res = SelectInst::Create(I->getOperand(0), True, False);
+ Res = SelectInst::Create(I->getOperand(0), NewOps[0], NewOps[1]);
break;
}
case Instruction::PHI: {
PHINode *OPN = cast<PHINode>(I);
PHINode *NPN = PHINode::Create(Ty, OPN->getNumIncomingValues());
- for (unsigned i = 0, e = OPN->getNumIncomingValues(); i != e; ++i) {
- Value *V = EvaluateInDifferentTypeImpl(OPN->getIncomingValue(i), Ty,
- isSigned, IC, Processed);
- NPN->addIncoming(V, OPN->getIncomingBlock(i));
- }
+ for (unsigned i = 0, e = OPN->getNumIncomingValues(); i != e; ++i)
+ NPN->addIncoming(NewOps[i], OPN->getIncomingBlock(i));
Res = NPN;
break;
}
@@ -138,36 +178,25 @@ static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
case Intrinsic::umax:
case Intrinsic::smin:
case Intrinsic::smax: {
- Value *Op0 = EvaluateInDifferentTypeImpl(II->getArgOperand(0), Ty,
- isSigned, IC, Processed);
- Value *Op1 = EvaluateInDifferentTypeImpl(II->getArgOperand(1), Ty,
- isSigned, IC, Processed);
Function *Fn = Intrinsic::getOrInsertDeclaration(
I->getModule(), II->getIntrinsicID(), {Ty});
- Res = CallInst::Create(Fn->getFunctionType(), Fn, {Op0, Op1});
+ Res =
+ CallInst::Create(Fn->getFunctionType(), Fn, {NewOps[0], NewOps[1]});
break;
}
case Intrinsic::abs: {
- Value *Arg = EvaluateInDifferentTypeImpl(II->getArgOperand(0), Ty,
- isSigned, IC, Processed);
Function *Fn = Intrinsic::getOrInsertDeclaration(
I->getModule(), II->getIntrinsicID(), {Ty});
- Res = CallInst::Create(Fn->getFunctionType(), Fn,
- {Arg, ConstantInt::getFalse(I->getContext())});
+ Res = CallInst::Create(
+ Fn->getFunctionType(), Fn,
+ {NewOps[0], ConstantInt::getFalse(I->getContext())});
break;
}
}
}
break;
case Instruction::ShuffleVector: {
- auto *ScalarTy = cast<VectorType>(Ty)->getElementType();
- auto *VTy = cast<VectorType>(I->getOperand(0)->getType());
- auto *FixedTy = VectorType::get(ScalarTy, VTy->getElementCount());
- Value *Op0 = EvaluateInDifferentTypeImpl(I->getOperand(0), FixedTy,
- isSigned, IC, Processed);
- Value *Op1 = EvaluateInDifferentTypeImpl(I->getOperand(1), FixedTy,
- isSigned, IC, Processed);
- Res = new ShuffleVectorInst(Op0, Op1,
+ Res = new ShuffleVectorInst(NewOps[0], NewOps[1],
cast<ShuffleVectorInst>(I)->getShuffleMask());
break;
}
@@ -177,14 +206,80 @@ static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
}
Res->takeName(I);
- Value *Result = IC.InsertNewInstWith(Res, I->getIterator());
- // There is no need in keeping track of the old value/new value relationship
- // when we have only one user, we came have here from that user and no-one
- // else cares.
- if (!V->hasOneUse())
- Processed[V] = Result;
+ return IC.InsertNewInstWith(Res, I->getIterator());
+}
- return Result;
+static Value *EvaluateInDifferentTypeImpl(Value *Root, Type *RootTy,
+ bool isSigned, InstCombinerImpl &IC,
+ EvaluatedMap &Processed) {
+ // Rebuild the expression bottom-up with an explicit stack, so that its depth
+ // is not bounded by the size of the native stack.
+ struct Frame {
+ Instruction *I;
+ Type *Ty;
+ Type *OpTy;
+ SmallVector<Value *, 3> Ops;
+ SmallVector<Value *, 3> NewOps;
+ };
+ SmallVector<Frame, 8> Stack;
+
+ // Produce the replacement for V if it is available right away, otherwise
+ // schedule V by pushing a frame for it.
+ auto tryVisit = [&](Value *V, Type *Ty, Value *&Out) -> bool {
+ // Since we cover transformation of instructions with multiple users, we
+ // might come to the same node via multiple paths. We should not create a
+ // replacement for every single one of them though.
+ if (Value *Result = Processed.lookup(V)) {
+ Out = Result;
+ return true;
+ }
+
+ if (Constant *C = dyn_cast<Constant>(V)) {
+ Out = ConstantFoldIntegerCast(C, Ty, isSigned, IC.getDataLayout());
+ return true;
+ }
+
+ Frame F;
+ F.I = cast<Instruction>(V);
+ F.Ty = Ty;
+ collectOperandsToEvaluate(F.I, Ty, F.OpTy, F.Ops);
+ Stack.push_back(std::move(F));
+ return false;
+ };
+
+ Value *RootResult = nullptr;
+ if (tryVisit(Root, RootTy, RootResult))
+ return RootResult;
+
+ while (true) {
+ // tryVisit below can grow Stack, so address the top frame by index instead
+ // of holding a reference that it would invalidate.
+ size_t Top = Stack.size() - 1;
+
+ if (Stack[Top].NewOps.size() != Stack[Top].Ops.size()) {
+ Value *Op = Stack[Top].Ops[Stack[Top].NewOps.size()];
+ Type *OpTy = Stack[Top].OpTy;
+ Value *NewOp = nullptr;
+ if (tryVisit(Op, OpTy, NewOp))
+ Stack[Top].NewOps.push_back(NewOp);
+ continue;
+ }
+
+ Instruction *I = Stack[Top].I;
+ Value *Result = createInDifferentType(I, Stack[Top].Ty, isSigned,
+ Stack[Top].NewOps, IC);
+ Stack.pop_back();
+
+ // There is no need in keeping track of the old value/new value relationship
+ // when we have only one user, we came have here from that user and no-one
+ // else cares.
+ if (!I->hasOneUse())
+ Processed[I] = Result;
+
+ if (Stack.empty())
+ return Result;
+ Stack.back().NewOps.push_back(Result);
+ }
}
/// Given an expression that CanEvaluateTruncated or CanEvaluateSExtd returns
@@ -332,38 +427,75 @@ class TypeEvaluationHelper {
[this](Value *V) { return Visited.contains(V); });
}
- /// A generic wrapper for canEvaluate* recursions to inject visitation
- /// tracking and enforce correct multi-use value evaluations.
- [[nodiscard]] bool
- canEvaluate(Value *V, Type *Ty,
- llvm::function_ref<bool(Value *, Type *Type)> Pred) {
+ enum class VisitState {
+ /// The value is on the traversal stack, its operands are still being
+ /// checked.
+ InProgress,
+ /// The value and all of its operands were accepted.
+ Finished,
+ };
+
+ struct VisitInfo {
+ VisitState State;
+ /// The verdict payload, i.e. BitsToClear for the zext traversal. Unused by
+ /// the traversals that only need a yes/no answer.
+ unsigned Result = 0;
+ };
+
+ /// The outcome of looking at a single value during the traversal.
+ enum class Visit {
+ /// The value can not be evaluated in the destination type.
+ Rejected,
+ /// The value needs no further traversal, Result is final.
+ Settled,
+ /// The value was pushed onto the stack, its operands come next.
+ Pushed,
+ };
+
+ struct Frame {
+ Value *V;
+ SmallVector<Value *, 3> Children;
+ SmallVector<unsigned, 3> ChildResults;
+ };
+
+ /// Decides whether \p V is admissible on its own and reports the operands
+ /// that have to be evaluated in the destination type as well.
+ using PreFn =
+ llvm::function_ref<bool(Value *, Type *, SmallVectorImpl<Value *> &)>;
+ /// Combines the verdicts of the operands reported by PreFn into the verdict
+ /// for the value itself.
+ using PostFn =
+ llvm::function_ref<bool(Value *, ArrayRef<unsigned>, unsigned &)>;
+
+ /// Look at a single value and either settle it right away or schedule its
+ /// operands by pushing a frame onto \p Stack.
+ [[nodiscard]] Visit tryVisit(Value *V, Type *Ty, PreFn Pre,
+ SmallVectorImpl<Frame> &Stack,
+ unsigned &Result) {
+ Result = 0;
+
if (canAlwaysEvaluateInType(V, Ty))
- return true;
+ return Visit::Settled;
auto *I = dyn_cast<Instruction>(V);
if (I == nullptr)
- return false;
-
- // We insert false by default to return false when we encounter user loops.
- const auto [It, Inserted] = Visited.insert({V, false});
-
- // There are three possible cases for us having information on this value
- // in the Visited map:
- // 1. We properly checked it and concluded that we can evaluate it (true)
- // 2. We properly checked it and concluded that we can't (false)
- // 3. We started to check it, but during the recursive traversal we came
- // back to it.
- //
- // For cases 1 and 2, we can safely return the stored result. For case 3, we
- // can potentially have a situation where we can evaluate recursive user
- // chains, but that can be quite tricky to do properly and isntead, we
- // return false.
- //
- // In any case, we should return whatever was there in the map to begin
- // with.
- if (!Inserted)
- return It->getSecond();
+ return Visit::Rejected;
+
+ // There are two possible cases for us having information on this value in
+ // the Visited map:
+ // 1. We properly checked it and concluded that we can evaluate it. Note
+ // that we never get to record a negative verdict, because the first
+ // rejection aborts the whole traversal.
+ // 2. We started to check it, but came back to it through a use chain
+ // loop. We can potentially have a situation where we can evaluate
+ // recursive user chains, but that can be quite tricky to do properly
+ // and instead, we reject.
+ if (auto It = Visited.find(V); It != Visited.end()) {
+ Result = It->getSecond().Result;
+ return It->getSecond().State == VisitState::Finished ? Visit::Settled
+ : Visit::Rejected;
+ }
// We can easily make a decision about single-user values whether they can
// be evaluated in a different type or not, we came from that user. This is
@@ -431,17 +563,85 @@ class TypeEvaluationHelper {
// conditions. Make sure we don't retain an extra use via the select
// condition.
if (isa<SelectInst>(U.getUser()) && U.getOperandNo() == 0)
- return false;
+ return Visit::Rejected;
Pending.push_back(U.getUser());
}
}
- const bool Result = Pred(V, Ty);
- // We have to set result this way and not via It because Pred is recursive
- // and it is very likely that we grew Visited and invalidated It.
- Visited[V] = Result;
- return Result;
+ Frame F;
+ F.V = V;
+ if (!Pre(V, Ty, F.Children))
+ return Visit::Rejected;
+
+ Visited[V] = {VisitState::InProgress};
+ Stack.push_back(std::move(F));
+ return Visit::Pushed;
+ }
+
+ /// A generic driver for the canEvaluate* traversals to inject visitation
+ /// tracking and enforce correct multi-use value evaluations. The expression
+ /// is walked in post-order with an explicit stack, so that its depth is not
+ /// bounded by the size of the native stack.
+ ///
+ /// \p RootResult carries the verdict payload of \p Root, which only the zext
+ /// traversal makes use of.
+ [[nodiscard]] bool canEvaluate(Value *Root, Type *Ty, PreFn Pre, PostFn Post,
+ unsigned &RootResult) {
+ SmallVector<Frame, 8> Stack;
+
+ switch (tryVisit(Root, Ty, Pre, Stack, RootResult)) {
+ case Visit::Rejected:
+ return false;
+ case Visit::Settled:
+ return true;
+ case Visit::Pushed:
+ break;
+ }
+
+ while (true) {
+ // tryVisit below can grow Stack, so address the top frame by index
+ // instead of holding a reference that it would invalidate.
+ size_t Top = Stack.size() - 1;
+
+ if (Stack[Top].ChildResults.size() != Stack[Top].Children.size()) {
+ Value *Child = Stack[Top].Children[Stack[Top].ChildResults.size()];
+ unsigned ChildResult;
+ switch (tryVisit(Child, Ty, Pre, Stack, ChildResult)) {
+ case Visit::Rejected:
+ return false;
+ case Visit::Settled:
+ Stack[Top].ChildResults.push_back(ChildResult);
+ break;
+ case Visit::Pushed:
+ break;
+ }
+ continue;
+ }
+
+ Value *V = Stack[Top].V;
+ unsigned Result = 0;
+ if (!Stack[Top].Children.empty() &&
+ !Post(V, Stack[Top].ChildResults, Result))
+ return false;
+
+ Stack.pop_back();
+ Visited[V] = {VisitState::Finished, Result};
+
+ if (Stack.empty()) {
+ RootResult = Result;
+ return true;
+ }
+ Stack.back().ChildResults.push_back(Result);
+ }
+ }
+
+ /// An overload for the traversals that have no verdict payload to combine.
+ [[nodiscard]] bool canEvaluate(Value *Root, Type *Ty, PreFn Pre) {
+ unsigned Unused = 0;
+ return canEvaluate(
+ Root, Ty, Pre,
+ [](Value *, ArrayRef<unsigned>, unsigned &) { return true; }, Unused);
}
/// Filter out values that we can not evaluate in the destination type for
@@ -451,19 +651,30 @@ class TypeEvaluationHelper {
[[nodiscard]] bool canEvaluateTruncatedImpl(Value *V, Type *Ty,
InstCombinerImpl &IC,
Instruction *CxtI);
- [[nodiscard]] bool canEvaluateTruncatedPred(Value *V, Type *Ty,
- InstCombinerImpl &IC,
- Instruction *CxtI);
+ [[nodiscard]] bool
+ canEvaluateTruncatedPred(Value *V, Type *Ty, InstCombinerImpl &IC,
+ Instruction *CxtI,
+ SmallVectorImpl<Value *> &Children);
[[nodiscard]] bool canEvaluateZExtdImpl(Value *V, Type *Ty,
unsigned &BitsToClear,
InstCombinerImpl &IC,
Instruction *CxtI);
+ /// The two halves of the zext predicate. canEvaluateZExtdPost has to handle
+ /// every opcode for which canEvaluateZExtdPre reports operands.
+ [[nodiscard]] bool canEvaluateZExtdPre(Value *V, Type *Ty,
+ SmallVectorImpl<Value *> &Children);
+ [[nodiscard]] static bool canEvaluateZExtdPost(Value *V,
+ ArrayRef<unsigned> ChildBits,
+ unsigned &BitsToClear,
+ InstCombinerImpl &IC,
+ Instruction *CxtI);
[[nodiscard]] bool canEvaluateSExtdImpl(Value *V, Type *Ty);
- [[nodiscard]] bool canEvaluateSExtdPred(Value *V, Type *Ty);
+ [[nodiscard]] bool canEvaluateSExtdPred(Value *V, Type *Ty,
+ SmallVectorImpl<Value *> &Children);
/// A bookkeeping map to memorize an already made decision for a traversed
/// value.
- SmallDenseMap<Value *, bool, 8> Visited;
+ SmallDenseMap<Value *, VisitInfo, 8> Visited;
/// A list of pending values to check in the end.
SmallVector<Value *, 8> Pending;
@@ -522,14 +733,17 @@ bool TypeEvaluationHelper::canEvaluateTruncated(Value *V, Type *Ty,
bool TypeEvaluationHelper::canEvaluateTruncatedImpl(Value *V, Type *Ty,
InstCombinerImpl...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217879
More information about the llvm-commits
mailing list