[llvm] [InstCombine] Evaluate expressions in a different type iteratively (PR #217879)
Maksim Shelegov via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 03:40:28 PDT 2026
https://github.com/mshelego updated https://github.com/llvm/llvm-project/pull/217879
>From fd93482a9a333ff49030314ccbde52e2ef0949c8 Mon Sep 17 00:00:00 2001
From: "Shelegov, Maksim" <maksim.shelegov at intel.com>
Date: Fri, 21 Aug 2026 19:39:41 +0200
Subject: [PATCH] [InstCombine] Limit recursion depth when evaluating in a
different type
canEvaluateTruncated(), canEvaluateZExtd() and canEvaluateSExtd() recurse once
per level of the expression they are asked about, with nothing bounding how far
they go, so a deep enough chain of eligible operations feeding a cast overflows
the stack. Chains like that are not exotic: C and OpenCL promote char and short
arithmetic to int, so an unrolled loop over a char array turns into a long chain
of i32 operations rooted at a truncate.
Give up past MaxTypeEvalDepth levels. Depth is bounded rather than the number of
values visited, because depth is what costs stack -- a wide but shallow
expression is still folded no matter how many values it has. Giving up early
only ever costs a fold.
EvaluateInDifferentType() walks the accepted expression the same way afterwards,
and needs no limit of its own: it is only ever reached for an expression one of
the predicates has just accepted, and it descends the same operands.
---
.../InstCombine/InstCombineCasts.cpp | 20 ++++
.../evaluate-in-different-type-limit.ll | 93 +++++++++++++++++++
2 files changed, 113 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/evaluate-in-different-type-limit.ll
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 2db6396b9d661..ddab197307179 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -24,7 +24,9 @@
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
+#include "llvm/Support/SaveAndRestore.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include <iterator>
#include <optional>
@@ -34,6 +36,11 @@ using namespace PatternMatch;
#define DEBUG_TYPE "instcombine"
+static cl::opt<unsigned> MaxTypeEvalDepth(
+ "instcombine-max-type-eval-depth", cl::Hidden, cl::init(128),
+ cl::desc("Maximum recursion depth when checking whether an expression can "
+ "be evaluated in a different type"));
+
using EvaluatedMap = SmallDenseMap<Value *, Value *, 8>;
static Value *EvaluateInDifferentTypeImpl(Value *V, Type *Ty, bool isSigned,
@@ -337,6 +344,9 @@ class TypeEvaluationHelper {
[[nodiscard]] bool
canEvaluate(Value *V, Type *Ty,
llvm::function_ref<bool(Value *, Type *Type)> Pred) {
+ if (Depth >= MaxTypeEvalDepth)
+ return false;
+
if (canAlwaysEvaluateInType(V, Ty))
return true;
@@ -437,6 +447,7 @@ class TypeEvaluationHelper {
}
}
+ SaveAndRestore RestoreDepth(Depth, Depth + 1);
const bool Result = Pred(V, Ty);
// We have to set result this way and not via It because Pred is recursive
// and it is very likely that we grew Visited and invalidated It.
@@ -461,6 +472,9 @@ class TypeEvaluationHelper {
[[nodiscard]] bool canEvaluateSExtdImpl(Value *V, Type *Ty);
[[nodiscard]] bool canEvaluateSExtdPred(Value *V, Type *Ty);
+ /// The recursion depth of this traversal.
+ unsigned Depth = 0;
+
/// A bookkeeping map to memorize an already made decision for a traversed
/// value.
SmallDenseMap<Value *, bool, 8> Visited;
@@ -1469,6 +1483,9 @@ bool TypeEvaluationHelper::canEvaluateZExtdImpl(Value *V, Type *Ty,
InstCombinerImpl &IC,
Instruction *CxtI) {
BitsToClear = 0;
+ if (Depth >= MaxTypeEvalDepth)
+ return false;
+
if (canAlwaysEvaluateInType(V, Ty))
return true;
// We stick to the one-user limit for the ZExt transform due to the fact
@@ -1476,6 +1493,9 @@ bool TypeEvaluationHelper::canEvaluateZExtdImpl(Value *V, Type *Ty,
if (canNotEvaluateInType(V, Ty))
return false;
+ // canEvaluateZExtd does not go through canEvaluate (it also returns
+ // BitsToClear), so it has to track the depth itself.
+ SaveAndRestore RestoreDepth(Depth, Depth + 1);
auto *I = cast<Instruction>(V);
unsigned Tmp;
switch (I->getOpcode()) {
diff --git a/llvm/test/Transforms/InstCombine/evaluate-in-different-type-limit.ll b/llvm/test/Transforms/InstCombine/evaluate-in-different-type-limit.ll
new file mode 100644
index 0000000000000..9ea0980bc6c2d
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/evaluate-in-different-type-limit.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s --check-prefixes=CHECK,DEFAULT
+; RUN: opt -passes=instcombine -instcombine-max-type-eval-depth=4 -S < %s | FileCheck %s --check-prefixes=CHECK,CAPPED
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i8 @trunc_too_deep(i8 %a, i8 %b) {
+; DEFAULT-LABEL: define i8 @trunc_too_deep(
+; DEFAULT-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; DEFAULT-NEXT: [[M:%.*]] = add i8 [[A]], [[B]]
+; DEFAULT-NEXT: [[X0:%.*]] = mul i8 [[M]], [[M]]
+; DEFAULT-NEXT: [[X1:%.*]] = xor i8 [[X0]], [[A]]
+; DEFAULT-NEXT: [[X2:%.*]] = or i8 [[X1]], [[M]]
+; DEFAULT-NEXT: ret i8 [[X2]]
+;
+; CAPPED-LABEL: define i8 @trunc_too_deep(
+; CAPPED-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CAPPED-NEXT: [[WA:%.*]] = zext i8 [[A]] to i32
+; CAPPED-NEXT: [[WB:%.*]] = zext i8 [[B]] to i32
+; CAPPED-NEXT: [[M:%.*]] = add nuw nsw i32 [[WA]], [[WB]]
+; CAPPED-NEXT: [[X0:%.*]] = mul nuw nsw i32 [[M]], [[M]]
+; CAPPED-NEXT: [[X1:%.*]] = xor i32 [[X0]], [[WA]]
+; CAPPED-NEXT: [[X2:%.*]] = or i32 [[X1]], [[M]]
+; CAPPED-NEXT: [[T:%.*]] = trunc i32 [[X2]] to i8
+; CAPPED-NEXT: ret i8 [[T]]
+;
+ %wa = zext i8 %a to i32
+ %wb = zext i8 %b to i32
+ %m = add i32 %wa, %wb
+ %x0 = mul i32 %m, %m
+ %x1 = xor i32 %x0, %wa
+ %x2 = or i32 %x1, %m
+ %t = trunc i32 %x2 to i8
+ ret i8 %t
+}
+
+define i8 @trunc_wide_but_shallow(i8 %a, i8 %b, i8 %c, i8 %d) {
+; CHECK-LABEL: define i8 @trunc_wide_but_shallow(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i8 [[D:%.*]]) {
+; CHECK-NEXT: [[L0:%.*]] = mul i8 [[A]], [[B]]
+; CHECK-NEXT: [[L1:%.*]] = mul i8 [[C]], [[D]]
+; CHECK-NEXT: [[L2:%.*]] = mul i8 [[B]], [[C]]
+; CHECK-NEXT: [[L3:%.*]] = mul i8 [[D]], [[A]]
+; CHECK-NEXT: [[M0:%.*]] = mul i8 [[L0]], [[L1]]
+; CHECK-NEXT: [[M1:%.*]] = mul i8 [[L2]], [[L3]]
+; CHECK-NEXT: [[R:%.*]] = mul i8 [[M0]], [[M1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %wa = zext i8 %a to i32
+ %wb = zext i8 %b to i32
+ %wc = zext i8 %c to i32
+ %wd = zext i8 %d to i32
+ %l0 = mul i32 %wa, %wb
+ %l1 = mul i32 %wc, %wd
+ %l2 = mul i32 %wb, %wc
+ %l3 = mul i32 %wd, %wa
+ %m0 = mul i32 %l0, %l1
+ %m1 = mul i32 %l2, %l3
+ %r = mul i32 %m0, %m1
+ %t = trunc i32 %r to i8
+ ret i8 %t
+}
+
+define i32 @zext_too_deep(i32 %p, i32 %q) {
+; DEFAULT-LABEL: define i32 @zext_too_deep(
+; DEFAULT-SAME: i32 [[P:%.*]], i32 [[Q:%.*]]) {
+; DEFAULT-NEXT: [[X0:%.*]] = add i32 [[P]], [[Q]]
+; DEFAULT-NEXT: [[X1:%.*]] = mul i32 [[X0]], [[P]]
+; DEFAULT-NEXT: [[X2:%.*]] = xor i32 [[X1]], [[Q]]
+; DEFAULT-NEXT: [[X3:%.*]] = or i32 [[X2]], [[P]]
+; DEFAULT-NEXT: [[T:%.*]] = and i32 [[X3]], 255
+; DEFAULT-NEXT: ret i32 [[T]]
+;
+; CAPPED-LABEL: define i32 @zext_too_deep(
+; CAPPED-SAME: i32 [[P:%.*]], i32 [[Q:%.*]]) {
+; CAPPED-NEXT: [[A:%.*]] = trunc i32 [[P]] to i8
+; CAPPED-NEXT: [[B:%.*]] = trunc i32 [[Q]] to i8
+; CAPPED-NEXT: [[X0:%.*]] = add i8 [[A]], [[B]]
+; CAPPED-NEXT: [[X1:%.*]] = mul i8 [[X0]], [[A]]
+; CAPPED-NEXT: [[X2:%.*]] = xor i8 [[X1]], [[B]]
+; CAPPED-NEXT: [[X3:%.*]] = or i8 [[X2]], [[A]]
+; CAPPED-NEXT: [[T:%.*]] = zext i8 [[X3]] to i32
+; CAPPED-NEXT: ret i32 [[T]]
+;
+ %a = trunc i32 %p to i8
+ %b = trunc i32 %q to i8
+ %x0 = add i8 %a, %b
+ %x1 = mul i8 %x0, %a
+ %x2 = xor i8 %x1, %b
+ %x3 = or i8 %x2, %a
+ %t = zext i8 %x3 to i32
+ ret i32 %t
+}
More information about the llvm-commits
mailing list