[llvm] [SLP] Conversion of icmp eq to icmp ult to enable vectorization (PR #202368)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 04:54:26 PDT 2026
https://github.com/Bhuvan1527 updated https://github.com/llvm/llvm-project/pull/202368
>From cf20937741b812b81ee84d1e196d569b81e638a7 Mon Sep 17 00:00:00 2001
From: Bhuvan1527 <balabhuvanvarma at gmail.com>
Date: Mon, 22 Jun 2026 19:15:34 +0530
Subject: [PATCH 1/2] Address review comment and failed test cases
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 101 +++++++++++++++++-
1 file changed, 98 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4dcf3243a20ad..8e56ec82099d5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1635,7 +1635,33 @@ static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
Value *Op0 = CI->getOperand(0);
Value *Op1 = CI->getOperand(1);
- return (BasePred == Pred &&
+ // Helper lambdas to match our target shapes: eq %x, 0 and ult %x, C (C != 0)
+ auto IsEqZero = [](CmpInst::Predicate P, Value *L, Value *R) {
+ if (P != CmpInst::ICMP_EQ)
+ return false;
+ auto *C = dyn_cast<ConstantInt>(R);
+ return C && C->isZero();
+ };
+
+ auto IsUltNonZero = [](CmpInst::Predicate P, Value *L, Value *R) {
+ if (P != CmpInst::ICMP_ULT)
+ return false;
+ auto *C = dyn_cast<ConstantInt>(R);
+ return C && !C->isZero();
+ };
+
+ bool MixedCompatible = false;
+ if (BaseOp0 == Op0) { // Ensure the compared value (%x) is the same
+ if ((IsEqZero(BasePred, BaseOp0, BaseOp1) &&
+ IsUltNonZero(Pred, Op0, Op1)) ||
+ (IsUltNonZero(BasePred, BaseOp0, BaseOp1) &&
+ IsEqZero(Pred, Op0, Op1))) {
+ MixedCompatible = true;
+ }
+ }
+
+ return MixedCompatible ||
+ (BasePred == Pred &&
areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1, TLI)) ||
(BasePred == SwappedPred &&
areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0, TLI));
@@ -4308,6 +4334,13 @@ class slpvectorizer::BoUpSLP {
/// Checks if the current node is a gather node.
bool isGather() const { return State == NeedToGather; }
+ /// Canonical predicate for comparison nodes that may be rewritten for
+ /// vector emission.
+ CmpInst::Predicate CmpPredicate = CmpInst::BAD_ICMP_PREDICATE;
+
+ void setCmpPredicate(CmpInst::Predicate P) { CmpPredicate = P; }
+ CmpInst::Predicate getCmpPredicate() const { return CmpPredicate; }
+
/// A vector of scalars.
ValueList Scalars;
@@ -11050,10 +11083,34 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
CmpInst::Predicate SwapP0 = CmpInst::getSwappedPredicate(P0);
Type *ComparedTy = VL0->getOperand(0)->getType();
+ Value *BaseOp0 = VL0->getOperand(0);
+ Value *BaseOp1 = VL0->getOperand(1);
for (Value *V : VL) {
if (isa<PoisonValue>(V))
continue;
auto *Cmp = cast<CmpInst>(V);
+ Value *Op0 = Cmp->getOperand(0);
+ Value *Op1 = Cmp->getOperand(1);
+
+ // Helper lambdas to match our target shapes: eq %x, 0 and ult %x, C (C !=
+ // 0)
+ auto IsEqZero = [](CmpInst::Predicate P, Value *L, Value *R) {
+ if (P != CmpInst::ICMP_EQ)
+ return false;
+ auto *C = dyn_cast<ConstantInt>(R);
+ return C && C->isZero();
+ };
+
+ auto IsUltNonZero = [](CmpInst::Predicate P, Value *L, Value *R) {
+ if (P != CmpInst::ICMP_ULT)
+ return false;
+ auto *C = dyn_cast<ConstantInt>(R);
+ return C && !C->isZero();
+ };
+
+ if (IsEqZero(P0, BaseOp0, BaseOp1) &&
+ IsUltNonZero(Cmp->getPredicate(), Op0, Op1))
+ continue;
if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
Cmp->getOperand(0)->getType() != ComparedTy) {
LLVM_DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");
@@ -13345,13 +13402,47 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
case Instruction::FCmp: {
// Check that all of the compares have the same predicate.
CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
+ auto IsEqZero = [](const Value *V) {
+ if (auto *Cmp = dyn_cast<ICmpInst>(V))
+ if (Cmp->getPredicate() == ICmpInst::ICMP_EQ)
+ if (auto *CI = dyn_cast<ConstantInt>(Cmp->getOperand(1)))
+ return CI->isZero();
+ return false;
+ };
+
+ auto IsUltNonZero = [](const Value *V) {
+ if (auto *Cmp = dyn_cast<ICmpInst>(V))
+ if (Cmp->getPredicate() == ICmpInst::ICMP_ULT)
+ if (auto *CI = dyn_cast<ConstantInt>(Cmp->getOperand(1)))
+ return !CI->isZero();
+ return false;
+ };
+
+ bool IsMixedCmp = any_of(VL, IsEqZero) && any_of(VL, IsUltNonZero);
+ // If the bundle is a mixture of eq 0 and ult lanes, canonicalize to ULT
+ if (IsMixedCmp) {
+ P0 = ICmpInst::ICMP_ULT; // Force vector generation to use ULT
+
+ for (auto [Idx, V] : enumerate(VL)) {
+ if (isa<PoisonValue>(V))
+ continue;
+ auto *Cmp = cast<ICmpInst>(V);
+ if (IsEqZero(Cmp)) {
+ // Rewrite the tracking constant for the 'eq 0' lane to be '1'
+ // so it builds perfectly into the right-hand side constant vector
+ Type *OpTy = Cmp->getOperand(1)->getType();
+ Operands.back()[Idx] = ConstantInt::get(OpTy, 1);
+ }
+ }
+ }
TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
ReuseShuffleIndices);
+ TE->setCmpPredicate(P0);
LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (CmpInst).\n";
TE->dump());
VLOperands Ops(VL, Operands, S, *this);
- if (cast<CmpInst>(VL0)->isCommutative()) {
+ if (!IsMixedCmp && cast<CmpInst>(VL0)->isCommutative()) {
// Commutative predicate - collect + sort operands of the instructions
// so that each side is more likely to have the same opcode.
assert(P0 == CmpInst::getSwappedPredicate(P0) &&
@@ -13365,6 +13456,8 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
if (isa<PoisonValue>(V))
continue;
auto *Cmp = cast<CmpInst>(V);
+ if (IsMixedCmp && IsEqZero(Cmp))
+ continue;
if (Cmp->getPredicate() != P0)
std::swap(Operands.front()[Idx], Operands.back()[Idx]);
}
@@ -23732,7 +23825,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
- CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
+ CmpInst::Predicate P0 = E->getCmpPredicate();
+ if (P0 == CmpInst::BAD_ICMP_PREDICATE)
+ P0 = cast<CmpInst>(VL0)->getPredicate();
Value *V = Builder.CreateCmp(P0, L, R);
V = PropagateIRFlags(V);
// Do not cast for cmps.
>From 477e99c9f89759a94d08457b962eeccd45a62032 Mon Sep 17 00:00:00 2001
From: Bhuvan1527 <balabhuvanvarma at gmail.com>
Date: Mon, 22 Jun 2026 19:15:34 +0530
Subject: [PATCH 2/2] [SLP] Conversion of icmp eq to icmp ult to enable
vectorization
Resolves: https://github.com/llvm/llvm-project/issues/190505
The current slp-vectorizer donot recognize the following pattern
```
icmp eq %x, 0
icmp ult %y, ..
icmp ult
..
```
It will not convert icmp eq to icmp ult. Hence, full vectorization is skipped. Instead it performs altshuffle vectorization. Creating two vector instructions (one for only eq, other for ult) followed by shuffleVec instructions to combine the results.
In this pr, we are handling this at three levels.
1. IsCmpSameOrSwapped() - making it treat icmp eq %x 0 same as icmp ult %x, ..
This is required for `InstructionState S = Legality.getInstructionsState();`
- Ensures the MainOp and AltOp are same instructions.
- **Here MainOp = icmp eq**
2. getScalarsVectorizationState() - Once MainOp and AltOp are same, still MainOp will not match with other 7 instructions. Hence this will return TreeEntry::NeedGather. However to vectorize we need it to return TreeEntry::Vectorize.
3. ICMP case in buildTreeRec() function: Once we have this mixed pattern, we are modifying the icmp eq to icmp ult before fetching their operands and vectorizing.
---
.../vectorizable_icmp_eq_icmp_ult.ll | 150 ++++++++++++++++++
1 file changed, 150 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/vectorizable_icmp_eq_icmp_ult.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/vectorizable_icmp_eq_icmp_ult.ll b/llvm/test/Transforms/SLPVectorizer/vectorizable_icmp_eq_icmp_ult.ll
new file mode 100644
index 0000000000000..738612ffc306b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/vectorizable_icmp_eq_icmp_ult.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
+source_filename = "vectorizable_icmp_eq_icmp_ult.ll"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128-ni:10:11:12:13"
+target triple = "x86_64-unknown-linux-gnu"
+
+define swiftcc void @julia_g_65(ptr noalias noundef nonnull sret([8 x i16]) align 2 captures(none) dereferenceable(16) %0, ptr nonnull swiftself %1, ptr addrspace(11) noundef nonnull readonly align 2 captures(none) dereferenceable(16) %2, ptr addrspace(11) noundef nonnull readonly align 2 captures(none) dereferenceable(16) %3, i64 signext %4) #0 {
+; CHECK-LABEL: define swiftcc void @julia_g_65(
+; CHECK-SAME: ptr noalias noundef nonnull sret([8 x i16]) align 2 captures(none) dereferenceable(16) [[TMP0:%.*]], ptr nonnull swiftself [[TMP1:%.*]], ptr addrspace(11) noundef nonnull readonly align 2 captures(none) dereferenceable(16) [[TMP2:%.*]], ptr addrspace(11) noundef nonnull readonly align 2 captures(none) dereferenceable(16) [[TMP3:%.*]], i64 signext [[TMP4:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP6:%.*]] = call ptr @julia.get_pgcstack()
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i64 16
+; CHECK-NEXT: [[TMP8:%.*]] = load ptr, ptr [[TMP7]], align 8, !tbaa [[JTBAA_GCFRAME_TBAA3:![0-9]+]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 16
+; CHECK-NEXT: [[TMP10:%.*]] = load ptr, ptr [[TMP9]], align 8, !tbaa [[JTBAA_CONST_TBAA7:![0-9]+]], !invariant.load [[META9:![0-9]+]]
+; CHECK-NEXT: fence syncscope("singlethread") seq_cst
+; CHECK-NEXT: call void @julia.safepoint(ptr [[TMP10]])
+; CHECK-NEXT: fence syncscope("singlethread") seq_cst
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP4]] to i16
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x i16> poison, i16 [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x i16> [[TMP12]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = icmp ult <8 x i16> [[TMP13]], <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>
+; CHECK-NEXT: [[TMP15:%.*]] = load <8 x i16>, ptr addrspace(11) [[TMP2]], align 2, !tbaa [[JTBAA_CONST_TBAA7]], !invariant.load [[META9]], !alias.scope [[META10:![0-9]+]], !noalias [[META13:![0-9]+]]
+; CHECK-NEXT: [[TMP16:%.*]] = load <8 x i16>, ptr addrspace(11) [[TMP3]], align 2, !tbaa [[JTBAA_CONST_TBAA7]], !invariant.load [[META9]], !alias.scope [[META10]], !noalias [[META13]]
+; CHECK-NEXT: [[TMP17:%.*]] = select <8 x i1> [[TMP14]], <8 x i16> [[TMP16]], <8 x i16> [[TMP15]]
+; CHECK-NEXT: store <8 x i16> [[TMP17]], ptr [[TMP0]], align 2
+; CHECK-NEXT: ret void
+;
+ %6 = call ptr @julia.get_pgcstack()
+ %7 = getelementptr inbounds i8, ptr %6, i64 16
+ %8 = load ptr, ptr %7, align 8, !tbaa !3
+ %9 = getelementptr inbounds i8, ptr %8, i64 16
+ %10 = load ptr, ptr %9, align 8, !tbaa !7, !invariant.load !9
+ fence syncscope("singlethread") seq_cst
+ call void @julia.safepoint(ptr %10)
+ fence syncscope("singlethread") seq_cst
+ %11 = trunc i64 %4 to i16
+ %12 = icmp eq i16 %11, 0
+ %13 = load i16, ptr addrspace(11) %2, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %14 = load i16, ptr addrspace(11) %3, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %15 = select i1 %12, i16 %14, i16 %13
+ %16 = icmp ult i16 %11, 2
+ %17 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 2
+ %18 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 2
+ %19 = load i16, ptr addrspace(11) %17, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %20 = load i16, ptr addrspace(11) %18, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %21 = select i1 %16, i16 %20, i16 %19
+ %22 = icmp ult i16 %11, 3
+ %23 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 4
+ %24 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 4
+ %25 = load i16, ptr addrspace(11) %23, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %26 = load i16, ptr addrspace(11) %24, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %27 = select i1 %22, i16 %26, i16 %25
+ %28 = icmp ult i16 %11, 4
+ %29 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 6
+ %30 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 6
+ %31 = load i16, ptr addrspace(11) %29, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %32 = load i16, ptr addrspace(11) %30, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %33 = select i1 %28, i16 %32, i16 %31
+ %34 = icmp ult i16 %11, 5
+ %35 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 8
+ %36 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 8
+ %37 = load i16, ptr addrspace(11) %35, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %38 = load i16, ptr addrspace(11) %36, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %39 = select i1 %34, i16 %38, i16 %37
+ %40 = icmp ult i16 %11, 6
+ %41 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 10
+ %42 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 10
+ %43 = load i16, ptr addrspace(11) %41, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %44 = load i16, ptr addrspace(11) %42, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %45 = select i1 %40, i16 %44, i16 %43
+ %46 = icmp ult i16 %11, 7
+ %47 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 12
+ %48 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 12
+ %49 = load i16, ptr addrspace(11) %47, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %50 = load i16, ptr addrspace(11) %48, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %51 = select i1 %46, i16 %50, i16 %49
+ %52 = icmp ult i16 %11, 8
+ %53 = getelementptr inbounds i8, ptr addrspace(11) %2, i64 14
+ %54 = getelementptr inbounds i8, ptr addrspace(11) %3, i64 14
+ %55 = load i16, ptr addrspace(11) %53, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %56 = load i16, ptr addrspace(11) %54, align 2, !tbaa !7, !invariant.load !9, !alias.scope !10, !noalias !13
+ %57 = select i1 %52, i16 %56, i16 %55
+ store i16 %15, ptr %0, align 2
+ %58 = getelementptr inbounds i8, ptr %0, i64 2
+ store i16 %21, ptr %58, align 2
+ %59 = getelementptr inbounds i8, ptr %0, i64 4
+ store i16 %27, ptr %59, align 2
+ %60 = getelementptr inbounds i8, ptr %0, i64 6
+ store i16 %33, ptr %60, align 2
+ %61 = getelementptr inbounds i8, ptr %0, i64 8
+ store i16 %39, ptr %61, align 2
+ %62 = getelementptr inbounds i8, ptr %0, i64 10
+ store i16 %45, ptr %62, align 2
+ %63 = getelementptr inbounds i8, ptr %0, i64 12
+ store i16 %51, ptr %63, align 2
+ %64 = getelementptr inbounds i8, ptr %0, i64 14
+ store i16 %57, ptr %64, align 2
+ ret void
+}
+
+declare ptr @julia.get_pgcstack()
+
+declare noalias nonnull ptr addrspace(10) @julia.gc_alloc_obj(ptr, i64, ptr addrspace(10)) #1
+
+declare void @llvm.memcpy.p11.p0.i64(ptr addrspace(11) noalias writeonly captures(none), ptr noalias readonly captures(none), i64, i1 immarg) #2
+
+declare void @julia.safepoint(ptr) #3
+
+attributes #0 = { "frame-pointer"="all" "probe-stack"="inline-asm" }
+attributes #1 = { mustprogress nounwind willreturn allockind("alloc") allocsize(1) memory(argmem: read, inaccessiblemem: readwrite) }
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
+attributes #3 = { memory(argmem: readwrite, inaccessiblemem: readwrite) }
+
+!llvm.module.flags = !{!0, !1, !2}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !{i32 2, !"julia.optlevel", i32 2}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"jtbaa_gcframe", !5, i64 0}
+!5 = !{!"jtbaa", !6, i64 0}
+!6 = !{!"jtbaa"}
+!7 = !{!8, !8, i64 0, i64 1}
+!8 = !{!"jtbaa_const", !5, i64 0}
+!9 = !{}
+!10 = !{!11}
+!11 = !{!"jnoalias_const", !12}
+!12 = !{!"jnoalias"}
+!13 = !{!14, !15, !16, !17}
+!14 = !{!"jnoalias_gcframe", !12}
+!15 = !{!"jnoalias_stack", !12}
+!16 = !{!"jnoalias_data", !12}
+!17 = !{!"jnoalias_typemd", !12}
+;.
+; CHECK: [[JTBAA_GCFRAME_TBAA3]] = !{[[META4:![0-9]+]], [[META4]], i64 0}
+; CHECK: [[META4]] = !{!"jtbaa_gcframe", [[META5:![0-9]+]], i64 0}
+; CHECK: [[META5]] = !{!"jtbaa", [[META6:![0-9]+]], i64 0}
+; CHECK: [[META6]] = !{!"jtbaa"}
+; CHECK: [[JTBAA_CONST_TBAA7]] = !{[[META8:![0-9]+]], [[META8]], i64 0, i64 1}
+; CHECK: [[META8]] = !{!"jtbaa_const", [[META5]], i64 0}
+; CHECK: [[META9]] = !{}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META11]] = !{!"jnoalias_const", [[META12:![0-9]+]]}
+; CHECK: [[META12]] = !{!"jnoalias"}
+; CHECK: [[META13]] = !{[[META14:![0-9]+]], [[META15:![0-9]+]], [[META16:![0-9]+]], [[META17:![0-9]+]]}
+; CHECK: [[META14]] = !{!"jnoalias_gcframe", [[META12]]}
+; CHECK: [[META15]] = !{!"jnoalias_stack", [[META12]]}
+; CHECK: [[META16]] = !{!"jnoalias_data", [[META12]]}
+; CHECK: [[META17]] = !{!"jnoalias_typemd", [[META12]]}
+;.
More information about the llvm-commits
mailing list