[llvm] [VPlan] Introduce m_[Specific]ICmp matcher (PR #151540)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 4 03:42:57 PDT 2025
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/151540
>From 421d7f7ef4ba7d1eb78f6e55904b6f70f269c6bb Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Thu, 31 Jul 2025 16:21:36 +0100
Subject: [PATCH 1/2] [VPlan] Introduce m_ICmp matcher
---
.../Transforms/Vectorize/VPlanPatternMatch.h | 49 +++++++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 25 +++++-----
2 files changed, 60 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index d133610ef4f75..fc88d7c8d798c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -461,6 +461,55 @@ m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
return m_BinaryOr<Op0_t, Op1_t, /*Commutative*/ true>(Op0, Op1);
}
+/// ICmp_match is a variant of BinaryRecipe_match that also binds the
+/// comparison predicate.
+template <typename Op0_t, typename Op1_t, bool Commutable = false>
+struct ICmp_match {
+ CmpPredicate *Predicate = nullptr;
+ Op0_t Op0;
+ Op1_t Op1;
+
+ ICmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
+ : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
+ ICmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
+
+ bool match(const VPValue *V) const {
+ auto *DefR = V->getDefiningRecipe();
+ return DefR && match(DefR);
+ }
+
+ bool match(const VPSingleDefRecipe *R) const {
+ return match(static_cast<const VPRecipeBase *>(R));
+ }
+
+ bool match(const VPRecipeBase *V) const {
+ if (m_Binary<Instruction::ICmp>(Op0, Op1).match(V)) {
+ if (Predicate)
+ *Predicate = cast<VPRecipeWithIRFlags>(V)->getPredicate();
+ return true;
+ }
+ if (Commutable && m_Binary<Instruction::ICmp>(Op1, Op0).match(V)) {
+ if (Predicate)
+ *Predicate = CmpPredicate::getSwapped(
+ cast<VPRecipeWithIRFlags>(V)->getPredicate());
+ return true;
+ }
+ return false;
+ }
+};
+
+template <typename Op0_t, typename Op1_t>
+inline ICmp_match<Op0_t, Op1_t, false> m_ICmp(const Op0_t &Op0,
+ const Op1_t &Op1) {
+ return ICmp_match<Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline ICmp_match<Op0_t, Op1_t, false>
+m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
+ return ICmp_match<Op0_t, Op1_t>(Pred, Op0, Op1);
+}
+
template <typename Op0_t, typename Op1_t>
using GEPLikeRecipe_match =
BinaryRecipe_match<Op0_t, Op1_t, Instruction::GetElementPtr, false,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3ecffc7593d49..2cefbbe0e7a11 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1381,11 +1381,10 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
// Currently only handle cases where the single user is a header-mask
// comparison with the backedge-taken-count.
- if (!match(
- *WideIV->user_begin(),
- m_Binary<Instruction::ICmp>(
- m_Specific(WideIV),
- m_Broadcast(m_Specific(Plan.getOrCreateBackedgeTakenCount())))))
+ if (!match(*WideIV->user_begin(),
+ m_ICmp(m_Specific(WideIV),
+ m_Broadcast(
+ m_Specific(Plan.getOrCreateBackedgeTakenCount())))))
continue;
// Update IV operands and comparison bound to use new narrower type.
@@ -1418,11 +1417,10 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
});
auto *CanIV = Plan.getCanonicalIV();
- if (!match(Cond, m_Binary<Instruction::ICmp>(
- m_Specific(CanIV->getBackedgeValue()),
- m_Specific(&Plan.getVectorTripCount()))) ||
- cast<VPRecipeWithIRFlags>(Cond->getDefiningRecipe())->getPredicate() !=
- CmpInst::ICMP_EQ)
+ CmpPredicate Pred;
+ if (!match(Cond, m_ICmp(Pred, m_Specific(CanIV->getBackedgeValue()),
+ m_Specific(&Plan.getVectorTripCount()))) ||
+ Pred != CmpInst::ICMP_EQ)
return false;
// The compare checks CanIV + VFxUF == vector trip count. The vector trip
@@ -1833,7 +1831,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
VPW->dropPoisonGeneratingFlags();
if (OldResSizeInBits != NewResSizeInBits &&
- !match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue()))) {
+ !match(&R, m_ICmp(m_VPValue(), m_VPValue()))) {
// Extend result to original width.
auto *Ext =
new VPWidenCastRecipe(Instruction::ZExt, ResultVPV, OldResTy);
@@ -1842,9 +1840,8 @@ void VPlanTransforms::truncateToMinimalBitwidths(
Ext->setOperand(0, ResultVPV);
assert(OldResSizeInBits > NewResSizeInBits && "Nothing to shrink?");
} else {
- assert(
- match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue())) &&
- "Only ICmps should not need extending the result.");
+ assert(match(&R, m_ICmp(m_VPValue(), m_VPValue())) &&
+ "Only ICmps should not need extending the result.");
}
assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
>From f52519706b619dd36b3cc6a35b66ea15324729dd Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Mon, 4 Aug 2025 11:31:05 +0100
Subject: [PATCH 2/2] [VPlan] m_SpecificICmp and fixes/cleanup
---
.../Transforms/Vectorize/VPlanPatternMatch.h | 52 ++++++++++++-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 7 ++-
2 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index fc88d7c8d798c..d7b91190af0bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -461,10 +461,9 @@ m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
return m_BinaryOr<Op0_t, Op1_t, /*Commutative*/ true>(Op0, Op1);
}
-/// ICmp_match is a variant of BinaryRecipe_match that also binds the
-/// comparison predicate.
-template <typename Op0_t, typename Op1_t, bool Commutable = false>
-struct ICmp_match {
+/// ICmp_match is a variant of BinaryRecipe_match that also binds the comparison
+/// predicate.
+template <typename Op0_t, typename Op1_t> struct ICmp_match {
CmpPredicate *Predicate = nullptr;
Op0_t Op0;
Op1_t Op1;
@@ -478,38 +477,55 @@ struct ICmp_match {
return DefR && match(DefR);
}
- bool match(const VPSingleDefRecipe *R) const {
- return match(static_cast<const VPRecipeBase *>(R));
- }
-
bool match(const VPRecipeBase *V) const {
if (m_Binary<Instruction::ICmp>(Op0, Op1).match(V)) {
if (Predicate)
*Predicate = cast<VPRecipeWithIRFlags>(V)->getPredicate();
return true;
}
- if (Commutable && m_Binary<Instruction::ICmp>(Op1, Op0).match(V)) {
- if (Predicate)
- *Predicate = CmpPredicate::getSwapped(
- cast<VPRecipeWithIRFlags>(V)->getPredicate());
- return true;
- }
return false;
}
};
+/// SpecificICmp_match is a variant of BinaryRecipe_match that also matches the
+/// comparison predicate.
+template <typename Op0_t, typename Op1_t> struct SpecificICmp_match {
+ const CmpPredicate Predicate;
+ Op0_t Op0;
+ Op1_t Op1;
+
+ SpecificICmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
+ : Predicate(Pred), Op0(LHS), Op1(RHS) {}
+
+ bool match(const VPValue *V) const {
+ auto *DefR = V->getDefiningRecipe();
+ return DefR && match(DefR);
+ }
+
+ bool match(const VPRecipeBase *V) const {
+ return m_Binary<Instruction::ICmp>(Op0, Op1).match(V) &&
+ CmpPredicate::getMatching(
+ cast<VPRecipeWithIRFlags>(V)->getPredicate(), Predicate);
+ }
+};
+
template <typename Op0_t, typename Op1_t>
-inline ICmp_match<Op0_t, Op1_t, false> m_ICmp(const Op0_t &Op0,
- const Op1_t &Op1) {
+inline ICmp_match<Op0_t, Op1_t> m_ICmp(const Op0_t &Op0, const Op1_t &Op1) {
return ICmp_match<Op0_t, Op1_t>(Op0, Op1);
}
template <typename Op0_t, typename Op1_t>
-inline ICmp_match<Op0_t, Op1_t, false>
-m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
+inline ICmp_match<Op0_t, Op1_t> m_ICmp(CmpPredicate &Pred, const Op0_t &Op0,
+ const Op1_t &Op1) {
return ICmp_match<Op0_t, Op1_t>(Pred, Op0, Op1);
}
+template <typename Op0_t, typename Op1_t>
+inline SpecificICmp_match<Op0_t, Op1_t>
+m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
+ return SpecificICmp_match<Op0_t, Op1_t>(MatchPred, Op0, Op1);
+}
+
template <typename Op0_t, typename Op1_t>
using GEPLikeRecipe_match =
BinaryRecipe_match<Op0_t, Op1_t, Instruction::GetElementPtr, false,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2cefbbe0e7a11..a128a43a23bb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1417,10 +1417,9 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
});
auto *CanIV = Plan.getCanonicalIV();
- CmpPredicate Pred;
- if (!match(Cond, m_ICmp(Pred, m_Specific(CanIV->getBackedgeValue()),
- m_Specific(&Plan.getVectorTripCount()))) ||
- Pred != CmpInst::ICMP_EQ)
+ if (!match(Cond, m_SpecificICmp(CmpInst::ICMP_EQ,
+ m_Specific(CanIV->getBackedgeValue()),
+ m_Specific(&Plan.getVectorTripCount()))))
return false;
// The compare checks CanIV + VFxUF == vector trip count. The vector trip
More information about the llvm-commits
mailing list