[llvm] 6b3cf50 - [IR][NFC] Hot-cold splitting in PatternMatch (#186777)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 16 06:34:09 PDT 2026
Author: Alexis Engelke
Date: 2026-03-16T13:34:03Z
New Revision: 6b3cf50d958c2f1f95502d7c86309c432b37eaae
URL: https://github.com/llvm/llvm-project/commit/6b3cf50d958c2f1f95502d7c86309c432b37eaae
DIFF: https://github.com/llvm/llvm-project/commit/6b3cf50d958c2f1f95502d7c86309c432b37eaae.diff
LOG: [IR][NFC] Hot-cold splitting in PatternMatch (#186777)
ConstantAggregates are rare, therefore split that check into a separate
function so that the fast path can be inlined.
Likewise for vectors, which occur much less frequently than scalar
values.
Added:
llvm/lib/IR/PatternMatch.cpp
Modified:
llvm/include/llvm/IR/PatternMatch.h
llvm/lib/IR/CMakeLists.txt
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 44defe2245152..226ae622add70 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -154,43 +154,16 @@ inline class_match<IntrinsicInst> m_AnyIntrinsic() {
}
struct undef_match {
+private:
+ static bool checkAggregate(const ConstantAggregate *CA);
+
+public:
static bool check(const Value *V) {
if (isa<UndefValue>(V))
return true;
-
- const auto *CA = dyn_cast<ConstantAggregate>(V);
- if (!CA)
- return false;
-
- SmallPtrSet<const ConstantAggregate *, 8> Seen;
- SmallVector<const ConstantAggregate *, 8> Worklist;
-
- // Either UndefValue, PoisonValue, or an aggregate that only contains
- // these is accepted by matcher.
- // CheckValue returns false if CA cannot satisfy this constraint.
- auto CheckValue = [&](const ConstantAggregate *CA) {
- for (const Value *Op : CA->operand_values()) {
- if (isa<UndefValue>(Op))
- continue;
-
- const auto *CA = dyn_cast<ConstantAggregate>(Op);
- if (!CA)
- return false;
- if (Seen.insert(CA).second)
- Worklist.emplace_back(CA);
- }
-
- return true;
- };
-
- if (!CheckValue(CA))
- return false;
-
- while (!Worklist.empty()) {
- if (!CheckValue(Worklist.pop_back_val()))
- return false;
- }
- return true;
+ if (const auto *CA = dyn_cast<ConstantAggregate>(V))
+ return checkAggregate(CA);
+ return false;
}
template <typename ITy> bool match(ITy *V) const { return check(V); }
};
@@ -402,38 +375,44 @@ template <int64_t Val> inline constantint_match<Val> m_ConstantInt() {
/// is true.
template <typename Predicate, typename ConstantVal, bool AllowPoison>
struct cstval_pred_ty : public Predicate {
+private:
+ bool matchVector(const Value *V) const {
+ if (const auto *C = dyn_cast<Constant>(V)) {
+ if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue()))
+ return this->isValue(CV->getValue());
+
+ // Number of elements of a scalable vector unknown at compile time
+ auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
+ if (!FVTy)
+ return false;
+
+ // Non-splat vector constant: check each element for a match.
+ unsigned NumElts = FVTy->getNumElements();
+ assert(NumElts != 0 && "Constant vector with no elements?");
+ bool HasNonPoisonElements = false;
+ for (unsigned i = 0; i != NumElts; ++i) {
+ Constant *Elt = C->getAggregateElement(i);
+ if (!Elt)
+ return false;
+ if (AllowPoison && isa<PoisonValue>(Elt))
+ continue;
+ auto *CV = dyn_cast<ConstantVal>(Elt);
+ if (!CV || !this->isValue(CV->getValue()))
+ return false;
+ HasNonPoisonElements = true;
+ }
+ return HasNonPoisonElements;
+ }
+ return false;
+ }
+
+public:
const Constant **Res = nullptr;
template <typename ITy> bool match_impl(ITy *V) const {
if (const auto *CV = dyn_cast<ConstantVal>(V))
return this->isValue(CV->getValue());
- if (const auto *VTy = dyn_cast<VectorType>(V->getType())) {
- if (const auto *C = dyn_cast<Constant>(V)) {
- if (const auto *CV = dyn_cast_or_null<ConstantVal>(C->getSplatValue()))
- return this->isValue(CV->getValue());
-
- // Number of elements of a scalable vector unknown at compile time
- auto *FVTy = dyn_cast<FixedVectorType>(VTy);
- if (!FVTy)
- return false;
-
- // Non-splat vector constant: check each element for a match.
- unsigned NumElts = FVTy->getNumElements();
- assert(NumElts != 0 && "Constant vector with no elements?");
- bool HasNonPoisonElements = false;
- for (unsigned i = 0; i != NumElts; ++i) {
- Constant *Elt = C->getAggregateElement(i);
- if (!Elt)
- return false;
- if (AllowPoison && isa<PoisonValue>(Elt))
- continue;
- auto *CV = dyn_cast<ConstantVal>(Elt);
- if (!CV || !this->isValue(CV->getValue()))
- return false;
- HasNonPoisonElements = true;
- }
- return HasNonPoisonElements;
- }
- }
+ if (isa<VectorType>(V->getType()))
+ return matchVector(V);
return false;
}
diff --git a/llvm/lib/IR/CMakeLists.txt b/llvm/lib/IR/CMakeLists.txt
index 08bac979d4c90..ff362356fe65a 100644
--- a/llvm/lib/IR/CMakeLists.txt
+++ b/llvm/lib/IR/CMakeLists.txt
@@ -55,6 +55,7 @@ add_llvm_component_library(LLVMCore
ModuleSummaryIndex.cpp
Operator.cpp
OptBisect.cpp
+ PatternMatch.cpp
Pass.cpp
PassInstrumentation.cpp
PassManager.cpp
diff --git a/llvm/lib/IR/PatternMatch.cpp b/llvm/lib/IR/PatternMatch.cpp
new file mode 100644
index 0000000000000..334aedeb48e2f
--- /dev/null
+++ b/llvm/lib/IR/PatternMatch.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Out-of-line implementations for PatternMatch.h.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Value.h"
+
+using namespace llvm;
+
+bool llvm::PatternMatch::undef_match::checkAggregate(
+ const ConstantAggregate *CA) {
+ SmallPtrSet<const ConstantAggregate *, 8> Seen;
+ SmallVector<const ConstantAggregate *, 8> Worklist;
+
+ // Either UndefValue, PoisonValue, or an aggregate that only contains
+ // these is accepted by matcher.
+ // CheckValue returns false if CA cannot satisfy this constraint.
+ auto CheckValue = [&](const ConstantAggregate *CA) {
+ for (const Value *Op : CA->operand_values()) {
+ if (isa<UndefValue>(Op))
+ continue;
+
+ const auto *CA = dyn_cast<ConstantAggregate>(Op);
+ if (!CA)
+ return false;
+ if (Seen.insert(CA).second)
+ Worklist.emplace_back(CA);
+ }
+
+ return true;
+ };
+
+ if (!CheckValue(CA))
+ return false;
+
+ while (!Worklist.empty()) {
+ if (!CheckValue(Worklist.pop_back_val()))
+ return false;
+ }
+ return true;
+}
More information about the llvm-commits
mailing list