[llvm] [LV][CSSPGO] Don't let pseudo probes block early-exit vectorization (PR #220149)
chandan singh via llvm-commits
llvm-commits at lists.llvm.org
Sat Sep 12 23:20:38 PDT 2026
https://github.com/chandankds updated https://github.com/llvm/llvm-project/pull/220149
>From 7020a71c9d42c68148e91d0cf61831eabac9b1d2 Mon Sep 17 00:00:00 2001
From: chandankds <chandankds at gmail.com>
Date: Mon, 31 Aug 2026 04:06:29 +0000
Subject: [PATCH 1/3] [CSSPGO] Don't let pseudo probes block early-exit
vectorization
llvm.pseudoprobe is modeled as accessing inaccessible memory, so
mayReadFromMemory()/mayWriteToMemory() return true even though the intrinsic
carries no real memory dependence. An otherwise vectorizable early-exit loop
is therefore rejected as soon as it contains a pseudo probe.
This patch skips pseudo probes in isVectorizableEarlyExitLoop(), isReadOnlyLoop() and
areAllLoadsDereferenceable() so the three checks agree and such loops
vectorize as they would without pseudo probe instrumentation.
Discussion: https://discourse.llvm.org/t/csspgo-unblocking-pseudo-probe-safe-optimizations/90946
---
llvm/lib/Analysis/Loads.cpp | 4 +
.../Vectorize/LoopVectorizationLegality.cpp | 10 ++
.../Vectorize/VPlanConstruction.cpp | 8 ++
.../LoopVectorize/early_exit_pseudo_probe.ll | 83 +++++++++++++
.../early_exit_pseudo_probe_legality.ll | 114 ++++++++++++++++++
llvm/unittests/Analysis/LoadsTest.cpp | 40 +++++-
6 files changed, 257 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe_legality.ll
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index de9022c540d42..398f7ddd8bb06 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -900,6 +900,10 @@ bool llvm::isReadOnlyLoop(
SmallVectorImpl<const SCEVPredicate *> *Predicates) {
for (BasicBlock *BB : L->blocks()) {
for (Instruction &I : *BB) {
+ // Debug values and pseudo-probes are placeholders with side effects but
+ // no real memory access, and do not affect dereferenceability.
+ if (I.isDebugOrPseudoInst())
+ continue;
if (auto *LI = dyn_cast<LoadInst>(&I)) {
if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
NonDereferenceableAndAlignedLoads.push_back(LI);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0c080d9434ea8..61ba789fcb6ee 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1730,6 +1730,10 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
bool HasSideEffects = false;
for (auto *BB : TheLoop->blocks())
for (auto &I : *BB) {
+ // Debug values and pseudo-probes are placeholders with side effects but
+ // no real memory dependence, and should not disqualify early exit loops.
+ if (I.isDebugOrPseudoInst())
+ continue;
if (I.mayWriteToMemory()) {
if (isa<StoreInst>(&I) && cast<StoreInst>(&I)->isSimple()) {
HasSideEffects = true;
@@ -1863,6 +1867,12 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
if (&I == Load)
continue;
+ // Debug values and pseudo-probes are placeholders with no real memory
+ // access, so they cannot alias the exit condition load and do not need
+ // to be masked.
+ if (I.isDebugOrPseudoInst())
+ continue;
+
if (I.mayReadOrWriteMemory()) {
// We need to mask all other memory ops.
ConditionallyExecutedOps.insert(&I);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c0c63f3e87842..efdcdb6e9329d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -29,6 +29,7 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -1208,6 +1209,13 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
for (VPRecipeBase &R : *VPBB) {
+ // Pseudo-probes are profiling placeholders with side effects but no real
+ // memory access. They are modeled as reading inaccessible memory, so skip
+ // them here to stay consistent with isReadOnlyLoop(), which classifies
+ // such loops as read-only.
+ if (auto *SDR = dyn_cast<VPSingleDefRecipe>(&R))
+ if (isa_and_nonnull<PseudoProbeInst>(SDR->getUnderlyingValue()))
+ continue;
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe.ll b/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe.ll
new file mode 100644
index 0000000000000..3b48f19094513
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
+
+declare void @init_mem(ptr, i64);
+
+;; llvm.pseudoprobe is modelled as accessing inaccessible memory, so it reports
+;; may-read and may-write, but it carries no real memory dependence. A read-only
+;; early exit loop must vectorize whether or not it has been instrumented for
+;; sample profiling.
+define i64 @early_exit_with_pseudo_probe() {
+; CHECK-LABEL: define i64 @early_exit_with_pseudo_probe() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP0]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64
+; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, %[[VECTOR_EARLY_EXIT]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ 0, %loop ], [ 1, %loop.inc ]
+ ret i64 %retval
+}
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+
+!llvm.pseudo_probe_desc = !{!0}
+
+!0 = !{i64 5116412291814990879, i64 52824598631, !"early_exit_with_pseudo_probe"}
+;.
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe_legality.ll
new file mode 100644
index 0000000000000..57e8da3c8ce8e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_pseudo_probe_legality.ll
@@ -0,0 +1,114 @@
+; REQUIRES: asserts
+;; Stores in uncountable early exit loops are only vectorized with
+;; -enable-early-exit-vectorization-with-side-effects; the flag does not affect
+;; the read-only loops below.
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization-with-side-effects -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+
+declare void @init_mem(ptr, i64);
+
+;; A pseudo probe reports may-read and may-write because it is modelled as
+;; accessing inaccessible memory, but it carries no real memory dependence, so
+;; the loop is still read-only for the purposes of early exit vectorization.
+define i64 @read_only_early_exit_with_pseudo_probe() {
+; CHECK-LABEL: LV: Checking a loop in 'read_only_early_exit_with_pseudo_probe'
+; CHECK: LV: Found an early exit loop with symbolic max backedge taken count: 63
+; CHECK-NEXT: LV: We can vectorize this loop!
+; CHECK-NOT: LV: Not vectorizing:
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ 0, %loop ], [ 1, %loop.inc ]
+ ret i64 %retval
+}
+
+;; Same loop as @loop_contains_store_condition_load_has_single_user in
+;; early_exit_store_legality.ll, with a pseudo probe added. The probe must
+;; neither trip the "complex writes to memory" check nor prevent the exit
+;; condition load from being moved.
+define void @store_early_exit_with_pseudo_probe(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'store_early_exit_with_pseudo_probe'
+; CHECK: LV: We can vectorize this loop!
+; CHECK-NOT: LV: Not vectorizing:
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ call void @llvm.pseudoprobe(i64 5116412291814990879, i64 2, i32 0, i64 -1)
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+;; llvm.sideeffect is modelled the same way as llvm.pseudoprobe, but it is not a
+;; profiling placeholder, so it must still block vectorization.
+define i64 @read_only_early_exit_with_sideeffect() {
+; CHECK-LABEL: LV: Checking a loop in 'read_only_early_exit_with_sideeffect'
+; CHECK: LV: Not vectorizing: Complex writes to memory unsupported in early exit loops.
+; CHECK-NOT: LV: We can vectorize this loop!
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ call void @llvm.sideeffect()
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ 0, %loop ], [ 1, %loop.inc ]
+ ret i64 %retval
+}
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
+declare void @llvm.sideeffect()
+
+!llvm.pseudo_probe_desc = !{!0}
+
+!0 = !{i64 5116412291814990879, i64 52824598631, !"read_only_early_exit_with_pseudo_probe"}
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index 8b15bda08485e..92d71fad5dd61 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -209,13 +209,43 @@ loop.end:
%retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
ret i64 %retval
}
+
+define i64 @f3() {
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+declare void @llvm.pseudoprobe(i64, i64, i32, i64)
)IR");
auto *GV1 = M->getNamedValue("f1");
auto *GV2 = M->getNamedValue("f2");
- ASSERT_TRUE(GV1 && GV2);
+ auto *GV3 = M->getNamedValue("f3");
+ ASSERT_TRUE(GV1 && GV2 && GV3);
auto *F1 = dyn_cast<Function>(GV1);
auto *F2 = dyn_cast<Function>(GV2);
- ASSERT_TRUE(F1 && F2);
+ auto *F3 = dyn_cast<Function>(GV3);
+ ASSERT_TRUE(F1 && F2 && F3);
TargetLibraryInfoImpl TLII(M->getTargetTriple());
TargetLibraryInfo TLI(TLII);
@@ -242,4 +272,10 @@ loop.end:
ASSERT_TRUE(IsReadOnlyLoop(F2, NonDerefLoads));
ASSERT_TRUE((NonDerefLoads.size() == 1) &&
(NonDerefLoads[0]->getName() == "ld1"));
+
+ // A pseudo probe is modelled as accessing inaccessible memory, but it does
+ // not make the loop non-read-only.
+ NonDerefLoads.clear();
+ ASSERT_TRUE(IsReadOnlyLoop(F3, NonDerefLoads));
+ ASSERT_TRUE(NonDerefLoads.empty());
}
>From 614fdea6466f5cd9d3ee32f57078bd2d78ad629b Mon Sep 17 00:00:00 2001
From: chandankds <chandankds at gmail.com>
Date: Mon, 7 Sep 2026 01:31:13 +0000
Subject: [PATCH 2/3] [CSSPGO][NFC] Drop debug values from pseudo-probe skip
comments
---
llvm/lib/Analysis/Loads.cpp | 2 +-
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 398f7ddd8bb06..01b9f632b0c84 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -900,7 +900,7 @@ bool llvm::isReadOnlyLoop(
SmallVectorImpl<const SCEVPredicate *> *Predicates) {
for (BasicBlock *BB : L->blocks()) {
for (Instruction &I : *BB) {
- // Debug values and pseudo-probes are placeholders with side effects but
+ // Pseudo-probes are placeholders with side effects but
// no real memory access, and do not affect dereferenceability.
if (I.isDebugOrPseudoInst())
continue;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 61ba789fcb6ee..954efdb2a7423 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1730,7 +1730,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
bool HasSideEffects = false;
for (auto *BB : TheLoop->blocks())
for (auto &I : *BB) {
- // Debug values and pseudo-probes are placeholders with side effects but
+ // Pseudo-probes are placeholders with side effects but
// no real memory dependence, and should not disqualify early exit loops.
if (I.isDebugOrPseudoInst())
continue;
@@ -1867,7 +1867,7 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
if (&I == Load)
continue;
- // Debug values and pseudo-probes are placeholders with no real memory
+ // Pseudo-probes are placeholders with no real memory
// access, so they cannot alias the exit condition load and do not need
// to be masked.
if (I.isDebugOrPseudoInst())
>From 57270e31540aae1273f0bae3acd72ec80c9293bb Mon Sep 17 00:00:00 2001
From: chandankds <chandankds at gmail.com>
Date: Sun, 13 Sep 2026 06:19:31 +0000
Subject: [PATCH 3/3] [LV][CSSPGO] Match pseudo probes with VPlan pattern
matching
Identify llvm.pseudoprobe via match(m_Intrinsic<Intrinsic::pseudoprobe>())
instead of inspecting the underlying IR value.
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index efdcdb6e9329d..5fc1568e9d4cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -29,7 +29,6 @@
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
@@ -1213,9 +1212,8 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
// memory access. They are modeled as reading inaccessible memory, so skip
// them here to stay consistent with isReadOnlyLoop(), which classifies
// such loops as read-only.
- if (auto *SDR = dyn_cast<VPSingleDefRecipe>(&R))
- if (isa_and_nonnull<PseudoProbeInst>(SDR->getUnderlyingValue()))
- continue;
+ if (match(&R, m_Intrinsic<Intrinsic::pseudoprobe>()))
+ continue;
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
assert(!R.mayReadFromMemory() && "unexpected recipe reading memory");
More information about the llvm-commits
mailing list