[llvm] [LV] Add initial legality checks for loops with unbound loads. (PR #152422)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 27 00:17:03 PDT 2025
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/152422
>From dc5b7c14e92f0850fc1f097401ed1c5c97c02613 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 6 Aug 2025 17:44:19 -0700
Subject: [PATCH 01/15] [LV] Add initial legality checks for loops with
non-dereferenceable load.
---
llvm/include/llvm/Analysis/Loads.h | 8 +++++
.../llvm/Analysis/TargetTransformInfo.h | 4 +++
.../llvm/Analysis/TargetTransformInfoImpl.h | 2 ++
.../Vectorize/LoopVectorizationLegality.h | 8 +++++
llvm/lib/Analysis/Loads.cpp | 16 ++++++++++
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +++
.../Target/RISCV/RISCVTargetTransformInfo.h | 3 ++
.../Vectorize/LoopVectorizationLegality.cpp | 31 +++++++++++++++++--
.../Transforms/Vectorize/LoopVectorize.cpp | 7 +++++
9 files changed, 80 insertions(+), 3 deletions(-)
diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index 84564563de8e3..080757b6d1fe0 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -91,6 +91,14 @@ LLVM_ABI bool isDereferenceableReadOnlyLoop(
Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
+/// Return true if the loop \p L cannot fault on any iteration and only
+/// contains read-only memory accesses. Also collect loads that are not
+/// guaranteed to be dereferenceable.
+LLVM_ABI bool isReadOnlyLoopWithSafeOrSpeculativeLoads(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<LoadInst *> *SpeculativeLoads,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
+
/// Return true if we know that executing a load from this value cannot trap.
///
/// If DT and ScanFrom are specified this method performs context-sensitive
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index aa4550de455e0..2b8e6be92238c 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1857,6 +1857,10 @@ class TargetTransformInfo {
/// \returns True if the target supports scalable vectors.
LLVM_ABI bool supportsScalableVectors() const;
+ /// \returns True if the target supports speculative load intrinsics (e.g.,
+ /// vp.load.ff).
+ LLVM_ABI bool supportsSpeculativeLoads() const;
+
/// \return true when scalable vectorization is preferred.
LLVM_ABI bool enableScalableVectorization() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index abdbca04488db..1df93ecc7ec16 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1106,6 +1106,8 @@ class TargetTransformInfoImplBase {
virtual bool supportsScalableVectors() const { return false; }
+ virtual bool supportsSpeculativeLoads() const { return false; }
+
virtual bool enableScalableVectorization() const { return false; }
virtual bool hasActiveVectorLength() const { return false; }
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 43ff084816d18..3b5638f3f570a 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -445,6 +445,11 @@ class LoopVectorizationLegality {
/// Returns a list of all known histogram operations in the loop.
bool hasHistograms() const { return !Histograms.empty(); }
+ /// Returns the loads that may fault and need to be speculative.
+ const SmallPtrSetImpl<const Instruction *> &getSpeculativeLoads() const {
+ return SpeculativeLoads;
+ }
+
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
return &PSE;
}
@@ -630,6 +635,9 @@ class LoopVectorizationLegality {
/// may work on the same memory location.
SmallVector<HistogramInfo, 1> Histograms;
+ /// Hold all loads that need to be speculative.
+ SmallPtrSet<const Instruction *, 4> SpeculativeLoads;
+
/// BFI and PSI are used to check for profile guided size optimizations.
BlockFrequencyInfo *BFI;
ProfileSummaryInfo *PSI;
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 78d0887d5d87e..c5a55e9903d41 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -870,3 +870,19 @@ bool llvm::isDereferenceableReadOnlyLoop(
}
return true;
}
+
+bool llvm::isReadOnlyLoopWithSafeOrSpeculativeLoads(
+ Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ SmallVectorImpl<LoadInst *> *SpeculativeLoads,
+ SmallVectorImpl<const SCEVPredicate *> *Predicates) {
+ for (BasicBlock *BB : L->blocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *LI = dyn_cast<LoadInst>(&I)) {
+ if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
+ SpeculativeLoads->push_back(LI);
+ } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow())
+ return false;
+ }
+ }
+ return true;
+}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index c7eb2ec18c679..9f05e01d34781 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1457,6 +1457,10 @@ bool TargetTransformInfo::supportsScalableVectors() const {
return TTIImpl->supportsScalableVectors();
}
+bool TargetTransformInfo::supportsSpeculativeLoads() const {
+ return TTIImpl->supportsSpeculativeLoads();
+}
+
bool TargetTransformInfo::enableScalableVectorization() const {
return TTIImpl->enableScalableVectorization();
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 05d504cbcb6bb..54e9c8346b6e2 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -110,6 +110,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool supportsScalableVectors() const override {
return ST->hasVInstructions();
}
+ bool supportsSpeculativeLoads() const override {
+ return ST->hasVInstructions();
+ }
bool enableOrderedReductions() const override { return true; }
bool enableScalableVectorization() const override {
return ST->hasVInstructions();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index c47fd9421fddd..46660866741ea 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1760,16 +1760,41 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
assert(LatchBB->getUniquePredecessor() == SingleUncountableExitingBlock &&
"Expected latch predecessor to be the early exiting block");
- // TODO: Handle loops that may fault.
Predicates.clear();
- if (!isDereferenceableReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC,
- &Predicates)) {
+ SmallVector<LoadInst *, 4> NonDerefLoads;
+ bool HasSafeAccess =
+ TTI->supportsSpeculativeLoads()
+ ? isReadOnlyLoopWithSafeOrSpeculativeLoads(
+ TheLoop, PSE.getSE(), DT, AC, &NonDerefLoads, &Predicates)
+ : isDereferenceableReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC,
+ &Predicates);
+ if (!HasSafeAccess) {
reportVectorizationFailure(
"Loop may fault",
"Cannot vectorize potentially faulting early exit loop",
"PotentiallyFaultingEarlyExitLoop", ORE, TheLoop);
return false;
}
+ // Speculative loads need to be unit-stride.
+ for (LoadInst *LI : NonDerefLoads) {
+ if (LI->getParent() != TheLoop->getHeader()) {
+ reportVectorizationFailure("Cannot vectorize predicated speculative load",
+ "SpeculativeLoadNeedsPredication", ORE,
+ TheLoop);
+ return false;
+ }
+ int Stride = isConsecutivePtr(LI->getType(), LI->getPointerOperand());
+ if (Stride != 1) {
+ reportVectorizationFailure("Loop contains non-unit-stride load",
+ "Cannot vectorize early exit loop with "
+ "speculative non-unit-stride load",
+ "SpeculativeNonUnitStrideLoadEarlyExitLoop",
+ ORE, TheLoop);
+ return false;
+ }
+ SpeculativeLoads.insert(LI);
+ LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
+ }
[[maybe_unused]] const SCEV *SymbolicMaxBTC =
PSE.getSymbolicMaxBackedgeTakenCount();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9667b506e594f..790a5236d4f04 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10041,6 +10041,13 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}
+ if (!LVL.getSpeculativeLoads().empty()) {
+ reportVectorizationFailure("Auto-vectorization of loops with speculative "
+ "load is not supported",
+ "SpeculativeLoadsNotSupported", ORE, L);
+ return false;
+ }
+
// Entrance to the VPlan-native vectorization path. Outer loops are processed
// here. They may require CFG and instruction level transformations before
// even evaluating whether vectorization is profitable. Since we cannot modify
>From 78f05df3753e6a7f167559560e9122a9f96b488c Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 8 Aug 2025 00:28:14 -0700
Subject: [PATCH 02/15] Fix braces
---
llvm/lib/Analysis/Loads.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index c5a55e9903d41..d05b89f86f554 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -880,8 +880,9 @@ bool llvm::isReadOnlyLoopWithSafeOrSpeculativeLoads(
if (auto *LI = dyn_cast<LoadInst>(&I)) {
if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
SpeculativeLoads->push_back(LI);
- } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow())
+ } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow()) {
return false;
+ }
}
}
return true;
>From f5315f8e262ec049b7e498157b48c5a5e3d36f8e Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sat, 9 Aug 2025 15:38:09 -0700
Subject: [PATCH 03/15] Limit to single unbound access. Add tests
---
.../Vectorize/LoopVectorizationLegality.cpp | 15 ++--
.../RISCV/unbound-access-legality.ll | 89 +++++++++++++++++++
2 files changed, 97 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 46660866741ea..b20fab14d5384 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1777,15 +1777,9 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
// Speculative loads need to be unit-stride.
for (LoadInst *LI : NonDerefLoads) {
- if (LI->getParent() != TheLoop->getHeader()) {
- reportVectorizationFailure("Cannot vectorize predicated speculative load",
- "SpeculativeLoadNeedsPredication", ORE,
- TheLoop);
- return false;
- }
int Stride = isConsecutivePtr(LI->getType(), LI->getPointerOperand());
if (Stride != 1) {
- reportVectorizationFailure("Loop contains non-unit-stride load",
+ reportVectorizationFailure("Loop contains strided unbound access",
"Cannot vectorize early exit loop with "
"speculative non-unit-stride load",
"SpeculativeNonUnitStrideLoadEarlyExitLoop",
@@ -1795,6 +1789,13 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
SpeculativeLoads.insert(LI);
LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
}
+ // Support single Speculative load for now.
+ if (NonDerefLoads.size() > 1) {
+ reportVectorizationFailure("Loop contains more than one unbound access",
+ "TooManySpeculativeLoadInEarlyExitLoop",
+ ORE, TheLoop);
+ return false;
+ }
[[maybe_unused]] const SCEV *SymbolicMaxBTC =
PSE.getSymbolicMaxBackedgeTakenCount();
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
new file mode 100644
index 0000000000000..e35d109304274
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
@@ -0,0 +1,89 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s
+
+define ptr @two_unbound_access(ptr %first, ptr %last, ptr %addr2) {
+; CHECK-LABEL: LV: Checking a loop in 'two_unbound_access'
+; CHECK: LV: Not vectorizing: Loop contains more than one unbound access.
+entry:
+ %cond = icmp eq ptr %first, %last
+ br i1 %cond, label %return, label %for.body
+
+for.body:
+ %first.addr = phi ptr [ %first, %entry], [ %first.next, %for.inc ]
+ %match.addr = phi ptr [ %addr2, %entry ], [ %match.next, %for.inc ]
+ %1 = load i32, ptr %first.addr, align 4
+ %match.value = load i32, ptr %match.addr, align 4
+ %cmp1 = icmp eq i32 %1, %match.value
+ br i1 %cmp1, label %early.exit, label %for.inc
+
+for.inc:
+ %match.next = getelementptr inbounds nuw i8, ptr %match.addr, i64 4
+ %first.next = getelementptr inbounds i8, ptr %first.addr, i64 4
+ %exit = icmp eq ptr %first.next, %last
+ br i1 %exit, label %main.exit, label %for.body
+
+early.exit:
+ br label %return
+
+main.exit:
+ br label %return
+
+return:
+ %retval = phi ptr [ %first, %entry ], [ %last, %main.exit ], [ %first.addr, %early.exit ]
+ ret ptr %retval
+}
+
+define ptr @unbound_strided_access(ptr %first, ptr %last, i32 %value) {
+; CHECK-LABEL: LV: Checking a loop in 'unbound_strided_access'
+; CHECK: LV: Not vectorizing: Loop contains strided unbound access.
+entry:
+ %cond = icmp eq ptr %first, %last
+ br i1 %cond, label %return, label %for.body
+
+for.body:
+ %first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
+ %1 = load i32, ptr %first.addr, align 4
+ %cond2= icmp eq i32 %1, %value
+ br i1 %cond2, label %for.end, label %for.inc
+
+for.inc:
+ %first.next = getelementptr inbounds i32, ptr %first.addr, i64 2
+ %cond3 = icmp eq ptr %first.next, %last
+ br i1 %cond3, label %for.end, label %for.body
+
+for.end:
+ %retval.ph = phi ptr [ %first.addr, %for.body ], [ %last, %for.inc ]
+ br label %return
+
+return:
+ %retval = phi ptr [ %first, %entry ], [ %retval.ph, %for.end ]
+ ret ptr %retval
+}
+
+define ptr @single_unbound_access(ptr %first, ptr %last, i32 %value) {
+; CHECK-LABEL: LV: Checking a loop in 'single_unbound_access'
+; CHECK: LV: We can vectorize this loop!
+; CHECK-NEXT: LV: Not vectorizing: Auto-vectorization of loops with speculative load is not supported.
+entry:
+ %cond = icmp eq ptr %first, %last
+ br i1 %cond, label %return, label %for.body
+
+for.body:
+ %first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
+ %1 = load i32, ptr %first.addr, align 4
+ %cond2= icmp eq i32 %1, %value
+ br i1 %cond2, label %for.end, label %for.inc
+
+for.inc:
+ %first.next = getelementptr inbounds i32, ptr %first.addr, i64 1
+ %cond3 = icmp eq ptr %first.next, %last
+ br i1 %cond3, label %for.end, label %for.body
+
+for.end:
+ %retval.ph = phi ptr [ %first.addr, %for.body ], [ %last, %for.inc ]
+ br label %return
+
+return:
+ %retval = phi ptr [ %first, %entry ], [ %retval.ph, %for.end ]
+ ret ptr %retval
+}
>From d3246bd1b132e63ede51b02417f37ae8f3d0cea3 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sat, 9 Aug 2025 15:39:30 -0700
Subject: [PATCH 04/15] clang-formatted
---
llvm/lib/Analysis/Loads.cpp | 3 ++-
.../Transforms/Vectorize/LoopVectorizationLegality.cpp | 8 ++++----
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index d05b89f86f554..b09dd227978b1 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -880,7 +880,8 @@ bool llvm::isReadOnlyLoopWithSafeOrSpeculativeLoads(
if (auto *LI = dyn_cast<LoadInst>(&I)) {
if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
SpeculativeLoads->push_back(LI);
- } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow()) {
+ } else if (I.mayReadFromMemory() || I.mayWriteToMemory() ||
+ I.mayThrow()) {
return false;
}
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index b20fab14d5384..0fa3a36db1a3e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1791,10 +1791,10 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
}
// Support single Speculative load for now.
if (NonDerefLoads.size() > 1) {
- reportVectorizationFailure("Loop contains more than one unbound access",
- "TooManySpeculativeLoadInEarlyExitLoop",
- ORE, TheLoop);
- return false;
+ reportVectorizationFailure("Loop contains more than one unbound access",
+ "TooManySpeculativeLoadInEarlyExitLoop", ORE,
+ TheLoop);
+ return false;
}
[[maybe_unused]] const SCEV *SymbolicMaxBTC =
>From eb316eeff7688d577e5a061ada36f7e68fada25a Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sat, 9 Aug 2025 15:42:06 -0700
Subject: [PATCH 05/15] trivial fix: use lower case
---
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0fa3a36db1a3e..37e1693e256bb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1789,7 +1789,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
SpeculativeLoads.insert(LI);
LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
}
- // Support single Speculative load for now.
+ // Support single speculative load for now.
if (NonDerefLoads.size() > 1) {
reportVectorizationFailure("Loop contains more than one unbound access",
"TooManySpeculativeLoadInEarlyExitLoop", ORE,
>From faa61124a3369bd72e0f01466102f6c22af26528 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Sat, 9 Aug 2025 15:46:12 -0700
Subject: [PATCH 06/15] trivial fix
---
.../Transforms/LoopVectorize/RISCV/unbound-access-legality.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
index e35d109304274..a5786a0d131f7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
@@ -43,7 +43,7 @@ entry:
for.body:
%first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
%1 = load i32, ptr %first.addr, align 4
- %cond2= icmp eq i32 %1, %value
+ %cond2 = icmp eq i32 %1, %value
br i1 %cond2, label %for.end, label %for.inc
for.inc:
@@ -71,7 +71,7 @@ entry:
for.body:
%first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
%1 = load i32, ptr %first.addr, align 4
- %cond2= icmp eq i32 %1, %value
+ %cond2 = icmp eq i32 %1, %value
br i1 %cond2, label %for.end, label %for.inc
for.inc:
>From b206c9f152010d330e2ec40c3822b5134b3e206f Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Mon, 11 Aug 2025 22:56:48 -0700
Subject: [PATCH 07/15] Address comment and refine TTI to check data type
---
llvm/include/llvm/Analysis/Loads.h | 6 ----
.../llvm/Analysis/TargetTransformInfo.h | 7 ++---
.../llvm/Analysis/TargetTransformInfoImpl.h | 6 ++--
llvm/lib/Analysis/Loads.cpp | 15 ---------
llvm/lib/Analysis/TargetTransformInfo.cpp | 9 +++---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 12 +++++++
llvm/lib/Target/RISCV/RISCVISelLowering.h | 4 +++
.../Target/RISCV/RISCVTargetTransformInfo.h | 8 +++--
.../Vectorize/LoopVectorizationLegality.cpp | 29 ++++++++---------
.../RISCV/unbound-access-legality.ll | 31 ++++++++-----------
llvm/unittests/Analysis/LoadsTest.cpp | 5 ++-
11 files changed, 63 insertions(+), 69 deletions(-)
diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index 080757b6d1fe0..7f28afafb3500 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -85,12 +85,6 @@ LLVM_ABI bool isDereferenceableAndAlignedInLoop(
AssumptionCache *AC = nullptr,
SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
-/// Return true if the loop \p L cannot fault on any iteration and only
-/// contains read-only memory accesses.
-LLVM_ABI bool isDereferenceableReadOnlyLoop(
- Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
-
/// Return true if the loop \p L cannot fault on any iteration and only
/// contains read-only memory accesses. Also collect loads that are not
/// guaranteed to be dereferenceable.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2b8e6be92238c..0fcc5d1b3fb98 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -843,6 +843,9 @@ class TargetTransformInfo {
/// Return true if the target supports strided load.
LLVM_ABI bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
+ /// Return true if the target supports speculative load.
+ LLVM_ABI bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const;
+
/// Return true is the target supports interleaved access for the given vector
/// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
/// address space \p AddrSpace.
@@ -1857,10 +1860,6 @@ class TargetTransformInfo {
/// \returns True if the target supports scalable vectors.
LLVM_ABI bool supportsScalableVectors() const;
- /// \returns True if the target supports speculative load intrinsics (e.g.,
- /// vp.load.ff).
- LLVM_ABI bool supportsSpeculativeLoads() const;
-
/// \return true when scalable vectorization is preferred.
LLVM_ABI bool enableScalableVectorization() const;
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1df93ecc7ec16..6aca7e1412271 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -374,6 +374,10 @@ class TargetTransformInfoImplBase {
return false;
}
+ virtual bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const {
+ return false;
+ }
+
virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
Align Alignment,
unsigned AddrSpace) const {
@@ -1106,8 +1110,6 @@ class TargetTransformInfoImplBase {
virtual bool supportsScalableVectors() const { return false; }
- virtual bool supportsSpeculativeLoads() const { return false; }
-
virtual bool enableScalableVectorization() const { return false; }
virtual bool hasActiveVectorLength() const { return false; }
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index b09dd227978b1..c39587131dac0 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -856,21 +856,6 @@ bool llvm::canReplacePointersIfEqual(const Value *From, const Value *To,
return isPointerAlwaysReplaceable(From, To, DL);
}
-bool llvm::isDereferenceableReadOnlyLoop(
- Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- SmallVectorImpl<const SCEVPredicate *> *Predicates) {
- for (BasicBlock *BB : L->blocks()) {
- for (Instruction &I : *BB) {
- if (auto *LI = dyn_cast<LoadInst>(&I)) {
- if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
- return false;
- } else if (I.mayReadFromMemory() || I.mayWriteToMemory() || I.mayThrow())
- return false;
- }
- }
- return true;
-}
-
bool llvm::isReadOnlyLoopWithSafeOrSpeculativeLoads(
Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
SmallVectorImpl<LoadInst *> *SpeculativeLoads,
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 9f05e01d34781..3e94c816488fd 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -531,6 +531,11 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+bool TargetTransformInfo::isLegalSpeculativeLoad(Type *DataType,
+ Align Alignment) const {
+ return TTIImpl->isLegalSpeculativeLoad(DataType, Alignment);
+}
+
bool TargetTransformInfo::isLegalInterleavedAccessType(
VectorType *VTy, unsigned Factor, Align Alignment,
unsigned AddrSpace) const {
@@ -1457,10 +1462,6 @@ bool TargetTransformInfo::supportsScalableVectors() const {
return TTIImpl->supportsScalableVectors();
}
-bool TargetTransformInfo::supportsSpeculativeLoads() const {
- return TTIImpl->supportsSpeculativeLoads();
-}
-
bool TargetTransformInfo::enableScalableVectorization() const {
return TTIImpl->enableScalableVectorization();
}
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 03e54b3d395e3..192435d134e0f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -24405,6 +24405,18 @@ bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
return true;
}
+bool RISCVTargetLowering::isLegalSpeculativeLoad(EVT DataType,
+ Align Alignment) const {
+ if (!Subtarget.hasVInstructions())
+ return false;
+
+ EVT ScalarType = DataType.getScalarType();
+ if (!isLegalElementTypeForRVV(ScalarType))
+ return false;
+
+ return true;
+}
+
MachineInstr *
RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
MachineBasicBlock::instr_iterator &MBBI,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 433b8be5c562e..a99f6fb38c8b6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -425,6 +425,10 @@ class RISCVTargetLowering : public TargetLowering {
/// alignment is legal.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const;
+ /// Return true if a speculative load of the given result type and
+ /// alignment is legal.
+ bool isLegalSpeculativeLoad(EVT DataType, Align Alignment) const;
+
unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
bool fallBackToDAGISel(const Instruction &Inst) const override;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 54e9c8346b6e2..ba13bdf89063a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -110,9 +110,6 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
bool supportsScalableVectors() const override {
return ST->hasVInstructions();
}
- bool supportsSpeculativeLoads() const override {
- return ST->hasVInstructions();
- }
bool enableOrderedReductions() const override { return true; }
bool enableScalableVectorization() const override {
return ST->hasVInstructions();
@@ -327,6 +324,11 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
}
+ bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const override {
+ EVT DataTypeVT = TLI->getValueType(DL, DataType);
+ return TLI->isLegalSpeculativeLoad(DataTypeVT, Alignment);
+ }
+
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
Align Alignment,
unsigned AddrSpace) const override {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 37e1693e256bb..1d7ec4f43311f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1762,40 +1762,37 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
Predicates.clear();
SmallVector<LoadInst *, 4> NonDerefLoads;
- bool HasSafeAccess =
- TTI->supportsSpeculativeLoads()
- ? isReadOnlyLoopWithSafeOrSpeculativeLoads(
- TheLoop, PSE.getSE(), DT, AC, &NonDerefLoads, &Predicates)
- : isDereferenceableReadOnlyLoop(TheLoop, PSE.getSE(), DT, AC,
- &Predicates);
- if (!HasSafeAccess) {
+ if (!isReadOnlyLoopWithSafeOrSpeculativeLoads(TheLoop, PSE.getSE(), DT, AC,
+ &NonDerefLoads, &Predicates)) {
reportVectorizationFailure(
"Loop may fault",
"Cannot vectorize potentially faulting early exit loop",
"PotentiallyFaultingEarlyExitLoop", ORE, TheLoop);
return false;
}
- // Speculative loads need to be unit-stride.
+ // Check non-dereferenceable loads if any.
for (LoadInst *LI : NonDerefLoads) {
+ // Only support unit-stride access for now.
int Stride = isConsecutivePtr(LI->getType(), LI->getPointerOperand());
if (Stride != 1) {
reportVectorizationFailure("Loop contains strided unbound access",
"Cannot vectorize early exit loop with "
- "speculative non-unit-stride load",
+ "speculative strided load",
"SpeculativeNonUnitStrideLoadEarlyExitLoop",
ORE, TheLoop);
return false;
}
+ if (!TTI->isLegalSpeculativeLoad(LI->getType(), LI->getAlign())) {
+ reportVectorizationFailure("Loop may fault",
+ "Cannot vectorize early exit loop with "
+ "illegal speculative load",
+ "IllegalSpeculativeLoadEarlyExitLoop", ORE,
+ TheLoop);
+ return false;
+ }
SpeculativeLoads.insert(LI);
LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
}
- // Support single speculative load for now.
- if (NonDerefLoads.size() > 1) {
- reportVectorizationFailure("Loop contains more than one unbound access",
- "TooManySpeculativeLoadInEarlyExitLoop", ORE,
- TheLoop);
- return false;
- }
[[maybe_unused]] const SCEV *SymbolicMaxBTC =
PSE.getSymbolicMaxBackedgeTakenCount();
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
index a5786a0d131f7..07e64784da84b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
@@ -1,35 +1,30 @@
; REQUIRES: asserts
; RUN: opt -passes=loop-vectorize -debug-only=loop-vectorize -disable-output 2>&1 -mtriple=riscv64 -mattr=+v -S %s | FileCheck %s
-define ptr @two_unbound_access(ptr %first, ptr %last, ptr %addr2) {
-; CHECK-LABEL: LV: Checking a loop in 'two_unbound_access'
-; CHECK: LV: Not vectorizing: Loop contains more than one unbound access.
+define ptr @unsupported_data_type(ptr %first, ptr %last, i128 %value) {
+; CHECK-LABEL: LV: Checking a loop in 'unsupported_data_type'
+; CHECK: LV: Not vectorizing: Loop may fault.
entry:
%cond = icmp eq ptr %first, %last
br i1 %cond, label %return, label %for.body
for.body:
- %first.addr = phi ptr [ %first, %entry], [ %first.next, %for.inc ]
- %match.addr = phi ptr [ %addr2, %entry ], [ %match.next, %for.inc ]
- %1 = load i32, ptr %first.addr, align 4
- %match.value = load i32, ptr %match.addr, align 4
- %cmp1 = icmp eq i32 %1, %match.value
- br i1 %cmp1, label %early.exit, label %for.inc
+ %first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
+ %1 = load i128, ptr %first.addr, align 4
+ %cond2 = icmp eq i128 %1, %value
+ br i1 %cond2, label %for.end, label %for.inc
for.inc:
- %match.next = getelementptr inbounds nuw i8, ptr %match.addr, i64 4
- %first.next = getelementptr inbounds i8, ptr %first.addr, i64 4
- %exit = icmp eq ptr %first.next, %last
- br i1 %exit, label %main.exit, label %for.body
-
-early.exit:
- br label %return
+ %first.next = getelementptr inbounds i128, ptr %first.addr, i64 1
+ %cond3 = icmp eq ptr %first.next, %last
+ br i1 %cond3, label %for.end, label %for.body
-main.exit:
+for.end:
+ %retval.ph = phi ptr [ %first.addr, %for.body ], [ %last, %for.inc ]
br label %return
return:
- %retval = phi ptr [ %first, %entry ], [ %last, %main.exit ], [ %first.addr, %early.exit ]
+ %retval = phi ptr [ %first, %entry ], [ %retval.ph, %for.end ]
ret ptr %retval
}
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index c4f5b22318e34..fab2aeb745ad0 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -195,7 +195,10 @@ loop.end:
assert(Header->getName() == "loop");
Loop *L = LI.getLoopFor(Header);
- return isDereferenceableReadOnlyLoop(L, &SE, &DT, &AC);
+ SmallVector<LoadInst *, 4> NonDerefLoads;
+ return isReadOnlyLoopWithSafeOrSpeculativeLoads(L, &SE, &DT, &AC,
+ &NonDerefLoads) &&
+ NonDerefLoads.empty();
};
ASSERT_TRUE(IsDerefReadOnlyLoop(F1));
>From 72383ddc007db4c577d817006fed1875a48569a5 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 20 Aug 2025 18:45:21 -0700
Subject: [PATCH 08/15] Query subtarget feature for misaligned access support
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 192435d134e0f..25305bd10f115 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -24414,6 +24414,10 @@ bool RISCVTargetLowering::isLegalSpeculativeLoad(EVT DataType,
if (!isLegalElementTypeForRVV(ScalarType))
return false;
+ if (!Subtarget.enableUnalignedVectorMem() &&
+ Alignment < ScalarType.getStoreSize())
+ return false;
+
return true;
}
>From 0c1d007f8226ce4c321f85ad6f70c2427ca4ae53 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Fri, 22 Aug 2025 00:24:07 -0700
Subject: [PATCH 09/15] Add align 1 case
---
.../RISCV/unbound-access-legality.ll | 28 +++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
index 07e64784da84b..1570b86f91c6e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/unbound-access-legality.ll
@@ -55,6 +55,34 @@ return:
ret ptr %retval
}
+; The alignment promise is 1 byte.
+define ptr @single_illegal_align1_access(ptr %first, ptr %last, i32 %value) {
+; CHECK-LABEL: LV: Checking a loop in 'single_illegal_align1_access'
+; CHECK: LV: Not vectorizing: Loop may fault.
+entry:
+ %cond = icmp eq ptr %first, %last
+ br i1 %cond, label %return, label %for.body
+
+for.body:
+ %first.addr = phi ptr [ %first, %entry ], [ %first.next, %for.inc ]
+ %1 = load i32, ptr %first.addr, align 1
+ %cond2 = icmp eq i32 %1, %value
+ br i1 %cond2, label %for.end, label %for.inc
+
+for.inc:
+ %first.next = getelementptr inbounds i32, ptr %first.addr, i64 1
+ %cond3 = icmp eq ptr %first.next, %last
+ br i1 %cond3, label %for.end, label %for.body
+
+for.end:
+ %retval.ph = phi ptr [ %first.addr, %for.body ], [ %last, %for.inc ]
+ br label %return
+
+return:
+ %retval = phi ptr [ %first, %entry ], [ %retval.ph, %for.end ]
+ ret ptr %retval
+}
+
define ptr @single_unbound_access(ptr %first, ptr %last, i32 %value) {
; CHECK-LABEL: LV: Checking a loop in 'single_unbound_access'
; CHECK: LV: We can vectorize this loop!
>From e22c4860b260f127f1b1d32ec9be807764cdcf90 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 26 Aug 2025 19:22:53 -0700
Subject: [PATCH 10/15] Rename isLegalSpeculativeLoad to
isLegalFaultOnlyFirstLoad
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 5 +++--
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h | 3 ++-
llvm/lib/Analysis/TargetTransformInfo.cpp | 6 +++---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 4 ++--
llvm/lib/Target/RISCV/RISCVISelLowering.h | 4 ++--
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h | 5 +++--
llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 +-
7 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 0fcc5d1b3fb98..709cec1fef8b4 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -843,8 +843,9 @@ class TargetTransformInfo {
/// Return true if the target supports strided load.
LLVM_ABI bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
- /// Return true if the target supports speculative load.
- LLVM_ABI bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const;
+ /// Return true if the target supports fault-only-first load.
+ LLVM_ABI bool isLegalFaultOnlyFirstLoad(Type *DataType,
+ Align Alignment) const;
/// Return true is the target supports interleaved access for the given vector
/// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 6aca7e1412271..8d5b9e0d98485 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -374,7 +374,8 @@ class TargetTransformInfoImplBase {
return false;
}
- virtual bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const {
+ virtual bool isLegalFaultOnlyFirstLoad(Type *DataType,
+ Align Alignment) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 3e94c816488fd..ca539fb1edb85 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -531,9 +531,9 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
-bool TargetTransformInfo::isLegalSpeculativeLoad(Type *DataType,
- Align Alignment) const {
- return TTIImpl->isLegalSpeculativeLoad(DataType, Alignment);
+bool TargetTransformInfo::isLegalFaultOnlyFirstLoad(Type *DataType,
+ Align Alignment) const {
+ return TTIImpl->isLegalFaultOnlyFirstLoad(DataType, Alignment);
}
bool TargetTransformInfo::isLegalInterleavedAccessType(
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 25305bd10f115..2ce0e4aa27647 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -24405,8 +24405,8 @@ bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
return true;
}
-bool RISCVTargetLowering::isLegalSpeculativeLoad(EVT DataType,
- Align Alignment) const {
+bool RISCVTargetLowering::isLegalFaultOnlyFirstLoad(EVT DataType,
+ Align Alignment) const {
if (!Subtarget.hasVInstructions())
return false;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index a99f6fb38c8b6..93e91f28bb5b8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -425,9 +425,9 @@ class RISCVTargetLowering : public TargetLowering {
/// alignment is legal.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const;
- /// Return true if a speculative load of the given result type and
+ /// Return true if a fault-only-first load of the given result type and
/// alignment is legal.
- bool isLegalSpeculativeLoad(EVT DataType, Align Alignment) const;
+ bool isLegalFaultOnlyFirstLoad(EVT DataType, Align Alignment) const;
unsigned getMaxSupportedInterleaveFactor() const override { return 8; }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index ba13bdf89063a..c076f53ee52cb 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -324,9 +324,10 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
}
- bool isLegalSpeculativeLoad(Type *DataType, Align Alignment) const override {
+ bool isLegalFaultOnlyFirstLoad(Type *DataType,
+ Align Alignment) const override {
EVT DataTypeVT = TLI->getValueType(DL, DataType);
- return TLI->isLegalSpeculativeLoad(DataTypeVT, Alignment);
+ return TLI->isLegalFaultOnlyFirstLoad(DataTypeVT, Alignment);
}
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 1d7ec4f43311f..dfe8324f2150d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1782,7 +1782,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
ORE, TheLoop);
return false;
}
- if (!TTI->isLegalSpeculativeLoad(LI->getType(), LI->getAlign())) {
+ if (!TTI->isLegalFaultOnlyFirstLoad(LI->getType(), LI->getAlign())) {
reportVectorizationFailure("Loop may fault",
"Cannot vectorize early exit loop with "
"illegal speculative load",
>From 2a372161d250ae8287b9f1ae47e12f8ee9900cde Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 26 Aug 2025 19:42:58 -0700
Subject: [PATCH 11/15] Rename isReadOnlyLoopWithSafeOrSpeculativeLoads to
isLoopSafeWithLoadOnlyFaults
---
llvm/include/llvm/Analysis/Loads.h | 9 ++++-----
llvm/lib/Analysis/Loads.cpp | 6 +++---
.../Transforms/Vectorize/LoopVectorizationLegality.cpp | 4 ++--
llvm/unittests/Analysis/LoadsTest.cpp | 3 +--
4 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/Analysis/Loads.h b/llvm/include/llvm/Analysis/Loads.h
index 7f28afafb3500..f8ff9f2d10dd2 100644
--- a/llvm/include/llvm/Analysis/Loads.h
+++ b/llvm/include/llvm/Analysis/Loads.h
@@ -85,12 +85,11 @@ LLVM_ABI bool isDereferenceableAndAlignedInLoop(
AssumptionCache *AC = nullptr,
SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
-/// Return true if the loop \p L cannot fault on any iteration and only
-/// contains read-only memory accesses. Also collect loads that are not
-/// guaranteed to be dereferenceable.
-LLVM_ABI bool isReadOnlyLoopWithSafeOrSpeculativeLoads(
+/// Returns true if the only potentially faulting operations in loop are loads.
+/// Also collect loads that are not guaranteed to be dereferenceable.
+LLVM_ABI bool isLoopSafeWithLoadOnlyFaults(
Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- SmallVectorImpl<LoadInst *> *SpeculativeLoads,
+ SmallVectorImpl<LoadInst *> *NonDereferenceableAndAlignedLoads,
SmallVectorImpl<const SCEVPredicate *> *Predicates = nullptr);
/// Return true if we know that executing a load from this value cannot trap.
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index c39587131dac0..0b90b539503f7 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -856,15 +856,15 @@ bool llvm::canReplacePointersIfEqual(const Value *From, const Value *To,
return isPointerAlwaysReplaceable(From, To, DL);
}
-bool llvm::isReadOnlyLoopWithSafeOrSpeculativeLoads(
+bool llvm::isLoopSafeWithLoadOnlyFaults(
Loop *L, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- SmallVectorImpl<LoadInst *> *SpeculativeLoads,
+ SmallVectorImpl<LoadInst *> *NonDereferenceableAndAlignedLoads,
SmallVectorImpl<const SCEVPredicate *> *Predicates) {
for (BasicBlock *BB : L->blocks()) {
for (Instruction &I : *BB) {
if (auto *LI = dyn_cast<LoadInst>(&I)) {
if (!isDereferenceableAndAlignedInLoop(LI, L, *SE, *DT, AC, Predicates))
- SpeculativeLoads->push_back(LI);
+ NonDereferenceableAndAlignedLoads->push_back(LI);
} else if (I.mayReadFromMemory() || I.mayWriteToMemory() ||
I.mayThrow()) {
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index dfe8324f2150d..0620804675920 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1762,8 +1762,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
Predicates.clear();
SmallVector<LoadInst *, 4> NonDerefLoads;
- if (!isReadOnlyLoopWithSafeOrSpeculativeLoads(TheLoop, PSE.getSE(), DT, AC,
- &NonDerefLoads, &Predicates)) {
+ if (!isLoopSafeWithLoadOnlyFaults(TheLoop, PSE.getSE(), DT, AC,
+ &NonDerefLoads, &Predicates)) {
reportVectorizationFailure(
"Loop may fault",
"Cannot vectorize potentially faulting early exit loop",
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index fab2aeb745ad0..59d6e01e90021 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -196,8 +196,7 @@ loop.end:
Loop *L = LI.getLoopFor(Header);
SmallVector<LoadInst *, 4> NonDerefLoads;
- return isReadOnlyLoopWithSafeOrSpeculativeLoads(L, &SE, &DT, &AC,
- &NonDerefLoads) &&
+ return isLoopSafeWithLoadOnlyFaults(L, &SE, &DT, &AC, &NonDerefLoads) &&
NonDerefLoads.empty();
};
>From abb0120cf621bcafab480db1f2222a705515aca6 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 26 Aug 2025 19:53:38 -0700
Subject: [PATCH 12/15] Rename SpeculativeLoads to FaultOnlyFirstLoads
---
.../Transforms/Vectorize/LoopVectorizationLegality.h | 10 +++++-----
.../Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 +-
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 3b5638f3f570a..b41ca453ca68d 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -445,9 +445,9 @@ class LoopVectorizationLegality {
/// Returns a list of all known histogram operations in the loop.
bool hasHistograms() const { return !Histograms.empty(); }
- /// Returns the loads that may fault and need to be speculative.
- const SmallPtrSetImpl<const Instruction *> &getSpeculativeLoads() const {
- return SpeculativeLoads;
+ /// Returns the loads that need to be fault-only-first.
+ const SmallPtrSetImpl<const Instruction *> &getFaultOnlyFirstLoads() const {
+ return FaultOnlyFirstLoads;
}
PredicatedScalarEvolution *getPredicatedScalarEvolution() const {
@@ -635,8 +635,8 @@ class LoopVectorizationLegality {
/// may work on the same memory location.
SmallVector<HistogramInfo, 1> Histograms;
- /// Hold all loads that need to be speculative.
- SmallPtrSet<const Instruction *, 4> SpeculativeLoads;
+ /// Hold all loads that need to be fault-only-first.
+ SmallPtrSet<const Instruction *, 4> FaultOnlyFirstLoads;
/// BFI and PSI are used to check for profile guided size optimizations.
BlockFrequencyInfo *BFI;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0620804675920..43d46af906750 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1790,7 +1790,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
TheLoop);
return false;
}
- SpeculativeLoads.insert(LI);
+ FaultOnlyFirstLoads.insert(LI);
LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 790a5236d4f04..836e4ca96a13d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10041,7 +10041,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}
- if (!LVL.getSpeculativeLoads().empty()) {
+ if (!LVL.getFaultOnlyFirstLoads().empty()) {
reportVectorizationFailure("Auto-vectorization of loops with speculative "
"load is not supported",
"SpeculativeLoadsNotSupported", ORE, L);
>From 6b920f472aab73d0167b7275bf6746e816f3beaa Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 26 Aug 2025 20:14:20 -0700
Subject: [PATCH 13/15] Refine comments with ff loads
---
.../Vectorize/LoopVectorizationLegality.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 43d46af906750..e741056a448ae 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1766,8 +1766,8 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
&NonDerefLoads, &Predicates)) {
reportVectorizationFailure(
"Loop may fault",
- "Cannot vectorize potentially faulting early exit loop",
- "PotentiallyFaultingEarlyExitLoop", ORE, TheLoop);
+ "Cannot vectorize early exit loop with non-load faults",
+ "EarlyExitLoopWithNonLoadFaults", ORE, TheLoop);
return false;
}
// Check non-dereferenceable loads if any.
@@ -1777,21 +1777,21 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
if (Stride != 1) {
reportVectorizationFailure("Loop contains strided unbound access",
"Cannot vectorize early exit loop with "
- "speculative strided load",
- "SpeculativeNonUnitStrideLoadEarlyExitLoop",
+ "fault-only-first strided load",
+ "EarlyExitLoopWithStridedFaultOnlyFirstLoad",
ORE, TheLoop);
return false;
}
if (!TTI->isLegalFaultOnlyFirstLoad(LI->getType(), LI->getAlign())) {
reportVectorizationFailure("Loop may fault",
"Cannot vectorize early exit loop with "
- "illegal speculative load",
- "IllegalSpeculativeLoadEarlyExitLoop", ORE,
- TheLoop);
+ "illegal fault-only-first load",
+ "EarlyExitLoopWithIllegalFaultOnlyFirstLoad",
+ ORE, TheLoop);
return false;
}
FaultOnlyFirstLoads.insert(LI);
- LLVM_DEBUG(dbgs() << "LV: Found speculative load: " << *LI << "\n");
+ LLVM_DEBUG(dbgs() << "LV: Found fault-only-first load: " << *LI << "\n");
}
[[maybe_unused]] const SCEV *SymbolicMaxBTC =
>From 22dee5b2f9ac2eb9592200fbd9f1515a7ec75219 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 27 Aug 2025 00:11:15 -0700
Subject: [PATCH 14/15] Update unittest to check the returned instructions
---
llvm/unittests/Analysis/LoadsTest.cpp | 23 +++++++++++++++++++----
1 file changed, 19 insertions(+), 4 deletions(-)
diff --git a/llvm/unittests/Analysis/LoadsTest.cpp b/llvm/unittests/Analysis/LoadsTest.cpp
index 59d6e01e90021..17679af730665 100644
--- a/llvm/unittests/Analysis/LoadsTest.cpp
+++ b/llvm/unittests/Analysis/LoadsTest.cpp
@@ -14,6 +14,7 @@
#include "llvm/AsmParser/Parser.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
@@ -120,7 +121,14 @@ define void @f(i32* %p1, i32* %p2, i64 %i) {
EXPECT_TRUE(canReplacePointersInUseIfEqual(IcmpUse, P2, DL));
}
-TEST(LoadsTest, IsDerefReadOnlyLoop) {
+static Instruction *getInstructionByName(Function &F, StringRef Name) {
+ for (auto &I : instructions(F))
+ if (I.getName() == Name)
+ return &I;
+ llvm_unreachable("Expected to find instruction!");
+}
+
+TEST(LoadsTest, IsLoadOnlyFaultingLoop) {
LLVMContext C;
std::unique_ptr<Module> M = parseIR(C,
R"IR(
@@ -180,10 +188,14 @@ loop.end:
auto *F2 = dyn_cast<Function>(GV2);
ASSERT_TRUE(F1 && F2);
+ auto *NonDerefLI = dyn_cast<LoadInst>(getInstructionByName(*F2, "ld1"));
+ ASSERT_TRUE(NonDerefLI);
+
TargetLibraryInfoImpl TLII(M->getTargetTriple());
TargetLibraryInfo TLI(TLII);
- auto IsDerefReadOnlyLoop = [&TLI](Function *F) -> bool {
+ auto IsLoadOnlyFaultingLoop = [&TLI](Function *F,
+ LoadInst *FFLI = nullptr) -> bool {
AssumptionCache AC(*F);
DominatorTree DT(*F);
LoopInfo LI(DT);
@@ -196,10 +208,13 @@ loop.end:
Loop *L = LI.getLoopFor(Header);
SmallVector<LoadInst *, 4> NonDerefLoads;
+ if (FFLI)
+ return isLoopSafeWithLoadOnlyFaults(L, &SE, &DT, &AC, &NonDerefLoads) &&
+ (NonDerefLoads.size() == 1) && (NonDerefLoads[0] == FFLI);
return isLoopSafeWithLoadOnlyFaults(L, &SE, &DT, &AC, &NonDerefLoads) &&
NonDerefLoads.empty();
};
- ASSERT_TRUE(IsDerefReadOnlyLoop(F1));
- ASSERT_FALSE(IsDerefReadOnlyLoop(F2));
+ ASSERT_TRUE(IsLoadOnlyFaultingLoop(F1));
+ ASSERT_TRUE(IsLoadOnlyFaultingLoop(F2, NonDerefLI));
}
>From e88d5d4d806f6200cfa4239fefd596870837bdd8 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 27 Aug 2025 00:16:39 -0700
Subject: [PATCH 15/15] Refine report messages
---
.../lib/Transforms/Vectorize/LoopVectorizationLegality.cpp | 2 +-
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index e741056a448ae..c82912f3d0c5b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1777,7 +1777,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
if (Stride != 1) {
reportVectorizationFailure("Loop contains strided unbound access",
"Cannot vectorize early exit loop with "
- "fault-only-first strided load",
+ "strided fault-only-first load",
"EarlyExitLoopWithStridedFaultOnlyFirstLoad",
ORE, TheLoop);
return false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 836e4ca96a13d..218b9e1cb1c27 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -10042,9 +10042,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
}
if (!LVL.getFaultOnlyFirstLoads().empty()) {
- reportVectorizationFailure("Auto-vectorization of loops with speculative "
- "load is not supported",
- "SpeculativeLoadsNotSupported", ORE, L);
+ reportVectorizationFailure(
+ "Auto-vectorization of loops with fault-only-first "
+ "load is not supported",
+ "FaultOnlyFirstLoadsNotSupported", ORE, L);
return false;
}
More information about the llvm-commits
mailing list