[llvm] [InterleavedAccessPass] Pass modifies its input and doesn't report it (PR #198773)
Sjoerd Meijer via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 1 01:55:52 PDT 2026
https://github.com/sjoerdmeijer updated https://github.com/llvm/llvm-project/pull/198773
>From feefe3b3cc5dafd7b339ecd3d802a7ae36da7d96 Mon Sep 17 00:00:00 2001
From: Sjoerd Meijer <smeijer at nvidia.com>
Date: Wed, 20 May 2026 03:59:13 -0700
Subject: [PATCH 1/2] [InterleavedAccessPass] Pass modifies its input and
doesn't report it
In a build with expensive checks enabled, it is very easy to trigger
this error in the InterleavedAccess pass:
Pass modifies its input and doesn't report it: Interleaved Access Pass
Pass modifies its input and doesn't report it
UNREACHABLE executed at /local/home/smeijer/llvm-project/llvm/lib/IR/LegacyPassManager.cpp:1404!
This patch removes (some) IR changes when the pass bails out.
---
llvm/lib/CodeGen/InterleavedAccessPass.cpp | 39 +++++++++++++++----
.../AArch64/masked-store-mask-cleanup.ll | 29 ++++++++++++++
2 files changed, 61 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 5498ce1d81647..a1a4966e40047 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -277,6 +277,14 @@ static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
return getMask(WideMask, Factor, LeafValueTy->getElementCount());
}
+// Function getMask() may insert new instructions to materialise the per-lane
+// mask before we bail out. This recursively deletes them so we don't leave dead
+// IR around.
+static void eraseDeadMaskInstructions(Value *Mask) {
+ if (auto *MaskI = dyn_cast_or_null<Instruction>(Mask))
+ RecursivelyDeleteTriviallyDeadInstructions(MaskI);
+}
+
bool InterleavedAccessImpl::lowerInterleavedLoad(
Instruction *Load, SmallSetVector<Instruction *, 32> &DeadInsts) {
if (isa<ScalableVectorType>(Load->getType()))
@@ -376,6 +384,16 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
bool BinOpShuffleChanged =
replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, Load);
+ // tryReplaceExtracts and replaceBinOpShuffles above can mutate the IR as a
+ // side-effect of succeeding. tryReplaceExtracts rewrites every entry in
+ // Extracts when it can rewrite all of them . And replaceBinOpShuffles returns
+ // true iff it rewrote at least one binop(shuffle()) pair. If we end up
+ // bailing out of the rest of the interleaved-load transformation below, we
+ // must still report those earlier mutations.
+ auto hasMutatedIR = [&] {
+ return !Extracts.empty() || BinOpShuffleChanged;
+ };
+
Value *Mask = nullptr;
auto GapMask = APInt::getAllOnes(Factor);
if (LI) {
@@ -384,7 +402,7 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
// Check mask operand. Handle both all-true/false and interleaved mask.
std::tie(Mask, GapMask) = getMask(getMaskOperand(II), Factor, VecTy);
if (!Mask)
- return false;
+ return hasMutatedIR();
LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load or masked.load: "
<< *Load << "\n");
@@ -395,9 +413,10 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
// Try to create target specific intrinsics to replace the load and
// shuffles.
if (!TLI->lowerInterleavedLoad(cast<Instruction>(Load), Mask, Shuffles,
- Indices, Factor, GapMask))
- // If Extracts is not empty, tryReplaceExtracts made changes earlier.
- return !Extracts.empty() || BinOpShuffleChanged;
+ Indices, Factor, GapMask)) {
+ eraseDeadMaskInstructions(Mask);
+ return hasMutatedIR();
+ }
DeadInsts.insert_range(Shuffles);
@@ -551,8 +570,10 @@ bool InterleavedAccessImpl::lowerInterleavedStore(
// Try to create target specific intrinsics to replace the store and
// shuffle.
- if (!TLI->lowerInterleavedStore(Store, Mask, SVI, Factor, GapMask))
+ if (!TLI->lowerInterleavedStore(Store, Mask, SVI, Factor, GapMask)) {
+ eraseDeadMaskInstructions(Mask);
return false;
+ }
// Already have a new target specific interleaved store. Erase the old store.
DeadInsts.insert(Store);
@@ -725,8 +746,10 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI, GapMask))
+ if (!TLI->lowerDeinterleaveIntrinsicToLoad(LoadedVal, Mask, DI, GapMask)) {
+ eraseDeadMaskInstructions(Mask);
return false;
+ }
DeadInsts.insert(DI);
// We now have a target-specific load, so delete the old one.
@@ -777,8 +800,10 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
}
// Try and match this with target specific intrinsics.
- if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues))
+ if (!TLI->lowerInterleaveIntrinsicToStore(StoredBy, Mask, InterleaveValues)) {
+ eraseDeadMaskInstructions(Mask);
return false;
+ }
// We now have a target-specific store, so delete the old one.
DeadInsts.insert(StoredBy);
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
new file mode 100644
index 0000000000000..079cfb1f3e080
--- /dev/null
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s
+
+; Make sure no llvm.vector.extract call is left over in the function body,
+; to avoid error "Pass modifies its input and doesn't report it". The
+; problem is that getMask() materialises this vector extract before the
+; pass bails out as it doesn't support a masked store.
+
+target triple = "aarch64-linux-gnu"
+
+define void @masked_store_splat_mask(ptr %p, i32 %x, i1 %m) {
+; CHECK-LABEL: define void @masked_store_splat_mask
+; CHECK-SAME: (ptr [[P:%.*]], i32 [[X:%.*]], i1 [[M:%.*]]) {
+; CHECK-NEXT: [[SPLAT_V_IN:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0
+; CHECK-NEXT: [[SPLAT_V:%.*]] = shufflevector <4 x i32> [[SPLAT_V_IN]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[SPLAT_M_IN:%.*]] = insertelement <4 x i1> poison, i1 [[M]], i32 0
+; CHECK-NEXT: [[SPLAT_M:%.*]] = shufflevector <4 x i1> [[SPLAT_M_IN]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: tail call void @llvm.masked.store.v4i32.p0(<4 x i32> [[SPLAT_V]], ptr align 8 [[P]], <4 x i1> [[SPLAT_M]])
+; CHECK-NEXT: ret void
+;
+ %splat_v_in = insertelement <4 x i32> poison, i32 %x, i32 0
+ %splat_v = shufflevector <4 x i32> %splat_v_in, <4 x i32> poison, <4 x i32> zeroinitializer
+ %splat_m_in = insertelement <4 x i1> poison, i1 %m, i32 0
+ %splat_m = shufflevector <4 x i1> %splat_m_in, <4 x i1> poison, <4 x i32> zeroinitializer
+ tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %splat_v, ptr %p, i32 8, <4 x i1> %splat_m)
+ ret void
+}
+
+declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)
>From 568813dba455b810284fe61e912a830303c3a1b8 Mon Sep 17 00:00:00 2001
From: Sjoerd Meijer <smeijer at nvidia.com>
Date: Mon, 1 Jun 2026 01:55:09 -0700
Subject: [PATCH 2/2] Addressed comments.
---
llvm/lib/CodeGen/InterleavedAccessPass.cpp | 25 ++++++++-----------
.../AArch64/masked-store-mask-cleanup.ll | 2 --
2 files changed, 10 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index a1a4966e40047..7e566276be170 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -384,16 +384,6 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
bool BinOpShuffleChanged =
replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, Load);
- // tryReplaceExtracts and replaceBinOpShuffles above can mutate the IR as a
- // side-effect of succeeding. tryReplaceExtracts rewrites every entry in
- // Extracts when it can rewrite all of them . And replaceBinOpShuffles returns
- // true iff it rewrote at least one binop(shuffle()) pair. If we end up
- // bailing out of the rest of the interleaved-load transformation below, we
- // must still report those earlier mutations.
- auto hasMutatedIR = [&] {
- return !Extracts.empty() || BinOpShuffleChanged;
- };
-
Value *Mask = nullptr;
auto GapMask = APInt::getAllOnes(Factor);
if (LI) {
@@ -401,8 +391,15 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
} else {
// Check mask operand. Handle both all-true/false and interleaved mask.
std::tie(Mask, GapMask) = getMask(getMaskOperand(II), Factor, VecTy);
+
+ // tryReplaceExtracts and replaceBinOpShuffles above can mutate the IR as a
+ // side-effect of succeeding. tryReplaceExtracts rewrites every entry in
+ // Extracts when it can rewrite all of them . And replaceBinOpShuffles returns
+ // true iff it rewrote at least one binop(shuffle()) pair. If we end up
+ // bailing out of the rest of the interleaved-load transformation below, we
+ // must still report those earlier mutations.
if (!Mask)
- return hasMutatedIR();
+ return !Extracts.empty() || BinOpShuffleChanged;
LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load or masked.load: "
<< *Load << "\n");
@@ -413,10 +410,8 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
// Try to create target specific intrinsics to replace the load and
// shuffles.
if (!TLI->lowerInterleavedLoad(cast<Instruction>(Load), Mask, Shuffles,
- Indices, Factor, GapMask)) {
- eraseDeadMaskInstructions(Mask);
- return hasMutatedIR();
- }
+ Indices, Factor, GapMask))
+ return true;
DeadInsts.insert_range(Shuffles);
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
index 079cfb1f3e080..97827ad439fd6 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/masked-store-mask-cleanup.ll
@@ -25,5 +25,3 @@ define void @masked_store_splat_mask(ptr %p, i32 %x, i1 %m) {
tail call void @llvm.masked.store.v4i32.p0(<4 x i32> %splat_v, ptr %p, i32 8, <4 x i1> %splat_m)
ret void
}
-
-declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)
More information about the llvm-commits
mailing list