[llvm] [VPlan] Don't bail creating interleave group if it has dead members (PR #208415)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 00:41:05 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/208415
>From 53744d5e8ce1de98c02cc3edc60a4689c4069b0a Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 9 Jul 2026 17:26:34 +0800
Subject: [PATCH 1/5] [VPlan] Don't bail creating interleave group if it has
dead members
This is an alternative to both #203555 and #205584.
To recap the issue:
1) The legacy cost model has the invariant that all CM_Interleave widened groups will eventually be converted to a VPInterleaveGroup, so it marks the addresses as uniform.
2) After #190191 a CM_Interleave isn't guaranteed to be converted to a VPInterleaveGroup if it has dead members, and it breaks this assumption in 1). So addresses aren't properly widened due to the uniform assumption.
Rather than try to fix up the address, this just maintains the invariant that a CM_Interleave will become a VPInterleaveGroup but not skipping over groups with dead members.
While there's slightly less code involved, the main benefit of this approach vs the others is that it keeps generating interleaved groups instead of falling back to gathers/scatters, so we don't get a codegen regression against 22.x.
Co-authored-by: Mel Chen <mel.chen at sifive.com>
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 58 +++++++++++--------
...e-to-widen-memory-with-wide-ops-chained.ll | 18 +++---
.../LoopVectorize/RISCV/dead-ops-cost.ll | 11 ++--
.../LoopVectorize/consecutive-ptr-uniforms.ll | 15 ++---
.../LoopVectorize/versioning-dead-load.ll | 18 +++---
5 files changed, 65 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1bae8b6402932..979a55b07e4ee 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3815,27 +3815,29 @@ void VPlanTransforms::createInterleaveGroups(
// single VPInterleaveRecipe at its insertion point.
VPDominatorTree VPDT(Plan);
for (const auto *IG : InterleaveGroups) {
- // Skip interleave groups where members don't have recipes. This can happen
- // when removeDeadRecipes removes recipes that are part of interleave groups
- // but have no users.
- if (llvm::any_of(IG->members(), [&IRMemberToRecipe](Instruction *Member) {
- return !IRMemberToRecipe.contains(Member);
- }))
+ VPWidenMemoryRecipe *Start = nullptr;
+ Instruction *StartMember = nullptr;
+ for (auto *Member : IG->members())
+ if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Member)) {
+ StartMember = Member;
+ Start = R;
+ break;
+ }
+ if (!StartMember) // All member recipes are dead, so the group is dead.
continue;
-
- auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));
VPIRMetadata InterleaveMD(*Start);
SmallVector<VPValue *, 4> StoredValues;
- if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Start->getAsRecipe()))
- StoredValues.push_back(StoreR->getStoredValue());
- for (unsigned I = 1; I < IG->getFactor(); ++I) {
+ for (unsigned I = 0; I < IG->getFactor(); ++I) {
Instruction *MemberI = IG->getMember(I);
if (!MemberI)
continue;
- VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(MemberI);
- if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(MemoryR->getAsRecipe()))
- StoredValues.push_back(StoreR->getStoredValue());
- InterleaveMD.intersect(*MemoryR);
+ if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(MemberI)) {
+ if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(MemoryR->getAsRecipe()))
+ StoredValues.push_back(StoreR->getStoredValue());
+ InterleaveMD.intersect(*MemoryR);
+ } else {
+ InterleaveMD.intersect(VPIRMetadata(*MemberI));
+ }
}
bool NeedsMaskForGaps =
@@ -3844,6 +3846,10 @@ void VPlanTransforms::createInterleaveGroups(
Instruction *IRInsertPos = IG->getInsertPos();
auto *InsertPos = IRMemberToRecipe.lookup(IRInsertPos);
+ if (!InsertPos) {
+ InsertPos = Start;
+ IRInsertPos = StartMember;
+ }
VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
GEPNoWrapFlags NW = GEPNoWrapFlags::none();
@@ -3854,11 +3860,12 @@ void VPlanTransforms::createInterleaveGroups(
// Get or create the start address for the interleave group.
VPValue *Addr = Start->getAddr();
VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
- if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPosR)) {
- // We cannot re-use the address of member zero because it does not
- // dominate the insert position. Instead, use the address of the insert
- // position and create a PtrAdd adjusting it to the address of member
- // zero.
+ if (IG->getIndex(StartMember) != 0 ||
+ (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPosR))) {
+ // Either member zero's recipe is dead, or we cannot re-use the address of
+ // member zero because it does not dominate the insert position. Instead,
+ // use the address of the insert position and create a PtrAdd adjusting it
+ // to the address of member zero.
// TODO: Hoist Addr's defining recipe (and any operands as needed) to
// InsertPos or sink loads above zero members to join it.
assert(IG->getIndex(IRInsertPos) != 0 &&
@@ -3891,13 +3898,16 @@ void VPlanTransforms::createInterleaveGroups(
unsigned J = 0;
for (unsigned i = 0; i < IG->getFactor(); ++i)
if (Instruction *Member = IG->getMember(i)) {
- VPRecipeBase *MemberR = IRMemberToRecipe.lookup(Member)->getAsRecipe();
+ VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Member);
if (!Member->getType()->isVoidTy()) {
- VPValue *OriginalV = MemberR->getVPSingleValue();
- OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
+ if (MemberR) {
+ VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
+ OriginalV->replaceAllUsesWith(VPIG->getVPValue(J));
+ }
J++;
}
- MemberR->eraseFromParent();
+ if (MemberR)
+ MemberR->getAsRecipe()->eraseFromParent();
}
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index f0f638f0dbbdd..469b801961cd9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -299,10 +299,11 @@ define void @test_2xi64_mul_sub_mismatched_ops2(ptr noalias %data, ptr noalias %
; VF2-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP9:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP9]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP10]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
-; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
+; VF2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 -8
+; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP11]], align 8
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], splat (i64 2)
; VF2-NEXT: [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
; VF2-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
@@ -599,10 +600,11 @@ define void @test_2xi64_mul_add_xor_mismatched_ops(ptr noalias %data, ptr noalia
; VF2-NEXT: [[TMP5:%.*]] = xor <2 x i64> splat (i64 4), [[TMP4]]
; VF2-NEXT: [[TMP11:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP12]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
-; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
-; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
+; VF2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 -8
+; VF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
+; VF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 0, i32 2>
+; VF2-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <4 x i64> [[WIDE_VEC]], <4 x i64> poison, <2 x i32> <i32 1, i32 3>
+; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC2]]
; VF2-NEXT: [[TMP7:%.*]] = add <2 x i64> [[TMP6]], splat (i64 2)
; VF2-NEXT: [[TMP8:%.*]] = xor <2 x i64> splat (i64 4), [[TMP7]]
; VF2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index a9db4458a34ff..5467148047200 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -363,10 +363,13 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[EVL_BASED_IV]], 1
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP11]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP22]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
-; CHECK-NEXT: [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP17]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP22]], i32 -1
+; CHECK-NEXT: [[INTERLEAVE_EVL:%.*]] = mul nuw nsw i32 [[TMP10]], 2
+; CHECK-NEXT: [[WIDE_VP_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[INTERLEAVE_EVL]])
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave2.nxv8i8(<vscale x 8 x i8> [[WIDE_VP_LOAD]])
+; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP12]] to <vscale x 4 x i32>
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index 50f5d417be7ad..1c736e9e8611e 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -1058,14 +1058,9 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; INTER-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; INTER-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 4
-; INTER-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP1]], i32 4
-; INTER-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP2]], i32 4
-; INTER-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP3]], i32 4
-; INTER-NEXT: [[TMP31:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP7]], i32 0
-; INTER-NEXT: [[TMP32:%.*]] = insertelement <4 x ptr> [[TMP31]], ptr [[TMP28]], i32 1
-; INTER-NEXT: [[TMP33:%.*]] = insertelement <4 x ptr> [[TMP32]], ptr [[TMP29]], i32 2
-; INTER-NEXT: [[TMP34:%.*]] = insertelement <4 x ptr> [[TMP33]], ptr [[TMP30]], i32 3
-; INTER-NEXT: [[STRIDED_VEC6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[TMP34]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; INTER-NEXT: [[WIDE_VEC5:%.*]] = load <16 x i32>, ptr [[TMP7]], align 8
+; INTER-NEXT: [[STRIDED_VEC6:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; INTER-NEXT: [[STRIDED_VEC7:%.*]] = shufflevector <16 x i32> [[WIDE_VEC5]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; INTER-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP18:%.*]] = sub <4 x i32> [[STRIDED_VEC]], [[STRIDED_VEC4]]
; INTER-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[NEXT_GEP]], i32 2
@@ -1099,11 +1094,11 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: br label %[[SCALAR_PH]]
; INTER: [[SCALAR_PH]]:
; INTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; INTER-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
+; INTER-NEXT: [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
; INTER-NEXT: br label %[[FOR_BODY:.*]]
; INTER: [[FOR_BODY]]:
; INTER-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ]
+; INTER-NEXT: [[P:%.*]] = phi ptr [ [[UNNAMEDTMP3:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL8]], %[[SCALAR_PH]] ]
; INTER-NEXT: [[UNNAMEDTMP00:%.*]] = load i32, ptr [[P]], align 8
; INTER-NEXT: [[UNNAMEDTMP03:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 1
; INTER-NEXT: [[UNNAMEDTMP04:%.*]] = load i32, ptr [[UNNAMEDTMP03]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll b/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
index ce8a20871922e..4013ee52bb5e8 100644
--- a/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/versioning-dead-load.ll
@@ -17,9 +17,9 @@ define void @f(ptr noalias %p, ptr noalias %q, ptr noalias %dst, i64 %stride) {
; VF4UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; VF4UF1-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF4UF1-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP0]]
-; VF4UF1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP1]], i64 0
-; VF4UF1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x ptr> [[BROADCAST_SPLATINSERT]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; VF4UF1-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; VF4UF1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP1]], align 4
+; VF4UF1-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; VF4UF1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; VF4UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF4UF1-NEXT: store <4 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP2]], align 4
; VF4UF1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[DST]], i64 [[INDEX]]
@@ -133,13 +133,13 @@ define void @f(ptr noalias %p, ptr noalias %q, ptr noalias %dst, i64 %stride) {
; VF2UF2-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[INDEX]], 3
; VF2UF2-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP0]], 3
; VF2UF2-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP1]]
-; VF2UF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP3]], i64 0
-; VF2UF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
; VF2UF2-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 [[TMP2]]
-; VF2UF2-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP4]], i64 0
-; VF2UF2-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT1]], <2 x ptr> poison, <2 x i32> zeroinitializer
-; VF2UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i32> poison)
-; VF2UF2-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[BROADCAST_SPLAT2]], <2 x i1> splat (i1 true), <2 x i32> poison)
+; VF2UF2-NEXT: [[WIDE_VEC:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; VF2UF2-NEXT: [[WIDE_MASKED_GATHER:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; VF2UF2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <4 x i32> [[WIDE_VEC]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+; VF2UF2-NEXT: [[WIDE_VEC2:%.*]] = load <4 x i32>, ptr [[TMP4]], align 4
+; VF2UF2-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+; VF2UF2-NEXT: [[STRIDED_VEC4:%.*]] = shufflevector <4 x i32> [[WIDE_VEC2]], <4 x i32> poison, <2 x i32> <i32 1, i32 3>
; VF2UF2-NEXT: [[TMP5:%.*]] = getelementptr inbounds [4 x i8], ptr [[Q]], i64 [[INDEX]]
; VF2UF2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 2
; VF2UF2-NEXT: store <2 x i32> [[WIDE_MASKED_GATHER]], ptr [[TMP5]], align 4
>From 0c8ee06b7f4b911f2989f0ab69f5230443a5aa0b Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 10 Jul 2026 01:55:20 +0800
Subject: [PATCH 2/5] Use an insert position that dominates all other members
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 ++-
.../interleaved-accesses-dead-member.ll | 68 +++++++++++++++++++
2 files changed, 76 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 979a55b07e4ee..0fbe718f2e5e2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3847,8 +3847,15 @@ void VPlanTransforms::createInterleaveGroups(
Instruction *IRInsertPos = IG->getInsertPos();
auto *InsertPos = IRMemberToRecipe.lookup(IRInsertPos);
if (!InsertPos) {
+ // InsertPos member is dead: find a new member that is alive.
+ assert(isa<VPWidenLoadRecipe>(Start) && "Dead member in non-load group?");
InsertPos = Start;
- IRInsertPos = StartMember;
+ for (Instruction *Member : IG->members())
+ if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Member))
+ if (VPDT.properlyDominates(MemberR->getAsRecipe(),
+ InsertPos->getAsRecipe()))
+ InsertPos = MemberR;
+ IRInsertPos = &InsertPos->getIngredient();
}
VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
new file mode 100644
index 0000000000000..eb7735ba11da8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt < %s -S -p loop-vectorize -force-vector-width=4 -enable-interleaved-mem-accesses=true | FileCheck %s
+
+; If the member holding the insert position is dead, make sure we choose a new member to insert at that dominates all other members.
+define void @dead_member_order(ptr noalias %src, ptr noalias %dst1, ptr noalias %dst2, i32 %N) {
+; CHECK-LABEL: define void @dead_member_order(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST1:%.*]], ptr noalias [[DST2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [3 x i8], ptr [[SRC]], i32 [[INDEX]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -1
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <12 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = shufflevector <12 x i8> [[WIDE_VEC]], <12 x i8> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST1]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[STRIDED_VEC1]], ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[DST2]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[TMP5]], ptr [[TMP6]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+
+ %gep.x = getelementptr [3 x i8], ptr %src, i32 %iv, i32 2
+ %x = load i8, ptr %gep.x ; dead
+
+ %gep.y = getelementptr [3 x i8], ptr %src, i32 %iv, i32 1
+ %y = load i8, ptr %gep.y
+
+ %gep.dst1 = getelementptr i8, ptr %dst1, i32 %iv
+ store i8 %y, ptr %gep.dst1
+
+ %gep.z = getelementptr [3 x i8], ptr %src, i32 %iv, i32 0
+ %z = load i8, ptr %gep.z
+
+ %res = add i8 %y, %z
+ %gep.dst2 = getelementptr i8, ptr %dst2, i32 %iv
+ store i8 %res, ptr %gep.dst2
+
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From f986ae560540eee8e38da8cac451dba5474201b8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 10 Jul 2026 10:48:22 +0800
Subject: [PATCH 3/5] Fix assert build error
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 0fbe718f2e5e2..98ea6e0553819 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3848,7 +3848,8 @@ void VPlanTransforms::createInterleaveGroups(
auto *InsertPos = IRMemberToRecipe.lookup(IRInsertPos);
if (!InsertPos) {
// InsertPos member is dead: find a new member that is alive.
- assert(isa<VPWidenLoadRecipe>(Start) && "Dead member in non-load group?");
+ assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
+ "Dead member in non-load group?");
InsertPos = Start;
for (Instruction *Member : IG->members())
if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Member))
>From 52ac228b834d6c9d4698bfd05935f2b9b029a35d Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 13 Jul 2026 15:39:15 +0800
Subject: [PATCH 4/5] Precommit metadata test
---
.../interleaved-accesses-dead-member.ll | 54 +++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
index eb7735ba11da8..61766804f4f25 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
@@ -66,3 +66,57 @@ loop:
exit:
ret void
}
+
+; Make sure we intersect the fact that the dead load doesn't have !invariant.load
+define void @dead_member_metadata(ptr noalias %src, ptr noalias %dst, i32 %N) {
+; CHECK-LABEL: define void @dead_member_metadata(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK: [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt i32 [[TMP0]], 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [2 x i8], ptr [[SRC]], i32 [[INDEX]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -1
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1, !invariant.load [[META4:![0-9]+]]
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i32 [[INDEX]]
+; CHECK-NEXT: store <4 x i8> [[STRIDED_VEC1]], ptr [[TMP3]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+
+ %gep.y = getelementptr [2 x i8], ptr %src, i32 %iv, i32 1
+ %y = load i8, ptr %gep.y, !invariant.load !{}
+ %gep.dst = getelementptr i8, ptr %dst, i32 %iv
+ store i8 %y, ptr %gep.dst
+
+ %gep.x = getelementptr [2 x i8], ptr %src, i32 %iv, i32 0
+ %x = load i8, ptr %gep.x ; dead
+
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %N
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
>From a48c1736f9867fcf62f948ede66523aabc490681 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 13 Jul 2026 15:40:22 +0800
Subject: [PATCH 5/5] Add changes w/ dead member metadata intersection
---
.../LoopVectorize/interleaved-accesses-dead-member.ll | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
index 61766804f4f25..df8780be88eec 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-dead-member.ll
@@ -86,14 +86,14 @@ define void @dead_member_metadata(ptr noalias %src, ptr noalias %dst, i32 %N) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr [2 x i8], ptr [[SRC]], i32 [[INDEX]], i32 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -1
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1, !invariant.load [[META4:![0-9]+]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP5]], align 1
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i32 [[INDEX]]
; CHECK-NEXT: store <4 x i8> [[STRIDED_VEC1]], ptr [[TMP3]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
More information about the llvm-commits
mailing list