[llvm] [LV] Fix incorrect address for interleave loads with dead members (PR #203555)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 02:59:05 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/203555
>From ab46ed202c25dfb4d1844478e6e622683758f6f5 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 12 Jun 2026 07:28:06 -0700
Subject: [PATCH 1/4] test
---
.../LoopVectorize/RISCV/pointer-induction.ll | 142 ++++++++++++++++++
1 file changed, 142 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 5047f0f669efa..80e6c7362708b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -S -tail-folding-policy=dont-fold-tail %s | FileCheck %s --check-prefix=NO-FOLDING
target triple = "riscv64-unknown-linux-gnu"
@@ -41,6 +42,69 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; NO-FOLDING-LABEL: define void @ptr_induction(
+; NO-FOLDING-SAME: ptr [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[P_END:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-FOLDING-NEXT: [[ENTRY:.*]]:
+; NO-FOLDING-NEXT: [[P2:%.*]] = ptrtoint ptr [[P]] to i64
+; NO-FOLDING-NEXT: [[P_END1:%.*]] = ptrtoint ptr [[P_END]] to i64
+; NO-FOLDING-NEXT: [[TMP0:%.*]] = add i64 [[P_END1]], 1
+; NO-FOLDING-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[P2]]
+; NO-FOLDING-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-FOLDING-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 4)
+; NO-FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
+; NO-FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NO-FOLDING: [[VECTOR_PH]]:
+; NO-FOLDING-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
+; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; NO-FOLDING-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[N_VEC]]
+; NO-FOLDING-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
+; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
+; NO-FOLDING: [[VECTOR_BODY]]:
+; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP7]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP7]]
+; NO-FOLDING-NEXT: [[TMP8:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
+; NO-FOLDING-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 2
+; NO-FOLDING-NEXT: [[TMP11:%.*]] = sub i32 [[TMP10]], 1
+; NO-FOLDING-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i64> [[TMP8]], i32 [[TMP11]]
+; NO-FOLDING-NEXT: store i64 [[TMP12]], ptr [[Q]], align 8
+; NO-FOLDING-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP14:%.*]] = mul nuw i32 [[TMP13]], 2
+; NO-FOLDING-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1
+; NO-FOLDING-NEXT: [[TMP16:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i32 [[TMP15]]
+; NO-FOLDING-NEXT: store i64 [[TMP16]], ptr [[P]], align 8
+; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-FOLDING-NEXT: [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
+; NO-FOLDING-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; NO-FOLDING: [[MIDDLE_BLOCK]]:
+; NO-FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; NO-FOLDING: [[SCALAR_PH]]:
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[P]], %[[ENTRY]] ]
+; NO-FOLDING-NEXT: br label %[[LOOP:.*]]
+; NO-FOLDING: [[LOOP]]:
+; NO-FOLDING-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[PTR_IND:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[PTR_IND_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[PTRI64:%.*]] = ptrtoint ptr [[PTR_IND]] to i64
+; NO-FOLDING-NEXT: store i64 [[PTRI64]], ptr [[Q]], align 8
+; NO-FOLDING-NEXT: store i64 [[IV]], ptr [[P]], align 8
+; NO-FOLDING-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; NO-FOLDING-NEXT: [[PTR_IND_NEXT]] = getelementptr i8, ptr [[PTR_IND]], i64 1
+; NO-FOLDING-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IND]], [[P_END]]
+; NO-FOLDING-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; NO-FOLDING: [[EXIT]]:
+; NO-FOLDING-NEXT: ret void
+;
entry:
br label %loop
@@ -137,6 +201,84 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i1 [[CMP_LCSSA]]
;
+; NO-FOLDING-LABEL: define i1 @scalarize_ptr_induction(
+; NO-FOLDING-SAME: ptr [[START:%.*]], ptr [[END:%.*]], ptr noalias [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] {
+; NO-FOLDING-NEXT: [[ENTRY:.*]]:
+; NO-FOLDING-NEXT: [[START5:%.*]] = ptrtoint ptr [[START]] to i64
+; NO-FOLDING-NEXT: [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; NO-FOLDING-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; NO-FOLDING-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; NO-FOLDING-NEXT: [[TMP0:%.*]] = add i64 [[END4]], -12
+; NO-FOLDING-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]]
+; NO-FOLDING-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12
+; NO-FOLDING-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; NO-FOLDING-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-FOLDING-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP5]], i64 24)
+; NO-FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], [[UMAX]]
+; NO-FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; NO-FOLDING: [[VECTOR_MEMCHECK]]:
+; NO-FOLDING-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
+; NO-FOLDING-NEXT: [[TMP6:%.*]] = add i64 [[END1]], -12
+; NO-FOLDING-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[START2]]
+; NO-FOLDING-NEXT: [[TMP8:%.*]] = udiv i64 [[TMP7]], 12
+; NO-FOLDING-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 12
+; NO-FOLDING-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 8
+; NO-FOLDING-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]]
+; NO-FOLDING-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]]
+; NO-FOLDING-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[START]], [[SCEVGEP]]
+; NO-FOLDING-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; NO-FOLDING-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; NO-FOLDING: [[VECTOR_PH]]:
+; NO-FOLDING-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
+; NO-FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP12]]
+; NO-FOLDING-NEXT: [[TMP13:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; NO-FOLDING-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[N_MOD_VF]]
+; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]]
+; NO-FOLDING-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12
+; NO-FOLDING-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
+; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
+; NO-FOLDING: [[VECTOR_BODY]]:
+; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[TMP17:%.*]] = mul i64 [[INDEX]], 12
+; NO-FOLDING-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
+; NO-FOLDING-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
+; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP18]], i64 0
+; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; NO-FOLDING-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x i64>
+; NO-FOLDING-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
+; NO-FOLDING-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
+; NO-FOLDING-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 2
+; NO-FOLDING-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; NO-FOLDING-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i32 [[TMP24]]
+; NO-FOLDING-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-FOLDING-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; NO-FOLDING: [[MIDDLE_BLOCK]]:
+; NO-FOLDING-NEXT: br label %[[SCALAR_PH]]
+; NO-FOLDING: [[SCALAR_PH]]:
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], [ [[START]], %[[VECTOR_MEMCHECK]] ]
+; NO-FOLDING-NEXT: br label %[[LOOP:.*]]
+; NO-FOLDING: [[LOOP]]:
+; NO-FOLDING-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4
+; NO-FOLDING-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-FOLDING-NEXT: [[EXT:%.*]] = zext i32 [[L]] to i64
+; NO-FOLDING-NEXT: [[UNUSED:%.*]] = load i32, ptr [[PTR_IV]], align 4
+; NO-FOLDING-NEXT: [[MUL1:%.*]] = mul i64 [[EXT]], -7070675565921424023
+; NO-FOLDING-NEXT: [[MUL2:%.*]] = add i64 [[MUL1]], -4
+; NO-FOLDING-NEXT: store i64 [[MUL2]], ptr [[DST]], align 1
+; NO-FOLDING-NEXT: [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 12
+; NO-FOLDING-NEXT: [[CMP:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; NO-FOLDING-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; NO-FOLDING: [[EXIT]]:
+; NO-FOLDING-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: ret i1 [[CMP_LCSSA]]
+;
entry:
br label %loop
>From 98b9cf46f71e3a4369e4675c9e732d131f6a9aa4 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 12 Jun 2026 07:40:57 -0700
Subject: [PATCH 2/4] repair address
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 28 ++++++++++++++++++-
...e-to-widen-memory-with-wide-ops-chained.ll | 6 ++--
.../LoopVectorize/RISCV/dead-ops-cost.ll | 4 +--
.../LoopVectorize/RISCV/pointer-induction.ll | 7 +++--
.../LoopVectorize/consecutive-ptr-uniforms.ll | 3 +-
5 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index adcfe30ff9561..dd79aa1c65f7f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3728,8 +3728,34 @@ void VPlanTransforms::createInterleaveGroups(
// but have no users.
if (llvm::any_of(IG->members(), [&IRMemberToRecipe](Instruction *Member) {
return !IRMemberToRecipe.contains(Member);
- }))
+ })) {
+ auto &DL = Plan.getDataLayout();
+ for (unsigned I = 0; I < IG->getFactor(); ++I)
+ if (Instruction *Member = IG->getMember(I)) {
+ // Fix addresses of members that still have recipes.
+ if (auto *MemberR = IRMemberToRecipe.lookup(Member)) {
+ auto *LoadR = cast<VPWidenLoadRecipe>(MemberR->getAsRecipe());
+ VPBuilder Builder(LoadR);
+ VPValue *Base = MemberR->getAddr();
+ Type *IndexTy = DL.getIndexType(Base->getScalarType());
+ VPValue *StepVec =
+ Builder.createNaryOp(VPInstruction::StepVector, {}, IndexTy);
+ int64_t Stride = IG->getFactor();
+ if (IG->isReverse())
+ Stride = -Stride;
+ VPValue *StrideVPV = Plan.getConstantInt(IndexTy, (uint64_t)Stride,
+ /*IsSigned=*/true);
+ VPValue *Idx = Builder.createOverflowingOp(Instruction::Mul,
+ {StepVec, StrideVPV});
+ auto *WidenGEP =
+ new VPWidenGEPRecipe(getLoadStoreType(Member), {Base, Idx}, {},
+ LoadR->getDebugLoc());
+ Builder.insert(WidenGEP);
+ LoadR->setOperand(0, WidenGEP);
+ }
+ }
continue;
+ }
auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));
VPIRMetadata InterleaveMD(*Start);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 11fd2748dca60..e2055c9e544a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -299,8 +299,7 @@ define void @test_2xi64_mul_sub_mismatched_ops2(ptr noalias %data, ptr noalias %
; VF2-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP9:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP9]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP10]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = getelementptr i64, ptr [[TMP10]], <2 x i64> <i64 0, i64 2>
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], splat (i64 2)
@@ -599,8 +598,7 @@ define void @test_2xi64_mul_add_xor_mismatched_ops(ptr noalias %data, ptr noalia
; VF2-NEXT: [[TMP5:%.*]] = xor <2 x i64> splat (i64 4), [[TMP4]]
; VF2-NEXT: [[TMP11:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP12]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = getelementptr i64, ptr [[TMP12]], <2 x i64> <i64 0, i64 2>
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP7:%.*]] = add <2 x i64> [[TMP6]], splat (i64 2)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index a9db4458a34ff..0c2291564eb80 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -349,6 +349,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl <vscale x 4 x i64> [[TMP9]], splat (i64 1)
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 2)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -363,8 +364,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[EVL_BASED_IV]], 1
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP11]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP22]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = getelementptr i8, ptr [[TMP22]], <vscale x 4 x i64> [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP17]] to <vscale x 4 x i32>
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 80e6c7362708b..18477cc21bfa1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -238,15 +238,16 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]]
; NO-FOLDING-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12
; NO-FOLDING-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
+; NO-FOLDING-NEXT: [[TMP27:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-FOLDING-NEXT: [[TMP28:%.*]] = mul <vscale x 2 x i64> [[TMP27]], splat (i64 3)
; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-FOLDING: [[VECTOR_BODY]]:
; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-FOLDING-NEXT: [[TMP17:%.*]] = mul i64 [[INDEX]], 12
; NO-FOLDING-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
; NO-FOLDING-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
-; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP18]], i64 0
-; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; NO-FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[TMP18]], <vscale x 2 x i64> [[TMP28]]
+; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[WIDE_GEP]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
; NO-FOLDING-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x i64>
; NO-FOLDING-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; NO-FOLDING-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index 6c6bb1f9a1cda..422a10ba6c4d2 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -1065,7 +1065,8 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: [[TMP32:%.*]] = insertelement <4 x ptr> [[TMP31]], ptr [[TMP28]], i32 1
; INTER-NEXT: [[TMP33:%.*]] = insertelement <4 x ptr> [[TMP32]], ptr [[TMP29]], i32 2
; INTER-NEXT: [[TMP34:%.*]] = insertelement <4 x ptr> [[TMP33]], ptr [[TMP30]], i32 3
-; INTER-NEXT: [[STRIDED_VEC6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[TMP34]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; INTER-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, <4 x ptr> [[TMP34]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
+; INTER-NEXT: [[STRIDED_VEC6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; INTER-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP17]], i64 0
; INTER-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP17]], i64 1
>From 5d8cd8cf0295b3a9938cca4940844171b6c993f7 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 23 Jun 2026 02:37:46 -0700
Subject: [PATCH 3/4] rebase and update
---
.../LoopVectorize/RISCV/pointer-induction.ll | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 18477cc21bfa1..46f501f146851 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -248,6 +248,10 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; NO-FOLDING-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
; NO-FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[TMP18]], <vscale x 2 x i64> [[TMP28]]
; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[WIDE_GEP]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-FOLDING-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; NO-FOLDING: [[MIDDLE_BLOCK]]:
; NO-FOLDING-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x i64>
; NO-FOLDING-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; NO-FOLDING-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
@@ -255,11 +259,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; NO-FOLDING-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 2
; NO-FOLDING-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1
; NO-FOLDING-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i32 [[TMP24]]
-; NO-FOLDING-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
-; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
-; NO-FOLDING-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NO-FOLDING-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; NO-FOLDING: [[MIDDLE_BLOCK]]:
+; NO-FOLDING-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META8:![0-9]+]], !noalias [[META4]]
; NO-FOLDING-NEXT: br label %[[SCALAR_PH]]
; NO-FOLDING: [[SCALAR_PH]]:
; NO-FOLDING-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], [ [[START]], %[[VECTOR_MEMCHECK]] ]
>From 75144434844fb7e145d74a5d938486d95239a2fb Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Tue, 23 Jun 2026 02:58:15 -0700
Subject: [PATCH 4/4] reduce nested if
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 50 ++++++++++---------
1 file changed, 26 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dd79aa1c65f7f..07b409404750b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3730,30 +3730,32 @@ void VPlanTransforms::createInterleaveGroups(
return !IRMemberToRecipe.contains(Member);
})) {
auto &DL = Plan.getDataLayout();
- for (unsigned I = 0; I < IG->getFactor(); ++I)
- if (Instruction *Member = IG->getMember(I)) {
- // Fix addresses of members that still have recipes.
- if (auto *MemberR = IRMemberToRecipe.lookup(Member)) {
- auto *LoadR = cast<VPWidenLoadRecipe>(MemberR->getAsRecipe());
- VPBuilder Builder(LoadR);
- VPValue *Base = MemberR->getAddr();
- Type *IndexTy = DL.getIndexType(Base->getScalarType());
- VPValue *StepVec =
- Builder.createNaryOp(VPInstruction::StepVector, {}, IndexTy);
- int64_t Stride = IG->getFactor();
- if (IG->isReverse())
- Stride = -Stride;
- VPValue *StrideVPV = Plan.getConstantInt(IndexTy, (uint64_t)Stride,
- /*IsSigned=*/true);
- VPValue *Idx = Builder.createOverflowingOp(Instruction::Mul,
- {StepVec, StrideVPV});
- auto *WidenGEP =
- new VPWidenGEPRecipe(getLoadStoreType(Member), {Base, Idx}, {},
- LoadR->getDebugLoc());
- Builder.insert(WidenGEP);
- LoadR->setOperand(0, WidenGEP);
- }
- }
+ for (unsigned I = 0; I < IG->getFactor(); ++I) {
+ Instruction *Member = IG->getMember(I);
+ if (!Member)
+ continue;
+ auto *MemberR = IRMemberToRecipe.lookup(Member);
+ if (!MemberR)
+ continue;
+ // Fix addresses of members that still have recipes.
+ auto *LoadR = cast<VPWidenLoadRecipe>(MemberR->getAsRecipe());
+ VPBuilder Builder(LoadR);
+ VPValue *Base = MemberR->getAddr();
+ Type *IndexTy = DL.getIndexType(Base->getScalarType());
+ VPValue *StepVec =
+ Builder.createNaryOp(VPInstruction::StepVector, {}, IndexTy);
+ int64_t Stride = IG->getFactor();
+ if (IG->isReverse())
+ Stride = -Stride;
+ VPValue *StrideVPV = Plan.getConstantInt(IndexTy, (uint64_t)Stride,
+ /*IsSigned=*/true);
+ VPValue *Idx =
+ Builder.createOverflowingOp(Instruction::Mul, {StepVec, StrideVPV});
+ auto *WidenGEP = new VPWidenGEPRecipe(
+ getLoadStoreType(Member), {Base, Idx}, {}, LoadR->getDebugLoc());
+ Builder.insert(WidenGEP);
+ LoadR->setOperand(0, WidenGEP);
+ }
continue;
}
More information about the llvm-commits
mailing list