[llvm] [LV] Fix incorrect address for interleave loads with dead members (PR #203555)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 07:55:58 PDT 2026
https://github.com/Mel-Chen created https://github.com/llvm/llvm-project/pull/203555
Some members in an interleave load group might be deleted from the plan early if they have no users, which prevents createInterleaveGroup from converting them into an Interleave recipe.
Meanwhile, the uniform analysis legacy cost model treats all interleave group member addresses as uniform. As a result, addresses for loads that still have users are generated incorrectly.
This patch addresses the issue by directly recomputing the address back to a VPWidenGEPRecipe.
Fixed #203188
>From ee460e0fdddb96c8cbe4b287cb3375de64b7a60a Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 12 Jun 2026 07:28:06 -0700
Subject: [PATCH 1/2] test
---
.../LoopVectorize/RISCV/pointer-induction.ll | 142 ++++++++++++++++++
1 file changed, 142 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 5047f0f669efa..80e6c7362708b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -S -tail-folding-policy=dont-fold-tail %s | FileCheck %s --check-prefix=NO-FOLDING
target triple = "riscv64-unknown-linux-gnu"
@@ -41,6 +42,69 @@ define void @ptr_induction(ptr %p, ptr noalias %q, ptr noalias %p.end) #0 {
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
+; NO-FOLDING-LABEL: define void @ptr_induction(
+; NO-FOLDING-SAME: ptr [[P:%.*]], ptr noalias [[Q:%.*]], ptr noalias [[P_END:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-FOLDING-NEXT: [[ENTRY:.*]]:
+; NO-FOLDING-NEXT: [[P2:%.*]] = ptrtoint ptr [[P]] to i64
+; NO-FOLDING-NEXT: [[P_END1:%.*]] = ptrtoint ptr [[P_END]] to i64
+; NO-FOLDING-NEXT: [[TMP0:%.*]] = add i64 [[P_END1]], 1
+; NO-FOLDING-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[P2]]
+; NO-FOLDING-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; NO-FOLDING-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP3]], i64 4)
+; NO-FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[UMAX]]
+; NO-FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NO-FOLDING: [[VECTOR_PH]]:
+; NO-FOLDING-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
+; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; NO-FOLDING-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[N_VEC]]
+; NO-FOLDING-NEXT: [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
+; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
+; NO-FOLDING: [[VECTOR_BODY]]:
+; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP7]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP7]]
+; NO-FOLDING-NEXT: [[TMP8:%.*]] = ptrtoint <vscale x 2 x ptr> [[VECTOR_GEP]] to <vscale x 2 x i64>
+; NO-FOLDING-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 2
+; NO-FOLDING-NEXT: [[TMP11:%.*]] = sub i32 [[TMP10]], 1
+; NO-FOLDING-NEXT: [[TMP12:%.*]] = extractelement <vscale x 2 x i64> [[TMP8]], i32 [[TMP11]]
+; NO-FOLDING-NEXT: store i64 [[TMP12]], ptr [[Q]], align 8
+; NO-FOLDING-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP14:%.*]] = mul nuw i32 [[TMP13]], 2
+; NO-FOLDING-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1
+; NO-FOLDING-NEXT: [[TMP16:%.*]] = extractelement <vscale x 2 x i64> [[VEC_IND]], i32 [[TMP15]]
+; NO-FOLDING-NEXT: store i64 [[TMP16]], ptr [[P]], align 8
+; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-FOLDING-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-FOLDING-NEXT: [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
+; NO-FOLDING-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; NO-FOLDING: [[MIDDLE_BLOCK]]:
+; NO-FOLDING-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; NO-FOLDING: [[SCALAR_PH]]:
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[P]], %[[ENTRY]] ]
+; NO-FOLDING-NEXT: br label %[[LOOP:.*]]
+; NO-FOLDING: [[LOOP]]:
+; NO-FOLDING-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[PTR_IND:%.*]] = phi ptr [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[PTR_IND_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[PTRI64:%.*]] = ptrtoint ptr [[PTR_IND]] to i64
+; NO-FOLDING-NEXT: store i64 [[PTRI64]], ptr [[Q]], align 8
+; NO-FOLDING-NEXT: store i64 [[IV]], ptr [[P]], align 8
+; NO-FOLDING-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; NO-FOLDING-NEXT: [[PTR_IND_NEXT]] = getelementptr i8, ptr [[PTR_IND]], i64 1
+; NO-FOLDING-NEXT: [[EC:%.*]] = icmp eq ptr [[PTR_IND]], [[P_END]]
+; NO-FOLDING-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; NO-FOLDING: [[EXIT]]:
+; NO-FOLDING-NEXT: ret void
+;
entry:
br label %loop
@@ -137,6 +201,84 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i1 [[CMP_LCSSA]]
;
+; NO-FOLDING-LABEL: define i1 @scalarize_ptr_induction(
+; NO-FOLDING-SAME: ptr [[START:%.*]], ptr [[END:%.*]], ptr noalias [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] {
+; NO-FOLDING-NEXT: [[ENTRY:.*]]:
+; NO-FOLDING-NEXT: [[START5:%.*]] = ptrtoint ptr [[START]] to i64
+; NO-FOLDING-NEXT: [[END4:%.*]] = ptrtoint ptr [[END]] to i64
+; NO-FOLDING-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; NO-FOLDING-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; NO-FOLDING-NEXT: [[TMP0:%.*]] = add i64 [[END4]], -12
+; NO-FOLDING-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]]
+; NO-FOLDING-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12
+; NO-FOLDING-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; NO-FOLDING-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; NO-FOLDING-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP5]], i64 24)
+; NO-FOLDING-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], [[UMAX]]
+; NO-FOLDING-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; NO-FOLDING: [[VECTOR_MEMCHECK]]:
+; NO-FOLDING-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8
+; NO-FOLDING-NEXT: [[TMP6:%.*]] = add i64 [[END1]], -12
+; NO-FOLDING-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[START2]]
+; NO-FOLDING-NEXT: [[TMP8:%.*]] = udiv i64 [[TMP7]], 12
+; NO-FOLDING-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 12
+; NO-FOLDING-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 8
+; NO-FOLDING-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]]
+; NO-FOLDING-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]]
+; NO-FOLDING-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[START]], [[SCEVGEP]]
+; NO-FOLDING-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; NO-FOLDING-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; NO-FOLDING: [[VECTOR_PH]]:
+; NO-FOLDING-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; NO-FOLDING-NEXT: [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
+; NO-FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP12]]
+; NO-FOLDING-NEXT: [[TMP13:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; NO-FOLDING-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[N_MOD_VF]]
+; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]]
+; NO-FOLDING-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12
+; NO-FOLDING-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
+; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
+; NO-FOLDING: [[VECTOR_BODY]]:
+; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-FOLDING-NEXT: [[TMP17:%.*]] = mul i64 [[INDEX]], 12
+; NO-FOLDING-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
+; NO-FOLDING-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
+; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP18]], i64 0
+; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
+; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; NO-FOLDING-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x i64>
+; NO-FOLDING-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
+; NO-FOLDING-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
+; NO-FOLDING-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
+; NO-FOLDING-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 2
+; NO-FOLDING-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1
+; NO-FOLDING-NEXT: [[TMP25:%.*]] = extractelement <vscale x 2 x i64> [[TMP21]], i32 [[TMP24]]
+; NO-FOLDING-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
+; NO-FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-FOLDING-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-FOLDING-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; NO-FOLDING: [[MIDDLE_BLOCK]]:
+; NO-FOLDING-NEXT: br label %[[SCALAR_PH]]
+; NO-FOLDING: [[SCALAR_PH]]:
+; NO-FOLDING-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], [ [[START]], %[[VECTOR_MEMCHECK]] ]
+; NO-FOLDING-NEXT: br label %[[LOOP:.*]]
+; NO-FOLDING: [[LOOP]]:
+; NO-FOLDING-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4
+; NO-FOLDING-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; NO-FOLDING-NEXT: [[EXT:%.*]] = zext i32 [[L]] to i64
+; NO-FOLDING-NEXT: [[UNUSED:%.*]] = load i32, ptr [[PTR_IV]], align 4
+; NO-FOLDING-NEXT: [[MUL1:%.*]] = mul i64 [[EXT]], -7070675565921424023
+; NO-FOLDING-NEXT: [[MUL2:%.*]] = add i64 [[MUL1]], -4
+; NO-FOLDING-NEXT: store i64 [[MUL2]], ptr [[DST]], align 1
+; NO-FOLDING-NEXT: [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 12
+; NO-FOLDING-NEXT: [[CMP:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; NO-FOLDING-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
+; NO-FOLDING: [[EXIT]]:
+; NO-FOLDING-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ]
+; NO-FOLDING-NEXT: ret i1 [[CMP_LCSSA]]
+;
entry:
br label %loop
>From 42e65e0901edbe7e26fb7b07f49f81aff76253b1 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Fri, 12 Jun 2026 07:40:57 -0700
Subject: [PATCH 2/2] repair address
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 28 ++++++++++++++++++-
...e-to-widen-memory-with-wide-ops-chained.ll | 6 ++--
.../LoopVectorize/RISCV/dead-ops-cost.ll | 4 +--
.../LoopVectorize/RISCV/pointer-induction.ll | 7 +++--
.../LoopVectorize/consecutive-ptr-uniforms.ll | 3 +-
5 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cd0fddd4489d8..ea5041c55b8f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3659,8 +3659,34 @@ void VPlanTransforms::createInterleaveGroups(
// but have no users.
if (llvm::any_of(IG->members(), [&IRMemberToRecipe](Instruction *Member) {
return !IRMemberToRecipe.contains(Member);
- }))
+ })) {
+ auto &DL = Plan.getDataLayout();
+ for (unsigned I = 0; I < IG->getFactor(); ++I)
+ if (Instruction *Member = IG->getMember(I)) {
+ // Fix addresses of members that still have recipes.
+ if (auto *MemberR = IRMemberToRecipe.lookup(Member)) {
+ auto *LoadR = cast<VPWidenLoadRecipe>(MemberR->getAsRecipe());
+ VPBuilder Builder(LoadR);
+ VPValue *Base = MemberR->getAddr();
+ Type *IndexTy = DL.getIndexType(Base->getScalarType());
+ VPValue *StepVec =
+ Builder.createNaryOp(VPInstruction::StepVector, {}, IndexTy);
+ int64_t Stride = IG->getFactor();
+ if (IG->isReverse())
+ Stride = -Stride;
+ VPValue *StrideVPV = Plan.getConstantInt(IndexTy, (uint64_t)Stride,
+ /*IsSigned=*/true);
+ VPValue *Idx = Builder.createOverflowingOp(Instruction::Mul,
+ {StepVec, StrideVPV});
+ auto *WidenGEP =
+ new VPWidenGEPRecipe(getLoadStoreType(Member), {Base, Idx}, {},
+ LoadR->getDebugLoc());
+ Builder.insert(WidenGEP);
+ LoadR->setOperand(0, WidenGEP);
+ }
+ }
continue;
+ }
auto *Start = IRMemberToRecipe.lookup(IG->getMember(0));
VPIRMetadata InterleaveMD(*Start);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
index 61891ff31ffae..223f759fb6ab1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-with-wide-ops-chained.ll
@@ -303,8 +303,7 @@ define void @test_2xi64_mul_sub_mismatched_ops2(ptr noalias %data, ptr noalias %
; VF2-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP3]], splat (i64 2)
; VF2-NEXT: [[TMP9:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP9]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP10]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = getelementptr i64, ptr [[TMP10]], <2 x i64> <i64 0, i64 2>
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
; VF2-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP6:%.*]] = sub <2 x i64> [[TMP5]], splat (i64 2)
@@ -603,8 +602,7 @@ define void @test_2xi64_mul_add_xor_mismatched_ops(ptr noalias %data, ptr noalia
; VF2-NEXT: [[TMP5:%.*]] = xor <2 x i64> splat (i64 4), [[TMP4]]
; VF2-NEXT: [[TMP11:%.*]] = or disjoint i64 [[TMP1]], 1
; VF2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]]
-; VF2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP12]], i64 0
-; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; VF2-NEXT: [[BROADCAST_SPLAT:%.*]] = getelementptr i64, ptr [[TMP12]], <2 x i64> <i64 0, i64 2>
; VF2-NEXT: [[STRIDED_VEC1:%.*]] = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 8 [[BROADCAST_SPLAT]], <2 x i1> splat (i1 true), <2 x i64> poison)
; VF2-NEXT: [[TMP6:%.*]] = mul <2 x i64> [[WIDE_LOAD]], [[STRIDED_VEC1]]
; VF2-NEXT: [[TMP7:%.*]] = add <2 x i64> [[TMP6]], splat (i64 2)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index a9db4458a34ff..0c2291564eb80 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -349,6 +349,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT: [[TMP5:%.*]] = shl <vscale x 4 x i64> [[TMP9]], splat (i64 1)
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <vscale x 4 x i64> [[TMP9]], splat (i64 2)
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
@@ -363,8 +364,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[EVL_BASED_IV]], 1
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], 1
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP11]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP22]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = getelementptr i8, ptr [[TMP22]], <vscale x 4 x i64> [[TMP5]]
; CHECK-NEXT: [[TMP17:%.*]] = call <vscale x 4 x i8> @llvm.vp.gather.nxv4i8.nxv4p0(<vscale x 4 x ptr> align 1 [[BROADCAST_SPLAT2]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
; CHECK-NEXT: [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP17]] to <vscale x 4 x i32>
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
index 80e6c7362708b..18477cc21bfa1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll
@@ -238,15 +238,16 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c
; NO-FOLDING-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]]
; NO-FOLDING-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12
; NO-FOLDING-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]]
+; NO-FOLDING-NEXT: [[TMP27:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-FOLDING-NEXT: [[TMP28:%.*]] = mul <vscale x 2 x i64> [[TMP27]], splat (i64 3)
; NO-FOLDING-NEXT: br label %[[VECTOR_BODY:.*]]
; NO-FOLDING: [[VECTOR_BODY]]:
; NO-FOLDING-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NO-FOLDING-NEXT: [[TMP17:%.*]] = mul i64 [[INDEX]], 12
; NO-FOLDING-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP17]]
; NO-FOLDING-NEXT: [[TMP18:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 4
-; NO-FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[TMP18]], i64 0
-; NO-FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
+; NO-FOLDING-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, ptr [[TMP18]], <vscale x 2 x i64> [[TMP28]]
+; NO-FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[WIDE_GEP]], <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META4:![0-9]+]]
; NO-FOLDING-NEXT: [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x i64>
; NO-FOLDING-NEXT: [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP19]], splat (i64 -7070675565921424023)
; NO-FOLDING-NEXT: [[TMP21:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 -4)
diff --git a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
index 6c6bb1f9a1cda..422a10ba6c4d2 100644
--- a/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
@@ -1065,7 +1065,8 @@ define void @pointer_iv_non_uniform_0(ptr %a, i64 %n) {
; INTER-NEXT: [[TMP32:%.*]] = insertelement <4 x ptr> [[TMP31]], ptr [[TMP28]], i32 1
; INTER-NEXT: [[TMP33:%.*]] = insertelement <4 x ptr> [[TMP32]], ptr [[TMP29]], i32 2
; INTER-NEXT: [[TMP34:%.*]] = insertelement <4 x ptr> [[TMP33]], ptr [[TMP30]], i32 3
-; INTER-NEXT: [[STRIDED_VEC6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[TMP34]], <4 x i1> splat (i1 true), <4 x i32> poison)
+; INTER-NEXT: [[WIDE_GEP:%.*]] = getelementptr i32, <4 x ptr> [[TMP34]], <4 x i64> <i64 0, i64 4, i64 8, i64 12>
+; INTER-NEXT: [[STRIDED_VEC6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 8 [[WIDE_GEP]], <4 x i1> splat (i1 true), <4 x i32> poison)
; INTER-NEXT: [[TMP17:%.*]] = sub <4 x i32> [[STRIDED_VEC6]], [[STRIDED_VEC]]
; INTER-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP17]], i64 0
; INTER-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP17]], i64 1
More information about the llvm-commits
mailing list