[llvm] [VPlan] Introduce VPGEPInstruction (PR #193510)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 01:53:34 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/193510
>From e2a89fb8677927de5c258706362058df3df8a498 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 22 Apr 2026 13:26:53 +0100
Subject: [PATCH 1/3] [VPlan] Introduce scalar GEP VPInstruction
Currently, WidenGEP, Replicate, and VPInstruction recipes can all
hold a Instruction::GetElementPtr. Introduce a first-class "scalar GEP"
VPInstructionWithType, similar to the existing "scalar cast", with the
additional benefit of being able to query the source element type of the
GEP directly without going to the underlying value. Planned follow-ups
include unifying WidenGEP and Replicate GEPs with this VPInstruction
GEP.
---
.../Vectorize/LoopVectorizationPlanner.h | 8 ++++
llvm/lib/Transforms/Vectorize/VPlan.h | 27 +++++++++++--
.../Vectorize/VPlanConstruction.cpp | 5 +++
.../Transforms/Vectorize/VPlanPatternMatch.h | 1 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 30 ++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 40 ++++++++++---------
.../LoopVectorize/VPlan/predicator.ll | 14 +++----
.../LoopVectorize/VPlan/tail-folding.ll | 6 +--
.../VPlan/vplan-printing-outer-loop.ll | 4 +-
.../Transforms/Vectorize/VPlanHCFGTest.cpp | 4 +-
10 files changed, 103 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index a6789974e0bd6..4c4a558bf8158 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -393,6 +393,14 @@ class VPBuilder {
new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
}
+ VPInstruction *createScalarGEP(Type *SourceElementTy, ArrayRef<VPValue *> Ops,
+ Type *ResultTy, DebugLoc DL,
+ const VPIRFlags &Flags,
+ const VPIRMetadata &Metadata = {}) {
+ return tryInsertInstruction(new VPInstructionWithType(
+ SourceElementTy, Ops, ResultTy, Flags, Metadata, DL));
+ }
+
VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
DebugLoc DL) {
if (ResultTy == SrcTy)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 6a1ea6b3439bf..bb6c7202713df 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1516,6 +1516,10 @@ class VPInstructionWithType : public VPInstruction {
/// Scalar result type produced by the recipe.
Type *ResultTy;
+ /// The source element type, which is present when the recipe has a
+ /// getelementptr opcode.
+ Type *SourceElementTy = nullptr;
+
public:
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
Type *ResultTy, const VPIRFlags &Flags = {},
@@ -1525,6 +1529,16 @@ class VPInstructionWithType : public VPInstruction {
: VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name),
ResultTy(ResultTy) {}
+ /// Constructor for GEPs.
+ VPInstructionWithType(Type *SourceElementTy, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown(),
+ const Twine &Name = "")
+ : VPInstruction(Instruction::GetElementPtr, Operands, Flags, Metadata, DL,
+ Name),
+ ResultTy(ResultTy), SourceElementTy(SourceElementTy) {}
+
static inline bool classof(const VPRecipeBase *R) {
// VPInstructionWithType are VPInstructions with specific opcodes requiring
// type information.
@@ -1537,6 +1551,7 @@ class VPInstructionWithType : public VPInstruction {
case VPInstruction::WideIVStep:
case VPInstruction::StepVector:
case VPInstruction::VScale:
+ case Instruction::GetElementPtr:
case Instruction::Load:
return true;
default:
@@ -1552,6 +1567,8 @@ class VPInstructionWithType : public VPInstruction {
auto *New =
new VPInstructionWithType(getOpcode(), operands(), getResultType(),
*this, *this, getDebugLoc(), getName());
+ if (getOpcode() == Instruction::GetElementPtr)
+ New->SourceElementTy = getSourceElementType();
New->setUnderlyingValue(getUnderlyingValue());
return New;
}
@@ -1560,12 +1577,14 @@ class VPInstructionWithType : public VPInstruction {
/// Return the cost of this VPInstruction.
InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override {
- // TODO: Compute accurate cost after retiring the legacy cost model.
- return 0;
- }
+ VPCostContext &Ctx) const override;
Type *getResultType() const { return ResultTy; }
+ Type *getSourceElementType() const {
+ assert(getOpcode() == Instruction::GetElementPtr &&
+ "Source element type requested for non-getelementptr");
+ return SourceElementTy;
+ }
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index e20d5d947ac54..ed8e066f61301 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -267,6 +267,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
CI->getType(), CI->getDebugLoc(),
VPIRFlags(*CI), MD);
NewR->setUnderlyingValue(CI);
+ } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
+ NewR = VPIRBuilder.createScalarGEP(
+ GEP->getSourceElementType(), VPOperands, GEP->getType(),
+ GEP->getDebugLoc(), VPIRFlags(*GEP), MD);
+ NewR->setUnderlyingValue(GEP);
} else if (auto *LI = dyn_cast<LoadInst>(Inst)) {
NewR = VPIRBuilder.createScalarLoad(LI->getType(), VPOperands[0],
LI->getDebugLoc(), MD);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 3cafeae7c4aea..d523cf5e9f5d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -751,6 +751,7 @@ inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
/*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
Op0, Op1),
+ VPInstruction_match<Instruction::GetElementPtr, Op0_t, Op1_t>(Op0, Op1),
VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 11c18babd1b74..ab9577d9ed730 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1147,6 +1147,12 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,
{TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);
}
+ case Instruction::GetElementPtr:
+ // We mark this instruction as zero-cost because the cost of GEPs in
+ // vectorized code depends on whether the corresponding memory instruction
+ // is scalarized or not. Therefore, we handle GEPs with the memory
+ // instruction cost.
+ return 0;
}
llvm_unreachable("called for unsupported opcode");
}
@@ -1321,6 +1327,7 @@ bool VPInstruction::isVectorToScalar() const {
bool VPInstruction::isSingleScalar() const {
switch (getOpcode()) {
+ case Instruction::GetElementPtr:
case Instruction::Load:
case Instruction::PHI:
case VPInstruction::ExplicitVectorLength:
@@ -1619,6 +1626,16 @@ void VPInstructionWithType::execute(VPTransformState &State) {
return;
}
switch (getOpcode()) {
+ case Instruction::GetElementPtr: {
+ Value *Ptr = State.get(getOperand(0), true);
+ auto IdxList =
+ to_vector(map_range(drop_begin(operands()),
+ [&](VPValue *Op) { return State.get(Op, true); }));
+ Value *GEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, IdxList,
+ "", getGEPNoWrapFlags());
+ State.set(this, GEP, true);
+ return;
+ }
case VPInstruction::StepVector: {
Value *StepVector =
State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));
@@ -1636,6 +1653,13 @@ void VPInstructionWithType::execute(VPTransformState &State) {
}
}
+InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
+ VPCostContext &Ctx) const {
+ if (!getUnderlyingValue())
+ return 0;
+ return getCostForRecipeWithOpcode(getOpcode(), VF, Ctx);
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -1658,6 +1682,12 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
O << "load ";
printOperands(O, SlotTracker);
break;
+ case Instruction::GetElementPtr:
+ O << "getelementptr";
+ printFlags(O);
+ O << *SourceElementTy << " ";
+ printOperands(O, SlotTracker);
+ break;
default:
assert(Instruction::isCast(getOpcode()) && "unhandled opcode");
O << Instruction::getOpcodeName(getOpcode()) << " ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3dadd9253ad57..3ced715dd1061 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1277,6 +1277,27 @@ getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) {
.Default([](auto *) { return std::nullopt; });
}
+/// If recipe \p R will lower to a GEP with a non-i8 source element type,
+/// return that source element type.
+static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
+ // All VPInstructions that lower to GEPs must have the i8 source element
+ // type (as they are PtrAdds), so we omit it.
+ return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
+ .Case([](const VPReplicateRecipe *I) -> Type * {
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(I->getUnderlyingValue()))
+ return GEP->getSourceElementType();
+ return nullptr;
+ })
+ .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
+ [](auto *I) { return I->getSourceElementType(); })
+ .Case<VPInstructionWithType>([](auto *I) {
+ return I->getOpcode() == Instruction::GetElementPtr
+ ? I->getSourceElementType()
+ : nullptr;
+ })
+ .Default([](auto *) { return nullptr; });
+}
+
/// Try to fold \p R using InstSimplifyFolder. Will succeed and return a
/// non-nullptr VPValue for a handled opcode or intrinsic ID if corresponding \p
/// Operands are foldable live-ins.
@@ -1329,8 +1350,7 @@ static VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R,
Ops[1]);
case Instruction::GetElementPtr: {
auto &RFlags = cast<VPRecipeWithIRFlags>(R);
- auto *GEP = cast<GetElementPtrInst>(RFlags.getUnderlyingInstr());
- return Folder.FoldGEP(GEP->getSourceElementType(), Ops[0],
+ return Folder.FoldGEP(getGEPSourceElementType(&RFlags), Ops[0],
drop_begin(Ops), RFlags.getGEPNoWrapFlags());
}
case VPInstruction::PtrAdd:
@@ -2360,22 +2380,6 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
return Def == getEmptyKey() || Def == getTombstoneKey();
}
- /// If recipe \p R will lower to a GEP with a non-i8 source element type,
- /// return that source element type.
- static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
- // All VPInstructions that lower to GEPs must have the i8 source element
- // type (as they are PtrAdds), so we omit it.
- return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
- .Case([](const VPReplicateRecipe *I) -> Type * {
- if (auto *GEP = dyn_cast<GetElementPtrInst>(I->getUnderlyingValue()))
- return GEP->getSourceElementType();
- return nullptr;
- })
- .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
- [](auto *I) { return I->getSourceElementType(); })
- .Default([](auto *) { return nullptr; });
- }
-
/// Returns true if recipe \p Def can be safely handed for CSE.
static bool canHandle(const VPSingleDefRecipe *Def) {
// We can extend the list of handled recipes in the future,
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 8e05e27d185a6..eac9dbe09eff0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -8,7 +8,7 @@ define void @diamond_phi(ptr %a) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
@@ -71,7 +71,7 @@ define void @mask_reuse(ptr %a) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb1
; CHECK-EMPTY:
@@ -154,7 +154,7 @@ define void @optimized_mask(ptr %a) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb6
; CHECK-EMPTY:
@@ -277,7 +277,7 @@ define void @switch(ptr %a) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0:%[0-9]+]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT ir<%c0> = icmp sle ir<%iv>, ir<0>
; CHECK-NEXT: Successor(s): bb2
; CHECK-EMPTY:
@@ -409,7 +409,7 @@ define void @diamond_phi2(ptr %a, i1 %c1, i1 %c2) {
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = or vp<[[VP5]]>, vp<[[VP6]]>
; CHECK-NEXT: BLEND ir<%phi> = ir<%add2>/vp<[[VP5]]> ir<%add1>/vp<[[VP6]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%a>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i64 ir<%a>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP7]]>
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
@@ -508,7 +508,7 @@ define void @blend_masks(ptr noalias %p, i1 %c0, i1 %c1, i1 %c2, i1 %c3, i1 %c4)
; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = logical-and vp<[[VP9]]>, ir<%c4>
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = or vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP15]]> ir<0>/vp<[[VP14]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>, vp<[[VP16]]>
; CHECK-NEXT: Successor(s): bb8
; CHECK-EMPTY:
@@ -592,7 +592,7 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) {
; CHECK-NEXT: bb3:
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and ir<%c0>, ir<%c1>
; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP7]]> ir<0>/vp<[[VP8]]>
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index 062ee47a43444..120b750bf96f2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -27,7 +27,7 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
@@ -116,7 +116,7 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: Successor(s): if, latch
; CHECK-EMPTY:
; CHECK-NEXT: if:
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
@@ -278,7 +278,7 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
-; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep> = getelementptr i32 ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%rdx.next> = add ir<%rdx>, ir<%x>
; CHECK-NEXT: Successor(s): vector.latch
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
index 6eff6d1df3523..def2b2d90888a 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
@@ -15,14 +15,14 @@ define void @foo(i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: outer.header:
; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep.1> = getelementptr inbounds [8 x i64] ir<@arr2>, ir<0>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n>
; CHECK-NEXT: Successor(s): inner
; CHECK-EMPTY:
; CHECK-NEXT: inner:
; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%gep.2> = getelementptr inbounds [8 x [8 x i64]] ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1>
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8>
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index d07c72c41de84..4bbfec0f330f9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -140,7 +140,7 @@ compound=true
N4 [label =
"vector.body:\l" +
" EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" +
- " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%indvars.iv\>\l" +
+ " EMIT-SCALAR ir\<%arr.idx\> = getelementptr inbounds i32 ir\<%A\>, ir\<%indvars.iv\>\l" +
" EMIT-SCALAR ir\<%l1\> = load ir\<%arr.idx\>\l" +
" EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
" EMIT store ir\<%res\>, ir\<%arr.idx\>\l" +
@@ -307,7 +307,7 @@ compound=true
N4 [label =
"vector.body:\l" +
" EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" +
- " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%iv\>\l" +
+ " EMIT-SCALAR ir\<%arr.idx\> = getelementptr inbounds i32 ir\<%A\>, ir\<%iv\>\l" +
" EMIT-SCALAR ir\<%l1\> = load ir\<%arr.idx\>\l" +
" EMIT ir\<%c\> = icmp eq ir\<%l1\>, ir\<0\>\l" +
"Successor(s): loop.latch\l"
>From 454c7b639c6b16161281d8ef449b263328f3c83a Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Tue, 5 May 2026 11:53:21 +0100
Subject: [PATCH 2/3] [VPlan] Introduce VPGEPInstruction
---
.../Vectorize/LoopVectorizationPlanner.h | 2 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 73 +++++++++++++------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 52 ++++++-------
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +-
4 files changed, 75 insertions(+), 58 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4c4a558bf8158..bdf84d8923cf8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -397,7 +397,7 @@ class VPBuilder {
Type *ResultTy, DebugLoc DL,
const VPIRFlags &Flags,
const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(new VPInstructionWithType(
+ return tryInsertInstruction(new VPGEPInstruction(
SourceElementTy, Ops, ResultTy, Flags, Metadata, DL));
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index bb6c7202713df..a91eb46b8b6e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1516,10 +1516,6 @@ class VPInstructionWithType : public VPInstruction {
/// Scalar result type produced by the recipe.
Type *ResultTy;
- /// The source element type, which is present when the recipe has a
- /// getelementptr opcode.
- Type *SourceElementTy = nullptr;
-
public:
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
Type *ResultTy, const VPIRFlags &Flags = {},
@@ -1529,16 +1525,6 @@ class VPInstructionWithType : public VPInstruction {
: VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name),
ResultTy(ResultTy) {}
- /// Constructor for GEPs.
- VPInstructionWithType(Type *SourceElementTy, ArrayRef<VPValue *> Operands,
- Type *ResultTy, const VPIRFlags &Flags = {},
- const VPIRMetadata &Metadata = {},
- DebugLoc DL = DebugLoc::getUnknown(),
- const Twine &Name = "")
- : VPInstruction(Instruction::GetElementPtr, Operands, Flags, Metadata, DL,
- Name),
- ResultTy(ResultTy), SourceElementTy(SourceElementTy) {}
-
static inline bool classof(const VPRecipeBase *R) {
// VPInstructionWithType are VPInstructions with specific opcodes requiring
// type information.
@@ -1551,7 +1537,6 @@ class VPInstructionWithType : public VPInstruction {
case VPInstruction::WideIVStep:
case VPInstruction::StepVector:
case VPInstruction::VScale:
- case Instruction::GetElementPtr:
case Instruction::Load:
return true;
default:
@@ -1567,8 +1552,6 @@ class VPInstructionWithType : public VPInstruction {
auto *New =
new VPInstructionWithType(getOpcode(), operands(), getResultType(),
*this, *this, getDebugLoc(), getName());
- if (getOpcode() == Instruction::GetElementPtr)
- New->SourceElementTy = getSourceElementType();
New->setUnderlyingValue(getUnderlyingValue());
return New;
}
@@ -1577,14 +1560,12 @@ class VPInstructionWithType : public VPInstruction {
/// Return the cost of this VPInstruction.
InstructionCost computeCost(ElementCount VF,
- VPCostContext &Ctx) const override;
+ VPCostContext &Ctx) const override {
+ // TODO: Compute accurate cost after retiring the legacy cost model.
+ return 0;
+ }
Type *getResultType() const { return ResultTy; }
- Type *getSourceElementType() const {
- assert(getOpcode() == Instruction::GetElementPtr &&
- "Source element type requested for non-getelementptr");
- return SourceElementTy;
- }
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
@@ -1594,6 +1575,52 @@ class VPInstructionWithType : public VPInstruction {
#endif
};
+class VPGEPInstruction : public VPInstructionWithType {
+ /// The source element type of the GEP.
+ Type *SourceElementTy;
+
+public:
+ VPGEPInstruction(Type *SourceElementTy, ArrayRef<VPValue *> Operands,
+ Type *ResultTy, const VPIRFlags &Flags = {},
+ const VPIRMetadata &Metadata = {},
+ DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
+ : VPInstructionWithType(Instruction::GetElementPtr, Operands, ResultTy,
+ Flags, Metadata, DL, Name),
+ SourceElementTy(SourceElementTy) {}
+
+ static inline bool classof(const VPRecipeBase *R) {
+ auto *VPI = dyn_cast<VPInstruction>(R);
+ return VPI && VPI->getOpcode() == Instruction::GetElementPtr;
+ }
+
+ Type *getSourceElementType() const { return SourceElementTy; }
+
+ VPGEPInstruction *clone() override {
+ auto *New = new VPGEPInstruction(getSourceElementType(), operands(),
+ getResultType(), *this, *this,
+ getDebugLoc(), getName());
+ New->setUnderlyingValue(getUnderlyingValue());
+ return New;
+ }
+
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override {
+ // We mark this instruction as zero-cost because the cost of GEPs in
+ // vectorized code depends on whether the corresponding memory instruction
+ // is scalarized or not. Therefore, we handle GEPs with the memory
+ // instruction cost.
+ return 0;
+ }
+
+ void execute(VPTransformState &State) override;
+
+protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// Helper type to provide functions to access incoming values and blocks for
/// phi-like recipes.
class VPPhiAccessors {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ab9577d9ed730..f4c1ed09d96e3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1147,12 +1147,6 @@ InstructionCost VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,
{TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);
}
- case Instruction::GetElementPtr:
- // We mark this instruction as zero-cost because the cost of GEPs in
- // vectorized code depends on whether the corresponding memory instruction
- // is scalarized or not. Therefore, we handle GEPs with the memory
- // instruction cost.
- return 0;
}
llvm_unreachable("called for unsupported opcode");
}
@@ -1448,6 +1442,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::Not:
// TODO: Cover additional opcodes.
return vputils::onlyFirstLaneUsed(this);
+ case Instruction::GetElementPtr:
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
case VPInstruction::ExplicitVectorLength:
@@ -1626,16 +1621,6 @@ void VPInstructionWithType::execute(VPTransformState &State) {
return;
}
switch (getOpcode()) {
- case Instruction::GetElementPtr: {
- Value *Ptr = State.get(getOperand(0), true);
- auto IdxList =
- to_vector(map_range(drop_begin(operands()),
- [&](VPValue *Op) { return State.get(Op, true); }));
- Value *GEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, IdxList,
- "", getGEPNoWrapFlags());
- State.set(this, GEP, true);
- return;
- }
case VPInstruction::StepVector: {
Value *StepVector =
State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));
@@ -1653,13 +1638,6 @@ void VPInstructionWithType::execute(VPTransformState &State) {
}
}
-InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
- VPCostContext &Ctx) const {
- if (!getUnderlyingValue())
- return 0;
- return getCostForRecipeWithOpcode(getOpcode(), VF, Ctx);
-}
-
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
@@ -1682,12 +1660,6 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
O << "load ";
printOperands(O, SlotTracker);
break;
- case Instruction::GetElementPtr:
- O << "getelementptr";
- printFlags(O);
- O << *SourceElementTy << " ";
- printOperands(O, SlotTracker);
- break;
default:
assert(Instruction::isCast(getOpcode()) && "unhandled opcode");
O << Instruction::getOpcodeName(getOpcode()) << " ";
@@ -1697,6 +1669,28 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
}
#endif
+void VPGEPInstruction::execute(VPTransformState &State) {
+ Value *Ptr = State.get(getOperand(0), true);
+ auto IdxList = to_vector(map_range(drop_begin(operands()), [&](VPValue *Op) {
+ return State.get(Op, true);
+ }));
+ Value *GEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, IdxList, "",
+ getGEPNoWrapFlags());
+ State.set(this, GEP, true);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPGEPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "EMIT-SCALAR ";
+ printAsOperand(O, SlotTracker);
+ O << " = getelementptr";
+ printFlags(O);
+ O << *getSourceElementType() << " ";
+ printOperands(O, SlotTracker);
+}
+#endif
+
void VPPhi::execute(VPTransformState &State) {
State.setDebugLocFrom(getDebugLoc());
PHINode *NewPhi = State.Builder.CreatePHI(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3ced715dd1061..73acbec3f58d8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1290,11 +1290,7 @@ static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
})
.Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
[](auto *I) { return I->getSourceElementType(); })
- .Case<VPInstructionWithType>([](auto *I) {
- return I->getOpcode() == Instruction::GetElementPtr
- ? I->getSourceElementType()
- : nullptr;
- })
+ .Case<VPGEPInstruction>([](auto *I) { return I->getSourceElementType(); })
.Default([](auto *) { return nullptr; });
}
>From a58536583cdf68c129e7db961d52afa7b61d8e5d Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 09:35:43 +0100
Subject: [PATCH 3/3] [VPlan] First-lane thinko, infer ResultTy, NFC
improvements
---
.../Vectorize/LoopVectorizationPlanner.h | 7 +++----
llvm/lib/Transforms/Vectorize/VPlan.h | 15 +++++++--------
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 8 ++++----
.../Transforms/Vectorize/VPlanConstruction.cpp | 6 +++---
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 14 +++++++-------
.../LoopVectorize/VPlan/AArch64/call-decisions.ll | 8 ++++----
6 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bdf84d8923cf8..cb78a4eb6c420 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -394,11 +394,10 @@ class VPBuilder {
}
VPInstruction *createScalarGEP(Type *SourceElementTy, ArrayRef<VPValue *> Ops,
- Type *ResultTy, DebugLoc DL,
- const VPIRFlags &Flags,
+ DebugLoc DL, const VPIRFlags &Flags,
const VPIRMetadata &Metadata = {}) {
- return tryInsertInstruction(new VPGEPInstruction(
- SourceElementTy, Ops, ResultTy, Flags, Metadata, DL));
+ return tryInsertInstruction(
+ new VPGEPInstruction(SourceElementTy, Ops, Flags, Metadata, DL));
}
VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a91eb46b8b6e3..f82b92b2ae35d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1575,17 +1575,17 @@ class VPInstructionWithType : public VPInstruction {
#endif
};
-class VPGEPInstruction : public VPInstructionWithType {
+class VPGEPInstruction : public VPInstruction {
/// The source element type of the GEP.
Type *SourceElementTy;
public:
VPGEPInstruction(Type *SourceElementTy, ArrayRef<VPValue *> Operands,
- Type *ResultTy, const VPIRFlags &Flags = {},
+ const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
- DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
- : VPInstructionWithType(Instruction::GetElementPtr, Operands, ResultTy,
- Flags, Metadata, DL, Name),
+ DebugLoc DL = DebugLoc::getUnknown())
+ : VPInstruction(Instruction::GetElementPtr, Operands, Flags, Metadata,
+ DL),
SourceElementTy(SourceElementTy) {}
static inline bool classof(const VPRecipeBase *R) {
@@ -1596,9 +1596,8 @@ class VPGEPInstruction : public VPInstructionWithType {
Type *getSourceElementType() const { return SourceElementTy; }
VPGEPInstruction *clone() override {
- auto *New = new VPGEPInstruction(getSourceElementType(), operands(),
- getResultType(), *this, *this,
- getDebugLoc(), getName());
+ auto *New = new VPGEPInstruction(getSourceElementType(), operands(), *this,
+ *this, getDebugLoc());
New->setUnderlyingValue(getUnderlyingValue());
return New;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 103dff1889a6a..451a30a1b5568 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -303,10 +303,10 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
[](const auto *R) { return R->getScalarType(); })
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,
- VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe>(
- [this](const VPRecipeBase *R) {
- return inferScalarType(R->getOperand(0));
- })
+ VPVectorEndPointerRecipe, VPWidenCanonicalIVRecipe,
+ VPGEPInstruction>([this](const VPRecipeBase *R) {
+ return inferScalarType(R->getOperand(0));
+ })
// VPInstructionWithType must be handled before VPInstruction.
.Case<VPInstructionWithType, VPWidenIntrinsicRecipe,
VPWidenCastRecipe>(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ed8e066f61301..e3b70eb356b5b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -268,9 +268,9 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
VPIRFlags(*CI), MD);
NewR->setUnderlyingValue(CI);
} else if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
- NewR = VPIRBuilder.createScalarGEP(
- GEP->getSourceElementType(), VPOperands, GEP->getType(),
- GEP->getDebugLoc(), VPIRFlags(*GEP), MD);
+ NewR = VPIRBuilder.createScalarGEP(GEP->getSourceElementType(),
+ VPOperands, GEP->getDebugLoc(),
+ VPIRFlags(*GEP), MD);
NewR->setUnderlyingValue(GEP);
} else if (auto *LI = dyn_cast<LoadInst>(Inst)) {
NewR = VPIRBuilder.createScalarLoad(LI->getType(), VPOperands[0],
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f4c1ed09d96e3..8a27ee9c2efed 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -19,6 +19,7 @@
#include "VPlanUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/IVDescriptors.h"
@@ -1438,11 +1439,11 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case Instruction::ICmp:
case Instruction::Select:
case Instruction::Or:
+ case Instruction::GetElementPtr:
case Instruction::Freeze:
case VPInstruction::Not:
// TODO: Cover additional opcodes.
return vputils::onlyFirstLaneUsed(this);
- case Instruction::GetElementPtr:
case Instruction::Load:
case VPInstruction::ActiveLaneMask:
case VPInstruction::ExplicitVectorLength:
@@ -1670,12 +1671,11 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
void VPGEPInstruction::execute(VPTransformState &State) {
- Value *Ptr = State.get(getOperand(0), true);
- auto IdxList = to_vector(map_range(drop_begin(operands()), [&](VPValue *Op) {
- return State.get(Op, true);
- }));
- Value *GEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, IdxList, "",
- getGEPNoWrapFlags());
+ auto Ops = map_to_vector(operands(),
+ [&](VPValue *Op) { return State.get(Op, true); });
+ Value *GEP =
+ State.Builder.CreateGEP(getSourceElementType(), Ops.front(),
+ drop_begin(Ops), "", getGEPNoWrapFlags());
State.set(this, GEP, true);
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
index 50c605d8aea62..292835a7f09d4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
@@ -24,11 +24,11 @@ define void @pick_wide_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%in.gep> = getelementptr inbounds double ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%call> = call ir<%in>, ir<@acos>
-; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%out.gep> = getelementptr inbounds double ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
@@ -76,11 +76,11 @@ define void @pick_intrinsic_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT ir<%in.gep> = getelementptr inbounds ir<%in.ptr>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%in.gep> = getelementptr inbounds i64 ir<%in.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer inbounds ir<%in.gep>
; CHECK-NEXT: WIDEN ir<%in> = load vp<[[VP4]]>
; CHECK-NEXT: EMIT ir<%call> = call ir<100>, ir<%in>, ir<@llvm.umin.i64>
-; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%out.gep> = getelementptr inbounds i64 ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
More information about the llvm-commits
mailing list