[llvm] [VPlan] Add VPInstruction::Intrinsic opcode, use for scalar intrinsics. (PR #207541)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 4 14:03:26 PDT 2026
https://github.com/fhahn created https://github.com/llvm/llvm-project/pull/207541
This patch adds a new Intrinsic opcode to VPInstruction, initially used for generating calls to scalar intrinsics. The intrinsic ID as integer is the last operand (i.e. the called function). Alternatively we could also create the needed intrinsic declarations and pass the function directly, but that would add potentially unused declarations, if we decide to not vectorize.
The first patch migrates just VScale, but there are other opcodes matching directly to intrinsics, which will be replaced in follow ups.
It also gives more flexibility going forward, e.g. allows emitting min/max intrinsics when expanding SCEV min/max expressions.
>From e809eb47919a672d74f313d675741ca244a48db0 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 3 Jul 2026 10:02:08 +0100
Subject: [PATCH] [VPlan] Add VPInstruction::Intrinsic opcode, use for scalar
intrinsics.
This patch adds a new Intrinsic opcode to VPInstruction, initially used
for generating calls to scalar intrinsics. The intrinsic ID as integer
is the last operand (i.e. the called function). Alternatively we could
also create the needed intrinsic declarations and pass the function
directly, but that would add potentially unused declarations, if we
decide to not vectorize.
The first patch migrates just VScale, but there are other opcodes
matching directly to intrinsics, which will be replaced in follow ups.
It also gives more flexibility going forward, e.g. allows emitting
min/max intrinsics when expanding SCEV min/max expressions.
---
.../Vectorize/LoopVectorizationPlanner.h | 16 ++++++-
llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++--
.../Transforms/Vectorize/VPlanPatternMatch.h | 9 ++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 48 +++++++++++++------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 7 ++-
.../LoopVectorize/VPlan/expand-scev.ll | 2 +-
.../Transforms/LoopVectorize/vscale-cost.ll | 4 +-
.../Transforms/Vectorize/VPDomTreeTest.cpp | 12 ++---
.../Transforms/Vectorize/VPlanTest.cpp | 19 ++++----
9 files changed, 81 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 4086f8cd04ac2..9dc13af7f669d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -435,10 +435,22 @@ class VPBuilder {
new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
}
- /// Create a VScale VPInstruction.
+ /// Create a scalar call to the intrinsic \p IntrinsicID with \p Operands, and
+ /// result type \p ResultTy
+ VPInstruction *createScalarIntrinsic(Intrinsic::ID IntrinsicID,
+ ArrayRef<VPValue *> Operands,
+ Type *ResultTy, DebugLoc DL) {
+ VPlan &Plan = getPlan();
+ SmallVector<VPValue *, 2> Ops(Operands);
+ Ops.push_back(Plan.getConstantInt(32, IntrinsicID));
+ return tryInsertInstruction(new VPInstructionWithType(
+ VPInstruction::Intrinsic, Ops, ResultTy, {}, {}, DL));
+ }
+
+ /// Create a scalar llvm.vscale call.
VPInstruction *createVScale(Type *ResultTy,
DebugLoc DL = DebugLoc::getUnknown()) {
- return createNaryOp(VPInstruction::VScale, {}, ResultTy, {}, DL);
+ return createScalarIntrinsic(Intrinsic::vscale, {}, ResultTy, DL);
}
VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b2d179d8e6d98..0d4d685a10e52 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1344,10 +1344,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
WideIVStep,
// Creates a step vector starting from 0 to VF with a step of 1.
StepVector,
- /// Returns the value for vscale.
- VScale,
+ /// Calls a scalar intrinsic. The intrinsic ID is the last operand.
+ Intrinsic,
- OpsEnd = VScale,
+ OpsEnd = Intrinsic,
};
/// Returns true if this VPInstruction generates scalar values for all lanes.
@@ -1559,7 +1559,7 @@ class VPInstructionWithType : public VPInstruction {
switch (Opc) {
case VPInstruction::WideIVStep:
case VPInstruction::StepVector:
- case VPInstruction::VScale:
+ case VPInstruction::Intrinsic:
case Instruction::Load:
return true;
default:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index c57e106021637..e03f6d0a9dc60 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -993,7 +993,8 @@ template <typename Opnd_t> struct Argument_match {
if (R->getOpcode() == Instruction::Call)
return Val.match(R->getOperand(OpI));
if (const auto *R = dyn_cast<VPInstruction>(V))
- if (R->getOpcode() == Instruction::Call)
+ if (R->getOpcode() == Instruction::Call ||
+ R->getOpcode() == VPInstruction::Intrinsic)
return Val.match(R->getOperand(OpI));
return false;
}
@@ -1091,12 +1092,10 @@ struct LiveIn_match {
}
};
-inline VPInstruction_match<VPInstruction::VScale> m_VScale() {
- return m_VPInstruction<VPInstruction::VScale>();
-}
-
inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
+inline IntrinsicID_match m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
+
/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
/// and bind the source element type and operands.
struct GetElementPtr_match {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 258ca7296006b..eacc90f9b85a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -24,6 +24,7 @@
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
@@ -625,7 +626,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
switch (Opcode) {
case VPInstruction::StepVector:
- case VPInstruction::VScale:
case VPInstruction::IncomingAliasMask:
return 0;
case Instruction::Alloca:
@@ -674,6 +674,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::AnyOf:
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
+ case VPInstruction::Intrinsic:
case VPInstruction::CanonicalIVIncrementForPart:
case VPInstruction::ComputeReductionResult:
case VPInstruction::FirstActiveLane:
@@ -1510,7 +1511,7 @@ bool VPInstruction::isSingleScalar() const {
case Instruction::PHI:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::ResumeForEpilogue:
- case VPInstruction::VScale:
+ case VPInstruction::Intrinsic:
return true;
default:
return Instruction::isCast(getOpcode());
@@ -1636,9 +1637,14 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::StepVector:
case VPInstruction::ReductionStartVector:
case VPInstruction::Reverse:
- case VPInstruction::VScale:
case VPInstruction::Unpack:
return false;
+ case VPInstruction::Intrinsic: {
+ LLVMContext &Ctx = getScalarType()->getContext();
+ AttributeSet Attrs =
+ Intrinsic::getFnAttributes(Ctx, vputils::getIntrinsicID(this));
+ return !Attrs.getMemoryEffects().doesNotAccessMemory();
+ }
case Instruction::Call:
return !getCalledFunction(ArrayRef<VPValue *>(op_begin(), op_end()))
->doesNotAccessMemory();
@@ -1678,6 +1684,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnTwoConds:
case VPInstruction::Broadcast:
+ case VPInstruction::Intrinsic:
case VPInstruction::ReductionStartVector:
case VPInstruction::ResumeForEpilogue:
return true;
@@ -1864,9 +1871,14 @@ void VPInstructionWithType::execute(VPTransformState &State) {
State.set(this, StepVector);
break;
}
- case VPInstruction::VScale: {
- Value *VScale = State.Builder.CreateVScale(ResultTy);
- State.set(this, VScale, true);
+ case VPInstruction::Intrinsic: {
+ SmallVector<Value *, 2> Args;
+ for (VPValue *Op : drop_end(operands()))
+ Args.push_back(State.get(Op, VPLane(0)));
+ Value *Call =
+ State.Builder.CreateIntrinsic(ResultTy, vputils::getIntrinsicID(this),
+ Args, /*FMFSource=*/nullptr, getName());
+ State.set(this, Call, true);
break;
}
@@ -1885,12 +1897,6 @@ InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
Ctx);
switch (getOpcode()) {
- case VPInstruction::VScale: {
- Type *Ty = this->getScalarType();
- ArrayRef<Type *> Tys;
- IntrinsicCostAttributes Attrs(Intrinsic::vscale, Ty, Tys);
- return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
- }
case VPInstruction::StepVector:
// TODO: This isn't quite right since even if the step-vector is hoisted
// out of the loop it has a non-zero cost in the middle block, etc.
@@ -1898,6 +1904,14 @@ InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
// licm transform we can add the cost here so that it doesn't incorrectly
// affect the choice of VF.
return 0;
+ case VPInstruction::Intrinsic: {
+ Type *Ty = getScalarType();
+ SmallVector<Type *, 2> ArgTys;
+ for (const VPValue *Op : drop_end(operands()))
+ ArgTys.push_back(Op->getScalarType());
+ IntrinsicCostAttributes Attrs(vputils::getIntrinsicID(this), Ty, ArgTys);
+ return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
+ }
default:
// Although VPInstructionWithType is also used for
// VPInstruction::WideIVStep it isn't currently possible to expose cases
@@ -1923,9 +1937,15 @@ void VPInstructionWithType::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::StepVector:
O << "step-vector " << *ResultTy;
break;
- case VPInstruction::VScale:
- O << "vscale " << *ResultTy;
+ case VPInstruction::Intrinsic: {
+ O << "call @" << Intrinsic::getBaseName(vputils::getIntrinsicID(this))
+ << "(";
+ interleaveComma(drop_end(operands()), O, [&O, &SlotTracker](VPValue *Op) {
+ Op->printAsOperand(O, SlotTracker);
+ });
+ O << ") " << *ResultTy;
break;
+ }
case Instruction::Load:
O << "load ";
printOperands(O, SlotTracker);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 3ebf8e63d5b36..16edfaf8ac2d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -95,9 +95,14 @@ template <typename Ty> Intrinsic::ID getIntrinsicID(const Ty *R) {
// The callee is the last operand, excluding the mask if predicated.
return GetCalleeIntrinsic(
Rep->getOperand(Rep->getNumOperandsWithoutMask() - 1));
- if (const auto *VPI = dyn_cast<VPInstruction>(R))
+ if (const auto *VPI = dyn_cast<VPInstruction>(R)) {
if (VPI->getOpcode() == Instruction::Call)
return GetCalleeIntrinsic(VPI->getOperand(VPI->getNumOperands() - 1));
+ if (VPI->getOpcode() == VPInstruction::Intrinsic) {
+ return cast<VPConstantInt>(VPI->getOperand(VPI->getNumOperands() - 1))
+ ->getZExtValue();
+ }
+ }
return Intrinsic::not_intrinsic;
}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
index 3c56f32e36721..3cbc253eba26e 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/expand-scev.ll
@@ -91,7 +91,7 @@ define void @scev_expand_vscale_mul(ptr %dst, i64 %n) {
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop.preheader>:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = vscale i64
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call @llvm.vscale() i64
; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = shl nuw vp<[[VP2]]>, ir<2>
; CHECK-NEXT: EMIT vp<%min.iters.check> = icmp ult ir<%n>, vp<[[VP3]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%min.iters.check>
diff --git a/llvm/test/Transforms/LoopVectorize/vscale-cost.ll b/llvm/test/Transforms/LoopVectorize/vscale-cost.ll
index 35af718a09f73..310b52d3b5613 100644
--- a/llvm/test/Transforms/LoopVectorize/vscale-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/vscale-cost.ll
@@ -5,8 +5,8 @@
define void @scalablevf(ptr %dst.start, i8 %a, i8 %b) {
; CHECK-LABEL: 'scalablevf'
-; CHECK: Cost of 1 for VF vscale x 4: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = vscale i64
-; CHECK: Cost of 1 for VF vscale x 4: EMIT-SCALAR vp<[[VP2]]> = vscale i64
+; CHECK: Cost of 1 for VF vscale x 4: EMIT-SCALAR vp<[[VP2:%[0-9]+]]> = call @llvm.vscale() i64
+; CHECK: Cost of 1 for VF vscale x 4: EMIT-SCALAR vp<[[VP2]]> = call @llvm.vscale() i64
;
entry:
br label %loop
diff --git a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
index de49dd6b57e61..321ea74f6d7b1 100644
--- a/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPDomTreeTest.cpp
@@ -239,29 +239,29 @@ TEST_F(VPDominatorTreeTest, DominanceRegionsTest) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPBasicBlock *R1BB1 = Plan.createVPBasicBlock("R1BB1");
VPInstruction *R1BB1I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R1BB1->appendRecipe(R1BB1I);
VPBasicBlock *R1BB2 = Plan.createVPBasicBlock("R1BB2");
VPInstruction *R1BB2I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R1BB2->appendRecipe(R1BB2I);
VPBasicBlock *R1BB3 = Plan.createVPBasicBlock("R1BB3");
VPInstruction *R1BB3I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R1BB3->appendRecipe(R1BB3I);
VPRegionBlock *R1 = Plan.createReplicateRegion(R1BB1, R1BB3, "R1");
VPBasicBlock *R2BB1 = Plan.createVPBasicBlock("R2BB1");
VPInstruction *R2BB1I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R2BB1->appendRecipe(R2BB1I);
VPBasicBlock *R2BB2 = Plan.createVPBasicBlock("R2BB2");
VPInstruction *R2BB2I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R2BB2->appendRecipe(R2BB2I);
VPBasicBlock *R2BB3 = Plan.createVPBasicBlock("R2BB3");
VPInstruction *R2BB3I =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
R2BB3->appendRecipe(R2BB3I);
VPRegionBlock *R2 = Plan.createReplicateRegion(R2BB1, R2BB3, "R2");
R2BB2->setParent(R2);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 7c49487dac459..e93cbf9140b8a 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -78,7 +78,7 @@ TEST_F(VPInstructionTest, insertBefore) {
VPInstruction *I1 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I3 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
@@ -97,7 +97,7 @@ TEST_F(VPInstructionTest, eraseFromParent) {
VPInstruction *I1 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I3 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
@@ -121,7 +121,7 @@ TEST_F(VPInstructionTest, moveAfter) {
VPInstruction *I1 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I3 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
@@ -135,7 +135,7 @@ TEST_F(VPInstructionTest, moveAfter) {
CHECK_ITERATOR(VPBB1, I2, I1, I3);
VPInstruction *I4 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I5 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPBasicBlock &VPBB2 = *getPlan().createVPBasicBlock("");
@@ -154,7 +154,7 @@ TEST_F(VPInstructionTest, moveBefore) {
VPInstruction *I1 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I2 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I3 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
@@ -168,7 +168,7 @@ TEST_F(VPInstructionTest, moveBefore) {
CHECK_ITERATOR(VPBB1, I2, I1, I3);
VPInstruction *I4 =
- new VPInstructionWithType(VPInstruction::VScale, {}, Int32);
+ new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPInstruction *I5 =
new VPInstructionWithType(VPInstruction::StepVector, {}, Int32);
VPBasicBlock &VPBB2 = *getPlan().createVPBasicBlock("");
@@ -1786,7 +1786,7 @@ TEST(VPDoubleValueDefTest, traverseUseLists) {
LLVMContext C;
IntegerType *Int32 = IntegerType::get(C, 32);
VPInstructionWithType Op0(VPInstruction::StepVector, {}, Int32);
- VPInstructionWithType Op1(VPInstruction::VScale, {}, Int32);
+ VPInstructionWithType Op1(VPInstruction::StepVector, {}, Int32);
VPDoubleValueDef DoubleValueDef({&Op0, &Op1}, IntegerType::get(C, 32));
// Create a new users of the defined values.
@@ -1859,9 +1859,10 @@ using VPUtilsTest = VPlanTestBase;
TEST_F(VPUtilsTest, IsUniformAcrossVFsAndUFsForSingleScalarOpcodes) {
VPlan &Plan = getPlan();
- // isSingleScalar opcode without operands.
+ // isSingleScalar opcode with a uniform (intrinsic ID) operand.
std::unique_ptr<VPInstruction> VScale(
- VPBuilder().createVScale(IntegerType::get(C, 32)));
+ VPBuilder(Plan.getEntry()).createVScale(IntegerType::get(C, 32)));
+ VScale->removeFromParent();
EXPECT_TRUE(vputils::isUniformAcrossVFsAndUFs(VScale.get()));
// isSingleScalar opcode with a uniform operand.
More information about the llvm-commits
mailing list