[llvm] [VPlan] Factor first-lane-scalar logic in VPI::execute (NFC) (PR #196207)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 1 03:17:46 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/196207
>From 95cb5d25763b17500a50d46c558553b245015f34 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Thu, 7 May 2026 00:07:53 +0100
Subject: [PATCH 1/2] [VPlan] Factor first-lane-scalar logic in VPI::execute
(NFC)
---
llvm/lib/Transforms/Vectorize/VPlan.h | 7 ++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 53 ++++++++-----------
2 files changed, 26 insertions(+), 34 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index eaf9d1433aff7..49842a96303a5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1374,10 +1374,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// needed.
bool canGenerateScalarForFirstLane() const;
- /// Utility methods serving execute(): generates a single vector instance of
- /// the modeled instruction. \returns the generated value. . In some cases an
- /// existing value is returned rather than a generated one.
- Value *generate(VPTransformState &State);
+ /// Utility method serving execute: Generates a scalar or vector value. \p
+ /// IsScalar determines whether to generate a scalar value.
+ Value *generate(VPTransformState &State, bool IsScalar);
/// Returns true if the VPInstruction does not need masking.
bool alwaysUnmasked() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f45b9e4f6c35b..4cf561739865e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -725,13 +725,12 @@ static Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
llvm_unreachable("RecurKind should be Sub/FSub.");
}
-Value *VPInstruction::generate(VPTransformState &State) {
+Value *VPInstruction::generate(VPTransformState &State, bool IsScalar) {
IRBuilderBase &Builder = State.Builder;
if (Instruction::isBinaryOp(getOpcode())) {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
- Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), IsScalar);
+ Value *B = State.get(getOperand(1), IsScalar);
auto *Res =
Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
if (auto *I = dyn_cast<Instruction>(Res))
@@ -741,8 +740,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
switch (getOpcode()) {
case VPInstruction::Not: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), IsScalar);
return Builder.CreateNot(A, Name);
}
case Instruction::ExtractElement: {
@@ -761,26 +759,23 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateInsertElement(Vec, Elt, Idx, Name);
}
case Instruction::Freeze: {
- Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
+ Value *Op = State.get(getOperand(0), IsScalar);
return Builder.CreateFreeze(Op, Name);
}
case Instruction::FCmp:
case Instruction::ICmp: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
- Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
+ Value *A = State.get(getOperand(0), IsScalar);
+ Value *B = State.get(getOperand(1), IsScalar);
return Builder.CreateCmp(getPredicate(), A, B, Name);
}
case Instruction::PHI: {
llvm_unreachable("should be handled by VPPhi::execute");
}
case Instruction::Select: {
- bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
- Value *Cond =
- State.get(getOperand(0),
- OnlyFirstLaneUsed || vputils::isSingleScalar(getOperand(0)));
- Value *Op1 = State.get(getOperand(1), OnlyFirstLaneUsed);
- Value *Op2 = State.get(getOperand(2), OnlyFirstLaneUsed);
+ Value *Cond = State.get(getOperand(0),
+ IsScalar || vputils::isSingleScalar(getOperand(0)));
+ Value *Op1 = State.get(getOperand(1), IsScalar);
+ Value *Op2 = State.get(getOperand(2), IsScalar);
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
Name);
}
@@ -1001,13 +996,13 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateLogicalOr(A, B, Name);
}
case VPInstruction::PtrAdd: {
- assert((State.VF.isScalar() || vputils::onlyFirstLaneUsed(this)) &&
- "can only generate first lane for PtrAdd");
- Value *Ptr = State.get(getOperand(0), VPLane(0));
- Value *Addend = State.get(getOperand(1), VPLane(0));
+ assert(IsScalar && "Can only generate first lane for PtrAdd");
+ Value *Ptr = State.get(getOperand(0), IsScalar);
+ Value *Addend = State.get(getOperand(1), IsScalar);
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::WidePtrAdd: {
+ assert(!IsScalar && "Cannot generate scalar value for WidePtrAdd");
Value *Ptr =
State.get(getOperand(0), vputils::isSingleScalar(getOperand(0)));
Value *Addend = State.get(getOperand(1));
@@ -1568,20 +1563,18 @@ void VPInstruction::execute(VPTransformState &State) {
assert(hasRequiredFlagsForOpcode(getOpcode()) &&
"Opcode requires specific flags to be set");
State.Builder.setFastMathFlags(getFastMathFlagsOrNone());
- Value *GeneratedValue = generate(State);
+ bool GenerateScalar =
+ State.VF.isScalar() || (canGenerateScalarForFirstLane() &&
+ (vputils::onlyFirstLaneUsed(this) ||
+ isVectorToScalar() || isSingleScalar()));
+ Value *GeneratedValue = generate(State, GenerateScalar);
if (!hasResult())
return;
assert(GeneratedValue && "generate must produce a value");
- bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
- (vputils::onlyFirstLaneUsed(this) ||
- isVectorToScalar() || isSingleScalar());
- assert((((GeneratedValue->getType()->isVectorTy() ||
- GeneratedValue->getType()->isStructTy()) ==
- !GeneratesPerFirstLaneOnly) ||
- State.VF.isScalar()) &&
+ assert(((GeneratedValue->getType()->isVectorTy() ||
+ GeneratedValue->getType()->isStructTy()) == !GenerateScalar) &&
"scalar value but not only first lane defined");
- State.set(this, GeneratedValue,
- /*IsScalar*/ GeneratesPerFirstLaneOnly);
+ State.set(this, GeneratedValue, GenerateScalar);
if (getOpcode() == VPInstruction::ResumeForEpilogue ||
getOpcode() == Instruction::Freeze) {
// FIXME: This is a workaround to enable reliable updates of the scalar loop
>From ce6dbbe50a3de94b63c31df47db3166289f65050 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 1 Jul 2026 11:14:55 +0100
Subject: [PATCH 2/2] [VPlan] Rename IsScalar -> IsSingleScalar
---
llvm/lib/Transforms/Vectorize/VPlan.h | 7 +++--
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 31 ++++++++++---------
2 files changed, 20 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 49842a96303a5..af68614cdbdf8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1374,9 +1374,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// needed.
bool canGenerateScalarForFirstLane() const;
- /// Utility method serving execute: Generates a scalar or vector value. \p
- /// IsScalar determines whether to generate a scalar value.
- Value *generate(VPTransformState &State, bool IsScalar);
+ /// Utility method serving execute: Generates either a single-scalar or vector
+ /// value. \p IsSingleScalar determines whether to generate a single-scalar
+ /// value.
+ Value *generate(VPTransformState &State, bool IsSingleScalar);
/// Returns true if the VPInstruction does not need masking.
bool alwaysUnmasked() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4cf561739865e..01a9e7ddf20db 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -725,12 +725,12 @@ static Instruction::BinaryOps getSubRecurOpcode(RecurKind Kind) {
llvm_unreachable("RecurKind should be Sub/FSub.");
}
-Value *VPInstruction::generate(VPTransformState &State, bool IsScalar) {
+Value *VPInstruction::generate(VPTransformState &State, bool IsSingleScalar) {
IRBuilderBase &Builder = State.Builder;
if (Instruction::isBinaryOp(getOpcode())) {
- Value *A = State.get(getOperand(0), IsScalar);
- Value *B = State.get(getOperand(1), IsScalar);
+ Value *A = State.get(getOperand(0), IsSingleScalar);
+ Value *B = State.get(getOperand(1), IsSingleScalar);
auto *Res =
Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
if (auto *I = dyn_cast<Instruction>(Res))
@@ -740,7 +740,7 @@ Value *VPInstruction::generate(VPTransformState &State, bool IsScalar) {
switch (getOpcode()) {
case VPInstruction::Not: {
- Value *A = State.get(getOperand(0), IsScalar);
+ Value *A = State.get(getOperand(0), IsSingleScalar);
return Builder.CreateNot(A, Name);
}
case Instruction::ExtractElement: {
@@ -759,23 +759,24 @@ Value *VPInstruction::generate(VPTransformState &State, bool IsScalar) {
return Builder.CreateInsertElement(Vec, Elt, Idx, Name);
}
case Instruction::Freeze: {
- Value *Op = State.get(getOperand(0), IsScalar);
+ Value *Op = State.get(getOperand(0), IsSingleScalar);
return Builder.CreateFreeze(Op, Name);
}
case Instruction::FCmp:
case Instruction::ICmp: {
- Value *A = State.get(getOperand(0), IsScalar);
- Value *B = State.get(getOperand(1), IsScalar);
+ Value *A = State.get(getOperand(0), IsSingleScalar);
+ Value *B = State.get(getOperand(1), IsSingleScalar);
return Builder.CreateCmp(getPredicate(), A, B, Name);
}
case Instruction::PHI: {
llvm_unreachable("should be handled by VPPhi::execute");
}
case Instruction::Select: {
- Value *Cond = State.get(getOperand(0),
- IsScalar || vputils::isSingleScalar(getOperand(0)));
- Value *Op1 = State.get(getOperand(1), IsScalar);
- Value *Op2 = State.get(getOperand(2), IsScalar);
+ Value *Cond =
+ State.get(getOperand(0),
+ IsSingleScalar || vputils::isSingleScalar(getOperand(0)));
+ Value *Op1 = State.get(getOperand(1), IsSingleScalar);
+ Value *Op2 = State.get(getOperand(2), IsSingleScalar);
return Builder.CreateSelectFMF(Cond, Op1, Op2, getFastMathFlagsOrNone(),
Name);
}
@@ -996,13 +997,13 @@ Value *VPInstruction::generate(VPTransformState &State, bool IsScalar) {
return Builder.CreateLogicalOr(A, B, Name);
}
case VPInstruction::PtrAdd: {
- assert(IsScalar && "Can only generate first lane for PtrAdd");
- Value *Ptr = State.get(getOperand(0), IsScalar);
- Value *Addend = State.get(getOperand(1), IsScalar);
+ assert(IsSingleScalar && "Can only generate first lane for PtrAdd");
+ Value *Ptr = State.get(getOperand(0), IsSingleScalar);
+ Value *Addend = State.get(getOperand(1), IsSingleScalar);
return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags());
}
case VPInstruction::WidePtrAdd: {
- assert(!IsScalar && "Cannot generate scalar value for WidePtrAdd");
+ assert(!IsSingleScalar && "Cannot generate scalar value for WidePtrAdd");
Value *Ptr =
State.get(getOperand(0), vputils::isSingleScalar(getOperand(0)));
Value *Addend = State.get(getOperand(1));
More information about the llvm-commits
mailing list