[llvm] [VPlan] Generalize usesScalars using WideInfo (NFCI) (PR #208910)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 03:49:28 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/208910
>From 4177de73e0a663aab5cf6c0711f3b6241546cd3c Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 16:26:11 +0100
Subject: [PATCH 1/3] [VPlan] Introduce distillation of widening semantics
Introduce VPWideningInfo, a distillation of widening semantics of
recipes, and demonstrate its utility in vputils.
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 231 ++++++++++++------
.../LoopVectorize/AArch64/predicated-costs.ll | 84 ++++++-
2 files changed, 230 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 205fc18a86c78..23382bc6c8b55 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -361,11 +361,42 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a single scalar value if only the SingleScalar
+/// bit is set, a wide value if only the Wide bit is set, and scalar values for
+/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
+/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
+/// be narrowed to single-scalar if legal and profitable. For instructions not
+/// producing values, like an assume or store, the bits talk about the
+/// appropriate operands. Finally, there is a class of instructions that
+/// necessarily take vector operands and produce a scalar result, like
+/// (Insert|Extract)Element, or necessarily take a scalar values and produce a
+/// vector, like Build(Struct)Vector, or could be interepreted as either a wide
+/// or narrow recipe like Broadcasts and non-constant live-ins: these are marked
+/// with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ SingleScalar = 1 << 0,
+ Wide = 1 << 1,
+ GenPerAllLanes = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool producesSingleScalarResult() const {
+ return Info == SingleScalar || Info == (SingleScalar | Agnostic);
+ }
+ bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
+};
+
+static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
+ VPWideningInfo WideOrRep) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
switch (Opcode) {
case Instruction::Freeze:
case Instruction::GetElementPtr:
@@ -373,15 +404,103 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
case VPInstruction::PtrAdd:
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
default:
- return false;
+ return WideOrRep;
}
}
+static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPDerivedIVSC:
+ case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPBranchOnMaskSC:
+ return VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return VPWideningInfo::GenPerAllLanes;
+ case VPRecipeBase::VPWidenCastSC:
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPBlendSC:
+ return VPWideningInfo::Wide | VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(&R);
+ // Broadcast is a special case of a vector-to-scalar.
+ if (VPI->isVectorToScalar() || VPI->getOpcode() == VPInstruction::Broadcast)
+ return VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ // These opcodes take multiple scalars are produce a vector.
+ if (is_contained({VPInstruction::BuildStructVector,
+ VPInstruction::BuildVector,
+ VPInstruction::ActiveLaneMask},
+ VPI->getOpcode()))
+ return VPWideningInfo::Wide | VPWideningInfo::Agnostic;
+ if (VPI->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ if (VPI->doesGeneratePerAllLanes())
+ return VPWideningInfo::GenPerAllLanes;
+ return getNarrowableWideningInfo(VPI->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPExpressionSC: {
+ auto *Expr = cast<VPExpressionRecipe>(&R);
+ return Expr->isVectorToScalar()
+ ? (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic)
+ : VPWideningInfo::Wide;
+ }
+ case VPRecipeBase::VPReductionSC:
+ case VPRecipeBase::VPReductionEVLSC: {
+ auto *Red = cast<VPReductionRecipe>(&R);
+ return Red->isPartialReduction()
+ ? VPWideningInfo::Wide
+ : (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic);
+ }
+ case VPRecipeBase::VPReplicateSC: {
+ auto *Rep = cast<VPReplicateRecipe>(&R);
+ if (Rep->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ return getNarrowableWideningInfo(Rep->getOpcode(),
+ VPWideningInfo::GenPerAllLanes);
+ }
+ case VPRecipeBase::VPWidenSC: {
+ auto *Wide = cast<VPWidenRecipe>(&R);
+ return getNarrowableWideningInfo(Wide->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPWidenCanonicalIVSC:
+ case VPRecipeBase::VPWidenPHISC:
+ case VPRecipeBase::VPWidenCallSC:
+ case VPRecipeBase::VPWidenIntrinsicSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
+ case VPRecipeBase::VPWidenLoadSC:
+ case VPRecipeBase::VPWidenLoadEVLSC:
+ case VPRecipeBase::VPWidenStoreSC:
+ case VPRecipeBase::VPWidenStoreEVLSC:
+ case VPRecipeBase::VPInterleaveSC:
+ case VPRecipeBase::VPInterleaveEVLSC:
+ case VPRecipeBase::VPHistogramSC:
+ case VPRecipeBase::VPCurrentIterationPHISC:
+ case VPRecipeBase::VPActiveLaneMaskPHISC:
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ case VPRecipeBase::VPWidenPointerInductionSC:
+ case VPRecipeBase::VPReductionPHISC:
+ return VPWideningInfo::Wide;
+ }
+ llvm_unreachable("Fell off end of switch: unknown recipe class");
+}
+
+static VPWideningInfo getWideningInfo(const VPValue *VPV) {
+ // A non-constant live-in may be introduce a Broadcast.
+ if (!VPV->hasDefiningRecipe())
+ return isa<VPConstant>(VPV)
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ return getWideningInfo(*VPV->getDefiningRecipe());
+}
+
bool vputils::isElementwise(const VPValue *V) {
unsigned Opcode = TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe>(
@@ -392,12 +511,6 @@ bool vputils::isElementwise(const VPValue *V) {
}
bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and canonical-IV region values are single-scalar.
- if (auto *RV = dyn_cast<VPRegionValue>(VPV))
- return RV == RV->getDefiningRegion()->getCanonicalIV();
- if (isa<VPIRValue, VPSymbolicValue>(VPV))
- return true;
-
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
@@ -405,29 +518,13 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isVectorToScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ // FIXME: Marking WidenCast as a single-scalar leads to regressions.
+ VPWideningInfo Info = getWideningInfo(VPV);
+ return Info.producesSingleScalarResult() ||
+ (!isa<VPWidenCastRecipe>(VPV) &&
+ Info.couldProduceSingleScalarResult() &&
+ all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar));
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -437,50 +534,38 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
- const VPRecipeBase *R = V->getDefiningRecipe();
- const VPBasicBlock *VPBB = R ? R->getParent() : nullptr;
- const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr;
- if (VPBB) {
- if ((VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
- if (match(V->getDefiningRecipe(),
+ // Bail out on VPPhi, as we can end up in infinite cycles.
+ if (isa<VPPhi>(V))
+ return false;
+
+ if (const VPRecipeBase *R = V->getDefiningRecipe()) {
+ const VPBasicBlock *VPBB = R->getParent();
+ const VPlan *Plan = VPBB->getPlan();
+ if (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry()) {
+ if (match(R,
m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()))
return false;
return all_of(R->operands(), isUniformAcrossVFsAndUFs);
}
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
+ // Be conservative about side-effects, except for the
+ // known-side-effecting assumes and stores, which we know will be
+ // uniform.
+ return RepR->isSingleScalar() &&
+ (!RepR->mayHaveSideEffects() ||
+ isa<AssumeInst, StoreInst>(RepR->getUnderlyingInstr())) &&
+ all_of(RepR->operands(), isUniformAcrossVFsAndUFs);
+ }
}
- return TypeSwitch<const VPRecipeBase *, bool>(R)
- .Case([](const VPDerivedIVRecipe *R) { return true; })
- .Case([](const VPReplicateRecipe *R) {
- // Be conservative about side-effects, except for the
- // known-side-effecting assumes and stores, which we know will be
- // uniform.
- return R->isSingleScalar() &&
- (!R->mayHaveSideEffects() ||
- isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPPhi *) {
- // Bail out on VPPhi, as we can end up in infinite cycles.
- return false;
- })
- .Case([](const VPInstruction *VPI) {
- return (VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- preservesUniformity(VPI->getOpcode())) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenCastRecipe *R) {
- // A cast is uniform according to its operand.
- return isUniformAcrossVFsAndUFs(R->getOperand(0));
- })
- .Default([](const VPRecipeBase *) { // A value is considered non-uniform
- // unless proven otherwise.
- return false;
- });
+ // TODO: Match more recipes.
+ if (!isa<VPDerivedIVRecipe, VPWidenRecipe, VPWidenCastRecipe, VPInstruction>(
+ V))
+ return false;
+
+ VPWideningInfo Info = getWideningInfo(V);
+ return Info.couldProduceSingleScalarResult() &&
+ all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
VPBasicBlock *vputils::getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 363223a66f398..000f690a6d46b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -326,22 +326,81 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[DIV]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK: [[PRED_SDIV_IF1]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK: [[PRED_SDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
+; CHECK: [[PRED_SDIV_IF3]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_CONTINUE4]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
+; CHECK: [[PRED_SDIV_IF5]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
+; CHECK: [[PRED_SDIV_CONTINUE6]]:
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[TMP3]], 2
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP10]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP11]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP12]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; CHECK: [[PRED_STORE_CONTINUE10]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
+; CHECK: [[PRED_STORE_IF11]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP14]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; CHECK: [[PRED_STORE_CONTINUE12]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -390,7 +449,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.sin.v4f64(<4 x double> [[TMP2]])
@@ -416,7 +475,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: store float [[PHI]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -457,9 +516,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF17:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF18:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF17]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF18]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -514,6 +573,7 @@ exit:
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META7]], [[META8]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META7]]}
-; CHECK: [[PROF17]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META7]], [[META8]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META8]], [[META7]]}
+; CHECK: [[PROF18]] = !{!"branch_weights", i32 0, i32 1}
;.
>From 268cb00406710eee5f0bf132a6105d6b019e451e Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 13 Jul 2026 11:10:56 +0100
Subject: [PATCH 2/3] [VPlan] Handle RegionValue correctly; fix regression
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 17 ++--
.../LoopVectorize/AArch64/predicated-costs.ll | 84 +++----------------
2 files changed, 22 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 23382bc6c8b55..c27d99acc3de4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -369,11 +369,9 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
/// be narrowed to single-scalar if legal and profitable. For instructions not
/// producing values, like an assume or store, the bits talk about the
/// appropriate operands. Finally, there is a class of instructions that
-/// necessarily take vector operands and produce a scalar result, like
-/// (Insert|Extract)Element, or necessarily take a scalar values and produce a
-/// vector, like Build(Struct)Vector, or could be interepreted as either a wide
-/// or narrow recipe like Broadcasts and non-constant live-ins: these are marked
-/// with the Agnostic bit.
+/// necessarily take vector operands and produce a scalar result termed
+/// VectorToScalar, or necessarily take a scalar values and produce a vector,
+/// termed ScalarToVector. These are marked with the Agnostic bit.
class VPWideningInfo {
unsigned char Info : 4;
@@ -493,11 +491,16 @@ static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
}
static VPWideningInfo getWideningInfo(const VPValue *VPV) {
- // A non-constant live-in may be introduce a Broadcast.
- if (!VPV->hasDefiningRecipe())
+ if (!VPV->hasDefiningRecipe()) {
+ if (auto *RV = dyn_cast<VPRegionValue>(VPV))
+ return RV == RV->getDefiningRegion()->getCanonicalIV()
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::Wide;
+ // A non-constant live-in may be introduce a Broadcast.
return isa<VPConstant>(VPV)
? VPWideningInfo::SingleScalar
: VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ }
return getWideningInfo(*VPV->getDefiningRecipe());
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 000f690a6d46b..363223a66f398 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -326,81 +326,22 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[DIV]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
-; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[TMP3]], 2
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP11]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP12]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], 3
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP13]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP14]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT1:.*]]
-; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -449,7 +390,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.sin.v4f64(<4 x double> [[TMP2]])
@@ -475,7 +416,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: store float [[PHI]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -516,9 +457,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF18:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF17:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF18]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF17]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -573,7 +514,6 @@ exit:
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META7]], [[META8]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META7]], [[META8]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META8]], [[META7]]}
-; CHECK: [[PROF18]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META7]]}
+; CHECK: [[PROF17]] = !{!"branch_weights", i32 0, i32 1}
;.
>From ca4888c4e645106643411eeaa0dc30f6eb39ed88 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sat, 11 Jul 2026 13:43:32 +0100
Subject: [PATCH 3/3] [VPlan] Generalize usesScalars using WideInfo (NFCI)
Generalize VPUser::usesScalars, making it non-virtual, by querying
vputils::getWideningInfo.
---
llvm/lib/Transforms/Vectorize/VPlan.h | 27 -------------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +++
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 38 ++-----------------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 35 +++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanValue.h | 9 +----
5 files changed, 47 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 46e2ad4716323..8605cc88541a9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1578,13 +1578,6 @@ class VPInstructionWithType : public VPInstruction {
Type *getResultType() const { return getScalarType(); }
- /// Cast recipes always use scalars of their operand.
- bool usesScalars(const VPValue *Op) const override {
- if (Instruction::isCast(getOpcode()))
- return true;
- return VPInstruction::usesScalars(Op);
- }
-
protected:
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
@@ -1742,12 +1735,6 @@ class VPIRInstruction : public VPRecipeBase {
Instruction &getInstruction() const { return I; }
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
bool usesFirstPartOnly(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
@@ -3439,13 +3426,6 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
return isSingleScalar();
}
- /// Returns true if the recipe uses scalars of operand \p Op.
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
-
/// Return the mask of a predicated VPReplicateRecipe.
VPValue *getMask() {
assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
@@ -3501,13 +3481,6 @@ class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
printOperands(O, SlotTracker);
}
#endif
-
- /// Returns true if the recipe uses scalars of operand \p Op.
- bool usesScalars(const VPValue *Op) const override {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return true;
- }
};
/// A recipe to combine multiple recipes into a single 'expression' recipe,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 3fbeb7e772a2e..82ff53ce4f396 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -56,6 +56,12 @@ static cl::opt<bool> VPlanPrintMetadata(
cl::desc("Controls the printing of recipe metadata when debugging."));
#endif
+bool VPUser::usesScalars(const VPValue *Op) const {
+ assert(is_contained(operands(), Op) && "Op must be an operand of the recipe");
+ VPWideningInfo WideInfo = vputils::getWideningInfo(*cast<VPRecipeBase>(this));
+ return WideInfo.usesScalarOperands() || usesFirstLaneOnly(Op);
+}
+
bool VPRecipeBase::mayWriteToMemory() const {
switch (getVPRecipeID()) {
case VPExpressionSC:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c27d99acc3de4..1cdc3082b3e90 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -361,36 +361,6 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// A class keeping track of widening information of various recipes.
-/// A recipe necessarily produces a single scalar value if only the SingleScalar
-/// bit is set, a wide value if only the Wide bit is set, and scalar values for
-/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
-/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
-/// be narrowed to single-scalar if legal and profitable. For instructions not
-/// producing values, like an assume or store, the bits talk about the
-/// appropriate operands. Finally, there is a class of instructions that
-/// necessarily take vector operands and produce a scalar result termed
-/// VectorToScalar, or necessarily take a scalar values and produce a vector,
-/// termed ScalarToVector. These are marked with the Agnostic bit.
-class VPWideningInfo {
- unsigned char Info : 4;
-
-public:
- using VPWideningTy = enum {
- SingleScalar = 1 << 0,
- Wide = 1 << 1,
- GenPerAllLanes = 1 << 2,
- Agnostic = 1 << 3
- };
-
- VPWideningInfo(unsigned char Info) : Info(Info) {}
- operator unsigned char() const { return Info; }
- bool producesSingleScalarResult() const {
- return Info == SingleScalar || Info == (SingleScalar | Agnostic);
- }
- bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
-};
-
static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
VPWideningInfo WideOrRep) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
@@ -410,7 +380,7 @@ static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
}
}
-static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
+VPWideningInfo vputils::getWideningInfo(const VPRecipeBase &R) {
switch (R.getVPRecipeID()) {
case VPRecipeBase::VPVectorPointerSC:
case VPRecipeBase::VPVectorEndPointerSC:
@@ -501,7 +471,7 @@ static VPWideningInfo getWideningInfo(const VPValue *VPV) {
? VPWideningInfo::SingleScalar
: VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
}
- return getWideningInfo(*VPV->getDefiningRecipe());
+ return vputils::getWideningInfo(*VPV->getDefiningRecipe());
}
bool vputils::isElementwise(const VPValue *V) {
@@ -523,7 +493,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
return false;
}
// FIXME: Marking WidenCast as a single-scalar leads to regressions.
- VPWideningInfo Info = getWideningInfo(VPV);
+ VPWideningInfo Info = ::getWideningInfo(VPV);
return Info.producesSingleScalarResult() ||
(!isa<VPWidenCastRecipe>(VPV) &&
Info.couldProduceSingleScalarResult() &&
@@ -566,7 +536,7 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
V))
return false;
- VPWideningInfo Info = getWideningInfo(V);
+ VPWideningInfo Info = ::getWideningInfo(V);
return Info.couldProduceSingleScalarResult() &&
all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2980b704ec8da..5980da69beedf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -21,6 +21,38 @@ class PredicatedScalarEvolution;
} // namespace llvm
namespace llvm {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a single scalar value if only the SingleScalar
+/// bit is set, a wide value if only the Wide bit is set, and scalar values for
+/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
+/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
+/// be narrowed to single-scalar if legal and profitable. For instructions not
+/// producing values, like an assume or store, the bits talk about the
+/// appropriate operands. Finally, there is a class of instructions that
+/// necessarily take vector operands and produce a scalar result termed
+/// VectorToScalar, or necessarily take a scalar values and produce a vector,
+/// termed ScalarToVector. These are marked with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ SingleScalar = 1 << 0,
+ Wide = 1 << 1,
+ GenPerAllLanes = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool producesSingleScalarResult() const {
+ return Info == SingleScalar || Info == (SingleScalar | Agnostic);
+ }
+ bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
+ bool usesScalarOperands() const {
+ return !(Info & (Agnostic | Wide)) || Info == (Wide | Agnostic);
+ }
+};
namespace vputils {
/// Returns true if only the first lane of \p Def is used.
@@ -61,6 +93,9 @@ bool isSingleScalar(const VPValue *VPV);
/// VPDerivedIV or the canonical IV).
bool isUniformAcrossVFsAndUFs(const VPValue *V);
+/// Get the WideningInfo for \p R.
+VPWideningInfo getWideningInfo(const VPRecipeBase &R);
+
/// Return true if \p V is elementwise, i.e. none of the lanes are permuted.
bool isElementwise(const VPValue *V);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 738902519fe90..3373a143243b3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -474,13 +474,8 @@ class LLVM_ABI_FOR_TEST VPUser {
return const_operand_range(op_begin(), op_end());
}
- /// Returns true if the VPUser uses scalars of operand \p Op. Conservatively
- /// returns if only first (scalar) lane is used, as default.
- virtual bool usesScalars(const VPValue *Op) const {
- assert(is_contained(operands(), Op) &&
- "Op must be an operand of the recipe");
- return usesFirstLaneOnly(Op);
- }
+ /// Returns true if the VPUser uses scalars of operand \p Op.
+ bool usesScalars(const VPValue *Op) const;
/// Returns true if the VPUser only uses the first lane of operand \p Op.
/// Conservatively returns false.
More information about the llvm-commits
mailing list