[llvm] [VPlan] Introduce distillation of widening semantics (NFC) (PR #196181)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 03:20:03 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/196181
>From 4177de73e0a663aab5cf6c0711f3b6241546cd3c Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 6 May 2026 16:26:11 +0100
Subject: [PATCH 1/2] [VPlan] Introduce distillation of widening semantics
Introduce VPWideningInfo, a distillation of widening semantics of
recipes, and demonstrate its utility in vputils.
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 231 ++++++++++++------
.../LoopVectorize/AArch64/predicated-costs.ll | 84 ++++++-
2 files changed, 230 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 205fc18a86c78..23382bc6c8b55 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -361,11 +361,42 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
match(Addr, m_scev_AffineAddRec(m_SCEV(), m_SCEV()));
}
-/// Returns true if \p Opcode preserves uniformity, i.e., if all operands are
-/// uniform, the result will also be uniform.
-static bool preservesUniformity(unsigned Opcode) {
+/// A class keeping track of widening information of various recipes.
+/// A recipe necessarily produces a single scalar value if only the SingleScalar
+/// bit is set, a wide value if only the Wide bit is set, and scalar values for
+/// all VF lanes only the GenPerAllLanes bit is set. The SingleScalar bit can be
+/// set on Wide or GenPerAllLanes recipes, which indicates that the recipe could
+/// be narrowed to single-scalar if legal and profitable. For instructions not
+/// producing values, like an assume or store, the bits talk about the
+/// appropriate operands. Finally, there is a class of instructions that
+/// necessarily take vector operands and produce a scalar result, like
+/// (Insert|Extract)Element, or necessarily take a scalar values and produce a
+/// vector, like Build(Struct)Vector, or could be interepreted as either a wide
+/// or narrow recipe like Broadcasts and non-constant live-ins: these are marked
+/// with the Agnostic bit.
+class VPWideningInfo {
+ unsigned char Info : 4;
+
+public:
+ using VPWideningTy = enum {
+ SingleScalar = 1 << 0,
+ Wide = 1 << 1,
+ GenPerAllLanes = 1 << 2,
+ Agnostic = 1 << 3
+ };
+
+ VPWideningInfo(unsigned char Info) : Info(Info) {}
+ operator unsigned char() const { return Info; }
+ bool producesSingleScalarResult() const {
+ return Info == SingleScalar || Info == (SingleScalar | Agnostic);
+ }
+ bool couldProduceSingleScalarResult() const { return Info & SingleScalar; }
+};
+
+static VPWideningInfo getNarrowableWideningInfo(unsigned Opcode,
+ VPWideningInfo WideOrRep) {
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
switch (Opcode) {
case Instruction::Freeze:
case Instruction::GetElementPtr:
@@ -373,15 +404,103 @@ static bool preservesUniformity(unsigned Opcode) {
case Instruction::FCmp:
case Instruction::Select:
case VPInstruction::Not:
- case VPInstruction::Broadcast:
case VPInstruction::MaskedCond:
case VPInstruction::PtrAdd:
- return true;
+ return WideOrRep | VPWideningInfo::SingleScalar;
default:
- return false;
+ return WideOrRep;
}
}
+static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPVectorPointerSC:
+ case VPRecipeBase::VPVectorEndPointerSC:
+ case VPRecipeBase::VPDerivedIVSC:
+ case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPIRInstructionSC:
+ case VPRecipeBase::VPBranchOnMaskSC:
+ return VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPScalarIVStepsSC:
+ return VPWideningInfo::GenPerAllLanes;
+ case VPRecipeBase::VPWidenCastSC:
+ case VPRecipeBase::VPWidenGEPSC:
+ case VPRecipeBase::VPPredInstPHISC:
+ case VPRecipeBase::VPBlendSC:
+ return VPWideningInfo::Wide | VPWideningInfo::SingleScalar;
+ case VPRecipeBase::VPInstructionSC: {
+ auto *VPI = cast<VPInstruction>(&R);
+ // Broadcast is a special case of a vector-to-scalar.
+ if (VPI->isVectorToScalar() || VPI->getOpcode() == VPInstruction::Broadcast)
+ return VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ // These opcodes take multiple scalars are produce a vector.
+ if (is_contained({VPInstruction::BuildStructVector,
+ VPInstruction::BuildVector,
+ VPInstruction::ActiveLaneMask},
+ VPI->getOpcode()))
+ return VPWideningInfo::Wide | VPWideningInfo::Agnostic;
+ if (VPI->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ if (VPI->doesGeneratePerAllLanes())
+ return VPWideningInfo::GenPerAllLanes;
+ return getNarrowableWideningInfo(VPI->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPExpressionSC: {
+ auto *Expr = cast<VPExpressionRecipe>(&R);
+ return Expr->isVectorToScalar()
+ ? (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic)
+ : VPWideningInfo::Wide;
+ }
+ case VPRecipeBase::VPReductionSC:
+ case VPRecipeBase::VPReductionEVLSC: {
+ auto *Red = cast<VPReductionRecipe>(&R);
+ return Red->isPartialReduction()
+ ? VPWideningInfo::Wide
+ : (VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic);
+ }
+ case VPRecipeBase::VPReplicateSC: {
+ auto *Rep = cast<VPReplicateRecipe>(&R);
+ if (Rep->isSingleScalar())
+ return VPWideningInfo::SingleScalar;
+ return getNarrowableWideningInfo(Rep->getOpcode(),
+ VPWideningInfo::GenPerAllLanes);
+ }
+ case VPRecipeBase::VPWidenSC: {
+ auto *Wide = cast<VPWidenRecipe>(&R);
+ return getNarrowableWideningInfo(Wide->getOpcode(), VPWideningInfo::Wide);
+ }
+ case VPRecipeBase::VPWidenCanonicalIVSC:
+ case VPRecipeBase::VPWidenPHISC:
+ case VPRecipeBase::VPWidenCallSC:
+ case VPRecipeBase::VPWidenIntrinsicSC:
+ case VPRecipeBase::VPWidenMemIntrinsicSC:
+ case VPRecipeBase::VPWidenLoadSC:
+ case VPRecipeBase::VPWidenLoadEVLSC:
+ case VPRecipeBase::VPWidenStoreSC:
+ case VPRecipeBase::VPWidenStoreEVLSC:
+ case VPRecipeBase::VPInterleaveSC:
+ case VPRecipeBase::VPInterleaveEVLSC:
+ case VPRecipeBase::VPHistogramSC:
+ case VPRecipeBase::VPCurrentIterationPHISC:
+ case VPRecipeBase::VPActiveLaneMaskPHISC:
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ case VPRecipeBase::VPWidenPointerInductionSC:
+ case VPRecipeBase::VPReductionPHISC:
+ return VPWideningInfo::Wide;
+ }
+ llvm_unreachable("Fell off end of switch: unknown recipe class");
+}
+
+static VPWideningInfo getWideningInfo(const VPValue *VPV) {
+ // A non-constant live-in may be introduce a Broadcast.
+ if (!VPV->hasDefiningRecipe())
+ return isa<VPConstant>(VPV)
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ return getWideningInfo(*VPV->getDefiningRecipe());
+}
+
bool vputils::isElementwise(const VPValue *V) {
unsigned Opcode = TypeSwitch<const VPValue *, unsigned>(V)
.Case<VPInstruction, VPWidenRecipe>(
@@ -392,12 +511,6 @@ bool vputils::isElementwise(const VPValue *V) {
}
bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and canonical-IV region values are single-scalar.
- if (auto *RV = dyn_cast<VPRegionValue>(VPV))
- return RV == RV->getDefiningRegion()->getCanonicalIV();
- if (isa<VPIRValue, VPSymbolicValue>(VPV))
- return true;
-
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
const VPRegionBlock *RegionOfR = Rep->getRegion();
// Don't consider recipes in replicate regions as uniform yet; their first
@@ -405,29 +518,13 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
// lanes.
if (RegionOfR && RegionOfR->isReplicator())
return false;
- return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&
- all_of(Rep->operands(), isSingleScalar));
- }
- if (isa<VPWidenGEPRecipe, VPBlendRecipe>(VPV))
- return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
- if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
- return preservesUniformity(WidenR->getOpcode()) &&
- all_of(WidenR->operands(), isSingleScalar);
}
- if (auto *VPI = dyn_cast<VPInstruction>(VPV))
- return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- (preservesUniformity(VPI->getOpcode()) &&
- all_of(VPI->operands(), isSingleScalar));
- if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
- return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
- return true;
- if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
- return Expr->isVectorToScalar();
-
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ // FIXME: Marking WidenCast as a single-scalar leads to regressions.
+ VPWideningInfo Info = getWideningInfo(VPV);
+ return Info.producesSingleScalarResult() ||
+ (!isa<VPWidenCastRecipe>(VPV) &&
+ Info.couldProduceSingleScalarResult() &&
+ all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar));
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
@@ -437,50 +534,38 @@ bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
- const VPRecipeBase *R = V->getDefiningRecipe();
- const VPBasicBlock *VPBB = R ? R->getParent() : nullptr;
- const VPlan *Plan = VPBB ? VPBB->getPlan() : nullptr;
- if (VPBB) {
- if ((VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry())) {
- if (match(V->getDefiningRecipe(),
+ // Bail out on VPPhi, as we can end up in infinite cycles.
+ if (isa<VPPhi>(V))
+ return false;
+
+ if (const VPRecipeBase *R = V->getDefiningRecipe()) {
+ const VPBasicBlock *VPBB = R->getParent();
+ const VPlan *Plan = VPBB->getPlan();
+ if (VPBB == Plan->getVectorPreheader() || VPBB == Plan->getEntry()) {
+ if (match(R,
m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()))
return false;
return all_of(R->operands(), isUniformAcrossVFsAndUFs);
}
+ if (auto *RepR = dyn_cast<VPReplicateRecipe>(R)) {
+ // Be conservative about side-effects, except for the
+ // known-side-effecting assumes and stores, which we know will be
+ // uniform.
+ return RepR->isSingleScalar() &&
+ (!RepR->mayHaveSideEffects() ||
+ isa<AssumeInst, StoreInst>(RepR->getUnderlyingInstr())) &&
+ all_of(RepR->operands(), isUniformAcrossVFsAndUFs);
+ }
}
- return TypeSwitch<const VPRecipeBase *, bool>(R)
- .Case([](const VPDerivedIVRecipe *R) { return true; })
- .Case([](const VPReplicateRecipe *R) {
- // Be conservative about side-effects, except for the
- // known-side-effecting assumes and stores, which we know will be
- // uniform.
- return R->isSingleScalar() &&
- (!R->mayHaveSideEffects() ||
- isa<AssumeInst, StoreInst>(R->getUnderlyingInstr())) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenRecipe *R) {
- return preservesUniformity(R->getOpcode()) &&
- all_of(R->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPPhi *) {
- // Bail out on VPPhi, as we can end up in infinite cycles.
- return false;
- })
- .Case([](const VPInstruction *VPI) {
- return (VPI->isSingleScalar() || VPI->isVectorToScalar() ||
- preservesUniformity(VPI->getOpcode())) &&
- all_of(VPI->operands(), isUniformAcrossVFsAndUFs);
- })
- .Case([](const VPWidenCastRecipe *R) {
- // A cast is uniform according to its operand.
- return isUniformAcrossVFsAndUFs(R->getOperand(0));
- })
- .Default([](const VPRecipeBase *) { // A value is considered non-uniform
- // unless proven otherwise.
- return false;
- });
+ // TODO: Match more recipes.
+ if (!isa<VPDerivedIVRecipe, VPWidenRecipe, VPWidenCastRecipe, VPInstruction>(
+ V))
+ return false;
+
+ VPWideningInfo Info = getWideningInfo(V);
+ return Info.couldProduceSingleScalarResult() &&
+ all_of(V->getDefiningRecipe()->operands(), isUniformAcrossVFsAndUFs);
}
VPBasicBlock *vputils::getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 363223a66f398..000f690a6d46b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -326,22 +326,81 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
+; CHECK: [[PRED_SDIV_IF]]:
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
+; CHECK: [[PRED_SDIV_CONTINUE]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[DIV]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
+; CHECK: [[PRED_SDIV_IF1]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
+; CHECK: [[PRED_SDIV_CONTINUE2]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
+; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
+; CHECK: [[PRED_SDIV_IF3]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
+; CHECK: [[PRED_SDIV_CONTINUE4]]:
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
+; CHECK: [[PRED_SDIV_IF5]]:
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
+; CHECK: [[PRED_SDIV_CONTINUE6]]:
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[TMP3]], 2
+; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK: [[PRED_STORE_IF]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
+; CHECK: [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK: [[PRED_STORE_IF7]]:
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP10]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
+; CHECK: [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT: br i1 [[TMP5]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP11]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP12]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
+; CHECK: [[PRED_STORE_CONTINUE10]]:
+; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
+; CHECK: [[PRED_STORE_IF11]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], 3
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP13]]
+; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP14]], align 4
+; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
+; CHECK: [[PRED_STORE_CONTINUE12]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -390,7 +449,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.sin.v4f64(<4 x double> [[TMP2]])
@@ -416,7 +475,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: store float [[PHI]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -457,9 +516,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF17:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF18:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF17]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF18]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -514,6 +573,7 @@ exit:
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META7]], [[META8]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META7]]}
-; CHECK: [[PROF17]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META7]], [[META8]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META8]], [[META7]]}
+; CHECK: [[PROF18]] = !{!"branch_weights", i32 0, i32 1}
;.
>From 268cb00406710eee5f0bf132a6105d6b019e451e Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 13 Jul 2026 11:10:56 +0100
Subject: [PATCH 2/2] [VPlan] Handle RegionValue correctly; fix regression
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 17 ++--
.../LoopVectorize/AArch64/predicated-costs.ll | 84 +++----------------
2 files changed, 22 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 23382bc6c8b55..c27d99acc3de4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -369,11 +369,9 @@ bool vputils::isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE,
/// be narrowed to single-scalar if legal and profitable. For instructions not
/// producing values, like an assume or store, the bits talk about the
/// appropriate operands. Finally, there is a class of instructions that
-/// necessarily take vector operands and produce a scalar result, like
-/// (Insert|Extract)Element, or necessarily take a scalar values and produce a
-/// vector, like Build(Struct)Vector, or could be interepreted as either a wide
-/// or narrow recipe like Broadcasts and non-constant live-ins: these are marked
-/// with the Agnostic bit.
+/// necessarily take vector operands and produce a scalar result termed
+/// VectorToScalar, or necessarily take a scalar values and produce a vector,
+/// termed ScalarToVector. These are marked with the Agnostic bit.
class VPWideningInfo {
unsigned char Info : 4;
@@ -493,11 +491,16 @@ static VPWideningInfo getWideningInfo(const VPRecipeBase &R) {
}
static VPWideningInfo getWideningInfo(const VPValue *VPV) {
- // A non-constant live-in may be introduce a Broadcast.
- if (!VPV->hasDefiningRecipe())
+ if (!VPV->hasDefiningRecipe()) {
+ if (auto *RV = dyn_cast<VPRegionValue>(VPV))
+ return RV == RV->getDefiningRegion()->getCanonicalIV()
+ ? VPWideningInfo::SingleScalar
+ : VPWideningInfo::Wide;
+ // A non-constant live-in may be introduce a Broadcast.
return isa<VPConstant>(VPV)
? VPWideningInfo::SingleScalar
: VPWideningInfo::SingleScalar | VPWideningInfo::Agnostic;
+ }
return getWideningInfo(*VPV->getDefiningRecipe());
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 000f690a6d46b..363223a66f398 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -326,81 +326,22 @@ exit:
define void @sdiv_power_of_2_divisor_in_replicate_region(i32 %x, ptr %dst, i64 %n) {
; CHECK-LABEL: define void @sdiv_power_of_2_divisor_in_replicate_region(
; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[C_1:%.*]] = icmp sgt i64 [[N]], 0
; CHECK-NEXT: [[C_2:%.*]] = icmp slt i64 [[N]], 9
; CHECK-NEXT: call void @llvm.assume(i1 [[C_1]])
; CHECK-NEXT: call void @llvm.assume(i1 [[C_2]])
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 99, [[X]]
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ poison, %[[VECTOR_BODY]] ], [ [[DIV]], %[[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; CHECK: [[PRED_SDIV_IF1]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; CHECK: [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
-; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; CHECK: [[PRED_SDIV_IF3]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; CHECK: [[PRED_SDIV_CONTINUE4]]:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; CHECK: [[PRED_SDIV_IF5]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; CHECK: [[PRED_SDIV_CONTINUE6]]:
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[TMP3]], 2
-; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
+; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV]], 2
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV]]
; CHECK-NEXT: store i32 [[DIV2]], ptr [[GEP]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; CHECK: [[PRED_STORE_IF7]]:
-; CHECK-NEXT: [[IV_NEXT:%.*]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP10]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE8]]
-; CHECK: [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT: br i1 [[TMP5]], label %[[EXIT:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], 2
-; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP11]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP12]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE10]]
-; CHECK: [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_IF11]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], 3
-; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST]], i64 [[TMP13]]
-; CHECK-NEXT: store i32 [[DIV2]], ptr [[TMP14]], align 4
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE12]]
-; CHECK: [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br label %[[EXIT1:.*]]
-; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
@@ -449,7 +390,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[STEP_ADD]] to <4 x double>
; CHECK-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.sin.v4f64(<4 x double> [[TMP2]])
@@ -475,7 +416,7 @@ define void @round_scalar_pred_divisor(ptr %dst, double %x) {
; CHECK-NEXT: store float [[PHI]], ptr [[DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1024
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP1]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -516,9 +457,9 @@ define void @getPredBlockCostDivisor_truncate(i32 %0, i1 %c1, i1 %c2, ptr %p) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF18:![0-9]+]]
+; CHECK-NEXT: br i1 [[C1]], label %[[IF_1:.*]], label %[[LATCH]], !prof [[PROF17:![0-9]+]]
; CHECK: [[IF_1]]:
-; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF18]]
+; CHECK-NEXT: br i1 [[C2]], label %[[IF_2:.*]], label %[[LATCH]], !prof [[PROF17]]
; CHECK: [[IF_2]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
; CHECK-NEXT: store i32 0, ptr [[GEP]], align 4
@@ -573,7 +514,6 @@ exit:
; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META7]], [[META8]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META7]], [[META8]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META8]], [[META7]]}
-; CHECK: [[PROF18]] = !{!"branch_weights", i32 0, i32 1}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META8]], [[META7]]}
+; CHECK: [[PROF17]] = !{!"branch_weights", i32 0, i32 1}
;.
More information about the llvm-commits
mailing list