[llvm] [LV] Vectorize predicated loop-invariant loads via masked load + broadcast (PR #200361)
Ashutosh Nema via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 2 05:32:32 PDT 2026
https://github.com/nema-ashutosh updated https://github.com/llvm/llvm-project/pull/200361
>From fca0eb65184ecd8be61689ab7bd3d00b3410d777 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Fri, 29 May 2026 15:04:07 +0530
Subject: [PATCH 1/2] [LV] Vectorize predicated loop-invariant loads via masked
load + broadcast
Today, a predicated load from a loop-invariant address is scalarised
inside a replicate region: every active lane independently issues the
same load and the results are reassembled into a vector. The cost
model assigns this a very high cost, which both blocks
auto-vectorization
and produces poor codegen when vectorization is forced.
This patch teaches the loop vectorizer to instead emit a single
masked load of one lane from the invariant pointer, guarded by an
any-active-lane reduction over the block mask, and broadcast the
result across the vector.
* New cost-model decision CM_CondInvar and helper
getConditionalInvarCost, picked when it beats gather/scatter
and scalarisation.
* New VPInstruction opcode BroadcastLane (dual of ExtractLane),
with type inference, cost, printing and a verifier check.
* New VPlan transform cseUniformMemoryReads, to fold the
structurally identical masked loads produced for each UF part
(VPWidenLoadRecipe is missed by the generic cse).
* Gated by -enable-masked-invariant-load (off by default).
Fixes #200355.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 106 ++++++++++-
.../Transforms/Vectorize/VPRecipeBuilder.h | 5 +
llvm/lib/Transforms/Vectorize/VPlan.h | 3 +
.../Transforms/Vectorize/VPlanAnalysis.cpp | 1 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 56 ++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 36 ++++
.../Transforms/Vectorize/VPlanTransforms.h | 4 +
.../Transforms/Vectorize/VPlanVerifier.cpp | 30 ++++
.../enable-masked-invariant-load-multiple.ll | 93 ++++++++++
...ble-masked-invariant-load-not-invariant.ll | 64 +++++++
.../X86/enable-masked-invariant-load.ll | 169 ++++++++++++++++++
11 files changed, 565 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 61c2d3cd228ec..064c1498e207d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,6 +198,11 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
+static cl::opt<bool> EnableMaskedInvariantLoad(
+ "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
+ cl::desc("Vectorize conditional invariant load with mask-load"
+ " and broadcast"));
+
/// Option tail-folding-policy indicates that an epilogue is undesired, that
/// tail folding is preferred, and this lists all options. I.e., the vectorizer
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
@@ -929,7 +934,8 @@ class LoopVectorizationCostModel {
CM_GatherScatter,
CM_Scalarize,
CM_VectorCall,
- CM_IntrinsicCall
+ CM_IntrinsicCall,
+ CM_CondInvar,
};
/// Save vectorization decision \p W and \p Cost taken by the cost model for
@@ -1330,6 +1336,10 @@ class LoopVectorizationCostModel {
/// The cost computation for Gather/Scatter instruction.
InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
+ /// Cost of lowering a predicated loop-invariant load as a single-lane
+ /// masked load + broadcast. Returns getInvalid() if not applicable.
+ InstructionCost getConditionalInvarCost(Instruction *I, ElementCount VF);
+
/// The cost computation for widening instruction \p I with consecutive
/// memory access.
InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
@@ -2714,9 +2724,11 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (IsUniformMemOpUse(I))
return true;
+ // CM_CondInvar uses its scalar invariant pointer uniformly.
return (WideningDecision == CM_Widen ||
WideningDecision == CM_Widen_Reverse ||
- WideningDecision == CM_Interleave);
+ WideningDecision == CM_Interleave ||
+ WideningDecision == CM_CondInvar);
};
// Returns true if Ptr is the pointer operand of a memory access instruction
@@ -4397,6 +4409,49 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
Config.CostKind);
}
+InstructionCost
+LoopVectorizationCostModel::getConditionalInvarCost(Instruction *I,
+ ElementCount VF) {
+ // Applicability checks. Any failure returns Invalid; the cost ladder
+ // already gates on .isValid().
+ if (!EnableMaskedInvariantLoad)
+ return InstructionCost::getInvalid();
+ if (!Legal->blockNeedsPredication(I->getParent()))
+ return InstructionCost::getInvalid();
+ if (!isMaskRequired(I))
+ return InstructionCost::getInvalid();
+ if (!isa<LoadInst>(I))
+ return InstructionCost::getInvalid();
+ const Align Alignment = getLoadStoreAlignment(I);
+ unsigned AS = getLoadStoreAddressSpace(I);
+ if (!TTI.isLegalMaskedLoad(I->getType(), Alignment, AS))
+ return InstructionCost::getInvalid();
+ Value *Ptr = getLoadStorePointerOperand(I);
+ if (!Legal->isInvariant(Ptr))
+ return InstructionCost::getInvalid();
+
+ // Components: AnyOf or-reduce + lane-0 mask materialisation
+ // (zero-splat + insertelement) + masked load + lane-0 broadcast shuffle.
+ // InstructionCost arithmetic propagates Invalid across the chain.
+ Type *ValTy = getLoadStoreType(I);
+ auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
+ auto *MaskTy = cast<VectorType>(
+ toVectorTy(Type::getInt1Ty(I->getContext()), VF));
+ InstructionCost Cost = TTI.getArithmeticReductionCost(
+ Instruction::Or, MaskTy, /*FMF*/ std::nullopt, Config.CostKind);
+ Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, MaskTy, MaskTy,
+ {}, Config.CostKind, 0);
+ Cost += TTI.getVectorInstrCost(Instruction::InsertElement, MaskTy,
+ Config.CostKind, /*Index=*/0);
+ Cost += TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::masked_load, VectorTy, Alignment,
+ AS),
+ Config.CostKind);
+ Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
+ VectorTy, {}, Config.CostKind, 0);
+ return Cost;
+}
+
InstructionCost
LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
ElementCount VF) {
@@ -4802,6 +4857,7 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
InstructionCost ScalarizationCost =
getMemInstScalarizationCost(&I, VF) * NumAccesses;
+ InstructionCost ConditionalInvarCost = getConditionalInvarCost(&I, VF);
// Choose better solution for the current VF,
// write down this decision and use it during vectorization.
InstructionCost Cost;
@@ -4810,6 +4866,11 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
InterleaveCost < ScalarizationCost) {
Decision = CM_Interleave;
Cost = InterleaveCost;
+ } else if (ConditionalInvarCost.isValid() &&
+ ConditionalInvarCost <= GatherScatterCost &&
+ ConditionalInvarCost < ScalarizationCost) {
+ Decision = CM_CondInvar;
+ Cost = ConditionalInvarCost;
} else if (GatherScatterCost < ScalarizationCost) {
Decision = CM_GatherScatter;
Cost = GatherScatterCost;
@@ -5458,6 +5519,9 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
: TTI::CastContextHint::Normal;
case LoopVectorizationCostModel::CM_Widen_Reverse:
return TTI::CastContextHint::Reversed;
+ case LoopVectorizationCostModel::CM_CondInvar:
+ return isPredicatedInst(I) ? TTI::CastContextHint::Masked
+ : TTI::CastContextHint::Normal;
case LoopVectorizationCostModel::CM_Unknown:
llvm_unreachable("Instr did not go through cost modelling?");
case LoopVectorizationCostModel::CM_VectorCall:
@@ -6166,6 +6230,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
MaxRuntimeStep);
VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
VPlanTransforms::cse(BestVPlan);
+ VPlanTransforms::cseUniformMemoryReads(BestVPlan);
VPlanTransforms::simplifyRecipes(BestVPlan);
VPlanTransforms::simplifyKnownEVL(BestVPlan, BestVF, PSE);
@@ -6316,6 +6381,41 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
});
}
+VPInstruction *
+VPRecipeBuilder::tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
+ VPValue *Mask, VPInstruction *VPI) {
+ assert(Mask && "CM_CondInvar requires a non-null block mask");
+ LLVMContext &Ctx = Load->getContext();
+ DebugLoc DL = Load->getDebugLoc();
+ Builder.setInsertPoint(VPI);
+
+ // any = AnyOf(BlockMask). Unroll appends per-part masks, so this becomes
+ // a single OR-reduction across all UF parts.
+ VPValue *AnyActive = Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
+
+ // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
+ Type *I1Ty = Type::getInt1Ty(Ctx);
+ VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
+ VPValue *ZeroVMask =
+ Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
+ VPValue *Lane0Idx =
+ Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
+ VPValue *VMask = Builder.createNaryOp(
+ Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
+
+ // Consecutive=true selects the masked-load (not masked-gather) lowering
+ // in VPWidenLoadRecipe::execute. The access is stride-0, not stride-1;
+ // promote to a real AddrKind enum if other passes start relying on
+ // isConsecutive() distinguishing the two.
+ auto *MaskedLoad =
+ new VPWidenLoadRecipe(*Load, Ptr, VMask, /*Consecutive*/ true, *VPI, DL);
+ Builder.insert(MaskedLoad);
+
+ return new VPInstruction(VPInstruction::BroadcastLane,
+ {Lane0Idx, MaskedLoad->getVPSingleValue()},
+ /*Flags*/ {}, /*MD*/ {}, DL);
+}
+
VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VFRange &Range) {
assert((VPI->getOpcode() == Instruction::Load ||
@@ -6381,6 +6481,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
+ if (Decision == LoopVectorizationCostModel::CM_CondInvar && Mask)
+ return tryToWidenCondInvarLoad(Load, Ptr, Mask, VPI);
auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
Load->getDebugLoc());
if (Reverse) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a84c77d614673..86f5c03ae501e 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -42,6 +42,11 @@ class VPRecipeBuilder {
/// Range. The function should not be called for memory instructions or calls.
bool shouldWiden(Instruction *I, VFRange &Range) const;
+ /// Lower a CM_CondInvar \p Load to: single-lane masked-load + broadcast.
+ /// \p Mask is the non-null block mask. Returns the BroadcastLane recipe.
+ VPInstruction *tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
+ VPValue *Mask, VPInstruction *VPI);
+
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 63436c79e9a98..19c286cfd319f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1305,6 +1305,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
LastActiveLane,
// Returns a reversed vector for the operand.
Reverse,
+ // Broadcast lane N of operand 1 across all VF lanes. Operand 0 is a
+ // constant i32 lane index. Operand order matches ExtractLane.
+ BroadcastLane,
// The opcodes below are used for VPInstructionWithType.
//
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index a42b631cd3304..4c3d50d8d8c44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -123,6 +123,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::Unpack:
return SetResultTyFromOp();
case VPInstruction::ExtractLane:
+ case VPInstruction::BroadcastLane:
return inferScalarType(R->getOperand(1));
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 11a91dcd46867..b1a4dac5ccd0d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -516,6 +516,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case Instruction::Store:
case VPInstruction::BranchOnCount:
case VPInstruction::BranchOnTwoConds:
+ case VPInstruction::BroadcastLane:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
case VPInstruction::LogicalOr:
@@ -578,6 +579,38 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
}
}
+/// Broadcast lane \p LaneNo of \p Vec across the result vector.
+/// - Fixed : shufflevector with mask <i32 N, ...>.
+/// - Scalable, lane 0 : shufflevector with zeroinitializer mask.
+/// - Scalable, lane k>0 : extractelement + CreateVectorSplat
+/// (non-zero scalable shuffle masks are rejected
+/// by the verifier today; see
+/// ShuffleVectorInst::isValidOperands).
+static Value *generateBroadcastLane(IRBuilderBase &Builder, Value *Vec,
+ unsigned LaneNo) {
+ auto *VecTy = cast<VectorType>(Vec->getType());
+ Value *Poison = PoisonValue::get(Vec->getType());
+
+ if (isa<ScalableVectorType>(VecTy)) {
+ if (LaneNo == 0) {
+ Type *I32Ty = Type::getInt32Ty(VecTy->getContext());
+ auto *MaskTy = VectorType::get(I32Ty, VecTy->getElementCount());
+ return Builder.CreateShuffleVector(Vec, Poison,
+ Constant::getNullValue(MaskTy),
+ "broadcast.lane");
+ }
+ Value *Scalar =
+ Builder.CreateExtractElement(Vec, Builder.getInt32(LaneNo));
+ return Builder.CreateVectorSplat(VecTy->getElementCount(), Scalar,
+ "broadcast.lane");
+ }
+
+ auto *FVT = cast<FixedVectorType>(VecTy);
+ SmallVector<int, 16> ShufMask(FVT->getNumElements(),
+ static_cast<int>(LaneNo));
+ return Builder.CreateShuffleVector(Vec, Poison, ShufMask, "broadcast.lane");
+}
+
Value *VPInstruction::generate(VPTransformState &State) {
IRBuilderBase &Builder = State.Builder;
@@ -926,6 +959,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
return State.get(getOperand(0), true);
case VPInstruction::Reverse:
return Builder.CreateVectorReverse(State.get(getOperand(0)), "reverse");
+ case VPInstruction::BroadcastLane: {
+ Value *Idx = State.get(getOperand(0), /*IsScalar*/ true);
+ Value *Vec = State.get(getOperand(1));
+ unsigned LaneNo = cast<ConstantInt>(Idx)->getZExtValue();
+ return generateBroadcastLane(Builder, Vec, LaneNo);
+ }
case VPInstruction::ExtractLastActive: {
Value *Result = State.get(getOperand(0), /*IsScalar=*/true);
for (unsigned Idx = 1; Idx < getNumOperands(); Idx += 2) {
@@ -1280,6 +1319,19 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
VectorTy, /*Mask=*/{}, Ctx.CostKind,
/*Index=*/0);
}
+ case VPInstruction::BroadcastLane: {
+ assert(VF.isVector() && "BroadcastLane requires a vector VF");
+ Type *EltTy = Ctx.Types.inferScalarType(this);
+ auto *VectorTy = cast<VectorType>(toVectorTy(EltTy, VF));
+ // Forward the constant lane index so targets can specialise lane-0.
+ unsigned LaneIdx = 0;
+ if (auto *LiveInIdx = getOperand(0)->getLiveInIRValue())
+ if (auto *CI = dyn_cast<ConstantInt>(LiveInIdx))
+ LaneIdx = CI->getZExtValue();
+ return Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
+ VectorTy, /*Mask=*/{}, Ctx.CostKind,
+ /*Index=*/LaneIdx);
+ }
case VPInstruction::ExtractLastLane: {
// Add on the cost of extracting the element.
auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
@@ -1466,6 +1518,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
return false;
case VPInstruction::ExitingIVValue:
case VPInstruction::ExtractLane:
+ case VPInstruction::BroadcastLane:
return Op == getOperand(0);
};
llvm_unreachable("switch should return");
@@ -1595,6 +1648,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::Reverse:
O << "reverse";
break;
+ case VPInstruction::BroadcastLane:
+ O << "broadcast-lane";
+ break;
case VPInstruction::Unpack:
O << "unpack";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32d89a34105a4..e036f823f8a1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2421,6 +2421,42 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
};
} // end anonymous namespace
+/// CSE duplicate masked VPWidenLoadRecipes (which are not VPSingleDefRecipes
+/// and so are missed by the generic cse()). Reset on any write within the
+/// block; unmasked loads are left to downstream EarlyCSE/GVN.
+void VPlanTransforms::cseUniformMemoryReads(VPlan &Plan) {
+ ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
+ Plan.getEntry());
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ SmallVector<VPWidenLoadRecipe *, 4> Candidates;
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (R.mayWriteToMemory()) {
+ Candidates.clear();
+ continue;
+ }
+ auto *Load = dyn_cast<VPWidenLoadRecipe>(&R);
+ if (!Load || !Load->isMasked())
+ continue;
+ VPWidenLoadRecipe *Match = nullptr;
+ for (VPWidenLoadRecipe *C : Candidates) {
+ if (!equal(C->operands(), Load->operands()))
+ continue;
+ // Same ingredient => same alignment/metadata/type.
+ if (&C->getIngredient() != &Load->getIngredient())
+ continue;
+ Match = C;
+ break;
+ }
+ if (Match) {
+ Load->getVPSingleValue()->replaceAllUsesWith(Match->getVPSingleValue());
+ Load->eraseFromParent();
+ continue;
+ }
+ Candidates.push_back(Load);
+ }
+ }
+}
+
/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
/// Plan.
void VPlanTransforms::cse(VPlan &Plan) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 75fc549167e03..0880b4ca4336c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -375,6 +375,10 @@ struct VPlanTransforms {
/// Perform common-subexpression-elimination on \p Plan.
static void cse(VPlan &Plan);
+ /// CSE masked VPWidenLoadRecipes with identical operands inside a basic
+ /// block, ignoring candidates across an intervening write. No AA needed.
+ static void cseUniformMemoryReads(VPlan &Plan);
+
/// If there's a single exit block, optimize its phi recipes that use exiting
/// IV values by feeding them precomputed end values instead, possibly taken
/// one step backwards.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 4b99829a21817..b7f70d7ba2469 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -42,6 +42,10 @@ class VPlanVerifier {
/// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
+ /// Operand 0 of \p BroadcastLane must be a live-in constant integer
+ /// (shufflevector masks are constant-only).
+ bool verifyBroadcastLaneRecipe(const VPInstruction &BroadcastLane) const;
+
bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
bool verifyBlock(const VPBlockBase *VPB);
@@ -199,6 +203,28 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
return true;
}
+bool VPlanVerifier::verifyBroadcastLaneRecipe(
+ const VPInstruction &BroadcastLane) const {
+ assert(BroadcastLane.getOpcode() == VPInstruction::BroadcastLane &&
+ "must be called with VPInstruction::BroadcastLane");
+
+ if (BroadcastLane.getNumOperands() != 2) {
+ errs() << "BroadcastLane must have exactly two operands "
+ "(lane index, vector)\n";
+ return false;
+ }
+
+ const VPValue *LaneIdx = BroadcastLane.getOperand(0);
+ Value *LiveInIdx = LaneIdx->getLiveInIRValue();
+ if (!LiveInIdx || !isa<ConstantInt>(LiveInIdx)) {
+ errs() << "BroadcastLane operand 0 must be a constant integer lane "
+ "index (live-in ConstantInt)\n";
+ return false;
+ }
+
+ return true;
+}
+
bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyPhiRecipes(VPBB))
return false;
@@ -307,6 +333,10 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
if (!verifyLastActiveLaneRecipe(*VPI))
return false;
break;
+ case VPInstruction::BroadcastLane:
+ if (!verifyBroadcastLaneRecipe(*VPI))
+ return false;
+ break;
default:
break;
}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
new file mode 100644
index 0000000000000..43be17e82ec54
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -enable-masked-invariant-load -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S \
+; RUN: | FileCheck %s
+
+;; Two distinct loop-invariant pointers loaded under the same predicate in the
+;; same block. Each load is independently lowered to a single-lane masked load
+;; + broadcast, but they share the SAME AnyOf(mask) and lane-0 mask material
+;; thanks to the standard VPlan CSE that runs before unrolling.
+;;
+;; for (i = 0; i < n; ++i)
+;; if (cond[i])
+;; a[i] = *inv1 + *inv2;
+
+define void @cond_invar_load_two_ptrs(ptr noalias %a, ptr noalias %cond, ptr noalias %inv1, ptr noalias %inv2, i64 %n) {
+; CHECK-LABEL: @cond_invar_load_two_ptrs(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i1> zeroinitializer, i1 [[TMP3]], i32 0
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV1:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT: [[BROADCAST_LANE:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV2:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT: [[BROADCAST_LANE2:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = add <4 x i32> [[BROADCAST_LANE]], [[BROADCAST_LANE2]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <4 x i1> [[TMP1]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT: [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-NEXT: [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK: if.then:
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[INV1]], align 4
+; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[INV2]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: store i32 [[SUM]], ptr [[A_GEP]], align 4
+; CHECK-NEXT: br label [[FOR_INC]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: for.end:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+ %cv = load i32, ptr %cond.gep, align 4
+ %tobool = icmp ne i32 %cv, 0
+ br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+ %v1 = load i32, ptr %inv1, align 4
+ %v2 = load i32, ptr %inv2, align 4
+ %sum = add i32 %v1, %v2
+ %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+ store i32 %sum, ptr %a.gep, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
new file mode 100644
index 0000000000000..e502761b8a7e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-masked-invariant-load -force-vector-width=4 \
+; RUN: -force-vector-interleave=1 -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S \
+; RUN: | FileCheck %s
+
+;; Negative test for CM_CondInvar.
+;;
+;; The conditional load's pointer (%src + i) is loop-VARIANT, so even with
+;; -enable-masked-invariant-load enabled, getConditionalInvarCost() returns
+;; Invalid and the cost model picks the regular consecutive masked load
+;; (CM_Widen).
+;;
+;; for (i = 0; i < n; ++i)
+;; if (cond[i])
+;; a[i] = src[i];
+;;
+;; The output:
+;; * Must contain a regular widened consecutive masked load on src + index.
+;; * Must NOT contain any AnyOf-style i1 OR-reduction.
+;; * Must NOT contain a broadcast.lane shuffle (the CondInvar lowering
+;; marker).
+;; * Must NOT contain a masked.load whose pointer is the loop-invariant
+;; %src argument.
+
+define void @cond_load_not_invariant(ptr noalias %a, ptr noalias %src, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: define void @cond_load_not_invariant(
+
+; The vector body widens the consecutive masked load on src[index].
+; CHECK: vector.body:
+; CHECK: [[SRC_GEP_V:%.*]] = getelementptr i32, ptr %src, i64 {{%.*}}
+; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SRC_GEP_V]],
+
+; The CondInvar lowering must NOT be applied.
+; CHECK-NOT: call i1 @llvm.vector.reduce.or
+; CHECK-NOT: %broadcast.lane = shufflevector
+; CHECK-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 %src,
+
+; CHECK: middle.block:
+; CHECK: ret void
+
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+ %cv = load i32, ptr %cond.gep, align 4
+ %tobool = icmp ne i32 %cv, 0
+ br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+ %src.gep = getelementptr inbounds i32, ptr %src, i64 %i
+ %v = load i32, ptr %src.gep, align 4
+ %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+ store i32 %v, ptr %a.gep, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll
new file mode 100644
index 0000000000000..4a66d907675d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S | FileCheck %s --check-prefix=CHECK-OFF
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN: -enable-masked-invariant-load -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S | FileCheck %s --check-prefix=CHECK-ON
+
+;; Conditional load from a loop-invariant pointer.
+;;
+;; for (i = 0; i < n; ++i)
+;; if (cond[i])
+;; a[i] = *inv;
+;;
+;; The pointer %inv is loop-invariant and the load is predicated by %tobool.
+;; Without -enable-masked-invariant-load the cost model falls back to per-lane
+;; scalarised predicated loads (CM_Scalarize) - one pred.load.if/continue
+;; chain per VF lane.
+;;
+;; With -enable-masked-invariant-load the load becomes:
+;; * AnyOf reduction across the block mask,
+;; * lane-0 mask materialisation (zeroinitializer + insertelement i1),
+;; * one masked.load from %inv,
+;; * shufflevector broadcast of lane 0 to all VF lanes.
+
+define void @cond_invar_load(ptr noalias %a, ptr noalias %cond, ptr noalias %inv, i64 %n) {
+; CHECK-OFF-LABEL: @cond_invar_load(
+; CHECK-OFF-NEXT: entry:
+; CHECK-OFF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-OFF-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-OFF: vector.ph:
+; CHECK-OFF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-OFF-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-OFF-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-OFF: vector.body:
+; CHECK-OFF-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-OFF-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-OFF-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-OFF-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-OFF-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-OFF-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-OFF: pred.load.if:
+; CHECK-OFF-NEXT: [[TMP3:%.*]] = load i32, ptr [[INV:%.*]], align 4
+; CHECK-OFF-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-OFF-NEXT: br label [[PRED_LOAD_CONTINUE]]
+; CHECK-OFF: pred.load.continue:
+; CHECK-OFF-NEXT: [[TMP5:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_LOAD_IF]] ]
+; CHECK-OFF-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-OFF-NEXT: br i1 [[TMP6]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK-OFF: pred.load.if1:
+; CHECK-OFF-NEXT: [[TMP7:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP7]], i64 1
+; CHECK-OFF-NEXT: br label [[PRED_LOAD_CONTINUE2]]
+; CHECK-OFF: pred.load.continue2:
+; CHECK-OFF-NEXT: [[TMP9:%.*]] = phi <4 x i32> [ [[TMP5]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], [[PRED_LOAD_IF1]] ]
+; CHECK-OFF-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-OFF-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK-OFF: pred.load.if3:
+; CHECK-OFF-NEXT: [[TMP11:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP11]], i64 2
+; CHECK-OFF-NEXT: br label [[PRED_LOAD_CONTINUE4]]
+; CHECK-OFF: pred.load.continue4:
+; CHECK-OFF-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP12]], [[PRED_LOAD_IF3]] ]
+; CHECK-OFF-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-OFF-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK-OFF: pred.load.if5:
+; CHECK-OFF-NEXT: [[TMP15:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP15]], i64 3
+; CHECK-OFF-NEXT: br label [[PRED_LOAD_CONTINUE6]]
+; CHECK-OFF: pred.load.continue6:
+; CHECK-OFF-NEXT: [[TMP17:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP16]], [[PRED_LOAD_IF5]] ]
+; CHECK-OFF-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-OFF-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP17]], ptr align 4 [[TMP18]], <4 x i1> [[TMP1]])
+; CHECK-OFF-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-OFF-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-OFF-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-OFF: middle.block:
+; CHECK-OFF-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-OFF-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-OFF: scalar.ph:
+; CHECK-OFF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-OFF-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK-OFF: for.body:
+; CHECK-OFF-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-OFF-NEXT: [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-OFF-NEXT: [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-OFF-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-OFF-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-OFF: if.then:
+; CHECK-OFF-NEXT: [[V:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT: [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-OFF-NEXT: store i32 [[V]], ptr [[A_GEP]], align 4
+; CHECK-OFF-NEXT: br label [[FOR_INC]]
+; CHECK-OFF: for.inc:
+; CHECK-OFF-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-OFF-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-OFF-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-OFF: for.end:
+; CHECK-OFF-NEXT: ret void
+;
+; CHECK-ON-LABEL: @cond_invar_load(
+; CHECK-ON-NEXT: entry:
+; CHECK-ON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-ON-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-ON: vector.ph:
+; CHECK-ON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-ON-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-ON-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-ON: vector.body:
+; CHECK-ON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ON-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-ON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-ON-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-ON-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-ON-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-ON-NEXT: [[TMP4:%.*]] = insertelement <4 x i1> zeroinitializer, i1 [[TMP3]], i32 0
+; CHECK-ON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-ON-NEXT: [[BROADCAST_LANE:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-ON-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-ON-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_LANE]], ptr align 4 [[TMP5]], <4 x i1> [[TMP1]])
+; CHECK-ON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-ON-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ON-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-ON: middle.block:
+; CHECK-ON-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-ON-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-ON: scalar.ph:
+; CHECK-ON-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-ON-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK-ON: for.body:
+; CHECK-ON-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-ON-NEXT: [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-ON-NEXT: [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-ON-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-ON-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-ON: if.then:
+; CHECK-ON-NEXT: [[V:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-ON-NEXT: [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-ON-NEXT: store i32 [[V]], ptr [[A_GEP]], align 4
+; CHECK-ON-NEXT: br label [[FOR_INC]]
+; CHECK-ON: for.inc:
+; CHECK-ON-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-ON-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-ON-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-ON: for.end:
+; CHECK-ON-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+ %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+ %cv = load i32, ptr %cond.gep, align 4
+ %tobool = icmp ne i32 %cv, 0
+ br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+ %v = load i32, ptr %inv, align 4
+ %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+ store i32 %v, ptr %a.gep, align 4
+ br label %for.inc
+
+for.inc:
+ %i.next = add nuw nsw i64 %i, 1
+ %exitcond = icmp eq i64 %i.next, %n
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+ ret void
+}
>From 3cb56a8c199ca0e461e41f7fdf07fdb73a534a60 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 2 Jun 2026 17:59:32 +0530
Subject: [PATCH 2/2] Replace CM_CondInvar and its recipe-builder dispatch with
a VPlan transform widenPredicatedInvariantLoads, run after
sinkPredicatedStores. It rewrites masked replicate loads from loop-invariant
addresses into AnyOf(mask) + masked VPWidenLoadRecipe + BroadcastLane(0);
---
.../Transforms/Vectorize/LoopVectorize.cpp | 106 +-----------------
.../Transforms/Vectorize/VPRecipeBuilder.h | 5 -
.../Transforms/Vectorize/VPlanTransforms.cpp | 78 +++++++++++++
.../Transforms/Vectorize/VPlanTransforms.h | 7 ++
.../VPlan/vplan-print-after-all.ll | 1 +
5 files changed, 89 insertions(+), 108 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 064c1498e207d..b2589f4641bc5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,11 +198,6 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
"vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
cl::desc("The maximum allowed number of runtime memory checks"));
-static cl::opt<bool> EnableMaskedInvariantLoad(
- "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
- cl::desc("Vectorize conditional invariant load with mask-load"
- " and broadcast"));
-
/// Option tail-folding-policy indicates that an epilogue is undesired, that
/// tail folding is preferred, and this lists all options. I.e., the vectorizer
/// will try to fold the tail-loop (epilogue) into the vector body and predicate
@@ -934,8 +929,7 @@ class LoopVectorizationCostModel {
CM_GatherScatter,
CM_Scalarize,
CM_VectorCall,
- CM_IntrinsicCall,
- CM_CondInvar,
+ CM_IntrinsicCall
};
/// Save vectorization decision \p W and \p Cost taken by the cost model for
@@ -1336,10 +1330,6 @@ class LoopVectorizationCostModel {
/// The cost computation for Gather/Scatter instruction.
InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
- /// Cost of lowering a predicated loop-invariant load as a single-lane
- /// masked load + broadcast. Returns getInvalid() if not applicable.
- InstructionCost getConditionalInvarCost(Instruction *I, ElementCount VF);
-
/// The cost computation for widening instruction \p I with consecutive
/// memory access.
InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
@@ -2724,11 +2714,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
if (IsUniformMemOpUse(I))
return true;
- // CM_CondInvar uses its scalar invariant pointer uniformly.
return (WideningDecision == CM_Widen ||
WideningDecision == CM_Widen_Reverse ||
- WideningDecision == CM_Interleave ||
- WideningDecision == CM_CondInvar);
+ WideningDecision == CM_Interleave);
};
// Returns true if Ptr is the pointer operand of a memory access instruction
@@ -4409,49 +4397,6 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
Config.CostKind);
}
-InstructionCost
-LoopVectorizationCostModel::getConditionalInvarCost(Instruction *I,
- ElementCount VF) {
- // Applicability checks. Any failure returns Invalid; the cost ladder
- // already gates on .isValid().
- if (!EnableMaskedInvariantLoad)
- return InstructionCost::getInvalid();
- if (!Legal->blockNeedsPredication(I->getParent()))
- return InstructionCost::getInvalid();
- if (!isMaskRequired(I))
- return InstructionCost::getInvalid();
- if (!isa<LoadInst>(I))
- return InstructionCost::getInvalid();
- const Align Alignment = getLoadStoreAlignment(I);
- unsigned AS = getLoadStoreAddressSpace(I);
- if (!TTI.isLegalMaskedLoad(I->getType(), Alignment, AS))
- return InstructionCost::getInvalid();
- Value *Ptr = getLoadStorePointerOperand(I);
- if (!Legal->isInvariant(Ptr))
- return InstructionCost::getInvalid();
-
- // Components: AnyOf or-reduce + lane-0 mask materialisation
- // (zero-splat + insertelement) + masked load + lane-0 broadcast shuffle.
- // InstructionCost arithmetic propagates Invalid across the chain.
- Type *ValTy = getLoadStoreType(I);
- auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
- auto *MaskTy = cast<VectorType>(
- toVectorTy(Type::getInt1Ty(I->getContext()), VF));
- InstructionCost Cost = TTI.getArithmeticReductionCost(
- Instruction::Or, MaskTy, /*FMF*/ std::nullopt, Config.CostKind);
- Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, MaskTy, MaskTy,
- {}, Config.CostKind, 0);
- Cost += TTI.getVectorInstrCost(Instruction::InsertElement, MaskTy,
- Config.CostKind, /*Index=*/0);
- Cost += TTI.getMemIntrinsicInstrCost(
- MemIntrinsicCostAttributes(Intrinsic::masked_load, VectorTy, Alignment,
- AS),
- Config.CostKind);
- Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
- VectorTy, {}, Config.CostKind, 0);
- return Cost;
-}
-
InstructionCost
LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
ElementCount VF) {
@@ -4857,7 +4802,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
InstructionCost ScalarizationCost =
getMemInstScalarizationCost(&I, VF) * NumAccesses;
- InstructionCost ConditionalInvarCost = getConditionalInvarCost(&I, VF);
// Choose better solution for the current VF,
// write down this decision and use it during vectorization.
InstructionCost Cost;
@@ -4866,11 +4810,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
InterleaveCost < ScalarizationCost) {
Decision = CM_Interleave;
Cost = InterleaveCost;
- } else if (ConditionalInvarCost.isValid() &&
- ConditionalInvarCost <= GatherScatterCost &&
- ConditionalInvarCost < ScalarizationCost) {
- Decision = CM_CondInvar;
- Cost = ConditionalInvarCost;
} else if (GatherScatterCost < ScalarizationCost) {
Decision = CM_GatherScatter;
Cost = GatherScatterCost;
@@ -5519,9 +5458,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
: TTI::CastContextHint::Normal;
case LoopVectorizationCostModel::CM_Widen_Reverse:
return TTI::CastContextHint::Reversed;
- case LoopVectorizationCostModel::CM_CondInvar:
- return isPredicatedInst(I) ? TTI::CastContextHint::Masked
- : TTI::CastContextHint::Normal;
case LoopVectorizationCostModel::CM_Unknown:
llvm_unreachable("Instr did not go through cost modelling?");
case LoopVectorizationCostModel::CM_VectorCall:
@@ -6381,41 +6317,6 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
});
}
-VPInstruction *
-VPRecipeBuilder::tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
- VPValue *Mask, VPInstruction *VPI) {
- assert(Mask && "CM_CondInvar requires a non-null block mask");
- LLVMContext &Ctx = Load->getContext();
- DebugLoc DL = Load->getDebugLoc();
- Builder.setInsertPoint(VPI);
-
- // any = AnyOf(BlockMask). Unroll appends per-part masks, so this becomes
- // a single OR-reduction across all UF parts.
- VPValue *AnyActive = Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
-
- // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
- Type *I1Ty = Type::getInt1Ty(Ctx);
- VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
- VPValue *ZeroVMask =
- Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
- VPValue *Lane0Idx =
- Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
- VPValue *VMask = Builder.createNaryOp(
- Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
-
- // Consecutive=true selects the masked-load (not masked-gather) lowering
- // in VPWidenLoadRecipe::execute. The access is stride-0, not stride-1;
- // promote to a real AddrKind enum if other passes start relying on
- // isConsecutive() distinguishing the two.
- auto *MaskedLoad =
- new VPWidenLoadRecipe(*Load, Ptr, VMask, /*Consecutive*/ true, *VPI, DL);
- Builder.insert(MaskedLoad);
-
- return new VPInstruction(VPInstruction::BroadcastLane,
- {Lane0Idx, MaskedLoad->getVPSingleValue()},
- /*Flags*/ {}, /*MD*/ {}, DL);
-}
-
VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
VFRange &Range) {
assert((VPI->getOpcode() == Instruction::Load ||
@@ -6481,8 +6382,6 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
- if (Decision == LoopVectorizationCostModel::CM_CondInvar && Mask)
- return tryToWidenCondInvarLoad(Load, Ptr, Mask, VPI);
auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
Load->getDebugLoc());
if (Reverse) {
@@ -6923,6 +6822,7 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
// Now optimize the initial VPlan.
RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
+ RUN_VPLAN_PASS(VPlanTransforms::widenPredicatedInvariantLoads, *Plan, TTI);
RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
Config.getMinimalBitwidths());
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 86f5c03ae501e..a84c77d614673 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -42,11 +42,6 @@ class VPRecipeBuilder {
/// Range. The function should not be called for memory instructions or calls.
bool shouldWiden(Instruction *I, VFRange &Range) const;
- /// Lower a CM_CondInvar \p Load to: single-lane masked-load + broadcast.
- /// \p Mask is the non-null block mask. Returns the BroadcastLane recipe.
- VPInstruction *tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
- VPValue *Mask, VPInstruction *VPI);
-
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e036f823f8a1f..c150b94361fb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4767,6 +4767,84 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
}
}
+static cl::opt<bool> EnableMaskedInvariantLoad(
+ "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
+ cl::desc("Rewrite predicated loads from loop-invariant addresses as a "
+ "single-lane masked load + broadcast"));
+
+void VPlanTransforms::widenPredicatedInvariantLoads(
+ VPlan &Plan, const TargetTransformInfo &TTI) {
+ if (!EnableMaskedInvariantLoad)
+ return;
+
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ if (!LoopRegion)
+ return;
+
+ // Collect masked, vector-producing replicate loads whose pointer is defined
+ // outside the loop region and which the target can lower as a masked load.
+ SmallVector<VPReplicateRecipe *> Worklist;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(LoopRegion->getEntry()))) {
+ for (VPRecipeBase &R : *VPBB) {
+ auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+ if (!RepR || !RepR->isPredicated() || RepR->isSingleScalar())
+ continue;
+ if (RepR->getOpcode() != Instruction::Load)
+ continue;
+ VPValue *Addr = RepR->getOperand(0);
+ if (!Addr->isDefinedOutsideLoopRegions())
+ continue;
+ auto *LI = cast<LoadInst>(RepR->getUnderlyingInstr());
+ if (!TTI.isLegalMaskedLoad(LI->getType(), LI->getAlign(),
+ LI->getPointerAddressSpace()))
+ continue;
+ Worklist.push_back(RepR);
+ }
+ }
+
+ for (VPReplicateRecipe *RepR : Worklist) {
+ auto *LI = cast<LoadInst>(RepR->getUnderlyingInstr());
+ LLVMContext &Ctx = LI->getContext();
+ DebugLoc DL = LI->getDebugLoc();
+ VPValue *Addr = RepR->getOperand(0);
+ VPValue *Mask = RepR->getMask();
+
+ VPBuilder Builder(RepR);
+
+ // any = AnyOf(mask). Unroll appends per-part masks, so this becomes a
+ // single OR-reduction across all UF parts.
+ VPValue *AnyActive =
+ Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
+
+ // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
+ Type *I1Ty = Type::getInt1Ty(Ctx);
+ VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
+ VPValue *ZeroVMask =
+ Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
+ VPValue *Lane0Idx =
+ Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
+ VPValue *VMask = Builder.createNaryOp(
+ Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
+
+ // Consecutive=true selects the masked-load (not masked-gather) lowering in
+ // VPWidenLoadRecipe::execute. The access is stride-0, not stride-1; the
+ // BroadcastLane below splats lane 0 across all VF lanes.
+ auto *MaskedLoad = new VPWidenLoadRecipe(*LI, Addr, VMask,
+ /*Consecutive*/ true, *RepR, DL);
+ Builder.insert(MaskedLoad);
+
+ auto *Bcast = new VPInstruction(
+ VPInstruction::BroadcastLane,
+ {Lane0Idx, MaskedLoad->getVPSingleValue()}, /*Flags*/ {},
+ /*MD*/ {}, DL);
+ Builder.insert(Bcast);
+
+ RepR->replaceAllUsesWith(Bcast);
+ RepR->eraseFromParent();
+ }
+}
+
static bool
canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
PredicatedScalarEvolution &PSE, const Loop &L,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0880b4ca4336c..4fe578692ba16 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -400,6 +400,13 @@ struct VPlanTransforms {
static void hoistPredicatedLoads(VPlan &Plan, PredicatedScalarEvolution &PSE,
const Loop *L);
+ /// Rewrite predicated VPReplicateRecipe loads from a loop-invariant address
+ /// into a single-lane masked load + broadcast pattern. Only applied when
+ /// the target reports isLegalMaskedLoad for the element type. Gated by
+ /// -enable-masked-invariant-load.
+ static void widenPredicatedInvariantLoads(VPlan &Plan,
+ const TargetTransformInfo &TTI);
+
/// Sink predicated stores to the same address with complementary predicates
/// (P and NOT P) to an unconditional store with select recipes for the
/// stored values. This eliminates branching overhead when all paths
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 061588317eba7..7f31efb6fc5fc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -29,6 +29,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::dropPoisonGeneratingRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::hoistPredicatedLoads
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::sinkPredicatedStores
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::widenPredicatedInvariantLoads
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::truncateToMinimalBitwidths
; CHECK: VPlan for loop in 'foo' after removeRedundantInductionCasts
; CHECK: VPlan for loop in 'foo' after reassociateHeaderMask
More information about the llvm-commits
mailing list