[llvm] [LV] Vectorize predicated loop-invariant loads via masked load + broadcast (PR #200361)

Ashutosh Nema via llvm-commits llvm-commits at lists.llvm.org
Tue Jun 2 05:32:32 PDT 2026


https://github.com/nema-ashutosh updated https://github.com/llvm/llvm-project/pull/200361

>From fca0eb65184ecd8be61689ab7bd3d00b3410d777 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Fri, 29 May 2026 15:04:07 +0530
Subject: [PATCH 1/2] [LV] Vectorize predicated loop-invariant loads via masked
 load + broadcast

Today, a predicated load from a loop-invariant address is scalarised
inside a replicate region: every active lane independently issues the
same load and the results are reassembled into a vector. The cost
model assigns this a very high cost, which both blocks
auto-vectorization

and produces poor codegen when vectorization is forced.
This patch teaches the loop vectorizer to instead emit a single
masked load of one lane from the invariant pointer, guarded by an
any-active-lane reduction over the block mask, and broadcast the
result across the vector.
  * New cost-model decision CM_CondInvar and helper
    getConditionalInvarCost, picked when it beats gather/scatter
    and scalarisation.
  * New VPInstruction opcode BroadcastLane (dual of ExtractLane),
    with type inference, cost, printing and a verifier check.
  * New VPlan transform cseUniformMemoryReads, to fold the
    structurally identical masked loads produced for each UF part
    (VPWidenLoadRecipe is missed by the generic cse).
  * Gated by -enable-masked-invariant-load (off by default).

Fixes #200355.
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 106 ++++++++++-
 .../Transforms/Vectorize/VPRecipeBuilder.h    |   5 +
 llvm/lib/Transforms/Vectorize/VPlan.h         |   3 +
 .../Transforms/Vectorize/VPlanAnalysis.cpp    |   1 +
 .../lib/Transforms/Vectorize/VPlanRecipes.cpp |  56 ++++++
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  36 ++++
 .../Transforms/Vectorize/VPlanTransforms.h    |   4 +
 .../Transforms/Vectorize/VPlanVerifier.cpp    |  30 ++++
 .../enable-masked-invariant-load-multiple.ll  |  93 ++++++++++
 ...ble-masked-invariant-load-not-invariant.ll |  64 +++++++
 .../X86/enable-masked-invariant-load.ll       | 169 ++++++++++++++++++
 11 files changed, 565 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 61c2d3cd228ec..064c1498e207d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,6 +198,11 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
     "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
     cl::desc("The maximum allowed number of runtime memory checks"));
 
+static cl::opt<bool> EnableMaskedInvariantLoad(
+    "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
+    cl::desc("Vectorize conditional invariant load with mask-load"
+             " and broadcast"));
+
 /// Option tail-folding-policy indicates that an epilogue is undesired, that
 /// tail folding is preferred, and this lists all options. I.e., the vectorizer
 /// will try to fold the tail-loop (epilogue) into the vector body and predicate
@@ -929,7 +934,8 @@ class LoopVectorizationCostModel {
     CM_GatherScatter,
     CM_Scalarize,
     CM_VectorCall,
-    CM_IntrinsicCall
+    CM_IntrinsicCall,
+    CM_CondInvar,
   };
 
   /// Save vectorization decision \p W and \p Cost taken by the cost model for
@@ -1330,6 +1336,10 @@ class LoopVectorizationCostModel {
   /// The cost computation for Gather/Scatter instruction.
   InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
 
+  /// Cost of lowering a predicated loop-invariant load as a single-lane
+  /// masked load + broadcast. Returns getInvalid() if not applicable.
+  InstructionCost getConditionalInvarCost(Instruction *I, ElementCount VF);
+
   /// The cost computation for widening instruction \p I with consecutive
   /// memory access.
   InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
@@ -2714,9 +2724,11 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
     if (IsUniformMemOpUse(I))
       return true;
 
+    // CM_CondInvar uses its scalar invariant pointer uniformly.
     return (WideningDecision == CM_Widen ||
             WideningDecision == CM_Widen_Reverse ||
-            WideningDecision == CM_Interleave);
+            WideningDecision == CM_Interleave ||
+            WideningDecision == CM_CondInvar);
   };
 
   // Returns true if Ptr is the pointer operand of a memory access instruction
@@ -4397,6 +4409,49 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
              Config.CostKind);
 }
 
+InstructionCost
+LoopVectorizationCostModel::getConditionalInvarCost(Instruction *I,
+                                                    ElementCount VF) {
+  // Applicability checks. Any failure returns Invalid; the cost ladder
+  // already gates on .isValid().
+  if (!EnableMaskedInvariantLoad)
+    return InstructionCost::getInvalid();
+  if (!Legal->blockNeedsPredication(I->getParent()))
+    return InstructionCost::getInvalid();
+  if (!isMaskRequired(I))
+    return InstructionCost::getInvalid();
+  if (!isa<LoadInst>(I))
+    return InstructionCost::getInvalid();
+  const Align Alignment = getLoadStoreAlignment(I);
+  unsigned AS = getLoadStoreAddressSpace(I);
+  if (!TTI.isLegalMaskedLoad(I->getType(), Alignment, AS))
+    return InstructionCost::getInvalid();
+  Value *Ptr = getLoadStorePointerOperand(I);
+  if (!Legal->isInvariant(Ptr))
+    return InstructionCost::getInvalid();
+
+  // Components: AnyOf or-reduce + lane-0 mask materialisation
+  // (zero-splat + insertelement) + masked load + lane-0 broadcast shuffle.
+  // InstructionCost arithmetic propagates Invalid across the chain.
+  Type *ValTy = getLoadStoreType(I);
+  auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
+  auto *MaskTy = cast<VectorType>(
+      toVectorTy(Type::getInt1Ty(I->getContext()), VF));
+  InstructionCost Cost = TTI.getArithmeticReductionCost(
+      Instruction::Or, MaskTy, /*FMF*/ std::nullopt, Config.CostKind);
+  Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, MaskTy, MaskTy,
+                             {}, Config.CostKind, 0);
+  Cost += TTI.getVectorInstrCost(Instruction::InsertElement, MaskTy,
+                                 Config.CostKind, /*Index=*/0);
+  Cost += TTI.getMemIntrinsicInstrCost(
+      MemIntrinsicCostAttributes(Intrinsic::masked_load, VectorTy, Alignment,
+                                 AS),
+      Config.CostKind);
+  Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
+                             VectorTy, {}, Config.CostKind, 0);
+  return Cost;
+}
+
 InstructionCost
 LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
                                                    ElementCount VF) {
@@ -4802,6 +4857,7 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
       InstructionCost ScalarizationCost =
           getMemInstScalarizationCost(&I, VF) * NumAccesses;
 
+      InstructionCost ConditionalInvarCost = getConditionalInvarCost(&I, VF);
       // Choose better solution for the current VF,
       // write down this decision and use it during vectorization.
       InstructionCost Cost;
@@ -4810,6 +4866,11 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
           InterleaveCost < ScalarizationCost) {
         Decision = CM_Interleave;
         Cost = InterleaveCost;
+      } else if (ConditionalInvarCost.isValid() &&
+                 ConditionalInvarCost <= GatherScatterCost &&
+                 ConditionalInvarCost < ScalarizationCost) {
+        Decision = CM_CondInvar;
+        Cost = ConditionalInvarCost;
       } else if (GatherScatterCost < ScalarizationCost) {
         Decision = CM_GatherScatter;
         Cost = GatherScatterCost;
@@ -5458,6 +5519,9 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
                                    : TTI::CastContextHint::Normal;
       case LoopVectorizationCostModel::CM_Widen_Reverse:
         return TTI::CastContextHint::Reversed;
+      case LoopVectorizationCostModel::CM_CondInvar:
+        return isPredicatedInst(I) ? TTI::CastContextHint::Masked
+                                   : TTI::CastContextHint::Normal;
       case LoopVectorizationCostModel::CM_Unknown:
         llvm_unreachable("Instr did not go through cost modelling?");
       case LoopVectorizationCostModel::CM_VectorCall:
@@ -6166,6 +6230,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
       MaxRuntimeStep);
   VPlanTransforms::materializeFactors(BestVPlan, VectorPH, BestVF);
   VPlanTransforms::cse(BestVPlan);
+  VPlanTransforms::cseUniformMemoryReads(BestVPlan);
   VPlanTransforms::simplifyRecipes(BestVPlan);
   VPlanTransforms::simplifyKnownEVL(BestVPlan, BestVF, PSE);
 
@@ -6316,6 +6381,41 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
   });
 }
 
+VPInstruction *
+VPRecipeBuilder::tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
+                                         VPValue *Mask, VPInstruction *VPI) {
+  assert(Mask && "CM_CondInvar requires a non-null block mask");
+  LLVMContext &Ctx = Load->getContext();
+  DebugLoc DL = Load->getDebugLoc();
+  Builder.setInsertPoint(VPI);
+
+  // any = AnyOf(BlockMask). Unroll appends per-part masks, so this becomes
+  // a single OR-reduction across all UF parts.
+  VPValue *AnyActive = Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
+
+  // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
+  Type *I1Ty = Type::getInt1Ty(Ctx);
+  VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
+  VPValue *ZeroVMask =
+      Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
+  VPValue *Lane0Idx =
+      Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
+  VPValue *VMask = Builder.createNaryOp(
+      Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
+
+  // Consecutive=true selects the masked-load (not masked-gather) lowering
+  // in VPWidenLoadRecipe::execute. The access is stride-0, not stride-1;
+  // promote to a real AddrKind enum if other passes start relying on
+  // isConsecutive() distinguishing the two.
+  auto *MaskedLoad =
+      new VPWidenLoadRecipe(*Load, Ptr, VMask, /*Consecutive*/ true, *VPI, DL);
+  Builder.insert(MaskedLoad);
+
+  return new VPInstruction(VPInstruction::BroadcastLane,
+                           {Lane0Idx, MaskedLoad->getVPSingleValue()},
+                           /*Flags*/ {}, /*MD*/ {}, DL);
+}
+
 VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
                                                 VFRange &Range) {
   assert((VPI->getOpcode() == Instruction::Load ||
@@ -6381,6 +6481,8 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
 
   if (VPI->getOpcode() == Instruction::Load) {
     auto *Load = cast<LoadInst>(I);
+    if (Decision == LoopVectorizationCostModel::CM_CondInvar && Mask)
+      return tryToWidenCondInvarLoad(Load, Ptr, Mask, VPI);
     auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
                                         Load->getDebugLoc());
     if (Reverse) {
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a84c77d614673..86f5c03ae501e 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -42,6 +42,11 @@ class VPRecipeBuilder {
   /// Range. The function should not be called for memory instructions or calls.
   bool shouldWiden(Instruction *I, VFRange &Range) const;
 
+  /// Lower a CM_CondInvar \p Load to: single-lane masked-load + broadcast.
+  /// \p Mask is the non-null block mask. Returns the BroadcastLane recipe.
+  VPInstruction *tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
+                                         VPValue *Mask, VPInstruction *VPI);
+
   /// Optimize the special case where the operand of \p VPI is a constant
   /// integer induction variable.
   VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 63436c79e9a98..19c286cfd319f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1305,6 +1305,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
     LastActiveLane,
     // Returns a reversed vector for the operand.
     Reverse,
+    // Broadcast lane N of operand 1 across all VF lanes. Operand 0 is a
+    // constant i32 lane index. Operand order matches ExtractLane.
+    BroadcastLane,
 
     // The opcodes below are used for VPInstructionWithType.
     //
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index a42b631cd3304..4c3d50d8d8c44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -123,6 +123,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case VPInstruction::Unpack:
     return SetResultTyFromOp();
   case VPInstruction::ExtractLane:
+  case VPInstruction::BroadcastLane:
     return inferScalarType(R->getOperand(1));
   case VPInstruction::FirstActiveLane:
   case VPInstruction::LastActiveLane:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 11a91dcd46867..b1a4dac5ccd0d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -516,6 +516,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
   case Instruction::Store:
   case VPInstruction::BranchOnCount:
   case VPInstruction::BranchOnTwoConds:
+  case VPInstruction::BroadcastLane:
   case VPInstruction::FirstOrderRecurrenceSplice:
   case VPInstruction::LogicalAnd:
   case VPInstruction::LogicalOr:
@@ -578,6 +579,38 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
   }
 }
 
+/// Broadcast lane \p LaneNo of \p Vec across the result vector.
+///   - Fixed              : shufflevector with mask <i32 N, ...>.
+///   - Scalable, lane 0   : shufflevector with zeroinitializer mask.
+///   - Scalable, lane k>0 : extractelement + CreateVectorSplat
+///                          (non-zero scalable shuffle masks are rejected
+///                          by the verifier today; see
+///                          ShuffleVectorInst::isValidOperands).
+static Value *generateBroadcastLane(IRBuilderBase &Builder, Value *Vec,
+                                    unsigned LaneNo) {
+  auto *VecTy = cast<VectorType>(Vec->getType());
+  Value *Poison = PoisonValue::get(Vec->getType());
+
+  if (isa<ScalableVectorType>(VecTy)) {
+    if (LaneNo == 0) {
+      Type *I32Ty = Type::getInt32Ty(VecTy->getContext());
+      auto *MaskTy = VectorType::get(I32Ty, VecTy->getElementCount());
+      return Builder.CreateShuffleVector(Vec, Poison,
+                                         Constant::getNullValue(MaskTy),
+                                         "broadcast.lane");
+    }
+    Value *Scalar =
+        Builder.CreateExtractElement(Vec, Builder.getInt32(LaneNo));
+    return Builder.CreateVectorSplat(VecTy->getElementCount(), Scalar,
+                                     "broadcast.lane");
+  }
+
+  auto *FVT = cast<FixedVectorType>(VecTy);
+  SmallVector<int, 16> ShufMask(FVT->getNumElements(),
+                                static_cast<int>(LaneNo));
+  return Builder.CreateShuffleVector(Vec, Poison, ShufMask, "broadcast.lane");
+}
+
 Value *VPInstruction::generate(VPTransformState &State) {
   IRBuilderBase &Builder = State.Builder;
 
@@ -926,6 +959,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
     return State.get(getOperand(0), true);
   case VPInstruction::Reverse:
     return Builder.CreateVectorReverse(State.get(getOperand(0)), "reverse");
+  case VPInstruction::BroadcastLane: {
+    Value *Idx = State.get(getOperand(0), /*IsScalar*/ true);
+    Value *Vec = State.get(getOperand(1));
+    unsigned LaneNo = cast<ConstantInt>(Idx)->getZExtValue();
+    return generateBroadcastLane(Builder, Vec, LaneNo);
+  }
   case VPInstruction::ExtractLastActive: {
     Value *Result = State.get(getOperand(0), /*IsScalar=*/true);
     for (unsigned Idx = 1; Idx < getNumOperands(); Idx += 2) {
@@ -1280,6 +1319,19 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
                                   VectorTy, /*Mask=*/{}, Ctx.CostKind,
                                   /*Index=*/0);
   }
+  case VPInstruction::BroadcastLane: {
+    assert(VF.isVector() && "BroadcastLane requires a vector VF");
+    Type *EltTy = Ctx.Types.inferScalarType(this);
+    auto *VectorTy = cast<VectorType>(toVectorTy(EltTy, VF));
+    // Forward the constant lane index so targets can specialise lane-0.
+    unsigned LaneIdx = 0;
+    if (auto *LiveInIdx = getOperand(0)->getLiveInIRValue())
+      if (auto *CI = dyn_cast<ConstantInt>(LiveInIdx))
+        LaneIdx = CI->getZExtValue();
+    return Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
+                                  VectorTy, /*Mask=*/{}, Ctx.CostKind,
+                                  /*Index=*/LaneIdx);
+  }
   case VPInstruction::ExtractLastLane: {
     // Add on the cost of extracting the element.
     auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
@@ -1466,6 +1518,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
     return false;
   case VPInstruction::ExitingIVValue:
   case VPInstruction::ExtractLane:
+  case VPInstruction::BroadcastLane:
     return Op == getOperand(0);
   };
   llvm_unreachable("switch should return");
@@ -1595,6 +1648,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
   case VPInstruction::Reverse:
     O << "reverse";
     break;
+  case VPInstruction::BroadcastLane:
+    O << "broadcast-lane";
+    break;
   case VPInstruction::Unpack:
     O << "unpack";
     break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 32d89a34105a4..e036f823f8a1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2421,6 +2421,42 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
 };
 } // end anonymous namespace
 
+/// CSE duplicate masked VPWidenLoadRecipes (which are not VPSingleDefRecipes
+/// and so are missed by the generic cse()). Reset on any write within the
+/// block; unmasked loads are left to downstream EarlyCSE/GVN.
+void VPlanTransforms::cseUniformMemoryReads(VPlan &Plan) {
+  ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
+      Plan.getEntry());
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+    SmallVector<VPWidenLoadRecipe *, 4> Candidates;
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+      if (R.mayWriteToMemory()) {
+        Candidates.clear();
+        continue;
+      }
+      auto *Load = dyn_cast<VPWidenLoadRecipe>(&R);
+      if (!Load || !Load->isMasked())
+        continue;
+      VPWidenLoadRecipe *Match = nullptr;
+      for (VPWidenLoadRecipe *C : Candidates) {
+        if (!equal(C->operands(), Load->operands()))
+          continue;
+        // Same ingredient => same alignment/metadata/type.
+        if (&C->getIngredient() != &Load->getIngredient())
+          continue;
+        Match = C;
+        break;
+      }
+      if (Match) {
+        Load->getVPSingleValue()->replaceAllUsesWith(Match->getVPSingleValue());
+        Load->eraseFromParent();
+        continue;
+      }
+      Candidates.push_back(Load);
+    }
+  }
+}
+
 /// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
 /// Plan.
 void VPlanTransforms::cse(VPlan &Plan) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 75fc549167e03..0880b4ca4336c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -375,6 +375,10 @@ struct VPlanTransforms {
   /// Perform common-subexpression-elimination on \p Plan.
   static void cse(VPlan &Plan);
 
+  /// CSE masked VPWidenLoadRecipes with identical operands inside a basic
+  /// block, ignoring candidates across an intervening write. No AA needed.
+  static void cseUniformMemoryReads(VPlan &Plan);
+
   /// If there's a single exit block, optimize its phi recipes that use exiting
   /// IV values by feeding them precomputed end values instead, possibly taken
   /// one step backwards.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 4b99829a21817..b7f70d7ba2469 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -42,6 +42,10 @@ class VPlanVerifier {
   /// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask.
   bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const;
 
+  /// Operand 0 of \p BroadcastLane must be a live-in constant integer
+  /// (shufflevector masks are constant-only).
+  bool verifyBroadcastLaneRecipe(const VPInstruction &BroadcastLane) const;
+
   bool verifyVPBasicBlock(const VPBasicBlock *VPBB);
 
   bool verifyBlock(const VPBlockBase *VPB);
@@ -199,6 +203,28 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
   return true;
 }
 
+bool VPlanVerifier::verifyBroadcastLaneRecipe(
+    const VPInstruction &BroadcastLane) const {
+  assert(BroadcastLane.getOpcode() == VPInstruction::BroadcastLane &&
+         "must be called with VPInstruction::BroadcastLane");
+
+  if (BroadcastLane.getNumOperands() != 2) {
+    errs() << "BroadcastLane must have exactly two operands "
+              "(lane index, vector)\n";
+    return false;
+  }
+
+  const VPValue *LaneIdx = BroadcastLane.getOperand(0);
+  Value *LiveInIdx = LaneIdx->getLiveInIRValue();
+  if (!LiveInIdx || !isa<ConstantInt>(LiveInIdx)) {
+    errs() << "BroadcastLane operand 0 must be a constant integer lane "
+              "index (live-in ConstantInt)\n";
+    return false;
+  }
+
+  return true;
+}
+
 bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
   if (!verifyPhiRecipes(VPBB))
     return false;
@@ -307,6 +333,10 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
         if (!verifyLastActiveLaneRecipe(*VPI))
           return false;
         break;
+      case VPInstruction::BroadcastLane:
+        if (!verifyBroadcastLaneRecipe(*VPI))
+          return false;
+        break;
       default:
         break;
       }
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
new file mode 100644
index 0000000000000..43be17e82ec54
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-multiple.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -enable-masked-invariant-load -force-vector-width=4 \
+; RUN:     -force-vector-interleave=1 -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S \
+; RUN:     | FileCheck %s
+
+;; Two distinct loop-invariant pointers loaded under the same predicate in the
+;; same block.  Each load is independently lowered to a single-lane masked load
+;; + broadcast, but they share the SAME AnyOf(mask) and lane-0 mask material
+;; thanks to the standard VPlan CSE that runs before unrolling.
+;;
+;;   for (i = 0; i < n; ++i)
+;;     if (cond[i])
+;;       a[i] = *inv1 + *inv2;
+
+define void @cond_invar_load_two_ptrs(ptr noalias %a, ptr noalias %cond, ptr noalias %inv1, ptr noalias %inv2, i64 %n) {
+; CHECK-LABEL: @cond_invar_load_two_ptrs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i1> zeroinitializer, i1 [[TMP3]], i32 0
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV1:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT:    [[BROADCAST_LANE:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV2:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-NEXT:    [[BROADCAST_LANE2:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD1]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[BROADCAST_LANE]], [[BROADCAST_LANE2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr align 4 [[TMP6]], <4 x i1> [[TMP1]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-NEXT:    [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[INV1]], align 4
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[INV2]], align 4
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-NEXT:    store i32 [[SUM]], ptr [[A_GEP]], align 4
+; CHECK-NEXT:    br label [[FOR_INC]]
+; CHECK:       for.inc:
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+  %cv = load i32, ptr %cond.gep, align 4
+  %tobool = icmp ne i32 %cv, 0
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+  %v1 = load i32, ptr %inv1, align 4
+  %v2 = load i32, ptr %inv2, align 4
+  %sum = add i32 %v1, %v2
+  %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+  store i32 %sum, ptr %a.gep, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
new file mode 100644
index 0000000000000..e502761b8a7e2
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load-not-invariant.ll
@@ -0,0 +1,64 @@
+; RUN: opt < %s -passes=loop-vectorize -enable-masked-invariant-load -force-vector-width=4 \
+; RUN:     -force-vector-interleave=1 -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S \
+; RUN:     | FileCheck %s
+
+;; Negative test for CM_CondInvar.
+;;
+;; The conditional load's pointer (%src + i) is loop-VARIANT, so even with
+;; -enable-masked-invariant-load enabled, getConditionalInvarCost() returns
+;; Invalid and the cost model picks the regular consecutive masked load
+;; (CM_Widen).
+;;
+;;   for (i = 0; i < n; ++i)
+;;     if (cond[i])
+;;       a[i] = src[i];
+;;
+;; The output:
+;;   * Must contain a regular widened consecutive masked load on src + index.
+;;   * Must NOT contain any AnyOf-style i1 OR-reduction.
+;;   * Must NOT contain a broadcast.lane shuffle (the CondInvar lowering
+;;     marker).
+;;   * Must NOT contain a masked.load whose pointer is the loop-invariant
+;;     %src argument.
+
+define void @cond_load_not_invariant(ptr noalias %a, ptr noalias %src, ptr noalias %cond, i64 %n) {
+; CHECK-LABEL: define void @cond_load_not_invariant(
+
+; The vector body widens the consecutive masked load on src[index].
+; CHECK:       vector.body:
+; CHECK:         [[SRC_GEP_V:%.*]] = getelementptr i32, ptr %src, i64 {{%.*}}
+; CHECK:         call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[SRC_GEP_V]],
+
+; The CondInvar lowering must NOT be applied.
+; CHECK-NOT:     call i1 @llvm.vector.reduce.or
+; CHECK-NOT:     %broadcast.lane = shufflevector
+; CHECK-NOT:     call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 %src,
+
+; CHECK:       middle.block:
+; CHECK:         ret void
+
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+  %cv = load i32, ptr %cond.gep, align 4
+  %tobool = icmp ne i32 %cv, 0
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+  %src.gep = getelementptr inbounds i32, ptr %src, i64 %i
+  %v = load i32, ptr %src.gep, align 4
+  %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+  store i32 %v, ptr %a.gep, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll
new file mode 100644
index 0000000000000..4a66d907675d0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/enable-masked-invariant-load.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN:     -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S | FileCheck %s --check-prefix=CHECK-OFF
+; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 \
+; RUN:     -enable-masked-invariant-load -mtriple=x86_64-unknown-linux-gnu -mattr=avx2 -S | FileCheck %s --check-prefix=CHECK-ON
+
+;; Conditional load from a loop-invariant pointer.
+;;
+;;   for (i = 0; i < n; ++i)
+;;     if (cond[i])
+;;       a[i] = *inv;
+;;
+;; The pointer %inv is loop-invariant and the load is predicated by %tobool.
+;; Without -enable-masked-invariant-load the cost model falls back to per-lane
+;; scalarised predicated loads (CM_Scalarize) - one pred.load.if/continue
+;; chain per VF lane.
+;;
+;; With -enable-masked-invariant-load the load becomes:
+;;   * AnyOf reduction across the block mask,
+;;   * lane-0 mask materialisation (zeroinitializer + insertelement i1),
+;;   * one masked.load from %inv,
+;;   * shufflevector broadcast of lane 0 to all VF lanes.
+
+define void @cond_invar_load(ptr noalias %a, ptr noalias %cond, ptr noalias %inv, i64 %n) {
+; CHECK-OFF-LABEL: @cond_invar_load(
+; CHECK-OFF-NEXT:  entry:
+; CHECK-OFF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-OFF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-OFF:       vector.ph:
+; CHECK-OFF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-OFF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-OFF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-OFF:       vector.body:
+; CHECK-OFF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
+; CHECK-OFF-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-OFF-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-OFF-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-OFF-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
+; CHECK-OFF-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-OFF:       pred.load.if:
+; CHECK-OFF-NEXT:    [[TMP3:%.*]] = load i32, ptr [[INV:%.*]], align 4
+; CHECK-OFF-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[TMP3]], i64 0
+; CHECK-OFF-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK-OFF:       pred.load.continue:
+; CHECK-OFF-NEXT:    [[TMP5:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_LOAD_IF]] ]
+; CHECK-OFF-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1
+; CHECK-OFF-NEXT:    br i1 [[TMP6]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK-OFF:       pred.load.if1:
+; CHECK-OFF-NEXT:    [[TMP7:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP7]], i64 1
+; CHECK-OFF-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
+; CHECK-OFF:       pred.load.continue2:
+; CHECK-OFF-NEXT:    [[TMP9:%.*]] = phi <4 x i32> [ [[TMP5]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP8]], [[PRED_LOAD_IF1]] ]
+; CHECK-OFF-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2
+; CHECK-OFF-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK-OFF:       pred.load.if3:
+; CHECK-OFF-NEXT:    [[TMP11:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP11]], i64 2
+; CHECK-OFF-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
+; CHECK-OFF:       pred.load.continue4:
+; CHECK-OFF-NEXT:    [[TMP13:%.*]] = phi <4 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP12]], [[PRED_LOAD_IF3]] ]
+; CHECK-OFF-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3
+; CHECK-OFF-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK-OFF:       pred.load.if5:
+; CHECK-OFF-NEXT:    [[TMP15:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT:    [[TMP16:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP15]], i64 3
+; CHECK-OFF-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
+; CHECK-OFF:       pred.load.continue6:
+; CHECK-OFF-NEXT:    [[TMP17:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP16]], [[PRED_LOAD_IF5]] ]
+; CHECK-OFF-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-OFF-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP17]], ptr align 4 [[TMP18]], <4 x i1> [[TMP1]])
+; CHECK-OFF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-OFF-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-OFF-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-OFF:       middle.block:
+; CHECK-OFF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-OFF-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-OFF:       scalar.ph:
+; CHECK-OFF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-OFF-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-OFF:       for.body:
+; CHECK-OFF-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-OFF-NEXT:    [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-OFF-NEXT:    [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-OFF-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-OFF-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-OFF:       if.then:
+; CHECK-OFF-NEXT:    [[V:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-OFF-NEXT:    [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-OFF-NEXT:    store i32 [[V]], ptr [[A_GEP]], align 4
+; CHECK-OFF-NEXT:    br label [[FOR_INC]]
+; CHECK-OFF:       for.inc:
+; CHECK-OFF-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-OFF-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-OFF-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-OFF:       for.end:
+; CHECK-OFF-NEXT:    ret void
+;
+; CHECK-ON-LABEL: @cond_invar_load(
+; CHECK-ON-NEXT:  entry:
+; CHECK-ON-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
+; CHECK-ON-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-ON:       vector.ph:
+; CHECK-ON-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-ON-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-ON-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-ON:       vector.body:
+; CHECK-ON-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ON-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]]
+; CHECK-ON-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-ON-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-ON-NEXT:    [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]]
+; CHECK-ON-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]])
+; CHECK-ON-NEXT:    [[TMP4:%.*]] = insertelement <4 x i1> zeroinitializer, i1 [[TMP3]], i32 0
+; CHECK-ON-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[INV:%.*]], <4 x i1> [[TMP4]], <4 x i32> poison)
+; CHECK-ON-NEXT:    [[BROADCAST_LANE:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_LOAD]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-ON-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-ON-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_LANE]], ptr align 4 [[TMP5]], <4 x i1> [[TMP1]])
+; CHECK-ON-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-ON-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ON-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-ON:       middle.block:
+; CHECK-ON-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-ON-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-ON:       scalar.ph:
+; CHECK-ON-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-ON-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-ON:       for.body:
+; CHECK-ON-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-ON-NEXT:    [[COND_GEP:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[I]]
+; CHECK-ON-NEXT:    [[CV:%.*]] = load i32, ptr [[COND_GEP]], align 4
+; CHECK-ON-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[CV]], 0
+; CHECK-ON-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
+; CHECK-ON:       if.then:
+; CHECK-ON-NEXT:    [[V:%.*]] = load i32, ptr [[INV]], align 4
+; CHECK-ON-NEXT:    [[A_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
+; CHECK-ON-NEXT:    store i32 [[V]], ptr [[A_GEP]], align 4
+; CHECK-ON-NEXT:    br label [[FOR_INC]]
+; CHECK-ON:       for.inc:
+; CHECK-ON-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-ON-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-ON-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-ON:       for.end:
+; CHECK-ON-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i64 [ 0, %entry ], [ %i.next, %for.inc ]
+  %cond.gep = getelementptr inbounds i32, ptr %cond, i64 %i
+  %cv = load i32, ptr %cond.gep, align 4
+  %tobool = icmp ne i32 %cv, 0
+  br i1 %tobool, label %if.then, label %for.inc
+
+if.then:
+  %v = load i32, ptr %inv, align 4
+  %a.gep = getelementptr inbounds i32, ptr %a, i64 %i
+  store i32 %v, ptr %a.gep, align 4
+  br label %for.inc
+
+for.inc:
+  %i.next = add nuw nsw i64 %i, 1
+  %exitcond = icmp eq i64 %i.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}

>From 3cb56a8c199ca0e461e41f7fdf07fdb73a534a60 Mon Sep 17 00:00:00 2001
From: Ashutosh Nema <ashu1212 at gmail.com>
Date: Tue, 2 Jun 2026 17:59:32 +0530
Subject: [PATCH 2/2] Replace CM_CondInvar and its recipe-builder dispatch with
 a VPlan transform widenPredicatedInvariantLoads, run after
 sinkPredicatedStores. It rewrites masked replicate loads from loop-invariant
 addresses into AnyOf(mask) + masked VPWidenLoadRecipe + BroadcastLane(0);

---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 106 +-----------------
 .../Transforms/Vectorize/VPRecipeBuilder.h    |   5 -
 .../Transforms/Vectorize/VPlanTransforms.cpp  |  78 +++++++++++++
 .../Transforms/Vectorize/VPlanTransforms.h    |   7 ++
 .../VPlan/vplan-print-after-all.ll            |   1 +
 5 files changed, 89 insertions(+), 108 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 064c1498e207d..b2589f4641bc5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -198,11 +198,6 @@ static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
     "vectorize-memory-check-threshold", cl::init(128), cl::Hidden,
     cl::desc("The maximum allowed number of runtime memory checks"));
 
-static cl::opt<bool> EnableMaskedInvariantLoad(
-    "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
-    cl::desc("Vectorize conditional invariant load with mask-load"
-             " and broadcast"));
-
 /// Option tail-folding-policy indicates that an epilogue is undesired, that
 /// tail folding is preferred, and this lists all options. I.e., the vectorizer
 /// will try to fold the tail-loop (epilogue) into the vector body and predicate
@@ -934,8 +929,7 @@ class LoopVectorizationCostModel {
     CM_GatherScatter,
     CM_Scalarize,
     CM_VectorCall,
-    CM_IntrinsicCall,
-    CM_CondInvar,
+    CM_IntrinsicCall
   };
 
   /// Save vectorization decision \p W and \p Cost taken by the cost model for
@@ -1336,10 +1330,6 @@ class LoopVectorizationCostModel {
   /// The cost computation for Gather/Scatter instruction.
   InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
 
-  /// Cost of lowering a predicated loop-invariant load as a single-lane
-  /// masked load + broadcast. Returns getInvalid() if not applicable.
-  InstructionCost getConditionalInvarCost(Instruction *I, ElementCount VF);
-
   /// The cost computation for widening instruction \p I with consecutive
   /// memory access.
   InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF);
@@ -2724,11 +2714,9 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
     if (IsUniformMemOpUse(I))
       return true;
 
-    // CM_CondInvar uses its scalar invariant pointer uniformly.
     return (WideningDecision == CM_Widen ||
             WideningDecision == CM_Widen_Reverse ||
-            WideningDecision == CM_Interleave ||
-            WideningDecision == CM_CondInvar);
+            WideningDecision == CM_Interleave);
   };
 
   // Returns true if Ptr is the pointer operand of a memory access instruction
@@ -4409,49 +4397,6 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
              Config.CostKind);
 }
 
-InstructionCost
-LoopVectorizationCostModel::getConditionalInvarCost(Instruction *I,
-                                                    ElementCount VF) {
-  // Applicability checks. Any failure returns Invalid; the cost ladder
-  // already gates on .isValid().
-  if (!EnableMaskedInvariantLoad)
-    return InstructionCost::getInvalid();
-  if (!Legal->blockNeedsPredication(I->getParent()))
-    return InstructionCost::getInvalid();
-  if (!isMaskRequired(I))
-    return InstructionCost::getInvalid();
-  if (!isa<LoadInst>(I))
-    return InstructionCost::getInvalid();
-  const Align Alignment = getLoadStoreAlignment(I);
-  unsigned AS = getLoadStoreAddressSpace(I);
-  if (!TTI.isLegalMaskedLoad(I->getType(), Alignment, AS))
-    return InstructionCost::getInvalid();
-  Value *Ptr = getLoadStorePointerOperand(I);
-  if (!Legal->isInvariant(Ptr))
-    return InstructionCost::getInvalid();
-
-  // Components: AnyOf or-reduce + lane-0 mask materialisation
-  // (zero-splat + insertelement) + masked load + lane-0 broadcast shuffle.
-  // InstructionCost arithmetic propagates Invalid across the chain.
-  Type *ValTy = getLoadStoreType(I);
-  auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
-  auto *MaskTy = cast<VectorType>(
-      toVectorTy(Type::getInt1Ty(I->getContext()), VF));
-  InstructionCost Cost = TTI.getArithmeticReductionCost(
-      Instruction::Or, MaskTy, /*FMF*/ std::nullopt, Config.CostKind);
-  Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, MaskTy, MaskTy,
-                             {}, Config.CostKind, 0);
-  Cost += TTI.getVectorInstrCost(Instruction::InsertElement, MaskTy,
-                                 Config.CostKind, /*Index=*/0);
-  Cost += TTI.getMemIntrinsicInstrCost(
-      MemIntrinsicCostAttributes(Intrinsic::masked_load, VectorTy, Alignment,
-                                 AS),
-      Config.CostKind);
-  Cost += TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
-                             VectorTy, {}, Config.CostKind, 0);
-  return Cost;
-}
-
 InstructionCost
 LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
                                                    ElementCount VF) {
@@ -4857,7 +4802,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
       InstructionCost ScalarizationCost =
           getMemInstScalarizationCost(&I, VF) * NumAccesses;
 
-      InstructionCost ConditionalInvarCost = getConditionalInvarCost(&I, VF);
       // Choose better solution for the current VF,
       // write down this decision and use it during vectorization.
       InstructionCost Cost;
@@ -4866,11 +4810,6 @@ void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
           InterleaveCost < ScalarizationCost) {
         Decision = CM_Interleave;
         Cost = InterleaveCost;
-      } else if (ConditionalInvarCost.isValid() &&
-                 ConditionalInvarCost <= GatherScatterCost &&
-                 ConditionalInvarCost < ScalarizationCost) {
-        Decision = CM_CondInvar;
-        Cost = ConditionalInvarCost;
       } else if (GatherScatterCost < ScalarizationCost) {
         Decision = CM_GatherScatter;
         Cost = GatherScatterCost;
@@ -5519,9 +5458,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
                                    : TTI::CastContextHint::Normal;
       case LoopVectorizationCostModel::CM_Widen_Reverse:
         return TTI::CastContextHint::Reversed;
-      case LoopVectorizationCostModel::CM_CondInvar:
-        return isPredicatedInst(I) ? TTI::CastContextHint::Masked
-                                   : TTI::CastContextHint::Normal;
       case LoopVectorizationCostModel::CM_Unknown:
         llvm_unreachable("Instr did not go through cost modelling?");
       case LoopVectorizationCostModel::CM_VectorCall:
@@ -6381,41 +6317,6 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
   });
 }
 
-VPInstruction *
-VPRecipeBuilder::tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
-                                         VPValue *Mask, VPInstruction *VPI) {
-  assert(Mask && "CM_CondInvar requires a non-null block mask");
-  LLVMContext &Ctx = Load->getContext();
-  DebugLoc DL = Load->getDebugLoc();
-  Builder.setInsertPoint(VPI);
-
-  // any = AnyOf(BlockMask). Unroll appends per-part masks, so this becomes
-  // a single OR-reduction across all UF parts.
-  VPValue *AnyActive = Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
-
-  // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
-  Type *I1Ty = Type::getInt1Ty(Ctx);
-  VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
-  VPValue *ZeroVMask =
-      Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
-  VPValue *Lane0Idx =
-      Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
-  VPValue *VMask = Builder.createNaryOp(
-      Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
-
-  // Consecutive=true selects the masked-load (not masked-gather) lowering
-  // in VPWidenLoadRecipe::execute. The access is stride-0, not stride-1;
-  // promote to a real AddrKind enum if other passes start relying on
-  // isConsecutive() distinguishing the two.
-  auto *MaskedLoad =
-      new VPWidenLoadRecipe(*Load, Ptr, VMask, /*Consecutive*/ true, *VPI, DL);
-  Builder.insert(MaskedLoad);
-
-  return new VPInstruction(VPInstruction::BroadcastLane,
-                           {Lane0Idx, MaskedLoad->getVPSingleValue()},
-                           /*Flags*/ {}, /*MD*/ {}, DL);
-}
-
 VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
                                                 VFRange &Range) {
   assert((VPI->getOpcode() == Instruction::Load ||
@@ -6481,8 +6382,6 @@ VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
 
   if (VPI->getOpcode() == Instruction::Load) {
     auto *Load = cast<LoadInst>(I);
-    if (Decision == LoopVectorizationCostModel::CM_CondInvar && Mask)
-      return tryToWidenCondInvarLoad(Load, Ptr, Mask, VPI);
     auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
                                         Load->getDebugLoc());
     if (Reverse) {
@@ -6923,6 +6822,7 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
     // Now optimize the initial VPlan.
     RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
     RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
+    RUN_VPLAN_PASS(VPlanTransforms::widenPredicatedInvariantLoads, *Plan, TTI);
     RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
                    Config.getMinimalBitwidths());
     RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 86f5c03ae501e..a84c77d614673 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -42,11 +42,6 @@ class VPRecipeBuilder {
   /// Range. The function should not be called for memory instructions or calls.
   bool shouldWiden(Instruction *I, VFRange &Range) const;
 
-  /// Lower a CM_CondInvar \p Load to: single-lane masked-load + broadcast.
-  /// \p Mask is the non-null block mask. Returns the BroadcastLane recipe.
-  VPInstruction *tryToWidenCondInvarLoad(LoadInst *Load, VPValue *Ptr,
-                                         VPValue *Mask, VPInstruction *VPI);
-
   /// Optimize the special case where the operand of \p VPI is a constant
   /// integer induction variable.
   VPWidenIntOrFpInductionRecipe *
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e036f823f8a1f..c150b94361fb3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4767,6 +4767,84 @@ void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
   }
 }
 
+static cl::opt<bool> EnableMaskedInvariantLoad(
+    "enable-masked-invariant-load", cl::init(false), cl::ReallyHidden,
+    cl::desc("Rewrite predicated loads from loop-invariant addresses as a "
+             "single-lane masked load + broadcast"));
+
+void VPlanTransforms::widenPredicatedInvariantLoads(
+    VPlan &Plan, const TargetTransformInfo &TTI) {
+  if (!EnableMaskedInvariantLoad)
+    return;
+
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  if (!LoopRegion)
+    return;
+
+  // Collect masked, vector-producing replicate loads whose pointer is defined
+  // outside the loop region and which the target can lower as a masked load.
+  SmallVector<VPReplicateRecipe *> Worklist;
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+           vp_depth_first_shallow(LoopRegion->getEntry()))) {
+    for (VPRecipeBase &R : *VPBB) {
+      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+      if (!RepR || !RepR->isPredicated() || RepR->isSingleScalar())
+        continue;
+      if (RepR->getOpcode() != Instruction::Load)
+        continue;
+      VPValue *Addr = RepR->getOperand(0);
+      if (!Addr->isDefinedOutsideLoopRegions())
+        continue;
+      auto *LI = cast<LoadInst>(RepR->getUnderlyingInstr());
+      if (!TTI.isLegalMaskedLoad(LI->getType(), LI->getAlign(),
+                                 LI->getPointerAddressSpace()))
+        continue;
+      Worklist.push_back(RepR);
+    }
+  }
+
+  for (VPReplicateRecipe *RepR : Worklist) {
+    auto *LI = cast<LoadInst>(RepR->getUnderlyingInstr());
+    LLVMContext &Ctx = LI->getContext();
+    DebugLoc DL = LI->getDebugLoc();
+    VPValue *Addr = RepR->getOperand(0);
+    VPValue *Mask = RepR->getMask();
+
+    VPBuilder Builder(RepR);
+
+    // any = AnyOf(mask). Unroll appends per-part masks, so this becomes a
+    // single OR-reduction across all UF parts.
+    VPValue *AnyActive =
+        Builder.createNaryOp(VPInstruction::AnyOf, {Mask}, DL);
+
+    // vmsk = insertelement <VF x i1> zeroinitializer, any, 0
+    Type *I1Ty = Type::getInt1Ty(Ctx);
+    VPValue *FalseI1 = Plan.getOrAddLiveIn(ConstantInt::getFalse(I1Ty));
+    VPValue *ZeroVMask =
+        Builder.createNaryOp(VPInstruction::Broadcast, {FalseI1}, DL);
+    VPValue *Lane0Idx =
+        Plan.getOrAddLiveIn(ConstantInt::get(Type::getInt32Ty(Ctx), 0));
+    VPValue *VMask = Builder.createNaryOp(
+        Instruction::InsertElement, {ZeroVMask, AnyActive, Lane0Idx}, DL);
+
+    // Consecutive=true selects the masked-load (not masked-gather) lowering in
+    // VPWidenLoadRecipe::execute. The access is stride-0, not stride-1; the
+    // BroadcastLane below splats lane 0 across all VF lanes.
+    auto *MaskedLoad = new VPWidenLoadRecipe(*LI, Addr, VMask,
+                                             /*Consecutive*/ true, *RepR, DL);
+    Builder.insert(MaskedLoad);
+
+    auto *Bcast = new VPInstruction(
+        VPInstruction::BroadcastLane,
+        {Lane0Idx, MaskedLoad->getVPSingleValue()}, /*Flags*/ {},
+        /*MD*/ {}, DL);
+    Builder.insert(Bcast);
+
+    RepR->replaceAllUsesWith(Bcast);
+    RepR->eraseFromParent();
+  }
+}
+
 static bool
 canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
                              PredicatedScalarEvolution &PSE, const Loop &L,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 0880b4ca4336c..4fe578692ba16 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -400,6 +400,13 @@ struct VPlanTransforms {
   static void hoistPredicatedLoads(VPlan &Plan, PredicatedScalarEvolution &PSE,
                                    const Loop *L);
 
+  /// Rewrite predicated VPReplicateRecipe loads from a loop-invariant address
+  /// into a single-lane masked load + broadcast pattern. Only applied when
+  /// the target reports isLegalMaskedLoad for the element type. Gated by
+  /// -enable-masked-invariant-load.
+  static void widenPredicatedInvariantLoads(VPlan &Plan,
+                                            const TargetTransformInfo &TTI);
+
   /// Sink predicated stores to the same address with complementary predicates
   /// (P and NOT P) to an unconditional store with select recipes for the
   /// stored values. This eliminates branching overhead when all paths
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 061588317eba7..7f31efb6fc5fc 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -29,6 +29,7 @@
 ; CHECK: VPlan for loop in 'foo' after VPlanTransforms::dropPoisonGeneratingRecipes
 ; CHECK: VPlan for loop in 'foo' after VPlanTransforms::hoistPredicatedLoads
 ; CHECK: VPlan for loop in 'foo' after VPlanTransforms::sinkPredicatedStores
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::widenPredicatedInvariantLoads
 ; CHECK: VPlan for loop in 'foo' after VPlanTransforms::truncateToMinimalBitwidths
 ; CHECK: VPlan for loop in 'foo' after removeRedundantInductionCasts
 ; CHECK: VPlan for loop in 'foo' after reassociateHeaderMask



More information about the llvm-commits mailing list