[llvm] [LoopVectorize] Widen selected-base loads with masked loads (PR #213227)

via llvm-commits llvm-commits at lists.llvm.org
Thu Aug 6 04:16:28 PDT 2026


https://github.com/OHNope updated https://github.com/llvm/llvm-project/pull/213227

>From 39dc61fe9d233cc522882ae67fdb984bcf67fb0d Mon Sep 17 00:00:00 2001
From: OHNope <ohnopexboy at gmail.com>
Date: Fri, 31 Jul 2026 17:56:04 +0900
Subject: [PATCH 1/4] [LoopVectorize] Widen selected-base loads with masked
 loads

---
 .../Vectorize/LoopVectorizationPlanner.h      |  15 +-
 llvm/lib/Transforms/Vectorize/VPlan.h         |   8 +-
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 189 ++++++++++++++
 .../X86/selected-base-masked-load.ll          | 235 ++++++++++++++++++
 4 files changed, 438 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index fa317f290022e..9cf41bb3e7001 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -520,13 +520,14 @@ class VPBuilder {
   }
 
   /// Create a recipe widening \p Load, loading from \p Addr with \p Mask (may
-  /// be null).
-  VPWidenLoadRecipe *createWidenLoad(LoadInst &Load, VPValue *Addr,
-                                     VPValue *Mask, bool Consecutive,
-                                     const VPIRMetadata &Metadata,
-                                     DebugLoc DL) {
-    return tryInsertInstruction(
-        new VPWidenLoadRecipe(Load, Addr, Mask, Consecutive, Metadata, DL));
+  /// be null). Use \p Alignment instead of the ingredient's alignment when it
+  /// is provided.
+  VPWidenLoadRecipe *
+  createWidenLoad(LoadInst &Load, VPValue *Addr, VPValue *Mask,
+                  bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL,
+                  std::optional<Align> Alignment = std::nullopt) {
+    return tryInsertInstruction(new VPWidenLoadRecipe(
+        Load, Addr, Mask, Consecutive, Metadata, DL, Alignment));
   }
 
   /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 814b77a96e825..ed8d7ac2525a2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3798,16 +3798,20 @@ class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPIRMetadata {
 struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPSingleDefRecipe,
                                                    public VPWidenMemoryRecipe {
   VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
-                    bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL)
+                    bool Consecutive, const VPIRMetadata &Metadata, DebugLoc DL,
+                    std::optional<Align> Alignment = std::nullopt)
       : VPSingleDefRecipe(VPRecipeBase::VPWidenLoadSC, {Addr}, Load.getType(),
                           &Load, DL),
         VPWidenMemoryRecipe(Load, Consecutive, Metadata) {
+    if (Alignment)
+      this->Alignment = *Alignment;
     setMask(Mask);
   }
 
   VPWidenLoadRecipe *clone() override {
     return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
-                                 getMask(), Consecutive, *this, getDebugLoc());
+                                 getMask(), Consecutive, *this, getDebugLoc(),
+                                 getAlign());
   }
 
   VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 348dbe514a56e..7db8358a430f5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -57,6 +57,93 @@ static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
   return getStrideFromAddRec(AddRec, L, AccessTy, /*Ptr=*/nullptr, PSE);
 }
 
+namespace {
+
+struct SelectedBaseLoadPattern {
+  VPInstruction *Addr;
+  SmallVector<VPValue *> GEPOperands;
+  VPValue *Cond;
+  VPValue *TrueBase;
+  VPValue *FalseBase;
+};
+
+/// Match a load through a GEP whose base pointer is selected per iteration
+/// between two loop-invariant candidates:
+///
+///   load (gep (select Cond, TrueBase, FalseBase), Indices...)
+///
+/// Legality, stride and profitability are checked separately by the caller.
+static std::optional<SelectedBaseLoadPattern>
+matchSelectedBaseLoad(VPInstruction *Load) {
+  if (Load->getOpcode() != Instruction::Load)
+    return std::nullopt;
+
+  // exclude volatile load, atomic load, other uneditable memory operations
+  auto *LI = dyn_cast_or_null<LoadInst>(Load->getUnderlyingInstr());
+  if (!LI || !LI->isSimple())
+    return std::nullopt;
+
+  VPValue *Addr = Load->getOperand(0);
+  auto *AddrI = dyn_cast<VPInstruction>(Addr);
+  if (!AddrI)
+    return std::nullopt;
+
+  Type *SourceElementTy = nullptr;
+  ArrayRef<VPValue *> GEPOperands;
+  // pattern with 3 params : cond + trueBase + falseBase
+  if (!match(Addr, m_GetElementPtr(SourceElementTy, GEPOperands)) ||
+      GEPOperands.size() < 2)
+    return std::nullopt;
+
+  VPValue *Cond = nullptr;
+  VPValue *TrueBase = nullptr;
+  VPValue *FalseBase = nullptr;
+  if (!match(GEPOperands.front(), m_Select(m_VPValue(Cond), m_VPValue(TrueBase),
+                                           m_VPValue(FalseBase))))
+    return std::nullopt;
+
+  if (!Cond->getScalarType()->isIntegerTy(1) || TrueBase == FalseBase)
+    return std::nullopt;
+
+  return SelectedBaseLoadPattern{AddrI, SmallVector<VPValue *>(GEPOperands),
+                                 Cond, TrueBase, FalseBase};
+}
+
+static InstructionCost getBlendedLoadCost(const TargetTransformInfo &TTI,
+                                          Type *VecTy, Align Alignment,
+                                          unsigned AddressSpace,
+                                          bool HasActiveMask,
+                                          TTI::TargetCostKind CostKind) {
+  Type *MaskTy = CmpInst::makeCmpResultType(VecTy);
+  InstructionCost Cost =
+      2 * TTI.getMemIntrinsicInstrCost(
+              MemIntrinsicCostAttributes(Intrinsic::masked_load, VecTy,
+                                         Alignment, AddressSpace),
+              CostKind) +
+      TTI.getArithmeticInstrCost(Instruction::Xor, MaskTy, CostKind) +
+      TTI.getCmpSelInstrCost(Instruction::Select, VecTy, MaskTy,
+                             CmpInst::BAD_ICMP_PREDICATE, CostKind);
+  if (HasActiveMask)
+    Cost += 2 * TTI.getArithmeticInstrCost(Instruction::And, MaskTy, CostKind);
+  return Cost;
+}
+
+static InstructionCost getGatherLoadCost(const LoadInst &LI, ElementCount VF,
+                                         bool HasActiveMask,
+                                         const VPCostContext &Ctx) {
+  Type *VecTy = VectorType::get(LI.getType(), VF);
+  Type *VecPtrTy = VectorType::get(LI.getPointerOperandType(), VF);
+  return Ctx.TTI.getAddressComputationCost(VecPtrTy, nullptr, nullptr,
+                                           Ctx.CostKind) +
+         Ctx.TTI.getMemIntrinsicInstrCost(
+             MemIntrinsicCostAttributes(Intrinsic::masked_gather, VecTy,
+                                        LI.getPointerOperand(), HasActiveMask,
+                                        LI.getAlign(), &LI),
+             Ctx.CostKind);
+}
+
+} // end anonymous namespace
+
 bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
     VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
     Loop *OuterLoop) {
@@ -5428,6 +5515,108 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
         return false;
       });
 
+  // Recognize loads through a pointer selected per iteration between two
+  // loop-invariant bases. Although the combined address is not consecutive,
+  // each selected address chain is. When profitable, use two complementary
+  // masked consecutive loads and blend their results instead of gathering a
+  // vector of selected pointers.
+  VPlanTransforms::runPass(
+      "widenSelectedBaseLoads", ProcessSubset, Plan, [&](VPInstruction *VPI) {
+        std::optional<SelectedBaseLoadPattern> Pattern =
+            matchSelectedBaseLoad(VPI);
+        if (!Pattern)
+          return false;
+
+        if (!vputils::isUniformAcrossVFsAndUFs(Pattern->TrueBase) ||
+            !vputils::isUniformAcrossVFsAndUFs(Pattern->FalseBase) ||
+            vputils::isUniformAcrossVFsAndUFs(Pattern->Cond))
+          return false;
+
+        auto *LI = cast<LoadInst>(VPI->getUnderlyingInstr());
+        Type *ScalarTy = LI->getType();
+        // The ingredient's alignment only applies to the address selected by
+        // each scalar iteration. It does not imply that either candidate base
+        // is aligned when that candidate is masked off. Use a conservative
+        // alignment for the newly-speculated base pointers.
+        const Align SplitAlignment(1);
+        if (!CostCtx.Config.isLegalMaskedLoadOrStore(
+                /*IsLoad=*/true, ScalarTy, SplitAlignment,
+                LI->getPointerAddressSpace()))
+          return false;
+
+        VPBuilder Builder(VPI);
+        SmallVector<VPValue *> TrueOps = Pattern->GEPOperands;
+        SmallVector<VPValue *> FalseOps = Pattern->GEPOperands;
+        // replace the baseOp
+        TrueOps[0] = Pattern->TrueBase;
+        FalseOps[0] = Pattern->FalseBase;
+        auto *TrueAddr =
+            Builder.insert(Pattern->Addr->cloneWithOperands(TrueOps));
+        auto *FalseAddr =
+            Builder.insert(Pattern->Addr->cloneWithOperands(FalseOps));
+
+        // Splitting the selected base must expose two unit-stride accesses.
+        // Remove the speculative address recipes again if either proof fails.
+        if (getConstantStride(TrueAddr, ScalarTy, CostCtx.PSE, CostCtx.L) !=
+                1 ||
+            getConstantStride(FalseAddr, ScalarTy, CostCtx.PSE, CostCtx.L) !=
+                1) {
+          TrueAddr->eraseFromParent();
+          FalseAddr->eraseFromParent();
+          return false;
+        }
+
+        bool HasActiveMask = RecipeBuilder.isPredicatedInst(LI);
+        auto IsProfitable = [&](ElementCount VF) {
+          // Do not replace a scalarization decision without comparing against
+          // its cost and updating the legacy scalarization bookkeeping.
+          if (VF.isScalable() || CostCtx.willBeScalarized(LI, VF) ||
+              !CostCtx.Config.isLegalGatherOrScatter(LI, VF))
+            return false;
+          Type *VecTy = VectorType::get(ScalarTy, VF);
+          InstructionCost BlendedCost = getBlendedLoadCost(
+              CostCtx.TTI, VecTy, SplitAlignment, LI->getPointerAddressSpace(),
+              HasActiveMask, CostCtx.CostKind);
+          return BlendedCost <
+                 getGatherLoadCost(*LI, VF, HasActiveMask, CostCtx);
+        };
+        // if fail, then erase the tmp recipes
+        if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
+                                                                Range)) {
+          TrueAddr->eraseFromParent();
+          FalseAddr->eraseFromParent();
+          return false;
+        }
+
+        VPSingleDefRecipe *TrueVectorPtr =
+            Builder.createConsecutiveVectorPointer(
+                TrueAddr, ScalarTy, /*Reverse=*/false, VPI->getDebugLoc());
+        VPSingleDefRecipe *FalseVectorPtr =
+            Builder.createConsecutiveVectorPointer(
+                FalseAddr, ScalarTy, /*Reverse=*/false, VPI->getDebugLoc());
+
+        VPValue *TrueMask = Pattern->Cond;
+        VPValue *FalseMask =
+            Builder.createNot(Pattern->Cond, VPI->getDebugLoc());
+        if (HasActiveMask) {
+          VPValue *ActiveMask = VPI->getMask();
+          TrueMask = Builder.createLogicalAnd(ActiveMask, TrueMask,
+                                              VPI->getDebugLoc());
+          FalseMask = Builder.createLogicalAnd(ActiveMask, FalseMask,
+                                               VPI->getDebugLoc());
+        }
+        // two load + merge
+        auto *TrueLoad = Builder.createWidenLoad(
+            *LI, TrueVectorPtr, TrueMask,
+            /*Consecutive=*/true, *VPI, VPI->getDebugLoc(), SplitAlignment);
+        auto *FalseLoad = Builder.createWidenLoad(
+            *LI, FalseVectorPtr, FalseMask, /*Consecutive=*/true, *VPI,
+            VPI->getDebugLoc(), SplitAlignment);
+        VPInstruction *Blend = Builder.createSelect(
+            Pattern->Cond, TrueLoad, FalseLoad, VPI->getDebugLoc());
+        return ReplaceWith(VPI, Blend);
+      });
+
   if (!RecipeBuilder.prefersVectorizedAddressing()) {
     VPlanTransforms::runPass(
         "makeVPlanMemOpDecision", ProcessSubset, Plan, [&](VPInstruction *VPI) {
diff --git a/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll b/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll
new file mode 100644
index 0000000000000..db925c80d529e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll
@@ -0,0 +1,235 @@
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -force-vector-width=8 -force-vector-interleave=1 \
+; RUN:   -tail-folding-policy=must-fold-tail -force-tail-folding-style=data \
+; RUN:   -S %s | FileCheck %s --check-prefix=TAIL
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -force-vector-width=8 -force-vector-interleave=1 -disable-output \
+; RUN:   -vplan-print-after=widenSelectedBaseLoads %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=VPLAN
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -S %s | FileCheck %s --check-prefix=ISSUE
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -force-vector-width=8 -force-vector-interleave=2 -S %s | \
+; RUN:   FileCheck %s --check-prefix=UNROLL
+; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux-gnu -mcpu=tigerlake \
+; RUN:   -force-vector-width=8 -force-vector-interleave=1 -disable-output \
+; RUN:   -vplan-print-after=dropPoisonGeneratingRecipes %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=VPLAN-DROP
+
+
+; A load from a per-lane selected base and a unit-stride index is cheaper as
+; two complementary masked loads than as a gather on AVX-512 targets.
+
+define void @selected_base_unit_stride(ptr noalias readonly %conditions,
+                                      ptr noalias writeonly %out,
+                                      ptr noalias readonly %a,
+                                      ptr noalias readonly %b, i64 %n) {
+; CHECK-LABEL: @selected_base_unit_stride(
+; CHECK: vector.body:
+; CHECK: [[MASK:%.*]] = icmp sgt <8 x i32> {{.*}}, zeroinitializer
+; CHECK: [[A_PTR:%.*]] = getelementptr i32, ptr %a, i64 {{.*}}
+; CHECK: [[B_PTR:%.*]] = getelementptr i32, ptr %b, i64 {{.*}}
+; CHECK: [[NOT_MASK:%.*]] = xor <8 x i1> [[MASK]], splat (i1 true)
+; CHECK: [[FROM_A:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[A_PTR]], <8 x i1> [[MASK]], <8 x i32> poison)
+; CHECK: [[FROM_B:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 [[B_PTR]], <8 x i1> [[NOT_MASK]], <8 x i32> poison)
+; CHECK: [[RESULT:%.*]] = select <8 x i1> [[MASK]], <8 x i32> [[FROM_A]], <8 x i32> [[FROM_B]]
+; CHECK-NOT: @llvm.masked.gather
+;
+; TAIL-LABEL: @selected_base_unit_stride(
+; TAIL: [[ACTIVE:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(
+; TAIL: [[MASK:%.*]] = icmp sgt <8 x i32> {{.*}}, zeroinitializer
+; TAIL: [[NOT_MASK:%.*]] = xor <8 x i1> [[MASK]], splat (i1 true)
+; TAIL: [[ACTIVE_A:%.*]] = select <8 x i1> [[ACTIVE]], <8 x i1> [[MASK]], <8 x i1> zeroinitializer
+; TAIL: [[ACTIVE_B:%.*]] = select <8 x i1> [[ACTIVE]], <8 x i1> [[NOT_MASK]], <8 x i1> zeroinitializer
+; TAIL: [[FROM_A:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 {{.*}}, <8 x i1> [[ACTIVE_A]], <8 x i32> poison)
+; TAIL: [[FROM_B:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 {{.*}}, <8 x i1> [[ACTIVE_B]], <8 x i32> poison)
+; TAIL: select <8 x i1> [[MASK]], <8 x i32> [[FROM_A]], <8 x i32> [[FROM_B]]
+;
+; VPLAN-LABEL: VPlan for loop in 'selected_base_unit_stride' after widenSelectedBaseLoads
+; VPLAN: EMIT ir<%src.ptr>.1 = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN: EMIT ir<%src.ptr>.2 = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN: [[A_PTR:vp<%[0-9]+>]] = vector-pointer inbounds i32, ir<%src.ptr>.1, ir<1>
+; VPLAN: [[B_PTR:vp<%[0-9]+>]] = vector-pointer inbounds i32, ir<%src.ptr>.2, ir<1>
+; VPLAN: [[NOT_MASK:vp<%[0-9]+>]] = not ir<%cmp>
+; VPLAN: WIDEN ir<%value> = load [[A_PTR]], ir<%cmp>
+; VPLAN: WIDEN ir<%value>.1 = load [[B_PTR]], [[NOT_MASK]]
+; VPLAN: EMIT {{.*}} = select ir<%cmp>, ir<%value>, ir<%value>.1
+;
+; Cloning the recipes for multiple parts must preserve the conservative
+; alignment selected by this transform.
+; UNROLL-LABEL: @selected_base_unit_stride(
+; UNROLL-COUNT-4: call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %condition.ptr = getelementptr inbounds i32, ptr %conditions, i64 %iv
+  %condition = load i32, ptr %condition.ptr, align 4
+  %cmp = icmp sgt i32 %condition, 0
+  %base = select i1 %cmp, ptr %a, ptr %b
+  %src.ptr = getelementptr inbounds i32, ptr %base, i64 %iv
+  ; The selected address is known to be 8-byte aligned, but that does not imply
+  ; the unselected candidate base has the same alignment.
+  %value = load i32, ptr %src.ptr, align 8
+  %out.ptr = getelementptr inbounds i32, ptr %out, i64 %iv
+  store i32 %value, ptr %out.ptr, align 4
+  %iv.next = add nuw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Fixed-trip-count forms of the two examples from llvm.org/PR206384.
+
+define void @issue_i32(ptr noalias readonly %conditions,
+                       ptr noalias writeonly %out,
+                       ptr noalias readonly %a,
+                       ptr noalias readonly %b) {
+; ISSUE-LABEL: @issue_i32(
+; ISSUE: [[MASK:%.*]] = icmp sgt <8 x i32> {{.*}}, zeroinitializer
+; ISSUE: [[NOT_MASK:%.*]] = xor <8 x i1> [[MASK]], splat (i1 true)
+; ISSUE: call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 %a, <8 x i1> [[MASK]], <8 x i32> poison)
+; ISSUE: call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 %b, <8 x i1> [[NOT_MASK]], <8 x i32> poison)
+; ISSUE-NOT: @llvm.masked.gather
+; ISSUE: ret void
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %condition.ptr = getelementptr inbounds i32, ptr %conditions, i64 %iv
+  %condition = load i32, ptr %condition.ptr, align 4
+  %cmp = icmp sgt i32 %condition, 0
+  %base = select i1 %cmp, ptr %a, ptr %b
+  %src.ptr = getelementptr inbounds i32, ptr %base, i64 %iv
+  %value = load i32, ptr %src.ptr, align 4
+  %out.ptr = getelementptr inbounds i32, ptr %out, i64 %iv
+  store i32 %value, ptr %out.ptr, align 4
+  %iv.next = add nuw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, 8
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @issue_f64(ptr noalias readonly %conditions,
+                       ptr noalias writeonly %out,
+                       ptr noalias readonly %a,
+                       ptr noalias readonly %b) {
+; ISSUE-LABEL: @issue_f64(
+; ISSUE: [[MASK:%.*]] = fcmp ogt <4 x double> {{.*}}, zeroinitializer
+; ISSUE: [[NOT_MASK:%.*]] = xor <4 x i1> [[MASK]], splat (i1 true)
+; ISSUE: call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 1 %a, <4 x i1> [[MASK]], <4 x double> poison)
+; ISSUE: call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 1 %b, <4 x i1> [[NOT_MASK]], <4 x double> poison)
+; ISSUE-NOT: @llvm.masked.gather
+; ISSUE: ret void
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %condition.ptr = getelementptr inbounds double, ptr %conditions, i64 %iv
+  %condition = load double, ptr %condition.ptr, align 8
+  %cmp = fcmp ogt double %condition, 0.0
+  %base = select i1 %cmp, ptr %a, ptr %b
+  %src.ptr = getelementptr inbounds double, ptr %base, i64 %iv
+  %value = load double, ptr %src.ptr, align 8
+  %out.ptr = getelementptr inbounds double, ptr %out, i64 %iv
+  store double %value, ptr %out.ptr, align 8
+  %iv.next = add nuw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, 4
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; A non-unit index is not representable by two consecutive masked loads and
+; must keep the gather.
+
+define void @selected_base_stride_two(ptr noalias readonly %conditions,
+                                      ptr noalias writeonly %out,
+                                      ptr noalias readonly %a,
+                                      ptr noalias readonly %b, i64 %n) {
+; CHECK-LABEL: @selected_base_stride_two(
+; CHECK: vector.body:
+; CHECK: call <8 x i32> @llvm.masked.gather.v8i32.v8p0(
+; CHECK-NOT: @llvm.masked.load
+; CHECK: ret void
+;
+; VPLAN-LABEL: VPlan for loop in 'selected_base_stride_two' after widenSelectedBaseLoads
+; VPLAN: EMIT ir<%src.ptr> = getelementptr inbounds ir<%base>, ir<%index>
+; VPLAN-NOT: vector-pointer
+; VPLAN: EMIT-SCALAR ir<%value> = load ir<%src.ptr>
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %condition.ptr = getelementptr inbounds i32, ptr %conditions, i64 %iv
+  %condition = load i32, ptr %condition.ptr, align 4
+  %cmp = icmp sgt i32 %condition, 0
+  %base = select i1 %cmp, ptr %a, ptr %b
+  %index = shl nuw i64 %iv, 1
+  %src.ptr = getelementptr inbounds i32, ptr %base, i64 %index
+  %value = load i32, ptr %src.ptr, align 4
+  %out.ptr = getelementptr inbounds i32, ptr %out, i64 %iv
+  store i32 %value, ptr %out.ptr, align 4
+  %iv.next = add nuw i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, %n
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @selected_base_inbounds_cleanup(
+    ptr noalias readonly dereferenceable(12) %a,
+    ptr noalias readonly dereferenceable(20) %b,
+    ptr noalias writeonly dereferenceable(32) %out) {
+; CHECK-LABEL: @selected_base_inbounds_cleanup(
+; CHECK: vector.body:
+; CHECK: call <8 x i32> @llvm.masked.load
+; CHECK: call <8 x i32> @llvm.masked.load
+; CHECK-NOT: @llvm.masked.gather
+; CHECK: ret void
+;
+; VPLAN-DROP-LABEL: VPlan for loop in 'selected_base_inbounds_cleanup' after VPlanTransforms::dropPoisonGeneratingRecipes
+; VPLAN-DROP: CLONE ir<%src.ptr>.1 = getelementptr ir<%a.end>, ir<%index>
+; VPLAN-DROP: CLONE ir<%src.ptr>.2 = getelementptr ir<%b>, ir<%index>
+; VPLAN-DROP: [[A_PTR:vp<%[0-9]+>]] = vector-pointer i32, ir<%src.ptr>.1, ir<1>
+; VPLAN-DROP: [[B_PTR:vp<%[0-9]+>]] = vector-pointer i32, ir<%src.ptr>.2, ir<1>
+
+entry:
+  %a.end = getelementptr inbounds i32, ptr %a, i64 3
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+
+  ; iv 0..2 use a.end + (-3..-1)
+  ; iv 3..7 use b + (0..4)
+  %use.a = icmp ult i64 %iv, 3
+  %base = select i1 %use.a, ptr %a.end, ptr %b
+  %index = add i64 %iv, -3
+
+  ; inbounds valid, only load the selected Addr
+  %src.ptr = getelementptr inbounds i32, ptr %base, i64 %index
+  %value = load i32, ptr %src.ptr, align 4
+
+  %out.ptr = getelementptr inbounds i32, ptr %out, i64 %iv
+  store i32 %value, ptr %out.ptr, align 4
+
+  %iv.next = add nuw i64 %iv, 1
+  %done = icmp eq i64 %iv.next, 8
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret void
+}
\ No newline at end of file

>From 9a01d9ad9768446011b442d2978b2fe20426d1e1 Mon Sep 17 00:00:00 2001
From: OHNope <ohnopexboy at gmail.com>
Date: Fri, 31 Jul 2026 20:48:31 +0900
Subject: [PATCH 2/4] [LoopVectorize] Update VPlan printing test for
 selected-base loads

---
 .../LoopVectorize/VPlan/vplan-print-before-after-all.ll          | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index a7271524f0191..97c0ed94f2946 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -20,6 +20,7 @@
 ; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] lowerMemoryIdioms
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] scalarizeMemOpsWithIrregularTypes
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenSelectedBaseLoads
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenConsecutiveMemOps
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] delegateMemOpWideningToLegacyCM
 ; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions

>From f184b8b76a5e2c495d191e6bec469d6350838859 Mon Sep 17 00:00:00 2001
From: OHNope <ohnopexboy at gmail.com>
Date: Tue, 4 Aug 2026 15:31:44 +0900
Subject: [PATCH 3/4] [LoopVectorize] Reuse existing costs for selected-base
 loads

---
 .../Transforms/Vectorize/VPlanPatternMatch.h  |  56 +++++++
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 140 ++++--------------
 2 files changed, 87 insertions(+), 109 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index d575d1f150e7a..b939147289687 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1122,6 +1122,62 @@ inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
   return GetElementPtr_match(SourceElementType, Operands);
 }
 
+/// The parts of a load through a pointer selected between two bases:
+///
+///   load (gep (select Cond, TrueBase, FalseBase), Indices...)
+struct SelectedBaseLoadPattern {
+  VPInstruction *Addr = nullptr;
+  SmallVector<VPValue *> GEPOperands;
+  VPValue *Cond = nullptr;
+  VPValue *TrueBase = nullptr;
+  VPValue *FalseBase = nullptr;
+};
+
+/// Match a simple load through a GEP whose base pointer is selected per
+/// iteration between two distinct candidates.
+struct SelectedBaseLoad_match {
+  SelectedBaseLoadPattern &Pattern;
+
+  bool match(const VPInstruction *Load) const {
+    if (Load->getOpcode() != Instruction::Load)
+      return false;
+
+    auto *LI = dyn_cast_or_null<LoadInst>(Load->getUnderlyingInstr());
+    if (!LI || !LI->isSimple())
+      return false;
+
+    Type *SourceElementTy = nullptr;
+    ArrayRef<VPValue *> GEPOperands;
+    if (!VPlanPatternMatch::match(
+            Load->getOperand(0),
+            m_GetElementPtr(SourceElementTy, GEPOperands)) ||
+        GEPOperands.size() < 2)
+      return false;
+
+    VPValue *Cond = nullptr;
+    VPValue *TrueBase = nullptr;
+    VPValue *FalseBase = nullptr;
+    if (!VPlanPatternMatch::match(GEPOperands.front(),
+                                  m_Select(m_VPValue(Cond), m_VPValue(TrueBase),
+                                           m_VPValue(FalseBase))) ||
+        !Cond->getScalarType()->isIntegerTy(1) || TrueBase == FalseBase)
+      return false;
+
+    auto *Addr = dyn_cast<VPInstruction>(Load->getOperand(0));
+    if (!Addr)
+      return false;
+
+    Pattern = {Addr, SmallVector<VPValue *>(GEPOperands), Cond, TrueBase,
+               FalseBase};
+    return true;
+  }
+};
+
+inline SelectedBaseLoad_match
+m_SelectedBaseLoad(SelectedBaseLoadPattern &Pattern) {
+  return {Pattern};
+}
+
 template <typename SubPattern_t> struct OneUse_match {
   SubPattern_t SubPattern;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a69f8496e2b3a..61a30bfab488b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "VPlanTransforms.h"
+#include "SLPVectorizer/SLPCostAnalysis.h"
 #include "VPRecipeBuilder.h"
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
@@ -57,93 +58,6 @@ static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
   return getStrideFromAddRec(AddRec, L, AccessTy, /*Ptr=*/nullptr, PSE);
 }
 
-namespace {
-
-struct SelectedBaseLoadPattern {
-  VPInstruction *Addr;
-  SmallVector<VPValue *> GEPOperands;
-  VPValue *Cond;
-  VPValue *TrueBase;
-  VPValue *FalseBase;
-};
-
-/// Match a load through a GEP whose base pointer is selected per iteration
-/// between two loop-invariant candidates:
-///
-///   load (gep (select Cond, TrueBase, FalseBase), Indices...)
-///
-/// Legality, stride and profitability are checked separately by the caller.
-static std::optional<SelectedBaseLoadPattern>
-matchSelectedBaseLoad(VPInstruction *Load) {
-  if (Load->getOpcode() != Instruction::Load)
-    return std::nullopt;
-
-  // exclude volatile load, atomic load, other uneditable memory operations
-  auto *LI = dyn_cast_or_null<LoadInst>(Load->getUnderlyingInstr());
-  if (!LI || !LI->isSimple())
-    return std::nullopt;
-
-  VPValue *Addr = Load->getOperand(0);
-  auto *AddrI = dyn_cast<VPInstruction>(Addr);
-  if (!AddrI)
-    return std::nullopt;
-
-  Type *SourceElementTy = nullptr;
-  ArrayRef<VPValue *> GEPOperands;
-  // pattern with 3 params : cond + trueBase + falseBase
-  if (!match(Addr, m_GetElementPtr(SourceElementTy, GEPOperands)) ||
-      GEPOperands.size() < 2)
-    return std::nullopt;
-
-  VPValue *Cond = nullptr;
-  VPValue *TrueBase = nullptr;
-  VPValue *FalseBase = nullptr;
-  if (!match(GEPOperands.front(), m_Select(m_VPValue(Cond), m_VPValue(TrueBase),
-                                           m_VPValue(FalseBase))))
-    return std::nullopt;
-
-  if (!Cond->getScalarType()->isIntegerTy(1) || TrueBase == FalseBase)
-    return std::nullopt;
-
-  return SelectedBaseLoadPattern{AddrI, SmallVector<VPValue *>(GEPOperands),
-                                 Cond, TrueBase, FalseBase};
-}
-
-static InstructionCost getBlendedLoadCost(const TargetTransformInfo &TTI,
-                                          Type *VecTy, Align Alignment,
-                                          unsigned AddressSpace,
-                                          bool HasActiveMask,
-                                          TTI::TargetCostKind CostKind) {
-  Type *MaskTy = CmpInst::makeCmpResultType(VecTy);
-  InstructionCost Cost =
-      2 * TTI.getMemIntrinsicInstrCost(
-              MemIntrinsicCostAttributes(Intrinsic::masked_load, VecTy,
-                                         Alignment, AddressSpace),
-              CostKind) +
-      TTI.getArithmeticInstrCost(Instruction::Xor, MaskTy, CostKind) +
-      TTI.getCmpSelInstrCost(Instruction::Select, VecTy, MaskTy,
-                             CmpInst::BAD_ICMP_PREDICATE, CostKind);
-  if (HasActiveMask)
-    Cost += 2 * TTI.getArithmeticInstrCost(Instruction::And, MaskTy, CostKind);
-  return Cost;
-}
-
-static InstructionCost getGatherLoadCost(const LoadInst &LI, ElementCount VF,
-                                         bool HasActiveMask,
-                                         const VPCostContext &Ctx) {
-  Type *VecTy = VectorType::get(LI.getType(), VF);
-  Type *VecPtrTy = VectorType::get(LI.getPointerOperandType(), VF);
-  return Ctx.TTI.getAddressComputationCost(VecPtrTy, nullptr, nullptr,
-                                           Ctx.CostKind) +
-         Ctx.TTI.getMemIntrinsicInstrCost(
-             MemIntrinsicCostAttributes(Intrinsic::masked_gather, VecTy,
-                                        LI.getPointerOperand(), HasActiveMask,
-                                        LI.getAlign(), &LI),
-             Ctx.CostKind);
-}
-
-} // end anonymous namespace
-
 bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
     VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
     Loop *OuterLoop) {
@@ -5533,14 +5447,13 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
   // vector of selected pointers.
   VPlanTransforms::runPass(
       "widenSelectedBaseLoads", ProcessSubset, Plan, [&](VPInstruction *VPI) {
-        std::optional<SelectedBaseLoadPattern> Pattern =
-            matchSelectedBaseLoad(VPI);
-        if (!Pattern)
+        SelectedBaseLoadPattern Pattern;
+        if (!match(VPI, m_SelectedBaseLoad(Pattern)))
           return false;
 
-        if (!vputils::isUniformAcrossVFsAndUFs(Pattern->TrueBase) ||
-            !vputils::isUniformAcrossVFsAndUFs(Pattern->FalseBase) ||
-            vputils::isUniformAcrossVFsAndUFs(Pattern->Cond))
+        if (!vputils::isUniformAcrossVFsAndUFs(Pattern.TrueBase) ||
+            !vputils::isUniformAcrossVFsAndUFs(Pattern.FalseBase) ||
+            vputils::isUniformAcrossVFsAndUFs(Pattern.Cond))
           return false;
 
         auto *LI = cast<LoadInst>(VPI->getUnderlyingInstr());
@@ -5556,15 +5469,15 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
           return false;
 
         VPBuilder Builder(VPI);
-        SmallVector<VPValue *> TrueOps = Pattern->GEPOperands;
-        SmallVector<VPValue *> FalseOps = Pattern->GEPOperands;
+        SmallVector<VPValue *> TrueOps = Pattern.GEPOperands;
+        SmallVector<VPValue *> FalseOps = Pattern.GEPOperands;
         // replace the baseOp
-        TrueOps[0] = Pattern->TrueBase;
-        FalseOps[0] = Pattern->FalseBase;
+        TrueOps[0] = Pattern.TrueBase;
+        FalseOps[0] = Pattern.FalseBase;
         auto *TrueAddr =
-            Builder.insert(Pattern->Addr->cloneWithOperands(TrueOps));
+            Builder.insert(Pattern.Addr->cloneWithOperands(TrueOps));
         auto *FalseAddr =
-            Builder.insert(Pattern->Addr->cloneWithOperands(FalseOps));
+            Builder.insert(Pattern.Addr->cloneWithOperands(FalseOps));
 
         // Splitting the selected base must expose two unit-stride accesses.
         // Remove the speculative address recipes again if either proof fails.
@@ -5579,19 +5492,28 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
 
         bool HasActiveMask = RecipeBuilder.isPredicatedInst(LI);
         auto IsProfitable = [&](ElementCount VF) {
-          // Do not replace a scalarization decision without comparing against
-          // its cost and updating the legacy scalarization bookkeeping.
+          // Compare against the legacy decision for each VF so a scalarization
+          // choice is not replaced just because the gather is expensive.
           if (VF.isScalable() || CostCtx.willBeScalarized(LI, VF) ||
               !CostCtx.Config.isLegalGatherOrScatter(LI, VF))
             return false;
           Type *VecTy = VectorType::get(ScalarTy, VF);
-          InstructionCost BlendedCost = getBlendedLoadCost(
+          InstructionCost BlendedCost = slpvectorizer::getBlendedLoadCost(
               CostCtx.TTI, VecTy, SplitAlignment, LI->getPointerAddressSpace(),
-              HasActiveMask, CostCtx.CostKind);
-          return BlendedCost <
-                 getGatherLoadCost(*LI, VF, HasActiveMask, CostCtx);
+              CostCtx.CostKind);
+          if (HasActiveMask) {
+            Type *MaskTy = CmpInst::makeCmpResultType(VecTy);
+            BlendedCost += 2 * CostCtx.TTI.getArithmeticInstrCost(
+                                   Instruction::And, MaskTy, CostCtx.CostKind);
+          }
+          // Reuse the legacy decision's cost. This is the gather cost for the
+          // usual case, but also keeps the comparison correct if the legacy
+          // model selected scalarization.
+          InstructionCost LegacyCost = CostCtx.getLegacyCost(LI, VF);
+          return BlendedCost.isValid() && LegacyCost.isValid() &&
+                 BlendedCost < LegacyCost;
         };
-        // if fail, then erase the tmp recipes
+        // Erase the speculative address recipes if the cost comparison fails.
         if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
                                                                 Range)) {
           TrueAddr->eraseFromParent();
@@ -5606,9 +5528,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
             Builder.createConsecutiveVectorPointer(
                 FalseAddr, ScalarTy, /*Reverse=*/false, VPI->getDebugLoc());
 
-        VPValue *TrueMask = Pattern->Cond;
+        VPValue *TrueMask = Pattern.Cond;
         VPValue *FalseMask =
-            Builder.createNot(Pattern->Cond, VPI->getDebugLoc());
+            Builder.createNot(Pattern.Cond, VPI->getDebugLoc());
         if (HasActiveMask) {
           VPValue *ActiveMask = VPI->getMask();
           TrueMask = Builder.createLogicalAnd(ActiveMask, TrueMask,
@@ -5624,7 +5546,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
             *LI, FalseVectorPtr, FalseMask, /*Consecutive=*/true, *VPI,
             VPI->getDebugLoc(), SplitAlignment);
         VPInstruction *Blend = Builder.createSelect(
-            Pattern->Cond, TrueLoad, FalseLoad, VPI->getDebugLoc());
+            Pattern.Cond, TrueLoad, FalseLoad, VPI->getDebugLoc());
         return ReplaceWith(VPI, Blend);
       });
 

>From 686e957fc70d211d6c1f5a8343133930be32aa4d Mon Sep 17 00:00:00 2001
From: OHNope <ohnopexboy at gmail.com>
Date: Thu, 6 Aug 2026 20:04:37 +0900
Subject: [PATCH 4/4] [LoopVectorize] Reuse VPLanTransforms' recipes to
 calculate the cost and build VPI

---
 .../Transforms/Vectorize/LoopVectorize.cpp    |   3 +
 .../Transforms/Vectorize/VPlanPatternMatch.h  |  31 ++-
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 256 ++++++++++--------
 .../Transforms/Vectorize/VPlanTransforms.h    |   6 +
 .../VPlan/vplan-print-before-after-all.ll     |   2 +-
 .../X86/selected-base-masked-load.ll          |  18 +-
 6 files changed, 183 insertions(+), 133 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index de385ae0a4511..ff2051f42d453 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6883,6 +6883,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
   RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
                  InterleaveGroups, CM.isEpilogueAllowed());
 
+  RUN_VPLAN_PASS(VPlanTransforms::widenSelectedBaseLoads, *Plan, Range,
+                 CostCtx);
+
   // Convert memory recipes to strided access recipes if the strided access is
   // legal and profitable.
   RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index b939147289687..a961cea312b82 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1126,7 +1126,8 @@ inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
 ///
 ///   load (gep (select Cond, TrueBase, FalseBase), Indices...)
 struct SelectedBaseLoadPattern {
-  VPInstruction *Addr = nullptr;
+  VPValue *Addr = nullptr;
+  Type *SourceElementType = nullptr;
   SmallVector<VPValue *> GEPOperands;
   VPValue *Cond = nullptr;
   VPValue *TrueBase = nullptr;
@@ -1138,19 +1139,18 @@ struct SelectedBaseLoadPattern {
 struct SelectedBaseLoad_match {
   SelectedBaseLoadPattern &Pattern;
 
-  bool match(const VPInstruction *Load) const {
-    if (Load->getOpcode() != Instruction::Load)
+  bool matchLoad(const Instruction *I, VPValue *Addr) const {
+    auto *LI = dyn_cast<LoadInst>(I);
+    if (!LI || !LI->isSimple())
       return false;
 
-    auto *LI = dyn_cast_or_null<LoadInst>(Load->getUnderlyingInstr());
-    if (!LI || !LI->isSimple())
+    if (!isa<VPWidenGEPRecipe, VPInstruction>(Addr))
       return false;
 
     Type *SourceElementTy = nullptr;
     ArrayRef<VPValue *> GEPOperands;
     if (!VPlanPatternMatch::match(
-            Load->getOperand(0),
-            m_GetElementPtr(SourceElementTy, GEPOperands)) ||
+            Addr, m_GetElementPtr(SourceElementTy, GEPOperands)) ||
         GEPOperands.size() < 2)
       return false;
 
@@ -1163,13 +1163,20 @@ struct SelectedBaseLoad_match {
         !Cond->getScalarType()->isIntegerTy(1) || TrueBase == FalseBase)
       return false;
 
-    auto *Addr = dyn_cast<VPInstruction>(Load->getOperand(0));
-    if (!Addr)
+    Pattern = {Addr, SourceElementTy, SmallVector<VPValue *>(GEPOperands),
+               Cond, TrueBase,        FalseBase};
+    return true;
+  }
+
+  bool match(const VPInstruction *Load) const {
+    if (Load->getOpcode() != Instruction::Load)
       return false;
+    return matchLoad(dyn_cast_or_null<LoadInst>(Load->getUnderlyingInstr()),
+                     Load->getOperand(0));
+  }
 
-    Pattern = {Addr, SmallVector<VPValue *>(GEPOperands), Cond, TrueBase,
-               FalseBase};
-    return true;
+  bool match(const VPWidenLoadRecipe *Load) const {
+    return matchLoad(&Load->getIngredient(), Load->getAddr());
   }
 };
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 61a30bfab488b..ede76f29737f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "VPlanTransforms.h"
-#include "SLPVectorizer/SLPCostAnalysis.h"
 #include "VPRecipeBuilder.h"
 #include "VPlan.h"
 #include "VPlanAnalysis.h"
@@ -5440,116 +5439,6 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
         return false;
       });
 
-  // Recognize loads through a pointer selected per iteration between two
-  // loop-invariant bases. Although the combined address is not consecutive,
-  // each selected address chain is. When profitable, use two complementary
-  // masked consecutive loads and blend their results instead of gathering a
-  // vector of selected pointers.
-  VPlanTransforms::runPass(
-      "widenSelectedBaseLoads", ProcessSubset, Plan, [&](VPInstruction *VPI) {
-        SelectedBaseLoadPattern Pattern;
-        if (!match(VPI, m_SelectedBaseLoad(Pattern)))
-          return false;
-
-        if (!vputils::isUniformAcrossVFsAndUFs(Pattern.TrueBase) ||
-            !vputils::isUniformAcrossVFsAndUFs(Pattern.FalseBase) ||
-            vputils::isUniformAcrossVFsAndUFs(Pattern.Cond))
-          return false;
-
-        auto *LI = cast<LoadInst>(VPI->getUnderlyingInstr());
-        Type *ScalarTy = LI->getType();
-        // The ingredient's alignment only applies to the address selected by
-        // each scalar iteration. It does not imply that either candidate base
-        // is aligned when that candidate is masked off. Use a conservative
-        // alignment for the newly-speculated base pointers.
-        const Align SplitAlignment(1);
-        if (!CostCtx.Config.isLegalMaskedLoadOrStore(
-                /*IsLoad=*/true, ScalarTy, SplitAlignment,
-                LI->getPointerAddressSpace()))
-          return false;
-
-        VPBuilder Builder(VPI);
-        SmallVector<VPValue *> TrueOps = Pattern.GEPOperands;
-        SmallVector<VPValue *> FalseOps = Pattern.GEPOperands;
-        // replace the baseOp
-        TrueOps[0] = Pattern.TrueBase;
-        FalseOps[0] = Pattern.FalseBase;
-        auto *TrueAddr =
-            Builder.insert(Pattern.Addr->cloneWithOperands(TrueOps));
-        auto *FalseAddr =
-            Builder.insert(Pattern.Addr->cloneWithOperands(FalseOps));
-
-        // Splitting the selected base must expose two unit-stride accesses.
-        // Remove the speculative address recipes again if either proof fails.
-        if (getConstantStride(TrueAddr, ScalarTy, CostCtx.PSE, CostCtx.L) !=
-                1 ||
-            getConstantStride(FalseAddr, ScalarTy, CostCtx.PSE, CostCtx.L) !=
-                1) {
-          TrueAddr->eraseFromParent();
-          FalseAddr->eraseFromParent();
-          return false;
-        }
-
-        bool HasActiveMask = RecipeBuilder.isPredicatedInst(LI);
-        auto IsProfitable = [&](ElementCount VF) {
-          // Compare against the legacy decision for each VF so a scalarization
-          // choice is not replaced just because the gather is expensive.
-          if (VF.isScalable() || CostCtx.willBeScalarized(LI, VF) ||
-              !CostCtx.Config.isLegalGatherOrScatter(LI, VF))
-            return false;
-          Type *VecTy = VectorType::get(ScalarTy, VF);
-          InstructionCost BlendedCost = slpvectorizer::getBlendedLoadCost(
-              CostCtx.TTI, VecTy, SplitAlignment, LI->getPointerAddressSpace(),
-              CostCtx.CostKind);
-          if (HasActiveMask) {
-            Type *MaskTy = CmpInst::makeCmpResultType(VecTy);
-            BlendedCost += 2 * CostCtx.TTI.getArithmeticInstrCost(
-                                   Instruction::And, MaskTy, CostCtx.CostKind);
-          }
-          // Reuse the legacy decision's cost. This is the gather cost for the
-          // usual case, but also keeps the comparison correct if the legacy
-          // model selected scalarization.
-          InstructionCost LegacyCost = CostCtx.getLegacyCost(LI, VF);
-          return BlendedCost.isValid() && LegacyCost.isValid() &&
-                 BlendedCost < LegacyCost;
-        };
-        // Erase the speculative address recipes if the cost comparison fails.
-        if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
-                                                                Range)) {
-          TrueAddr->eraseFromParent();
-          FalseAddr->eraseFromParent();
-          return false;
-        }
-
-        VPSingleDefRecipe *TrueVectorPtr =
-            Builder.createConsecutiveVectorPointer(
-                TrueAddr, ScalarTy, /*Reverse=*/false, VPI->getDebugLoc());
-        VPSingleDefRecipe *FalseVectorPtr =
-            Builder.createConsecutiveVectorPointer(
-                FalseAddr, ScalarTy, /*Reverse=*/false, VPI->getDebugLoc());
-
-        VPValue *TrueMask = Pattern.Cond;
-        VPValue *FalseMask =
-            Builder.createNot(Pattern.Cond, VPI->getDebugLoc());
-        if (HasActiveMask) {
-          VPValue *ActiveMask = VPI->getMask();
-          TrueMask = Builder.createLogicalAnd(ActiveMask, TrueMask,
-                                              VPI->getDebugLoc());
-          FalseMask = Builder.createLogicalAnd(ActiveMask, FalseMask,
-                                               VPI->getDebugLoc());
-        }
-        // two load + merge
-        auto *TrueLoad = Builder.createWidenLoad(
-            *LI, TrueVectorPtr, TrueMask,
-            /*Consecutive=*/true, *VPI, VPI->getDebugLoc(), SplitAlignment);
-        auto *FalseLoad = Builder.createWidenLoad(
-            *LI, FalseVectorPtr, FalseMask, /*Consecutive=*/true, *VPI,
-            VPI->getDebugLoc(), SplitAlignment);
-        VPInstruction *Blend = Builder.createSelect(
-            Pattern.Cond, TrueLoad, FalseLoad, VPI->getDebugLoc());
-        return ReplaceWith(VPI, Blend);
-      });
-
   if (!RecipeBuilder.prefersVectorizedAddressing()) {
     VPlanTransforms::runPass(
         "makeVPlanMemOpDecision", ProcessSubset, Plan, [&](VPInstruction *VPI) {
@@ -5645,6 +5534,151 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
                            });
 }
 
+void VPlanTransforms::widenSelectedBaseLoads(VPlan &Plan, VFRange &Range,
+                                             VPCostContext &CostCtx) {
+  if (Plan.hasScalarVFOnly())
+    return;
+
+  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+           vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+      auto *LoadR = dyn_cast<VPWidenLoadRecipe>(&R);
+      if (!LoadR || LoadR->isConsecutive())
+        continue;
+
+      SelectedBaseLoadPattern Pattern;
+      if (!m_SelectedBaseLoad(Pattern).match(LoadR))
+        continue;
+
+      if (!vputils::isUniformAcrossVFsAndUFs(Pattern.TrueBase) ||
+          !vputils::isUniformAcrossVFsAndUFs(Pattern.FalseBase) ||
+          vputils::isUniformAcrossVFsAndUFs(Pattern.Cond))
+        continue;
+
+      auto &LI = cast<LoadInst>(LoadR->getIngredient());
+      Type *ScalarTy = LI.getType();
+      // The ingredient's alignment only applies to the address selected by
+      // each scalar iteration. It does not imply that either candidate base
+      // is aligned when that candidate is masked off. Use a conservative
+      // alignment for the newly-speculated base pointers.
+      const Align SplitAlignment(1);
+      if (!CostCtx.Config.isLegalMaskedLoadOrStore(
+              /*IsLoad=*/true, ScalarTy, SplitAlignment,
+              LI.getPointerAddressSpace()))
+        continue;
+
+      VPBuilder Builder(LoadR);
+      SmallVector<VPRecipeBase *> NewRecipes;
+      auto EraseNewRecipes = [&]() {
+        for (VPRecipeBase *NewRecipe : reverse(NewRecipes))
+          NewRecipe->eraseFromParent();
+      };
+
+      SmallVector<VPValue *> TrueOps = Pattern.GEPOperands;
+      SmallVector<VPValue *> FalseOps = Pattern.GEPOperands;
+      TrueOps[0] = Pattern.TrueBase;
+      FalseOps[0] = Pattern.FalseBase;
+      auto CloneAddr = [&](ArrayRef<VPValue *> Ops) -> VPSingleDefRecipe * {
+        if (auto *GEP = dyn_cast<VPWidenGEPRecipe>(Pattern.Addr)) {
+          auto *GEPInst = cast<GetElementPtrInst>(GEP->getUnderlyingValue());
+          // The address is consumed by a consecutive pointer recipe, so keep
+          // the split GEP scalar and let the vector pointer provide the lane
+          // addressing.
+          return VPBuilder::createSingleScalarOp(
+              Instruction::GetElementPtr, Ops, /*Mask=*/nullptr, *GEP,
+              VPIRMetadata{}, GEP->getDebugLoc(), GEPInst);
+        }
+        return cast<VPInstruction>(Pattern.Addr)->cloneWithOperands(Ops);
+      };
+      auto *TrueAddr = Builder.insert(CloneAddr(TrueOps));
+      auto *FalseAddr = Builder.insert(CloneAddr(FalseOps));
+      NewRecipes.push_back(TrueAddr);
+      NewRecipes.push_back(FalseAddr);
+
+      // Splitting the selected base must expose two unit-stride accesses.
+      if (getConstantStride(TrueAddr, ScalarTy, CostCtx.PSE, CostCtx.L) != 1 ||
+          getConstantStride(FalseAddr, ScalarTy, CostCtx.PSE, CostCtx.L) != 1) {
+        EraseNewRecipes();
+        continue;
+      }
+
+      auto *TrueVectorPtr = Builder.createConsecutiveVectorPointer(
+          TrueAddr, ScalarTy, /*Reverse=*/false, LoadR->getDebugLoc());
+      auto *FalseVectorPtr = Builder.createConsecutiveVectorPointer(
+          FalseAddr, ScalarTy, /*Reverse=*/false, LoadR->getDebugLoc());
+      NewRecipes.push_back(TrueVectorPtr);
+      NewRecipes.push_back(FalseVectorPtr);
+
+      auto *NotCond = Builder.createNot(Pattern.Cond, LoadR->getDebugLoc());
+      NewRecipes.push_back(NotCond);
+      VPValue *TrueMask = Pattern.Cond;
+      VPValue *FalseMask = NotCond;
+      VPInstruction *TrueMaskAnd = nullptr;
+      VPInstruction *FalseMaskAnd = nullptr;
+      if (VPValue *ActiveMask = LoadR->getMask()) {
+        TrueMaskAnd = Builder.createLogicalAnd(ActiveMask, TrueMask,
+                                               LoadR->getDebugLoc());
+        FalseMaskAnd = Builder.createLogicalAnd(ActiveMask, FalseMask,
+                                                LoadR->getDebugLoc());
+        TrueMask = TrueMaskAnd;
+        FalseMask = FalseMaskAnd;
+        NewRecipes.push_back(TrueMaskAnd);
+        NewRecipes.push_back(FalseMaskAnd);
+      }
+
+      auto *TrueLoad = Builder.createWidenLoad(
+          LI, TrueVectorPtr, TrueMask, /*Consecutive=*/true, *LoadR,
+          LoadR->getDebugLoc(), SplitAlignment);
+      auto *FalseLoad = Builder.createWidenLoad(
+          LI, FalseVectorPtr, FalseMask, /*Consecutive=*/true, *LoadR,
+          LoadR->getDebugLoc(), SplitAlignment);
+      NewRecipes.push_back(TrueLoad);
+      NewRecipes.push_back(FalseLoad);
+      auto *Blend = Builder.createSelect(Pattern.Cond, TrueLoad, FalseLoad,
+                                         LoadR->getDebugLoc());
+      NewRecipes.push_back(Blend);
+
+      auto IsProfitable = [&](ElementCount VF) {
+        if (VF.isScalable())
+          return false;
+
+        const InstructionCost CurrentCost = LoadR->computeCost(VF, CostCtx);
+        InstructionCost ReplacementCost = TrueLoad->computeCost(VF, CostCtx) +
+                                          FalseLoad->computeCost(VF, CostCtx) +
+                                          NotCond->computeCost(VF, CostCtx) +
+                                          Blend->computeCost(VF, CostCtx);
+
+        // LogicalAnd is currently modeled as a poison-safe VPlan operation and
+        // has no dedicated computeCost implementation. Account for the two
+        // vector mask-and operations here using the same VPlan cost context.
+        if (TrueMaskAnd) {
+          Type *MaskTy =
+              VectorType::get(Type::getInt1Ty(Plan.getContext()), VF);
+          ReplacementCost +=
+              2 * CostCtx.TTI.getArithmeticInstrCost(Instruction::And, MaskTy,
+                                                     CostCtx.CostKind);
+        }
+        return CurrentCost.isValid() && ReplacementCost.isValid() &&
+               ReplacementCost < CurrentCost;
+      };
+
+      if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsProfitable,
+                                                              Range)) {
+        EraseNewRecipes();
+        continue;
+      }
+
+      // Invalidate the legacy widening decision for the replaced load, as the
+      // VPlan recipe costs now account for the replacement accesses.
+      for (ElementCount VF : Range)
+        CostCtx.invalidateWideningDecision(&LoadR->getIngredient(), VF);
+
+      LoadR->replaceAllUsesWith(Blend);
+      LoadR->eraseFromParent();
+    }
+  }
+}
+
 void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
   if (LoopVectorizationPlanner::getDecisionAndClampRange(
           [&](ElementCount VF) { return VF.isScalar(); }, Range))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index af8f6de26e0f9..6ee98c79fd4c3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -589,6 +589,12 @@ struct VPlanTransforms {
                                          VPRecipeBuilder &RecipeBuilder,
                                          VPCostContext &CostCtx);
 
+  /// Widen loads through a pointer selected per iteration between two
+  /// loop-invariant bases into two masked consecutive loads and a select when
+  /// the replacement is legal and profitable according to VPlan recipe costs.
+  static void widenSelectedBaseLoads(VPlan &Plan, VFRange &Range,
+                                     VPCostContext &CostCtx);
+
   /// Make VPlan-based scalarization decision prior to delegating to the ones
   /// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
   /// enabled by prior widening of consecutive memory operations for now.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 97c0ed94f2946..c9e0149d56226 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -20,7 +20,6 @@
 ; CHECK-BEFORE: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] lowerMemoryIdioms
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] scalarizeMemOpsWithIrregularTypes
-; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenSelectedBaseLoads
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] widenConsecutiveMemOps
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] delegateMemOpWideningToLegacyCM
 ; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::makeMemOpWideningDecisions
@@ -37,6 +36,7 @@
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createPartialReductions
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToAbstractRecipes
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::createInterleaveGroups
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::widenSelectedBaseLoads
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::convertToStridedAccesses
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::dropPoisonGeneratingRecipes
 ; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::hoistPredicatedLoads
diff --git a/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll b/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll
index db925c80d529e..f50ef542cb3c6 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/selected-base-masked-load.ll
@@ -47,11 +47,11 @@ define void @selected_base_unit_stride(ptr noalias readonly %conditions,
 ; TAIL: [[FROM_B:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 {{.*}}, <8 x i1> [[ACTIVE_B]], <8 x i32> poison)
 ; TAIL: select <8 x i1> [[MASK]], <8 x i32> [[FROM_A]], <8 x i32> [[FROM_B]]
 ;
-; VPLAN-LABEL: VPlan for loop in 'selected_base_unit_stride' after widenSelectedBaseLoads
-; VPLAN: EMIT ir<%src.ptr>.1 = getelementptr inbounds ir<%a>, ir<%iv>
-; VPLAN: EMIT ir<%src.ptr>.2 = getelementptr inbounds ir<%b>, ir<%iv>
-; VPLAN: [[A_PTR:vp<%[0-9]+>]] = vector-pointer inbounds i32, ir<%src.ptr>.1, ir<1>
-; VPLAN: [[B_PTR:vp<%[0-9]+>]] = vector-pointer inbounds i32, ir<%src.ptr>.2, ir<1>
+; VPLAN-LABEL: VPlan for loop in 'selected_base_unit_stride' after VPlanTransforms::widenSelectedBaseLoads
+; VPLAN: CLONE ir<%src.ptr>.1 = getelementptr inbounds ir<%a>, ir<%iv>
+; VPLAN: CLONE ir<%src.ptr>.2 = getelementptr inbounds ir<%b>, ir<%iv>
+; VPLAN: [[A_PTR:vp<%[0-9]+>]] = vector-pointer i32, ir<%src.ptr>.1, ir<1>
+; VPLAN: [[B_PTR:vp<%[0-9]+>]] = vector-pointer i32, ir<%src.ptr>.2, ir<1>
 ; VPLAN: [[NOT_MASK:vp<%[0-9]+>]] = not ir<%cmp>
 ; VPLAN: WIDEN ir<%value> = load [[A_PTR]], ir<%cmp>
 ; VPLAN: WIDEN ir<%value>.1 = load [[B_PTR]], [[NOT_MASK]]
@@ -163,10 +163,10 @@ define void @selected_base_stride_two(ptr noalias readonly %conditions,
 ; CHECK-NOT: @llvm.masked.load
 ; CHECK: ret void
 ;
-; VPLAN-LABEL: VPlan for loop in 'selected_base_stride_two' after widenSelectedBaseLoads
-; VPLAN: EMIT ir<%src.ptr> = getelementptr inbounds ir<%base>, ir<%index>
+; VPLAN-LABEL: VPlan for loop in 'selected_base_stride_two' after VPlanTransforms::widenSelectedBaseLoads
+; VPLAN: WIDEN-GEP ir<%src.ptr> = getelementptr inbounds ir<%base>, ir<%index>
 ; VPLAN-NOT: vector-pointer
-; VPLAN: EMIT-SCALAR ir<%value> = load ir<%src.ptr>
+; VPLAN: WIDEN ir<%value> = load ir<%src.ptr>
 entry:
   br label %loop
 
@@ -232,4 +232,4 @@ loop:
 
 exit:
   ret void
-}
\ No newline at end of file
+}



More information about the llvm-commits mailing list