[llvm] [SLP] Vectorize select-addressed loads as masked-load blends (PR #210455)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 15:45:55 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/210455
Recognize loads whose address is chosen per lane via
select(cond, A, B) and vectorize them as two masked loads blended
by a select, instead of gathering.
Fixes case 6 from #206367
>From 933c5adbbebdee4b5a7587ad756760d0f9160edd Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 17 Jul 2026 15:45:42 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 176 +++++++++++++++---
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 42 +++++
.../Vectorize/SLPVectorizer/SLPUtils.h | 14 ++
.../SLPVectorizer/X86/masked-blended-loads.ll | 83 +--------
4 files changed, 212 insertions(+), 103 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6dde1850d58d5..28e145f297840 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1469,7 +1469,8 @@ class slpvectorizer::BoUpSLP {
Vectorize,
ScatterVectorize,
StridedVectorize,
- CompressVectorize
+ CompressVectorize,
+ BlendedLoadVectorize
};
using ValueList = SmallVector<Value *, 8>;
@@ -3766,13 +3767,15 @@ class slpvectorizer::BoUpSLP {
/// (either with vector instruction or with scatter/gather
/// intrinsics for store/load)?
enum EntryState {
- Vectorize, ///< The node is regularly vectorized.
- ScatterVectorize, ///< Masked scatter/gather node.
- StridedVectorize, ///< Strided loads (and stores)
- ExpandVectorize, ///< Masked stores, the values are expanded into
- ///< a wider vector and vectorized with a mask.
- CompressVectorize, ///< (Masked) load with compress.
- NeedToGather, ///< Gather/buildvector node.
+ Vectorize, ///< The node is regularly vectorized.
+ ScatterVectorize, ///< Masked scatter/gather node.
+ StridedVectorize, ///< Strided loads (and stores)
+ ExpandVectorize, ///< Masked stores, the values are expanded into
+ ///< a wider vector and vectorized with a mask.
+ CompressVectorize, ///< (Masked) load with compress.
+ BlendedLoadVectorize, ///< (Masked) loads blended via `select` from two
+ ///< candidate base pointers.
+ NeedToGather, ///< Gather/buildvector node.
CombinedVectorize, ///< Vectorized node, combined with its user into more
///< complex node like select/cmp to minmax, mul/add to
///< fma, etc. Must be used for the following nodes in
@@ -4063,6 +4066,9 @@ class slpvectorizer::BoUpSLP {
case CompressVectorize:
dbgs() << "CompressVectorize\n";
break;
+ case BlendedLoadVectorize:
+ dbgs() << "BlendedLoadVectorize\n";
+ break;
case NeedToGather:
dbgs() << "NeedToGather\n";
break;
@@ -6209,7 +6215,8 @@ struct llvm::DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
if (Entry->State == TreeEntry::ScatterVectorize ||
Entry->State == TreeEntry::StridedVectorize ||
Entry->State == TreeEntry::ExpandVectorize ||
- Entry->State == TreeEntry::CompressVectorize)
+ Entry->State == TreeEntry::CompressVectorize ||
+ Entry->State == TreeEntry::BlendedLoadVectorize)
return "color=blue";
return "";
}
@@ -6995,6 +7002,29 @@ isMaskedLoadCompress(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
CompressMask, LoadVecTy);
}
+/// Returns the cost of a BlendedLoadVectorize node loading \p VecTy: two
+/// masked loads (one per candidate base) blended by a select, plus building
+/// the blend mask from the scalar conditions and negating it for the
+/// false-lane load.
+static InstructionCost getBlendedLoadCost(const TargetTransformInfo &TTI,
+ Type *VecTy, Align Alignment,
+ unsigned AddressSpace,
+ TTI::TargetCostKind CostKind) {
+ Type *CmpTy = CmpInst::makeCmpResultType(VecTy);
+ InstructionCost MaskBuildCost = getScalarizationOverhead(
+ TTI, CmpTy->getScalarType(), cast<VectorType>(CmpTy),
+ APInt::getAllOnes(getNumElements(VecTy)), /*Insert=*/true,
+ /*Extract=*/false, CostKind);
+ return 2 * TTI.getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(Intrinsic::masked_load, VecTy,
+ Alignment, AddressSpace),
+ CostKind) +
+ MaskBuildCost +
+ TTI.getArithmeticInstrCost(Instruction::Xor, CmpTy, CostKind) +
+ TTI.getCmpSelInstrCost(Instruction::Select, VecTy, CmpTy,
+ CmpInst::BAD_ICMP_PREDICATE, CostKind);
+}
+
/// Checks if the stores \p VL with pointers \p PointerOps can be lowered as a
/// single masked store. On success \p StoreVecTy is the widened store type and
/// \p ReuseShuffleIndices is the expand mask that places each stored value at
@@ -7450,6 +7480,24 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
return *MaskedGatherLegal;
};
if (!IsSorted) {
+ // Check for a group of loads, each selecting its address (directly, or
+ // via a constant-offset GEP) between the same two candidate base
+ // pointers - the shape if-converted, fully-unrolled loop bodies of the
+ // form `x = cond ? A[i] : B[i]` take. If found, model it as two masked
+ // loads (one per candidate) blended by the (vectorized) condition,
+ // rather than falling back to a gather of the individual scalar loads.
+ Value *TrueBase = nullptr;
+ Value *FalseBase = nullptr;
+ SmallVector<Value *> Conditions;
+ if (isSelectedBaseLoad(VL, PointerOps, *DL, TrueBase, FalseBase,
+ Conditions) &&
+ TTI->isLegalMaskedLoad(VecTy, CommonAlignment,
+ cast<LoadInst>(VL0)->getPointerAddressSpace())) {
+ // Keep PointerOps/Order untouched: there is no per-lane address operand
+ // to recurse into, so the operand list stays the real pointer operands.
+ return LoadsState::BlendedLoadVectorize;
+ }
+
if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order,
SPtrInfo, /*isLoad=*/true))
return LoadsState::StridedVectorize;
@@ -7677,6 +7725,13 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
CostKind) +
VectorGEPCost;
break;
+ case LoadsState::BlendedLoadVectorize:
+ // Two masked loads (one per candidate base) plus a select; no address
+ // vector is materialized, so VectorGEPCost is skipped.
+ VecLdCost +=
+ getBlendedLoadCost(TTI, SubVecTy, CommonAlignment,
+ LI0->getPointerAddressSpace(), CostKind);
+ break;
case LoadsState::Gather:
llvm_unreachable("Gathers are not added to States");
}
@@ -8047,7 +8102,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
((TE.State == TreeEntry::Vectorize ||
TE.State == TreeEntry::StridedVectorize ||
TE.State == TreeEntry::ExpandVectorize ||
- TE.State == TreeEntry::CompressVectorize) &&
+ TE.State == TreeEntry::CompressVectorize ||
+ TE.State == TreeEntry::BlendedLoadVectorize) &&
(isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
(TopToBottom && isa<StoreInst, InsertElementInst, InsertValueInst>(
TE.getMainOp()))))) {
@@ -8261,7 +8317,8 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
LoadsState Res = canVectorizeLoads(TE.Scalars, TE.Scalars.front(),
CurrentOrder, PointerOps, SPtrInfo);
if (Res == LoadsState::Vectorize || Res == LoadsState::StridedVectorize ||
- Res == LoadsState::CompressVectorize)
+ Res == LoadsState::CompressVectorize ||
+ Res == LoadsState::BlendedLoadVectorize)
return std::move(CurrentOrder);
}
if (std::optional<OrdersType> CurrentOrder =
@@ -8729,7 +8786,8 @@ void BoUpSLP::reorderTopToBottom() {
((TE->State == TreeEntry::Vectorize ||
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
- TE->State == TreeEntry::CompressVectorize) &&
+ TE->State == TreeEntry::CompressVectorize ||
+ TE->State == TreeEntry::BlendedLoadVectorize) &&
(isa<ExtractElementInst, ExtractValueInst, LoadInst, StoreInst,
InsertElementInst, InsertValueInst>(TE->getMainOp()) ||
(SLPReVec && isa<ShuffleVectorInst>(TE->getMainOp()))))) {
@@ -8782,6 +8840,7 @@ void BoUpSLP::buildReorderableOperands(
OpData.second->State == TreeEntry::StridedVectorize ||
OpData.second->State == TreeEntry::ExpandVectorize ||
OpData.second->State == TreeEntry::CompressVectorize ||
+ OpData.second->State == TreeEntry::BlendedLoadVectorize ||
OpData.second->State == TreeEntry::SplitVectorize);
}))
continue;
@@ -8802,7 +8861,8 @@ void BoUpSLP::buildReorderableOperands(
if (UserTE->getOpcode() == Instruction::Load &&
(UserTE->State == TreeEntry::Vectorize ||
UserTE->State == TreeEntry::StridedVectorize ||
- UserTE->State == TreeEntry::CompressVectorize))
+ UserTE->State == TreeEntry::CompressVectorize ||
+ UserTE->State == TreeEntry::BlendedLoadVectorize))
continue;
}
TreeEntry *TE = getOperandEntry(UserTE, I);
@@ -8845,6 +8905,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
TE->State != TreeEntry::StridedVectorize &&
TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
+ TE->State != TreeEntry::BlendedLoadVectorize &&
TE->State != TreeEntry::SplitVectorize)
NonVectorized.insert(TE.get());
if (std::optional<OrdersType> CurrentOrder =
@@ -8884,6 +8945,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
TE->State == TreeEntry::StridedVectorize ||
TE->State == TreeEntry::ExpandVectorize ||
TE->State == TreeEntry::CompressVectorize ||
+ TE->State == TreeEntry::BlendedLoadVectorize ||
TE->State == TreeEntry::SplitVectorize ||
(TE->isGather() && GathersToOrders.contains(TE))) ||
!TE->UserTreeIndex || !TE->ReuseShuffleIndices.empty() ||
@@ -9194,6 +9256,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
TE->State != TreeEntry::StridedVectorize &&
TE->State != TreeEntry::ExpandVectorize &&
TE->State != TreeEntry::CompressVectorize &&
+ TE->State != TreeEntry::BlendedLoadVectorize &&
TE->State != TreeEntry::SplitVectorize &&
(TE->State != TreeEntry::ScatterVectorize ||
TE->ReorderIndices.empty()))
@@ -10060,7 +10123,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
LoadsState State = canVectorizeLoads(
VL, VL.front(), Order, PointerOps, SPtrInfo);
if (State == LoadsState::ScatterVectorize ||
- State == LoadsState::CompressVectorize)
+ State == LoadsState::CompressVectorize ||
+ State == LoadsState::BlendedLoadVectorize)
return false;
ConsecutiveNodesSize += VL.size();
size_t Start = std::distance(Slice.begin(), It);
@@ -10718,6 +10782,14 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
}
return IsGatheredNode() ? TreeEntry::NeedToGather
: TreeEntry::StridedVectorize;
+ case LoadsState::BlendedLoadVectorize:
+ if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
+ // Delay slow vectorized nodes for better vectorization attempts.
+ LoadEntriesToVectorize.insert(VectorizableTree.size());
+ return TreeEntry::NeedToGather;
+ }
+ return IsGatheredNode() ? TreeEntry::NeedToGather
+ : TreeEntry::BlendedLoadVectorize;
case LoadsState::Gather:
#ifndef NDEBUG
Type *ScalarTy = VL0->getType();
@@ -11262,7 +11334,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
return (IncludeGather && Res == BoUpSLP::LoadsState::Gather) ||
Res == BoUpSLP::LoadsState::ScatterVectorize ||
Res == BoUpSLP::LoadsState::StridedVectorize ||
- Res == BoUpSLP::LoadsState::CompressVectorize;
+ Res == BoUpSLP::LoadsState::CompressVectorize ||
+ Res == BoUpSLP::LoadsState::BlendedLoadVectorize;
};
// Operand of the root tree entry on the vectorize path: always pack the
// scalars (PackProfitable=true). Choose between keeping the original VL
@@ -13017,6 +13090,16 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
<< "SLP: added a new TreeEntry (non-consecutive LoadInst).\n";
TE->dump());
break;
+ case TreeEntry::BlendedLoadVectorize:
+ // Load blended (via `select`) from two candidate bases, emitted as two
+ // masked loads plus a select (see isSelectedBaseLoad). No per-lane
+ // address operand to recurse into, so Operands stays the real pointer
+ // operand list.
+ TE = newTreeEntry(VL, TreeEntry::BlendedLoadVectorize, Bundle, S,
+ UserTreeIdx, ReuseShuffleIndices);
+ LLVM_DEBUG(dbgs() << "SLP: added a new TreeEntry (blended LoadInst).\n";
+ TE->dump());
+ break;
case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
@@ -16338,7 +16421,8 @@ TTI::CastContextHint BoUpSLP::getCastContextHint(const TreeEntry &TE) const {
if (TE.State == TreeEntry::ScatterVectorize ||
TE.State == TreeEntry::StridedVectorize)
return TTI::CastContextHint::GatherScatter;
- if (TE.State == TreeEntry::CompressVectorize)
+ if (TE.State == TreeEntry::CompressVectorize ||
+ TE.State == TreeEntry::BlendedLoadVectorize)
return TTI::CastContextHint::Masked;
if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::Load &&
!TE.isAltShuffle()) {
@@ -16767,7 +16851,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
E->State == TreeEntry::ScatterVectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize ||
- E->State == TreeEntry::CompressVectorize) &&
+ E->State == TreeEntry::CompressVectorize ||
+ E->State == TreeEntry::BlendedLoadVectorize) &&
"Unhandled state");
assert(E->getOpcode() &&
((allSameType(VL) && allSameBlock(VL)) ||
@@ -17653,6 +17738,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
CostKind);
break;
}
+ case TreeEntry::BlendedLoadVectorize: {
+ // Two masked loads (one per candidate base) blended by a select.
+ Align CommonAlignment =
+ computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
+ VecLdCost = getBlendedLoadCost(*TTI, VecTy, CommonAlignment,
+ LI0->getPointerAddressSpace(), CostKind);
+ break;
+ }
case TreeEntry::ExpandVectorize:
case TreeEntry::CombinedVectorize:
case TreeEntry::SplitVectorize:
@@ -17663,9 +17756,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
};
InstructionCost Cost = GetCostDiff(GetScalarCost, GetVectorCost);
- // If this node generates masked gather load then it is not a terminal node.
- // Hence address operand cost is estimated separately.
- if (E->State == TreeEntry::ScatterVectorize)
+ // Masked gather and blended loads are not terminal nodes: their address
+ // cost is estimated separately (blended loads have no per-lane address).
+ if (E->State == TreeEntry::ScatterVectorize ||
+ E->State == TreeEntry::BlendedLoadVectorize)
return Cost;
// Estimate cost of GEPs since this tree node is a terminator.
@@ -17965,6 +18059,7 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
VectorizableTree[0]->State == TreeEntry::StridedVectorize ||
VectorizableTree[0]->State == TreeEntry::ExpandVectorize ||
VectorizableTree[0]->State == TreeEntry::CompressVectorize ||
+ VectorizableTree[0]->State == TreeEntry::BlendedLoadVectorize ||
(ForReduction &&
AreVectorizableGathers(VectorizableTree[0].get(),
VectorizableTree[0]->Scalars.size()) &&
@@ -17989,7 +18084,8 @@ bool BoUpSLP::isFullyVectorizableTinyTree(bool ForReduction) const {
VectorizableTree[0]->State != TreeEntry::ScatterVectorize &&
VectorizableTree[0]->State != TreeEntry::StridedVectorize &&
VectorizableTree[0]->State != TreeEntry::ExpandVectorize &&
- VectorizableTree[0]->State != TreeEntry::CompressVectorize))
+ VectorizableTree[0]->State != TreeEntry::CompressVectorize &&
+ VectorizableTree[0]->State != TreeEntry::BlendedLoadVectorize))
return false;
return true;
@@ -23828,6 +23924,34 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
/*ArgNo=*/0,
Attribute::getWithAlignment(Inst->getContext(), CommonAlignment));
NewLI = Inst;
+ } else if (E->State == TreeEntry::BlendedLoadVectorize) {
+ // E->Scalars stays in the dense order isSelectedBaseLoad validated;
+ // ReorderIndices, if any, is handled below by the shared FinalShuffle,
+ // same as for StridedVectorize.
+ SmallVector<Value *> BlendPointerOps(E->Scalars.size());
+ for (auto [I, V] : enumerate(E->Scalars))
+ BlendPointerOps[I] = cast<LoadInst>(V)->getPointerOperand();
+ Value *TrueBase = nullptr;
+ Value *FalseBase = nullptr;
+ SmallVector<Value *> Conditions;
+ bool Found = isSelectedBaseLoad(E->Scalars, BlendPointerOps, *DL,
+ TrueBase, FalseBase, Conditions);
+ assert(Found && "Expected a valid blended-load pattern");
+ (void)Found;
+ Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
+ Value *BlendMask = PoisonValue::get(
+ FixedVectorType::get(Builder.getInt1Ty(), Conditions.size()));
+ for (auto [I, Cond] : enumerate(Conditions))
+ BlendMask =
+ Builder.CreateInsertElement(BlendMask, Cond, Builder.getInt32(I));
+ Value *NotMask = Builder.CreateNot(BlendMask);
+ Value *VecPoison = PoisonValue::get(VecTy);
+ Value *TrueVal = Builder.CreateMaskedLoad(
+ VecTy, TrueBase, CommonAlignment, BlendMask, VecPoison);
+ Value *FalseVal = Builder.CreateMaskedLoad(
+ VecTy, FalseBase, CommonAlignment, NotMask, VecPoison);
+ NewLI = cast<Instruction>(
+ Builder.CreateSelect(BlendMask, TrueVal, FalseVal));
} else {
assert(E->State == TreeEntry::ScatterVectorize && "Unhandled state");
Value *VecPtr = vectorizeOperand(E, 0);
@@ -23854,7 +23978,8 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
NewLI = Builder.CreateMaskedGather(VecTy, VecPtr, CommonAlignment);
}
- Value *V = E->State == TreeEntry::CompressVectorize
+ Value *V = (E->State == TreeEntry::CompressVectorize ||
+ E->State == TreeEntry::BlendedLoadVectorize)
? NewLI
: PropagateIRFlags(NewLI);
@@ -25169,7 +25294,8 @@ Value *BoUpSLP::vectorizeTree(
(E->State == TreeEntry::Vectorize ||
E->State == TreeEntry::StridedVectorize ||
E->State == TreeEntry::ExpandVectorize ||
- E->State == TreeEntry::CompressVectorize) &&
+ E->State == TreeEntry::CompressVectorize ||
+ E->State == TreeEntry::BlendedLoadVectorize) &&
any_of(UseEntries, [&, TTI = TTI](TreeEntry *UseEntry) {
return (UseEntry->State == TreeEntry::Vectorize ||
UseEntry->State ==
@@ -25177,7 +25303,9 @@ Value *BoUpSLP::vectorizeTree(
UseEntry->State ==
TreeEntry::ExpandVectorize ||
UseEntry->State ==
- TreeEntry::CompressVectorize) &&
+ TreeEntry::CompressVectorize ||
+ UseEntry->State ==
+ TreeEntry::BlendedLoadVectorize) &&
doesInTreeUserNeedToExtract(
Scalar, getRootEntryInstruction(*UseEntry),
TLI, TTI);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 93239b7cd2725..765fdb33cda97 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -13,6 +13,7 @@
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
@@ -539,6 +540,47 @@ bool isSimple(Instruction *I) {
return true;
}
+bool isSelectedBaseLoad(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
+ const DataLayout &DL, Value *&TrueBase,
+ Value *&FalseBase,
+ SmallVectorImpl<Value *> &Conditions) {
+ TrueBase = nullptr;
+ FalseBase = nullptr;
+ Type *ScalarTy = VL.front()->getType();
+ uint64_t ScalarSize = DL.getTypeStoreSize(ScalarTy);
+ Conditions.assign(PointerOps.size(), nullptr);
+ for (auto [Idx, P] : enumerate(PointerOps)) {
+ Value *Base = P;
+ uint64_t Offset = 0;
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(P)) {
+ APInt OffsetAP(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
+ if (!GEP->accumulateConstantOffset(DL, OffsetAP) || OffsetAP.isNegative())
+ return false;
+ Offset = OffsetAP.getZExtValue();
+ Base = GEP->getPointerOperand();
+ }
+ auto *Sel = dyn_cast<SelectInst>(Base);
+ if (!Sel)
+ return false;
+ Value *T = Sel->getTrueValue();
+ Value *F = Sel->getFalseValue();
+ if (!TrueBase) {
+ if (T == F)
+ return false;
+ TrueBase = T;
+ FalseBase = F;
+ } else if (TrueBase != T || FalseBase != F) {
+ return false;
+ }
+ // Lane Idx must be at exactly Base + Idx * sizeof(ScalarTy); codegen reads
+ // contiguously from TrueBase/FalseBase starting at lane 0.
+ if (Offset != static_cast<uint64_t>(Idx) * ScalarSize)
+ return false;
+ Conditions[Idx] = Sel->getCondition();
+ }
+ return TrueBase != nullptr;
+}
+
void addMask(SmallVectorImpl<int> &Mask, ArrayRef<int> SubMask,
bool ExtendingManyInputs) {
if (SubMask.empty())
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index 51181b99078c5..b466cb117dee8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -26,6 +26,7 @@
namespace llvm {
class Constant;
+class DataLayout;
class Instruction;
class TargetLibraryInfo;
class TargetTransformInfo;
@@ -247,6 +248,19 @@ MemoryLocation getLocation(Instruction *I);
/// \returns True if the instruction is not a volatile or atomic load/store.
bool isSimple(Instruction *I);
+/// Checks if the loads \p VL with pointer operands \p PointerOps are each
+/// (optionally via a constant-offset GEP) a `select Cond, A, B` picking between
+/// the same two base pointers A/B on every lane - the shape a fully unrolled
+/// `x = cond ? A[i] : B[i]` takes. On success \p TrueBase / \p FalseBase are
+/// the candidate bases and \p Conditions holds each lane's `select` condition,
+/// used to build the blend mask. Lane \p Idx must be at `Base + Idx *
+/// sizeof(ScalarTy)`; only dense, natural lane order starting at the base is
+/// recognized (reordered or partial groups fall back to Gather/Scatter).
+bool isSelectedBaseLoad(ArrayRef<Value *> VL, ArrayRef<Value *> PointerOps,
+ const DataLayout &DL, Value *&TrueBase,
+ Value *&FalseBase,
+ SmallVectorImpl<Value *> &Conditions);
+
/// Shuffles \p Mask in accordance with the given \p SubMask.
/// \param ExtendingManyInputs Supports reshuffling of the mask with not only
/// one but two input vectors.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll
index dd8b0e584bcfc..b9dae4219f312 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/masked-blended-loads.ll
@@ -6,85 +6,10 @@ define <16 x i32> @test(ptr %a, ptr %b, <16 x i32> %c, <16 x i32> %d) {
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], <16 x i32> [[C:%.*]], <16 x i32> [[D:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <16 x i32> [[D]], zeroinitializer
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x i1> [[TMP0]], i64 0
-; CHECK-NEXT: [[A_B:%.*]] = select i1 [[TMP1]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[A_B]], align 4
-; CHECK-NEXT: [[VECINS:%.*]] = insertelement <16 x i32> [[C]], i32 [[L0]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x i1> [[TMP0]], i64 1
-; CHECK-NEXT: [[A_SINK_1:%.*]] = select i1 [[TMP2]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_1]], i64 4
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[GEP_1]], align 4
-; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <16 x i32> [[VECINS]], i32 [[L1]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x i1> [[TMP0]], i64 2
-; CHECK-NEXT: [[A_SINK_2:%.*]] = select i1 [[TMP3]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_2]], i64 8
-; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[GEP_2]], align 4
-; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <16 x i32> [[VECINS_1]], i32 [[L2]], i64 2
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x i1> [[TMP0]], i64 3
-; CHECK-NEXT: [[A_SINK_3:%.*]] = select i1 [[TMP4]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_3:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_3]], i64 12
-; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[GEP_3]], align 4
-; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <16 x i32> [[VECINS_2]], i32 [[L3]], i64 3
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <16 x i1> [[TMP0]], i64 4
-; CHECK-NEXT: [[A_SINK_4:%.*]] = select i1 [[TMP5]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_4:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_4]], i64 16
-; CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[GEP_4]], align 4
-; CHECK-NEXT: [[VECINS_4:%.*]] = insertelement <16 x i32> [[VECINS_3]], i32 [[L4]], i64 4
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x i1> [[TMP0]], i64 5
-; CHECK-NEXT: [[A_SINK_5:%.*]] = select i1 [[TMP6]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_5]], i64 20
-; CHECK-NEXT: [[L5:%.*]] = load i32, ptr [[GEP_5]], align 4
-; CHECK-NEXT: [[VECINS_5:%.*]] = insertelement <16 x i32> [[VECINS_4]], i32 [[L5]], i64 5
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x i1> [[TMP0]], i64 6
-; CHECK-NEXT: [[A_SINK_6:%.*]] = select i1 [[TMP7]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_6]], i64 24
-; CHECK-NEXT: [[L6:%.*]] = load i32, ptr [[GEP_6]], align 4
-; CHECK-NEXT: [[VECINS_6:%.*]] = insertelement <16 x i32> [[VECINS_5]], i32 [[L6]], i64 6
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <16 x i1> [[TMP0]], i64 7
-; CHECK-NEXT: [[A_SINK_7:%.*]] = select i1 [[TMP8]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_7]], i64 28
-; CHECK-NEXT: [[L7:%.*]] = load i32, ptr [[GEP_7]], align 4
-; CHECK-NEXT: [[VECINS_7:%.*]] = insertelement <16 x i32> [[VECINS_6]], i32 [[L7]], i64 7
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x i1> [[TMP0]], i64 8
-; CHECK-NEXT: [[A_SINK_8:%.*]] = select i1 [[TMP9]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_8:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_8]], i64 32
-; CHECK-NEXT: [[L8:%.*]] = load i32, ptr [[GEP_8]], align 4
-; CHECK-NEXT: [[VECINS_8:%.*]] = insertelement <16 x i32> [[VECINS_7]], i32 [[L8]], i64 8
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x i1> [[TMP0]], i64 9
-; CHECK-NEXT: [[A_SINK_9:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_9:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_9]], i64 36
-; CHECK-NEXT: [[L9:%.*]] = load i32, ptr [[GEP_9]], align 4
-; CHECK-NEXT: [[VECINS_9:%.*]] = insertelement <16 x i32> [[VECINS_8]], i32 [[L9]], i64 9
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x i1> [[TMP0]], i64 10
-; CHECK-NEXT: [[A_SINK_10:%.*]] = select i1 [[TMP11]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_10:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_10]], i64 40
-; CHECK-NEXT: [[L10:%.*]] = load i32, ptr [[GEP_10]], align 4
-; CHECK-NEXT: [[VECINS_10:%.*]] = insertelement <16 x i32> [[VECINS_9]], i32 [[L10]], i64 10
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <16 x i1> [[TMP0]], i64 11
-; CHECK-NEXT: [[A_SINK_11:%.*]] = select i1 [[TMP12]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_11:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_11]], i64 44
-; CHECK-NEXT: [[L11:%.*]] = load i32, ptr [[GEP_11]], align 4
-; CHECK-NEXT: [[VECINS_11:%.*]] = insertelement <16 x i32> [[VECINS_10]], i32 [[L11]], i64 11
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <16 x i1> [[TMP0]], i64 12
-; CHECK-NEXT: [[A_SINK_12:%.*]] = select i1 [[TMP13]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_12:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_12]], i64 48
-; CHECK-NEXT: [[L12:%.*]] = load i32, ptr [[GEP_12]], align 4
-; CHECK-NEXT: [[VECINS_12:%.*]] = insertelement <16 x i32> [[VECINS_11]], i32 [[L12]], i64 12
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x i1> [[TMP0]], i64 13
-; CHECK-NEXT: [[A_SINK_13:%.*]] = select i1 [[TMP14]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_13:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_13]], i64 52
-; CHECK-NEXT: [[L13:%.*]] = load i32, ptr [[GEP_13]], align 4
-; CHECK-NEXT: [[VECINS_13:%.*]] = insertelement <16 x i32> [[VECINS_12]], i32 [[L13]], i64 13
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x i1> [[TMP0]], i64 14
-; CHECK-NEXT: [[A_SINK_14:%.*]] = select i1 [[TMP15]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_14:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_14]], i64 56
-; CHECK-NEXT: [[L14:%.*]] = load i32, ptr [[GEP_14]], align 4
-; CHECK-NEXT: [[VECINS_14:%.*]] = insertelement <16 x i32> [[VECINS_13]], i32 [[L14]], i64 14
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <16 x i1> [[TMP0]], i64 15
-; CHECK-NEXT: [[A_SINK_15:%.*]] = select i1 [[TMP16]], ptr [[A]], ptr [[B]]
-; CHECK-NEXT: [[GEP_15:%.*]] = getelementptr inbounds i8, ptr [[A_SINK_15]], i64 60
-; CHECK-NEXT: [[L15:%.*]] = load i32, ptr [[GEP_15]], align 4
-; CHECK-NEXT: [[VECINS_15:%.*]] = insertelement <16 x i32> [[VECINS_14]], i32 [[L15]], i64 15
+; CHECK-NEXT: [[TMP1:%.*]] = xor <16 x i1> [[TMP0]], splat (i1 true)
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[A]], <16 x i1> [[TMP0]], <16 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 4 [[B]], <16 x i1> [[TMP1]], <16 x i32> poison)
+; CHECK-NEXT: [[VECINS_15:%.*]] = select <16 x i1> [[TMP0]], <16 x i32> [[TMP2]], <16 x i32> [[TMP3]]
; CHECK-NEXT: ret <16 x i32> [[VECINS_15]]
;
entry:
More information about the llvm-commits
mailing list