[llvm-branch-commits] [llvm] [SLP][modularisation][NFC] Move full-vector width helpers to SLPTypeUtils (2/2) (PR #214673)
Madhur Amilkanthwar via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 1 07:58:57 PDT 2026
https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/214673
>From 137b05aa3a43898ebf5f268b8c14cf1afaea9e84 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Wed, 5 Aug 2026 23:51:35 -0700
Subject: [PATCH 1/3] [SLP][modularisation][NFC] Move full-vector width helpers
to SLPTypeUtils (2/2)
Move the following BoUpSLP-independent helpers out of SLPVectorizer.cpp
into SLPVectorizer/SLPTypeUtils.{h,cpp}:
getFullVectorNumberOfElements
getFloorFullVectorNumberOfElements
getMaskedDivRemType
hasFullVectorsOrPowerOf2
They build on the type helpers moved in (1/2). Behavior is unchanged.
Part of the SLPVectorizer.cpp modularization effort:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 66 -------------------
.../Vectorize/SLPVectorizer/SLPTypeUtils.cpp | 56 ++++++++++++++++
.../Vectorize/SLPVectorizer/SLPTypeUtils.h | 28 ++++++++
3 files changed, 84 insertions(+), 66 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3fd0934be1653..2069efe4f3b28 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -351,54 +351,6 @@ static const int MinScheduleRegionSize = 16;
/// Maximum allowed number of operands in the PHI nodes.
static const unsigned MaxPHINumOperands = 128;
-/// Returns the number of elements of the given type \p Ty, not less than \p Sz,
-/// which forms type, which splits by \p TTI into whole vector types during
-/// legalization.
-static unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI,
- Type *Ty, unsigned Sz) {
- if (!isValidElementType(Ty, SLPReVec) || isa<StructType>(Ty))
- return bit_ceil(Sz);
- // Find the number of elements, which forms full vectors.
- const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
- if (NumParts == 0 || NumParts >= Sz)
- return bit_ceil(Sz);
- return bit_ceil(divideCeil(Sz, NumParts)) * NumParts;
-}
-
-/// Returns the number of elements of the given type \p Ty, not greater than \p
-/// Sz, which forms type, which splits by \p TTI into whole vector types during
-/// legalization.
-static unsigned
-getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz) {
- if (!isValidElementType(Ty, SLPReVec) || isa<StructType>(Ty))
- return bit_floor(Sz);
- // Find the number of elements, which forms full vectors.
- unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
- if (NumParts == 0 || NumParts >= Sz)
- return bit_floor(Sz);
- unsigned RegVF = bit_ceil(divideCeil(Sz, NumParts));
- if (RegVF > Sz)
- return bit_floor(Sz);
- return (Sz / RegVF) * RegVF;
-}
-
-/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the
-/// padded full-register vector type if padding is structurally possible, or
-/// nullptr if the vector already fills a register or the opcode is not
-/// div/rem. Does not check profitability; see getMaskedDivRemCost for that.
-static FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
- unsigned Opcode, Type *ScalarTy,
- unsigned NumElts) {
- if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts))
- return nullptr;
- unsigned PaddedNumElts =
- getFullVectorNumberOfElements(TTI, ScalarTy, NumElts);
- if (PaddedNumElts == NumElts)
- return nullptr;
- return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts));
-}
-
/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, checks if
/// padding to a full register and using the masked div/rem intrinsic is
/// cheaper than the direct vector op. Returns the cost of the masked
@@ -540,24 +492,6 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
: TargetTransformInfo::SK_PermuteSingleSrc;
}
-/// Returns true if widened type of \p Ty elements with size \p Sz represents
-/// full vector type, i.e. adding extra element results in extra parts upon type
-/// legalization.
-static bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz) {
- if (Sz <= 1)
- return false;
- if (!isValidElementType(Ty, SLPReVec) && !isa<FixedVectorType>(Ty))
- return false;
- if (has_single_bit(Sz))
- return true;
- if (isa<StructType>(Ty))
- return false;
- const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
- return NumParts > 0 && NumParts < Sz && has_single_bit(Sz / NumParts) &&
- Sz % NumParts == 0;
-}
-
/// Returns number of parts, the type \p VecTy will be split at the codegen
/// phase. If the type is going to be scalarized or does not uses whole
/// registers, returns 1.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
index 54ad1aff29b94..8fc27fb2a43c6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
@@ -11,12 +11,16 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVectorExtras.h"
+#include "llvm/ADT/bit.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/VectorTypeUtils.h"
#include "llvm/Support/Casting.h"
+#include "llvm/Support/MathExtras.h"
#include <cassert>
@@ -69,4 +73,56 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF) {
ElementCount::getFixed(VF * getNumElements(ScalarTy)));
}
+unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
+ unsigned Sz) {
+ if (!isValidElementType(Ty) || isa<StructType>(Ty))
+ return bit_ceil(Sz);
+ // Find the number of elements, which forms full vectors.
+ const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
+ if (NumParts == 0 || NumParts >= Sz)
+ return bit_ceil(Sz);
+ return bit_ceil(divideCeil(Sz, NumParts)) * NumParts;
+}
+
+unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
+ Type *Ty, unsigned Sz) {
+ if (!isValidElementType(Ty) || isa<StructType>(Ty))
+ return bit_floor(Sz);
+ // Find the number of elements, which forms full vectors.
+ unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
+ if (NumParts == 0 || NumParts >= Sz)
+ return bit_floor(Sz);
+ unsigned RegVF = bit_ceil(divideCeil(Sz, NumParts));
+ if (RegVF > Sz)
+ return bit_floor(Sz);
+ return (Sz / RegVF) * RegVF;
+}
+
+FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
+ unsigned Opcode, Type *ScalarTy,
+ unsigned NumElts) {
+ if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts))
+ return nullptr;
+ unsigned PaddedNumElts =
+ getFullVectorNumberOfElements(TTI, ScalarTy, NumElts);
+ if (PaddedNumElts == NumElts)
+ return nullptr;
+ return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts));
+}
+
+bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
+ unsigned Sz) {
+ if (Sz <= 1)
+ return false;
+ if (!isValidElementType(Ty) && !isa<FixedVectorType>(Ty))
+ return false;
+ if (has_single_bit(Sz))
+ return true;
+ if (isa<StructType>(Ty))
+ return false;
+ const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
+ return NumParts > 0 && NumParts < Sz && has_single_bit(Sz / NumParts) &&
+ Sz % NumParts == 0;
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
index f92c83ba660ee..dc9b0c6c0644f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
@@ -15,6 +15,8 @@
#define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPTYPEUTILS_H
namespace llvm {
+class FixedVectorType;
+class TargetTransformInfo;
class Type;
class Value;
} // namespace llvm
@@ -42,6 +44,32 @@ Type *getValueType(Value *V, bool ReVec, bool LookThroughCmp = false);
/// \returns the vector type of ScalarTy based on vectorization factor.
Type *getWidenedType(Type *ScalarTy, unsigned VF);
+/// Returns the number of elements of the given type \p Ty, not less than \p Sz,
+/// which forms type, which splits by \p TTI into whole vector types during
+/// legalization.
+unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
+ unsigned Sz);
+
+/// Returns the number of elements of the given type \p Ty, not greater than \p
+/// Sz, which forms type, which splits by \p TTI into whole vector types during
+/// legalization.
+unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
+ Type *Ty, unsigned Sz);
+
+/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the
+/// padded full-register vector type if padding is structurally possible, or
+/// nullptr if the vector already fills a register or the opcode is not
+/// div/rem. Does not check profitability; see getMaskedDivRemCost for that.
+FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
+ unsigned Opcode, Type *ScalarTy,
+ unsigned NumElts);
+
+/// Returns true if widened type of \p Ty elements with size \p Sz represents
+/// full vector type, i.e. adding extra element results in extra parts upon type
+/// legalization.
+bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
+ unsigned Sz);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPTYPEUTILS_H
>From 23366225d974203df565a6a2967a4ae8965ce35b Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 30 Aug 2026 21:36:13 -0700
Subject: [PATCH 2/3] fixup! [SLP][modularisation][NFC] Move full-vector width
helpers to SLPTypeUtils (2/2)
---
llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
index dc9b0c6c0644f..30c6f059d36e8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
@@ -59,7 +59,7 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the
/// padded full-register vector type if padding is structurally possible, or
/// nullptr if the vector already fills a register or the opcode is not
-/// div/rem. Does not check profitability; see getMaskedDivRemCost for that.
+/// div/rem. Does not check profitability.
FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
unsigned Opcode, Type *ScalarTy,
unsigned NumElts);
>From 0fb8db6d6e1baa53013025dd498e56f99377135c Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 30 Aug 2026 21:55:10 -0700
Subject: [PATCH 3/3] fixup! [SLP][modularisation][NFC] Move full-vector width
helpers to SLPTypeUtils (2/2)
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 93 ++++++++++---------
.../Vectorize/SLPVectorizer/SLPTypeUtils.cpp | 16 ++--
.../Vectorize/SLPVectorizer/SLPTypeUtils.h | 8 +-
3 files changed, 62 insertions(+), 55 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2069efe4f3b28..76b69eb767a26 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -361,7 +361,7 @@ getMaskedDivRemCost(const TargetTransformInfo &TTI, unsigned Opcode,
const TTI::TargetCostKind CostKind,
FixedVectorType **PaddedTy = nullptr) {
FixedVectorType *PaddedVecTy =
- getMaskedDivRemType(TTI, Opcode, ScalarTy, NumElts);
+ getMaskedDivRemType(TTI, Opcode, ScalarTy, NumElts, SLPReVec);
if (!PaddedVecTy)
return InstructionCost::getInvalid();
// One mask bit per element of the padded vector, not per padded lane.
@@ -506,10 +506,10 @@ getNumberOfParts(const TargetTransformInfo &TTI, Type *VecTy, Type *ScalarTy,
unsigned Sz = getNumElements(VecTy);
unsigned ScalarSz = getNumElements(ScalarTy);
Type *ElementTy = toScalarizedTy(VecTy);
- unsigned PWSz = getFullVectorNumberOfElements(TTI, ElementTy, Sz);
+ unsigned PWSz = getFullVectorNumberOfElements(TTI, ElementTy, Sz, SLPReVec);
if (NumParts >= Sz || PWSz % NumParts != 0 ||
(PWSz / NumParts) % ScalarSz != 0 ||
- !hasFullVectorsOrPowerOf2(TTI, ElementTy, PWSz / NumParts))
+ !hasFullVectorsOrPowerOf2(TTI, ElementTy, PWSz / NumParts, SLPReVec))
return 1;
const unsigned NumElts = PWSz / NumParts;
if (divideCeil(Sz, NumElts) != NumParts)
@@ -6295,7 +6295,8 @@ static bool isMaskedLoadCompress(
if (IsStrided && !IsMasked && Order.empty()) {
// Check for potential segmented(interleaved) loads.
VectorType *AlignedLoadVecTy = cast<VectorType>(getWidenedType(
- ScalarTy, getFullVectorNumberOfElements(TTI, ScalarTy, *Diff + 1)));
+ ScalarTy,
+ getFullVectorNumberOfElements(TTI, ScalarTy, *Diff + 1, SLPReVec)));
SimplifyQuery SQ(DL, &TLI, &DT, &AC, cast<LoadInst>(VL.back()));
if (!isSafeToLoadUnconditionally(Ptr0, AlignedLoadVecTy, CommonAlignment,
SQ))
@@ -6941,10 +6942,10 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
DemandedElts.clearAllBits();
// Iterate through possible vectorization factors and check if vectorized +
// shuffles is better than just gather.
- for (unsigned VF =
- getFloorFullVectorNumberOfElements(TTI, ScalarTy, VL.size() - 1);
- VF >= MinVF;
- VF = getFloorFullVectorNumberOfElements(TTI, ScalarTy, VF - 1)) {
+ for (unsigned VF = getFloorFullVectorNumberOfElements(
+ TTI, ScalarTy, VL.size() - 1, SLPReVec);
+ VF >= MinVF; VF = getFloorFullVectorNumberOfElements(
+ TTI, ScalarTy, VF - 1, SLPReVec)) {
SmallVector<std::pair<unsigned, LoadsState>> States;
for (unsigned Cnt = 0, End = VL.size(); Cnt < End; Cnt += VF) {
const unsigned SliceVF = std::min(VF, End - Cnt);
@@ -9236,9 +9237,10 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
if (isAllowedNonPowerOf2VF(MaxVF))
CandidateVFs.push_back(MaxVF);
for (int NumElts = getFloorFullVectorNumberOfElements(
- *TTI, Loads.front()->getType(), MaxVF);
- NumElts > 1; NumElts = getFloorFullVectorNumberOfElements(
- *TTI, Loads.front()->getType(), NumElts - 1)) {
+ *TTI, Loads.front()->getType(), MaxVF, SLPReVec);
+ NumElts > 1;
+ NumElts = getFloorFullVectorNumberOfElements(
+ *TTI, Loads.front()->getType(), NumElts - 1, SLPReVec)) {
CandidateVFs.push_back(NumElts);
if (VectorizeNonPowerOf2 && NumElts > 2)
CandidateVFs.push_back(NumElts - 1);
@@ -10894,8 +10896,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
S.getMainOp()->getDataLayout().getTypeSizeInBits(ScalarTy);
unsigned MinVF = R.getMinVF(EltBits);
auto RegWidth = [&](unsigned N) {
- return std::max(getFullVectorNumberOfElements(TTI, ScalarTy, N),
- MinVF);
+ return std::max(
+ getFullVectorNumberOfElements(TTI, ScalarTy, N, SLPReVec), MinVF);
};
// Keeping the originals just moves the reshuffle to the operand
// columns with duplicates; keep them only if at most one column has
@@ -11044,11 +11046,12 @@ bool BoUpSLP::canBuildSplitNode(ArrayRef<Value *> VL,
ReorderIndices.clear();
// When VL fills a power-of-2 register but the split halves do not, the
// reorder shuffle makes the split unprofitable - reject.
- else if (hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), VL.size()) &&
- (!hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(),
- Op1.size()) ||
- !hasFullVectorsOrPowerOf2(*TTI, Op2.front()->getType(),
- Op2.size())))
+ else if (hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), VL.size(),
+ SLPReVec) &&
+ (!hasFullVectorsOrPowerOf2(*TTI, Op1.front()->getType(), Op1.size(),
+ SLPReVec) ||
+ !hasFullVectorsOrPowerOf2(*TTI, Op2.front()->getType(), Op2.size(),
+ SLPReVec)))
return false;
SmallVector<int> Mask;
if (!ReorderIndices.empty())
@@ -14366,7 +14369,7 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
}
if (Sz > 1 && isa<Instruction>(P.second.front())) {
const unsigned SubVF = getFloorFullVectorNumberOfElements(
- *TTI, TE.Scalars.front()->getType(), Sz);
+ *TTI, TE.Scalars.front()->getType(), Sz, SLPReVec);
SubVectors.emplace_back(Cnt - Sz, SubVF);
for (unsigned I : seq<unsigned>(Cnt - Sz, Cnt - Sz + SubVF))
DemandedElts.clearBit(I);
@@ -14979,9 +14982,9 @@ void BoUpSLP::transformNodes() {
unsigned End = VL.size();
SmallBitVector Processed(End);
for (unsigned VF = getFloorFullVectorNumberOfElements(
- *TTI, VL.front()->getType(), VL.size() - 1);
+ *TTI, VL.front()->getType(), VL.size() - 1, SLPReVec);
VF >= MinVF; VF = getFloorFullVectorNumberOfElements(
- *TTI, VL.front()->getType(), VF - 1)) {
+ *TTI, VL.front()->getType(), VF - 1, SLPReVec)) {
if (StartIdx + VF > End)
continue;
SmallVector<std::pair<unsigned, unsigned>> Slices;
@@ -15667,7 +15670,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
SubMask);
}
const unsigned BaseVF = getFullVectorNumberOfElements(
- *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector));
+ *R.TTI, VL.front()->getType(), alignTo(NumElts, EltsPerVector),
+ SLPReVec);
for (const auto [Idx, SubVecSize] : zip(Indices, SubVecSizes)) {
assert((Idx + SubVecSize) <= BaseVF &&
"SK_ExtractSubvector index out of range");
@@ -21341,7 +21345,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
unsigned MinIdx = MinElement % VF;
if (MinIdx > 1) {
unsigned RegFloor = getFloorFullVectorNumberOfElements(
- *TTI, VL.front()->getType(), MinIdx);
+ *TTI, VL.front()->getType(), MinIdx, SLPReVec);
auto *RegFloorTy = getWidenedType(VL.front()->getType(), RegFloor);
unsigned RegFloorParts =
getNumberOfParts(RegFloorTy, VL.front()->getType(), RegFloor);
@@ -21487,8 +21491,8 @@ BoUpSLP::isGatherShuffledEntry(
Mask.assign(VL.size(), PoisonMaskElem);
assert((TE->UserTreeIndex || TE == &getRootNode()) &&
"Expected only single user of the gather node.");
- unsigned PWSz =
- getFullVectorNumberOfElements(*TTI, VL.front()->getType(), VL.size());
+ unsigned PWSz = getFullVectorNumberOfElements(*TTI, VL.front()->getType(),
+ VL.size(), SLPReVec);
if (TE->UserTreeIndex && TE->UserTreeIndex.UserTE->isGather() &&
TE->UserTreeIndex.EdgeIdx == UINT_MAX &&
(TE->Idx == 0 ||
@@ -28838,7 +28842,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
if (!has_single_bit(Sz) ||
!hasFullVectorsOrPowerOf2(
*TTI, cast<StoreInst>(Chain.front())->getValueOperand()->getType(),
- VF) ||
+ VF, SLPReVec) ||
VF < 2 || VF < MinVF) {
// Check if vectorizing with a non-power-of-2 VF should be considered; see
// isAllowedNonPowerOf2VF for supported widths.
@@ -28858,9 +28862,10 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
Analysis.buildInstructionsState(ValOps.getArrayRef(), R);
if (all_of(ValOps, IsaPred<Instruction>) && ValOps.size() > 1) {
DenseSet<Value *> Stores(Chain.begin(), Chain.end());
- bool IsAllowedSize = hasFullVectorsOrPowerOf2(
- *TTI, ValOps.front()->getType(), ValOps.size()) ||
- isAllowedNonPowerOf2VF(ValOps.size());
+ bool IsAllowedSize =
+ hasFullVectorsOrPowerOf2(*TTI, ValOps.front()->getType(), ValOps.size(),
+ SLPReVec) ||
+ isAllowedNonPowerOf2VF(ValOps.size());
if ((!IsAllowedSize && S && S.getOpcode() != Instruction::Load &&
(!S.getMainOp()->isSafeToRemove() ||
any_of(ValOps.getArrayRef(),
@@ -29305,7 +29310,7 @@ bool StoreChainContext::updateCandidateVFs(const TargetTransformInfo &TTI) {
// Attempt again to vectorize even larger chains if all previous
// attempts were unsuccessful because of the cost issues.
unsigned Limit =
- getFloorFullVectorNumberOfElements(TTI, StoreTy, MaxTotalNum);
+ getFloorFullVectorNumberOfElements(TTI, StoreTy, MaxTotalNum, SLPReVec);
if (bit_floor(Limit) == VF && Limit != VF)
CandidateVFs.push(Limit);
CandidateVFs.push(VF);
@@ -29879,11 +29884,12 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
Type *ScalarTy = getValueType(VL[0], SLPReVec, /*LookThroughCmp=*/true);
unsigned Sz = R.getVectorElementSize(I0);
unsigned MinVF = R.getMinVF(Sz);
- unsigned MaxVF = std::max<unsigned>(
- isAllowedNonPowerOf2VF(VL.size())
- ? VL.size()
- : getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()),
- MinVF);
+ unsigned MaxVF =
+ std::max<unsigned>(isAllowedNonPowerOf2VF(VL.size())
+ ? VL.size()
+ : getFloorFullVectorNumberOfElements(
+ *TTI, ScalarTy, VL.size(), SLPReVec),
+ MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
// Standalone seeds only need one register worth of lanes.
if (StandaloneSeeds && Sz != 0)
@@ -29903,7 +29909,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
unsigned NextInst = 0, MaxInst = VL.size();
for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF;
- VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1)) {
+ VF = getFloorFullVectorNumberOfElements(*TTI, I0->getType(), VF - 1,
+ SLPReVec)) {
// No actual vectorization should happen, if number of parts is the same as
// provided vectorization factor (i.e. the scalar type is used for vector
// code during codegen).
@@ -29913,7 +29920,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
for (unsigned I = NextInst; I < MaxInst; ++I) {
unsigned ActualVF = std::min(MaxInst - I, VF);
- if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF) &&
+ if (!hasFullVectorsOrPowerOf2(*TTI, ScalarTy, ActualVF, SLPReVec) &&
(ActualVF != VL.size() || !isAllowedNonPowerOf2VF(ActualVF)))
continue;
@@ -31251,8 +31258,8 @@ class HorizontalReduction {
auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) {
unsigned NumParts, NumRegs;
Type *ScalarTy = Candidates.front()->getType();
- ReduxWidth =
- getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth);
+ ReduxWidth = getFloorFullVectorNumberOfElements(TTI, ScalarTy,
+ ReduxWidth, SLPReVec);
VectorType *Tp = cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth));
NumParts = ::getNumberOfParts(TTI, Tp, ScalarTy);
NumRegs =
@@ -31446,10 +31453,10 @@ class HorizontalReduction {
if (ReduxWidth > ReductionLimit && V.isTreeNotExtendable()) {
// Add subvectors of VL to the list of the analyzed values.
for (unsigned VF = getFloorFullVectorNumberOfElements(
- *TTI, VL.front()->getType(), ReduxWidth - 1);
+ *TTI, VL.front()->getType(), ReduxWidth - 1, SLPReVec);
VF >= ReductionLimit;
VF = getFloorFullVectorNumberOfElements(
- *TTI, VL.front()->getType(), VF - 1)) {
+ *TTI, VL.front()->getType(), VF - 1, SLPReVec)) {
if (has_single_bit(VF) &&
V.getCanonicalGraphSize() != V.getTreeSize())
continue;
@@ -31819,8 +31826,8 @@ class HorizontalReduction {
unsigned ReduxWidth = 0;
auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) {
Type *ScalarTy = Candidates.front()->getType();
- ReduxWidth =
- getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth);
+ ReduxWidth = getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth,
+ SLPReVec);
Type *Tp = getWidenedType(ScalarTy, ReduxWidth);
unsigned NumParts = ::getNumberOfParts(TTI, Tp, ScalarTy);
unsigned NumRegs =
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
index 8fc27fb2a43c6..39e114561b689 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.cpp
@@ -74,8 +74,8 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF) {
}
unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz) {
- if (!isValidElementType(Ty) || isa<StructType>(Ty))
+ unsigned Sz, bool ReVec) {
+ if (!isValidElementType(Ty, ReVec) || isa<StructType>(Ty))
return bit_ceil(Sz);
// Find the number of elements, which forms full vectors.
const unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
@@ -85,8 +85,8 @@ unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
}
unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
- Type *Ty, unsigned Sz) {
- if (!isValidElementType(Ty) || isa<StructType>(Ty))
+ Type *Ty, unsigned Sz, bool ReVec) {
+ if (!isValidElementType(Ty, ReVec) || isa<StructType>(Ty))
return bit_floor(Sz);
// Find the number of elements, which forms full vectors.
unsigned NumParts = TTI.getNumberOfParts(getWidenedType(Ty, Sz));
@@ -100,21 +100,21 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
unsigned Opcode, Type *ScalarTy,
- unsigned NumElts) {
+ unsigned NumElts, bool ReVec) {
if (!Instruction::isIntDivRem(Opcode) || has_single_bit(NumElts))
return nullptr;
unsigned PaddedNumElts =
- getFullVectorNumberOfElements(TTI, ScalarTy, NumElts);
+ getFullVectorNumberOfElements(TTI, ScalarTy, NumElts, ReVec);
if (PaddedNumElts == NumElts)
return nullptr;
return cast<FixedVectorType>(getWidenedType(ScalarTy, PaddedNumElts));
}
bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz) {
+ unsigned Sz, bool ReVec) {
if (Sz <= 1)
return false;
- if (!isValidElementType(Ty) && !isa<FixedVectorType>(Ty))
+ if (!isValidElementType(Ty, ReVec) && !isa<FixedVectorType>(Ty))
return false;
if (has_single_bit(Sz))
return true;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
index 30c6f059d36e8..6e0ed47b6b966 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPTypeUtils.h
@@ -48,13 +48,13 @@ Type *getWidenedType(Type *ScalarTy, unsigned VF);
/// which forms type, which splits by \p TTI into whole vector types during
/// legalization.
unsigned getFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz);
+ unsigned Sz, bool ReVec);
/// Returns the number of elements of the given type \p Ty, not greater than \p
/// Sz, which forms type, which splits by \p TTI into whole vector types during
/// legalization.
unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
- Type *Ty, unsigned Sz);
+ Type *Ty, unsigned Sz, bool ReVec);
/// For a non-power-of-2 \p NumElts-wide integer div/rem \p Opcode, returns the
/// padded full-register vector type if padding is structurally possible, or
@@ -62,13 +62,13 @@ unsigned getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI,
/// div/rem. Does not check profitability.
FixedVectorType *getMaskedDivRemType(const TargetTransformInfo &TTI,
unsigned Opcode, Type *ScalarTy,
- unsigned NumElts);
+ unsigned NumElts, bool ReVec);
/// Returns true if widened type of \p Ty elements with size \p Sz represents
/// full vector type, i.e. adding extra element results in extra parts upon type
/// legalization.
bool hasFullVectorsOrPowerOf2(const TargetTransformInfo &TTI, Type *Ty,
- unsigned Sz);
+ unsigned Sz, bool ReVec);
} // namespace llvm::slpvectorizer
More information about the llvm-branch-commits
mailing list