[llvm] [SLP][REVEC] Make ShuffleCostEstimator and ShuffleInstructionBuilder support vector instructions. (PR #99499)
Han-Kuan Chen via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 6 07:17:28 PDT 2024
https://github.com/HanKuanChen updated https://github.com/llvm/llvm-project/pull/99499
>From 968f97d048b5c415eacbdf437db14a8faa47160e Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 00:24:24 -0700
Subject: [PATCH 01/10] [SLP][REVEC] Pre-commit test.
---
llvm/test/Transforms/SLPVectorizer/revec.ll | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index a6e1061189980..85f06dc72cc8a 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -88,3 +88,18 @@ entry:
store <4 x i32> %9, ptr %10, align 4
ret void
}
+
+define void @test4(ptr %in, ptr %out) {
+entry:
+ %0 = load <8 x float>, ptr %in, align 4
+ %1 = fmul <8 x float> %0, zeroinitializer
+ %2 = fmul <8 x float> %0, zeroinitializer
+ %3 = fadd <8 x float> zeroinitializer, %1
+ %4 = fadd <8 x float> %0, %2
+ %5 = fcmp ogt <8 x float> %3, zeroinitializer
+ %6 = fcmp ogt <8 x float> %4, zeroinitializer
+ %7 = getelementptr i1, ptr %out, i64 8
+ store <8 x i1> %5, ptr %out, align 1
+ store <8 x i1> %6, ptr %7, align 1
+ ret void
+}
>From c43c8e4ae83df46ed7db2c58b8ed8b80e6637840 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 00:33:11 -0700
Subject: [PATCH 02/10] [SLP][REVEC] Make ShuffleCostEstimator::gather support
vector instructions.
When REVEC is enabled, we need to expand vector types into scalar types.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 20 ++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8d2ce6bad6af7..56fd7ccbc3c75 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9172,12 +9172,30 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
Vals.push_back(Constant::getNullValue(V->getType()));
}
+ if (auto *VecTy = dyn_cast<FixedVectorType>(Vals.front()->getType())) {
+ // When REVEC is enabled, we need to expand vector types into scalar
+ // types.
+ unsigned VecTyNumElements = VecTy->getNumElements();
+ SmallVector<Constant *> NewVals;
+ NewVals.reserve(VL.size() * VecTyNumElements);
+ for (Constant *V : Vals)
+ for (unsigned I = 0; I != VecTyNumElements; ++I) {
+ Type *ScalarTy = V->getType()->getScalarType();
+ if (isa<PoisonValue>(V))
+ NewVals.push_back(PoisonValue::get(ScalarTy));
+ else if (isa<UndefValue>(V))
+ NewVals.push_back(UndefValue::get(ScalarTy));
+ else
+ NewVals.push_back(Constant::getNullValue(ScalarTy));
+ }
+ Vals.swap(NewVals);
+ }
return ConstantVector::get(Vals);
}
return ConstantVector::getSplat(
ElementCount::getFixed(
cast<FixedVectorType>(Root->getType())->getNumElements()),
- getAllOnesValue(*R.DL, ScalarTy));
+ getAllOnesValue(*R.DL, ScalarTy->getScalarType()));
}
InstructionCost createFreeze(InstructionCost Cost) { return Cost; }
/// Finalize emission of the shuffles.
>From fdaef5929ce66139772c07a56d11054c734a54d4 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 00:56:08 -0700
Subject: [PATCH 03/10] [SLP][REVEC] Make BoUpSLP::gather support vector
instructions.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 22 ++++++++++++++-----
1 file changed, 16 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 56fd7ccbc3c75..0931f6b725594 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -11695,8 +11695,8 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
Type *Ty) {
Value *Scalar = V;
if (Scalar->getType() != Ty) {
- assert(Scalar->getType()->isIntegerTy() && Ty->isIntegerTy() &&
- "Expected integer types only.");
+ assert(Scalar->getType()->isIntOrIntVectorTy() &&
+ Ty->isIntOrIntVectorTy() && "Expected integer types only.");
Value *V = Scalar;
if (auto *CI = dyn_cast<CastInst>(Scalar);
isa_and_nonnull<SExtInst, ZExtInst>(CI)) {
@@ -11709,10 +11709,20 @@ Value *BoUpSLP::gather(ArrayRef<Value *> VL, Value *Root, Type *ScalarTy) {
V, Ty, !isKnownNonNegative(Scalar, SimplifyQuery(*DL)));
}
- Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
- auto *InsElt = dyn_cast<InsertElementInst>(Vec);
- if (!InsElt)
- return Vec;
+ Instruction *InsElt;
+ if (auto *VecTy = dyn_cast<FixedVectorType>(Scalar->getType())) {
+ Vec = InsElt = Builder.CreateInsertVector(
+ Vec->getType(), Vec, V,
+ Builder.getInt64(Pos * VecTy->getNumElements()));
+ auto *II = dyn_cast<IntrinsicInst>(InsElt);
+ if (!II || II->getIntrinsicID() != Intrinsic::vector_insert)
+ return Vec;
+ } else {
+ Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
+ InsElt = dyn_cast<InsertElementInst>(Vec);
+ if (!InsElt)
+ return Vec;
+ }
GatherShuffleExtractSeq.insert(InsElt);
CSEBlocks.insert(InsElt->getParent());
// Add to our 'need-to-extract' list.
>From 1bfa95d06fbb8fcf2ef796fa26158f31d4f18888 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 01:19:03 -0700
Subject: [PATCH 04/10] [SLP][REVEC] NFC. Add
transformScalarShuffleIndiciesToVector.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0931f6b725594..82c8db069b7d8 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -253,6 +253,21 @@ static FixedVectorType *getWidenedType(Type *ScalarTy, unsigned VF) {
VF * getNumElements(ScalarTy));
}
+static void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements,
+ SmallVectorImpl<int> &Mask) {
+ // The ShuffleBuilder implementation use shufflevector to splat an "element".
+ // But the element have different meaning for SLP (scalar) and REVEC
+ // (vector). We need to expand Mask into masks which shufflevector can use
+ // directly.
+ SmallVector<int> NewMask(Mask.size() * VecTyNumElements);
+ for (unsigned I : seq<unsigned>(Mask.size()))
+ for (auto [J, MaskV] : enumerate(MutableArrayRef(NewMask).slice(
+ I * VecTyNumElements, VecTyNumElements)))
+ MaskV = Mask[I] == PoisonMaskElem ? PoisonMaskElem
+ : Mask[I] * VecTyNumElements + J;
+ Mask.swap(NewMask);
+}
+
/// \returns True if the value is a constant (but not globals/constant
/// expressions).
static bool isConstant(Value *V) {
>From 534f25c23707b98336e20bc921432cd674da26ed Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 01:22:42 -0700
Subject: [PATCH 05/10] [SLP][REVEC] Make ShuffleInstructionBuilder::finalize
support vector instructions.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 82c8db069b7d8..983f813d0ce03 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -12244,6 +12244,14 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
finalize(ArrayRef<int> ExtMask, unsigned VF = 0,
function_ref<void(Value *&, SmallVectorImpl<int> &)> Action = {}) {
IsFinalized = true;
+ SmallVector<int> NewExtMask(ExtMask);
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
+ CommonMask);
+ transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
+ NewExtMask);
+ ExtMask = NewExtMask;
+ }
if (Action) {
Value *Vec = InVectors.front();
if (InVectors.size() == 2) {
>From 3e632e1056c0dd9588f479667cc3860607dcee55 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Mon, 5 Aug 2024 23:08:30 -0700
Subject: [PATCH 06/10] [SLP][REVEC] NFC. Add a helper function to get VF for
ShuffleCostEstimator.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 31 ++++++++++++++++---
1 file changed, 27 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 983f813d0ce03..4f5374fc77531 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7777,6 +7777,31 @@ namespace {
/// The base class for shuffle instruction emission and shuffle cost estimation.
class BaseShuffleAnalysis {
protected:
+ Type *ScalarTy = nullptr;
+
+ BaseShuffleAnalysis(Type *ScalarTy) : ScalarTy(ScalarTy) {}
+
+ /// V is expected to be a vectorized value.
+ /// When REVEC is disabled, there is no difference between VF and
+ /// VNumElements.
+ /// When REVEC is enabled, VF is VNumElements / ScalarTyNumElements.
+ /// e.g., if ScalarTy is <4 x Ty> and V1 is <8 x Ty>, 2 is returned instead
+ /// of 8.
+ unsigned getVF(Value *V) const {
+ assert(V && "V cannot be nullptr");
+ assert(isa<FixedVectorType>(V->getType()) &&
+ "V does not have FixedVectorType");
+ assert(ScalarTy && "ScalarTy cannot be nullptr");
+ unsigned ScalarTyNumElements = getNumElements(ScalarTy);
+ unsigned VNumElements =
+ cast<FixedVectorType>(V->getType())->getNumElements();
+ assert(VNumElements > ScalarTyNumElements &&
+ "the number of elements of V is not large enough");
+ assert(VNumElements % ScalarTyNumElements == 0 &&
+ "the number of elements of V is not a vectorized value");
+ return VNumElements / ScalarTyNumElements;
+ }
+
/// Checks if the mask is an identity mask.
/// \param IsStrict if is true the function returns false if mask size does
/// not match vector size.
@@ -8273,7 +8298,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
bool IsFinalized = false;
SmallVector<int> CommonMask;
SmallVector<PointerUnion<Value *, const TreeEntry *>, 2> InVectors;
- Type *ScalarTy = nullptr;
const TargetTransformInfo &TTI;
InstructionCost Cost = 0;
SmallDenseSet<Value *> VectorizedVals;
@@ -8948,7 +8972,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
ShuffleCostEstimator(Type *ScalarTy, TargetTransformInfo &TTI,
ArrayRef<Value *> VectorizedVals, BoUpSLP &R,
SmallPtrSetImpl<Value *> &CheckedExtracts)
- : ScalarTy(ScalarTy), TTI(TTI),
+ : BaseShuffleAnalysis(ScalarTy), TTI(TTI),
VectorizedVals(VectorizedVals.begin(), VectorizedVals.end()), R(R),
CheckedExtracts(CheckedExtracts) {}
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
@@ -11838,7 +11862,6 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
/// resulting shuffle and the second operand sets to be the newly added
/// operand. The \p CommonMask is transformed in the proper way after that.
SmallVector<Value *, 2> InVectors;
- Type *ScalarTy = nullptr;
IRBuilderBase &Builder;
BoUpSLP &R;
@@ -11964,7 +11987,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
public:
ShuffleInstructionBuilder(Type *ScalarTy, IRBuilderBase &Builder, BoUpSLP &R)
- : ScalarTy(ScalarTy), Builder(Builder), R(R) {}
+ : BaseShuffleAnalysis(ScalarTy), Builder(Builder), R(R) {}
/// Adjusts extractelements after reusing them.
Value *adjustExtracts(const TreeEntry *E, MutableArrayRef<int> Mask,
>From a58871fcdde35039d364f61aede25481d44ca560 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Mon, 1 Jul 2024 11:46:06 -0700
Subject: [PATCH 07/10] [SLP][REVEC] Make ShuffleCostEstimator::add and
ShuffleInstructionBuilder::add support vector instructions.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4f5374fc77531..917cb8e473adf 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9177,7 +9177,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
}
assert(!InVectors.empty() && !CommonMask.empty() &&
"Expected only tree entries from extracts/reused buildvectors.");
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ unsigned VF = getVF(V1);
if (InVectors.size() == 2) {
Cost += createShuffle(InVectors.front(), InVectors.back(), CommonMask);
transformMaskAfterShuffle(CommonMask, CommonMask);
@@ -12244,7 +12244,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis {
break;
}
}
- int VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ int VF = getVF(V1);
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
if (Mask[Idx] != PoisonMaskElem && CommonMask[Idx] == PoisonMaskElem)
CommonMask[Idx] = Mask[Idx] + (It == InVectors.begin() ? 0 : VF);
>From 201cf72fa3c0818b1ecf16e87585051f17dbe453 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Mon, 1 Jul 2024 10:44:57 -0700
Subject: [PATCH 08/10] [SLP][REVEC] Make ShuffleCostEstimator::createShuffle
support vector instructions.
The VF is relative to the number of elements in ScalarTy instead of the
size of mask.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 917cb8e473adf..523295355c752 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -8879,14 +8879,14 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
} else if (V1 && P2.isNull()) {
// Shuffle single vector.
ExtraCost += GetValueMinBWAffectedCost(V1);
- CommonVF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ CommonVF = getVF(V1);
assert(
all_of(Mask,
[=](int Idx) { return Idx < static_cast<int>(CommonVF); }) &&
"All elements in mask must be less than CommonVF.");
} else if (V1 && !V2) {
// Shuffle vector and tree node.
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
+ unsigned VF = getVF(V1);
const TreeEntry *E2 = P2.get<const TreeEntry *>();
CommonVF = std::max(VF, E2->getVectorFactor());
assert(all_of(Mask,
@@ -8912,7 +8912,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else if (!V1 && V2) {
// Shuffle vector and tree node.
- unsigned VF = cast<FixedVectorType>(V2->getType())->getNumElements();
+ unsigned VF = getVF(V2);
const TreeEntry *E1 = P1.get<const TreeEntry *>();
CommonVF = std::max(VF, E1->getVectorFactor());
assert(all_of(Mask,
@@ -8940,9 +8940,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
} else {
assert(V1 && V2 && "Expected both vectors.");
- unsigned VF = cast<FixedVectorType>(V1->getType())->getNumElements();
- CommonVF =
- std::max(VF, cast<FixedVectorType>(V2->getType())->getNumElements());
+ unsigned VF = getVF(V1);
+ CommonVF = std::max(VF, getVF(V2));
assert(all_of(Mask,
[=](int Idx) {
return Idx < 2 * static_cast<int>(CommonVF);
@@ -8960,6 +8959,9 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
V2 = getAllOnesValue(*R.DL, getWidenedType(ScalarTy, CommonVF));
}
}
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy))
+ transformScalarShuffleIndiciesToVector(VecTy->getNumElements(),
+ CommonMask);
InVectors.front() =
Constant::getNullValue(getWidenedType(ScalarTy, CommonMask.size()));
if (InVectors.size() == 2)
>From ed8f565d5924899a0d9f76ecb05f94e16cd5bfa1 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Thu, 27 Jun 2024 01:38:32 -0700
Subject: [PATCH 09/10] [SLP][REVEC] Make ExtractAndExtendIfNeeded support
vector instructions.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 10 +++++++++
llvm/test/Transforms/SLPVectorizer/revec.ll | 21 +++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 523295355c752..5d1287e767b17 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14060,6 +14060,16 @@ Value *BoUpSLP::vectorizeTree(
if (GEP->hasName())
CloneGEP->takeName(GEP);
Ex = CloneGEP;
+ } else if (auto *VecTy =
+ dyn_cast<FixedVectorType>(Scalar->getType())) {
+ unsigned VecTyNumElements = VecTy->getNumElements();
+ // When REVEC is enabled, we need to extract a vector.
+ // Note: The element size of Scalar may be different from the
+ // element size of Vec.
+ Ex = Builder.CreateExtractVector(
+ FixedVectorType::get(Vec->getType()->getScalarType(),
+ VecTyNumElements),
+ Vec, Builder.getInt64(ExternalUse.Lane * VecTyNumElements));
} else {
Ex = Builder.CreateExtractElement(Vec, Lane);
}
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 85f06dc72cc8a..d6dd4128de9c7 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -90,6 +90,27 @@ entry:
}
define void @test4(ptr %in, ptr %out) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[IN:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> poison, i64 8)
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP1]], <8 x float> [[TMP0]], i64 0)
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> zeroinitializer, i64 0)
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP4]], <8 x float> zeroinitializer, i64 8)
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <16 x float> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> poison, <8 x float> poison, i64 0)
+; CHECK-NEXT: [[TMP8:%.*]] = call <16 x float> @llvm.vector.insert.v16f32.v8f32(<16 x float> [[TMP7]], <8 x float> zeroinitializer, i64 8)
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> [[TMP8]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP10:%.*]] = fadd <16 x float> [[TMP9]], [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = fcmp ogt <16 x float> [[TMP10]], [[TMP5]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i1, ptr [[OUT:%.*]], i64 8
+; CHECK-NEXT: [[TMP13:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[TMP11]], i64 8)
+; CHECK-NEXT: store <8 x i1> [[TMP13]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = call <8 x i1> @llvm.vector.extract.v8i1.v16i1(<16 x i1> [[TMP11]], i64 0)
+; CHECK-NEXT: store <8 x i1> [[TMP14]], ptr [[TMP12]], align 1
+; CHECK-NEXT: ret void
+;
entry:
%0 = load <8 x float>, ptr %in, align 4
%1 = fmul <8 x float> %0, zeroinitializer
>From ccd6f2bbb0671d12dc93e30f6d70340d9fe163ac Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Tue, 6 Aug 2024 07:15:52 -0700
Subject: [PATCH 10/10] [SLP][REVEC] Apply comments.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 25 ++++++++++---------
1 file changed, 13 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5d1287e767b17..33db28dff9385 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9217,18 +9217,19 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
// When REVEC is enabled, we need to expand vector types into scalar
// types.
unsigned VecTyNumElements = VecTy->getNumElements();
- SmallVector<Constant *> NewVals;
- NewVals.reserve(VL.size() * VecTyNumElements);
- for (Constant *V : Vals)
- for (unsigned I = 0; I != VecTyNumElements; ++I) {
- Type *ScalarTy = V->getType()->getScalarType();
- if (isa<PoisonValue>(V))
- NewVals.push_back(PoisonValue::get(ScalarTy));
- else if (isa<UndefValue>(V))
- NewVals.push_back(UndefValue::get(ScalarTy));
- else
- NewVals.push_back(Constant::getNullValue(ScalarTy));
- }
+ SmallVector<Constant *> NewVals(VF * VecTyNumElements, nullptr);
+ for (auto [I, V] : enumerate(Vals)) {
+ Type *ScalarTy = V->getType()->getScalarType();
+ Constant *NewVal;
+ if (isa<PoisonValue>(V))
+ NewVal = PoisonValue::get(ScalarTy);
+ else if (isa<UndefValue>(V))
+ NewVal = UndefValue::get(ScalarTy);
+ else
+ NewVal = Constant::getNullValue(ScalarTy);
+ std::fill_n(NewVals.begin() + I * VecTyNumElements, VecTyNumElements,
+ NewVal);
+ }
Vals.swap(NewVals);
}
return ConstantVector::get(Vals);
More information about the llvm-commits
mailing list