[llvm] 8d307f5 - [SLP]Fix PR69246: do not treat resizing maskas identity.
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 24 08:17:54 PDT 2023
Author: Alexey Bataev
Date: 2023-10-24T08:14:13-07:00
New Revision: 8d307f59eed3ab6de688a555b0c6720fd8f18d10
URL: https://github.com/llvm/llvm-project/commit/8d307f59eed3ab6de688a555b0c6720fd8f18d10
DIFF: https://github.com/llvm/llvm-project/commit/8d307f59eed3ab6de688a555b0c6720fd8f18d10.diff
LOG: [SLP]Fix PR69246: do not treat resizing maskas identity.
If the mask is resizing and the mask size is greater than than the
length of the vector, being reused from extractelement instructions, the
mask for undefs cannot be treated as identity, must be treated as
a broadcast.
Added:
llvm/test/Transforms/SLPVectorizer/X86/extract-with-undefs-shuffle.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2a7129e17831b9d..c26bf5e1fb1b004 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -647,13 +647,16 @@ tryToGatherSingleRegisterExtractElements(MutableArrayRef<Value *> VL,
// Restore unused scalars from mask, if some of the extractelements were not
// selected for shuffle.
for (int I = 0, E = GatheredExtracts.size(); I < E; ++I) {
+ if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]) &&
+ isa<UndefValue>(GatheredExtracts[I])) {
+ std::swap(VL[I], GatheredExtracts[I]);
+ continue;
+ }
auto *EI = dyn_cast<ExtractElementInst>(VL[I]);
if (!EI || !isa<FixedVectorType>(EI->getVectorOperandType()) ||
!isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
is_contained(UndefVectorExtracts, I))
continue;
- if (Mask[I] == PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[I]))
- std::swap(VL[I], GatheredExtracts[I]);
}
return Res;
}
@@ -10142,7 +10145,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
inversePermutation(E->ReorderIndices, ReorderMask);
if (!ReorderMask.empty())
reorderScalars(GatheredScalars, ReorderMask);
- auto FindReusedSplat = [&](MutableArrayRef<int> Mask) {
+ auto FindReusedSplat = [&](MutableArrayRef<int> Mask, unsigned InputVF) {
if (!isSplat(E->Scalars) || none_of(E->Scalars, [](Value *V) {
return isa<UndefValue>(V) && !isa<PoisonValue>(V);
}))
@@ -10159,12 +10162,14 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
});
if (It == VectorizableTree.end())
return false;
- unsigned I =
- *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
- if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
+ if (Mask.size() <= InputVF &&
+ ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) {
std::iota(Mask.begin(), Mask.end(), 0);
- else
+ } else {
+ unsigned I =
+ *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; });
std::fill(Mask.begin(), Mask.end(), I);
+ }
return true;
};
BVTy ShuffleBuilder(Params...);
@@ -10370,7 +10375,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
isGuaranteedNotToBePoison(Vec1) && isGuaranteedNotToBePoison(Vec2);
ShuffleBuilder.add(Vec1, Vec2, ExtractMask);
} else if (Vec1) {
- IsUsedInExpr = FindReusedSplat(ExtractMask);
+ IsUsedInExpr = FindReusedSplat(
+ ExtractMask,
+ cast<FixedVectorType>(Vec1->getType())->getNumElements());
ShuffleBuilder.add(Vec1, ExtractMask);
IsNonPoisoned &= isGuaranteedNotToBePoison(Vec1);
} else {
@@ -10381,7 +10388,10 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) {
}
if (GatherShuffle) {
if (Entries.size() == 1) {
- IsUsedInExpr = FindReusedSplat(Mask);
+ IsUsedInExpr = FindReusedSplat(
+ Mask,
+ cast<FixedVectorType>(Entries.front()->VectorizedValue->getType())
+ ->getNumElements());
ShuffleBuilder.add(Entries.front()->VectorizedValue, Mask);
IsNonPoisoned &=
isGuaranteedNotToBePoison(Entries.front()->VectorizedValue);
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-with-undefs-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-with-undefs-shuffle.ll
new file mode 100644
index 000000000000000..491f3efbceb9ac9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-with-undefs-shuffle.ll
@@ -0,0 +1,54 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -S --passes=slp-vectorizer -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(i32 %arg) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT: bb:
+; CHECK-NEXT: br label [[BB1:%.*]]
+; CHECK: bb1:
+; CHECK-NEXT: br label [[BB13:%.*]]
+; CHECK: bb4:
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ poison, [[BB24:%.*]] ], [ [[TMP1:%.*]], [[BB13]] ]
+; CHECK-NEXT: br label [[BB1]]
+; CHECK: bb12:
+; CHECK-NEXT: br label [[BB13]]
+; CHECK: bb13:
+; CHECK-NEXT: [[TMP1]] = phi <2 x i32> [ zeroinitializer, [[BB1]] ], [ poison, [[BB12:%.*]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <8 x i32> [ zeroinitializer, [[BB1]] ], [ poison, [[BB12]] ]
+; CHECK-NEXT: br label [[BB4:%.*]]
+; CHECK: bb24:
+; CHECK-NEXT: br label [[BB4]]
+;
+bb:
+ br label %bb1
+
+bb1:
+ %0 = extractelement <2 x i32> zeroinitializer, i32 0
+ br label %bb13
+
+bb4:
+ %phi26 = phi i32 [ 0, %bb24 ], [ %phi14, %bb13 ]
+ %phi27 = phi i32 [ 0, %bb24 ], [ %phi16, %bb13 ]
+ br label %bb1
+
+bb12:
+ %add = add i32 0, 0
+ br label %bb13
+
+bb13:
+ %phi14 = phi i32 [ 0, %bb1 ], [ 0, %bb12 ]
+ %phi15 = phi i32 [ %0, %bb1 ], [ 0, %bb12 ]
+ %phi16 = phi i32 [ 0, %bb1 ], [ 0, %bb12 ]
+ %phi17 = phi i32 [ undef, %bb1 ], [ %arg, %bb12 ]
+ %phi18 = phi i32 [ undef, %bb1 ], [ %arg, %bb12 ]
+ %phi19 = phi i32 [ undef, %bb1 ], [ %arg, %bb12 ]
+ %phi20 = phi i32 [ undef, %bb1 ], [ %add, %bb12 ]
+ %phi21 = phi i32 [ undef, %bb1 ], [ 0, %bb12 ]
+ %phi22 = phi i32 [ undef, %bb1 ], [ 0, %bb12 ]
+ %phi23 = phi i32 [ undef, %bb1 ], [ 0, %bb12 ]
+ br label %bb4
+
+bb24:
+ br label %bb4
+}
More information about the llvm-commits
mailing list