[llvm] [SLP][REVEC] Make tryToReduce and related functions support vector instructions. (PR #102327)
Han-Kuan Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 12 00:46:15 PDT 2024
https://github.com/HanKuanChen updated https://github.com/llvm/llvm-project/pull/102327
>From daffaf3a0859ababf196b95f076ce7adff3bbb9f Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Mon, 1 Jul 2024 21:29:29 -0700
Subject: [PATCH 1/4] [SLP][REVEC] Pre-commit test.
---
llvm/test/Transforms/SLPVectorizer/revec.ll | 32 +++++++++++++++++++++
1 file changed, 32 insertions(+)
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index d6dd4128de9c7e..9f41bcda5515dc 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -124,3 +124,35 @@ entry:
store <8 x i1> %6, ptr %7, align 1
ret void
}
+
+define <4 x i1> @test5(ptr %in1, ptr %in2) {
+entry:
+ %0 = load <4 x i32>, ptr %in1, align 4
+ %1 = load <4 x i16>, ptr %in2, align 2
+ %cmp000 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp001 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp002 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp003 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp100 = icmp eq <4 x i16> %1, zeroinitializer
+ %cmp101 = icmp eq <4 x i16> %1, zeroinitializer
+ %cmp102 = icmp eq <4 x i16> %1, zeroinitializer
+ %cmp103 = icmp eq <4 x i16> %1, zeroinitializer
+ %and.cmp0 = and <4 x i1> %cmp000, %cmp100
+ %and.cmp1 = and <4 x i1> %cmp001, %cmp101
+ %and.cmp2 = and <4 x i1> %cmp002, %cmp102
+ %and.cmp3 = and <4 x i1> %cmp003, %cmp103
+ %cmp004 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp005 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp006 = icmp ugt <4 x i32> %0, zeroinitializer
+ %cmp007 = icmp ugt <4 x i32> %0, zeroinitializer
+ %and.cmp4 = and <4 x i1> %and.cmp0, %cmp004
+ %and.cmp5 = and <4 x i1> %and.cmp1, %cmp005
+ %and.cmp6 = and <4 x i1> %and.cmp2, %cmp006
+ %and.cmp7 = and <4 x i1> %and.cmp3, %cmp007
+ %or0 = or <4 x i1> %and.cmp5, %and.cmp4
+ %or1 = or <4 x i1> %or0, %and.cmp6
+ %or2 = or <4 x i1> %or1, %and.cmp7
+ %vbsl = select <4 x i1> %or2, <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>
+ %cmp = icmp ugt <4 x i32> %vbsl, <i32 2, i32 3, i32 4, i32 5>
+ ret <4 x i1> %cmp
+}
>From 36c26e826cc2daaaafd43fb97ec6d662332703f4 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Fri, 28 Jun 2024 02:49:59 -0700
Subject: [PATCH 2/4] [SLP][REVEC] Make tryToReduce and related functions
support vector instructions.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 41 +++++++++++++++---
llvm/test/Transforms/SLPVectorizer/revec.ll | 43 +++++++++++++++++++
2 files changed, 79 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4186b17e644b0b..7f96c918109105 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17710,8 +17710,25 @@ class HorizontalReduction {
SameValuesCounter, TrackedToOrig);
}
- Value *ReducedSubTree =
- emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
+ Value *ReducedSubTree;
+ Type *ScalarTy = VL.front()->getType();
+ if (isa<FixedVectorType>(ScalarTy)) {
+ assert(SLPReVec && "FixedVectorType is not expected.");
+ unsigned ScalarTyNumElements = getNumElements(ScalarTy);
+ ReducedSubTree = PoisonValue::get(FixedVectorType::get(
+ VectorizedRoot->getType()->getScalarType(), ScalarTyNumElements));
+ for (unsigned I = 0; I != ScalarTyNumElements; ++I) {
+ // Do reduction for each lane.
+ SmallVector<int, 16> Mask =
+ createStrideMask(I, ScalarTyNumElements, VL.size());
+ Value *Lane = Builder.CreateShuffleVector(VectorizedRoot, Mask);
+ ReducedSubTree = Builder.CreateInsertElement(
+ ReducedSubTree, emitReduction(Lane, Builder, ReduxWidth, TTI),
+ I);
+ }
+ } else
+ ReducedSubTree =
+ emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
if (ReducedSubTree->getType() != VL.front()->getType()) {
assert(ReducedSubTree->getType() != VL.front()->getType() &&
"Expected different reduction type.");
@@ -17939,9 +17956,23 @@ class HorizontalReduction {
case RecurKind::FAdd:
case RecurKind::FMul: {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
- if (!AllConsts)
- VectorCost =
- TTI->getArithmeticReductionCost(RdxOpcode, VectorTy, FMF, CostKind);
+ if (!AllConsts) {
+ if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ assert(SLPReVec && "FixedVectorType is not expected.");
+ unsigned ScalarTyNumElements = VecTy->getNumElements();
+ for (unsigned I = 0, End = ReducedVals.size(); I != End; ++I) {
+ VectorCost += TTI->getShuffleCost(
+ TTI::SK_PermuteSingleSrc, VectorTy,
+ createStrideMask(I, ScalarTyNumElements, End));
+ VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy, FMF,
+ CostKind);
+ VectorCost += TTI->getVectorInstrCost(
+ Instruction::InsertElement, VecTy, TTI::TCK_RecipThroughput, I);
+ }
+ } else
+ VectorCost = TTI->getArithmeticReductionCost(RdxOpcode, VectorTy, FMF,
+ CostKind);
+ }
ScalarCost = EvaluateScalarCost([&]() {
return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
});
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 9f41bcda5515dc..39fc906535948c 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -126,6 +126,49 @@ entry:
}
define <4 x i1> @test5(ptr %in1, ptr %in2) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[IN1:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i16>, ptr [[IN2:%.*]], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> poison, <4 x i32> poison, i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP2]], <4 x i32> poison, i64 8)
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP3]], <4 x i32> poison, i64 12)
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP4]], <4 x i32> [[TMP0]], i64 0)
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x i32> [[TMP5]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> poison, <4 x i32> zeroinitializer, i64 0)
+; CHECK-NEXT: [[TMP8:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP7]], <4 x i32> zeroinitializer, i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP8]], <4 x i32> zeroinitializer, i64 8)
+; CHECK-NEXT: [[TMP10:%.*]] = call <16 x i32> @llvm.vector.insert.v16i32.v4i32(<16 x i32> [[TMP9]], <4 x i32> zeroinitializer, i64 12)
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt <16 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> poison, <4 x i16> poison, i64 4)
+; CHECK-NEXT: [[TMP13:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP12]], <4 x i16> poison, i64 8)
+; CHECK-NEXT: [[TMP14:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP13]], <4 x i16> poison, i64 12)
+; CHECK-NEXT: [[TMP15:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP14]], <4 x i16> [[TMP1]], i64 0)
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <16 x i16> [[TMP15]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP17:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> poison, <4 x i16> zeroinitializer, i64 0)
+; CHECK-NEXT: [[TMP18:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP17]], <4 x i16> zeroinitializer, i64 4)
+; CHECK-NEXT: [[TMP19:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP18]], <4 x i16> zeroinitializer, i64 8)
+; CHECK-NEXT: [[TMP20:%.*]] = call <16 x i16> @llvm.vector.insert.v16i16.v4i16(<16 x i16> [[TMP19]], <4 x i16> zeroinitializer, i64 12)
+; CHECK-NEXT: [[TMP21:%.*]] = icmp eq <16 x i16> [[TMP16]], [[TMP20]]
+; CHECK-NEXT: [[TMP22:%.*]] = and <16 x i1> [[TMP11]], [[TMP21]]
+; CHECK-NEXT: [[TMP23:%.*]] = icmp ugt <16 x i32> [[TMP6]], [[TMP10]]
+; CHECK-NEXT: [[TMP24:%.*]] = and <16 x i1> [[TMP22]], [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
+; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP28]])
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i1> [[TMP27]], i1 [[TMP29]], i64 1
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-NEXT: [[TMP32:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP31]])
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i1> [[TMP30]], i1 [[TMP32]], i64 2
+; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-NEXT: [[TMP35:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP34]])
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i1> [[TMP33]], i1 [[TMP35]], i64 3
+; CHECK-NEXT: [[VBSL:%.*]] = select <4 x i1> [[TMP36]], <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <4 x i32> [[VBSL]], <i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT: ret <4 x i1> [[CMP]]
+;
entry:
%0 = load <4 x i32>, ptr %in1, align 4
%1 = load <4 x i16>, ptr %in2, align 2
>From d832702d3c38b672b84f335b50f3400f0d881a81 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Wed, 7 Aug 2024 19:51:56 -0700
Subject: [PATCH 3/4] [SLP][REVEC] Apply comments.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 23 +++++++++++++++----
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7f96c918109105..a2bd4a65fdc1f6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17717,8 +17717,19 @@ class HorizontalReduction {
unsigned ScalarTyNumElements = getNumElements(ScalarTy);
ReducedSubTree = PoisonValue::get(FixedVectorType::get(
VectorizedRoot->getType()->getScalarType(), ScalarTyNumElements));
- for (unsigned I = 0; I != ScalarTyNumElements; ++I) {
+ for (unsigned I : seq<unsigned>(ScalarTyNumElements)) {
// Do reduction for each lane.
+ // e.g., do reduce add for
+ // VL[0] = <4 x Ty> <a, b, c, d>
+ // VL[1] = <4 x Ty> <e, f, g, h>
+ // Lane[0] = <2 x Ty> <a, e>
+ // Lane[1] = <2 x Ty> <b, f>
+ // Lane[2] = <2 x Ty> <c, g>
+ // Lane[3] = <2 x Ty> <d, h>
+ // result[0] = reduce add Lane[0]
+ // result[1] = reduce add Lane[1]
+ // result[2] = reduce add Lane[2]
+ // result[3] = reduce add Lane[3]
SmallVector<int, 16> Mask =
createStrideMask(I, ScalarTyNumElements, VL.size());
Value *Lane = Builder.CreateShuffleVector(VectorizedRoot, Mask);
@@ -17726,9 +17737,10 @@ class HorizontalReduction {
ReducedSubTree, emitReduction(Lane, Builder, ReduxWidth, TTI),
I);
}
- } else
+ } else {
ReducedSubTree =
emitReduction(VectorizedRoot, Builder, ReduxWidth, TTI);
+ }
if (ReducedSubTree->getType() != VL.front()->getType()) {
assert(ReducedSubTree->getType() != VL.front()->getType() &&
"Expected different reduction type.");
@@ -17960,18 +17972,19 @@ class HorizontalReduction {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
- for (unsigned I = 0, End = ReducedVals.size(); I != End; ++I) {
+ for (unsigned I : seq<unsigned>(ReducedVals.size())) {
VectorCost += TTI->getShuffleCost(
TTI::SK_PermuteSingleSrc, VectorTy,
- createStrideMask(I, ScalarTyNumElements, End));
+ createStrideMask(I, ScalarTyNumElements, ReducedVals.size()));
VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy, FMF,
CostKind);
VectorCost += TTI->getVectorInstrCost(
Instruction::InsertElement, VecTy, TTI::TCK_RecipThroughput, I);
}
- } else
+ } else {
VectorCost = TTI->getArithmeticReductionCost(RdxOpcode, VectorTy, FMF,
CostKind);
+ }
}
ScalarCost = EvaluateScalarCost([&]() {
return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
>From 52a9f8119eb07eaa0bd40dd526af188a0636f9b3 Mon Sep 17 00:00:00 2001
From: Han-Kuan Chen <hankuan.chen at sifive.com>
Date: Mon, 12 Aug 2024 00:45:56 -0700
Subject: [PATCH 4/4] [SLP][REVEC] Apply comments.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a2bd4a65fdc1f6..2bd491ea2f865d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -17978,9 +17978,10 @@ class HorizontalReduction {
createStrideMask(I, ScalarTyNumElements, ReducedVals.size()));
VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy, FMF,
CostKind);
- VectorCost += TTI->getVectorInstrCost(
- Instruction::InsertElement, VecTy, TTI::TCK_RecipThroughput, I);
}
+ VectorCost += TTI->getScalarizationOverhead(
+ VecTy, APInt::getAllOnes(ScalarTyNumElements), /*Insert*/ true,
+ /*Extract*/ false, TTI::TCK_RecipThroughput);
} else {
VectorCost = TTI->getArithmeticReductionCost(RdxOpcode, VectorTy, FMF,
CostKind);
More information about the llvm-commits
mailing list