[llvm] 3e47f00 - [LV] Consider ExtractValue as uniform.
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 5 08:21:50 PDT 2021
Author: Sander de Smalen
Date: 2021-08-05T16:20:50+01:00
New Revision: 3e47f009ff2c28c2a24150b1da46b61afa847a44
URL: https://github.com/llvm/llvm-project/commit/3e47f009ff2c28c2a24150b1da46b61afa847a44
DIFF: https://github.com/llvm/llvm-project/commit/3e47f009ff2c28c2a24150b1da46b61afa847a44.diff
LOG: [LV] Consider ExtractValue as uniform.
Since all operands to ExtractValue must be loop-invariant when we deem
the loop vectorizable, we can consider ExtractValue to be uniform.
Reviewed By: david-arm
Differential Revision: https://reviews.llvm.org/D107286
Added:
llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3c191e076858..49c853799f10 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5441,6 +5441,15 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
}
}
+ // ExtractValue instructions must be uniform, because the operands are
+ // known to be loop-invariant.
+ if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
+ assert(isOutOfScope(EVI->getAggregateOperand()) &&
+ "Expected aggregate value to be loop invariant");
+ addToWorklistIfAllowed(EVI);
+ continue;
+ }
+
// If there's no pointer operand, there's nothing to do.
auto *Ptr = getLoadStorePointerOperand(&I);
if (!Ptr)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
index 4bec0ad3eea1..14ac57cd8e88 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/extractvalue-no-scalarization-required.ll
@@ -8,6 +8,9 @@
; Check scalar cost for extractvalue. The constant and loop invariant operands are free,
; leaving cost 3 for scalarizing the result + 2 for executing the op with VF 2.
+; CM: LV: Found uniform instruction: %a = extractvalue { i64, i64 } %sv, 0
+; CM: LV: Found uniform instruction: %b = extractvalue { i64, i64 } %sv, 1
+
; CM: LV: Scalar loop costs: 5.
; CM: LV: Found an estimated cost of 0 for VF 2 For instruction: %a = extractvalue { i64, i64 } %sv, 0
; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction: %b = extractvalue { i64, i64 } %sv, 1
@@ -18,21 +21,19 @@
; FORCED-NEXT: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
; FORCED-NEXT: %0 = add i32 %index, 0
; FORCED-NEXT: %1 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT: %2 = extractvalue { i64, i64 } %sv, 0
-; FORCED-NEXT: %3 = insertelement <2 x i64> poison, i64 %1, i32 0
-; FORCED-NEXT: %4 = insertelement <2 x i64> %3, i64 %2, i32 1
-; FORCED-NEXT: %5 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT: %6 = extractvalue { i64, i64 } %sv, 1
-; FORCED-NEXT: %7 = insertelement <2 x i64> poison, i64 %5, i32 0
-; FORCED-NEXT: %8 = insertelement <2 x i64> %7, i64 %6, i32 1
-; FORCED-NEXT: %9 = getelementptr i64, i64* %dst, i32 %0
-; FORCED-NEXT: %10 = add <2 x i64> %4, %8
-; FORCED-NEXT: %11 = getelementptr i64, i64* %9, i32 0
-; FORCED-NEXT: %12 = bitcast i64* %11 to <2 x i64>*
-; FORCED-NEXT: store <2 x i64> %10, <2 x i64>* %12, align 4
+; FORCED-NEXT: %broadcast.splatinsert = insertelement <2 x i64> poison, i64 %1, i32 0
+; FORCED-NEXT: %broadcast.splat = shufflevector <2 x i64> %broadcast.splatinsert, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT: %2 = extractvalue { i64, i64 } %sv, 1
+; FORCED-NEXT: %broadcast.splatinsert1 = insertelement <2 x i64> poison, i64 %2, i32 0
+; FORCED-NEXT: %broadcast.splat2 = shufflevector <2 x i64> %broadcast.splatinsert1, <2 x i64> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT: %3 = getelementptr i64, i64* %dst, i32 %0
+; FORCED-NEXT: %4 = add <2 x i64> %broadcast.splat, %broadcast.splat2
+; FORCED-NEXT: %5 = getelementptr i64, i64* %3, i32 0
+; FORCED-NEXT: %6 = bitcast i64* %5 to <2 x i64>*
+; FORCED-NEXT: store <2 x i64> %4, <2 x i64>* %6, align 4
; FORCED-NEXT: %index.next = add nuw i32 %index, 2
-; FORCED-NEXT: %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT: br i1 %13, label %middle.block, label %vector.body, !llvm.loop !0
+; FORCED-NEXT: %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT: br i1 %7, label %middle.block, label %vector.body, !llvm.loop !0
define void @test1(i64* %dst, {i64, i64} %sv) {
entry:
@@ -57,6 +58,9 @@ exit:
; Similar to the test case above, but checks getVectorCallCost as well.
declare float @pow(float, float) readnone nounwind
+; CM: LV: Found uniform instruction: %a = extractvalue { float, float } %sv, 0
+; CM: LV: Found uniform instruction: %b = extractvalue { float, float } %sv, 1
+
; CM: LV: Scalar loop costs: 14.
; CM: LV: Found an estimated cost of 0 for VF 2 For instruction: %a = extractvalue { float, float } %sv, 0
; CM-NEXT: LV: Found an estimated cost of 0 for VF 2 For instruction: %b = extractvalue { float, float } %sv, 1
@@ -67,21 +71,19 @@ declare float @pow(float, float) readnone nounwind
; FORCED-NEXT: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
; FORCED-NEXT: %0 = add i32 %index, 0
; FORCED-NEXT: %1 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT: %2 = extractvalue { float, float } %sv, 0
-; FORCED-NEXT: %3 = insertelement <2 x float> poison, float %1, i32 0
-; FORCED-NEXT: %4 = insertelement <2 x float> %3, float %2, i32 1
-; FORCED-NEXT: %5 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT: %6 = extractvalue { float, float } %sv, 1
-; FORCED-NEXT: %7 = insertelement <2 x float> poison, float %5, i32 0
-; FORCED-NEXT: %8 = insertelement <2 x float> %7, float %6, i32 1
-; FORCED-NEXT: %9 = getelementptr float, float* %dst, i32 %0
-; FORCED-NEXT: %10 = call <2 x float> @llvm.pow.v2f32(<2 x float> %4, <2 x float> %8)
-; FORCED-NEXT: %11 = getelementptr float, float* %9, i32 0
-; FORCED-NEXT: %12 = bitcast float* %11 to <2 x float>*
-; FORCED-NEXT: store <2 x float> %10, <2 x float>* %12, align 4
+; FORCED-NEXT: %broadcast.splatinsert = insertelement <2 x float> poison, float %1, i32 0
+; FORCED-NEXT: %broadcast.splat = shufflevector <2 x float> %broadcast.splatinsert, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT: %2 = extractvalue { float, float } %sv, 1
+; FORCED-NEXT: %broadcast.splatinsert1 = insertelement <2 x float> poison, float %2, i32 0
+; FORCED-NEXT: %broadcast.splat2 = shufflevector <2 x float> %broadcast.splatinsert1, <2 x float> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT: %3 = getelementptr float, float* %dst, i32 %0
+; FORCED-NEXT: %4 = call <2 x float> @llvm.pow.v2f32(<2 x float> %broadcast.splat, <2 x float> %broadcast.splat2)
+; FORCED-NEXT: %5 = getelementptr float, float* %3, i32 0
+; FORCED-NEXT: %6 = bitcast float* %5 to <2 x float>*
+; FORCED-NEXT: store <2 x float> %4, <2 x float>* %6, align 4
; FORCED-NEXT: %index.next = add nuw i32 %index, 2
-; FORCED-NEXT: %13 = icmp eq i32 %index.next, 0
-; FORCED-NEXT: br i1 %13, label %middle.block, label %vector.body, !llvm.loop !4
+; FORCED-NEXT: %7 = icmp eq i32 %index.next, 0
+; FORCED-NEXT: br i1 %7, label %middle.block, label %vector.body, !llvm.loop !4
define void @test_getVectorCallCost(float* %dst, {float, float} %sv) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
new file mode 100644
index 000000000000..13d619b030f1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
@@ -0,0 +1,41 @@
+; RUN: opt -S -loop-vectorize -scalable-vectorization=on < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define void @widen_extractvalue(i64* %dst, {i64, i64} %sv) #0 {
+; CHECK-LABEL: @widen_extractvalue(
+; CHECK: vector.body:
+; CHECK: [[EXTRACT0:%.*]] = extractvalue { i64, i64 } [[SV:%.*]], 0
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT0]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT: [[EXTRACT1:%.*]] = extractvalue { i64, i64 } [[SV]], 1
+; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT1]], i32 0
+; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK: [[ADD:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[DOTSPLAT2]]
+entry:
+ br label %loop.body
+
+loop.body:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.body ]
+ %a = extractvalue { i64, i64 } %sv, 0
+ %b = extractvalue { i64, i64 } %sv, 1
+ %addr = getelementptr i64, i64* %dst, i32 %iv
+ %add = add i64 %a, %b
+ store i64 %add, i64* %addr
+ %iv.next = add nsw i32 %iv, 1
+ %cond = icmp ne i32 %iv.next, 0
+ br i1 %cond, label %loop.body, label %exit, !llvm.loop !0
+
+exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+!0 = distinct !{!0, !1, !2, !3, !4, !5}
+!1 = !{!"llvm.loop.mustprogress"}
+!2 = !{!"llvm.loop.vectorize.width", i32 2}
+!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!4 = !{!"llvm.loop.interleave.count", i32 1}
+!5 = !{!"llvm.loop.vectorize.enable", i1 true}
+
More information about the llvm-commits
mailing list