[llvm] a04f615 - [LV] Check for innermost loop instead of EnableVPlanNativePath in CM.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 25 04:50:36 PST 2024
Author: Florian Hahn
Date: 2024-01-25T12:49:52Z
New Revision: a04f6152914ea21f3068aaba9d8fc21d2e703d3e
URL: https://github.com/llvm/llvm-project/commit/a04f6152914ea21f3068aaba9d8fc21d2e703d3e
DIFF: https://github.com/llvm/llvm-project/commit/a04f6152914ea21f3068aaba9d8fc21d2e703d3e.diff
LOG: [LV] Check for innermost loop instead of EnableVPlanNativePath in CM.
Replace EnableVPlanNativePath checks in the cost-model by assertions
that the code is only called for innermost loops. This ensures that the
cost model isn't used in the VPlanNativePath, which is only used for
outer-loop vectorization.
Even with EnableVPlanNativePath, inner loops are processed by the
inner loop vectorization path, not the native path, so checking for
EnableVPlanNativePath may impact decisions for inner loops and can
cause crashes, like in the attached test case.
Added:
llvm/test/Transforms/LoopVectorize/vplan-native-inner-loop-only.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 6ca93e15719fb2..9a82d52c84a8a4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1298,11 +1298,9 @@ class LoopVectorizationCostModel {
bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
assert(VF.isVector() &&
"Profitable to scalarize relevant only for VF > 1.");
-
- // Cost model is not run in the VPlan-native path - return conservative
- // result until this changes.
- if (EnableVPlanNativePath)
- return false;
+ assert(
+ TheLoop->isInnermost() &&
+ "cost-model should not be used for outer loops (in VPlan-native path)");
auto Scalars = InstsToScalarize.find(VF);
assert(Scalars != InstsToScalarize.end() &&
@@ -1312,6 +1310,9 @@ class LoopVectorizationCostModel {
/// Returns true if \p I is known to be uniform after vectorization.
bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
+ assert(
+ TheLoop->isInnermost() &&
+ "cost-model should not be used for outer loops (in VPlan-native path)");
// Pseudo probe needs to be duplicated for each unrolled iteration and
// vector lane so that profiled loop trip count can be accurately
// accumulated instead of being under counted.
@@ -1321,11 +1322,6 @@ class LoopVectorizationCostModel {
if (VF.isScalar())
return true;
- // Cost model is not run in the VPlan-native path - return conservative
- // result until this changes.
- if (EnableVPlanNativePath)
- return false;
-
auto UniformsPerVF = Uniforms.find(VF);
assert(UniformsPerVF != Uniforms.end() &&
"VF not yet analyzed for uniformity");
@@ -1334,14 +1330,12 @@ class LoopVectorizationCostModel {
/// Returns true if \p I is known to be scalar after vectorization.
bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
+ assert(
+ TheLoop->isInnermost() &&
+ "cost-model should not be used for outer loops (in VPlan-native path)");
if (VF.isScalar())
return true;
- // Cost model is not run in the VPlan-native path - return conservative
- // result until this changes.
- if (EnableVPlanNativePath)
- return false;
-
auto ScalarsPerVF = Scalars.find(VF);
assert(ScalarsPerVF != Scalars.end() &&
"Scalar values are not calculated for VF");
@@ -1399,10 +1393,9 @@ class LoopVectorizationCostModel {
/// through the cost modeling.
InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
assert(VF.isVector() && "Expected VF to be a vector VF");
- // Cost model is not run in the VPlan-native path - return conservative
- // result until this changes.
- if (EnableVPlanNativePath)
- return CM_GatherScatter;
+ assert(
+ TheLoop->isInnermost() &&
+ "cost-model should not be used for outer loops (in VPlan-native path)");
std::pair<Instruction *, ElementCount> InstOnVF = std::make_pair(I, VF);
auto Itr = WideningDecisions.find(InstOnVF);
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-inner-loop-only.ll b/llvm/test/Transforms/LoopVectorize/vplan-native-inner-loop-only.ll
new file mode 100644
index 00000000000000..887feb7c03ef94
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-native-inner-loop-only.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -p loop-vectorize -enable-vplan-native-path -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; -enable-vplan-native-path should not impact codegen for inner loops.
+
+define void @test(ptr %A) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = shl nsw i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i64 [[TMP3]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 0
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[STRIDED_VEC]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
+; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
+; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP9]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
+; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
+; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP11]], align 4
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[SCALAR_PH]]
+; CHECK: scalar.ph:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[MUL:%.*]] = shl nsw i64 [[IV]], 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL]]
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add i32 [[L]], 2
+; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %mul = shl nsw i64 %iv, 1
+ %gep = getelementptr inbounds i32, ptr %A, i64 %mul
+ %l = load i32, ptr %gep, align 4
+ %add = add i32 %l, 2
+ store i32 %add, ptr %gep
+ %iv.next = add nsw i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, 100
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
More information about the llvm-commits
mailing list