[llvm] bad4de1 - Don't disable loop unroll for vectorized loops on AMDGPU target

[Alexander Timofeev via llvm-commits]

Author: Alexander Timofeev
Date: 2023-05-25T22:54:41+02:00
New Revision: bad4de1ae7fa6dffec2f038118d6acedb9fce38e

URL: https://github.com/llvm/llvm-project/commit/bad4de1ae7fa6dffec2f038118d6acedb9fce38e
DIFF: https://github.com/llvm/llvm-project/commit/bad4de1ae7fa6dffec2f038118d6acedb9fce38e.diff

LOG: Don't disable loop unroll for vectorized loops on AMDGPU target

We've got a performance regression after the https://reviews.llvm.org/D115261.
Despite the loop being vectorized unroll is still required.

Reviewed By: rampitec

Differential Revision: https://reviews.llvm.org/D149281

Added: 
    llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll

Modified: 
    llvm/include/llvm/Analysis/TargetTransformInfo.h
    llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index df3b24ae1a8c8..7f03fa801298a 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -569,6 +569,8 @@ class TargetTransformInfo {
     /// Don't allow loop unrolling to simulate more than this number of
     /// iterations when checking full unroll profitability
     unsigned MaxIterationsCountToAnalyze;
+    /// Don't disable runtime unroll for the loops which were vectorized.
+    bool UnrollVectorizedLoop = false;
   };
 
   /// Get target-customized preferences for the generic loop unrolling

diff  --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index e4b5eac6c31bb..c2fd67790d9b6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -113,6 +113,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   // manipulations in average.
   UP.BEInsns += 3;
 
+  // We want to run unroll even for the loops which have been vectorized.
+  UP.UnrollVectorizedLoop = true;
+
   // TODO: Do we want runtime unrolling?
 
   // Maximum alloca size than can fit registers. Reserve 16 registers.

diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ad1326c761661..0bf3b97161f92 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7743,7 +7743,10 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
     LoopVectorizeHints Hints(L, true, *ORE);
     Hints.setAlreadyVectorized();
   }
-  AddRuntimeUnrollDisableMetaData(L);
+  TargetTransformInfo::UnrollingPreferences UP;
+  TTI.getUnrollingPreferences(L, *PSE.getSE(), UP, ORE);
+  if (!UP.UnrollVectorizedLoop || CanonicalIVStartValue)
+    AddRuntimeUnrollDisableMetaData(L);
 
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
   //    predication, updating analyses.

diff  --git a/llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll b/llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll
new file mode 100644
index 0000000000000..eccaae6cc7de3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll
@@ -0,0 +1,34 @@
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx90a -passes=loop-vectorize %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: @test
+; CHECK-LABEL: vector.body:
+; CHECK: br i1 %{{[0-9]+}}, label %middle.block, label %vector.body, !llvm.loop !0
+; CHECK-LABEL: middle.block:
+; CHECK-LABEL: scalar.ph:
+; CHECK-LABEL: loop.body:
+; CHECK: br i1 %cond, label %exit, label %loop.body, !llvm.loop !2
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: !2 = distinct !{!2, !3, !1}
+; CHECK: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+
+
+define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(3) %lds, i32 %n) {
+
+entry:
+  br label %loop.body
+
+loop.body:
+  %counter = phi i32 [0, %entry], [%inc, %loop.body]
+  %ptr_lds = getelementptr i32, ptr addrspace(3) %lds, i32 %counter
+  %val = load i32, ptr addrspace(3) %ptr_lds
+  %ptr_out = getelementptr i32, ptr addrspace(1) %out, i32 %counter
+  store i32 %val, ptr addrspace(1) %ptr_out
+  %inc = add i32 %counter, 1
+  %cond = icmp sge i32 %counter, %n
+  br i1 %cond, label  %exit, label %loop.body
+
+exit:
+  ret void
+}
+