[llvm] c94d32a - [AMDGPU] Increase max iterations count to analyze complete unroll
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 6 06:33:04 PDT 2020
Author: dfukalov
Date: 2020-06-06T16:32:45+03:00
New Revision: c94d32a6b30263ea448ac6ae0f95641b8574c5f5
URL: https://github.com/llvm/llvm-project/commit/c94d32a6b30263ea448ac6ae0f95641b8574c5f5
DIFF: https://github.com/llvm/llvm-project/commit/c94d32a6b30263ea448ac6ae0f95641b8574c5f5.diff
LOG: [AMDGPU] Increase max iterations count to analyze complete unroll
Summary: In some cases inner loops may not get boosts so try to analyze them deeper.
Reviewers: rampitec, mzolotukhin
Reviewed By: rampitec
Subscribers: arsenm, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, zzheng, kerbowa, llvm-commits
Tags: #llvm
Differential Revision: https://reviews.llvm.org/D81204
Added:
llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-analyze-small-loops.ll
Modified:
llvm/include/llvm/Analysis/TargetTransformInfo.h
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 9cb388c18e8c..06051552b9d5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -503,6 +503,9 @@ class TargetTransformInfo {
/// This value is used in the same manner to limit the size of the inner
/// loop.
unsigned UnrollAndJamInnerLoopThreshold;
+ /// Don't allow loop unrolling to simulate more than this number of
+ /// iterations when checking full unroll profitability
+ unsigned MaxIterationsCountToAnalyze;
};
/// Get target-customized preferences for the generic loop unrolling
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 094fb5f7b4d0..085ba4702b5c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -79,6 +79,11 @@ static cl::opt<bool> UseLegacyDA(
cl::desc("Enable legacy divergence analysis for AMDGPU"),
cl::init(false), cl::Hidden);
+static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
+ "amdgpu-unroll-max-block-to-analyze",
+ cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),
+ cl::init(20), cl::Hidden);
+
static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
unsigned Depth = 0) {
const Instruction *I = dyn_cast<Instruction>(Cond);
@@ -223,6 +228,11 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
if (UP.Threshold >= MaxBoost)
return;
}
+
+ // If we got a GEP in a small BB from inner loop then increase max trip
+ // count to analyze for better estimation cost in unroll
+ if (L->empty() && BB->size() < UnrollMaxBlockToAnalyze)
+ UP.MaxIterationsCountToAnalyze = 32;
}
}
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index e94eb4009d98..ec56610e41e5 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -223,6 +223,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.UnrollAndJam = false;
UP.PeelProfiledIterations = true;
UP.UnrollAndJamInnerLoopThreshold = 60;
+ UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP);
@@ -264,6 +265,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
if (UnrollUnrollRemainder.getNumOccurrences() > 0)
UP.UnrollRemainder = UnrollUnrollRemainder;
+ if (UnrollMaxIterationsCountToAnalyze.getNumOccurrences() > 0)
+ UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
// Apply user values provided by argument
if (UserThreshold.hasValue()) {
@@ -353,11 +356,12 @@ struct EstimatedUnrollCost {
static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
const Loop *L, unsigned TripCount, DominatorTree &DT, ScalarEvolution &SE,
const SmallPtrSetImpl<const Value *> &EphValues,
- const TargetTransformInfo &TTI, unsigned MaxUnrolledLoopSize) {
+ const TargetTransformInfo &TTI, unsigned MaxUnrolledLoopSize,
+ unsigned MaxIterationsCountToAnalyze) {
// We want to be able to scale offsets by the trip count and add more offsets
// to them without checking for overflows, and we already don't want to
// analyze *massive* trip counts, so we force the max to be reasonably small.
- assert(UnrollMaxIterationsCountToAnalyze <
+ assert(MaxIterationsCountToAnalyze <
(unsigned)(std::numeric_limits<int>::max() / 2) &&
"The unroll iterations max is too large!");
@@ -367,8 +371,7 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
return None;
// Don't simulate loops with a big or unknown tripcount
- if (!UnrollMaxIterationsCountToAnalyze || !TripCount ||
- TripCount > UnrollMaxIterationsCountToAnalyze)
+ if (!TripCount || TripCount > MaxIterationsCountToAnalyze)
return None;
SmallSetVector<BasicBlock *, 16> BBWorklist;
@@ -845,7 +848,8 @@ bool llvm::computeUnrollCount(
// To check that, run additional analysis on the loop.
if (Optional<EstimatedUnrollCost> Cost = analyzeLoopUnrollCost(
L, FullUnrollTripCount, DT, SE, EphValues, TTI,
- UP.Threshold * UP.MaxPercentThresholdBoost / 100)) {
+ UP.Threshold * UP.MaxPercentThresholdBoost / 100,
+ UP.MaxIterationsCountToAnalyze)) {
unsigned Boost =
getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);
if (Cost->UnrolledCost < UP.Threshold * Boost / 100) {
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-analyze-small-loops.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-analyze-small-loops.ll
new file mode 100644
index 000000000000..423d65c7d9fa
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-analyze-small-loops.ll
@@ -0,0 +1,49 @@
+; RUN: opt -S -mtriple=amdgcn-unknown-amdhsa -loop-unroll -unroll-threshold=150 < %s | FileCheck %s
+
+; Test that max iterations count to analyze (specific for the target)
+; is enough to make the inner loop completely unrolled
+define hidden void @foo(float addrspace(1)* %ptrG, float addrspace(3)* %ptrL, i32 %A, i32 %A2, i32 %M) {
+bb:
+ br label %bb2
+
+bb2: ; preds = %bb7, %bb
+ %i = phi i32 [ 0, %bb ], [ %i8, %bb7 ]
+ br label %bb4
+
+bb3: ; preds = %bb7
+ ret void
+
+bb4: ; preds = %bb10, %bb2
+ %i5 = phi i32 [ 0, %bb2 ], [ %i11, %bb10 ]
+ %i6 = add nuw nsw i32 %i5, %i
+ br label %for.body
+
+bb7: ; preds = %bb10
+ %i8 = add nuw nsw i32 %i, 1
+ %i9 = icmp eq i32 %i8, 8
+ br i1 %i9, label %bb3, label %bb2
+
+bb10: ; preds = %for.body
+ %i11 = add nuw nsw i32 %i5, 1
+ %cmpj = icmp ult i32 %i11, 8
+ br i1 %cmpj, label %bb7, label %bb4
+
+; CHECK: for.body:
+; CHECK-NOT: %phi = phi {{.*}}
+for.body: ; preds = %for.body, %bb4
+ %phi = phi i32 [ 0, %bb4 ], [ %k, %for.body ]
+ %mul = shl nuw nsw i32 %phi, 5
+ %add1 = add i32 %A, %mul
+ %add2 = add i32 %add1, %M
+ %arrayidx = getelementptr inbounds float, float addrspace(3)* %ptrL, i32 %add2
+ %bc = bitcast float addrspace(3)* %arrayidx to i32 addrspace(3)*
+ %ld = load i32, i32 addrspace(3)* %bc, align 4
+ %mul2 = shl nuw nsw i32 %phi, 3
+ %add3 = add nuw nsw i32 %mul2, %A2
+ %arrayidx2 = getelementptr inbounds float, float addrspace(1)* %ptrG, i32 %add3
+ %bc2 = bitcast float addrspace(1)* %arrayidx2 to i32 addrspace(1)*
+ store i32 %ld, i32 addrspace(1)* %bc2, align 4
+ %k = add nuw nsw i32 %phi, 1
+ %cmpk = icmp ult i32 %k, 32
+ br i1 %cmpk, label %for.body, label %bb10
+}
More information about the llvm-commits
mailing list