[llvm] b7cfa6c - [Loop Peeling] Separate the Loop Peeling Utilities from the Loop Unrolling Utilities
Sidharth Baveja via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 11:32:43 PDT 2020
Author: Sidharth Baveja
Date: 2020-07-31T18:31:58Z
New Revision: b7cfa6ca92830b3c331cb44706bb279996663439
URL: https://github.com/llvm/llvm-project/commit/b7cfa6ca92830b3c331cb44706bb279996663439
DIFF: https://github.com/llvm/llvm-project/commit/b7cfa6ca92830b3c331cb44706bb279996663439.diff
LOG: [Loop Peeling] Separate the Loop Peeling Utilities from the Loop Unrolling Utilities
Summary: This patch separates the Loop Peeling Utilities from Loop Unrolling.
The reason for this change is that Loop Peeling is no longer only being used by
loop unrolling; Patch D82927 introduces loop peeling with fusion, such that
loops can be modified to have to same trip count, making them legal to be
peeled.
Reviewed By: Meinersbur
Differential Revision: https://reviews.llvm.org/D83056
Added:
llvm/include/llvm/Transforms/Utils/LoopPeel.h
llvm/lib/Transforms/Utils/LoopPeel.cpp
Modified:
llvm/include/llvm/Transforms/Utils/UnrollLoop.h
llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
llvm/lib/Transforms/Scalar/LoopFuse.cpp
llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
llvm/lib/Transforms/Utils/CMakeLists.txt
llvm/lib/Transforms/Utils/LoopUnroll.cpp
Removed:
llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/LoopPeel.h b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
new file mode 100644
index 000000000000..8f857e1e5c21
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/LoopPeel.h
@@ -0,0 +1,40 @@
+//===- llvm/Transforms/Utils/LoopPeel.h ----- Peeling utilities -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines some loop peeling utilities. It does not define any
+// actual pass or policy.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
+#define LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
+
+#include "llvm/Analysis/TargetTransformInfo.h"
+
+namespace llvm {
+
+bool canPeel(Loop *L);
+
+bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE,
+ DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA);
+
+TargetTransformInfo::PeelingPreferences
+gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
+ const TargetTransformInfo &TTI,
+ Optional<bool> UserAllowPeeling,
+ Optional<bool> UserAllowProfileBasedPeeling,
+ bool UnrollingSpecficValues = false);
+
+void computePeelCount(Loop *L, unsigned LoopSize,
+ TargetTransformInfo::PeelingPreferences &PP,
+ unsigned &TripCount, ScalarEvolution &SE,
+ unsigned Threshold = UINT_MAX);
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_LOOPPEEL_H
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index bb3d02b95956..4254bd71a41c 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -92,16 +92,6 @@ bool UnrollRuntimeLoopRemainder(
const TargetTransformInfo *TTI, bool PreserveLCSSA,
Loop **ResultLoop = nullptr);
-void computePeelCount(Loop *L, unsigned LoopSize,
- TargetTransformInfo::UnrollingPreferences &UP,
- TargetTransformInfo::PeelingPreferences &PP,
- unsigned &TripCount, ScalarEvolution &SE);
-
-bool canPeel(Loop *L);
-
-bool peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI, ScalarEvolution *SE,
- DominatorTree *DT, AssumptionCache *AC, bool PreserveLCSSA);
-
LoopUnrollResult UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
LoopInfo *LI, ScalarEvolution *SE,
@@ -121,7 +111,6 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
unsigned &TripMultiple, unsigned LoopSize,
TargetTransformInfo::UnrollingPreferences &UP,
TargetTransformInfo::PeelingPreferences &PP,
-
bool &UseUpperBound);
void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
@@ -138,12 +127,6 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
Optional<bool> UserUpperBound, Optional<unsigned> UserFullUnrollMaxCount);
-TargetTransformInfo::PeelingPreferences
-gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
- const TargetTransformInfo &TTI,
- Optional<bool> UserAllowPeeling,
- Optional<bool> UserAllowProfileBasedPeeling);
-
unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
bool &NotDuplicatable, bool &Convergent,
const TargetTransformInfo &TTI,
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 68efaf767502..0cdb383eb924 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -21,6 +21,7 @@
#include "llvm/IR/User.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
using namespace llvm;
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 5573af834c59..90356ed0f2a7 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -66,7 +66,7 @@
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/CodeMoverUtils.h"
-#include "llvm/Transforms/Utils/UnrollLoop.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
using namespace llvm;
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index 285cba6ee205..bd6241932306 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -41,6 +41,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 87f40bb7ba85..2b610392dcfd 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -56,6 +56,7 @@
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/SizeOpts.h"
@@ -115,10 +116,6 @@ static cl::opt<unsigned> UnrollFullMaxCount(
cl::desc(
"Set the max unroll count for full unrolling, for testing purposes"));
-static cl::opt<unsigned> UnrollPeelCount(
- "unroll-peel-count", cl::Hidden,
- cl::desc("Set the unroll peeling count, for testing purposes"));
-
static cl::opt<bool>
UnrollAllowPartial("unroll-allow-partial", cl::Hidden,
cl::desc("Allows loops to be partially unrolled until "
@@ -149,15 +146,6 @@ static cl::opt<unsigned> FlatLoopTripCountThreshold(
"threshold, the loop is considered as flat and will be less "
"aggressively unrolled."));
-static cl::opt<bool>
- UnrollAllowPeeling("unroll-allow-peeling", cl::init(true), cl::Hidden,
- cl::desc("Allows loops to be peeled when the dynamic "
- "trip count is known to be low."));
-
-static cl::opt<bool> UnrollAllowLoopNestsPeeling(
- "unroll-allow-loop-nests-peeling", cl::init(false), cl::Hidden,
- cl::desc("Allows loop nests to be peeled."));
-
static cl::opt<bool> UnrollUnrollRemainder(
"unroll-remainder", cl::Hidden,
cl::desc("Allow the loop remainder to be unrolled."));
@@ -275,39 +263,6 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
return UP;
}
-TargetTransformInfo::PeelingPreferences
-llvm::gatherPeelingPreferences(Loop *L, ScalarEvolution &SE,
- const TargetTransformInfo &TTI,
- Optional<bool> UserAllowPeeling,
- Optional<bool> UserAllowProfileBasedPeeling) {
- TargetTransformInfo::PeelingPreferences PP;
-
- // Default values
- PP.PeelCount = 0;
- PP.AllowPeeling = true;
- PP.AllowLoopNestsPeeling = false;
- PP.PeelProfiledIterations = true;
-
- // Get Target Specifc Values
- TTI.getPeelingPreferences(L, SE, PP);
-
- // User Specified Values using cl::opt
- if (UnrollPeelCount.getNumOccurrences() > 0)
- PP.PeelCount = UnrollPeelCount;
- if (UnrollAllowPeeling.getNumOccurrences() > 0)
- PP.AllowPeeling = UnrollAllowPeeling;
- if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
- PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
-
- // User Specifed values provided by argument
- if (UserAllowPeeling.hasValue())
- PP.AllowPeeling = *UserAllowPeeling;
- if (UserAllowProfileBasedPeeling.hasValue())
- PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
-
- return PP;
-}
-
namespace {
/// A struct to densely store the state of an instruction after unrolling at
@@ -881,7 +836,7 @@ bool llvm::computeUnrollCount(
}
// 4th priority is loop peeling.
- computePeelCount(L, LoopSize, UP, PP, TripCount, SE);
+ computePeelCount(L, LoopSize, PP, TripCount, SE, UP.Threshold);
if (PP.PeelCount) {
UP.Runtime = false;
UP.Count = 1;
@@ -1087,7 +1042,7 @@ static LoopUnrollResult tryToUnrollLoop(
ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
ProvidedFullUnrollMaxCount);
TargetTransformInfo::PeelingPreferences PP = gatherPeelingPreferences(
- L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
+ L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling, true);
// Exit early if unrolling is disabled. For OptForSize, we pick the loop size
// as threshold later on.
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 19f655c3a78f..2ef17dc595fa 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -35,11 +35,11 @@ add_llvm_component_library(LLVMTransformUtils
LCSSA.cpp
LibCallsShrinkWrap.cpp
Local.cpp
+ LoopPeel.cpp
LoopRotationUtils.cpp
LoopSimplify.cpp
LoopUnroll.cpp
LoopUnrollAndJam.cpp
- LoopUnrollPeel.cpp
LoopUnrollRuntime.cpp
LoopUtils.cpp
LoopVersioning.cpp
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
similarity index 92%
rename from llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
rename to llvm/lib/Transforms/Utils/LoopPeel.cpp
index c653aacbee6c..ff9b9eca8fdf 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -1,4 +1,4 @@
-//===- UnrollLoopPeel.cpp - Loop peeling utilities ------------------------===//
+//===- LoopPeel.cpp -------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,12 +6,10 @@
//
//===----------------------------------------------------------------------===//
//
-// This file implements some loop unrolling utilities for peeling loops
-// with dynamically inferred (from PGO) trip counts. See LoopUnroll.cpp for
-// unrolling loops with compile-time constant trip counts.
-//
+// Loop Peeling Utilities.
//===----------------------------------------------------------------------===//
+#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallVector.h"
@@ -49,10 +47,24 @@
using namespace llvm;
using namespace llvm::PatternMatch;
-#define DEBUG_TYPE "loop-unroll"
+#define DEBUG_TYPE "loop-peel"
STATISTIC(NumPeeled, "Number of loops peeled");
+static cl::opt<unsigned> UnrollPeelCount(
+ "unroll-peel-count", cl::Hidden,
+ cl::desc("Set the unroll peeling count, for testing purposes"));
+
+static cl::opt<bool>
+ UnrollAllowPeeling("unroll-allow-peeling", cl::init(true), cl::Hidden,
+ cl::desc("Allows loops to be peeled when the dynamic "
+ "trip count is known to be low."));
+
+static cl::opt<bool>
+ UnrollAllowLoopNestsPeeling("unroll-allow-loop-nests-peeling",
+ cl::init(false), cl::Hidden,
+ cl::desc("Allows loop nests to be peeled."));
+
static cl::opt<unsigned> UnrollPeelMaxCount(
"unroll-peel-max-count", cl::init(7), cl::Hidden,
cl::desc("Max average trip count which will cause loop peeling."));
@@ -278,9 +290,9 @@ static unsigned countToEliminateCompares(Loop &L, unsigned MaxPeelCount,
// Return the number of iterations we want to peel off.
void llvm::computePeelCount(Loop *L, unsigned LoopSize,
- TargetTransformInfo::UnrollingPreferences &UP,
TargetTransformInfo::PeelingPreferences &PP,
- unsigned &TripCount, ScalarEvolution &SE) {
+ unsigned &TripCount, ScalarEvolution &SE,
+ unsigned Threshold) {
assert(LoopSize > 0 && "Zero loop size is not allowed!");
// Save the PP.PeelCount value set by the target in
// TTI.getPeelingPreferences or by the flag -unroll-peel-count.
@@ -322,7 +334,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
// maximum number of iterations among these values, thus turning all those
// Phis into invariants.
// First, check that we can peel at least one iteration.
- if (2 * LoopSize <= UP.Threshold && UnrollPeelMaxCount > 0) {
+ if (2 * LoopSize <= Threshold && UnrollPeelMaxCount > 0) {
// Store the pre-calculated values here.
SmallDenseMap<PHINode *, unsigned> IterationsToInvariance;
// Now go through all Phis to calculate their the number of iterations they
@@ -342,7 +354,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
// Pay respect to limitations implied by loop size and the max peel count.
unsigned MaxPeelCount = UnrollPeelMaxCount;
- MaxPeelCount = std::min(MaxPeelCount, UP.Threshold / LoopSize - 1);
+ MaxPeelCount = std::min(MaxPeelCount, Threshold / LoopSize - 1);
DesiredPeelCount = std::max(DesiredPeelCount,
countToEliminateCompares(*L, MaxPeelCount, SE));
@@ -385,7 +397,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
if (*PeelCount) {
if ((*PeelCount + AlreadyPeeled <= UnrollPeelMaxCount) &&
- (LoopSize * (*PeelCount + 1) <= UP.Threshold)) {
+ (LoopSize * (*PeelCount + 1) <= Threshold)) {
LLVM_DEBUG(dbgs() << "Peeling first " << *PeelCount
<< " iterations.\n");
PP.PeelCount = *PeelCount;
@@ -396,7 +408,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
LLVM_DEBUG(dbgs() << "Max peel count: " << UnrollPeelMaxCount << "\n");
LLVM_DEBUG(dbgs() << "Peel cost: " << LoopSize * (*PeelCount + 1)
<< "\n");
- LLVM_DEBUG(dbgs() << "Max peel cost: " << UP.Threshold << "\n");
+ LLVM_DEBUG(dbgs() << "Max peel cost: " << Threshold << "\n");
}
}
}
@@ -491,7 +503,7 @@ static void fixupBranchWeights(BasicBlock *Header, BranchInst *LatchBR,
/// instructions in the last peeled-off iteration.
static void cloneLoopBlocks(
Loop *L, unsigned IterNumber, BasicBlock *InsertTop, BasicBlock *InsertBot,
- SmallVectorImpl<std::pair<BasicBlock *, BasicBlock *> > &ExitEdges,
+ SmallVectorImpl<std::pair<BasicBlock *, BasicBlock *>> &ExitEdges,
SmallVectorImpl<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks,
ValueToValueMapTy &VMap, ValueToValueMapTy &LVMap, DominatorTree *DT,
LoopInfo *LI) {
@@ -599,6 +611,40 @@ static void cloneLoopBlocks(
LVMap[KV.first] = KV.second;
}
+TargetTransformInfo::PeelingPreferences llvm::gatherPeelingPreferences(
+ Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI,
+ Optional<bool> UserAllowPeeling,
+ Optional<bool> UserAllowProfileBasedPeeling, bool UnrollingSpecficValues) {
+ TargetTransformInfo::PeelingPreferences PP;
+
+ // Set the default values.
+ PP.PeelCount = 0;
+ PP.AllowPeeling = true;
+ PP.AllowLoopNestsPeeling = false;
+ PP.PeelProfiledIterations = true;
+
+ // Get the target specifc values.
+ TTI.getPeelingPreferences(L, SE, PP);
+
+ // User specified values using cl::opt.
+ if (UnrollingSpecficValues) {
+ if (UnrollPeelCount.getNumOccurrences() > 0)
+ PP.PeelCount = UnrollPeelCount;
+ if (UnrollAllowPeeling.getNumOccurrences() > 0)
+ PP.AllowPeeling = UnrollAllowPeeling;
+ if (UnrollAllowLoopNestsPeeling.getNumOccurrences() > 0)
+ PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
+ }
+
+ // User specifed values provided by argument.
+ if (UserAllowPeeling.hasValue())
+ PP.AllowPeeling = *UserAllowPeeling;
+ if (UserAllowProfileBasedPeeling.hasValue())
+ PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
+
+ return PP;
+}
+
/// Peel off the first \p PeelCount iterations of loop \p L.
///
/// Note that this does not peel them off as a single straight-line block.
@@ -609,8 +655,8 @@ static void cloneLoopBlocks(
/// for the bulk of dynamic execution, can be further simplified by scalar
/// optimizations.
bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
- ScalarEvolution *SE, DominatorTree *DT,
- AssumptionCache *AC, bool PreserveLCSSA) {
+ ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
+ bool PreserveLCSSA) {
assert(PeelCount > 0 && "Attempt to peel out zero iterations?");
assert(canPeel(L) && "Attempt to peel a loop which is not peelable?");
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index 8e8aeea15dbf..7bea696a853a 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -59,6 +59,7 @@
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/LoopPeel.h"
#include "llvm/Transforms/Utils/LoopSimplify.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/SimplifyIndVar.h"
More information about the llvm-commits
mailing list