[compiler-rt] 63a8969 - [PGO] Extend the value profile buckets for mem op sizes.
Hiroshi Yamauchi via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 25 10:46:56 PDT 2020
Author: Hiroshi Yamauchi
Date: 2020-06-25T10:22:56-07:00
New Revision: 63a89693f09f6b24ce4f2350d828150bd9c4f3e8
URL: https://github.com/llvm/llvm-project/commit/63a89693f09f6b24ce4f2350d828150bd9c4f3e8
DIFF: https://github.com/llvm/llvm-project/commit/63a89693f09f6b24ce4f2350d828150bd9c4f3e8.diff
LOG: [PGO] Extend the value profile buckets for mem op sizes.
Extend the memop value profile buckets to be more flexible (could accommodate a
mix of individual values and ranges) and to cover more value ranges (from 11 to
22 buckets).
Disabled behind a flag (to be enabled separately) and the existing code to be
removed later.
Differential Revision: https://reviews.llvm.org/D81682
Added:
llvm/unittests/ProfileData/InstrProfDataTest.cpp
Modified:
compiler-rt/include/profile/InstrProfData.inc
compiler-rt/lib/profile/InstrProfilingValue.c
llvm/include/llvm/ProfileData/InstrProf.h
llvm/include/llvm/ProfileData/InstrProfData.inc
llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
llvm/lib/ProfileData/InstrProf.cpp
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
llvm/test/Transforms/PGOProfile/memcpy.ll
llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
llvm/unittests/ProfileData/CMakeLists.txt
Removed:
################################################################################
diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc
index a6913527e67f..f4cb2524ad71 100644
--- a/compiler-rt/include/profile/InstrProfData.inc
+++ b/compiler-rt/include/profile/InstrProfData.inc
@@ -157,6 +157,8 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
#ifndef VALUE_RANGE_PROF
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#else /* VALUE_RANGE_PROF */
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -753,9 +755,14 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
@@ -783,3 +790,78 @@ typedef struct InstrProfValueData {
#endif
#undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ * [0, 0]
+ * [1, 1]
+ * [2, 2]
+ * [3, 3]
+ * [4, 4]
+ * [5, 5]
+ * [6, 6]
+ * [7, 7]
+ * [8, 8]
+ * [9, 15]
+ * [16, 16]
+ * [17, 31]
+ * [32, 32]
+ * [33, 63]
+ * [64, 64]
+ * [65, 127]
+ * [128, 128]
+ * [129, 255]
+ * [256, 256]
+ * [257, 511]
+ * [512, 512]
+ * [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked. Use the value as is.
+ return Value;
+ else if (Value >= 513)
+ // The last range is mapped to its lowest value.
+ return 513;
+ else if (__builtin_popcountll(Value) == 1)
+ // If it's a power of two, use it as is.
+ return Value;
+ else
+ // Otherwise, take to the previous power of two + 1.
+ return (1 << (64 - __builtin_clzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked.
+ return 1;
+ else if (__builtin_popcountll(Value) == 1)
+ // If it's a power of two, there's only one value.
+ return 1;
+ else
+ // Otherwise, there's more than one value in the range.
+ return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/compiler-rt/lib/profile/InstrProfilingValue.c b/compiler-rt/lib/profile/InstrProfilingValue.c
index fd53cac3dff3..76e1d3fa11b8 100644
--- a/compiler-rt/lib/profile/InstrProfilingValue.c
+++ b/compiler-rt/lib/profile/InstrProfilingValue.c
@@ -17,13 +17,14 @@
#define INSTR_PROF_VALUE_PROF_DATA
#define INSTR_PROF_COMMON_API_IMPL
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
#include "profile/InstrProfData.inc"
static int hasStaticCounters = 1;
static int OutOfNodesWarnings = 0;
static int hasNonDefaultValsPerSite = 0;
#define INSTR_PROF_MAX_VP_WARNS 10
-#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24
#define INSTR_PROF_VNODE_POOL_SIZE 1024
#ifndef _MSC_VER
@@ -250,6 +251,8 @@ __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data,
* The range for large values is optional. The default value of INT64_MIN
* indicates it is not specified.
*/
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
uint64_t TargetValue, void *Data, uint32_t CounterIndex,
int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) {
@@ -263,6 +266,18 @@ COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range(
__llvm_profile_instrument_target(TargetValue, Data, CounterIndex);
}
+/*
+ * The target values are partitioned into multiple ranges. The range spec is
+ * defined in InstrProfData.inc.
+ */
+COMPILER_RT_VISIBILITY void
+__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data,
+ uint32_t CounterIndex) {
+ // Map the target value to the representative value of its range.
+ uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue);
+ __llvm_profile_instrument_target(RepValue, Data, CounterIndex);
+}
+
/*
* A wrapper struct that represents value profile runtime data.
* Like InstrProfRecord class which is used by profiling host tools,
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index 62a0c6955708..3d9fca9422c4 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -75,10 +75,18 @@ inline StringRef getInstrProfValueProfFuncName() {
}
/// Return the name profile runtime entry point to do value range profiling.
+// FIXME: This is to be removed after switching to the new memop value
+// profiling.
inline StringRef getInstrProfValueRangeProfFuncName() {
return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR;
}
+/// Return the name profile runtime entry point to do memop size value
+/// profiling.
+inline StringRef getInstrProfValueProfMemOpFuncName() {
+ return INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR;
+}
+
/// Return the name prefix of variables containing instrumented function names.
inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; }
diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc
index a6913527e67f..f4cb2524ad71 100644
--- a/llvm/include/llvm/ProfileData/InstrProfData.inc
+++ b/llvm/include/llvm/ProfileData/InstrProfData.inc
@@ -157,6 +157,8 @@ VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA
#ifndef VALUE_RANGE_PROF
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx))
#else /* VALUE_RANGE_PROF */
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \
INSTR_PROF_COMMA
VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \
@@ -753,9 +755,14 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target
#define INSTR_PROF_VALUE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC)
+/* FIXME: This is to be removed after switching to the new memop value
+ * profiling. */
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range
#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \
INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC)
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop
+#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \
+ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC)
/* InstrProfile per-function control data alignment. */
#define INSTR_PROF_DATA_ALIGNMENT 8
@@ -783,3 +790,78 @@ typedef struct InstrProfValueData {
#endif
#undef COVMAP_V2_OR_V3
+
+#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API
+
+#ifdef __cplusplus
+#define INSTR_PROF_INLINE inline
+#else
+#define INSTR_PROF_INLINE
+#endif
+
+/* The value range buckets (22 buckets) for the memop size value profiling looks
+ * like:
+ *
+ * [0, 0]
+ * [1, 1]
+ * [2, 2]
+ * [3, 3]
+ * [4, 4]
+ * [5, 5]
+ * [6, 6]
+ * [7, 7]
+ * [8, 8]
+ * [9, 15]
+ * [16, 16]
+ * [17, 31]
+ * [32, 32]
+ * [33, 63]
+ * [64, 64]
+ * [65, 127]
+ * [128, 128]
+ * [129, 255]
+ * [256, 256]
+ * [257, 511]
+ * [512, 512]
+ * [513, UINT64_MAX]
+ *
+ * Each range has a 'representative value' which is the lower end value of the
+ * range and used to store in the runtime profile data records and the VP
+ * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127].
+ */
+
+/* Map an (observed) memop size value to the representative value of its range.
+ * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t
+InstrProfGetRangeRepValue(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked. Use the value as is.
+ return Value;
+ else if (Value >= 513)
+ // The last range is mapped to its lowest value.
+ return 513;
+ else if (__builtin_popcountll(Value) == 1)
+ // If it's a power of two, use it as is.
+ return Value;
+ else
+ // Otherwise, take to the previous power of two + 1.
+ return (1 << (64 - __builtin_clzll(Value) - 1)) + 1;
+}
+
+/* Return true if the range that an (observed) memop size value belongs to has
+ * only a single value in the range. For example, 0 -> true, 8 -> true, 10 ->
+ * false, 64 -> true, 100 -> false, 513 -> false. */
+INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned
+InstrProfIsSingleValRange(uint64_t Value) {
+ if (Value <= 8)
+ // The first ranges are individually tracked.
+ return 1;
+ else if (__builtin_popcountll(Value) == 1)
+ // If it's a power of two, there's only one value.
+ return 1;
+ else
+ // Otherwise, there's more than one value in the range.
+ return 0;
+}
+
+#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */
diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 263d3b629589..a7052f7b6a2b 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -68,6 +68,8 @@ class InstrProfiling : public PassInfoMixin<InstrProfiling> {
// vector of counter load/store pairs to be register promoted.
std::vector<LoadStorePair> PromotionCandidates;
+ // FIXME: These are to be removed after switching to the new memop value
+ // profiling.
// The start value of precise value profile range for memory intrinsic sizes.
int64_t MemOPSizeRangeStart;
// The end value of precise value profile range for memory intrinsic sizes.
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index b9d8ae9ba60d..8879674c324d 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1111,6 +1111,8 @@ bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken) {
return true;
}
+// FIXME: This is to be removed after switching to the new memop value
+// profiling.
// Parse the value profile options.
void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
int64_t &RangeLast) {
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index ceceafa86030..8147a1dbc784 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -57,6 +57,8 @@ using namespace llvm;
#define DEBUG_TYPE "instrprof"
+// FIXME: These are to be removed after switching to the new memop value
+// profiling.
// The start and end values of precise value profile range for memory
// intrinsic sizes
cl::opt<std::string> MemOPSizeRange(
@@ -72,6 +74,12 @@ cl::opt<unsigned> MemOPSizeLarge(
"Value of 0 disables the large value profiling."),
cl::init(8192));
+cl::opt<bool> UseOldMemOpValueProf(
+ "use-old-memop-value-prof",
+ cl::desc("Use the old memop value profiling buckets. This is "
+ "transitional and to be removed after switching. "),
+ cl::init(true));
+
namespace {
cl::opt<bool> DoHashBasedCounterSplit(
@@ -389,6 +397,19 @@ class PGOCounterPromoter {
BlockFrequencyInfo *BFI;
};
+enum class ValueProfilingCallType {
+ // Individual values are tracked. Currently used for indiret call target
+ // profiling.
+ Default,
+
+ // The old memop size value profiling. FIXME: To be removed after switching to
+ // the new one.
+ OldMemOp,
+
+ // MemOp: the (new) memop size value profiling with extended buckets.
+ MemOp
+};
+
} // end anonymous namespace
PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
@@ -573,9 +594,9 @@ bool InstrProfiling::run(
return true;
}
-static FunctionCallee
-getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
- bool IsRange = false) {
+static FunctionCallee getOrInsertValueProfilingCall(
+ Module &M, const TargetLibraryInfo &TLI,
+ ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
LLVMContext &Ctx = M.getContext();
auto *ReturnTy = Type::getVoidTy(M.getContext());
@@ -583,16 +604,22 @@ getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
if (auto AK = TLI.getExtAttrForI32Param(false))
AL = AL.addParamAttribute(M.getContext(), 2, AK);
- if (!IsRange) {
+ if (CallType == ValueProfilingCallType::Default ||
+ CallType == ValueProfilingCallType::MemOp) {
Type *ParamTypes[] = {
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
#include "llvm/ProfileData/InstrProfData.inc"
};
auto *ValueProfilingCallTy =
FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
- return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
- ValueProfilingCallTy, AL);
+ StringRef FuncName = CallType == ValueProfilingCallType::Default
+ ? getInstrProfValueProfFuncName()
+ : getInstrProfValueProfMemOpFuncName();
+ return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
} else {
+ // FIXME: This code is to be removed after switching to the new memop value
+ // profiling.
+ assert(CallType == ValueProfilingCallType::OldMemOp);
Type *RangeParamTypes[] = {
#define VALUE_RANGE_PROF 1
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
@@ -632,8 +659,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Index += It->second.NumValueSites[Kind];
IRBuilder<> Builder(Ind);
- bool IsRange = (Ind->getValueKind()->getZExtValue() ==
- llvm::InstrProfValueKind::IPVK_MemOPSize);
+ bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
+ llvm::InstrProfValueKind::IPVK_MemOPSize);
CallInst *Call = nullptr;
auto *TLI = &GetTLI(*Ind->getFunction());
@@ -643,12 +670,19 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
// WinEHPrepare pass.
SmallVector<OperandBundleDef, 1> OpBundles;
Ind->getOperandBundlesAsDefs(OpBundles);
- if (!IsRange) {
+ if (!IsMemOpSize) {
Value *Args[3] = {Ind->getTargetValue(),
Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
Builder.getInt32(Index)};
Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
OpBundles);
+ } else if (!UseOldMemOpValueProf) {
+ Value *Args[3] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index)};
+ Call = Builder.CreateCall(
+ getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
+ Args, OpBundles);
} else {
Value *Args[6] = {
Ind->getTargetValue(),
@@ -657,7 +691,8 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Builder.getInt64(MemOPSizeRangeStart),
Builder.getInt64(MemOPSizeRangeLast),
Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
- Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true),
+ Call = Builder.CreateCall(getOrInsertValueProfilingCall(
+ *M, *TLI, ValueProfilingCallType::OldMemOp),
Args, OpBundles);
}
if (auto AK = TLI->getExtAttrForI32Param(false))
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 2b7b859891dc..43a1434ae2d3 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -38,6 +38,8 @@
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/ProfileData/InstrProf.h"
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "llvm/ProfileData/InstrProfData.inc"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@@ -89,17 +91,25 @@ static cl::opt<bool>
cl::desc("Scale the memop size counts using the basic "
" block count value"));
+// FIXME: These are to be removed after switching to the new memop value
+// profiling.
// This option sets the rangge of precise profile memop sizes.
extern cl::opt<std::string> MemOPSizeRange;
// This option sets the value that groups large memop sizes
extern cl::opt<unsigned> MemOPSizeLarge;
+extern cl::opt<bool> UseOldMemOpValueProf;
+
cl::opt<bool>
MemOPOptMemcmpBcmp("pgo-memop-optimize-memcmp-bcmp", cl::init(true),
cl::Hidden,
cl::desc("Size-specialize memcmp and bcmp calls"));
+static cl::opt<unsigned>
+ MemOpMaxOptSize("memop-value-prof-max-opt-size", cl::Hidden, cl::init(128),
+ cl::desc("Optimize the memop size <= this value"));
+
namespace {
class PGOMemOPSizeOptLegacyPass : public FunctionPass {
public:
@@ -269,6 +279,8 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
TargetLibraryInfo &TLI;
bool Changed;
std::vector<MemOp> WorkList;
+ // FIXME: These are to be removed after switching to the new memop value
+ // profiling.
// Start of the previse range.
int64_t PreciseRangeStart;
// Last value of the previse range.
@@ -277,6 +289,8 @@ class MemOPSizeOpt : public InstVisitor<MemOPSizeOpt> {
std::unique_ptr<InstrProfValueData[]> ValueDataArray;
bool perform(MemOp MO);
+ // FIXME: This is to be removed after switching to the new memop value
+ // profiling.
// This kind shows which group the value falls in. For PreciseValue, we have
// the profile count for that value. LargeGroup groups the values that are in
// range [LargeValue, +inf). NonLargeGroup groups the rest of values.
@@ -365,8 +379,11 @@ bool MemOPSizeOpt::perform(MemOp MO) {
if (MemOPScaleCount)
C = getScaledCount(C, ActualCount, SavedTotalCount);
- // Only care precise value here.
- if (getMemOPSizeKind(V) != PreciseValue)
+ if (UseOldMemOpValueProf) {
+ // Only care precise value here.
+ if (getMemOPSizeKind(V) != PreciseValue)
+ continue;
+ } else if (!InstrProfIsSingleValRange(V) || V > MemOpMaxOptSize)
continue;
// ValueCounts are sorted on the count. Break at the first un-profitable
diff --git a/llvm/test/Transforms/PGOProfile/memcpy.ll b/llvm/test/Transforms/PGOProfile/memcpy.ll
index 6047c95e7c08..e00e1d350871 100644
--- a/llvm/test/Transforms/PGOProfile/memcpy.ll
+++ b/llvm/test/Transforms/PGOProfile/memcpy.ll
@@ -1,5 +1,7 @@
-; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s
-; RUN: opt <%s -passes=pgo-instr-gen,instrprof -S | FileCheck %s
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefix=OLDMEMOPVP
+; RUN: opt <%s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefix=NEWMEMOPVP
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
@@ -23,7 +25,8 @@ for.cond1:
for.body3:
%conv = sext i32 %add to i64
-; CHECK: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
+; OLDMEMOPVP: call void @__llvm_profile_instrument_range(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0, i64 0, i64 8, i64 8192)
+; NEWMEMOPVP: call void @__llvm_profile_instrument_memop(i64 %conv, i8* bitcast ({ i64, i64, i64*, i8*, i8*, i32, [2 x i16] }* @__profd_foo to i8*), i32 0)
call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %conv, i1 false)
%inc = add nsw i32 %j.0, 1
br label %for.cond1
diff --git a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
index b79431e3128e..43c85ed25baa 100644
--- a/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_profile_funclet.ll
@@ -1,8 +1,10 @@
; RUN: opt < %s -pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -pgo-instr-gen -instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
+; RUN: opt < %s -pgo-instr-gen -instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
; RUN: opt < %s -passes=pgo-instr-gen -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -passes=pgo-instr-gen,instrprof -S | FileCheck %s --check-prefix=LOWER
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=true -S | FileCheck %s --check-prefixes=LOWER,LOWEROLDMEMOPVP
+; RUN: opt < %s -passes=pgo-instr-gen,instrprof -use-old-memop-value-prof=false -S | FileCheck %s --check-prefixes=LOWER,LOWERNEWMEMOPVP
; This test is to verify that PGO runtime library calls get created with the
; appropriate operand bundle funclet information when a memory intrinsic
@@ -63,7 +65,8 @@ try.cont: ; preds = %entry
; GEN-SAME: [ "funclet"(token %tmp1) ]
; LOWER: catch:
-; LOWER: call void @__llvm_profile_instrument_range(
+; LOWEROLDMEMOPVP: call void @__llvm_profile_instrument_range(
+; LOWERNEWMEMOPVP: call void @__llvm_profile_instrument_memop(
; LOWER-SAME: [ "funclet"(token %tmp1) ]
declare dso_local void @"?may_throw@@YAXH at Z"(i32)
diff --git a/llvm/unittests/ProfileData/CMakeLists.txt b/llvm/unittests/ProfileData/CMakeLists.txt
index 366ed5482bf2..00a0079e675a 100644
--- a/llvm/unittests/ProfileData/CMakeLists.txt
+++ b/llvm/unittests/ProfileData/CMakeLists.txt
@@ -7,6 +7,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_unittest(ProfileDataTests
CoverageMappingTest.cpp
+ InstrProfDataTest.cpp
InstrProfTest.cpp
SampleProfTest.cpp
)
diff --git a/llvm/unittests/ProfileData/InstrProfDataTest.cpp b/llvm/unittests/ProfileData/InstrProfDataTest.cpp
new file mode 100644
index 000000000000..af1a3de0657c
--- /dev/null
+++ b/llvm/unittests/ProfileData/InstrProfDataTest.cpp
@@ -0,0 +1,68 @@
+//===- unittest/ProfileData/InstProfDataTest.cpp ----------------------------=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include <cstdint>
+
+#define INSTR_PROF_VALUE_PROF_MEMOP_API
+#include "llvm/ProfileData/InstrProfData.inc"
+
+namespace {
+
+TEST(InstrProfDataTest, MapValueToRangeRepValue) {
+ EXPECT_EQ(0ULL, InstrProfGetRangeRepValue(0));
+ EXPECT_EQ(1ULL, InstrProfGetRangeRepValue(1));
+ EXPECT_EQ(2ULL, InstrProfGetRangeRepValue(2));
+ EXPECT_EQ(3ULL, InstrProfGetRangeRepValue(3));
+ EXPECT_EQ(4ULL, InstrProfGetRangeRepValue(4));
+ EXPECT_EQ(5ULL, InstrProfGetRangeRepValue(5));
+ EXPECT_EQ(6ULL, InstrProfGetRangeRepValue(6));
+ EXPECT_EQ(7ULL, InstrProfGetRangeRepValue(7));
+ EXPECT_EQ(8ULL, InstrProfGetRangeRepValue(8));
+ EXPECT_EQ(9ULL, InstrProfGetRangeRepValue(9));
+ EXPECT_EQ(16ULL, InstrProfGetRangeRepValue(16));
+ EXPECT_EQ(17ULL, InstrProfGetRangeRepValue(30));
+ EXPECT_EQ(32ULL, InstrProfGetRangeRepValue(32));
+ EXPECT_EQ(33ULL, InstrProfGetRangeRepValue(54));
+ EXPECT_EQ(64ULL, InstrProfGetRangeRepValue(64));
+ EXPECT_EQ(65ULL, InstrProfGetRangeRepValue(127));
+ EXPECT_EQ(128ULL, InstrProfGetRangeRepValue(128));
+ EXPECT_EQ(129ULL, InstrProfGetRangeRepValue(200));
+ EXPECT_EQ(256ULL, InstrProfGetRangeRepValue(256));
+ EXPECT_EQ(257ULL, InstrProfGetRangeRepValue(397));
+ EXPECT_EQ(512ULL, InstrProfGetRangeRepValue(512));
+ EXPECT_EQ(513ULL, InstrProfGetRangeRepValue(2832048023));
+}
+
+TEST(InstrProfDataTest, IsInOneValueRange) {
+ EXPECT_EQ(true, InstrProfIsSingleValRange(0));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(1));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(2));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(3));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(4));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(5));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(6));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(7));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(8));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(9));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(16));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(30));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(32));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(54));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(64));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(127));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(128));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(200));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(256));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(397));
+ EXPECT_EQ(true, InstrProfIsSingleValRange(512));
+ EXPECT_EQ(false, InstrProfIsSingleValRange(2832048023344));
+}
+
+} // end anonymous namespace
More information about the llvm-commits
mailing list