[llvm] ef0e0ad - [llvm-profdata] Do not create numerical strings for MD5 function names read from a Sample Profile. (#66164)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 17 14:09:44 PDT 2023
Author: William Junda Huang
Date: 2023-10-17T21:09:39Z
New Revision: ef0e0adccd94ffdb10546491ef2719669754d3c9
URL: https://github.com/llvm/llvm-project/commit/ef0e0adccd94ffdb10546491ef2719669754d3c9
DIFF: https://github.com/llvm/llvm-project/commit/ef0e0adccd94ffdb10546491ef2719669754d3c9.diff
LOG: [llvm-profdata] Do not create numerical strings for MD5 function names read from a Sample Profile. (#66164)
This is phase 2 of the MD5 refactoring on Sample Profile following
https://reviews.llvm.org/D147740
In previous implementation, when a MD5 Sample Profile is read, the
reader first converts the MD5 values to strings, and then create a
StringRef as if the numerical strings are regular function names, and
later on IPO transformation passes perform string comparison over these
numerical strings for profile matching. This is inefficient since it
causes many small heap allocations.
In this patch I created a class `ProfileFuncRef` that is similar to
`StringRef` but it can represent a hash value directly without any
conversion, and it will be more efficient (I will attach some benchmark
results later) when being used in associative containers.
ProfileFuncRef guarantees the same function name in string form or in
MD5 form has the same hash value, which also fix a few issue in IPO
passes where function matching/lookup only check for function name
string, while returns a no-match if the profile is MD5.
When testing on an internal large profile (> 1 GB, with more than 10
million functions), the full profile load time is reduced from 28 sec to
25 sec in average, and reading function offset table from 0.78s to 0.7s
Added:
llvm/include/llvm/ProfileData/FunctionId.h
llvm/include/llvm/ProfileData/HashKeyMap.h
Modified:
llvm/include/llvm/ProfileData/SampleProf.h
llvm/include/llvm/ProfileData/SampleProfReader.h
llvm/include/llvm/ProfileData/SampleProfWriter.h
llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
llvm/lib/ProfileData/SampleProf.cpp
llvm/lib/ProfileData/SampleProfReader.cpp
llvm/lib/ProfileData/SampleProfWriter.cpp
llvm/lib/Target/X86/X86InsertPrefetch.cpp
llvm/lib/Transforms/IPO/SampleContextTracker.cpp
llvm/lib/Transforms/IPO/SampleProfile.cpp
llvm/tools/llvm-profdata/llvm-profdata.cpp
llvm/tools/llvm-profgen/CSPreInliner.cpp
llvm/tools/llvm-profgen/CSPreInliner.h
llvm/tools/llvm-profgen/CallContext.h
llvm/tools/llvm-profgen/ProfileGenerator.cpp
llvm/tools/llvm-profgen/ProfileGenerator.h
llvm/tools/llvm-profgen/ProfiledBinary.cpp
llvm/tools/llvm-profgen/ProfiledBinary.h
llvm/unittests/ProfileData/SampleProfTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/ProfileData/FunctionId.h b/llvm/include/llvm/ProfileData/FunctionId.h
new file mode 100644
index 000000000000000..0076cdc090459f4
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/FunctionId.h
@@ -0,0 +1,213 @@
+//===--- FunctionId.h - Sample profile function object ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines FunctionId class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_FUNCTIONID_H
+#define LLVM_PROFILEDATA_FUNCTIONID_H
+
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MD5.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdint>
+
+namespace llvm {
+namespace sampleprof {
+
+/// This class represents a function that is read from a sample profile. It
+/// comes with two forms: a string or a hash code. The latter form is the 64-bit
+/// MD5 of the function name for efficient storage supported by ExtBinary
+/// profile format, and when reading the profile, this class can represent it
+/// without converting it to a string first.
+/// When representing a hash code, we utilize the LengthOrHashCode field to
+/// store it, and Name is set to null. When representing a string, it is same as
+/// StringRef.
+class FunctionId {
+
+ const char *Data = nullptr;
+
+ // Use uint64_t instead of size_t so that it can also hold a MD5 value on
+ // 32-bit system.
+ uint64_t LengthOrHashCode = 0;
+
+ /// Extension to memcmp to handle hash code representation. If both are hash
+ /// values, Lhs and Rhs are both null, function returns 0 (and needs an extra
+ /// comparison using getIntValue). If only one is hash code, it is considered
+ /// less than the StringRef one. Otherwise perform normal string comparison.
+ static int compareMemory(const char *Lhs, const char *Rhs, uint64_t Length) {
+ if (Lhs == Rhs)
+ return 0;
+ if (!Lhs)
+ return -1;
+ if (!Rhs)
+ return 1;
+ return ::memcmp(Lhs, Rhs, (size_t)Length);
+ }
+
+public:
+ FunctionId() = default;
+
+ /// Constructor from a StringRef.
+ explicit FunctionId(StringRef Str)
+ : Data(Str.data()), LengthOrHashCode(Str.size()) {
+ }
+
+ /// Constructor from a hash code.
+ explicit FunctionId(uint64_t HashCode)
+ : LengthOrHashCode(HashCode) {
+ assert(HashCode != 0);
+ }
+
+ /// Check for equality. Similar to StringRef::equals, but will also cover for
+ /// the case where one or both are hash codes. Comparing their int values are
+ /// sufficient. A hash code FunctionId is considered not equal to a StringRef
+ /// FunctionId regardless of actual contents.
+ bool equals(const FunctionId &Other) const {
+ return LengthOrHashCode == Other.LengthOrHashCode &&
+ compareMemory(Data, Other.Data, LengthOrHashCode) == 0;
+ }
+
+ /// Total order comparison. If both FunctionId are StringRef, this is the same
+ /// as StringRef::compare. If one of them is StringRef, it is considered
+ /// greater than the hash code FunctionId. Otherwise this is the the same
+ /// as comparing their int values.
+ int compare(const FunctionId &Other) const {
+ auto Res = compareMemory(
+ Data, Other.Data, std::min(LengthOrHashCode, Other.LengthOrHashCode));
+ if (Res != 0)
+ return Res;
+ if (LengthOrHashCode == Other.LengthOrHashCode)
+ return 0;
+ return LengthOrHashCode < Other.LengthOrHashCode ? -1 : 1;
+ }
+
+ /// Convert to a string, usually for output purpose. Use caution on return
+ /// value's lifetime when converting to StringRef.
+ std::string str() const {
+ if (Data)
+ return std::string(Data, LengthOrHashCode);
+ if (LengthOrHashCode != 0)
+ return std::to_string(LengthOrHashCode);
+ return std::string();
+ }
+
+ /// Convert to StringRef. This is only allowed when it is known this object is
+ /// representing a StringRef, not a hash code. Calling this function on a hash
+ /// code is considered an error.
+ StringRef stringRef() const {
+ if (Data)
+ return StringRef(Data, LengthOrHashCode);
+ assert(LengthOrHashCode == 0 &&
+ "Cannot convert MD5 FunctionId to StringRef");
+ return StringRef();
+ }
+
+ friend raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj);
+
+ /// Get hash code of this object. Returns this object's hash code if it is
+ /// already representing one, otherwise returns the MD5 of its string content.
+ /// Note that it is not the same as std::hash because we want to keep the
+ /// consistency that the same sample profile function in string form or MD5
+ /// form has the same hash code.
+ uint64_t getHashCode() const {
+ if (Data)
+ return MD5Hash(StringRef(Data, LengthOrHashCode));
+ return LengthOrHashCode;
+ }
+
+ bool empty() const { return LengthOrHashCode == 0; }
+
+ /// Check if this object represents a StringRef, or a hash code.
+ bool isStringRef() const { return Data != nullptr; }
+};
+
+inline bool operator==(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.equals(RHS);
+}
+
+inline bool operator!=(const FunctionId &LHS, const FunctionId &RHS) {
+ return !LHS.equals(RHS);
+}
+
+inline bool operator<(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) < 0;
+}
+
+inline bool operator<=(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) <= 0;
+}
+
+inline bool operator>(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) > 0;
+}
+
+inline bool operator>=(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) >= 0;
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj) {
+ if (Obj.Data)
+ return OS << StringRef(Obj.Data, Obj.LengthOrHashCode);
+ if (Obj.LengthOrHashCode != 0)
+ return OS << Obj.LengthOrHashCode;
+ return OS;
+}
+
+inline uint64_t MD5Hash(const FunctionId &Obj) {
+ return Obj.getHashCode();
+}
+
+inline uint64_t hash_value(const FunctionId &Obj) {
+ return Obj.getHashCode();
+}
+
+} // end namespace sampleprof
+
+/// Template specialization for FunctionId so that it can be used in LLVM map
+/// containers.
+template <> struct DenseMapInfo<sampleprof::FunctionId, void> {
+
+ static inline sampleprof::FunctionId getEmptyKey() {
+ return sampleprof::FunctionId(~0ULL);
+ }
+
+ static inline sampleprof::FunctionId getTombstoneKey() {
+ return sampleprof::FunctionId(~1ULL);
+ }
+
+ static unsigned getHashValue(const sampleprof::FunctionId &Val) {
+ return Val.getHashCode();
+ }
+
+ static bool isEqual(const sampleprof::FunctionId &LHS,
+ const sampleprof::FunctionId &RHS) {
+ return LHS == RHS;
+ }
+};
+
+} // end namespace llvm
+
+namespace std {
+
+/// Template specialization for FunctionId so that it can be used in STL
+/// containers.
+template <> struct hash<llvm::sampleprof::FunctionId> {
+ size_t operator()(const llvm::sampleprof::FunctionId &Val) const {
+ return Val.getHashCode();
+ }
+};
+
+} // end namespace std
+
+#endif // LLVM_PROFILEDATA_FUNCTIONID_H
diff --git a/llvm/include/llvm/ProfileData/HashKeyMap.h b/llvm/include/llvm/ProfileData/HashKeyMap.h
new file mode 100644
index 000000000000000..b2f1bf222157b7b
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/HashKeyMap.h
@@ -0,0 +1,129 @@
+//===--- HashKeyMap.h - Wrapper for maps using hash value key ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines HashKeyMap template.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_HASHKEYMAP_H
+#define LLVM_PROFILEDATA_HASHKEYMAP_H
+
+#include "llvm/ADT/Hashing.h"
+#include <iterator>
+#include <utility>
+
+namespace llvm {
+
+namespace sampleprof {
+
+/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
+/// the hash value of the original key as the new key. This greatly improves the
+/// performance of insert and query operations especially when hash values of
+/// keys are available a priori, and reduces memory usage if KeyT has a large
+/// size.
+/// All keys with the same hash value are considered equivalent (i.e. hash
+/// collision is silently ignored). Given such feature this class should only be
+/// used where it does not affect compilation correctness, for example, when
+/// loading a sample profile. The original key is not stored, so if the user
+/// needs to preserve it, it should be stored in the mapped type.
+/// Assuming the hashing algorithm is uniform, we use the formula
+/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
+/// elements chosen at random to calculate the probability of collision. With
+/// 1,000,000 entries the probability is negligible:
+/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
+/// Source: https://en.wikipedia.org/wiki/Birthday_problem
+///
+/// \param MapT The underlying associative container type.
+/// \param KeyT The original key type, which requires the implementation of
+/// llvm::hash_value(KeyT).
+/// \param ValueT The original mapped type, which has the same requirement as
+/// the underlying container.
+/// \param MapTArgs Additional template parameters passed to the underlying
+/// container.
+template <template <typename, typename, typename...> typename MapT,
+ typename KeyT, typename ValueT, typename... MapTArgs>
+class HashKeyMap :
+ public MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...> {
+public:
+ using base_type = MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...>;
+ using key_type = decltype(hash_value(KeyT()));
+ using original_key_type = KeyT;
+ using mapped_type = ValueT;
+ using value_type = typename base_type::value_type;
+
+ using iterator = typename base_type::iterator;
+ using const_iterator = typename base_type::const_iterator;
+
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(const key_type &Hash,
+ const original_key_type &Key,
+ Ts &&...Args) {
+ assert(Hash == hash_value(Key));
+ return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
+ }
+
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(const original_key_type &Key,
+ Ts &&...Args) {
+ return try_emplace(hash_value(Key), Key, std::forward<Ts>(Args)...);
+ }
+
+ template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
+ return try_emplace(std::forward<Ts>(Args)...);
+ }
+
+ mapped_type &operator[](const original_key_type &Key) {
+ return try_emplace(Key, mapped_type()).first->second;
+ }
+
+ iterator find(const original_key_type &Key) {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It;
+ return base_type::end();
+ }
+
+ const_iterator find(const original_key_type &Key) const {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It;
+ return base_type::end();
+ }
+
+ mapped_type lookup(const original_key_type &Key) const {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It->second;
+ return mapped_type();
+ }
+
+ size_t count(const original_key_type &Key) const {
+ return base_type::count(hash_value(Key));
+ }
+
+ size_t erase(const original_key_type &Ctx) {
+ auto It = find(Ctx);
+ if (It != base_type::end()) {
+ base_type::erase(It);
+ return 1;
+ }
+ return 0;
+ }
+
+ iterator erase(const_iterator It) {
+ return base_type::erase(It);
+ }
+};
+
+}
+
+}
+
+#endif // LLVM_PROFILEDATA_HASHKEYMAP_H
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 78a16499accd64a..57ea144532a3cb1 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -21,10 +21,12 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/FunctionId.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/ProfileData/HashKeyMap.h"
#include <algorithm>
#include <cstdint>
#include <list>
@@ -109,16 +111,6 @@ static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
uint64_t('2') << (64 - 56) | uint64_t(Format);
}
-/// Get the proper representation of a string according to whether the
-/// current Format uses MD5 to represent the string.
-static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
- std::string &GUIDBuf) {
- if (Name.empty() || !UseMD5)
- return Name;
- GUIDBuf = std::to_string(Function::getGUID(Name));
- return GUIDBuf;
-}
-
static inline uint64_t SPVersion() { return 103; }
// Section Type used by SampleProfileExtBinaryBaseReader and
@@ -305,27 +297,22 @@ struct LineLocation {
return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
}
+ uint64_t getHashCode() const {
+ return ((uint64_t) Discriminator << 32) | LineOffset;
+ }
+
uint32_t LineOffset;
uint32_t Discriminator;
};
struct LineLocationHash {
uint64_t operator()(const LineLocation &Loc) const {
- return std::hash<std::uint64_t>{}((((uint64_t)Loc.LineOffset) << 32) |
- Loc.Discriminator);
+ return Loc.getHashCode();
}
};
raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
-static inline uint64_t hashFuncName(StringRef F) {
- // If function name is already MD5 string, do not hash again.
- uint64_t Hash;
- if (F.getAsInteger(10, Hash))
- Hash = MD5Hash(F);
- return Hash;
-}
-
/// Representation of a single sample record.
///
/// A sample record is represented by a positive integer value, which
@@ -338,7 +325,7 @@ static inline uint64_t hashFuncName(StringRef F) {
/// will be a list of one or more functions.
class SampleRecord {
public:
- using CallTarget = std::pair<StringRef, uint64_t>;
+ using CallTarget = std::pair<FunctionId, uint64_t>;
struct CallTargetComparator {
bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
if (LHS.second != RHS.second)
@@ -349,7 +336,7 @@ class SampleRecord {
};
using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
- using CallTargetMap = StringMap<uint64_t>;
+ using CallTargetMap = std::unordered_map<FunctionId, uint64_t>;
SampleRecord() = default;
/// Increment the number of samples for this record by \p S.
@@ -378,7 +365,7 @@ class SampleRecord {
///
/// Sample counts accumulate using saturating arithmetic, to avoid wrapping
/// around unsigned integers.
- sampleprof_error addCalledTarget(StringRef F, uint64_t S,
+ sampleprof_error addCalledTarget(FunctionId F, uint64_t S,
uint64_t Weight = 1) {
uint64_t &TargetSamples = CallTargets[F];
bool Overflowed;
@@ -390,7 +377,7 @@ class SampleRecord {
/// Remove called function from the call target map. Return the target sample
/// count of the called function.
- uint64_t removeCalledTarget(StringRef F) {
+ uint64_t removeCalledTarget(FunctionId F) {
uint64_t Count = 0;
auto I = CallTargets.find(F);
if (I != CallTargets.end()) {
@@ -474,18 +461,18 @@ enum ContextAttributeMask {
0x4, // Leaf of context is duplicated into the base profile
};
-// Represents a context frame with function name and line location
+// Represents a context frame with profile function and line location
struct SampleContextFrame {
- StringRef FuncName;
+ FunctionId Func;
LineLocation Location;
SampleContextFrame() : Location(0, 0) {}
-
- SampleContextFrame(StringRef FuncName, LineLocation Location)
- : FuncName(FuncName), Location(Location) {}
+
+ SampleContextFrame(FunctionId Func, LineLocation Location)
+ : Func(Func), Location(Location) {}
bool operator==(const SampleContextFrame &That) const {
- return Location == That.Location && FuncName == That.FuncName;
+ return Location == That.Location && Func == That.Func;
}
bool operator!=(const SampleContextFrame &That) const {
@@ -494,7 +481,7 @@ struct SampleContextFrame {
std::string toString(bool OutputLineLocation) const {
std::ostringstream OContextStr;
- OContextStr << FuncName.str();
+ OContextStr << Func.str();
if (OutputLineLocation) {
OContextStr << ":" << Location.LineOffset;
if (Location.Discriminator)
@@ -502,11 +489,16 @@ struct SampleContextFrame {
}
return OContextStr.str();
}
+
+ uint64_t getHashCode() const {
+ uint64_t NameHash = Func.getHashCode();
+ uint64_t LocId = Location.getHashCode();
+ return NameHash + (LocId << 5) + LocId;
+ }
};
static inline hash_code hash_value(const SampleContextFrame &arg) {
- return hash_combine(arg.FuncName, arg.Location.LineOffset,
- arg.Location.Discriminator);
+ return arg.getHashCode();
}
using SampleContextFrameVector = SmallVector<SampleContextFrame, 1>;
@@ -533,9 +525,12 @@ class SampleContext {
SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
SampleContext(StringRef Name)
- : Name(Name), State(UnknownContext), Attributes(ContextNone) {
+ : Func(Name), State(UnknownContext), Attributes(ContextNone) {
assert(!Name.empty() && "Name is empty");
}
+
+ SampleContext(FunctionId Func)
+ : Func(Func), State(UnknownContext), Attributes(ContextNone) {}
SampleContext(SampleContextFrames Context,
ContextStateMask CState = RawContext)
@@ -557,7 +552,7 @@ class SampleContext {
bool HasContext = ContextStr.startswith("[");
if (!HasContext) {
State = UnknownContext;
- Name = ContextStr;
+ Func = FunctionId(ContextStr);
} else {
CSNameTable.emplace_back();
SampleContextFrameVector &Context = CSNameTable.back();
@@ -574,24 +569,25 @@ class SampleContext {
ContextStr = ContextStr.substr(1, ContextStr.size() - 2);
StringRef ContextRemain = ContextStr;
StringRef ChildContext;
- StringRef CalleeName;
+ FunctionId Callee;
while (!ContextRemain.empty()) {
auto ContextSplit = ContextRemain.split(" @ ");
ChildContext = ContextSplit.first;
ContextRemain = ContextSplit.second;
LineLocation CallSiteLoc(0, 0);
- decodeContextString(ChildContext, CalleeName, CallSiteLoc);
- Context.emplace_back(CalleeName, CallSiteLoc);
+ decodeContextString(ChildContext, Callee, CallSiteLoc);
+ Context.emplace_back(Callee, CallSiteLoc);
}
}
// Decode context string for a frame to get function name and location.
// `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
- static void decodeContextString(StringRef ContextStr, StringRef &FName,
+ static void decodeContextString(StringRef ContextStr,
+ FunctionId &Func,
LineLocation &LineLoc) {
// Get function name
auto EntrySplit = ContextStr.split(':');
- FName = EntrySplit.first;
+ Func = FunctionId(EntrySplit.first);
LineLoc = {0, 0};
if (!EntrySplit.second.empty()) {
@@ -618,7 +614,7 @@ class SampleContext {
void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
bool hasContext() const { return State != UnknownContext; }
bool isBaseContext() const { return FullContext.size() == 1; }
- StringRef getName() const { return Name; }
+ FunctionId getFunction() const { return Func; }
SampleContextFrames getContextFrames() const { return FullContext; }
static std::string getContextString(SampleContextFrames Context,
@@ -636,22 +632,19 @@ class SampleContext {
std::string toString() const {
if (!hasContext())
- return Name.str();
+ return Func.str();
return getContextString(FullContext, false);
}
uint64_t getHashCode() const {
if (hasContext())
return hash_value(getContextFrames());
-
- // For non-context function name, use its MD5 as hash value, so that it is
- // consistent with the profile map's key.
- return hashFuncName(getName());
+ return getFunction().getHashCode();
}
/// Set the name of the function and clear the current context.
- void setName(StringRef FunctionName) {
- Name = FunctionName;
+ void setFunction(FunctionId newFunction) {
+ Func = newFunction;
FullContext = SampleContextFrames();
State = UnknownContext;
}
@@ -660,12 +653,12 @@ class SampleContext {
ContextStateMask CState = RawContext) {
assert(CState != UnknownContext);
FullContext = Context;
- Name = Context.back().FuncName;
+ Func = Context.back().Func;
State = CState;
}
bool operator==(const SampleContext &That) const {
- return State == That.State && Name == That.Name &&
+ return State == That.State && Func == That.Func &&
FullContext == That.FullContext;
}
@@ -676,14 +669,14 @@ class SampleContext {
return State < That.State;
if (!hasContext()) {
- return Name < That.Name;
+ return Func < That.Func;
}
uint64_t I = 0;
while (I < std::min(FullContext.size(), That.FullContext.size())) {
auto &Context1 = FullContext[I];
auto &Context2 = That.FullContext[I];
- auto V = Context1.FuncName.compare(Context2.FuncName);
+ auto V = Context1.Func.compare(Context2.Func);
if (V)
return V < 0;
if (Context1.Location != Context2.Location)
@@ -707,15 +700,16 @@ class SampleContext {
return false;
ThatContext = ThatContext.take_front(ThisContext.size());
// Compare Leaf frame first
- if (ThisContext.back().FuncName != ThatContext.back().FuncName)
+ if (ThisContext.back().Func != ThatContext.back().Func)
return false;
// Compare leading context
return ThisContext.drop_back() == ThatContext.drop_back();
}
private:
- /// Mangled name of the function.
- StringRef Name;
+ // The function associated with this context. If CS profile, this is the leaf
+ // function.
+ FunctionId Func;
// Full context including calling context and leaf function name
SampleContextFrames FullContext;
// State of the associated sample profile
@@ -738,7 +732,7 @@ class SampleProfileReaderItaniumRemapper;
using BodySampleMap = std::map<LineLocation, SampleRecord>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
-using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
+using FunctionSamplesMap = std::map<FunctionId, FunctionSamples>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
using LocToLocMap =
std::unordered_map<LineLocation, LineLocation, LineLocationHash>;
@@ -790,14 +784,16 @@ class FunctionSamples {
sampleprof_error addCalledTargetSamples(uint32_t LineOffset,
uint32_t Discriminator,
- StringRef FName, uint64_t Num,
+ FunctionId Func,
+ uint64_t Num,
uint64_t Weight = 1) {
return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
- FName, Num, Weight);
+ Func, Num, Weight);
}
sampleprof_error addSampleRecord(LineLocation Location,
- const SampleRecord &SampleRecord, uint64_t Weight = 1) {
+ const SampleRecord &SampleRecord,
+ uint64_t Weight = 1) {
return BodySamples[Location].merge(SampleRecord, Weight);
}
@@ -805,11 +801,11 @@ class FunctionSamples {
// the number of body samples actually decreased.
uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,
uint32_t Discriminator,
- StringRef FName) {
+ FunctionId Func) {
uint64_t Count = 0;
auto I = BodySamples.find(LineLocation(LineOffset, Discriminator));
if (I != BodySamples.end()) {
- Count = I->second.removeCalledTarget(FName);
+ Count = I->second.removeCalledTarget(Func);
Count = I->second.removeSamples(Count);
if (!I->second.getSamples())
BodySamples.erase(I);
@@ -1002,7 +998,7 @@ class FunctionSamples {
sampleprof_error Result = sampleprof_error::success;
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
- if (Context.getName().empty())
+ if (Context.getFunction().empty())
Context = Other.getContext();
if (FunctionHash == 0) {
// Set the function hash code for the target profile.
@@ -1039,25 +1035,26 @@ class FunctionSamples {
/// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
/// to \p S.
void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S,
- const StringMap<Function *> &SymbolMap,
+ const HashKeyMap<std::unordered_map, FunctionId,
+ Function *> &SymbolMap,
uint64_t Threshold) const {
if (TotalSamples <= Threshold)
return;
auto isDeclaration = [](const Function *F) {
return !F || F->isDeclaration();
};
- if (isDeclaration(SymbolMap.lookup(getFuncName()))) {
+ if (isDeclaration(SymbolMap.lookup(getFunction()))) {
// Add to the import list only when it's defined out of module.
- S.insert(getGUID(getName()));
+ S.insert(getGUID());
}
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
- if (TS.getValue() > Threshold) {
- const Function *Callee = SymbolMap.lookup(getFuncName(TS.getKey()));
+ if (TS.second > Threshold) {
+ const Function *Callee = SymbolMap.lookup(TS.first);
if (isDeclaration(Callee))
- S.insert(getGUID(TS.getKey()));
+ S.insert(TS.first.getHashCode());
}
for (const auto &CS : CallsiteSamples)
for (const auto &NameFS : CS.second)
@@ -1065,13 +1062,15 @@ class FunctionSamples {
}
/// Set the name of the function.
- void setName(StringRef FunctionName) { Context.setName(FunctionName); }
+ void setFunction(FunctionId newFunction) {
+ Context.setFunction(newFunction);
+ }
/// Return the function name.
- StringRef getName() const { return Context.getName(); }
+ FunctionId getFunction() const { return Context.getFunction(); }
/// Return the original function name.
- StringRef getFuncName() const { return getFuncName(getName()); }
+ StringRef getFuncName() const { return getFuncName(getFunction()); }
void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
@@ -1128,19 +1127,19 @@ class FunctionSamples {
return FnName;
}
- /// Translate \p Name into its original name.
- /// When profile doesn't use MD5, \p Name needs no translation.
- /// When profile uses MD5, \p Name in current FunctionSamples
+ /// Translate \p Func into its original name.
+ /// When profile doesn't use MD5, \p Func needs no translation.
+ /// When profile uses MD5, \p Func in current FunctionSamples
/// is actually GUID of the original function name. getFuncName will
- /// translate \p Name in current FunctionSamples into its original name
+ /// translate \p Func in current FunctionSamples into its original name
/// by looking up in the function map GUIDToFuncNameMap.
/// If the original name doesn't exist in the map, return empty StringRef.
- StringRef getFuncName(StringRef Name) const {
+ StringRef getFuncName(FunctionId Func) const {
if (!UseMD5)
- return Name;
+ return Func.stringRef();
assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
- return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
+ return GUIDToFuncNameMap->lookup(Func.getHashCode());
}
/// Returns the line offset to the start line of the subprogram.
@@ -1156,8 +1155,12 @@ class FunctionSamples {
/// Returns a unique hash code for a combination of a callsite location and
/// the callee function name.
- static uint64_t getCallSiteHash(StringRef CalleeName,
- const LineLocation &Callsite);
+ /// Guarantee MD5 and non-MD5 representation of the same function results in
+ /// the same hash.
+ static uint64_t getCallSiteHash(FunctionId Callee,
+ const LineLocation &Callsite) {
+ return SampleContextFrame(Callee, Callsite).getHashCode();
+ }
/// Get the FunctionSamples of the inline instance where DIL originates
/// from.
@@ -1197,16 +1200,15 @@ class FunctionSamples {
/// all the function symbols defined or declared in current module.
DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
- // Assume the input \p Name is a name coming from FunctionSamples itself.
- // If UseMD5 is true, the name is already a GUID and we
- // don't want to return the GUID of GUID.
- static uint64_t getGUID(StringRef Name) {
- return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
+ /// Return the GUID of the context's name. If the context is already using
+ /// MD5, don't hash it again.
+ uint64_t getGUID() const {
+ return getFunction().getHashCode();
}
// Find all the names in the current FunctionSamples including names in
// all the inline instances and names of call targets.
- void findAllNames(DenseSet<StringRef> &NameSet) const;
+ void findAllNames(DenseSet<FunctionId> &NameSet) const;
bool operator==(const FunctionSamples &Other) const {
return (GUIDToFuncNameMap == Other.GUIDToFuncNameMap ||
@@ -1223,9 +1225,6 @@ class FunctionSamples {
return !(*this == Other);
}
- template <typename T>
- const T &getKey() const;
-
private:
/// CFG hash value for the function.
uint64_t FunctionHash = 0;
@@ -1289,90 +1288,16 @@ class FunctionSamples {
const LocToLocMap *IRToProfileLocationMap = nullptr;
};
-template <>
-inline const SampleContext &FunctionSamples::getKey<SampleContext>() const {
- return getContext();
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline FunctionId getRepInFormat(StringRef Name) {
+ if (Name.empty() || !FunctionSamples::UseMD5)
+ return FunctionId(Name);
+ return FunctionId(Function::getGUID(Name));
}
raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
-/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
-/// the hash value of the original key as the new key. This greatly improves the
-/// performance of insert and query operations especially when hash values of
-/// keys are available a priori, and reduces memory usage if KeyT has a large
-/// size.
-/// All keys with the same hash value are considered equivalent (i.e. hash
-/// collision is silently ignored). Given such feature this class should only be
-/// used where it does not affect compilation correctness, for example, when
-/// loading a sample profile.
-/// Assuming the hashing algorithm is uniform, we use the formula
-/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
-/// elements chosen at random to calculate the probability of collision. With
-/// 1,000,000 entries the probability is negligible:
-/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
-/// Source: https://en.wikipedia.org/wiki/Birthday_problem
-template <template <typename, typename, typename...> typename MapT,
- typename KeyT, typename ValueT, typename... MapTArgs>
-class HashKeyMap : public MapT<hash_code, ValueT, MapTArgs...> {
-public:
- using base_type = MapT<hash_code, ValueT, MapTArgs...>;
- using key_type = hash_code;
- using original_key_type = KeyT;
- using mapped_type = ValueT;
- using value_type = typename base_type::value_type;
-
- using iterator = typename base_type::iterator;
- using const_iterator = typename base_type::const_iterator;
-
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(const key_type &Hash,
- const original_key_type &Key,
- Ts &&...Args) {
- assert(Hash == hash_value(Key));
- return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
- }
-
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(const original_key_type &Key,
- Ts &&...Args) {
- key_type Hash = hash_value(Key);
- return try_emplace(Hash, Key, std::forward<Ts>(Args)...);
- }
-
- template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
- return try_emplace(std::forward<Ts>(Args)...);
- }
-
- mapped_type &operator[](const original_key_type &Key) {
- return try_emplace(Key, mapped_type()).first->second;
- }
-
- iterator find(const original_key_type &Key) {
- key_type Hash = hash_value(Key);
- auto It = base_type::find(Hash);
- if (It != base_type::end())
- return It;
- return base_type::end();
- }
-
- const_iterator find(const original_key_type &Key) const {
- key_type Hash = hash_value(Key);
- auto It = base_type::find(Hash);
- if (It != base_type::end())
- return It;
- return base_type::end();
- }
-
- size_t erase(const original_key_type &Ctx) {
- auto It = find(Ctx);
- if (It != base_type::end()) {
- base_type::erase(It);
- return 1;
- }
- return 0;
- }
-};
-
/// This class provides operator overloads to the map container using MD5 as the
/// key type, so that existing code can still work in most cases using
/// SampleContext as key.
@@ -1400,11 +1325,6 @@ class SampleProfileMap
Ctx);
}
- // Overloaded find() to lookup a function by name.
- iterator find(StringRef Fname) {
- return base_type::find(hashFuncName(Fname));
- }
-
size_t erase(const SampleContext &Ctx) {
return HashKeyMap<std::unordered_map, SampleContext, FunctionSamples>::
erase(Ctx);
@@ -1475,7 +1395,7 @@ class ProfileConverter {
// profile.
void convertCSProfiles();
struct FrameNode {
- FrameNode(StringRef FName = StringRef(),
+ FrameNode(FunctionId FName = FunctionId(),
FunctionSamples *FSamples = nullptr,
LineLocation CallLoc = {0, 0})
: FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){};
@@ -1483,14 +1403,14 @@ class ProfileConverter {
// Map line+discriminator location to child frame
std::map<uint64_t, FrameNode> AllChildFrames;
// Function name for current frame
- StringRef FuncName;
+ FunctionId FuncName;
// Function Samples for current frame
FunctionSamples *FuncSamples;
// Callsite location in parent context
LineLocation CallSiteLoc;
FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
- StringRef CalleeName);
+ FunctionId CalleeName);
};
static void flattenProfile(SampleProfileMap &ProfileMap,
@@ -1507,7 +1427,7 @@ class ProfileConverter {
for (const auto &I : InputProfiles) {
// Retain the profile name and clear the full context for each function
// profile.
- FunctionSamples &FS = OutputProfiles.Create(I.second.getName());
+ FunctionSamples &FS = OutputProfiles.Create(I.second.getFunction());
FS.merge(I.second);
}
} else {
@@ -1553,7 +1473,8 @@ class ProfileConverter {
CalleeProfile.getHeadSamplesEstimate());
// Add callsite sample.
Profile.addCalledTargetSamples(
- I.first.LineOffset, I.first.Discriminator, CalleeProfile.getName(),
+ I.first.LineOffset, I.first.Discriminator,
+ CalleeProfile.getFunction(),
CalleeProfile.getHeadSamplesEstimate());
// Update total samples.
TotalSamples = TotalSamples >= CalleeProfile.getTotalSamples()
@@ -1625,7 +1546,9 @@ using namespace sampleprof;
template <> struct DenseMapInfo<SampleContext> {
static inline SampleContext getEmptyKey() { return SampleContext(); }
- static inline SampleContext getTombstoneKey() { return SampleContext("@"); }
+ static inline SampleContext getTombstoneKey() {
+ return SampleContext(FunctionId(~1ULL));
+ }
static unsigned getHashValue(const SampleContext &Val) {
return Val.getHashCode();
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index a5b1df3ef550b2b..9e8f543909cdbd1 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -409,13 +409,13 @@ class SampleProfileReader {
/// Return the samples collected for function \p F.
FunctionSamples *getSamplesFor(StringRef Fname) {
- auto It = Profiles.find(Fname);
+ auto It = Profiles.find(FunctionId(Fname));
if (It != Profiles.end())
return &It->second;
if (Remapper) {
if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) {
- auto It = Profiles.find(*NameInProfile);
+ auto It = Profiles.find(FunctionId(*NameInProfile));
if (It != Profiles.end())
return &It->second;
}
@@ -474,7 +474,7 @@ class SampleProfileReader {
/// It includes all the names that have samples either in outline instance
/// or inline instance.
- virtual std::vector<StringRef> *getNameTable() { return nullptr; }
+ virtual std::vector<FunctionId> *getNameTable() { return nullptr; }
virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
/// Return whether names in the profile are all MD5 numbers.
@@ -508,10 +508,6 @@ class SampleProfileReader {
/// Memory buffer holding the profile file.
std::unique_ptr<MemoryBuffer> Buffer;
- /// Extra name buffer holding names created on demand.
- /// This should only be needed for md5 profiles.
- std::unordered_set<std::string> MD5NameBuffer;
-
/// Profile summary information.
std::unique_ptr<ProfileSummary> Summary;
@@ -595,7 +591,9 @@ class SampleProfileReaderBinary : public SampleProfileReader {
/// It includes all the names that have samples either in outline instance
/// or inline instance.
- std::vector<StringRef> *getNameTable() override { return &NameTable; }
+ std::vector<FunctionId> *getNameTable() override {
+ return &NameTable;
+ }
protected:
/// Read a numeric value of type T from the profile.
@@ -637,7 +635,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
std::error_code readNameTable();
/// Read a string indirectly via the name table. Optionally return the index.
- ErrorOr<StringRef> readStringFromTable(size_t *RetIdx = nullptr);
+ ErrorOr<FunctionId> readStringFromTable(size_t *RetIdx = nullptr);
/// Read a context indirectly via the CSNameTable. Optionally return the
/// index.
@@ -654,19 +652,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
const uint8_t *End = nullptr;
/// Function name table.
- std::vector<StringRef> NameTable;
-
- /// If MD5 is used in NameTable section, the section saves uint64_t data.
- /// The uint64_t data has to be converted to a string and then the string
- /// will be used to initialize StringRef in NameTable.
- /// Note NameTable contains StringRef so it needs another buffer to own
- /// the string data. MD5StringBuf serves as the string buffer that is
- /// referenced by NameTable (vector of StringRef). We make sure
- /// the lifetime of MD5StringBuf is not shorter than that of NameTable.
- std::vector<std::string> MD5StringBuf;
-
- /// The starting address of fixed length MD5 name table section.
- const uint8_t *MD5NameMemStart = nullptr;
+ std::vector<FunctionId> NameTable;
/// CSNameTable is used to save full context vectors. It is the backing buffer
/// for SampleContextFrames.
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 1f19283ea1dd0a8..963a4d4918e567b 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -196,20 +196,20 @@ class SampleProfileWriterBinary : public SampleProfileWriter {
std::error_code writeSample(const FunctionSamples &S) override;
protected:
- virtual MapVector<StringRef, uint32_t> &getNameTable() { return NameTable; }
+ virtual MapVector<FunctionId, uint32_t> &getNameTable() { return NameTable; }
virtual std::error_code writeMagicIdent(SampleProfileFormat Format);
virtual std::error_code writeNameTable();
std::error_code writeHeader(const SampleProfileMap &ProfileMap) override;
std::error_code writeSummary();
virtual std::error_code writeContextIdx(const SampleContext &Context);
- std::error_code writeNameIdx(StringRef FName);
+ std::error_code writeNameIdx(FunctionId FName);
std::error_code writeBody(const FunctionSamples &S);
- inline void stablizeNameTable(MapVector<StringRef, uint32_t> &NameTable,
- std::set<StringRef> &V);
-
- MapVector<StringRef, uint32_t> NameTable;
-
- void addName(StringRef FName);
+ inline void stablizeNameTable(MapVector<FunctionId, uint32_t> &NameTable,
+ std::set<FunctionId> &V);
+
+ MapVector<FunctionId, uint32_t> NameTable;
+
+ void addName(FunctionId FName);
virtual void addContext(const SampleContext &Context);
void addNames(const FunctionSamples &S);
diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
index bc8360a80bc02bd..9436e7b41a98f64 100644
--- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
+++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
@@ -52,10 +52,11 @@ struct ProfiledCallGraphNode {
using edges = std::set<edge, ProfiledCallGraphEdgeComparer>;
using iterator = edges::iterator;
using const_iterator = edges::const_iterator;
-
- ProfiledCallGraphNode(StringRef FName = StringRef()) : Name(FName) {}
-
- StringRef Name;
+
+ ProfiledCallGraphNode(FunctionId FName = FunctionId()) : Name(FName)
+ {}
+
+ FunctionId Name;
edges Edges;
};
@@ -85,7 +86,7 @@ class ProfiledCallGraph {
std::queue<ContextTrieNode *> Queue;
for (auto &Child : ContextTracker.getRootContext().getAllChildContext()) {
ContextTrieNode *Callee = &Child.second;
- addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+ addProfiledFunction(Callee->getFuncName());
Queue.push(Callee);
}
@@ -102,7 +103,7 @@ class ProfiledCallGraph {
// context-based one, which may in turn block context-based inlining.
for (auto &Child : Caller->getAllChildContext()) {
ContextTrieNode *Callee = &Child.second;
- addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+ addProfiledFunction(Callee->getFuncName());
Queue.push(Callee);
// Fetch edge weight from the profile.
@@ -116,15 +117,14 @@ class ProfiledCallGraph {
LineLocation Callsite = Callee->getCallSiteLoc();
if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
- auto It = TargetCounts.find(CalleeSamples->getName());
+ auto It = TargetCounts.find(CalleeSamples->getFunction());
if (It != TargetCounts.end())
CallsiteCount = It->second;
}
Weight = std::max(CallsiteCount, CalleeEntryCount);
}
- addProfiledCall(ContextTracker.getFuncNameFor(Caller),
- ContextTracker.getFuncNameFor(Callee), Weight);
+ addProfiledCall(Caller->getFuncName(), Callee->getFuncName(), Weight);
}
}
@@ -136,8 +136,8 @@ class ProfiledCallGraph {
iterator begin() { return Root.Edges.begin(); }
iterator end() { return Root.Edges.end(); }
ProfiledCallGraphNode *getEntryNode() { return &Root; }
-
- void addProfiledFunction(StringRef Name) {
+
+ void addProfiledFunction(FunctionId Name) {
if (!ProfiledFunctions.count(Name)) {
// Link to synthetic root to make sure every node is reachable
// from root. This does not affect SCC order.
@@ -147,7 +147,7 @@ class ProfiledCallGraph {
}
private:
- void addProfiledCall(StringRef CallerName, StringRef CalleeName,
+ void addProfiledCall(FunctionId CallerName, FunctionId CalleeName,
uint64_t Weight = 0) {
assert(ProfiledFunctions.count(CallerName));
auto CalleeIt = ProfiledFunctions.find(CalleeName);
@@ -168,19 +168,19 @@ class ProfiledCallGraph {
}
void addProfiledCalls(const FunctionSamples &Samples) {
- addProfiledFunction(Samples.getFuncName());
+ addProfiledFunction(Samples.getFunction());
for (const auto &Sample : Samples.getBodySamples()) {
for (const auto &[Target, Frequency] : Sample.second.getCallTargets()) {
addProfiledFunction(Target);
- addProfiledCall(Samples.getFuncName(), Target, Frequency);
+ addProfiledCall(Samples.getFunction(), Target, Frequency);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
for (const auto &InlinedSamples : CallsiteSamples.second) {
addProfiledFunction(InlinedSamples.first);
- addProfiledCall(Samples.getFuncName(), InlinedSamples.first,
+ addProfiledCall(Samples.getFunction(), InlinedSamples.first,
InlinedSamples.second.getHeadSamplesEstimate());
addProfiledCalls(InlinedSamples.second);
}
@@ -206,7 +206,8 @@ class ProfiledCallGraph {
}
ProfiledCallGraphNode Root;
- StringMap<ProfiledCallGraphNode> ProfiledFunctions;
+ HashKeyMap<std::unordered_map, FunctionId, ProfiledCallGraphNode>
+ ProfiledFunctions;
};
} // end namespace sampleprof
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 347dac1e9684db1..f4c999ab93d8aac 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -35,20 +35,20 @@ class Instruction;
class ContextTrieNode {
public:
ContextTrieNode(ContextTrieNode *Parent = nullptr,
- StringRef FName = StringRef(),
+ FunctionId FName = FunctionId(),
FunctionSamples *FSamples = nullptr,
LineLocation CallLoc = {0, 0})
: ParentContext(Parent), FuncName(FName), FuncSamples(FSamples),
CallSiteLoc(CallLoc){};
ContextTrieNode *getChildContext(const LineLocation &CallSite,
- StringRef ChildName);
+ FunctionId ChildName);
ContextTrieNode *getHottestChildContext(const LineLocation &CallSite);
ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite,
- StringRef ChildName,
+ FunctionId ChildName,
bool AllowCreate = true);
- void removeChildContext(const LineLocation &CallSite, StringRef ChildName);
+ void removeChildContext(const LineLocation &CallSite, FunctionId ChildName);
std::map<uint64_t, ContextTrieNode> &getAllChildContext();
- StringRef getFuncName() const;
+ FunctionId getFuncName() const;
FunctionSamples *getFunctionSamples() const;
void setFunctionSamples(FunctionSamples *FSamples);
std::optional<uint32_t> getFunctionSize() const;
@@ -68,7 +68,7 @@ class ContextTrieNode {
ContextTrieNode *ParentContext;
// Function name for current context
- StringRef FuncName;
+ FunctionId FuncName;
// Function Samples for current context
FunctionSamples *FuncSamples;
@@ -118,7 +118,8 @@ class SampleContextTracker {
FunctionSamples *getBaseSamplesFor(const Function &Func,
bool MergeContext = true);
// Query base profile for a given function by name.
- FunctionSamples *getBaseSamplesFor(StringRef Name, bool MergeContext = true);
+ FunctionSamples *getBaseSamplesFor(FunctionId Name,
+ bool MergeContext = true);
// Retrieve the context trie node for given profile context
ContextTrieNode *getContextFor(const SampleContext &Context);
// Get real function name for a given trie node.
@@ -129,7 +130,7 @@ class SampleContextTracker {
void markContextSamplesInlined(const FunctionSamples *InlinedSamples);
ContextTrieNode &getRootContext();
void promoteMergeContextSamplesTree(const Instruction &Inst,
- StringRef CalleeName);
+ FunctionId CalleeName);
// Create a merged conext-less profile map.
void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles);
@@ -140,7 +141,8 @@ class SampleContextTracker {
return nullptr;
return I->second;
}
- StringMap<ContextSamplesTy> &getFuncToCtxtProfiles() {
+ HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+ &getFuncToCtxtProfiles() {
return FuncToCtxtProfiles;
}
@@ -189,9 +191,9 @@ class SampleContextTracker {
private:
ContextTrieNode *getContextFor(const DILocation *DIL);
ContextTrieNode *getCalleeContextFor(const DILocation *DIL,
- StringRef CalleeName);
- ContextTrieNode *getTopLevelContextNode(StringRef FName);
- ContextTrieNode &addTopLevelContextNode(StringRef FName);
+ FunctionId CalleeName);
+ ContextTrieNode *getTopLevelContextNode(FunctionId FName);
+ ContextTrieNode &addTopLevelContextNode(FunctionId FName);
ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo);
void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode);
ContextTrieNode &
@@ -204,7 +206,8 @@ class SampleContextTracker {
ProfileToNodeMap[FSample] = Node;
}
// Map from function name to context profiles (excluding base profile)
- StringMap<ContextSamplesTy> FuncToCtxtProfiles;
+ HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+ FuncToCtxtProfiles;
// Map from current FunctionSample to the belonged context trie.
std::unordered_map<const FunctionSamples *, ContextTrieNode *>
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index b14dc01be236245..59fa71899ed47b2 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -121,7 +121,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
sampleprof_error Result;
Result = addSamples(Other.getSamples(), Weight);
for (const auto &I : Other.getCallTargets()) {
- MergeResult(Result, addCalledTarget(I.first(), I.second, Weight));
+ MergeResult(Result, addCalledTarget(I.first, I.second, Weight));
}
return Result;
}
@@ -181,7 +181,8 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
for (const auto &CS : SortedCallsiteSamples.get()) {
for (const auto &FS : CS->second) {
OS.indent(Indent + 2);
- OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
+ OS << CS->first << ": inlined callee: " << FS.second.getFunction()
+ << ": ";
FS.second.print(OS, Indent + 4);
}
}
@@ -234,14 +235,6 @@ LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL,
}
}
-uint64_t FunctionSamples::getCallSiteHash(StringRef CalleeName,
- const LineLocation &Callsite) {
- uint64_t NameHash = std::hash<std::string>{}(CalleeName.str());
- uint64_t LocId =
- (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator;
- return NameHash + (LocId << 5) + LocId;
-}
-
const FunctionSamples *FunctionSamples::findFunctionSamples(
const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
assert(DIL);
@@ -268,11 +261,11 @@ const FunctionSamples *FunctionSamples::findFunctionSamples(
return FS;
}
-void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
- NameSet.insert(getName());
+void FunctionSamples::findAllNames(DenseSet<FunctionId> &NameSet) const {
+ NameSet.insert(getFunction());
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
- NameSet.insert(TS.getKey());
+ NameSet.insert(TS.first);
for (const auto &CS : CallsiteSamples) {
for (const auto &NameFS : CS.second) {
@@ -287,18 +280,15 @@ const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
SampleProfileReaderItaniumRemapper *Remapper) const {
CalleeName = getCanonicalFnName(CalleeName);
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
-
auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
if (iter == CallsiteSamples.end())
return nullptr;
- auto FS = iter->second.find(CalleeName);
+ auto FS = iter->second.find(getRepInFormat(CalleeName));
if (FS != iter->second.end())
return &FS->second;
if (Remapper) {
if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
- auto FS = iter->second.find(*NameInProfile);
+ auto FS = iter->second.find(getRepInFormat(*NameInProfile));
if (FS != iter->second.end())
return &FS->second;
}
@@ -422,7 +412,7 @@ void ProfileSymbolList::dump(raw_ostream &OS) const {
ProfileConverter::FrameNode *
ProfileConverter::FrameNode::getOrCreateChildFrame(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
auto It = AllChildFrames.find(Hash);
if (It != AllChildFrames.end()) {
@@ -450,7 +440,7 @@ ProfileConverter::getOrCreateContextPath(const SampleContext &Context) {
auto Node = &RootFrame;
LineLocation CallSiteLoc(0, 0);
for (auto &Callsite : Context.getContextFrames()) {
- Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.FuncName);
+ Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.Func);
CallSiteLoc = Callsite.Location;
}
return Node;
@@ -468,23 +458,23 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
if (!ChildProfile)
continue;
SampleContext OrigChildContext = ChildProfile->getContext();
- hash_code OrigChildContextHash = OrigChildContext.getHashCode();
+ uint64_t OrigChildContextHash = OrigChildContext.getHashCode();
// Reset the child context to be contextless.
- ChildProfile->getContext().setName(OrigChildContext.getName());
+ ChildProfile->getContext().setFunction(OrigChildContext.getFunction());
if (NodeProfile) {
// Add child profile to the callsite profile map.
auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
- SamplesMap.emplace(OrigChildContext.getName().str(), *ChildProfile);
+ SamplesMap.emplace(OrigChildContext.getFunction(), *ChildProfile);
NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
// Remove the corresponding body sample for the callsite and update the
// total weight.
auto Count = NodeProfile->removeCalledTargetAndBodySample(
ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,
- OrigChildContext.getName());
+ OrigChildContext.getFunction());
NodeProfile->removeTotalSamples(Count);
}
- hash_code NewChildProfileHash(0);
+ uint64_t NewChildProfileHash = 0;
// Separate child profile to be a standalone profile, if the current parent
// profile doesn't exist. This is a duplicating operation when the child
// profile is already incorporated into the parent which is still useful and
@@ -498,7 +488,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
NewChildProfileHash = ChildProfile->getContext().getHashCode();
auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
- SamplesMap[ChildProfile->getName().str()].getContext().setAttribute(
+ SamplesMap[ChildProfile->getFunction()].getContext().setAttribute(
ContextDuplicatedIntoBase);
}
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 256bdb833a0b187..a69e9d5849047ab 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -91,7 +91,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
JOS.attributeArray("calls", [&] {
for (const auto &J : CallTargets) {
JOS.object([&] {
- JOS.attribute("function", J.first);
+ JOS.attribute("function", J.first.str());
JOS.attribute("samples", J.second);
});
}
@@ -117,7 +117,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
};
JOS.object([&] {
- JOS.attribute("name", S.getName());
+ JOS.attribute("name", S.getFunction().str());
JOS.attribute("total", S.getTotalSamples());
if (TopLevel)
JOS.attribute("head", S.getHeadSamples());
@@ -392,8 +392,8 @@ std::error_code SampleProfileReaderText::readImpl() {
switch (LineTy) {
case LineType::CallSiteProfile: {
FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
- LineLocation(LineOffset, Discriminator))[std::string(FName)];
- FSamples.setName(FName);
+ LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
+ FSamples.setFunction(FunctionId(FName));
MergeResult(Result, FSamples.addTotalSamples(NumSamples));
InlineStack.push_back(&FSamples);
DepthMetadata = 0;
@@ -406,7 +406,8 @@ std::error_code SampleProfileReaderText::readImpl() {
FunctionSamples &FProfile = *InlineStack.back();
for (const auto &name_count : TargetCountMap) {
MergeResult(Result, FProfile.addCalledTargetSamples(
- LineOffset, Discriminator, name_count.first,
+ LineOffset, Discriminator,
+ FunctionId(name_count.first),
name_count.second));
}
MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
@@ -516,28 +517,14 @@ inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
return *Idx;
}
-ErrorOr<StringRef>
+ErrorOr<FunctionId>
SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
auto Idx = readStringIndex(NameTable);
if (std::error_code EC = Idx.getError())
return EC;
-
- // Lazy loading, if the string has not been materialized from memory storing
- // MD5 values, then it is default initialized with the null pointer. This can
- // only happen when using fixed length MD5, that bounds check is performed
- // while parsing the name table to ensure MD5NameMemStart points to an array
- // with enough MD5 entries.
- StringRef &SR = NameTable[*Idx];
- if (!SR.data()) {
- assert(MD5NameMemStart);
- using namespace support;
- uint64_t FID = endian::read<uint64_t, llvm::endianness::little>(
- MD5NameMemStart + (*Idx) * sizeof(uint64_t));
- SR = MD5StringBuf.emplace_back(std::to_string(FID));
- }
if (RetIdx)
*RetIdx = *Idx;
- return SR;
+ return NameTable[*Idx];
}
ErrorOr<SampleContextFrames>
@@ -655,8 +642,8 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
- LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)];
- CalleeProfile.setName(*FName);
+ LineLocation(*LineOffset, DiscriminatorVal))[*FName];
+ CalleeProfile.setFunction(*FName);
if (std::error_code EC = readProfile(CalleeProfile))
return EC;
}
@@ -894,13 +881,17 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
const SampleContext *CommonContext = nullptr;
for (const auto &NameOffset : FuncOffsetList) {
const auto &FContext = NameOffset.first;
- auto FName = FContext.getName();
+ FunctionId FName = FContext.getFunction();
+ StringRef FNameString;
+ if (!useMD5())
+ FNameString = FName.stringRef();
+
// For function in the current module, keep its farthest ancestor
// context. This can be used to load itself and its child and
// sibling contexts.
- if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) ||
- (!useMD5() && (FuncsToUse.count(FName) ||
- (Remapper && Remapper->exist(FName))))) {
+ if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
+ (!useMD5() && (FuncsToUse.count(FNameString) ||
+ (Remapper && Remapper->exist(FNameString))))) {
if (!CommonContext || !CommonContext->IsPrefixOf(FContext))
CommonContext = &FContext;
}
@@ -929,8 +920,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
assert(useFuncOffsetList());
for (auto NameOffset : FuncOffsetList) {
SampleContext FContext(NameOffset.first);
- auto FuncName = FContext.getName();
- if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName))
+ auto FuncName = FContext.getFunction();
+ StringRef FuncNameStr = FuncName.stringRef();
+ if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
continue;
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
@@ -1062,8 +1054,6 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
// tables mixing string and MD5, all of them have to be normalized to use MD5,
// because optimization passes can only handle either type.
bool UseMD5 = useMD5();
- if (UseMD5)
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
NameTable.reserve(*Size);
@@ -1082,12 +1072,12 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
if (std::error_code EC = Name.getError())
return EC;
if (UseMD5) {
- uint64_t FID = hashFuncName(*Name);
+ FunctionId FID(*Name);
if (!ProfileIsCS)
- MD5SampleContextTable.emplace_back(FID);
- NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID)));
+ MD5SampleContextTable.emplace_back(FID.getHashCode());
+ NameTable.emplace_back(FID);
} else
- NameTable.push_back(*Name);
+ NameTable.push_back(FunctionId(*Name));
}
if (!ProfileIsCS)
MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1110,14 +1100,14 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
if (Data + (*Size) * sizeof(uint64_t) > End)
return sampleprof_error::truncated;
- // Preallocate and initialize NameTable so we can check whether a name
- // index has been read before by checking whether the element in the
- // NameTable is empty, meanwhile readStringIndex can do the boundary
- // check using the size of NameTable.
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
- NameTable.resize(*Size);
- MD5NameMemStart = Data;
+ NameTable.reserve(*Size);
+ for (size_t I = 0; I < *Size; ++I) {
+ using namespace support;
+ uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
+ Data + I * sizeof(uint64_t));
+ NameTable.emplace_back(FunctionId(FID));
+ }
if (!ProfileIsCS)
MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
Data = Data + (*Size) * sizeof(uint64_t);
@@ -1130,7 +1120,6 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
if (std::error_code EC = Size.getError())
return EC;
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
NameTable.reserve(*Size);
if (!ProfileIsCS)
@@ -1141,7 +1130,7 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
return EC;
if (!ProfileIsCS)
support::endian::write64le(&MD5SampleContextTable[I], *FID);
- NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID)));
+ NameTable.emplace_back(FunctionId(*FID));
}
if (!ProfileIsCS)
MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1243,7 +1232,7 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
CalleeProfile = const_cast<FunctionSamples *>(
&FProfile->functionSamplesAt(LineLocation(
*LineOffset,
- *Discriminator))[std::string(FContext.getName())]);
+ *Discriminator))[FContext.getFunction()]);
}
if (std::error_code EC =
readFuncMetadata(ProfileHasAttribute, CalleeProfile))
@@ -1653,7 +1642,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
// body, there will be identical replicated profiles for the
// original function. In this case, we simply not bother updating
// the profile of the original function.
- FProfile = &Profiles[Name];
+ FProfile = &Profiles[FunctionId(Name)];
FProfile->addHeadSamples(HeadCount);
if (FProfile->getTotalSamples() > 0)
Update = false;
@@ -1665,9 +1654,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
uint32_t LineOffset = Offset >> 16;
uint32_t Discriminator = Offset & 0xffff;
FProfile = &CallerProfile->functionSamplesAt(
- LineLocation(LineOffset, Discriminator))[std::string(Name)];
+ LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
}
- FProfile->setName(Name);
+ FProfile->setFunction(FunctionId(Name));
for (uint32_t I = 0; I < NumPosCounts; ++I) {
uint32_t Offset;
@@ -1723,7 +1712,8 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
if (Update)
FProfile->addCalledTargetSamples(LineOffset, Discriminator,
- TargetName, TargetCount);
+ FunctionId(TargetName),
+ TargetCount);
}
}
@@ -1784,11 +1774,13 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
// We will need to remap the entire context string.
assert(Remappings && "should be initialized while creating remapper");
for (auto &Sample : Reader.getProfiles()) {
- DenseSet<StringRef> NamesInSample;
+ DenseSet<FunctionId> NamesInSample;
Sample.second.findAllNames(NamesInSample);
- for (auto &Name : NamesInSample)
- if (auto Key = Remappings->insert(Name))
- NameMap.insert({Key, Name});
+ for (auto &Name : NamesInSample) {
+ StringRef NameStr = Name.stringRef();
+ if (auto Key = Remappings->insert(NameStr))
+ NameMap.insert({Key, NameStr});
+ }
}
RemappingApplied = true;
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index c11cd4dfa6a5a97..625e523f13cec0e 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -240,7 +240,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
if (Context.hasContext())
return writeCSNameIdx(Context);
else
- return SampleProfileWriterBinary::writeNameIdx(Context.getName());
+ return SampleProfileWriterBinary::writeNameIdx(Context.getFunction());
}
std::error_code
@@ -346,7 +346,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
return SampleProfileWriterBinary::writeNameTable();
auto &OS = *OutputStream;
- std::set<StringRef> V;
+ std::set<FunctionId> V;
stablizeNameTable(NameTable, V);
// Write out the MD5 name table. We wrote unencoded MD5 so reader can
@@ -355,7 +355,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
encodeULEB128(NameTable.size(), OS);
support::endian::Writer Writer(OS, llvm::endianness::little);
for (auto N : V)
- Writer.write(hashFuncName(N));
+ Writer.write(N.getHashCode());
return sampleprof_error::success;
}
@@ -369,10 +369,13 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
// If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
// so compiler won't strip the suffix during profile matching after
// seeing the flag in the profile.
- for (const auto &I : NameTable) {
- if (I.first.contains(FunctionSamples::UniqSuffix)) {
- addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
- break;
+ // Original names are unavailable if using MD5, so this option has no use.
+ if (!UseMD5) {
+ for (const auto &I : NameTable) {
+ if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {
+ addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
+ break;
+ }
}
}
@@ -399,7 +402,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
auto Frames = Context.getContextFrames();
encodeULEB128(Frames.size(), OS);
for (auto &Callsite : Frames) {
- if (std::error_code EC = writeNameIdx(Callsite.FuncName))
+ if (std::error_code EC = writeNameIdx(Callsite.Func))
return EC;
encodeULEB128(Callsite.Location.LineOffset, OS);
encodeULEB128(Callsite.Location.Discriminator, OS);
@@ -567,7 +570,7 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
if (FunctionSamples::ProfileIsCS)
OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
else
- OS << S.getName() << ":" << S.getTotalSamples();
+ OS << S.getFunction() << ":" << S.getTotalSamples();
if (Indent == 0)
OS << ":" << S.getHeadSamples();
@@ -627,10 +630,10 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
std::error_code
SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
assert(!Context.hasContext() && "cs profile is not supported");
- return writeNameIdx(Context.getName());
+ return writeNameIdx(Context.getFunction());
}
-std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
+std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
auto &NTable = getNameTable();
const auto &Ret = NTable.find(FName);
if (Ret == NTable.end())
@@ -639,13 +642,13 @@ std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
return sampleprof_error::success;
}
-void SampleProfileWriterBinary::addName(StringRef FName) {
+void SampleProfileWriterBinary::addName(FunctionId FName) {
auto &NTable = getNameTable();
NTable.insert(std::make_pair(FName, 0));
}
void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
- addName(Context.getName());
+ addName(Context.getFunction());
}
void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
@@ -653,14 +656,14 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
for (const auto &I : S.getBodySamples()) {
const SampleRecord &Sample = I.second;
for (const auto &J : Sample.getCallTargets())
- addName(J.first());
+ addName(J.first);
}
// Recursively add all the names for inlined callsites.
for (const auto &J : S.getCallsiteSamples())
for (const auto &FS : J.second) {
const FunctionSamples &CalleeSamples = FS.second;
- addName(CalleeSamples.getName());
+ addName(CalleeSamples.getFunction());
addNames(CalleeSamples);
}
}
@@ -669,26 +672,26 @@ void SampleProfileWriterExtBinaryBase::addContext(
const SampleContext &Context) {
if (Context.hasContext()) {
for (auto &Callsite : Context.getContextFrames())
- SampleProfileWriterBinary::addName(Callsite.FuncName);
+ SampleProfileWriterBinary::addName(Callsite.Func);
CSNameTable.insert(std::make_pair(Context, 0));
} else {
- SampleProfileWriterBinary::addName(Context.getName());
+ SampleProfileWriterBinary::addName(Context.getFunction());
}
}
void SampleProfileWriterBinary::stablizeNameTable(
- MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) {
+ MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
// Sort the names to make NameTable deterministic.
for (const auto &I : NameTable)
V.insert(I.first);
int i = 0;
- for (const StringRef &N : V)
+ for (const FunctionId &N : V)
NameTable[N] = i++;
}
std::error_code SampleProfileWriterBinary::writeNameTable() {
auto &OS = *OutputStream;
- std::set<StringRef> V;
+ std::set<FunctionId> V;
stablizeNameTable(NameTable, V);
// Write out the name table.
@@ -835,7 +838,7 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
encodeULEB128(Sample.getSamples(), OS);
encodeULEB128(Sample.getCallTargets().size(), OS);
for (const auto &J : Sample.getSortedCallTargets()) {
- StringRef Callee = J.first;
+ FunctionId Callee = J.first;
uint64_t CalleeSamples = J.second;
if (std::error_code EC = writeNameIdx(Callee))
return EC;
diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
index 29ae05bf0c94673..3c9738be547f33b 100644
--- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
@@ -110,6 +110,11 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
Prefetches &Prefetches) const {
assert(Prefetches.empty() &&
"Expected caller passed empty PrefetchInfo vector.");
+
+ // There is no point to match prefetch hints if the profile is using MD5.
+ if (FunctionSamples::UseMD5)
+ return false;
+
static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
{"_nta_", X86::PREFETCHNTA},
{"_t0_", X86::PREFETCHT0},
@@ -125,7 +130,7 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
// Convert serialized prefetch hints into PrefetchInfo objects, and populate
// the Prefetches vector.
for (const auto &S_V : *T) {
- StringRef Name = S_V.getKey();
+ StringRef Name = S_V.first.stringRef();
if (Name.consume_front(SerializedPrefetchPrefix)) {
int64_t D = static_cast<int64_t>(S_V.second);
unsigned IID = 0;
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index 79bee8362ef8a9c..09dd5c11b3094bd 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -29,7 +29,7 @@ using namespace sampleprof;
namespace llvm {
ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
if (CalleeName.empty())
return getHottestChildContext(CallSite);
@@ -104,7 +104,7 @@ SampleContextTracker::moveContextSamples(ContextTrieNode &ToNodeParent,
}
void ContextTrieNode::removeChildContext(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
// Note this essentially calls dtor and destroys that child context
AllChildContext.erase(Hash);
@@ -114,7 +114,7 @@ std::map<uint64_t, ContextTrieNode> &ContextTrieNode::getAllChildContext() {
return AllChildContext;
}
-StringRef ContextTrieNode::getFuncName() const { return FuncName; }
+FunctionId ContextTrieNode::getFuncName() const { return FuncName; }
FunctionSamples *ContextTrieNode::getFunctionSamples() const {
return FuncSamples;
@@ -178,7 +178,7 @@ void ContextTrieNode::dumpTree() {
}
ContextTrieNode *ContextTrieNode::getOrCreateChildContext(
- const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) {
+ const LineLocation &CallSite, FunctionId CalleeName, bool AllowCreate) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
auto It = AllChildContext.find(Hash);
if (It != AllChildContext.end()) {
@@ -232,14 +232,12 @@ SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst,
return nullptr;
CalleeName = FunctionSamples::getCanonicalFnName(CalleeName);
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::string FGUID;
- CalleeName = getRepInFormat(CalleeName, FunctionSamples::UseMD5, FGUID);
+
+ FunctionId FName = getRepInFormat(CalleeName);
// For indirect call, CalleeName will be empty, in which case the context
// profile for callee with largest total samples will be returned.
- ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName);
+ ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, FName);
if (CalleeContext) {
FunctionSamples *FSamples = CalleeContext->getFunctionSamples();
LLVM_DEBUG(if (FSamples) {
@@ -305,27 +303,23 @@ SampleContextTracker::getContextSamplesFor(const SampleContext &Context) {
SampleContextTracker::ContextSamplesTy &
SampleContextTracker::getAllContextSamplesFor(const Function &Func) {
StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
- return FuncToCtxtProfiles[CanonName];
+ return FuncToCtxtProfiles[getRepInFormat(CanonName)];
}
SampleContextTracker::ContextSamplesTy &
SampleContextTracker::getAllContextSamplesFor(StringRef Name) {
- return FuncToCtxtProfiles[Name];
+ return FuncToCtxtProfiles[getRepInFormat(Name)];
}
FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func,
bool MergeContext) {
StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
- return getBaseSamplesFor(CanonName, MergeContext);
+ return getBaseSamplesFor(getRepInFormat(CanonName), MergeContext);
}
-FunctionSamples *SampleContextTracker::getBaseSamplesFor(StringRef Name,
+FunctionSamples *SampleContextTracker::getBaseSamplesFor(FunctionId Name,
bool MergeContext) {
LLVM_DEBUG(dbgs() << "Getting base profile for function: " << Name << "\n");
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::string FGUID;
- Name = getRepInFormat(Name, FunctionSamples::UseMD5, FGUID);
// Base profile is top-level node (child of root node), so try to retrieve
// existing top-level node for given function first. If it exists, it could be
@@ -373,7 +367,7 @@ void SampleContextTracker::markContextSamplesInlined(
ContextTrieNode &SampleContextTracker::getRootContext() { return RootContext; }
void SampleContextTracker::promoteMergeContextSamplesTree(
- const Instruction &Inst, StringRef CalleeName) {
+ const Instruction &Inst, FunctionId CalleeName) {
LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n"
<< Inst << "\n");
// Get the caller context for the call instruction, we don't use callee
@@ -458,9 +452,9 @@ void SampleContextTracker::dump() { RootContext.dumpTree(); }
StringRef SampleContextTracker::getFuncNameFor(ContextTrieNode *Node) const {
if (!FunctionSamples::UseMD5)
- return Node->getFuncName();
+ return Node->getFuncName().stringRef();
assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
- return GUIDToFuncNameMap->lookup(std::stoull(Node->getFuncName().data()));
+ return GUIDToFuncNameMap->lookup(Node->getFuncName().getHashCode());
}
ContextTrieNode *
@@ -470,7 +464,7 @@ SampleContextTracker::getContextFor(const SampleContext &Context) {
ContextTrieNode *
SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
assert(DIL && "Expect non-null location");
ContextTrieNode *CallContext = getContextFor(DIL);
@@ -485,7 +479,7 @@ SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
assert(DIL && "Expect non-null location");
- SmallVector<std::pair<LineLocation, StringRef>, 10> S;
+ SmallVector<std::pair<LineLocation, FunctionId>, 10> S;
// Use C++ linkage name if possible.
const DILocation *PrevDIL = DIL;
@@ -494,7 +488,8 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
if (Name.empty())
Name = PrevDIL->getScope()->getSubprogram()->getName();
S.push_back(
- std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), Name));
+ std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL),
+ getRepInFormat(Name)));
PrevDIL = DIL;
}
@@ -503,24 +498,14 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
StringRef RootName = PrevDIL->getScope()->getSubprogram()->getLinkageName();
if (RootName.empty())
RootName = PrevDIL->getScope()->getSubprogram()->getName();
- S.push_back(std::make_pair(LineLocation(0, 0), RootName));
-
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::list<std::string> MD5Names;
- if (FunctionSamples::UseMD5) {
- for (auto &Location : S) {
- MD5Names.emplace_back();
- getRepInFormat(Location.second, FunctionSamples::UseMD5, MD5Names.back());
- Location.second = MD5Names.back();
- }
- }
+ S.push_back(std::make_pair(LineLocation(0, 0),
+ getRepInFormat(RootName)));
ContextTrieNode *ContextNode = &RootContext;
int I = S.size();
while (--I >= 0 && ContextNode) {
LineLocation &CallSite = S[I].first;
- StringRef CalleeName = S[I].second;
+ FunctionId CalleeName = S[I].second;
ContextNode = ContextNode->getChildContext(CallSite, CalleeName);
}
@@ -540,10 +525,10 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
// Create child node at parent line/disc location
if (AllowCreate) {
ContextNode =
- ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.FuncName);
+ ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.Func);
} else {
ContextNode =
- ContextNode->getChildContext(CallSiteLoc, Callsite.FuncName);
+ ContextNode->getChildContext(CallSiteLoc, Callsite.Func);
}
CallSiteLoc = Callsite.Location;
}
@@ -553,12 +538,14 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
return ContextNode;
}
-ContextTrieNode *SampleContextTracker::getTopLevelContextNode(StringRef FName) {
+ContextTrieNode *
+SampleContextTracker::getTopLevelContextNode(FunctionId FName) {
assert(!FName.empty() && "Top level node query must provide valid name");
return RootContext.getChildContext(LineLocation(0, 0), FName);
}
-ContextTrieNode &SampleContextTracker::addTopLevelContextNode(StringRef FName) {
+ContextTrieNode &
+SampleContextTracker::addTopLevelContextNode(FunctionId FName) {
assert(!getTopLevelContextNode(FName) && "Node to add must not exist");
return *RootContext.getOrCreateChildContext(LineLocation(0, 0), FName);
}
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 67a2e167b6bf7dd..a7773737ef16bd3 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -424,7 +424,7 @@ struct CandidateComparer {
return LCS->getBodySamples().size() > RCS->getBodySamples().size();
// Tie breaker using GUID so we have stable/deterministic inlining order
- return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName());
+ return LCS->getGUID() < RCS->getGUID();
}
};
@@ -467,7 +467,7 @@ class SampleProfileMatcher {
private:
FunctionSamples *getFlattenedSamplesFor(const Function &F) {
StringRef CanonFName = FunctionSamples::getCanonicalFnName(F);
- auto It = FlattenedProfiles.find(CanonFName);
+ auto It = FlattenedProfiles.find(FunctionId(CanonFName));
if (It != FlattenedProfiles.end())
return &It->second;
return nullptr;
@@ -475,18 +475,21 @@ class SampleProfileMatcher {
void runOnFunction(const Function &F);
void findIRAnchors(const Function &F,
std::map<LineLocation, StringRef> &IRAnchors);
- void findProfileAnchors(const FunctionSamples &FS,
- std::map<LineLocation, StringSet<>> &ProfileAnchors);
+ void findProfileAnchors(
+ const FunctionSamples &FS,
+ std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors);
void countMismatchedSamples(const FunctionSamples &FS);
void countProfileMismatches(
const Function &F, const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors);
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors);
void countProfileCallsiteMismatches(
const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
-
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites);
LocToLocMap &getIRToProfileLocationMap(const Function &F) {
auto Ret = FuncMappings.try_emplace(
@@ -497,7 +500,8 @@ class SampleProfileMatcher {
void distributeIRToProfileLocationMap(FunctionSamples &FS);
void runStaleProfileMatching(
const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
LocToLocMap &IRToProfileLocationMap);
};
@@ -539,7 +543,6 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
void findExternalInlineCandidate(CallBase *CB, const FunctionSamples *Samples,
DenseSet<GlobalValue::GUID> &InlinedGUIDs,
- const StringMap<Function *> &SymbolMap,
uint64_t Threshold);
// Attempt to promote indirect call and also inline the promoted call
bool tryPromoteAndInlineCandidate(
@@ -574,7 +577,7 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
/// the function name. If the function name contains suffix, additional
/// entry is added to map from the stripped name to the function if there
/// is one-to-one mapping.
- StringMap<Function *> SymbolMap;
+ HashKeyMap<std::unordered_map, FunctionId, Function *> SymbolMap;
std::function<AssumptionCache &(Function &)> GetAC;
std::function<TargetTransformInfo &(Function &)> GetTTI;
@@ -616,6 +619,11 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
// All the Names used in FunctionSamples including outline function
// names, inline instance names and call target names.
StringSet<> NamesInProfile;
+ // MD5 version of NamesInProfile. Either NamesInProfile or GUIDsInProfile is
+ // populated, depends on whether the profile uses MD5. Because the name table
+ // generally contains several magnitude more entries than the number of
+ // functions, we do not want to convert all names from one form to another.
+ llvm::DenseSet<uint64_t> GUIDsInProfile;
// For symbol in profile symbol list, whether to regard their profiles
// to be accurate. It is mainly decided by existance of profile symbol
@@ -760,8 +768,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
assert(L && R && "Expect non-null FunctionSamples");
if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate())
return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate();
- return FunctionSamples::getGUID(L->getName()) <
- FunctionSamples::getGUID(R->getName());
+ return L->getGUID() < R->getGUID();
};
if (FunctionSamples::ProfileIsCS) {
@@ -971,13 +978,13 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
// This prevents allocating an array of zero length in callees below.
if (MaxNumPromotions == 0)
return false;
- auto CalleeFunctionName = Candidate.CalleeSamples->getFuncName();
+ auto CalleeFunctionName = Candidate.CalleeSamples->getFunction();
auto R = SymbolMap.find(CalleeFunctionName);
- if (R == SymbolMap.end() || !R->getValue())
+ if (R == SymbolMap.end() || !R->second)
return false;
auto &CI = *Candidate.CallInstr;
- if (!doesHistoryAllowICP(CI, R->getValue()->getName()))
+ if (!doesHistoryAllowICP(CI, R->second->getName()))
return false;
const char *Reason = "Callee function not available";
@@ -987,17 +994,17 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
// clone the caller first, and inline the cloned caller if it is
// recursive. As llvm does not inline recursive calls, we will
// simply ignore it instead of handling it explicitly.
- if (!R->getValue()->isDeclaration() && R->getValue()->getSubprogram() &&
- R->getValue()->hasFnAttribute("use-sample-profile") &&
- R->getValue() != &F && isLegalToPromote(CI, R->getValue(), &Reason)) {
+ if (!R->second->isDeclaration() && R->second->getSubprogram() &&
+ R->second->hasFnAttribute("use-sample-profile") &&
+ R->second != &F && isLegalToPromote(CI, R->second, &Reason)) {
// For promoted target, set its value with NOMORE_ICP_MAGICNUM count
// in the value profile metadata so the target won't be promoted again.
SmallVector<InstrProfValueData, 1> SortedCallTargets = {InstrProfValueData{
- Function::getGUID(R->getValue()->getName()), NOMORE_ICP_MAGICNUM}};
+ Function::getGUID(R->second->getName()), NOMORE_ICP_MAGICNUM}};
updateIDTMetaData(CI, SortedCallTargets, 0);
auto *DI = &pgo::promoteIndirectCall(
- CI, R->getValue(), Candidate.CallsiteCount, Sum, false, ORE);
+ CI, R->second, Candidate.CallsiteCount, Sum, false, ORE);
if (DI) {
Sum -= Candidate.CallsiteCount;
// Do not prorate the indirect callsite distribution since the original
@@ -1026,7 +1033,8 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
}
} else {
LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to "
- << Candidate.CalleeSamples->getFuncName() << " because "
+ << FunctionSamples::getCanonicalFnName(
+ Candidate.CallInstr->getName())<< " because "
<< Reason << "\n");
}
return false;
@@ -1071,8 +1079,7 @@ void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
void SampleProfileLoader::findExternalInlineCandidate(
CallBase *CB, const FunctionSamples *Samples,
- DenseSet<GlobalValue::GUID> &InlinedGUIDs,
- const StringMap<Function *> &SymbolMap, uint64_t Threshold) {
+ DenseSet<GlobalValue::GUID> &InlinedGUIDs, uint64_t Threshold) {
// If ExternalInlineAdvisor(ReplayInlineAdvisor) wants to inline an external
// function make sure it's imported
@@ -1081,7 +1088,7 @@ void SampleProfileLoader::findExternalInlineCandidate(
// just add the direct GUID and move on
if (!Samples) {
InlinedGUIDs.insert(
- FunctionSamples::getGUID(CB->getCalledFunction()->getName()));
+ Function::getGUID(CB->getCalledFunction()->getName()));
return;
}
// Otherwise, drop the threshold to import everything that we can
@@ -1122,22 +1129,20 @@ void SampleProfileLoader::findExternalInlineCandidate(
CalleeSample->getContext().hasAttribute(ContextShouldBeInlined);
if (!PreInline && CalleeSample->getHeadSamplesEstimate() < Threshold)
continue;
-
- StringRef Name = CalleeSample->getFuncName();
- Function *Func = SymbolMap.lookup(Name);
+
+ Function *Func = SymbolMap.lookup(CalleeSample->getFunction());
// Add to the import list only when it's defined out of module.
if (!Func || Func->isDeclaration())
- InlinedGUIDs.insert(FunctionSamples::getGUID(CalleeSample->getName()));
+ InlinedGUIDs.insert(CalleeSample->getGUID());
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : CalleeSample->getBodySamples())
for (const auto &TS : BS.second.getCallTargets())
- if (TS.getValue() > Threshold) {
- StringRef CalleeName = CalleeSample->getFuncName(TS.getKey());
- const Function *Callee = SymbolMap.lookup(CalleeName);
+ if (TS.second > Threshold) {
+ const Function *Callee = SymbolMap.lookup(TS.first);
if (!Callee || Callee->isDeclaration())
- InlinedGUIDs.insert(FunctionSamples::getGUID(TS.getKey()));
+ InlinedGUIDs.insert(TS.first.getHashCode());
}
// Import hot child context profile associted with callees. Note that this
@@ -1235,7 +1240,7 @@ bool SampleProfileLoader::inlineHotFunctions(
for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
uint64_t SumOrigin = Sum;
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1256,7 +1261,7 @@ bool SampleProfileLoader::inlineHotFunctions(
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
- InlinedGUIDs, SymbolMap,
+ InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
}
}
@@ -1505,7 +1510,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
for (const auto *FS : CalleeSamples) {
// TODO: Consider disable pre-lTO ICP for MonoLTO as well
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1558,7 +1563,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
- InlinedGUIDs, SymbolMap,
+ InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
}
}
@@ -1644,7 +1649,7 @@ GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
SmallVector<InstrProfValueData, 2> R;
for (const auto &I : SampleRecord::SortCallTargets(M)) {
R.emplace_back(
- InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
+ InstrProfValueData{I.first.getHashCode(), I.second});
}
return R;
}
@@ -1871,7 +1876,8 @@ SampleProfileLoader::buildProfiledCallGraph(Module &M) {
for (Function &F : M) {
if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile"))
continue;
- ProfiledCG->addProfiledFunction(FunctionSamples::getCanonicalFnName(F));
+ ProfiledCG->addProfiledFunction(
+ getRepInFormat(FunctionSamples::getCanonicalFnName(F)));
}
return ProfiledCG;
@@ -2022,8 +2028,16 @@ bool SampleProfileLoader::doInitialization(Module &M,
ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
if (ProfAccForSymsInList) {
NamesInProfile.clear();
- if (auto NameTable = Reader->getNameTable())
- NamesInProfile.insert(NameTable->begin(), NameTable->end());
+ GUIDsInProfile.clear();
+ if (auto NameTable = Reader->getNameTable()) {
+ if (FunctionSamples::UseMD5) {
+ for (auto Name : *NameTable)
+ GUIDsInProfile.insert(Name.getHashCode());
+ } else {
+ for (auto Name : *NameTable)
+ NamesInProfile.insert(Name.stringRef());
+ }
+ }
CoverageTracker.setProfAccForSymsInList(true);
}
@@ -2177,7 +2191,7 @@ void SampleProfileMatcher::findIRAnchors(
}
void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
- const auto *FuncDesc = ProbeManager->getDesc(FS.getName());
+ const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID());
// Skip the function that is external or renamed.
if (!FuncDesc)
return;
@@ -2194,7 +2208,8 @@ void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
void SampleProfileMatcher::countProfileMismatches(
const Function &F, const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors) {
[[maybe_unused]] bool IsFuncHashMismatch = false;
if (FunctionSamples::ProfileIsProbeBased) {
TotalFuncHashSamples += FS.getTotalSamples();
@@ -2228,7 +2243,8 @@ void SampleProfileMatcher::countProfileMismatches(
void SampleProfileMatcher::countProfileCallsiteMismatches(
const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites) {
// Check if there are any callsites in the profile that does not match to any
@@ -2263,7 +2279,7 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
// reported as mismatching.
if (IRCalleeName == UnknownIndirectCallee)
CallsiteIsMatched = true;
- else if (Callees.size() == 1 && Callees.count(IRCalleeName))
+ else if (Callees.size() == 1 && Callees.count(getRepInFormat(IRCalleeName)))
CallsiteIsMatched = true;
FuncProfiledCallsites++;
@@ -2275,9 +2291,8 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
}
}
-void SampleProfileMatcher::findProfileAnchors(
- const FunctionSamples &FS,
- std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
+ std::map<LineLocation, std::unordered_set<FunctionId>> &ProfileAnchors) {
auto isInvalidLineOffset = [](uint32_t LineOffset) {
return LineOffset & 0x8000;
};
@@ -2287,8 +2302,9 @@ void SampleProfileMatcher::findProfileAnchors(
if (isInvalidLineOffset(Loc.LineOffset))
continue;
for (const auto &I : I.second.getCallTargets()) {
- auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
- Ret.first->second.insert(I.first());
+ auto Ret = ProfileAnchors.try_emplace(Loc,
+ std::unordered_set<FunctionId>());
+ Ret.first->second.insert(I.first);
}
}
@@ -2298,7 +2314,8 @@ void SampleProfileMatcher::findProfileAnchors(
continue;
const auto &CalleeMap = I.second;
for (const auto &I : CalleeMap) {
- auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
+ auto Ret = ProfileAnchors.try_emplace(Loc,
+ std::unordered_set<FunctionId>());
Ret.first->second.insert(I.first);
}
}
@@ -2322,21 +2339,24 @@ void SampleProfileMatcher::findProfileAnchors(
// [1, 2, 3(foo), 4, 7, 8(bar), 9]
// The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
void SampleProfileMatcher::runStaleProfileMatching(
- const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const Function &F,
+ const std::map<LineLocation, StringRef> &IRAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
LocToLocMap &IRToProfileLocationMap) {
LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
<< "\n");
assert(IRToProfileLocationMap.empty() &&
"Run stale profile matching only once per function");
- StringMap<std::set<LineLocation>> CalleeToCallsitesMap;
+ std::unordered_map<FunctionId, std::set<LineLocation>>
+ CalleeToCallsitesMap;
for (const auto &I : ProfileAnchors) {
const auto &Loc = I.first;
const auto &Callees = I.second;
// Filter out possible indirect calls, use direct callee name as anchor.
if (Callees.size() == 1) {
- StringRef CalleeName = Callees.begin()->first();
+ FunctionId CalleeName = *Callees.begin();
const auto &Candidates = CalleeToCallsitesMap.try_emplace(
CalleeName, std::set<LineLocation>());
Candidates.first->second.insert(Loc);
@@ -2355,11 +2375,12 @@ void SampleProfileMatcher::runStaleProfileMatching(
for (const auto &IR : IRAnchors) {
const auto &Loc = IR.first;
- StringRef CalleeName = IR.second;
+ auto CalleeName = IR.second;
bool IsMatchedAnchor = false;
// Match the anchor location in lexical order.
if (!CalleeName.empty()) {
- auto CandidateAnchors = CalleeToCallsitesMap.find(CalleeName);
+ auto CandidateAnchors = CalleeToCallsitesMap.find(
+ getRepInFormat(CalleeName));
if (CandidateAnchors != CalleeToCallsitesMap.end() &&
!CandidateAnchors->second.empty()) {
auto CI = CandidateAnchors->second.begin();
@@ -2420,7 +2441,7 @@ void SampleProfileMatcher::runOnFunction(const Function &F) {
findIRAnchors(F, IRAnchors);
// Anchors for profile. It's a map from callsite location to a set of callee
// name.
- std::map<LineLocation, StringSet<>> ProfileAnchors;
+ std::map<LineLocation, std::unordered_set<FunctionId>> ProfileAnchors;
findProfileAnchors(*FSFlattened, ProfileAnchors);
// Detect profile mismatch for profile staleness metrics report.
@@ -2499,7 +2520,7 @@ void SampleProfileMatcher::runOnModule() {
void SampleProfileMatcher::distributeIRToProfileLocationMap(
FunctionSamples &FS) {
- const auto ProfileMappings = FuncMappings.find(FS.getName());
+ const auto ProfileMappings = FuncMappings.find(FS.getFuncName());
if (ProfileMappings != FuncMappings.end()) {
FS.setIRToProfileLocationMap(&(ProfileMappings->second));
}
@@ -2541,10 +2562,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
Function *F = dyn_cast<Function>(N_F.getValue());
if (F == nullptr || OrigName.empty())
continue;
- SymbolMap[OrigName] = F;
+ SymbolMap[FunctionId(OrigName)] = F;
StringRef NewName = FunctionSamples::getCanonicalFnName(*F);
if (OrigName != NewName && !NewName.empty()) {
- auto r = SymbolMap.insert(std::make_pair(NewName, F));
+ auto r = SymbolMap.emplace(FunctionId(NewName), F);
// Failiing to insert means there is already an entry in SymbolMap,
// thus there are multiple functions that are mapped to the same
// stripped name. In this case of name conflicting, set the value
@@ -2557,11 +2578,11 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
if (Remapper) {
if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
if (*MapName != OrigName && !MapName->empty())
- SymbolMap.insert(std::make_pair(*MapName, F));
+ SymbolMap.emplace(FunctionId(*MapName), F);
}
}
}
- assert(SymbolMap.count(StringRef()) == 0 &&
+ assert(SymbolMap.count(FunctionId()) == 0 &&
"No empty StringRef should be added in SymbolMap");
if (ReportProfileStaleness || PersistProfileStaleness ||
@@ -2625,7 +2646,9 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
// but not cold accumulatively...), so the outline function showing up as
// cold in sampled binary will actually not be cold after current build.
StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
- if (NamesInProfile.count(CanonName))
+ if ((FunctionSamples::UseMD5 &&
+ GUIDsInProfile.count(Function::getGUID(CanonName))) ||
+ (!FunctionSamples::UseMD5 && NamesInProfile.count(CanonName)))
initialEntryCount = -1;
}
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 18307935c7558b0..7d665a8005b0d62 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -199,6 +199,14 @@ class SymbolRemapper {
StringRef New = RemappingTable.lookup(Name);
return New.empty() ? Name : New;
}
+
+ FunctionId operator()(FunctionId Name) {
+ // MD5 name cannot be remapped.
+ if (!Name.isStringRef())
+ return Name;
+ StringRef New = RemappingTable.lookup(Name.stringRef());
+ return New.empty() ? Name : FunctionId(New);
+ }
};
}
@@ -709,14 +717,15 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
//
// Note that goo's count will remain in bar.cc:bar() as it does not have an
// entry in InstrProfile.
- DenseMap<StringRef, std::pair<uint64_t, uint64_t>> FlattenSampleMap;
+ llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
&InstrProfileMap](const FunctionSamples &FS,
const StringRef &RootName) {
auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
const StringRef &RootName,
auto &BuildImpl) -> void {
- const StringRef &Name = FS.getName();
+ std::string NameStr = FS.getFunction().str();
+ const StringRef Name = NameStr;
const StringRef *NewRootName = &RootName;
uint64_t EntrySample = FS.getHeadSamplesEstimate();
uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
@@ -770,7 +779,8 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
for (auto &PD : Reader->getProfiles()) {
sampleprof::FunctionSamples &FS = PD.second;
- BuildMaxSampleMap(FS, FS.getName());
+ std::string Name = FS.getFunction().str();
+ BuildMaxSampleMap(FS, Name);
}
ProfileSummary InstrPS = *IPBuilder.getSummary();
@@ -806,7 +816,7 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
uint64_t SampleMaxCount = std::max(E.second.first, E.second.second);
if (SampleMaxCount < ColdSampleThreshold)
continue;
- const StringRef &Name = E.first;
+ StringRef Name = E.first();
auto It = InstrProfileMap.find(Name);
if (It == InstrProfileMap.end()) {
auto NewName = StaticFuncMap.find(Name);
@@ -885,7 +895,7 @@ static sampleprof::FunctionSamples
remapSamples(const sampleprof::FunctionSamples &Samples,
SymbolRemapper &Remapper, sampleprof_error &Error) {
sampleprof::FunctionSamples Result;
- Result.setName(Remapper(Samples.getName()));
+ Result.setFunction(Remapper(Samples.getFunction()));
Result.addTotalSamples(Samples.getTotalSamples());
Result.addHeadSamples(Samples.getHeadSamples());
for (const auto &BodySample : Samples.getBodySamples()) {
@@ -896,7 +906,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
for (const auto &Target : BodySample.second.getCallTargets()) {
Result.addCalledTargetSamples(BodySample.first.LineOffset,
MaskedDiscriminator,
- Remapper(Target.first()), Target.second);
+ Remapper(Target.first), Target.second);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
@@ -905,8 +915,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
for (const auto &Callsite : CallsiteSamples.second) {
sampleprof::FunctionSamples Remapped =
remapSamples(Callsite.second, Remapper, Error);
- MergeResult(Error,
- Target[std::string(Remapped.getName())].merge(Remapped));
+ MergeResult(Error, Target[Remapped.getFunction()].merge(Remapped));
}
}
return Result;
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index e891ea8df4907f1..025d3ca5a6da5ad 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -74,8 +74,8 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
ProfileInlineLimitMax = 50000;
}
-std::vector<StringRef> CSPreInliner::buildTopDownOrder() {
- std::vector<StringRef> Order;
+std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
+ std::vector<FunctionId> Order;
// Trim cold edges to get a more stable call graph. This allows for a more
// stable top-down order which in turns helps the stablity of the generated
// profile from run to run.
@@ -129,7 +129,7 @@ bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
LineLocation Callsite = CalleeNode->getCallSiteLoc();
if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
- auto It = TargetCounts.find(CalleeSamples->getName());
+ auto It = TargetCounts.find(CalleeSamples->getFunction());
if (It != TargetCounts.end())
CallsiteCount = It->second;
}
@@ -196,7 +196,7 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
return (Candidate.SizeCost < SampleThreshold);
}
-void CSPreInliner::processFunction(const StringRef Name) {
+void CSPreInliner::processFunction(const FunctionId Name) {
FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name);
if (!FSamples)
return;
@@ -297,7 +297,7 @@ void CSPreInliner::run() {
// It also helps better compress context profile to control profile
// size, as we now only need context profile for functions going to
// be inlined.
- for (StringRef FuncName : buildTopDownOrder()) {
+ for (FunctionId FuncName : buildTopDownOrder()) {
processFunction(FuncName);
}
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 4d848aafdab914d..8a3f16a4f13cb8d 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -57,8 +57,7 @@ struct ProfiledCandidateComparer {
// Tie breaker using GUID so we have stable/deterministic inlining order
assert(LHS.CalleeSamples && RHS.CalleeSamples &&
"Expect non-null FunctionSamples");
- return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) <
- RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName());
+ return LHS.CalleeSamples->getGUID() < RHS.CalleeSamples->getGUID();
}
};
@@ -81,8 +80,8 @@ class CSPreInliner {
private:
bool getInlineCandidates(ProfiledCandidateQueue &CQueue,
const FunctionSamples *FCallerContextSamples);
- std::vector<StringRef> buildTopDownOrder();
- void processFunction(StringRef Name);
+ std::vector<FunctionId> buildTopDownOrder();
+ void processFunction(FunctionId Name);
bool shouldInline(ProfiledInlineCandidate &Candidate);
uint32_t getFuncSize(const ContextTrieNode *ContextNode);
bool UseContextCost;
diff --git a/llvm/tools/llvm-profgen/CallContext.h b/llvm/tools/llvm-profgen/CallContext.h
index 5e552130d03c757..ce9423f118dee98 100644
--- a/llvm/tools/llvm-profgen/CallContext.h
+++ b/llvm/tools/llvm-profgen/CallContext.h
@@ -18,7 +18,7 @@ namespace llvm {
namespace sampleprof {
inline std::string getCallSite(const SampleContextFrame &Callsite) {
- std::string CallsiteStr = Callsite.FuncName.str();
+ std::string CallsiteStr = Callsite.Func.str();
CallsiteStr += ":";
CallsiteStr += Twine(Callsite.Location.LineOffset).str();
if (Callsite.Location.Discriminator > 0) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index d307ab465676296..c4028e6b132871b 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -208,7 +208,7 @@ double ProfileGeneratorBase::calculateDensity(const SampleProfileMap &Profiles,
}
for (auto *FuncSamples : HotFuncs) {
- auto *Func = Binary->getBinaryFunction(FuncSamples->getName());
+ auto *Func = Binary->getBinaryFunction(FuncSamples->getFunction());
if (!Func)
continue;
uint64_t FuncSize = Func->getFuncSize();
@@ -449,7 +449,7 @@ bool ProfileGeneratorBase::collectFunctionsFromRawProfile(
bool ProfileGenerator::collectFunctionsFromLLVMProfile(
std::unordered_set<const BinaryFunction *> &ProfiledFunctions) {
for (const auto &FS : ProfileMap) {
- if (auto *Func = Binary->getBinaryFunction(FS.second.getName()))
+ if (auto *Func = Binary->getBinaryFunction(FS.second.getFunction()))
ProfiledFunctions.insert(Func);
}
return true;
@@ -466,7 +466,7 @@ bool CSProfileGenerator::collectFunctionsFromLLVMProfile(
}
FunctionSamples &
-ProfileGenerator::getTopLevelFunctionProfile(StringRef FuncName) {
+ProfileGenerator::getTopLevelFunctionProfile(FunctionId FuncName) {
SampleContext Context(FuncName);
return ProfileMap.Create(Context);
}
@@ -586,7 +586,7 @@ void ProfileGenerator::populateBoundarySamplesWithProbesForAllFunctions(
FunctionProfile.addCalledTargetSamples(
FrameVec.back().Location.LineOffset,
FrameVec.back().Location.Discriminator,
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
}
}
@@ -595,11 +595,11 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
const SampleContextFrameVector &FrameVec, uint64_t Count) {
// Get top level profile
FunctionSamples *FunctionProfile =
- &getTopLevelFunctionProfile(FrameVec[0].FuncName);
+ &getTopLevelFunctionProfile(FrameVec[0].Func);
FunctionProfile->addTotalSamples(Count);
if (Binary->usePseudoProbes()) {
const auto *FuncDesc = Binary->getFuncDescForGUID(
- Function::getGUID(FunctionProfile->getName()));
+ FunctionProfile->getFunction().getHashCode());
FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
}
@@ -610,16 +610,16 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
FunctionSamplesMap &SamplesMap =
FunctionProfile->functionSamplesAt(Callsite);
auto Ret =
- SamplesMap.emplace(FrameVec[I].FuncName.str(), FunctionSamples());
+ SamplesMap.emplace(FrameVec[I].Func, FunctionSamples());
if (Ret.second) {
- SampleContext Context(FrameVec[I].FuncName);
+ SampleContext Context(FrameVec[I].Func);
Ret.first->second.setContext(Context);
}
FunctionProfile = &Ret.first->second;
FunctionProfile->addTotalSamples(Count);
if (Binary->usePseudoProbes()) {
const auto *FuncDesc = Binary->getFuncDescForGUID(
- Function::getGUID(FunctionProfile->getName()));
+ FunctionProfile->getFunction().getHashCode());
FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
}
}
@@ -716,10 +716,11 @@ void ProfileGenerator::populateBoundarySamplesForAllFunctions(
FunctionProfile.addCalledTargetSamples(
FrameVec.back().Location.LineOffset,
getBaseDiscriminator(FrameVec.back().Location.Discriminator),
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
// Add head samples for callee.
- FunctionSamples &CalleeProfile = getTopLevelFunctionProfile(CalleeName);
+ FunctionSamples &CalleeProfile =
+ getTopLevelFunctionProfile(FunctionId(CalleeName));
CalleeProfile.addHeadSamples(Count);
}
}
@@ -737,7 +738,7 @@ CSProfileGenerator::getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
if (!FProfile) {
FSamplesList.emplace_back();
FProfile = &FSamplesList.back();
- FProfile->setName(ContextNode->getFuncName());
+ FProfile->setFunction(ContextNode->getFuncName());
ContextNode->setFunctionSamples(FProfile);
}
// Update ContextWasInlined attribute for existing contexts.
@@ -899,7 +900,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
if (LeafLoc) {
CallerNode->getFunctionSamples()->addCalledTargetSamples(
LeafLoc->Location.LineOffset,
- getBaseDiscriminator(LeafLoc->Location.Discriminator), CalleeName,
+ getBaseDiscriminator(LeafLoc->Location.Discriminator),
+ FunctionId(CalleeName),
Count);
// Record head sample for called target(callee)
CalleeCallSite = LeafLoc->Location;
@@ -907,7 +909,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
}
ContextTrieNode *CalleeNode =
- CallerNode->getOrCreateChildContext(CalleeCallSite, CalleeName);
+ CallerNode->getOrCreateChildContext(CalleeCallSite,
+ FunctionId(CalleeName));
FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode);
CalleeProfile->addHeadSamples(Count);
}
@@ -1212,7 +1215,7 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes(
continue;
FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(),
CallProbe->getDiscriminator(),
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
}
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 471792ec713cd53..88cf2dc0d49f373 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -157,7 +157,7 @@ class ProfileGenerator : public ProfileGeneratorBase {
void generateLineNumBasedProfile();
void generateProbeBasedProfile();
RangeSample preprocessRangeCounter(const RangeSample &RangeCounter);
- FunctionSamples &getTopLevelFunctionProfile(StringRef FuncName);
+ FunctionSamples &getTopLevelFunctionProfile(FunctionId FuncName);
// Helper function to get the leaf frame's FunctionProfile by traversing the
// inline stack and meanwhile it adds the total samples for each frame's
// function profile.
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 85d7c1123fecaf6..6db25b5541b4581 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -77,7 +77,7 @@ void BinarySizeContextTracker::addInstructionForContext(
ContextTrieNode *CurNode = &RootContext;
bool IsLeaf = true;
for (const auto &Callsite : reverse(Context)) {
- StringRef CallerName = Callsite.FuncName;
+ FunctionId CallerName = Callsite.Func;
LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
CurNode = CurNode->getOrCreateChildContext(CallsiteLoc, CallerName);
IsLeaf = false;
@@ -145,7 +145,8 @@ void BinarySizeContextTracker::trackInlineesOptimizedAway(
StringRef CallerName = ProbeFrame.first;
LineLocation CallsiteLoc(ProbeFrame.second, 0);
SizeContext =
- SizeContext->getOrCreateChildContext(CallsiteLoc, CallerName);
+ SizeContext->getOrCreateChildContext(CallsiteLoc,
+ FunctionId(CallerName));
}
// Add 0 size to make known.
SizeContext->addFunctionSize(0);
@@ -838,6 +839,14 @@ void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
if (BinaryFunctions.empty())
WithColor::warning() << "Loading of DWARF info completed, but no binary "
"functions have been retrieved.\n";
+
+
+ // Populate the hash binary function map for MD5 function name lookup. This
+ // is done after BinaryFunctions are finalized.
+ for (auto &BinaryFunction : BinaryFunctions) {
+ HashBinaryFunctions[MD5Hash(StringRef(BinaryFunction.first))] =
+ &BinaryFunction.second;
+ }
}
void ProfiledBinary::populateSymbolListFromDWARF(
@@ -889,7 +898,7 @@ SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
LineLocation Line(LineOffset, Discriminator);
auto It = NameStrings.insert(FunctionName.str());
- CallStack.emplace_back(*It.first, Line);
+ CallStack.emplace_back(FunctionId(StringRef(*It.first)), Line);
}
return CallStack;
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index a6d78c661cc1c31..8c607d665dee81e 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -220,6 +220,10 @@ class ProfiledBinary {
// A map of mapping function name to BinaryFunction info.
std::unordered_map<std::string, BinaryFunction> BinaryFunctions;
+ // Lookup BinaryFunctions using the function name's MD5 hash. Needed if the
+ // profile is using MD5.
+ std::unordered_map<uint64_t, BinaryFunction *> HashBinaryFunctions;
+
// A list of binary functions that have samples.
std::unordered_set<const BinaryFunction *> ProfiledFunctions;
@@ -476,12 +480,18 @@ class ProfiledBinary {
void setProfiledFunctions(std::unordered_set<const BinaryFunction *> &Funcs) {
ProfiledFunctions = Funcs;
}
-
- BinaryFunction *getBinaryFunction(StringRef FName) {
- auto I = BinaryFunctions.find(FName.str());
- if (I == BinaryFunctions.end())
+
+ BinaryFunction *getBinaryFunction(FunctionId FName) {
+ if (FName.isStringRef()) {
+ auto I = BinaryFunctions.find(FName.str());
+ if (I == BinaryFunctions.end())
+ return nullptr;
+ return &I->second;
+ }
+ auto I = HashBinaryFunctions.find(FName.getHashCode());
+ if (I == HashBinaryFunctions.end())
return nullptr;
- return &I->second;
+ return I->second;
}
uint32_t getFuncSizeForContext(const ContextTrieNode *ContextNode) {
@@ -556,7 +566,7 @@ class ProfiledBinary {
InlineContextStack.clear();
continue;
}
- InlineContextStack.emplace_back(Callsite.first,
+ InlineContextStack.emplace_back(FunctionId(Callsite.first),
LineLocation(Callsite.second, 0));
}
}
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index e01025440e4a56a..3abba4743093258 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -145,7 +145,7 @@ struct SampleProfTest : ::testing::Test {
StringRef FooName("_Z3fooi");
FunctionSamples FooSamples;
- FooSamples.setName(FooName);
+ FooSamples.setFunction(FunctionId(FooName));
FooSamples.addTotalSamples(7711);
FooSamples.addHeadSamples(610);
FooSamples.addBodySamples(1, 0, 610);
@@ -155,43 +155,43 @@ struct SampleProfTest : ::testing::Test {
FooSamples.addBodySamples(10, 0, 605);
// Add inline instance with name "_Z3gooi".
- StringRef GooName("_Z3gooi");
+ FunctionId GooName(StringRef("_Z3gooi"));
auto &GooSamples =
- FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName.str()];
- GooSamples.setName(GooName);
+ FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName];
+ GooSamples.setFunction(GooName);
GooSamples.addTotalSamples(502);
GooSamples.addBodySamples(3, 0, 502);
// Add inline instance with name "_Z3hooi".
- StringRef HooName("_Z3hooi");
+ FunctionId HooName(StringRef("_Z3hooi"));
auto &HooSamples =
- GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName.str()];
- HooSamples.setName(HooName);
+ GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName];
+ HooSamples.setFunction(HooName);
HooSamples.addTotalSamples(317);
HooSamples.addBodySamples(4, 0, 317);
StringRef BarName("_Z3bari");
FunctionSamples BarSamples;
- BarSamples.setName(BarName);
+ BarSamples.setFunction(FunctionId(BarName));
BarSamples.addTotalSamples(20301);
BarSamples.addHeadSamples(1437);
BarSamples.addBodySamples(1, 0, 1437);
// Test how reader/writer handles unmangled names.
StringRef MconstructName("_M_construct<char *>");
StringRef StringviewName("string_view<std::allocator<char> >");
- BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000);
- BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437);
+ BarSamples.addCalledTargetSamples(1, 0, FunctionId(MconstructName), 1000);
+ BarSamples.addCalledTargetSamples(1, 0, FunctionId(StringviewName), 437);
StringRef BazName("_Z3bazi");
FunctionSamples BazSamples;
- BazSamples.setName(BazName);
+ BazSamples.setFunction(FunctionId(BazName));
BazSamples.addTotalSamples(12557);
BazSamples.addHeadSamples(1257);
BazSamples.addBodySamples(1, 0, 12557);
StringRef BooName("_Z3booi");
FunctionSamples BooSamples;
- BooSamples.setName(BooName);
+ BooSamples.setFunction(FunctionId(BooName));
BooSamples.addTotalSamples(1232);
BooSamples.addHeadSamples(1);
BooSamples.addBodySamples(1, 0, 1232);
@@ -210,8 +210,8 @@ struct SampleProfTest : ::testing::Test {
if (Remap) {
FooName = "_Z4fauxi";
BarName = "_Z3barl";
- GooName = "_Z3gool";
- HooName = "_Z3hool";
+ GooName = FunctionId(StringRef("_Z3gool"));
+ HooName = FunctionId(StringRef("_Z3hool"));
}
M.getOrInsertFunction(FooName, fn_type);
@@ -245,7 +245,7 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
ASSERT_TRUE(ReadFooSamples != nullptr);
if (!UseMD5) {
- ASSERT_EQ("_Z3fooi", ReadFooSamples->getName());
+ ASSERT_EQ("_Z3fooi", ReadFooSamples->getFunction().str());
}
ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples());
ASSERT_EQ(610u, ReadFooSamples->getHeadSamples());
@@ -254,7 +254,8 @@ struct SampleProfTest : ::testing::Test {
// inline instance for GooName. Test the correct FunctionSamples can be
// found with Remapper support.
const FunctionSamples *ReadGooSamples =
- ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0), GooName,
+ ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0),
+ GooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadGooSamples != nullptr);
ASSERT_EQ(502u, ReadGooSamples->getTotalSamples());
@@ -263,7 +264,8 @@ struct SampleProfTest : ::testing::Test {
// no inline instance for GooName. Test no FunctionSamples will be
// found with Remapper support.
const FunctionSamples *ReadGooSamplesAgain =
- ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0), GooName,
+ ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+ GooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadGooSamplesAgain == nullptr);
@@ -272,7 +274,8 @@ struct SampleProfTest : ::testing::Test {
// inline instance for HooName. Test the correct FunctionSamples can be
// found with Remapper support.
const FunctionSamples *ReadHooSamples =
- ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0), HooName,
+ ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+ HooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadHooSamples != nullptr);
ASSERT_EQ(317u, ReadHooSamples->getTotalSamples());
@@ -280,7 +283,7 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName);
ASSERT_TRUE(ReadBarSamples != nullptr);
if (!UseMD5) {
- ASSERT_EQ("_Z3bari", ReadBarSamples->getName());
+ ASSERT_EQ("_Z3bari", ReadBarSamples->getFunction().str());
}
ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples());
ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples());
@@ -304,13 +307,9 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadBooSamples = Reader->getSamplesFor(BooName);
ASSERT_TRUE(ReadBooSamples != nullptr);
ASSERT_EQ(1232u, ReadBooSamples->getTotalSamples());
-
- std::string MconstructGUID;
- StringRef MconstructRep =
- getRepInFormat(MconstructName, UseMD5, MconstructGUID);
- std::string StringviewGUID;
- StringRef StringviewRep =
- getRepInFormat(StringviewName, UseMD5, StringviewGUID);
+
+ FunctionId MconstructRep = getRepInFormat(MconstructName);
+ FunctionId StringviewRep = getRepInFormat(StringviewName);
ASSERT_EQ(1000u, CTMap.get()[MconstructRep]);
ASSERT_EQ(437u, CTMap.get()[StringviewRep]);
@@ -333,7 +332,7 @@ struct SampleProfTest : ::testing::Test {
uint64_t TotalSamples, uint64_t HeadSamples) {
StringRef Name(Fname);
FunctionSamples FcnSamples;
- FcnSamples.setName(Name);
+ FcnSamples.setFunction(FunctionId(Name));
FcnSamples.addTotalSamples(TotalSamples);
FcnSamples.addHeadSamples(HeadSamples);
FcnSamples.addBodySamples(1, 0, HeadSamples);
More information about the llvm-commits
mailing list