[llvm] [llvm-profdata] Do not create numerical strings for MD5 function names read from a Sample Profile. (PR #66164)
William Junda Huang via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 17 12:49:38 PDT 2023
https://github.com/huangjd updated https://github.com/llvm/llvm-project/pull/66164
>From 92dac9f101a8d90e79d71df2f83bb47334972619 Mon Sep 17 00:00:00 2001
From: William Huang <williamjhuang at google.com>
Date: Tue, 17 Oct 2023 09:03:48 +0000
Subject: [PATCH] Merge with main
---
llvm/include/llvm/ProfileData/FunctionId.h | 213 +++++++++++++
llvm/include/llvm/ProfileData/HashKeyMap.h | 129 ++++++++
llvm/include/llvm/ProfileData/SampleProf.h | 283 +++++++-----------
.../llvm/ProfileData/SampleProfReader.h | 30 +-
.../llvm/ProfileData/SampleProfWriter.h | 16 +-
.../llvm/Transforms/IPO/ProfiledCallGraph.h | 33 +-
.../Transforms/IPO/SampleContextTracker.h | 29 +-
llvm/lib/ProfileData/SampleProf.cpp | 42 +--
llvm/lib/ProfileData/SampleProfReader.cpp | 98 +++---
llvm/lib/ProfileData/SampleProfWriter.cpp | 45 +--
llvm/lib/Target/X86/X86InsertPrefetch.cpp | 7 +-
.../Transforms/IPO/SampleContextTracker.cpp | 67 ++---
llvm/lib/Transforms/IPO/SampleProfile.cpp | 149 +++++----
llvm/tools/llvm-profdata/llvm-profdata.cpp | 25 +-
llvm/tools/llvm-profgen/CSPreInliner.cpp | 10 +-
llvm/tools/llvm-profgen/CSPreInliner.h | 7 +-
llvm/tools/llvm-profgen/CallContext.h | 2 +-
llvm/tools/llvm-profgen/ProfileGenerator.cpp | 33 +-
llvm/tools/llvm-profgen/ProfileGenerator.h | 2 +-
llvm/tools/llvm-profgen/ProfiledBinary.cpp | 15 +-
llvm/tools/llvm-profgen/ProfiledBinary.h | 22 +-
llvm/unittests/ProfileData/SampleProfTest.cpp | 53 ++--
22 files changed, 797 insertions(+), 513 deletions(-)
create mode 100644 llvm/include/llvm/ProfileData/FunctionId.h
create mode 100644 llvm/include/llvm/ProfileData/HashKeyMap.h
diff --git a/llvm/include/llvm/ProfileData/FunctionId.h b/llvm/include/llvm/ProfileData/FunctionId.h
new file mode 100644
index 000000000000000..0076cdc090459f4
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/FunctionId.h
@@ -0,0 +1,213 @@
+//===--- FunctionId.h - Sample profile function object ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines FunctionId class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_FUNCTIONID_H
+#define LLVM_PROFILEDATA_FUNCTIONID_H
+
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/MD5.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstdint>
+
+namespace llvm {
+namespace sampleprof {
+
+/// This class represents a function that is read from a sample profile. It
+/// comes with two forms: a string or a hash code. The latter form is the 64-bit
+/// MD5 of the function name for efficient storage supported by ExtBinary
+/// profile format, and when reading the profile, this class can represent it
+/// without converting it to a string first.
+/// When representing a hash code, we utilize the LengthOrHashCode field to
+/// store it, and Name is set to null. When representing a string, it is same as
+/// StringRef.
+class FunctionId {
+
+ const char *Data = nullptr;
+
+ // Use uint64_t instead of size_t so that it can also hold a MD5 value on
+ // 32-bit system.
+ uint64_t LengthOrHashCode = 0;
+
+ /// Extension to memcmp to handle hash code representation. If both are hash
+ /// values, Lhs and Rhs are both null, function returns 0 (and needs an extra
+ /// comparison using getIntValue). If only one is hash code, it is considered
+ /// less than the StringRef one. Otherwise perform normal string comparison.
+ static int compareMemory(const char *Lhs, const char *Rhs, uint64_t Length) {
+ if (Lhs == Rhs)
+ return 0;
+ if (!Lhs)
+ return -1;
+ if (!Rhs)
+ return 1;
+ return ::memcmp(Lhs, Rhs, (size_t)Length);
+ }
+
+public:
+ FunctionId() = default;
+
+ /// Constructor from a StringRef.
+ explicit FunctionId(StringRef Str)
+ : Data(Str.data()), LengthOrHashCode(Str.size()) {
+ }
+
+ /// Constructor from a hash code.
+ explicit FunctionId(uint64_t HashCode)
+ : LengthOrHashCode(HashCode) {
+ assert(HashCode != 0);
+ }
+
+ /// Check for equality. Similar to StringRef::equals, but will also cover for
+ /// the case where one or both are hash codes. Comparing their int values are
+ /// sufficient. A hash code FunctionId is considered not equal to a StringRef
+ /// FunctionId regardless of actual contents.
+ bool equals(const FunctionId &Other) const {
+ return LengthOrHashCode == Other.LengthOrHashCode &&
+ compareMemory(Data, Other.Data, LengthOrHashCode) == 0;
+ }
+
+ /// Total order comparison. If both FunctionId are StringRef, this is the same
+ /// as StringRef::compare. If one of them is StringRef, it is considered
+ /// greater than the hash code FunctionId. Otherwise this is the the same
+ /// as comparing their int values.
+ int compare(const FunctionId &Other) const {
+ auto Res = compareMemory(
+ Data, Other.Data, std::min(LengthOrHashCode, Other.LengthOrHashCode));
+ if (Res != 0)
+ return Res;
+ if (LengthOrHashCode == Other.LengthOrHashCode)
+ return 0;
+ return LengthOrHashCode < Other.LengthOrHashCode ? -1 : 1;
+ }
+
+ /// Convert to a string, usually for output purpose. Use caution on return
+ /// value's lifetime when converting to StringRef.
+ std::string str() const {
+ if (Data)
+ return std::string(Data, LengthOrHashCode);
+ if (LengthOrHashCode != 0)
+ return std::to_string(LengthOrHashCode);
+ return std::string();
+ }
+
+ /// Convert to StringRef. This is only allowed when it is known this object is
+ /// representing a StringRef, not a hash code. Calling this function on a hash
+ /// code is considered an error.
+ StringRef stringRef() const {
+ if (Data)
+ return StringRef(Data, LengthOrHashCode);
+ assert(LengthOrHashCode == 0 &&
+ "Cannot convert MD5 FunctionId to StringRef");
+ return StringRef();
+ }
+
+ friend raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj);
+
+ /// Get hash code of this object. Returns this object's hash code if it is
+ /// already representing one, otherwise returns the MD5 of its string content.
+ /// Note that it is not the same as std::hash because we want to keep the
+ /// consistency that the same sample profile function in string form or MD5
+ /// form has the same hash code.
+ uint64_t getHashCode() const {
+ if (Data)
+ return MD5Hash(StringRef(Data, LengthOrHashCode));
+ return LengthOrHashCode;
+ }
+
+ bool empty() const { return LengthOrHashCode == 0; }
+
+ /// Check if this object represents a StringRef, or a hash code.
+ bool isStringRef() const { return Data != nullptr; }
+};
+
+inline bool operator==(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.equals(RHS);
+}
+
+inline bool operator!=(const FunctionId &LHS, const FunctionId &RHS) {
+ return !LHS.equals(RHS);
+}
+
+inline bool operator<(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) < 0;
+}
+
+inline bool operator<=(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) <= 0;
+}
+
+inline bool operator>(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) > 0;
+}
+
+inline bool operator>=(const FunctionId &LHS, const FunctionId &RHS) {
+ return LHS.compare(RHS) >= 0;
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, const FunctionId &Obj) {
+ if (Obj.Data)
+ return OS << StringRef(Obj.Data, Obj.LengthOrHashCode);
+ if (Obj.LengthOrHashCode != 0)
+ return OS << Obj.LengthOrHashCode;
+ return OS;
+}
+
+inline uint64_t MD5Hash(const FunctionId &Obj) {
+ return Obj.getHashCode();
+}
+
+inline uint64_t hash_value(const FunctionId &Obj) {
+ return Obj.getHashCode();
+}
+
+} // end namespace sampleprof
+
+/// Template specialization for FunctionId so that it can be used in LLVM map
+/// containers.
+template <> struct DenseMapInfo<sampleprof::FunctionId, void> {
+
+ static inline sampleprof::FunctionId getEmptyKey() {
+ return sampleprof::FunctionId(~0ULL);
+ }
+
+ static inline sampleprof::FunctionId getTombstoneKey() {
+ return sampleprof::FunctionId(~1ULL);
+ }
+
+ static unsigned getHashValue(const sampleprof::FunctionId &Val) {
+ return Val.getHashCode();
+ }
+
+ static bool isEqual(const sampleprof::FunctionId &LHS,
+ const sampleprof::FunctionId &RHS) {
+ return LHS == RHS;
+ }
+};
+
+} // end namespace llvm
+
+namespace std {
+
+/// Template specialization for FunctionId so that it can be used in STL
+/// containers.
+template <> struct hash<llvm::sampleprof::FunctionId> {
+ size_t operator()(const llvm::sampleprof::FunctionId &Val) const {
+ return Val.getHashCode();
+ }
+};
+
+} // end namespace std
+
+#endif // LLVM_PROFILEDATA_FUNCTIONID_H
diff --git a/llvm/include/llvm/ProfileData/HashKeyMap.h b/llvm/include/llvm/ProfileData/HashKeyMap.h
new file mode 100644
index 000000000000000..b2f1bf222157b7b
--- /dev/null
+++ b/llvm/include/llvm/ProfileData/HashKeyMap.h
@@ -0,0 +1,129 @@
+//===--- HashKeyMap.h - Wrapper for maps using hash value key ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+///
+/// Defines HashKeyMap template.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_HASHKEYMAP_H
+#define LLVM_PROFILEDATA_HASHKEYMAP_H
+
+#include "llvm/ADT/Hashing.h"
+#include <iterator>
+#include <utility>
+
+namespace llvm {
+
+namespace sampleprof {
+
+/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
+/// the hash value of the original key as the new key. This greatly improves the
+/// performance of insert and query operations especially when hash values of
+/// keys are available a priori, and reduces memory usage if KeyT has a large
+/// size.
+/// All keys with the same hash value are considered equivalent (i.e. hash
+/// collision is silently ignored). Given such feature this class should only be
+/// used where it does not affect compilation correctness, for example, when
+/// loading a sample profile. The original key is not stored, so if the user
+/// needs to preserve it, it should be stored in the mapped type.
+/// Assuming the hashing algorithm is uniform, we use the formula
+/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
+/// elements chosen at random to calculate the probability of collision. With
+/// 1,000,000 entries the probability is negligible:
+/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
+/// Source: https://en.wikipedia.org/wiki/Birthday_problem
+///
+/// \param MapT The underlying associative container type.
+/// \param KeyT The original key type, which requires the implementation of
+/// llvm::hash_value(KeyT).
+/// \param ValueT The original mapped type, which has the same requirement as
+/// the underlying container.
+/// \param MapTArgs Additional template parameters passed to the underlying
+/// container.
+template <template <typename, typename, typename...> typename MapT,
+ typename KeyT, typename ValueT, typename... MapTArgs>
+class HashKeyMap :
+ public MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...> {
+public:
+ using base_type = MapT<decltype(hash_value(KeyT())), ValueT, MapTArgs...>;
+ using key_type = decltype(hash_value(KeyT()));
+ using original_key_type = KeyT;
+ using mapped_type = ValueT;
+ using value_type = typename base_type::value_type;
+
+ using iterator = typename base_type::iterator;
+ using const_iterator = typename base_type::const_iterator;
+
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(const key_type &Hash,
+ const original_key_type &Key,
+ Ts &&...Args) {
+ assert(Hash == hash_value(Key));
+ return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
+ }
+
+ template <typename... Ts>
+ std::pair<iterator, bool> try_emplace(const original_key_type &Key,
+ Ts &&...Args) {
+ return try_emplace(hash_value(Key), Key, std::forward<Ts>(Args)...);
+ }
+
+ template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
+ return try_emplace(std::forward<Ts>(Args)...);
+ }
+
+ mapped_type &operator[](const original_key_type &Key) {
+ return try_emplace(Key, mapped_type()).first->second;
+ }
+
+ iterator find(const original_key_type &Key) {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It;
+ return base_type::end();
+ }
+
+ const_iterator find(const original_key_type &Key) const {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It;
+ return base_type::end();
+ }
+
+ mapped_type lookup(const original_key_type &Key) const {
+ auto It = base_type::find(hash_value(Key));
+ if (It != base_type::end())
+ return It->second;
+ return mapped_type();
+ }
+
+ size_t count(const original_key_type &Key) const {
+ return base_type::count(hash_value(Key));
+ }
+
+ size_t erase(const original_key_type &Ctx) {
+ auto It = find(Ctx);
+ if (It != base_type::end()) {
+ base_type::erase(It);
+ return 1;
+ }
+ return 0;
+ }
+
+ iterator erase(const_iterator It) {
+ return base_type::erase(It);
+ }
+};
+
+}
+
+}
+
+#endif // LLVM_PROFILEDATA_HASHKEYMAP_H
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 78a16499accd64a..57ea144532a3cb1 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -21,10 +21,12 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/ProfileData/FunctionId.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/ProfileData/HashKeyMap.h"
#include <algorithm>
#include <cstdint>
#include <list>
@@ -109,16 +111,6 @@ static inline uint64_t SPMagic(SampleProfileFormat Format = SPF_Binary) {
uint64_t('2') << (64 - 56) | uint64_t(Format);
}
-/// Get the proper representation of a string according to whether the
-/// current Format uses MD5 to represent the string.
-static inline StringRef getRepInFormat(StringRef Name, bool UseMD5,
- std::string &GUIDBuf) {
- if (Name.empty() || !UseMD5)
- return Name;
- GUIDBuf = std::to_string(Function::getGUID(Name));
- return GUIDBuf;
-}
-
static inline uint64_t SPVersion() { return 103; }
// Section Type used by SampleProfileExtBinaryBaseReader and
@@ -305,27 +297,22 @@ struct LineLocation {
return LineOffset != O.LineOffset || Discriminator != O.Discriminator;
}
+ uint64_t getHashCode() const {
+ return ((uint64_t) Discriminator << 32) | LineOffset;
+ }
+
uint32_t LineOffset;
uint32_t Discriminator;
};
struct LineLocationHash {
uint64_t operator()(const LineLocation &Loc) const {
- return std::hash<std::uint64_t>{}((((uint64_t)Loc.LineOffset) << 32) |
- Loc.Discriminator);
+ return Loc.getHashCode();
}
};
raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc);
-static inline uint64_t hashFuncName(StringRef F) {
- // If function name is already MD5 string, do not hash again.
- uint64_t Hash;
- if (F.getAsInteger(10, Hash))
- Hash = MD5Hash(F);
- return Hash;
-}
-
/// Representation of a single sample record.
///
/// A sample record is represented by a positive integer value, which
@@ -338,7 +325,7 @@ static inline uint64_t hashFuncName(StringRef F) {
/// will be a list of one or more functions.
class SampleRecord {
public:
- using CallTarget = std::pair<StringRef, uint64_t>;
+ using CallTarget = std::pair<FunctionId, uint64_t>;
struct CallTargetComparator {
bool operator()(const CallTarget &LHS, const CallTarget &RHS) const {
if (LHS.second != RHS.second)
@@ -349,7 +336,7 @@ class SampleRecord {
};
using SortedCallTargetSet = std::set<CallTarget, CallTargetComparator>;
- using CallTargetMap = StringMap<uint64_t>;
+ using CallTargetMap = std::unordered_map<FunctionId, uint64_t>;
SampleRecord() = default;
/// Increment the number of samples for this record by \p S.
@@ -378,7 +365,7 @@ class SampleRecord {
///
/// Sample counts accumulate using saturating arithmetic, to avoid wrapping
/// around unsigned integers.
- sampleprof_error addCalledTarget(StringRef F, uint64_t S,
+ sampleprof_error addCalledTarget(FunctionId F, uint64_t S,
uint64_t Weight = 1) {
uint64_t &TargetSamples = CallTargets[F];
bool Overflowed;
@@ -390,7 +377,7 @@ class SampleRecord {
/// Remove called function from the call target map. Return the target sample
/// count of the called function.
- uint64_t removeCalledTarget(StringRef F) {
+ uint64_t removeCalledTarget(FunctionId F) {
uint64_t Count = 0;
auto I = CallTargets.find(F);
if (I != CallTargets.end()) {
@@ -474,18 +461,18 @@ enum ContextAttributeMask {
0x4, // Leaf of context is duplicated into the base profile
};
-// Represents a context frame with function name and line location
+// Represents a context frame with profile function and line location
struct SampleContextFrame {
- StringRef FuncName;
+ FunctionId Func;
LineLocation Location;
SampleContextFrame() : Location(0, 0) {}
-
- SampleContextFrame(StringRef FuncName, LineLocation Location)
- : FuncName(FuncName), Location(Location) {}
+
+ SampleContextFrame(FunctionId Func, LineLocation Location)
+ : Func(Func), Location(Location) {}
bool operator==(const SampleContextFrame &That) const {
- return Location == That.Location && FuncName == That.FuncName;
+ return Location == That.Location && Func == That.Func;
}
bool operator!=(const SampleContextFrame &That) const {
@@ -494,7 +481,7 @@ struct SampleContextFrame {
std::string toString(bool OutputLineLocation) const {
std::ostringstream OContextStr;
- OContextStr << FuncName.str();
+ OContextStr << Func.str();
if (OutputLineLocation) {
OContextStr << ":" << Location.LineOffset;
if (Location.Discriminator)
@@ -502,11 +489,16 @@ struct SampleContextFrame {
}
return OContextStr.str();
}
+
+ uint64_t getHashCode() const {
+ uint64_t NameHash = Func.getHashCode();
+ uint64_t LocId = Location.getHashCode();
+ return NameHash + (LocId << 5) + LocId;
+ }
};
static inline hash_code hash_value(const SampleContextFrame &arg) {
- return hash_combine(arg.FuncName, arg.Location.LineOffset,
- arg.Location.Discriminator);
+ return arg.getHashCode();
}
using SampleContextFrameVector = SmallVector<SampleContextFrame, 1>;
@@ -533,9 +525,12 @@ class SampleContext {
SampleContext() : State(UnknownContext), Attributes(ContextNone) {}
SampleContext(StringRef Name)
- : Name(Name), State(UnknownContext), Attributes(ContextNone) {
+ : Func(Name), State(UnknownContext), Attributes(ContextNone) {
assert(!Name.empty() && "Name is empty");
}
+
+ SampleContext(FunctionId Func)
+ : Func(Func), State(UnknownContext), Attributes(ContextNone) {}
SampleContext(SampleContextFrames Context,
ContextStateMask CState = RawContext)
@@ -557,7 +552,7 @@ class SampleContext {
bool HasContext = ContextStr.startswith("[");
if (!HasContext) {
State = UnknownContext;
- Name = ContextStr;
+ Func = FunctionId(ContextStr);
} else {
CSNameTable.emplace_back();
SampleContextFrameVector &Context = CSNameTable.back();
@@ -574,24 +569,25 @@ class SampleContext {
ContextStr = ContextStr.substr(1, ContextStr.size() - 2);
StringRef ContextRemain = ContextStr;
StringRef ChildContext;
- StringRef CalleeName;
+ FunctionId Callee;
while (!ContextRemain.empty()) {
auto ContextSplit = ContextRemain.split(" @ ");
ChildContext = ContextSplit.first;
ContextRemain = ContextSplit.second;
LineLocation CallSiteLoc(0, 0);
- decodeContextString(ChildContext, CalleeName, CallSiteLoc);
- Context.emplace_back(CalleeName, CallSiteLoc);
+ decodeContextString(ChildContext, Callee, CallSiteLoc);
+ Context.emplace_back(Callee, CallSiteLoc);
}
}
// Decode context string for a frame to get function name and location.
// `ContextStr` is in the form of `FuncName:StartLine.Discriminator`.
- static void decodeContextString(StringRef ContextStr, StringRef &FName,
+ static void decodeContextString(StringRef ContextStr,
+ FunctionId &Func,
LineLocation &LineLoc) {
// Get function name
auto EntrySplit = ContextStr.split(':');
- FName = EntrySplit.first;
+ Func = FunctionId(EntrySplit.first);
LineLoc = {0, 0};
if (!EntrySplit.second.empty()) {
@@ -618,7 +614,7 @@ class SampleContext {
void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
bool hasContext() const { return State != UnknownContext; }
bool isBaseContext() const { return FullContext.size() == 1; }
- StringRef getName() const { return Name; }
+ FunctionId getFunction() const { return Func; }
SampleContextFrames getContextFrames() const { return FullContext; }
static std::string getContextString(SampleContextFrames Context,
@@ -636,22 +632,19 @@ class SampleContext {
std::string toString() const {
if (!hasContext())
- return Name.str();
+ return Func.str();
return getContextString(FullContext, false);
}
uint64_t getHashCode() const {
if (hasContext())
return hash_value(getContextFrames());
-
- // For non-context function name, use its MD5 as hash value, so that it is
- // consistent with the profile map's key.
- return hashFuncName(getName());
+ return getFunction().getHashCode();
}
/// Set the name of the function and clear the current context.
- void setName(StringRef FunctionName) {
- Name = FunctionName;
+ void setFunction(FunctionId newFunction) {
+ Func = newFunction;
FullContext = SampleContextFrames();
State = UnknownContext;
}
@@ -660,12 +653,12 @@ class SampleContext {
ContextStateMask CState = RawContext) {
assert(CState != UnknownContext);
FullContext = Context;
- Name = Context.back().FuncName;
+ Func = Context.back().Func;
State = CState;
}
bool operator==(const SampleContext &That) const {
- return State == That.State && Name == That.Name &&
+ return State == That.State && Func == That.Func &&
FullContext == That.FullContext;
}
@@ -676,14 +669,14 @@ class SampleContext {
return State < That.State;
if (!hasContext()) {
- return Name < That.Name;
+ return Func < That.Func;
}
uint64_t I = 0;
while (I < std::min(FullContext.size(), That.FullContext.size())) {
auto &Context1 = FullContext[I];
auto &Context2 = That.FullContext[I];
- auto V = Context1.FuncName.compare(Context2.FuncName);
+ auto V = Context1.Func.compare(Context2.Func);
if (V)
return V < 0;
if (Context1.Location != Context2.Location)
@@ -707,15 +700,16 @@ class SampleContext {
return false;
ThatContext = ThatContext.take_front(ThisContext.size());
// Compare Leaf frame first
- if (ThisContext.back().FuncName != ThatContext.back().FuncName)
+ if (ThisContext.back().Func != ThatContext.back().Func)
return false;
// Compare leading context
return ThisContext.drop_back() == ThatContext.drop_back();
}
private:
- /// Mangled name of the function.
- StringRef Name;
+ // The function associated with this context. If CS profile, this is the leaf
+ // function.
+ FunctionId Func;
// Full context including calling context and leaf function name
SampleContextFrames FullContext;
// State of the associated sample profile
@@ -738,7 +732,7 @@ class SampleProfileReaderItaniumRemapper;
using BodySampleMap = std::map<LineLocation, SampleRecord>;
// NOTE: Using a StringMap here makes parsed profiles consume around 17% more
// memory, which is *very* significant for large profiles.
-using FunctionSamplesMap = std::map<std::string, FunctionSamples, std::less<>>;
+using FunctionSamplesMap = std::map<FunctionId, FunctionSamples>;
using CallsiteSampleMap = std::map<LineLocation, FunctionSamplesMap>;
using LocToLocMap =
std::unordered_map<LineLocation, LineLocation, LineLocationHash>;
@@ -790,14 +784,16 @@ class FunctionSamples {
sampleprof_error addCalledTargetSamples(uint32_t LineOffset,
uint32_t Discriminator,
- StringRef FName, uint64_t Num,
+ FunctionId Func,
+ uint64_t Num,
uint64_t Weight = 1) {
return BodySamples[LineLocation(LineOffset, Discriminator)].addCalledTarget(
- FName, Num, Weight);
+ Func, Num, Weight);
}
sampleprof_error addSampleRecord(LineLocation Location,
- const SampleRecord &SampleRecord, uint64_t Weight = 1) {
+ const SampleRecord &SampleRecord,
+ uint64_t Weight = 1) {
return BodySamples[Location].merge(SampleRecord, Weight);
}
@@ -805,11 +801,11 @@ class FunctionSamples {
// the number of body samples actually decreased.
uint64_t removeCalledTargetAndBodySample(uint32_t LineOffset,
uint32_t Discriminator,
- StringRef FName) {
+ FunctionId Func) {
uint64_t Count = 0;
auto I = BodySamples.find(LineLocation(LineOffset, Discriminator));
if (I != BodySamples.end()) {
- Count = I->second.removeCalledTarget(FName);
+ Count = I->second.removeCalledTarget(Func);
Count = I->second.removeSamples(Count);
if (!I->second.getSamples())
BodySamples.erase(I);
@@ -1002,7 +998,7 @@ class FunctionSamples {
sampleprof_error Result = sampleprof_error::success;
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
- if (Context.getName().empty())
+ if (Context.getFunction().empty())
Context = Other.getContext();
if (FunctionHash == 0) {
// Set the function hash code for the target profile.
@@ -1039,25 +1035,26 @@ class FunctionSamples {
/// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
/// to \p S.
void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S,
- const StringMap<Function *> &SymbolMap,
+ const HashKeyMap<std::unordered_map, FunctionId,
+ Function *> &SymbolMap,
uint64_t Threshold) const {
if (TotalSamples <= Threshold)
return;
auto isDeclaration = [](const Function *F) {
return !F || F->isDeclaration();
};
- if (isDeclaration(SymbolMap.lookup(getFuncName()))) {
+ if (isDeclaration(SymbolMap.lookup(getFunction()))) {
// Add to the import list only when it's defined out of module.
- S.insert(getGUID(getName()));
+ S.insert(getGUID());
}
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
- if (TS.getValue() > Threshold) {
- const Function *Callee = SymbolMap.lookup(getFuncName(TS.getKey()));
+ if (TS.second > Threshold) {
+ const Function *Callee = SymbolMap.lookup(TS.first);
if (isDeclaration(Callee))
- S.insert(getGUID(TS.getKey()));
+ S.insert(TS.first.getHashCode());
}
for (const auto &CS : CallsiteSamples)
for (const auto &NameFS : CS.second)
@@ -1065,13 +1062,15 @@ class FunctionSamples {
}
/// Set the name of the function.
- void setName(StringRef FunctionName) { Context.setName(FunctionName); }
+ void setFunction(FunctionId newFunction) {
+ Context.setFunction(newFunction);
+ }
/// Return the function name.
- StringRef getName() const { return Context.getName(); }
+ FunctionId getFunction() const { return Context.getFunction(); }
/// Return the original function name.
- StringRef getFuncName() const { return getFuncName(getName()); }
+ StringRef getFuncName() const { return getFuncName(getFunction()); }
void setFunctionHash(uint64_t Hash) { FunctionHash = Hash; }
@@ -1128,19 +1127,19 @@ class FunctionSamples {
return FnName;
}
- /// Translate \p Name into its original name.
- /// When profile doesn't use MD5, \p Name needs no translation.
- /// When profile uses MD5, \p Name in current FunctionSamples
+ /// Translate \p Func into its original name.
+ /// When profile doesn't use MD5, \p Func needs no translation.
+ /// When profile uses MD5, \p Func in current FunctionSamples
/// is actually GUID of the original function name. getFuncName will
- /// translate \p Name in current FunctionSamples into its original name
+ /// translate \p Func in current FunctionSamples into its original name
/// by looking up in the function map GUIDToFuncNameMap.
/// If the original name doesn't exist in the map, return empty StringRef.
- StringRef getFuncName(StringRef Name) const {
+ StringRef getFuncName(FunctionId Func) const {
if (!UseMD5)
- return Name;
+ return Func.stringRef();
assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
- return GUIDToFuncNameMap->lookup(std::stoull(Name.data()));
+ return GUIDToFuncNameMap->lookup(Func.getHashCode());
}
/// Returns the line offset to the start line of the subprogram.
@@ -1156,8 +1155,12 @@ class FunctionSamples {
/// Returns a unique hash code for a combination of a callsite location and
/// the callee function name.
- static uint64_t getCallSiteHash(StringRef CalleeName,
- const LineLocation &Callsite);
+ /// Guarantee MD5 and non-MD5 representation of the same function results in
+ /// the same hash.
+ static uint64_t getCallSiteHash(FunctionId Callee,
+ const LineLocation &Callsite) {
+ return SampleContextFrame(Callee, Callsite).getHashCode();
+ }
/// Get the FunctionSamples of the inline instance where DIL originates
/// from.
@@ -1197,16 +1200,15 @@ class FunctionSamples {
/// all the function symbols defined or declared in current module.
DenseMap<uint64_t, StringRef> *GUIDToFuncNameMap = nullptr;
- // Assume the input \p Name is a name coming from FunctionSamples itself.
- // If UseMD5 is true, the name is already a GUID and we
- // don't want to return the GUID of GUID.
- static uint64_t getGUID(StringRef Name) {
- return UseMD5 ? std::stoull(Name.data()) : Function::getGUID(Name);
+ /// Return the GUID of the context's name. If the context is already using
+ /// MD5, don't hash it again.
+ uint64_t getGUID() const {
+ return getFunction().getHashCode();
}
// Find all the names in the current FunctionSamples including names in
// all the inline instances and names of call targets.
- void findAllNames(DenseSet<StringRef> &NameSet) const;
+ void findAllNames(DenseSet<FunctionId> &NameSet) const;
bool operator==(const FunctionSamples &Other) const {
return (GUIDToFuncNameMap == Other.GUIDToFuncNameMap ||
@@ -1223,9 +1225,6 @@ class FunctionSamples {
return !(*this == Other);
}
- template <typename T>
- const T &getKey() const;
-
private:
/// CFG hash value for the function.
uint64_t FunctionHash = 0;
@@ -1289,90 +1288,16 @@ class FunctionSamples {
const LocToLocMap *IRToProfileLocationMap = nullptr;
};
-template <>
-inline const SampleContext &FunctionSamples::getKey<SampleContext>() const {
- return getContext();
+/// Get the proper representation of a string according to whether the
+/// current Format uses MD5 to represent the string.
+static inline FunctionId getRepInFormat(StringRef Name) {
+ if (Name.empty() || !FunctionSamples::UseMD5)
+ return FunctionId(Name);
+ return FunctionId(Function::getGUID(Name));
}
raw_ostream &operator<<(raw_ostream &OS, const FunctionSamples &FS);
-/// This class is a wrapper to associative container MapT<KeyT, ValueT> using
-/// the hash value of the original key as the new key. This greatly improves the
-/// performance of insert and query operations especially when hash values of
-/// keys are available a priori, and reduces memory usage if KeyT has a large
-/// size.
-/// All keys with the same hash value are considered equivalent (i.e. hash
-/// collision is silently ignored). Given such feature this class should only be
-/// used where it does not affect compilation correctness, for example, when
-/// loading a sample profile.
-/// Assuming the hashing algorithm is uniform, we use the formula
-/// 1 - Permute(n, k) / n ^ k where n is the universe size and k is number of
-/// elements chosen at random to calculate the probability of collision. With
-/// 1,000,000 entries the probability is negligible:
-/// 1 - (2^64)!/((2^64-1000000)!*(2^64)^1000000) ~= 3*10^-8.
-/// Source: https://en.wikipedia.org/wiki/Birthday_problem
-template <template <typename, typename, typename...> typename MapT,
- typename KeyT, typename ValueT, typename... MapTArgs>
-class HashKeyMap : public MapT<hash_code, ValueT, MapTArgs...> {
-public:
- using base_type = MapT<hash_code, ValueT, MapTArgs...>;
- using key_type = hash_code;
- using original_key_type = KeyT;
- using mapped_type = ValueT;
- using value_type = typename base_type::value_type;
-
- using iterator = typename base_type::iterator;
- using const_iterator = typename base_type::const_iterator;
-
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(const key_type &Hash,
- const original_key_type &Key,
- Ts &&...Args) {
- assert(Hash == hash_value(Key));
- return base_type::try_emplace(Hash, std::forward<Ts>(Args)...);
- }
-
- template <typename... Ts>
- std::pair<iterator, bool> try_emplace(const original_key_type &Key,
- Ts &&...Args) {
- key_type Hash = hash_value(Key);
- return try_emplace(Hash, Key, std::forward<Ts>(Args)...);
- }
-
- template <typename... Ts> std::pair<iterator, bool> emplace(Ts &&...Args) {
- return try_emplace(std::forward<Ts>(Args)...);
- }
-
- mapped_type &operator[](const original_key_type &Key) {
- return try_emplace(Key, mapped_type()).first->second;
- }
-
- iterator find(const original_key_type &Key) {
- key_type Hash = hash_value(Key);
- auto It = base_type::find(Hash);
- if (It != base_type::end())
- return It;
- return base_type::end();
- }
-
- const_iterator find(const original_key_type &Key) const {
- key_type Hash = hash_value(Key);
- auto It = base_type::find(Hash);
- if (It != base_type::end())
- return It;
- return base_type::end();
- }
-
- size_t erase(const original_key_type &Ctx) {
- auto It = find(Ctx);
- if (It != base_type::end()) {
- base_type::erase(It);
- return 1;
- }
- return 0;
- }
-};
-
/// This class provides operator overloads to the map container using MD5 as the
/// key type, so that existing code can still work in most cases using
/// SampleContext as key.
@@ -1400,11 +1325,6 @@ class SampleProfileMap
Ctx);
}
- // Overloaded find() to lookup a function by name.
- iterator find(StringRef Fname) {
- return base_type::find(hashFuncName(Fname));
- }
-
size_t erase(const SampleContext &Ctx) {
return HashKeyMap<std::unordered_map, SampleContext, FunctionSamples>::
erase(Ctx);
@@ -1475,7 +1395,7 @@ class ProfileConverter {
// profile.
void convertCSProfiles();
struct FrameNode {
- FrameNode(StringRef FName = StringRef(),
+ FrameNode(FunctionId FName = FunctionId(),
FunctionSamples *FSamples = nullptr,
LineLocation CallLoc = {0, 0})
: FuncName(FName), FuncSamples(FSamples), CallSiteLoc(CallLoc){};
@@ -1483,14 +1403,14 @@ class ProfileConverter {
// Map line+discriminator location to child frame
std::map<uint64_t, FrameNode> AllChildFrames;
// Function name for current frame
- StringRef FuncName;
+ FunctionId FuncName;
// Function Samples for current frame
FunctionSamples *FuncSamples;
// Callsite location in parent context
LineLocation CallSiteLoc;
FrameNode *getOrCreateChildFrame(const LineLocation &CallSite,
- StringRef CalleeName);
+ FunctionId CalleeName);
};
static void flattenProfile(SampleProfileMap &ProfileMap,
@@ -1507,7 +1427,7 @@ class ProfileConverter {
for (const auto &I : InputProfiles) {
// Retain the profile name and clear the full context for each function
// profile.
- FunctionSamples &FS = OutputProfiles.Create(I.second.getName());
+ FunctionSamples &FS = OutputProfiles.Create(I.second.getFunction());
FS.merge(I.second);
}
} else {
@@ -1553,7 +1473,8 @@ class ProfileConverter {
CalleeProfile.getHeadSamplesEstimate());
// Add callsite sample.
Profile.addCalledTargetSamples(
- I.first.LineOffset, I.first.Discriminator, CalleeProfile.getName(),
+ I.first.LineOffset, I.first.Discriminator,
+ CalleeProfile.getFunction(),
CalleeProfile.getHeadSamplesEstimate());
// Update total samples.
TotalSamples = TotalSamples >= CalleeProfile.getTotalSamples()
@@ -1625,7 +1546,9 @@ using namespace sampleprof;
template <> struct DenseMapInfo<SampleContext> {
static inline SampleContext getEmptyKey() { return SampleContext(); }
- static inline SampleContext getTombstoneKey() { return SampleContext("@"); }
+ static inline SampleContext getTombstoneKey() {
+ return SampleContext(FunctionId(~1ULL));
+ }
static unsigned getHashValue(const SampleContext &Val) {
return Val.getHashCode();
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index a5b1df3ef550b2b..9e8f543909cdbd1 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -409,13 +409,13 @@ class SampleProfileReader {
/// Return the samples collected for function \p F.
FunctionSamples *getSamplesFor(StringRef Fname) {
- auto It = Profiles.find(Fname);
+ auto It = Profiles.find(FunctionId(Fname));
if (It != Profiles.end())
return &It->second;
if (Remapper) {
if (auto NameInProfile = Remapper->lookUpNameInProfile(Fname)) {
- auto It = Profiles.find(*NameInProfile);
+ auto It = Profiles.find(FunctionId(*NameInProfile));
if (It != Profiles.end())
return &It->second;
}
@@ -474,7 +474,7 @@ class SampleProfileReader {
/// It includes all the names that have samples either in outline instance
/// or inline instance.
- virtual std::vector<StringRef> *getNameTable() { return nullptr; }
+ virtual std::vector<FunctionId> *getNameTable() { return nullptr; }
virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
/// Return whether names in the profile are all MD5 numbers.
@@ -508,10 +508,6 @@ class SampleProfileReader {
/// Memory buffer holding the profile file.
std::unique_ptr<MemoryBuffer> Buffer;
- /// Extra name buffer holding names created on demand.
- /// This should only be needed for md5 profiles.
- std::unordered_set<std::string> MD5NameBuffer;
-
/// Profile summary information.
std::unique_ptr<ProfileSummary> Summary;
@@ -595,7 +591,9 @@ class SampleProfileReaderBinary : public SampleProfileReader {
/// It includes all the names that have samples either in outline instance
/// or inline instance.
- std::vector<StringRef> *getNameTable() override { return &NameTable; }
+ std::vector<FunctionId> *getNameTable() override {
+ return &NameTable;
+ }
protected:
/// Read a numeric value of type T from the profile.
@@ -637,7 +635,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
std::error_code readNameTable();
/// Read a string indirectly via the name table. Optionally return the index.
- ErrorOr<StringRef> readStringFromTable(size_t *RetIdx = nullptr);
+ ErrorOr<FunctionId> readStringFromTable(size_t *RetIdx = nullptr);
/// Read a context indirectly via the CSNameTable. Optionally return the
/// index.
@@ -654,19 +652,7 @@ class SampleProfileReaderBinary : public SampleProfileReader {
const uint8_t *End = nullptr;
/// Function name table.
- std::vector<StringRef> NameTable;
-
- /// If MD5 is used in NameTable section, the section saves uint64_t data.
- /// The uint64_t data has to be converted to a string and then the string
- /// will be used to initialize StringRef in NameTable.
- /// Note NameTable contains StringRef so it needs another buffer to own
- /// the string data. MD5StringBuf serves as the string buffer that is
- /// referenced by NameTable (vector of StringRef). We make sure
- /// the lifetime of MD5StringBuf is not shorter than that of NameTable.
- std::vector<std::string> MD5StringBuf;
-
- /// The starting address of fixed length MD5 name table section.
- const uint8_t *MD5NameMemStart = nullptr;
+ std::vector<FunctionId> NameTable;
/// CSNameTable is used to save full context vectors. It is the backing buffer
/// for SampleContextFrames.
diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h
index 1f19283ea1dd0a8..963a4d4918e567b 100644
--- a/llvm/include/llvm/ProfileData/SampleProfWriter.h
+++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h
@@ -196,20 +196,20 @@ class SampleProfileWriterBinary : public SampleProfileWriter {
std::error_code writeSample(const FunctionSamples &S) override;
protected:
- virtual MapVector<StringRef, uint32_t> &getNameTable() { return NameTable; }
+ virtual MapVector<FunctionId, uint32_t> &getNameTable() { return NameTable; }
virtual std::error_code writeMagicIdent(SampleProfileFormat Format);
virtual std::error_code writeNameTable();
std::error_code writeHeader(const SampleProfileMap &ProfileMap) override;
std::error_code writeSummary();
virtual std::error_code writeContextIdx(const SampleContext &Context);
- std::error_code writeNameIdx(StringRef FName);
+ std::error_code writeNameIdx(FunctionId FName);
std::error_code writeBody(const FunctionSamples &S);
- inline void stablizeNameTable(MapVector<StringRef, uint32_t> &NameTable,
- std::set<StringRef> &V);
-
- MapVector<StringRef, uint32_t> NameTable;
-
- void addName(StringRef FName);
+ inline void stablizeNameTable(MapVector<FunctionId, uint32_t> &NameTable,
+ std::set<FunctionId> &V);
+
+ MapVector<FunctionId, uint32_t> NameTable;
+
+ void addName(FunctionId FName);
virtual void addContext(const SampleContext &Context);
void addNames(const FunctionSamples &S);
diff --git a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
index bc8360a80bc02bd..9436e7b41a98f64 100644
--- a/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
+++ b/llvm/include/llvm/Transforms/IPO/ProfiledCallGraph.h
@@ -52,10 +52,11 @@ struct ProfiledCallGraphNode {
using edges = std::set<edge, ProfiledCallGraphEdgeComparer>;
using iterator = edges::iterator;
using const_iterator = edges::const_iterator;
-
- ProfiledCallGraphNode(StringRef FName = StringRef()) : Name(FName) {}
-
- StringRef Name;
+
+ ProfiledCallGraphNode(FunctionId FName = FunctionId()) : Name(FName)
+ {}
+
+ FunctionId Name;
edges Edges;
};
@@ -85,7 +86,7 @@ class ProfiledCallGraph {
std::queue<ContextTrieNode *> Queue;
for (auto &Child : ContextTracker.getRootContext().getAllChildContext()) {
ContextTrieNode *Callee = &Child.second;
- addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+ addProfiledFunction(Callee->getFuncName());
Queue.push(Callee);
}
@@ -102,7 +103,7 @@ class ProfiledCallGraph {
// context-based one, which may in turn block context-based inlining.
for (auto &Child : Caller->getAllChildContext()) {
ContextTrieNode *Callee = &Child.second;
- addProfiledFunction(ContextTracker.getFuncNameFor(Callee));
+ addProfiledFunction(Callee->getFuncName());
Queue.push(Callee);
// Fetch edge weight from the profile.
@@ -116,15 +117,14 @@ class ProfiledCallGraph {
LineLocation Callsite = Callee->getCallSiteLoc();
if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
- auto It = TargetCounts.find(CalleeSamples->getName());
+ auto It = TargetCounts.find(CalleeSamples->getFunction());
if (It != TargetCounts.end())
CallsiteCount = It->second;
}
Weight = std::max(CallsiteCount, CalleeEntryCount);
}
- addProfiledCall(ContextTracker.getFuncNameFor(Caller),
- ContextTracker.getFuncNameFor(Callee), Weight);
+ addProfiledCall(Caller->getFuncName(), Callee->getFuncName(), Weight);
}
}
@@ -136,8 +136,8 @@ class ProfiledCallGraph {
iterator begin() { return Root.Edges.begin(); }
iterator end() { return Root.Edges.end(); }
ProfiledCallGraphNode *getEntryNode() { return &Root; }
-
- void addProfiledFunction(StringRef Name) {
+
+ void addProfiledFunction(FunctionId Name) {
if (!ProfiledFunctions.count(Name)) {
// Link to synthetic root to make sure every node is reachable
// from root. This does not affect SCC order.
@@ -147,7 +147,7 @@ class ProfiledCallGraph {
}
private:
- void addProfiledCall(StringRef CallerName, StringRef CalleeName,
+ void addProfiledCall(FunctionId CallerName, FunctionId CalleeName,
uint64_t Weight = 0) {
assert(ProfiledFunctions.count(CallerName));
auto CalleeIt = ProfiledFunctions.find(CalleeName);
@@ -168,19 +168,19 @@ class ProfiledCallGraph {
}
void addProfiledCalls(const FunctionSamples &Samples) {
- addProfiledFunction(Samples.getFuncName());
+ addProfiledFunction(Samples.getFunction());
for (const auto &Sample : Samples.getBodySamples()) {
for (const auto &[Target, Frequency] : Sample.second.getCallTargets()) {
addProfiledFunction(Target);
- addProfiledCall(Samples.getFuncName(), Target, Frequency);
+ addProfiledCall(Samples.getFunction(), Target, Frequency);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
for (const auto &InlinedSamples : CallsiteSamples.second) {
addProfiledFunction(InlinedSamples.first);
- addProfiledCall(Samples.getFuncName(), InlinedSamples.first,
+ addProfiledCall(Samples.getFunction(), InlinedSamples.first,
InlinedSamples.second.getHeadSamplesEstimate());
addProfiledCalls(InlinedSamples.second);
}
@@ -206,7 +206,8 @@ class ProfiledCallGraph {
}
ProfiledCallGraphNode Root;
- StringMap<ProfiledCallGraphNode> ProfiledFunctions;
+ HashKeyMap<std::unordered_map, FunctionId, ProfiledCallGraphNode>
+ ProfiledFunctions;
};
} // end namespace sampleprof
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 347dac1e9684db1..f4c999ab93d8aac 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -35,20 +35,20 @@ class Instruction;
class ContextTrieNode {
public:
ContextTrieNode(ContextTrieNode *Parent = nullptr,
- StringRef FName = StringRef(),
+ FunctionId FName = FunctionId(),
FunctionSamples *FSamples = nullptr,
LineLocation CallLoc = {0, 0})
: ParentContext(Parent), FuncName(FName), FuncSamples(FSamples),
CallSiteLoc(CallLoc){};
ContextTrieNode *getChildContext(const LineLocation &CallSite,
- StringRef ChildName);
+ FunctionId ChildName);
ContextTrieNode *getHottestChildContext(const LineLocation &CallSite);
ContextTrieNode *getOrCreateChildContext(const LineLocation &CallSite,
- StringRef ChildName,
+ FunctionId ChildName,
bool AllowCreate = true);
- void removeChildContext(const LineLocation &CallSite, StringRef ChildName);
+ void removeChildContext(const LineLocation &CallSite, FunctionId ChildName);
std::map<uint64_t, ContextTrieNode> &getAllChildContext();
- StringRef getFuncName() const;
+ FunctionId getFuncName() const;
FunctionSamples *getFunctionSamples() const;
void setFunctionSamples(FunctionSamples *FSamples);
std::optional<uint32_t> getFunctionSize() const;
@@ -68,7 +68,7 @@ class ContextTrieNode {
ContextTrieNode *ParentContext;
// Function name for current context
- StringRef FuncName;
+ FunctionId FuncName;
// Function Samples for current context
FunctionSamples *FuncSamples;
@@ -118,7 +118,8 @@ class SampleContextTracker {
FunctionSamples *getBaseSamplesFor(const Function &Func,
bool MergeContext = true);
// Query base profile for a given function by name.
- FunctionSamples *getBaseSamplesFor(StringRef Name, bool MergeContext = true);
+ FunctionSamples *getBaseSamplesFor(FunctionId Name,
+ bool MergeContext = true);
// Retrieve the context trie node for given profile context
ContextTrieNode *getContextFor(const SampleContext &Context);
// Get real function name for a given trie node.
@@ -129,7 +130,7 @@ class SampleContextTracker {
void markContextSamplesInlined(const FunctionSamples *InlinedSamples);
ContextTrieNode &getRootContext();
void promoteMergeContextSamplesTree(const Instruction &Inst,
- StringRef CalleeName);
+ FunctionId CalleeName);
// Create a merged conext-less profile map.
void createContextLessProfileMap(SampleProfileMap &ContextLessProfiles);
@@ -140,7 +141,8 @@ class SampleContextTracker {
return nullptr;
return I->second;
}
- StringMap<ContextSamplesTy> &getFuncToCtxtProfiles() {
+ HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+ &getFuncToCtxtProfiles() {
return FuncToCtxtProfiles;
}
@@ -189,9 +191,9 @@ class SampleContextTracker {
private:
ContextTrieNode *getContextFor(const DILocation *DIL);
ContextTrieNode *getCalleeContextFor(const DILocation *DIL,
- StringRef CalleeName);
- ContextTrieNode *getTopLevelContextNode(StringRef FName);
- ContextTrieNode &addTopLevelContextNode(StringRef FName);
+ FunctionId CalleeName);
+ ContextTrieNode *getTopLevelContextNode(FunctionId FName);
+ ContextTrieNode &addTopLevelContextNode(FunctionId FName);
ContextTrieNode &promoteMergeContextSamplesTree(ContextTrieNode &NodeToPromo);
void mergeContextNode(ContextTrieNode &FromNode, ContextTrieNode &ToNode);
ContextTrieNode &
@@ -204,7 +206,8 @@ class SampleContextTracker {
ProfileToNodeMap[FSample] = Node;
}
// Map from function name to context profiles (excluding base profile)
- StringMap<ContextSamplesTy> FuncToCtxtProfiles;
+ HashKeyMap<std::unordered_map, FunctionId, ContextSamplesTy>
+ FuncToCtxtProfiles;
// Map from current FunctionSample to the belonged context trie.
std::unordered_map<const FunctionSamples *, ContextTrieNode *>
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index b14dc01be236245..59fa71899ed47b2 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -121,7 +121,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other,
sampleprof_error Result;
Result = addSamples(Other.getSamples(), Weight);
for (const auto &I : Other.getCallTargets()) {
- MergeResult(Result, addCalledTarget(I.first(), I.second, Weight));
+ MergeResult(Result, addCalledTarget(I.first, I.second, Weight));
}
return Result;
}
@@ -181,7 +181,8 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const {
for (const auto &CS : SortedCallsiteSamples.get()) {
for (const auto &FS : CS->second) {
OS.indent(Indent + 2);
- OS << CS->first << ": inlined callee: " << FS.second.getName() << ": ";
+ OS << CS->first << ": inlined callee: " << FS.second.getFunction()
+ << ": ";
FS.second.print(OS, Indent + 4);
}
}
@@ -234,14 +235,6 @@ LineLocation FunctionSamples::getCallSiteIdentifier(const DILocation *DIL,
}
}
-uint64_t FunctionSamples::getCallSiteHash(StringRef CalleeName,
- const LineLocation &Callsite) {
- uint64_t NameHash = std::hash<std::string>{}(CalleeName.str());
- uint64_t LocId =
- (((uint64_t)Callsite.LineOffset) << 32) | Callsite.Discriminator;
- return NameHash + (LocId << 5) + LocId;
-}
-
const FunctionSamples *FunctionSamples::findFunctionSamples(
const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper) const {
assert(DIL);
@@ -268,11 +261,11 @@ const FunctionSamples *FunctionSamples::findFunctionSamples(
return FS;
}
-void FunctionSamples::findAllNames(DenseSet<StringRef> &NameSet) const {
- NameSet.insert(getName());
+void FunctionSamples::findAllNames(DenseSet<FunctionId> &NameSet) const {
+ NameSet.insert(getFunction());
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
- NameSet.insert(TS.getKey());
+ NameSet.insert(TS.first);
for (const auto &CS : CallsiteSamples) {
for (const auto &NameFS : CS.second) {
@@ -287,18 +280,15 @@ const FunctionSamples *FunctionSamples::findFunctionSamplesAt(
SampleProfileReaderItaniumRemapper *Remapper) const {
CalleeName = getCanonicalFnName(CalleeName);
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, UseMD5, CalleeGUID);
-
auto iter = CallsiteSamples.find(mapIRLocToProfileLoc(Loc));
if (iter == CallsiteSamples.end())
return nullptr;
- auto FS = iter->second.find(CalleeName);
+ auto FS = iter->second.find(getRepInFormat(CalleeName));
if (FS != iter->second.end())
return &FS->second;
if (Remapper) {
if (auto NameInProfile = Remapper->lookUpNameInProfile(CalleeName)) {
- auto FS = iter->second.find(*NameInProfile);
+ auto FS = iter->second.find(getRepInFormat(*NameInProfile));
if (FS != iter->second.end())
return &FS->second;
}
@@ -422,7 +412,7 @@ void ProfileSymbolList::dump(raw_ostream &OS) const {
ProfileConverter::FrameNode *
ProfileConverter::FrameNode::getOrCreateChildFrame(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
auto It = AllChildFrames.find(Hash);
if (It != AllChildFrames.end()) {
@@ -450,7 +440,7 @@ ProfileConverter::getOrCreateContextPath(const SampleContext &Context) {
auto Node = &RootFrame;
LineLocation CallSiteLoc(0, 0);
for (auto &Callsite : Context.getContextFrames()) {
- Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.FuncName);
+ Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.Func);
CallSiteLoc = Callsite.Location;
}
return Node;
@@ -468,23 +458,23 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
if (!ChildProfile)
continue;
SampleContext OrigChildContext = ChildProfile->getContext();
- hash_code OrigChildContextHash = OrigChildContext.getHashCode();
+ uint64_t OrigChildContextHash = OrigChildContext.getHashCode();
// Reset the child context to be contextless.
- ChildProfile->getContext().setName(OrigChildContext.getName());
+ ChildProfile->getContext().setFunction(OrigChildContext.getFunction());
if (NodeProfile) {
// Add child profile to the callsite profile map.
auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
- SamplesMap.emplace(OrigChildContext.getName().str(), *ChildProfile);
+ SamplesMap.emplace(OrigChildContext.getFunction(), *ChildProfile);
NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());
// Remove the corresponding body sample for the callsite and update the
// total weight.
auto Count = NodeProfile->removeCalledTargetAndBodySample(
ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,
- OrigChildContext.getName());
+ OrigChildContext.getFunction());
NodeProfile->removeTotalSamples(Count);
}
- hash_code NewChildProfileHash(0);
+ uint64_t NewChildProfileHash = 0;
// Separate child profile to be a standalone profile, if the current parent
// profile doesn't exist. This is a duplicating operation when the child
// profile is already incorporated into the parent which is still useful and
@@ -498,7 +488,7 @@ void ProfileConverter::convertCSProfiles(ProfileConverter::FrameNode &Node) {
ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);
NewChildProfileHash = ChildProfile->getContext().getHashCode();
auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);
- SamplesMap[ChildProfile->getName().str()].getContext().setAttribute(
+ SamplesMap[ChildProfile->getFunction()].getContext().setAttribute(
ContextDuplicatedIntoBase);
}
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 256bdb833a0b187..a69e9d5849047ab 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -91,7 +91,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
JOS.attributeArray("calls", [&] {
for (const auto &J : CallTargets) {
JOS.object([&] {
- JOS.attribute("function", J.first);
+ JOS.attribute("function", J.first.str());
JOS.attribute("samples", J.second);
});
}
@@ -117,7 +117,7 @@ static void dumpFunctionProfileJson(const FunctionSamples &S,
};
JOS.object([&] {
- JOS.attribute("name", S.getName());
+ JOS.attribute("name", S.getFunction().str());
JOS.attribute("total", S.getTotalSamples());
if (TopLevel)
JOS.attribute("head", S.getHeadSamples());
@@ -392,8 +392,8 @@ std::error_code SampleProfileReaderText::readImpl() {
switch (LineTy) {
case LineType::CallSiteProfile: {
FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt(
- LineLocation(LineOffset, Discriminator))[std::string(FName)];
- FSamples.setName(FName);
+ LineLocation(LineOffset, Discriminator))[FunctionId(FName)];
+ FSamples.setFunction(FunctionId(FName));
MergeResult(Result, FSamples.addTotalSamples(NumSamples));
InlineStack.push_back(&FSamples);
DepthMetadata = 0;
@@ -406,7 +406,8 @@ std::error_code SampleProfileReaderText::readImpl() {
FunctionSamples &FProfile = *InlineStack.back();
for (const auto &name_count : TargetCountMap) {
MergeResult(Result, FProfile.addCalledTargetSamples(
- LineOffset, Discriminator, name_count.first,
+ LineOffset, Discriminator,
+ FunctionId(name_count.first),
name_count.second));
}
MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator,
@@ -516,28 +517,14 @@ inline ErrorOr<size_t> SampleProfileReaderBinary::readStringIndex(T &Table) {
return *Idx;
}
-ErrorOr<StringRef>
+ErrorOr<FunctionId>
SampleProfileReaderBinary::readStringFromTable(size_t *RetIdx) {
auto Idx = readStringIndex(NameTable);
if (std::error_code EC = Idx.getError())
return EC;
-
- // Lazy loading, if the string has not been materialized from memory storing
- // MD5 values, then it is default initialized with the null pointer. This can
- // only happen when using fixed length MD5, that bounds check is performed
- // while parsing the name table to ensure MD5NameMemStart points to an array
- // with enough MD5 entries.
- StringRef &SR = NameTable[*Idx];
- if (!SR.data()) {
- assert(MD5NameMemStart);
- using namespace support;
- uint64_t FID = endian::read<uint64_t, llvm::endianness::little>(
- MD5NameMemStart + (*Idx) * sizeof(uint64_t));
- SR = MD5StringBuf.emplace_back(std::to_string(FID));
- }
if (RetIdx)
*RetIdx = *Idx;
- return SR;
+ return NameTable[*Idx];
}
ErrorOr<SampleContextFrames>
@@ -655,8 +642,8 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) {
uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask();
FunctionSamples &CalleeProfile = FProfile.functionSamplesAt(
- LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)];
- CalleeProfile.setName(*FName);
+ LineLocation(*LineOffset, DiscriminatorVal))[*FName];
+ CalleeProfile.setFunction(*FName);
if (std::error_code EC = readProfile(CalleeProfile))
return EC;
}
@@ -894,13 +881,17 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
const SampleContext *CommonContext = nullptr;
for (const auto &NameOffset : FuncOffsetList) {
const auto &FContext = NameOffset.first;
- auto FName = FContext.getName();
+ FunctionId FName = FContext.getFunction();
+ StringRef FNameString;
+ if (!useMD5())
+ FNameString = FName.stringRef();
+
// For function in the current module, keep its farthest ancestor
// context. This can be used to load itself and its child and
// sibling contexts.
- if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) ||
- (!useMD5() && (FuncsToUse.count(FName) ||
- (Remapper && Remapper->exist(FName))))) {
+ if ((useMD5() && FuncGuidsToUse.count(FName.getHashCode())) ||
+ (!useMD5() && (FuncsToUse.count(FNameString) ||
+ (Remapper && Remapper->exist(FNameString))))) {
if (!CommonContext || !CommonContext->IsPrefixOf(FContext))
CommonContext = &FContext;
}
@@ -929,8 +920,9 @@ std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() {
assert(useFuncOffsetList());
for (auto NameOffset : FuncOffsetList) {
SampleContext FContext(NameOffset.first);
- auto FuncName = FContext.getName();
- if (!FuncsToUse.count(FuncName) && !Remapper->exist(FuncName))
+ auto FuncName = FContext.getFunction();
+ StringRef FuncNameStr = FuncName.stringRef();
+ if (!FuncsToUse.count(FuncNameStr) && !Remapper->exist(FuncNameStr))
continue;
const uint8_t *FuncProfileAddr = Start + NameOffset.second;
if (std::error_code EC = readFuncProfile(FuncProfileAddr))
@@ -1062,8 +1054,6 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
// tables mixing string and MD5, all of them have to be normalized to use MD5,
// because optimization passes can only handle either type.
bool UseMD5 = useMD5();
- if (UseMD5)
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
NameTable.reserve(*Size);
@@ -1082,12 +1072,12 @@ std::error_code SampleProfileReaderBinary::readNameTable() {
if (std::error_code EC = Name.getError())
return EC;
if (UseMD5) {
- uint64_t FID = hashFuncName(*Name);
+ FunctionId FID(*Name);
if (!ProfileIsCS)
- MD5SampleContextTable.emplace_back(FID);
- NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(FID)));
+ MD5SampleContextTable.emplace_back(FID.getHashCode());
+ NameTable.emplace_back(FID);
} else
- NameTable.push_back(*Name);
+ NameTable.push_back(FunctionId(*Name));
}
if (!ProfileIsCS)
MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1110,14 +1100,14 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
if (Data + (*Size) * sizeof(uint64_t) > End)
return sampleprof_error::truncated;
- // Preallocate and initialize NameTable so we can check whether a name
- // index has been read before by checking whether the element in the
- // NameTable is empty, meanwhile readStringIndex can do the boundary
- // check using the size of NameTable.
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
- NameTable.resize(*Size);
- MD5NameMemStart = Data;
+ NameTable.reserve(*Size);
+ for (size_t I = 0; I < *Size; ++I) {
+ using namespace support;
+ uint64_t FID = endian::read<uint64_t, endianness::little, unaligned>(
+ Data + I * sizeof(uint64_t));
+ NameTable.emplace_back(FunctionId(FID));
+ }
if (!ProfileIsCS)
MD5SampleContextStart = reinterpret_cast<const uint64_t *>(Data);
Data = Data + (*Size) * sizeof(uint64_t);
@@ -1130,7 +1120,6 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
if (std::error_code EC = Size.getError())
return EC;
- MD5StringBuf.reserve(MD5StringBuf.size() + *Size);
NameTable.clear();
NameTable.reserve(*Size);
if (!ProfileIsCS)
@@ -1141,7 +1130,7 @@ SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5,
return EC;
if (!ProfileIsCS)
support::endian::write64le(&MD5SampleContextTable[I], *FID);
- NameTable.emplace_back(MD5StringBuf.emplace_back(std::to_string(*FID)));
+ NameTable.emplace_back(FunctionId(*FID));
}
if (!ProfileIsCS)
MD5SampleContextStart = MD5SampleContextTable.data();
@@ -1243,7 +1232,7 @@ SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute,
CalleeProfile = const_cast<FunctionSamples *>(
&FProfile->functionSamplesAt(LineLocation(
*LineOffset,
- *Discriminator))[std::string(FContext.getName())]);
+ *Discriminator))[FContext.getFunction()]);
}
if (std::error_code EC =
readFuncMetadata(ProfileHasAttribute, CalleeProfile))
@@ -1653,7 +1642,7 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
// body, there will be identical replicated profiles for the
// original function. In this case, we simply not bother updating
// the profile of the original function.
- FProfile = &Profiles[Name];
+ FProfile = &Profiles[FunctionId(Name)];
FProfile->addHeadSamples(HeadCount);
if (FProfile->getTotalSamples() > 0)
Update = false;
@@ -1665,9 +1654,9 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
uint32_t LineOffset = Offset >> 16;
uint32_t Discriminator = Offset & 0xffff;
FProfile = &CallerProfile->functionSamplesAt(
- LineLocation(LineOffset, Discriminator))[std::string(Name)];
+ LineLocation(LineOffset, Discriminator))[FunctionId(Name)];
}
- FProfile->setName(Name);
+ FProfile->setFunction(FunctionId(Name));
for (uint32_t I = 0; I < NumPosCounts; ++I) {
uint32_t Offset;
@@ -1723,7 +1712,8 @@ std::error_code SampleProfileReaderGCC::readOneFunctionProfile(
if (Update)
FProfile->addCalledTargetSamples(LineOffset, Discriminator,
- TargetName, TargetCount);
+ FunctionId(TargetName),
+ TargetCount);
}
}
@@ -1784,11 +1774,13 @@ void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) {
// We will need to remap the entire context string.
assert(Remappings && "should be initialized while creating remapper");
for (auto &Sample : Reader.getProfiles()) {
- DenseSet<StringRef> NamesInSample;
+ DenseSet<FunctionId> NamesInSample;
Sample.second.findAllNames(NamesInSample);
- for (auto &Name : NamesInSample)
- if (auto Key = Remappings->insert(Name))
- NameMap.insert({Key, Name});
+ for (auto &Name : NamesInSample) {
+ StringRef NameStr = Name.stringRef();
+ if (auto Key = Remappings->insert(NameStr))
+ NameMap.insert({Key, NameStr});
+ }
}
RemappingApplied = true;
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
index c11cd4dfa6a5a97..625e523f13cec0e 100644
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -240,7 +240,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeContextIdx(
if (Context.hasContext())
return writeCSNameIdx(Context);
else
- return SampleProfileWriterBinary::writeNameIdx(Context.getName());
+ return SampleProfileWriterBinary::writeNameIdx(Context.getFunction());
}
std::error_code
@@ -346,7 +346,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
return SampleProfileWriterBinary::writeNameTable();
auto &OS = *OutputStream;
- std::set<StringRef> V;
+ std::set<FunctionId> V;
stablizeNameTable(NameTable, V);
// Write out the MD5 name table. We wrote unencoded MD5 so reader can
@@ -355,7 +355,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTable() {
encodeULEB128(NameTable.size(), OS);
support::endian::Writer Writer(OS, llvm::endianness::little);
for (auto N : V)
- Writer.write(hashFuncName(N));
+ Writer.write(N.getHashCode());
return sampleprof_error::success;
}
@@ -369,10 +369,13 @@ std::error_code SampleProfileWriterExtBinaryBase::writeNameTableSection(
// If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
// so compiler won't strip the suffix during profile matching after
// seeing the flag in the profile.
- for (const auto &I : NameTable) {
- if (I.first.contains(FunctionSamples::UniqSuffix)) {
- addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
- break;
+ // Original names are unavailable if using MD5, so this option has no use.
+ if (!UseMD5) {
+ for (const auto &I : NameTable) {
+ if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {
+ addSectionFlag(SecNameTable, SecNameTableFlags::SecFlagUniqSuffix);
+ break;
+ }
}
}
@@ -399,7 +402,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeCSNameTableSection() {
auto Frames = Context.getContextFrames();
encodeULEB128(Frames.size(), OS);
for (auto &Callsite : Frames) {
- if (std::error_code EC = writeNameIdx(Callsite.FuncName))
+ if (std::error_code EC = writeNameIdx(Callsite.Func))
return EC;
encodeULEB128(Callsite.Location.LineOffset, OS);
encodeULEB128(Callsite.Location.Discriminator, OS);
@@ -567,7 +570,7 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
if (FunctionSamples::ProfileIsCS)
OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
else
- OS << S.getName() << ":" << S.getTotalSamples();
+ OS << S.getFunction() << ":" << S.getTotalSamples();
if (Indent == 0)
OS << ":" << S.getHeadSamples();
@@ -627,10 +630,10 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) {
std::error_code
SampleProfileWriterBinary::writeContextIdx(const SampleContext &Context) {
assert(!Context.hasContext() && "cs profile is not supported");
- return writeNameIdx(Context.getName());
+ return writeNameIdx(Context.getFunction());
}
-std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
+std::error_code SampleProfileWriterBinary::writeNameIdx(FunctionId FName) {
auto &NTable = getNameTable();
const auto &Ret = NTable.find(FName);
if (Ret == NTable.end())
@@ -639,13 +642,13 @@ std::error_code SampleProfileWriterBinary::writeNameIdx(StringRef FName) {
return sampleprof_error::success;
}
-void SampleProfileWriterBinary::addName(StringRef FName) {
+void SampleProfileWriterBinary::addName(FunctionId FName) {
auto &NTable = getNameTable();
NTable.insert(std::make_pair(FName, 0));
}
void SampleProfileWriterBinary::addContext(const SampleContext &Context) {
- addName(Context.getName());
+ addName(Context.getFunction());
}
void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
@@ -653,14 +656,14 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) {
for (const auto &I : S.getBodySamples()) {
const SampleRecord &Sample = I.second;
for (const auto &J : Sample.getCallTargets())
- addName(J.first());
+ addName(J.first);
}
// Recursively add all the names for inlined callsites.
for (const auto &J : S.getCallsiteSamples())
for (const auto &FS : J.second) {
const FunctionSamples &CalleeSamples = FS.second;
- addName(CalleeSamples.getName());
+ addName(CalleeSamples.getFunction());
addNames(CalleeSamples);
}
}
@@ -669,26 +672,26 @@ void SampleProfileWriterExtBinaryBase::addContext(
const SampleContext &Context) {
if (Context.hasContext()) {
for (auto &Callsite : Context.getContextFrames())
- SampleProfileWriterBinary::addName(Callsite.FuncName);
+ SampleProfileWriterBinary::addName(Callsite.Func);
CSNameTable.insert(std::make_pair(Context, 0));
} else {
- SampleProfileWriterBinary::addName(Context.getName());
+ SampleProfileWriterBinary::addName(Context.getFunction());
}
}
void SampleProfileWriterBinary::stablizeNameTable(
- MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) {
+ MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
// Sort the names to make NameTable deterministic.
for (const auto &I : NameTable)
V.insert(I.first);
int i = 0;
- for (const StringRef &N : V)
+ for (const FunctionId &N : V)
NameTable[N] = i++;
}
std::error_code SampleProfileWriterBinary::writeNameTable() {
auto &OS = *OutputStream;
- std::set<StringRef> V;
+ std::set<FunctionId> V;
stablizeNameTable(NameTable, V);
// Write out the name table.
@@ -835,7 +838,7 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
encodeULEB128(Sample.getSamples(), OS);
encodeULEB128(Sample.getCallTargets().size(), OS);
for (const auto &J : Sample.getSortedCallTargets()) {
- StringRef Callee = J.first;
+ FunctionId Callee = J.first;
uint64_t CalleeSamples = J.second;
if (std::error_code EC = writeNameIdx(Callee))
return EC;
diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
index 29ae05bf0c94673..3c9738be547f33b 100644
--- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
@@ -110,6 +110,11 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
Prefetches &Prefetches) const {
assert(Prefetches.empty() &&
"Expected caller passed empty PrefetchInfo vector.");
+
+ // There is no point to match prefetch hints if the profile is using MD5.
+ if (FunctionSamples::UseMD5)
+ return false;
+
static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
{"_nta_", X86::PREFETCHNTA},
{"_t0_", X86::PREFETCHT0},
@@ -125,7 +130,7 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
// Convert serialized prefetch hints into PrefetchInfo objects, and populate
// the Prefetches vector.
for (const auto &S_V : *T) {
- StringRef Name = S_V.getKey();
+ StringRef Name = S_V.first.stringRef();
if (Name.consume_front(SerializedPrefetchPrefix)) {
int64_t D = static_cast<int64_t>(S_V.second);
unsigned IID = 0;
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
index 79bee8362ef8a9c..09dd5c11b3094bd 100644
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -29,7 +29,7 @@ using namespace sampleprof;
namespace llvm {
ContextTrieNode *ContextTrieNode::getChildContext(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
if (CalleeName.empty())
return getHottestChildContext(CallSite);
@@ -104,7 +104,7 @@ SampleContextTracker::moveContextSamples(ContextTrieNode &ToNodeParent,
}
void ContextTrieNode::removeChildContext(const LineLocation &CallSite,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
// Note this essentially calls dtor and destroys that child context
AllChildContext.erase(Hash);
@@ -114,7 +114,7 @@ std::map<uint64_t, ContextTrieNode> &ContextTrieNode::getAllChildContext() {
return AllChildContext;
}
-StringRef ContextTrieNode::getFuncName() const { return FuncName; }
+FunctionId ContextTrieNode::getFuncName() const { return FuncName; }
FunctionSamples *ContextTrieNode::getFunctionSamples() const {
return FuncSamples;
@@ -178,7 +178,7 @@ void ContextTrieNode::dumpTree() {
}
ContextTrieNode *ContextTrieNode::getOrCreateChildContext(
- const LineLocation &CallSite, StringRef CalleeName, bool AllowCreate) {
+ const LineLocation &CallSite, FunctionId CalleeName, bool AllowCreate) {
uint64_t Hash = FunctionSamples::getCallSiteHash(CalleeName, CallSite);
auto It = AllChildContext.find(Hash);
if (It != AllChildContext.end()) {
@@ -232,14 +232,12 @@ SampleContextTracker::getCalleeContextSamplesFor(const CallBase &Inst,
return nullptr;
CalleeName = FunctionSamples::getCanonicalFnName(CalleeName);
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::string FGUID;
- CalleeName = getRepInFormat(CalleeName, FunctionSamples::UseMD5, FGUID);
+
+ FunctionId FName = getRepInFormat(CalleeName);
// For indirect call, CalleeName will be empty, in which case the context
// profile for callee with largest total samples will be returned.
- ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, CalleeName);
+ ContextTrieNode *CalleeContext = getCalleeContextFor(DIL, FName);
if (CalleeContext) {
FunctionSamples *FSamples = CalleeContext->getFunctionSamples();
LLVM_DEBUG(if (FSamples) {
@@ -305,27 +303,23 @@ SampleContextTracker::getContextSamplesFor(const SampleContext &Context) {
SampleContextTracker::ContextSamplesTy &
SampleContextTracker::getAllContextSamplesFor(const Function &Func) {
StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
- return FuncToCtxtProfiles[CanonName];
+ return FuncToCtxtProfiles[getRepInFormat(CanonName)];
}
SampleContextTracker::ContextSamplesTy &
SampleContextTracker::getAllContextSamplesFor(StringRef Name) {
- return FuncToCtxtProfiles[Name];
+ return FuncToCtxtProfiles[getRepInFormat(Name)];
}
FunctionSamples *SampleContextTracker::getBaseSamplesFor(const Function &Func,
bool MergeContext) {
StringRef CanonName = FunctionSamples::getCanonicalFnName(Func);
- return getBaseSamplesFor(CanonName, MergeContext);
+ return getBaseSamplesFor(getRepInFormat(CanonName), MergeContext);
}
-FunctionSamples *SampleContextTracker::getBaseSamplesFor(StringRef Name,
+FunctionSamples *SampleContextTracker::getBaseSamplesFor(FunctionId Name,
bool MergeContext) {
LLVM_DEBUG(dbgs() << "Getting base profile for function: " << Name << "\n");
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::string FGUID;
- Name = getRepInFormat(Name, FunctionSamples::UseMD5, FGUID);
// Base profile is top-level node (child of root node), so try to retrieve
// existing top-level node for given function first. If it exists, it could be
@@ -373,7 +367,7 @@ void SampleContextTracker::markContextSamplesInlined(
ContextTrieNode &SampleContextTracker::getRootContext() { return RootContext; }
void SampleContextTracker::promoteMergeContextSamplesTree(
- const Instruction &Inst, StringRef CalleeName) {
+ const Instruction &Inst, FunctionId CalleeName) {
LLVM_DEBUG(dbgs() << "Promoting and merging context tree for instr: \n"
<< Inst << "\n");
// Get the caller context for the call instruction, we don't use callee
@@ -458,9 +452,9 @@ void SampleContextTracker::dump() { RootContext.dumpTree(); }
StringRef SampleContextTracker::getFuncNameFor(ContextTrieNode *Node) const {
if (!FunctionSamples::UseMD5)
- return Node->getFuncName();
+ return Node->getFuncName().stringRef();
assert(GUIDToFuncNameMap && "GUIDToFuncNameMap needs to be populated first");
- return GUIDToFuncNameMap->lookup(std::stoull(Node->getFuncName().data()));
+ return GUIDToFuncNameMap->lookup(Node->getFuncName().getHashCode());
}
ContextTrieNode *
@@ -470,7 +464,7 @@ SampleContextTracker::getContextFor(const SampleContext &Context) {
ContextTrieNode *
SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
- StringRef CalleeName) {
+ FunctionId CalleeName) {
assert(DIL && "Expect non-null location");
ContextTrieNode *CallContext = getContextFor(DIL);
@@ -485,7 +479,7 @@ SampleContextTracker::getCalleeContextFor(const DILocation *DIL,
ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
assert(DIL && "Expect non-null location");
- SmallVector<std::pair<LineLocation, StringRef>, 10> S;
+ SmallVector<std::pair<LineLocation, FunctionId>, 10> S;
// Use C++ linkage name if possible.
const DILocation *PrevDIL = DIL;
@@ -494,7 +488,8 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
if (Name.empty())
Name = PrevDIL->getScope()->getSubprogram()->getName();
S.push_back(
- std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL), Name));
+ std::make_pair(FunctionSamples::getCallSiteIdentifier(DIL),
+ getRepInFormat(Name)));
PrevDIL = DIL;
}
@@ -503,24 +498,14 @@ ContextTrieNode *SampleContextTracker::getContextFor(const DILocation *DIL) {
StringRef RootName = PrevDIL->getScope()->getSubprogram()->getLinkageName();
if (RootName.empty())
RootName = PrevDIL->getScope()->getSubprogram()->getName();
- S.push_back(std::make_pair(LineLocation(0, 0), RootName));
-
- // Convert real function names to MD5 names, if the input profile is
- // MD5-based.
- std::list<std::string> MD5Names;
- if (FunctionSamples::UseMD5) {
- for (auto &Location : S) {
- MD5Names.emplace_back();
- getRepInFormat(Location.second, FunctionSamples::UseMD5, MD5Names.back());
- Location.second = MD5Names.back();
- }
- }
+ S.push_back(std::make_pair(LineLocation(0, 0),
+ getRepInFormat(RootName)));
ContextTrieNode *ContextNode = &RootContext;
int I = S.size();
while (--I >= 0 && ContextNode) {
LineLocation &CallSite = S[I].first;
- StringRef CalleeName = S[I].second;
+ FunctionId CalleeName = S[I].second;
ContextNode = ContextNode->getChildContext(CallSite, CalleeName);
}
@@ -540,10 +525,10 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
// Create child node at parent line/disc location
if (AllowCreate) {
ContextNode =
- ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.FuncName);
+ ContextNode->getOrCreateChildContext(CallSiteLoc, Callsite.Func);
} else {
ContextNode =
- ContextNode->getChildContext(CallSiteLoc, Callsite.FuncName);
+ ContextNode->getChildContext(CallSiteLoc, Callsite.Func);
}
CallSiteLoc = Callsite.Location;
}
@@ -553,12 +538,14 @@ SampleContextTracker::getOrCreateContextPath(const SampleContext &Context,
return ContextNode;
}
-ContextTrieNode *SampleContextTracker::getTopLevelContextNode(StringRef FName) {
+ContextTrieNode *
+SampleContextTracker::getTopLevelContextNode(FunctionId FName) {
assert(!FName.empty() && "Top level node query must provide valid name");
return RootContext.getChildContext(LineLocation(0, 0), FName);
}
-ContextTrieNode &SampleContextTracker::addTopLevelContextNode(StringRef FName) {
+ContextTrieNode &
+SampleContextTracker::addTopLevelContextNode(FunctionId FName) {
assert(!getTopLevelContextNode(FName) && "Node to add must not exist");
return *RootContext.getOrCreateChildContext(LineLocation(0, 0), FName);
}
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 67a2e167b6bf7dd..a7773737ef16bd3 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -424,7 +424,7 @@ struct CandidateComparer {
return LCS->getBodySamples().size() > RCS->getBodySamples().size();
// Tie breaker using GUID so we have stable/deterministic inlining order
- return LCS->getGUID(LCS->getName()) < RCS->getGUID(RCS->getName());
+ return LCS->getGUID() < RCS->getGUID();
}
};
@@ -467,7 +467,7 @@ class SampleProfileMatcher {
private:
FunctionSamples *getFlattenedSamplesFor(const Function &F) {
StringRef CanonFName = FunctionSamples::getCanonicalFnName(F);
- auto It = FlattenedProfiles.find(CanonFName);
+ auto It = FlattenedProfiles.find(FunctionId(CanonFName));
if (It != FlattenedProfiles.end())
return &It->second;
return nullptr;
@@ -475,18 +475,21 @@ class SampleProfileMatcher {
void runOnFunction(const Function &F);
void findIRAnchors(const Function &F,
std::map<LineLocation, StringRef> &IRAnchors);
- void findProfileAnchors(const FunctionSamples &FS,
- std::map<LineLocation, StringSet<>> &ProfileAnchors);
+ void findProfileAnchors(
+ const FunctionSamples &FS,
+ std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors);
void countMismatchedSamples(const FunctionSamples &FS);
void countProfileMismatches(
const Function &F, const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors);
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors);
void countProfileCallsiteMismatches(
const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
-
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites);
LocToLocMap &getIRToProfileLocationMap(const Function &F) {
auto Ret = FuncMappings.try_emplace(
@@ -497,7 +500,8 @@ class SampleProfileMatcher {
void distributeIRToProfileLocationMap(FunctionSamples &FS);
void runStaleProfileMatching(
const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
LocToLocMap &IRToProfileLocationMap);
};
@@ -539,7 +543,6 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
void findExternalInlineCandidate(CallBase *CB, const FunctionSamples *Samples,
DenseSet<GlobalValue::GUID> &InlinedGUIDs,
- const StringMap<Function *> &SymbolMap,
uint64_t Threshold);
// Attempt to promote indirect call and also inline the promoted call
bool tryPromoteAndInlineCandidate(
@@ -574,7 +577,7 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
/// the function name. If the function name contains suffix, additional
/// entry is added to map from the stripped name to the function if there
/// is one-to-one mapping.
- StringMap<Function *> SymbolMap;
+ HashKeyMap<std::unordered_map, FunctionId, Function *> SymbolMap;
std::function<AssumptionCache &(Function &)> GetAC;
std::function<TargetTransformInfo &(Function &)> GetTTI;
@@ -616,6 +619,11 @@ class SampleProfileLoader final : public SampleProfileLoaderBaseImpl<Function> {
// All the Names used in FunctionSamples including outline function
// names, inline instance names and call target names.
StringSet<> NamesInProfile;
+ // MD5 version of NamesInProfile. Either NamesInProfile or GUIDsInProfile is
+ // populated, depends on whether the profile uses MD5. Because the name table
+ // generally contains several magnitude more entries than the number of
+ // functions, we do not want to convert all names from one form to another.
+ llvm::DenseSet<uint64_t> GUIDsInProfile;
// For symbol in profile symbol list, whether to regard their profiles
// to be accurate. It is mainly decided by existance of profile symbol
@@ -760,8 +768,7 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
assert(L && R && "Expect non-null FunctionSamples");
if (L->getHeadSamplesEstimate() != R->getHeadSamplesEstimate())
return L->getHeadSamplesEstimate() > R->getHeadSamplesEstimate();
- return FunctionSamples::getGUID(L->getName()) <
- FunctionSamples::getGUID(R->getName());
+ return L->getGUID() < R->getGUID();
};
if (FunctionSamples::ProfileIsCS) {
@@ -971,13 +978,13 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
// This prevents allocating an array of zero length in callees below.
if (MaxNumPromotions == 0)
return false;
- auto CalleeFunctionName = Candidate.CalleeSamples->getFuncName();
+ auto CalleeFunctionName = Candidate.CalleeSamples->getFunction();
auto R = SymbolMap.find(CalleeFunctionName);
- if (R == SymbolMap.end() || !R->getValue())
+ if (R == SymbolMap.end() || !R->second)
return false;
auto &CI = *Candidate.CallInstr;
- if (!doesHistoryAllowICP(CI, R->getValue()->getName()))
+ if (!doesHistoryAllowICP(CI, R->second->getName()))
return false;
const char *Reason = "Callee function not available";
@@ -987,17 +994,17 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
// clone the caller first, and inline the cloned caller if it is
// recursive. As llvm does not inline recursive calls, we will
// simply ignore it instead of handling it explicitly.
- if (!R->getValue()->isDeclaration() && R->getValue()->getSubprogram() &&
- R->getValue()->hasFnAttribute("use-sample-profile") &&
- R->getValue() != &F && isLegalToPromote(CI, R->getValue(), &Reason)) {
+ if (!R->second->isDeclaration() && R->second->getSubprogram() &&
+ R->second->hasFnAttribute("use-sample-profile") &&
+ R->second != &F && isLegalToPromote(CI, R->second, &Reason)) {
// For promoted target, set its value with NOMORE_ICP_MAGICNUM count
// in the value profile metadata so the target won't be promoted again.
SmallVector<InstrProfValueData, 1> SortedCallTargets = {InstrProfValueData{
- Function::getGUID(R->getValue()->getName()), NOMORE_ICP_MAGICNUM}};
+ Function::getGUID(R->second->getName()), NOMORE_ICP_MAGICNUM}};
updateIDTMetaData(CI, SortedCallTargets, 0);
auto *DI = &pgo::promoteIndirectCall(
- CI, R->getValue(), Candidate.CallsiteCount, Sum, false, ORE);
+ CI, R->second, Candidate.CallsiteCount, Sum, false, ORE);
if (DI) {
Sum -= Candidate.CallsiteCount;
// Do not prorate the indirect callsite distribution since the original
@@ -1026,7 +1033,8 @@ bool SampleProfileLoader::tryPromoteAndInlineCandidate(
}
} else {
LLVM_DEBUG(dbgs() << "\nFailed to promote indirect call to "
- << Candidate.CalleeSamples->getFuncName() << " because "
+ << FunctionSamples::getCanonicalFnName(
+ Candidate.CallInstr->getName())<< " because "
<< Reason << "\n");
}
return false;
@@ -1071,8 +1079,7 @@ void SampleProfileLoader::emitOptimizationRemarksForInlineCandidates(
void SampleProfileLoader::findExternalInlineCandidate(
CallBase *CB, const FunctionSamples *Samples,
- DenseSet<GlobalValue::GUID> &InlinedGUIDs,
- const StringMap<Function *> &SymbolMap, uint64_t Threshold) {
+ DenseSet<GlobalValue::GUID> &InlinedGUIDs, uint64_t Threshold) {
// If ExternalInlineAdvisor(ReplayInlineAdvisor) wants to inline an external
// function make sure it's imported
@@ -1081,7 +1088,7 @@ void SampleProfileLoader::findExternalInlineCandidate(
// just add the direct GUID and move on
if (!Samples) {
InlinedGUIDs.insert(
- FunctionSamples::getGUID(CB->getCalledFunction()->getName()));
+ Function::getGUID(CB->getCalledFunction()->getName()));
return;
}
// Otherwise, drop the threshold to import everything that we can
@@ -1122,22 +1129,20 @@ void SampleProfileLoader::findExternalInlineCandidate(
CalleeSample->getContext().hasAttribute(ContextShouldBeInlined);
if (!PreInline && CalleeSample->getHeadSamplesEstimate() < Threshold)
continue;
-
- StringRef Name = CalleeSample->getFuncName();
- Function *Func = SymbolMap.lookup(Name);
+
+ Function *Func = SymbolMap.lookup(CalleeSample->getFunction());
// Add to the import list only when it's defined out of module.
if (!Func || Func->isDeclaration())
- InlinedGUIDs.insert(FunctionSamples::getGUID(CalleeSample->getName()));
+ InlinedGUIDs.insert(CalleeSample->getGUID());
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : CalleeSample->getBodySamples())
for (const auto &TS : BS.second.getCallTargets())
- if (TS.getValue() > Threshold) {
- StringRef CalleeName = CalleeSample->getFuncName(TS.getKey());
- const Function *Callee = SymbolMap.lookup(CalleeName);
+ if (TS.second > Threshold) {
+ const Function *Callee = SymbolMap.lookup(TS.first);
if (!Callee || Callee->isDeclaration())
- InlinedGUIDs.insert(FunctionSamples::getGUID(TS.getKey()));
+ InlinedGUIDs.insert(TS.first.getHashCode());
}
// Import hot child context profile associted with callees. Note that this
@@ -1235,7 +1240,7 @@ bool SampleProfileLoader::inlineHotFunctions(
for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
uint64_t SumOrigin = Sum;
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1256,7 +1261,7 @@ bool SampleProfileLoader::inlineHotFunctions(
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
- InlinedGUIDs, SymbolMap,
+ InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
}
}
@@ -1505,7 +1510,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
for (const auto *FS : CalleeSamples) {
// TODO: Consider disable pre-lTO ICP for MonoLTO as well
if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
- findExternalInlineCandidate(I, FS, InlinedGUIDs, SymbolMap,
+ findExternalInlineCandidate(I, FS, InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
continue;
}
@@ -1558,7 +1563,7 @@ bool SampleProfileLoader::inlineHotFunctionsWithPriority(
}
} else if (LTOPhase == ThinOrFullLTOPhase::ThinLTOPreLink) {
findExternalInlineCandidate(I, findCalleeFunctionSamples(*I),
- InlinedGUIDs, SymbolMap,
+ InlinedGUIDs,
PSI->getOrCompHotCountThreshold());
}
}
@@ -1644,7 +1649,7 @@ GetSortedValueDataFromCallTargets(const SampleRecord::CallTargetMap &M) {
SmallVector<InstrProfValueData, 2> R;
for (const auto &I : SampleRecord::SortCallTargets(M)) {
R.emplace_back(
- InstrProfValueData{FunctionSamples::getGUID(I.first), I.second});
+ InstrProfValueData{I.first.getHashCode(), I.second});
}
return R;
}
@@ -1871,7 +1876,8 @@ SampleProfileLoader::buildProfiledCallGraph(Module &M) {
for (Function &F : M) {
if (F.isDeclaration() || !F.hasFnAttribute("use-sample-profile"))
continue;
- ProfiledCG->addProfiledFunction(FunctionSamples::getCanonicalFnName(F));
+ ProfiledCG->addProfiledFunction(
+ getRepInFormat(FunctionSamples::getCanonicalFnName(F)));
}
return ProfiledCG;
@@ -2022,8 +2028,16 @@ bool SampleProfileLoader::doInitialization(Module &M,
ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
if (ProfAccForSymsInList) {
NamesInProfile.clear();
- if (auto NameTable = Reader->getNameTable())
- NamesInProfile.insert(NameTable->begin(), NameTable->end());
+ GUIDsInProfile.clear();
+ if (auto NameTable = Reader->getNameTable()) {
+ if (FunctionSamples::UseMD5) {
+ for (auto Name : *NameTable)
+ GUIDsInProfile.insert(Name.getHashCode());
+ } else {
+ for (auto Name : *NameTable)
+ NamesInProfile.insert(Name.stringRef());
+ }
+ }
CoverageTracker.setProfAccForSymsInList(true);
}
@@ -2177,7 +2191,7 @@ void SampleProfileMatcher::findIRAnchors(
}
void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
- const auto *FuncDesc = ProbeManager->getDesc(FS.getName());
+ const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID());
// Skip the function that is external or renamed.
if (!FuncDesc)
return;
@@ -2194,7 +2208,8 @@ void SampleProfileMatcher::countMismatchedSamples(const FunctionSamples &FS) {
void SampleProfileMatcher::countProfileMismatches(
const Function &F, const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors) {
[[maybe_unused]] bool IsFuncHashMismatch = false;
if (FunctionSamples::ProfileIsProbeBased) {
TotalFuncHashSamples += FS.getTotalSamples();
@@ -2228,7 +2243,8 @@ void SampleProfileMatcher::countProfileMismatches(
void SampleProfileMatcher::countProfileCallsiteMismatches(
const FunctionSamples &FS,
const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
uint64_t &FuncMismatchedCallsites, uint64_t &FuncProfiledCallsites) {
// Check if there are any callsites in the profile that does not match to any
@@ -2263,7 +2279,7 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
// reported as mismatching.
if (IRCalleeName == UnknownIndirectCallee)
CallsiteIsMatched = true;
- else if (Callees.size() == 1 && Callees.count(IRCalleeName))
+ else if (Callees.size() == 1 && Callees.count(getRepInFormat(IRCalleeName)))
CallsiteIsMatched = true;
FuncProfiledCallsites++;
@@ -2275,9 +2291,8 @@ void SampleProfileMatcher::countProfileCallsiteMismatches(
}
}
-void SampleProfileMatcher::findProfileAnchors(
- const FunctionSamples &FS,
- std::map<LineLocation, StringSet<>> &ProfileAnchors) {
+void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,
+ std::map<LineLocation, std::unordered_set<FunctionId>> &ProfileAnchors) {
auto isInvalidLineOffset = [](uint32_t LineOffset) {
return LineOffset & 0x8000;
};
@@ -2287,8 +2302,9 @@ void SampleProfileMatcher::findProfileAnchors(
if (isInvalidLineOffset(Loc.LineOffset))
continue;
for (const auto &I : I.second.getCallTargets()) {
- auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
- Ret.first->second.insert(I.first());
+ auto Ret = ProfileAnchors.try_emplace(Loc,
+ std::unordered_set<FunctionId>());
+ Ret.first->second.insert(I.first);
}
}
@@ -2298,7 +2314,8 @@ void SampleProfileMatcher::findProfileAnchors(
continue;
const auto &CalleeMap = I.second;
for (const auto &I : CalleeMap) {
- auto Ret = ProfileAnchors.try_emplace(Loc, StringSet<>());
+ auto Ret = ProfileAnchors.try_emplace(Loc,
+ std::unordered_set<FunctionId>());
Ret.first->second.insert(I.first);
}
}
@@ -2322,21 +2339,24 @@ void SampleProfileMatcher::findProfileAnchors(
// [1, 2, 3(foo), 4, 7, 8(bar), 9]
// The output mapping: [2->3, 3->4, 5->7, 6->8, 7->9].
void SampleProfileMatcher::runStaleProfileMatching(
- const Function &F, const std::map<LineLocation, StringRef> &IRAnchors,
- const std::map<LineLocation, StringSet<>> &ProfileAnchors,
+ const Function &F,
+ const std::map<LineLocation, StringRef> &IRAnchors,
+ const std::map<LineLocation, std::unordered_set<FunctionId>>
+ &ProfileAnchors,
LocToLocMap &IRToProfileLocationMap) {
LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()
<< "\n");
assert(IRToProfileLocationMap.empty() &&
"Run stale profile matching only once per function");
- StringMap<std::set<LineLocation>> CalleeToCallsitesMap;
+ std::unordered_map<FunctionId, std::set<LineLocation>>
+ CalleeToCallsitesMap;
for (const auto &I : ProfileAnchors) {
const auto &Loc = I.first;
const auto &Callees = I.second;
// Filter out possible indirect calls, use direct callee name as anchor.
if (Callees.size() == 1) {
- StringRef CalleeName = Callees.begin()->first();
+ FunctionId CalleeName = *Callees.begin();
const auto &Candidates = CalleeToCallsitesMap.try_emplace(
CalleeName, std::set<LineLocation>());
Candidates.first->second.insert(Loc);
@@ -2355,11 +2375,12 @@ void SampleProfileMatcher::runStaleProfileMatching(
for (const auto &IR : IRAnchors) {
const auto &Loc = IR.first;
- StringRef CalleeName = IR.second;
+ auto CalleeName = IR.second;
bool IsMatchedAnchor = false;
// Match the anchor location in lexical order.
if (!CalleeName.empty()) {
- auto CandidateAnchors = CalleeToCallsitesMap.find(CalleeName);
+ auto CandidateAnchors = CalleeToCallsitesMap.find(
+ getRepInFormat(CalleeName));
if (CandidateAnchors != CalleeToCallsitesMap.end() &&
!CandidateAnchors->second.empty()) {
auto CI = CandidateAnchors->second.begin();
@@ -2420,7 +2441,7 @@ void SampleProfileMatcher::runOnFunction(const Function &F) {
findIRAnchors(F, IRAnchors);
// Anchors for profile. It's a map from callsite location to a set of callee
// name.
- std::map<LineLocation, StringSet<>> ProfileAnchors;
+ std::map<LineLocation, std::unordered_set<FunctionId>> ProfileAnchors;
findProfileAnchors(*FSFlattened, ProfileAnchors);
// Detect profile mismatch for profile staleness metrics report.
@@ -2499,7 +2520,7 @@ void SampleProfileMatcher::runOnModule() {
void SampleProfileMatcher::distributeIRToProfileLocationMap(
FunctionSamples &FS) {
- const auto ProfileMappings = FuncMappings.find(FS.getName());
+ const auto ProfileMappings = FuncMappings.find(FS.getFuncName());
if (ProfileMappings != FuncMappings.end()) {
FS.setIRToProfileLocationMap(&(ProfileMappings->second));
}
@@ -2541,10 +2562,10 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
Function *F = dyn_cast<Function>(N_F.getValue());
if (F == nullptr || OrigName.empty())
continue;
- SymbolMap[OrigName] = F;
+ SymbolMap[FunctionId(OrigName)] = F;
StringRef NewName = FunctionSamples::getCanonicalFnName(*F);
if (OrigName != NewName && !NewName.empty()) {
- auto r = SymbolMap.insert(std::make_pair(NewName, F));
+ auto r = SymbolMap.emplace(FunctionId(NewName), F);
// Failiing to insert means there is already an entry in SymbolMap,
// thus there are multiple functions that are mapped to the same
// stripped name. In this case of name conflicting, set the value
@@ -2557,11 +2578,11 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
if (Remapper) {
if (auto MapName = Remapper->lookUpNameInProfile(OrigName)) {
if (*MapName != OrigName && !MapName->empty())
- SymbolMap.insert(std::make_pair(*MapName, F));
+ SymbolMap.emplace(FunctionId(*MapName), F);
}
}
}
- assert(SymbolMap.count(StringRef()) == 0 &&
+ assert(SymbolMap.count(FunctionId()) == 0 &&
"No empty StringRef should be added in SymbolMap");
if (ReportProfileStaleness || PersistProfileStaleness ||
@@ -2625,7 +2646,9 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
// but not cold accumulatively...), so the outline function showing up as
// cold in sampled binary will actually not be cold after current build.
StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
- if (NamesInProfile.count(CanonName))
+ if ((FunctionSamples::UseMD5 &&
+ GUIDsInProfile.count(Function::getGUID(CanonName))) ||
+ (!FunctionSamples::UseMD5 && NamesInProfile.count(CanonName)))
initialEntryCount = -1;
}
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 18307935c7558b0..7d665a8005b0d62 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -199,6 +199,14 @@ class SymbolRemapper {
StringRef New = RemappingTable.lookup(Name);
return New.empty() ? Name : New;
}
+
+ FunctionId operator()(FunctionId Name) {
+ // MD5 name cannot be remapped.
+ if (!Name.isStringRef())
+ return Name;
+ StringRef New = RemappingTable.lookup(Name.stringRef());
+ return New.empty() ? Name : FunctionId(New);
+ }
};
}
@@ -709,14 +717,15 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
//
// Note that goo's count will remain in bar.cc:bar() as it does not have an
// entry in InstrProfile.
- DenseMap<StringRef, std::pair<uint64_t, uint64_t>> FlattenSampleMap;
+ llvm::StringMap<std::pair<uint64_t, uint64_t>> FlattenSampleMap;
auto BuildMaxSampleMap = [&FlattenSampleMap, &StaticFuncMap,
&InstrProfileMap](const FunctionSamples &FS,
const StringRef &RootName) {
auto BuildMaxSampleMapImpl = [&](const FunctionSamples &FS,
const StringRef &RootName,
auto &BuildImpl) -> void {
- const StringRef &Name = FS.getName();
+ std::string NameStr = FS.getFunction().str();
+ const StringRef Name = NameStr;
const StringRef *NewRootName = &RootName;
uint64_t EntrySample = FS.getHeadSamplesEstimate();
uint64_t MaxBodySample = FS.getMaxCountInside(/* SkipCallSite*/ true);
@@ -770,7 +779,8 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
for (auto &PD : Reader->getProfiles()) {
sampleprof::FunctionSamples &FS = PD.second;
- BuildMaxSampleMap(FS, FS.getName());
+ std::string Name = FS.getFunction().str();
+ BuildMaxSampleMap(FS, Name);
}
ProfileSummary InstrPS = *IPBuilder.getSummary();
@@ -806,7 +816,7 @@ adjustInstrProfile(std::unique_ptr<WriterContext> &WC,
uint64_t SampleMaxCount = std::max(E.second.first, E.second.second);
if (SampleMaxCount < ColdSampleThreshold)
continue;
- const StringRef &Name = E.first;
+ StringRef Name = E.first();
auto It = InstrProfileMap.find(Name);
if (It == InstrProfileMap.end()) {
auto NewName = StaticFuncMap.find(Name);
@@ -885,7 +895,7 @@ static sampleprof::FunctionSamples
remapSamples(const sampleprof::FunctionSamples &Samples,
SymbolRemapper &Remapper, sampleprof_error &Error) {
sampleprof::FunctionSamples Result;
- Result.setName(Remapper(Samples.getName()));
+ Result.setFunction(Remapper(Samples.getFunction()));
Result.addTotalSamples(Samples.getTotalSamples());
Result.addHeadSamples(Samples.getHeadSamples());
for (const auto &BodySample : Samples.getBodySamples()) {
@@ -896,7 +906,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
for (const auto &Target : BodySample.second.getCallTargets()) {
Result.addCalledTargetSamples(BodySample.first.LineOffset,
MaskedDiscriminator,
- Remapper(Target.first()), Target.second);
+ Remapper(Target.first), Target.second);
}
}
for (const auto &CallsiteSamples : Samples.getCallsiteSamples()) {
@@ -905,8 +915,7 @@ remapSamples(const sampleprof::FunctionSamples &Samples,
for (const auto &Callsite : CallsiteSamples.second) {
sampleprof::FunctionSamples Remapped =
remapSamples(Callsite.second, Remapper, Error);
- MergeResult(Error,
- Target[std::string(Remapped.getName())].merge(Remapped));
+ MergeResult(Error, Target[Remapped.getFunction()].merge(Remapped));
}
}
return Result;
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.cpp b/llvm/tools/llvm-profgen/CSPreInliner.cpp
index e891ea8df4907f1..025d3ca5a6da5ad 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.cpp
+++ b/llvm/tools/llvm-profgen/CSPreInliner.cpp
@@ -74,8 +74,8 @@ CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
ProfileInlineLimitMax = 50000;
}
-std::vector<StringRef> CSPreInliner::buildTopDownOrder() {
- std::vector<StringRef> Order;
+std::vector<FunctionId> CSPreInliner::buildTopDownOrder() {
+ std::vector<FunctionId> Order;
// Trim cold edges to get a more stable call graph. This allows for a more
// stable top-down order which in turns helps the stablity of the generated
// profile from run to run.
@@ -129,7 +129,7 @@ bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
LineLocation Callsite = CalleeNode->getCallSiteLoc();
if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
- auto It = TargetCounts.find(CalleeSamples->getName());
+ auto It = TargetCounts.find(CalleeSamples->getFunction());
if (It != TargetCounts.end())
CallsiteCount = It->second;
}
@@ -196,7 +196,7 @@ bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
return (Candidate.SizeCost < SampleThreshold);
}
-void CSPreInliner::processFunction(const StringRef Name) {
+void CSPreInliner::processFunction(const FunctionId Name) {
FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name);
if (!FSamples)
return;
@@ -297,7 +297,7 @@ void CSPreInliner::run() {
// It also helps better compress context profile to control profile
// size, as we now only need context profile for functions going to
// be inlined.
- for (StringRef FuncName : buildTopDownOrder()) {
+ for (FunctionId FuncName : buildTopDownOrder()) {
processFunction(FuncName);
}
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 4d848aafdab914d..8a3f16a4f13cb8d 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -57,8 +57,7 @@ struct ProfiledCandidateComparer {
// Tie breaker using GUID so we have stable/deterministic inlining order
assert(LHS.CalleeSamples && RHS.CalleeSamples &&
"Expect non-null FunctionSamples");
- return LHS.CalleeSamples->getGUID(LHS.CalleeSamples->getName()) <
- RHS.CalleeSamples->getGUID(RHS.CalleeSamples->getName());
+ return LHS.CalleeSamples->getGUID() < RHS.CalleeSamples->getGUID();
}
};
@@ -81,8 +80,8 @@ class CSPreInliner {
private:
bool getInlineCandidates(ProfiledCandidateQueue &CQueue,
const FunctionSamples *FCallerContextSamples);
- std::vector<StringRef> buildTopDownOrder();
- void processFunction(StringRef Name);
+ std::vector<FunctionId> buildTopDownOrder();
+ void processFunction(FunctionId Name);
bool shouldInline(ProfiledInlineCandidate &Candidate);
uint32_t getFuncSize(const ContextTrieNode *ContextNode);
bool UseContextCost;
diff --git a/llvm/tools/llvm-profgen/CallContext.h b/llvm/tools/llvm-profgen/CallContext.h
index 5e552130d03c757..ce9423f118dee98 100644
--- a/llvm/tools/llvm-profgen/CallContext.h
+++ b/llvm/tools/llvm-profgen/CallContext.h
@@ -18,7 +18,7 @@ namespace llvm {
namespace sampleprof {
inline std::string getCallSite(const SampleContextFrame &Callsite) {
- std::string CallsiteStr = Callsite.FuncName.str();
+ std::string CallsiteStr = Callsite.Func.str();
CallsiteStr += ":";
CallsiteStr += Twine(Callsite.Location.LineOffset).str();
if (Callsite.Location.Discriminator > 0) {
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index d307ab465676296..c4028e6b132871b 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -208,7 +208,7 @@ double ProfileGeneratorBase::calculateDensity(const SampleProfileMap &Profiles,
}
for (auto *FuncSamples : HotFuncs) {
- auto *Func = Binary->getBinaryFunction(FuncSamples->getName());
+ auto *Func = Binary->getBinaryFunction(FuncSamples->getFunction());
if (!Func)
continue;
uint64_t FuncSize = Func->getFuncSize();
@@ -449,7 +449,7 @@ bool ProfileGeneratorBase::collectFunctionsFromRawProfile(
bool ProfileGenerator::collectFunctionsFromLLVMProfile(
std::unordered_set<const BinaryFunction *> &ProfiledFunctions) {
for (const auto &FS : ProfileMap) {
- if (auto *Func = Binary->getBinaryFunction(FS.second.getName()))
+ if (auto *Func = Binary->getBinaryFunction(FS.second.getFunction()))
ProfiledFunctions.insert(Func);
}
return true;
@@ -466,7 +466,7 @@ bool CSProfileGenerator::collectFunctionsFromLLVMProfile(
}
FunctionSamples &
-ProfileGenerator::getTopLevelFunctionProfile(StringRef FuncName) {
+ProfileGenerator::getTopLevelFunctionProfile(FunctionId FuncName) {
SampleContext Context(FuncName);
return ProfileMap.Create(Context);
}
@@ -586,7 +586,7 @@ void ProfileGenerator::populateBoundarySamplesWithProbesForAllFunctions(
FunctionProfile.addCalledTargetSamples(
FrameVec.back().Location.LineOffset,
FrameVec.back().Location.Discriminator,
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
}
}
@@ -595,11 +595,11 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
const SampleContextFrameVector &FrameVec, uint64_t Count) {
// Get top level profile
FunctionSamples *FunctionProfile =
- &getTopLevelFunctionProfile(FrameVec[0].FuncName);
+ &getTopLevelFunctionProfile(FrameVec[0].Func);
FunctionProfile->addTotalSamples(Count);
if (Binary->usePseudoProbes()) {
const auto *FuncDesc = Binary->getFuncDescForGUID(
- Function::getGUID(FunctionProfile->getName()));
+ FunctionProfile->getFunction().getHashCode());
FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
}
@@ -610,16 +610,16 @@ FunctionSamples &ProfileGenerator::getLeafProfileAndAddTotalSamples(
FunctionSamplesMap &SamplesMap =
FunctionProfile->functionSamplesAt(Callsite);
auto Ret =
- SamplesMap.emplace(FrameVec[I].FuncName.str(), FunctionSamples());
+ SamplesMap.emplace(FrameVec[I].Func, FunctionSamples());
if (Ret.second) {
- SampleContext Context(FrameVec[I].FuncName);
+ SampleContext Context(FrameVec[I].Func);
Ret.first->second.setContext(Context);
}
FunctionProfile = &Ret.first->second;
FunctionProfile->addTotalSamples(Count);
if (Binary->usePseudoProbes()) {
const auto *FuncDesc = Binary->getFuncDescForGUID(
- Function::getGUID(FunctionProfile->getName()));
+ FunctionProfile->getFunction().getHashCode());
FunctionProfile->setFunctionHash(FuncDesc->FuncHash);
}
}
@@ -716,10 +716,11 @@ void ProfileGenerator::populateBoundarySamplesForAllFunctions(
FunctionProfile.addCalledTargetSamples(
FrameVec.back().Location.LineOffset,
getBaseDiscriminator(FrameVec.back().Location.Discriminator),
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
// Add head samples for callee.
- FunctionSamples &CalleeProfile = getTopLevelFunctionProfile(CalleeName);
+ FunctionSamples &CalleeProfile =
+ getTopLevelFunctionProfile(FunctionId(CalleeName));
CalleeProfile.addHeadSamples(Count);
}
}
@@ -737,7 +738,7 @@ CSProfileGenerator::getOrCreateFunctionSamples(ContextTrieNode *ContextNode,
if (!FProfile) {
FSamplesList.emplace_back();
FProfile = &FSamplesList.back();
- FProfile->setName(ContextNode->getFuncName());
+ FProfile->setFunction(ContextNode->getFuncName());
ContextNode->setFunctionSamples(FProfile);
}
// Update ContextWasInlined attribute for existing contexts.
@@ -899,7 +900,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
if (LeafLoc) {
CallerNode->getFunctionSamples()->addCalledTargetSamples(
LeafLoc->Location.LineOffset,
- getBaseDiscriminator(LeafLoc->Location.Discriminator), CalleeName,
+ getBaseDiscriminator(LeafLoc->Location.Discriminator),
+ FunctionId(CalleeName),
Count);
// Record head sample for called target(callee)
CalleeCallSite = LeafLoc->Location;
@@ -907,7 +909,8 @@ void CSProfileGenerator::populateBoundarySamplesForFunction(
}
ContextTrieNode *CalleeNode =
- CallerNode->getOrCreateChildContext(CalleeCallSite, CalleeName);
+ CallerNode->getOrCreateChildContext(CalleeCallSite,
+ FunctionId(CalleeName));
FunctionSamples *CalleeProfile = getOrCreateFunctionSamples(CalleeNode);
CalleeProfile->addHeadSamples(Count);
}
@@ -1212,7 +1215,7 @@ void CSProfileGenerator::populateBoundarySamplesWithProbes(
continue;
FunctionProfile.addCalledTargetSamples(CallProbe->getIndex(),
CallProbe->getDiscriminator(),
- CalleeName, Count);
+ FunctionId(CalleeName), Count);
}
}
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 471792ec713cd53..88cf2dc0d49f373 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -157,7 +157,7 @@ class ProfileGenerator : public ProfileGeneratorBase {
void generateLineNumBasedProfile();
void generateProbeBasedProfile();
RangeSample preprocessRangeCounter(const RangeSample &RangeCounter);
- FunctionSamples &getTopLevelFunctionProfile(StringRef FuncName);
+ FunctionSamples &getTopLevelFunctionProfile(FunctionId FuncName);
// Helper function to get the leaf frame's FunctionProfile by traversing the
// inline stack and meanwhile it adds the total samples for each frame's
// function profile.
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 85d7c1123fecaf6..6db25b5541b4581 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -77,7 +77,7 @@ void BinarySizeContextTracker::addInstructionForContext(
ContextTrieNode *CurNode = &RootContext;
bool IsLeaf = true;
for (const auto &Callsite : reverse(Context)) {
- StringRef CallerName = Callsite.FuncName;
+ FunctionId CallerName = Callsite.Func;
LineLocation CallsiteLoc = IsLeaf ? LineLocation(0, 0) : Callsite.Location;
CurNode = CurNode->getOrCreateChildContext(CallsiteLoc, CallerName);
IsLeaf = false;
@@ -145,7 +145,8 @@ void BinarySizeContextTracker::trackInlineesOptimizedAway(
StringRef CallerName = ProbeFrame.first;
LineLocation CallsiteLoc(ProbeFrame.second, 0);
SizeContext =
- SizeContext->getOrCreateChildContext(CallsiteLoc, CallerName);
+ SizeContext->getOrCreateChildContext(CallsiteLoc,
+ FunctionId(CallerName));
}
// Add 0 size to make known.
SizeContext->addFunctionSize(0);
@@ -838,6 +839,14 @@ void ProfiledBinary::loadSymbolsFromDWARF(ObjectFile &Obj) {
if (BinaryFunctions.empty())
WithColor::warning() << "Loading of DWARF info completed, but no binary "
"functions have been retrieved.\n";
+
+
+ // Populate the hash binary function map for MD5 function name lookup. This
+ // is done after BinaryFunctions are finalized.
+ for (auto &BinaryFunction : BinaryFunctions) {
+ HashBinaryFunctions[MD5Hash(StringRef(BinaryFunction.first))] =
+ &BinaryFunction.second;
+ }
}
void ProfiledBinary::populateSymbolListFromDWARF(
@@ -889,7 +898,7 @@ SampleContextFrameVector ProfiledBinary::symbolize(const InstructionPointer &IP,
LineLocation Line(LineOffset, Discriminator);
auto It = NameStrings.insert(FunctionName.str());
- CallStack.emplace_back(*It.first, Line);
+ CallStack.emplace_back(FunctionId(StringRef(*It.first)), Line);
}
return CallStack;
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index a6d78c661cc1c31..8c607d665dee81e 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -220,6 +220,10 @@ class ProfiledBinary {
// A map of mapping function name to BinaryFunction info.
std::unordered_map<std::string, BinaryFunction> BinaryFunctions;
+ // Lookup BinaryFunctions using the function name's MD5 hash. Needed if the
+ // profile is using MD5.
+ std::unordered_map<uint64_t, BinaryFunction *> HashBinaryFunctions;
+
// A list of binary functions that have samples.
std::unordered_set<const BinaryFunction *> ProfiledFunctions;
@@ -476,12 +480,18 @@ class ProfiledBinary {
void setProfiledFunctions(std::unordered_set<const BinaryFunction *> &Funcs) {
ProfiledFunctions = Funcs;
}
-
- BinaryFunction *getBinaryFunction(StringRef FName) {
- auto I = BinaryFunctions.find(FName.str());
- if (I == BinaryFunctions.end())
+
+ BinaryFunction *getBinaryFunction(FunctionId FName) {
+ if (FName.isStringRef()) {
+ auto I = BinaryFunctions.find(FName.str());
+ if (I == BinaryFunctions.end())
+ return nullptr;
+ return &I->second;
+ }
+ auto I = HashBinaryFunctions.find(FName.getHashCode());
+ if (I == HashBinaryFunctions.end())
return nullptr;
- return &I->second;
+ return I->second;
}
uint32_t getFuncSizeForContext(const ContextTrieNode *ContextNode) {
@@ -556,7 +566,7 @@ class ProfiledBinary {
InlineContextStack.clear();
continue;
}
- InlineContextStack.emplace_back(Callsite.first,
+ InlineContextStack.emplace_back(FunctionId(Callsite.first),
LineLocation(Callsite.second, 0));
}
}
diff --git a/llvm/unittests/ProfileData/SampleProfTest.cpp b/llvm/unittests/ProfileData/SampleProfTest.cpp
index e01025440e4a56a..3abba4743093258 100644
--- a/llvm/unittests/ProfileData/SampleProfTest.cpp
+++ b/llvm/unittests/ProfileData/SampleProfTest.cpp
@@ -145,7 +145,7 @@ struct SampleProfTest : ::testing::Test {
StringRef FooName("_Z3fooi");
FunctionSamples FooSamples;
- FooSamples.setName(FooName);
+ FooSamples.setFunction(FunctionId(FooName));
FooSamples.addTotalSamples(7711);
FooSamples.addHeadSamples(610);
FooSamples.addBodySamples(1, 0, 610);
@@ -155,43 +155,43 @@ struct SampleProfTest : ::testing::Test {
FooSamples.addBodySamples(10, 0, 605);
// Add inline instance with name "_Z3gooi".
- StringRef GooName("_Z3gooi");
+ FunctionId GooName(StringRef("_Z3gooi"));
auto &GooSamples =
- FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName.str()];
- GooSamples.setName(GooName);
+ FooSamples.functionSamplesAt(LineLocation(7, 0))[GooName];
+ GooSamples.setFunction(GooName);
GooSamples.addTotalSamples(502);
GooSamples.addBodySamples(3, 0, 502);
// Add inline instance with name "_Z3hooi".
- StringRef HooName("_Z3hooi");
+ FunctionId HooName(StringRef("_Z3hooi"));
auto &HooSamples =
- GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName.str()];
- HooSamples.setName(HooName);
+ GooSamples.functionSamplesAt(LineLocation(9, 0))[HooName];
+ HooSamples.setFunction(HooName);
HooSamples.addTotalSamples(317);
HooSamples.addBodySamples(4, 0, 317);
StringRef BarName("_Z3bari");
FunctionSamples BarSamples;
- BarSamples.setName(BarName);
+ BarSamples.setFunction(FunctionId(BarName));
BarSamples.addTotalSamples(20301);
BarSamples.addHeadSamples(1437);
BarSamples.addBodySamples(1, 0, 1437);
// Test how reader/writer handles unmangled names.
StringRef MconstructName("_M_construct<char *>");
StringRef StringviewName("string_view<std::allocator<char> >");
- BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000);
- BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437);
+ BarSamples.addCalledTargetSamples(1, 0, FunctionId(MconstructName), 1000);
+ BarSamples.addCalledTargetSamples(1, 0, FunctionId(StringviewName), 437);
StringRef BazName("_Z3bazi");
FunctionSamples BazSamples;
- BazSamples.setName(BazName);
+ BazSamples.setFunction(FunctionId(BazName));
BazSamples.addTotalSamples(12557);
BazSamples.addHeadSamples(1257);
BazSamples.addBodySamples(1, 0, 12557);
StringRef BooName("_Z3booi");
FunctionSamples BooSamples;
- BooSamples.setName(BooName);
+ BooSamples.setFunction(FunctionId(BooName));
BooSamples.addTotalSamples(1232);
BooSamples.addHeadSamples(1);
BooSamples.addBodySamples(1, 0, 1232);
@@ -210,8 +210,8 @@ struct SampleProfTest : ::testing::Test {
if (Remap) {
FooName = "_Z4fauxi";
BarName = "_Z3barl";
- GooName = "_Z3gool";
- HooName = "_Z3hool";
+ GooName = FunctionId(StringRef("_Z3gool"));
+ HooName = FunctionId(StringRef("_Z3hool"));
}
M.getOrInsertFunction(FooName, fn_type);
@@ -245,7 +245,7 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
ASSERT_TRUE(ReadFooSamples != nullptr);
if (!UseMD5) {
- ASSERT_EQ("_Z3fooi", ReadFooSamples->getName());
+ ASSERT_EQ("_Z3fooi", ReadFooSamples->getFunction().str());
}
ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples());
ASSERT_EQ(610u, ReadFooSamples->getHeadSamples());
@@ -254,7 +254,8 @@ struct SampleProfTest : ::testing::Test {
// inline instance for GooName. Test the correct FunctionSamples can be
// found with Remapper support.
const FunctionSamples *ReadGooSamples =
- ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0), GooName,
+ ReadFooSamples->findFunctionSamplesAt(LineLocation(7, 0),
+ GooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadGooSamples != nullptr);
ASSERT_EQ(502u, ReadGooSamples->getTotalSamples());
@@ -263,7 +264,8 @@ struct SampleProfTest : ::testing::Test {
// no inline instance for GooName. Test no FunctionSamples will be
// found with Remapper support.
const FunctionSamples *ReadGooSamplesAgain =
- ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0), GooName,
+ ReadFooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+ GooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadGooSamplesAgain == nullptr);
@@ -272,7 +274,8 @@ struct SampleProfTest : ::testing::Test {
// inline instance for HooName. Test the correct FunctionSamples can be
// found with Remapper support.
const FunctionSamples *ReadHooSamples =
- ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0), HooName,
+ ReadGooSamples->findFunctionSamplesAt(LineLocation(9, 0),
+ HooName.stringRef(),
Reader->getRemapper());
ASSERT_TRUE(ReadHooSamples != nullptr);
ASSERT_EQ(317u, ReadHooSamples->getTotalSamples());
@@ -280,7 +283,7 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName);
ASSERT_TRUE(ReadBarSamples != nullptr);
if (!UseMD5) {
- ASSERT_EQ("_Z3bari", ReadBarSamples->getName());
+ ASSERT_EQ("_Z3bari", ReadBarSamples->getFunction().str());
}
ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples());
ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples());
@@ -304,13 +307,9 @@ struct SampleProfTest : ::testing::Test {
FunctionSamples *ReadBooSamples = Reader->getSamplesFor(BooName);
ASSERT_TRUE(ReadBooSamples != nullptr);
ASSERT_EQ(1232u, ReadBooSamples->getTotalSamples());
-
- std::string MconstructGUID;
- StringRef MconstructRep =
- getRepInFormat(MconstructName, UseMD5, MconstructGUID);
- std::string StringviewGUID;
- StringRef StringviewRep =
- getRepInFormat(StringviewName, UseMD5, StringviewGUID);
+
+ FunctionId MconstructRep = getRepInFormat(MconstructName);
+ FunctionId StringviewRep = getRepInFormat(StringviewName);
ASSERT_EQ(1000u, CTMap.get()[MconstructRep]);
ASSERT_EQ(437u, CTMap.get()[StringviewRep]);
@@ -333,7 +332,7 @@ struct SampleProfTest : ::testing::Test {
uint64_t TotalSamples, uint64_t HeadSamples) {
StringRef Name(Fname);
FunctionSamples FcnSamples;
- FcnSamples.setName(Name);
+ FcnSamples.setFunction(FunctionId(Name));
FcnSamples.addTotalSamples(TotalSamples);
FcnSamples.addHeadSamples(HeadSamples);
FcnSamples.addBodySamples(1, 0, HeadSamples);
More information about the llvm-commits
mailing list