[lld] [lld][ELF] Extend profile guided function ordering to ELF binaries (PR #117514)

[via llvm-commits]

================
@@ -11,425 +11,41 @@
 #include "lld/Common/ErrorHandler.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
-#include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/BalancedPartitioning.h"
 #include "llvm/Support/TimeProfiler.h"
-#include "llvm/Support/VirtualFileSystem.h"
-#include "llvm/Support/xxhash.h"
 
-#define DEBUG_TYPE "bp-section-orderer"
 using namespace llvm;
 using namespace lld::macho;
 
-using UtilityNodes = SmallVector<BPFunctionNode::UtilityNodeT>;
-
-/// Symbols can be appended with "(.__uniq.xxxx)?.llvm.yyyy" where "xxxx" and
-/// "yyyy" are numbers that could change between builds. We need to use the root
-/// symbol name before this suffix so these symbols can be matched with profiles
-/// which may have different suffixes.
-static StringRef getRootSymbol(StringRef Name) {
-  auto [P0, S0] = Name.rsplit(".llvm.");
-  auto [P1, S1] = P0.rsplit(".__uniq.");
-  return P1;
-}
-
-static uint64_t getRelocHash(StringRef kind, uint64_t sectionIdx,
-                             uint64_t offset, uint64_t addend) {
-  return xxHash64((kind + ": " + Twine::utohexstr(sectionIdx) + " + " +
-                   Twine::utohexstr(offset) + " + " + Twine::utohexstr(addend))
-                      .str());
-}
-
-static uint64_t
-getRelocHash(const Reloc &reloc,
-             const DenseMap<const InputSection *, uint64_t> &sectionToIdx) {
-  auto *isec = reloc.getReferentInputSection();
-  std::optional<uint64_t> sectionIdx;
-  auto sectionIdxIt = sectionToIdx.find(isec);
-  if (sectionIdxIt != sectionToIdx.end())
-    sectionIdx = sectionIdxIt->getSecond();
-  std::string kind;
-  if (isec)
-    kind = ("Section " + Twine(static_cast<uint8_t>(isec->kind()))).str();
-  if (auto *sym = reloc.referent.dyn_cast<Symbol *>()) {
-    kind += (" Symbol " + Twine(static_cast<uint8_t>(sym->kind()))).str();
-    if (auto *d = dyn_cast<Defined>(sym))
-      return getRelocHash(kind, sectionIdx.value_or(0), d->value, reloc.addend);
-  }
-  return getRelocHash(kind, sectionIdx.value_or(0), 0, reloc.addend);
-}
-
-/// Given \p sectionIdxs, a list of section indexes, return a list of utility
-/// nodes for each section index. If \p duplicateSectionIdx is provided,
-/// populate it with nearly identical sections. Increment \p maxUN to be the
-/// largest utility node we have used so far.
-static SmallVector<std::pair<unsigned, UtilityNodes>> getUnsForCompression(
-    ArrayRef<const InputSection *> sections,
-    const DenseMap<const InputSection *, uint64_t> &sectionToIdx,
-    ArrayRef<unsigned> sectionIdxs,
-    DenseMap<unsigned, SmallVector<unsigned>> *duplicateSectionIdxs,
-    BPFunctionNode::UtilityNodeT &maxUN) {
-  TimeTraceScope timeScope("Build nodes for compression");
-
-  SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> sectionHashes;
-  sectionHashes.reserve(sectionIdxs.size());
-  SmallVector<uint64_t> hashes;
-  for (unsigned sectionIdx : sectionIdxs) {
-    const auto *isec = sections[sectionIdx];
-    constexpr unsigned windowSize = 4;
-
-    for (size_t i = 0; i < isec->data.size(); i++) {
-      auto window = isec->data.drop_front(i).take_front(windowSize);
-      hashes.push_back(xxHash64(window));
-    }
-    for (const auto &r : isec->relocs) {
-      if (r.length == 0 || r.referent.isNull() || r.offset >= isec->data.size())
-        continue;
-      uint64_t relocHash = getRelocHash(r, sectionToIdx);
-      uint32_t start = (r.offset < windowSize) ? 0 : r.offset - windowSize + 1;
-      for (uint32_t i = start; i < r.offset + r.length; i++) {
-        auto window = isec->data.drop_front(i).take_front(windowSize);
-        hashes.push_back(xxHash64(window) + relocHash);
-      }
-    }
-
-    llvm::sort(hashes);
-    hashes.erase(std::unique(hashes.begin(), hashes.end()), hashes.end());
-
-    sectionHashes.emplace_back(sectionIdx, hashes);
-    hashes.clear();
-  }
-
-  DenseMap<uint64_t, unsigned> hashFrequency;
-  for (auto &[sectionIdx, hashes] : sectionHashes)
-    for (auto hash : hashes)
-      ++hashFrequency[hash];
-
-  if (duplicateSectionIdxs) {
-    // Merge section that are nearly identical
-    SmallVector<std::pair<unsigned, SmallVector<uint64_t>>> newSectionHashes;
-    DenseMap<uint64_t, unsigned> wholeHashToSectionIdx;
-    for (auto &[sectionIdx, hashes] : sectionHashes) {
-      uint64_t wholeHash = 0;
-      for (auto hash : hashes)
-        if (hashFrequency[hash] > 5)
-          wholeHash ^= hash;
-      auto [it, wasInserted] =
-          wholeHashToSectionIdx.insert(std::make_pair(wholeHash, sectionIdx));
-      if (wasInserted) {
-        newSectionHashes.emplace_back(sectionIdx, hashes);
-      } else {
-        (*duplicateSectionIdxs)[it->getSecond()].push_back(sectionIdx);
-      }
-    }
-    sectionHashes = newSectionHashes;
-
-    // Recompute hash frequencies
-    hashFrequency.clear();
-    for (auto &[sectionIdx, hashes] : sectionHashes)
-      for (auto hash : hashes)
-        ++hashFrequency[hash];
-  }
-
-  // Filter rare and common hashes and assign each a unique utility node that
-  // doesn't conflict with the trace utility nodes
-  DenseMap<uint64_t, BPFunctionNode::UtilityNodeT> hashToUN;
-  for (auto &[hash, frequency] : hashFrequency) {
-    if (frequency <= 1 || frequency * 2 > sectionHashes.size())
-      continue;
-    hashToUN[hash] = ++maxUN;
-  }
-
-  SmallVector<std::pair<unsigned, UtilityNodes>> sectionUns;
-  for (auto &[sectionIdx, hashes] : sectionHashes) {
-    UtilityNodes uns;
-    for (auto &hash : hashes) {
-      auto it = hashToUN.find(hash);
-      if (it != hashToUN.end())
-        uns.push_back(it->second);
-    }
-    sectionUns.emplace_back(sectionIdx, uns);
-  }
-  return sectionUns;
-}
-
 DenseMap<const InputSection *, size_t> lld::macho::runBalancedPartitioning(
     size_t &highestAvailablePriority, StringRef profilePath,
     bool forFunctionCompression, bool forDataCompression,
     bool compressionSortStartupFunctions, bool verbose) {
 
-  SmallVector<const InputSection *> sections;
-  DenseMap<const InputSection *, uint64_t> sectionToIdx;
-  StringMap<DenseSet<unsigned>> symbolToSectionIdxs;
+  SmallVector<BPSectionBase *> sections;
----------------
Colibrow wrote:

I'm not sure how to pass the sections into the `reorderSectionsByBalancedPartitioning` function which locates in lld/common since the BPSectionELF and BPSectionMacho both need to be accepted. now I use the `SmallVector<std::unique_ptr<BPSectionBase>>` and `SmallVector<std::unique_ptr<BPSectionMacho>>`. Could you give some more details?

https://github.com/llvm/llvm-project/pull/117514