[PATCH] D36351: [lld][ELF] Add profile guided section layout
Rafael Avila de Espindola via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 8 09:50:33 PST 2018
I now get this warning:
/home/espindola/llvm/llvm/tools/lld/ELF/CallGraphSort.cpp:204:28:
warning: missing field 'NormalizedWeight' initializer
[-Wmissing-field-initializers]
Edge E{From, To, Weight};
And git-clang-format still produces this diff:
- if (A.getDensity() >
- NewDensity * Double(MAX_DENSITY_DEGRADATION))
+ if (A.getDensity() > NewDensity * Double(MAX_DENSITY_DEGRADATION))
Cheers,
Rafael
Michael Spencer via Phabricator via llvm-commits
<llvm-commits at lists.llvm.org> writes:
> Bigcheese updated this revision to Diff 137539.
> Bigcheese added a comment.
>
> Address review comments.
>
>
> https://reviews.llvm.org/D36351
>
> Files:
> ELF/CMakeLists.txt
> ELF/CallGraphSort.cpp
> ELF/CallGraphSort.h
> ELF/Config.h
> ELF/Driver.cpp
> ELF/Options.td
> ELF/Writer.cpp
> test/ELF/cgprofile-txt.s
>
> Index: test/ELF/cgprofile-txt.s
> ===================================================================
> --- /dev/null
> +++ test/ELF/cgprofile-txt.s
> @@ -0,0 +1,106 @@
> +# REQUIRES: x86
> +
> +# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
> +# RUN: ld.lld -e A %t -o %t2
> +# RUN: llvm-readobj -symbols %t2 | FileCheck %s --check-prefix=NOSORT
> +
> +# RUN: echo "A B 100" > %t.call_graph
> +# RUN: echo "A C 40" >> %t.call_graph
> +# RUN: echo "B C 30" >> %t.call_graph
> +# RUN: echo "C D 90" >> %t.call_graph
> +# RUN: echo "PP TS 100" >> %t.call_graph
> +# RUN: echo "_init2 _init 24567837" >> %t.call_graph
> +# RUN: echo "TS QC 9001" >> %t.call_graph
> +# RUN: ld.lld -e A %t --call-graph-ordering-file %t.call_graph -o %t2
> +# RUN: llvm-readobj -symbols %t2 | FileCheck %s
> +
> + .section .text.D,"ax", at progbits
> +D:
> + retq
> +
> + .section .text.C,"ax", at progbits
> + .globl C
> +C:
> + retq
> +
> + .section .text.B,"ax", at progbits
> + .globl B
> +B:
> + retq
> +
> + .section .text.A,"ax", at progbits
> + .globl A
> +A:
> + retq
> +
> + .section .ponies,"ax", at progbits,unique,1
> + .globl TS
> +TS:
> + retq
> +
> + .section .ponies,"ax", at progbits,unique,2
> + .globl PP
> +PP:
> + retq
> +
> + .section .other,"ax", at progbits,unique,1
> + .globl QC
> +QC:
> + retq
> +
> + .section .other,"ax", at progbits,unique,2
> + .globl GB
> +GB:
> + retq
> +
> + .section .init,"ax", at progbits,unique,1
> + .globl _init
> +_init:
> + retq
> +
> + .section .init,"ax", at progbits,unique,2
> + .globl _init2
> +_init2:
> + retq
> +
> +# CHECK: Name: D
> +# CHECK-NEXT: Value: 0x201003
> +# CHECK: Name: A
> +# CHECK-NEXT: Value: 0x201000
> +# CHECK: Name: B
> +# CHECK-NEXT: Value: 0x201001
> +# CHECK: Name: C
> +# CHECK-NEXT: Value: 0x201002
> +# CHECK: Name: GB
> +# CHECK-NEXT: Value: 0x201007
> +# CHECK: Name: PP
> +# CHECK-NEXT: Value: 0x201004
> +# CHECK: Name: QC
> +# CHECK-NEXT: Value: 0x201006
> +# CHECK: Name: TS
> +# CHECK-NEXT: Value: 0x201005
> +# CHECK: Name: _init
> +# CHECK-NEXT: Value: 0x201008
> +# CHECK: Name: _init2
> +# CHECK-NEXT: Value: 0x201009
> +
> +# NOSORT: Name: D
> +# NOSORT-NEXT: Value: 0x201000
> +# NOSORT: Name: A
> +# NOSORT-NEXT: Value: 0x201003
> +# NOSORT: Name: B
> +# NOSORT-NEXT: Value: 0x201002
> +# NOSORT: Name: C
> +# NOSORT-NEXT: Value: 0x201001
> +# NOSORT: Name: GB
> +# NOSORT-NEXT: Value: 0x201007
> +# NOSORT: Name: PP
> +# NOSORT-NEXT: Value: 0x201005
> +# NOSORT: Name: QC
> +# NOSORT-NEXT: Value: 0x201006
> +# NOSORT: Name: TS
> +# NOSORT-NEXT: Value: 0x201004
> +# NOSORT: Name: _init
> +# NOSORT-NEXT: Value: 0x201008
> +# NOSORT: Name: _init2
> +# NOSORT-NEXT: Value: 0x201009
> Index: ELF/Writer.cpp
> ===================================================================
> --- ELF/Writer.cpp
> +++ ELF/Writer.cpp
> @@ -9,6 +9,7 @@
>
> #include "Writer.h"
> #include "AArch64ErrataFix.h"
> +#include "CallGraphSort.h"
> #include "Config.h"
> #include "Filesystem.h"
> #include "LinkerScript.h"
> @@ -1020,6 +1021,10 @@
> // Builds section order for handling --symbol-ordering-file.
> static DenseMap<const InputSectionBase *, int> buildSectionOrder() {
> DenseMap<const InputSectionBase *, int> SectionOrder;
> + // Use the rarely used option -call-graph-ordering-file to sort sections.
> + if (!Config->CallGraphProfile.empty())
> + return computeCallGraphProfileOrder();
> +
> if (Config->SymbolOrderingFile.empty())
> return SectionOrder;
>
> Index: ELF/Options.td
> ===================================================================
> --- ELF/Options.td
> +++ ELF/Options.td
> @@ -66,6 +66,9 @@
> "Only set DT_NEEDED for shared libraries if used",
> "Always set DT_NEEDED for shared libraries">;
>
> +defm call_graph_ordering_file: Eq<"call-graph-ordering-file">,
> + HelpText<"Layout sections to optimize the given callgraph">;
> +
> // -chroot doesn't have a help text because it is an internal option.
> defm chroot: Eq<"chroot">;
>
> Index: ELF/Driver.cpp
> ===================================================================
> --- ELF/Driver.cpp
> +++ ELF/Driver.cpp
> @@ -571,6 +571,31 @@
> return {BuildIdKind::None, {}};
> }
>
> +static void readCallGraph(MemoryBufferRef MB) {
> + // Build a map from symbol name to section
> + DenseMap<StringRef, InputSectionBase *> SymbolSection;
> + for (InputFile *File : ObjectFiles)
> + for (Symbol *Sym : File->getSymbols())
> + if (auto *D = dyn_cast<Defined>(Sym))
> + if (auto *IS = dyn_cast_or_null<InputSectionBase>(D->Section))
> + SymbolSection[D->getName()] = IS;
> +
> + std::vector<StringRef> Lines = args::getLines(MB);
> + for (StringRef L : Lines) {
> + SmallVector<StringRef, 3> Fields;
> + L.split(Fields, ' ');
> + if (Fields.size() != 3)
> + fatal("parse error");
> + uint64_t Count;
> + if (!to_integer(Fields[2], Count))
> + fatal("parse error");
> + InputSectionBase *FromSec = SymbolSection.lookup(Fields[0]);
> + InputSectionBase *ToSec = SymbolSection.lookup(Fields[1]);
> + if (FromSec && ToSec)
> + Config->CallGraphProfile[std::make_pair(FromSec, ToSec)] += Count;
> + }
> +}
> +
> static bool getCompressDebugSections(opt::InputArgList &Args) {
> StringRef S = Args.getLastArgValue(OPT_compress_debug_sections, "none");
> if (S == "none")
> @@ -1124,6 +1149,10 @@
> // Apply symbol renames for -wrap.
> Symtab->applySymbolWrap();
>
> + if (auto *Arg = Args.getLastArg(OPT_call_graph_ordering_file))
> + if (Optional<MemoryBufferRef> Buffer = readFile(Arg->getValue()))
> + readCallGraph(*Buffer);
> +
> // Now that we have a complete list of input files.
> // Beyond this point, no new files are added.
> // Aggregate all input sections into one place.
> Index: ELF/Config.h
> ===================================================================
> --- ELF/Config.h
> +++ ELF/Config.h
> @@ -24,6 +24,7 @@
> namespace elf {
>
> class InputFile;
> +class InputSectionBase;
>
> enum ELFKind {
> ELFNoneKind,
> @@ -103,6 +104,9 @@
> std::vector<SymbolVersion> VersionScriptGlobals;
> std::vector<SymbolVersion> VersionScriptLocals;
> std::vector<uint8_t> BuildIdVector;
> + llvm::MapVector<std::pair<const InputSectionBase *, const InputSectionBase *>,
> + uint64_t>
> + CallGraphProfile;
> bool AllowMultipleDefinition;
> bool AndroidPackDynRelocs = false;
> bool ARMHasBlx = false;
> Index: ELF/CallGraphSort.h
> ===================================================================
> --- /dev/null
> +++ ELF/CallGraphSort.h
> @@ -0,0 +1,23 @@
> +//===- CallGraphSort.h ------------------------------------------*- C++ -*-===//
> +//
> +// The LLVM Linker
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +
> +#ifndef LLD_ELF_CALL_GRAPH_SORT_H
> +#define LLD_ELF_CALL_GRAPH_SORT_H
> +
> +#include "llvm/ADT/DenseMap.h"
> +
> +namespace lld {
> +namespace elf {
> +class InputSectionBase;
> +
> +llvm::DenseMap<const InputSectionBase *, int> computeCallGraphProfileOrder();
> +} // namespace elf
> +} // namespace lld
> +
> +#endif
> Index: ELF/CallGraphSort.cpp
> ===================================================================
> --- /dev/null
> +++ ELF/CallGraphSort.cpp
> @@ -0,0 +1,350 @@
> +//===- CallGraphSort.cpp --------------------------------------------------===//
> +//
> +// The LLVM Linker
> +//
> +// This file is distributed under the University of Illinois Open Source
> +// License. See LICENSE.TXT for details.
> +//
> +//===----------------------------------------------------------------------===//
> +///
> +/// Implementation of Call-Chain Clustering from: Optimizing Function Placement
> +/// for Large-Scale Data-Center Applications
> +/// https://research.fb.com/wp-content/uploads/2017/01/cgo2017-hfsort-final1.pdf
> +///
> +/// The goal of this algorithm is to improve runtime performance of the final
> +/// executable by arranging code sections such that page table and i-cache
> +/// misses are minimized.
> +///
> +/// Definitions:
> +/// * Cluster
> +/// * An ordered list of input sections which are layed out as a unit. At the
> +/// beginning of the algorithm each input section has its own cluster and
> +/// the weight of the cluster is the sum of the weight of all incomming
> +/// edges.
> +/// * Call-Chain Clustering (C³) Heuristic
> +/// * Defines when and how clusters are combined. Pick the highest weighted
> +/// input section then add it to its most likely predecessor if it wouldn't
> +/// penalize it too much.
> +/// * Density
> +/// * The weight of the cluster divided by the size of the cluster. This is a
> +/// proxy for the ammount of execution time spent per byte of the cluster.
> +///
> +/// It does so given a call graph profile by the following:
> +/// * Build a weighted call graph from the profile
> +/// * Sort input sections by weight
> +/// * For each input section starting with the highest weight
> +/// * Find its most likely predecessor cluster
> +/// * Check if the combined cluster would be too large, or would have too low
> +/// a density.
> +/// * If not, then combine the clusters.
> +/// * Sort non-empty clusters by density
> +///
> +//===----------------------------------------------------------------------===//
> +
> +#include "CallGraphSort.h"
> +#include "OutputSections.h"
> +#include "SymbolTable.h"
> +#include "Symbols.h"
> +
> +#include "llvm/Support/MathExtras.h"
> +
> +using namespace llvm;
> +using namespace lld;
> +using namespace lld::elf;
> +
> +namespace {
> +using ClusterIndex = std::ptrdiff_t;
> +using SectionIndex = std::ptrdiff_t;
> +using EdgeIndex = std::ptrdiff_t;
> +
> +// Used for calculating an comparing density. Use soft-float for determinism.
> +struct Double : APFloat {
> + Double() : APFloat(APFloat::IEEEdouble(), 0) {}
> + Double(uint64_t Val) : APFloat(APFloat::IEEEdouble(), Val) {}
> + Double(APFloat A) : APFloat(A) {}
> + bool operator>(const Double Other) const {
> + return compare(Other) == cmpGreaterThan;
> + }
> + bool operator<(const Double Other) const {
> + return compare(Other) == cmpLessThan;
> + }
> +};
> +
> +struct Cluster {
> + Cluster(SectionIndex Sec, const InputSectionBase *IS);
> +
> + Double getDensity() const {
> + if (Size == 0)
> + return 0;
> + return Double(Weight) / Double(Size);
> + }
> +
> + std::vector<const InputSectionBase *> ISBs;
> + std::vector<SectionIndex> Sections;
> + size_t Size = 0;
> + uint64_t Weight = 0;
> +};
> +
> +struct Section {
> + Section(const InputSectionBase *IS) : ISB(IS) { Size = ISB->getSize(); }
> +
> + Double getDensity() const {
> + if (Size == 0)
> + return 0;
> + return Double(Weight) / Double(Size);
> + }
> +
> + size_t Size = 0;
> + uint64_t Weight = 0;
> + const InputSectionBase *ISB;
> + std::vector<SectionIndex> Preds;
> + std::vector<SectionIndex> Succs;
> +};
> +
> +struct Edge {
> + SectionIndex From;
> + SectionIndex To;
> + uint64_t Weight;
> + Double NormalizedWeight;
> +
> + bool operator==(const Edge Other) const;
> +};
> +
> +struct EdgeDenseMapInfo {
> + static Edge getEmptyKey() {
> + return {DenseMapInfo<SectionIndex>::getEmptyKey(),
> + DenseMapInfo<SectionIndex>::getEmptyKey(), 0, 0};
> + }
> + static Edge getTombstoneKey() {
> + return {DenseMapInfo<SectionIndex>::getTombstoneKey(),
> + DenseMapInfo<SectionIndex>::getTombstoneKey(), 0, 0};
> + }
> + static unsigned getHashValue(const Edge &Val) {
> + return hash_combine(DenseMapInfo<SectionIndex>::getHashValue(Val.From),
> + DenseMapInfo<SectionIndex>::getHashValue(Val.To));
> + }
> + static bool isEqual(const Edge &LHS, const Edge &RHS) { return LHS == RHS; }
> +};
> +
> +class CallGraphSort {
> +public:
> + CallGraphSort();
> +
> + DenseMap<const InputSectionBase *, int> run();
> +
> +private:
> + DenseMap<Edge, EdgeIndex, EdgeDenseMapInfo> EdgeMap;
> + std::vector<Cluster> Clusters;
> + std::vector<Edge> Edges;
> + std::vector<Section> Sections;
> +
> + void normalizeEdgeWeights();
> + void generateClusters();
> +};
> +
> +// Maximum ammount the combined cluster density can be worse than the original
> +// cluster to consider merging.
> +constexpr int MAX_DENSITY_DEGRADATION = 8;
> +
> +// Maximum cluster size in bytes.
> +constexpr uint64_t MAX_CLUSTER_SIZE = 1024 * 1024;
> +} // end anonymous namespace
> +
> +Cluster::Cluster(SectionIndex Sec, const InputSectionBase *IS) {
> + ISBs.push_back(IS);
> + Sections.push_back(Sec);
> + Size = IS->getSize();
> +}
> +
> +bool Edge::operator==(const Edge Other) const {
> + return From == Other.From && To == Other.To;
> +}
> +
> +// Take the edge list in Config->CallGraphProfile, resolve symbol names to
> +// Symbols, and generate a graph between InputSections with the provided
> +// weights.
> +CallGraphSort::CallGraphSort() {
> + MapVector<std::pair<const InputSectionBase *, const InputSectionBase *>,
> + uint64_t> &Profile = Config->CallGraphProfile;
> + DenseMap<const InputSectionBase *, SectionIndex> SecToSec;
> +
> + auto GetOrCreateNode = [&](const InputSectionBase *IS) -> SectionIndex {
> + auto Res = SecToSec.insert(std::make_pair(IS, Sections.size()));
> + if (Res.second)
> + Sections.emplace_back(IS);
> + return Res.first->second;
> + };
> +
> + // Create the graph.
> + for (const auto &C : Profile) {
> + const InputSectionBase *FromSB = C.first.first;
> + const InputSectionBase *ToSB = C.first.second;
> + uint64_t Weight = C.second;
> +
> + if (Weight == 0)
> + continue;
> +
> + // Ignore edges between input sections belonging to different output
> + // sections. This is done because otherwise we would end up with clusters
> + // containing input sections that can't actually be placed adjacently in the
> + // output. This messes with the cluster size and density calculations. We
> + // would also end up moving input sections in other output sections without
> + // moving them closer to what calls them.
> + if (FromSB->getOutputSection() != ToSB->getOutputSection())
> + continue;
> +
> + SectionIndex From = GetOrCreateNode(FromSB);
> + SectionIndex To = GetOrCreateNode(ToSB);
> +
> + Sections[To].Weight = SaturatingAdd(Sections[To].Weight, Weight);
> +
> + if (From == To)
> + continue;
> +
> + Edge E{From, To, Weight};
> +
> + // Add or increment an edge
> + auto Res = EdgeMap.insert(std::make_pair(E, Edges.size()));
> + EdgeIndex EI = Res.first->second;
> + if (Res.second) {
> + Edges.push_back(E);
> + Sections[From].Succs.push_back(To);
> + Sections[To].Preds.push_back(From);
> + } else
> + Edges[EI].Weight = SaturatingAdd(Edges[EI].Weight, Weight);
> + }
> + normalizeEdgeWeights();
> +}
> +
> +// Normalize the edge weights so that we can reject edges which have a low
> +// probibility.
> +void CallGraphSort::normalizeEdgeWeights() {
> + for (SectionIndex SI = 0, SE = Sections.size(); SI != SE; ++SI) {
> + Section &S = Sections[SI];
> + for (SectionIndex PI : S.Preds) {
> + Edge &E = Edges[EdgeMap[{PI, SI, 0, 0}]];
> + if (S.Weight == 0)
> + continue;
> + E.NormalizedWeight = Double(E.Weight) / Double(S.Weight);
> + }
> + }
> +}
> +
> +// It's bad to merge clusters which would degrade the density too much.
> +static bool isNewDensityBad(Cluster &A, Cluster &B) {
> + Double NewDensity = Double(A.Weight + B.Weight) / Double(A.Size + B.Size);
> + if (A.getDensity() >
> + NewDensity * Double(MAX_DENSITY_DEGRADATION))
> + return true;
> + return false;
> +}
> +
> +static void mergeClusters(Cluster &Into, Cluster &From) {
> + Into.ISBs.insert(Into.ISBs.end(), From.ISBs.begin(), From.ISBs.end());
> + Into.Sections.insert(Into.Sections.end(), From.Sections.begin(),
> + From.Sections.end());
> + Into.Size += From.Size;
> + Into.Weight += From.Weight;
> + From.ISBs.clear();
> + From.Sections.clear();
> + From.Size = 0;
> + From.Weight = 0;
> +}
> +
> +// Group InputSections into clusters using the Call-Chain Clustering heuristic
> +// then sort the clusters by density.
> +void CallGraphSort::generateClusters() {
> + // Minimum edge probability to consider merging.
> + const Double MIN_EDGE_PROBABILITY = Double(1) / Double(10);
> +
> + std::vector<SectionIndex> SortedSecs;
> + std::vector<Cluster *> SecToCluster(Sections.size());
> +
> + Clusters.reserve(Sections.size());
> +
> + for (SectionIndex SI = 0, SE = Sections.size(); SI != SE; ++SI) {
> + Cluster C(SI, Sections[SI].ISB);
> + C.Size = Sections[SI].Size;
> + C.Weight = Sections[SI].Weight;
> + Clusters.push_back(C);
> + SortedSecs.push_back(SI);
> + }
> +
> + for (Cluster &C : Clusters) {
> + SecToCluster[C.Sections.front()] = &C;
> + }
> +
> + std::stable_sort(SortedSecs.begin(), SortedSecs.end(),
> + [&](SectionIndex A, SectionIndex B) {
> + return Sections[A].getDensity() > Sections[B].getDensity();
> + });
> +
> + for (SectionIndex SI : SortedSecs) {
> + Cluster &C = *SecToCluster[SI];
> +
> + SectionIndex BestPred = -1;
> + Double BestWeight = 0;
> +
> + for (SectionIndex PI : Sections[SI].Preds) {
> + Edge &E = Edges[EdgeMap[{PI, SI, 0, 0}]];
> + if (BestPred == -1 || E.NormalizedWeight > BestWeight) {
> + BestPred = PI;
> + BestWeight = E.NormalizedWeight;
> + }
> + }
> +
> + if (BestWeight < MIN_EDGE_PROBABILITY)
> + continue;
> +
> + Cluster *PredC = SecToCluster[BestPred];
> + if (PredC == nullptr || PredC == &C)
> + continue;
> +
> + if (C.Size + PredC->Size > MAX_CLUSTER_SIZE)
> + continue;
> +
> + if (isNewDensityBad(*PredC, C))
> + continue;
> +
> + for (SectionIndex SI : C.Sections)
> + SecToCluster[SI] = PredC;
> +
> + mergeClusters(*PredC, C);
> + }
> +
> + // Remove empty or dead nodes.
> + Clusters.erase(std::remove_if(Clusters.begin(), Clusters.end(),
> + [](const Cluster &C) {
> + return C.Size == 0 || C.Sections.empty();
> + }),
> + Clusters.end());
> +
> + // Sort by density. Invalidates all NodeIndexs.
> + std::sort(Clusters.begin(), Clusters.end(),
> + [](const Cluster &A, const Cluster &B) {
> + return A.getDensity() > B.getDensity();
> + });
> +}
> +
> +DenseMap<const InputSectionBase *, int> CallGraphSort::run() {
> + generateClusters();
> +
> + // Generate order.
> + llvm::DenseMap<const InputSectionBase *, int> OrderMap;
> + ssize_t CurOrder = 1;
> +
> + for (const Cluster &C : Clusters)
> + for (const InputSectionBase *IS : C.ISBs)
> + OrderMap[IS] = CurOrder++;
> +
> + return OrderMap;
> +}
> +
> +// Sort sections by the profile data provided by -callgraph-profile-file
> +//
> +// This first builds a call graph based on the profile data then merges sections
> +// according to the C³ huristic. All clusters are then sorted by a density
> +// metric to further improve locality.
> +DenseMap<const InputSectionBase *, int> elf::computeCallGraphProfileOrder() {
> + return CallGraphSort().run();
> +}
> Index: ELF/CMakeLists.txt
> ===================================================================
> --- ELF/CMakeLists.txt
> +++ ELF/CMakeLists.txt
> @@ -19,6 +19,7 @@
> Arch/SPARCV9.cpp
> Arch/X86.cpp
> Arch/X86_64.cpp
> + CallGraphSort.cpp
> Driver.cpp
> DriverUtils.cpp
> EhFrame.cpp
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits
More information about the llvm-commits
mailing list