[llvm] 9849291 - [PseudoProbe] Encode/Decode FS discriminator
Hongtao Yu via llvm-commits
llvm-commits at lists.llvm.org
Wed May 10 11:34:06 PDT 2023
Author: Hongtao Yu
Date: 2023-05-10T11:27:54-07:00
New Revision: 9849291dccee91001dbd0473943f6958413c74af
URL: https://github.com/llvm/llvm-project/commit/9849291dccee91001dbd0473943f6958413c74af
DIFF: https://github.com/llvm/llvm-project/commit/9849291dccee91001dbd0473943f6958413c74af.diff
LOG: [PseudoProbe] Encode/Decode FS discriminator
Encoding FS discriminators for block probes. Decoding them correspondingly.
The encoding/decoding of FS discriminators are conditional, only for probes with a non-zero discriminator. This saves encoding size, also ensures downwards-compatiblity.
Reviewed By: wenlei
Differential Revision: https://reviews.llvm.org/D147651
Added:
llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin
llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test
Modified:
llvm/include/llvm/IR/PseudoProbe.h
llvm/include/llvm/MC/MCPseudoProbe.h
llvm/include/llvm/MC/MCStreamer.h
llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
llvm/lib/MC/MCAsmStreamer.cpp
llvm/lib/MC/MCParser/AsmParser.cpp
llvm/lib/MC/MCPseudoProbe.cpp
llvm/lib/MC/MCStreamer.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/PseudoProbe.h b/llvm/include/llvm/IR/PseudoProbe.h
index 79726c0eee8be..60b1be169f87d 100644
--- a/llvm/include/llvm/IR/PseudoProbe.h
+++ b/llvm/include/llvm/IR/PseudoProbe.h
@@ -30,7 +30,8 @@ enum class PseudoProbeType { Block = 0, IndirectCall, DirectCall };
enum class PseudoProbeAttributes {
Reserved = 0x1,
- Sentinel = 0x2, // A place holder for split function entry address.
+ Sentinel = 0x2, // A place holder for split function entry address.
+ HasDiscriminator = 0x4, // for probes with a discriminator
};
// The saturated distrution factor representing 100% for block probes.
@@ -91,6 +92,10 @@ static inline bool isSentinelProbe(uint32_t Flags) {
return Flags & (uint32_t)PseudoProbeAttributes::Sentinel;
}
+static inline bool hasDiscriminator(uint32_t Flags) {
+ return Flags & (uint32_t)PseudoProbeAttributes::HasDiscriminator;
+}
+
std::optional<PseudoProbe> extractProbe(const Instruction &Inst);
void setProbeDistributionFactor(Instruction &Inst, float Factor);
diff --git a/llvm/include/llvm/MC/MCPseudoProbe.h b/llvm/include/llvm/MC/MCPseudoProbe.h
index cd1bf805b50a0..4904cb4ca5456 100644
--- a/llvm/include/llvm/MC/MCPseudoProbe.h
+++ b/llvm/include/llvm/MC/MCPseudoProbe.h
@@ -30,12 +30,15 @@
// 0 - block probe, 1 - indirect call, 2 - direct call
// ATTRIBUTE (uint3)
// 1 - reserved
+// 2 - Sentinel
+// 4 - HasDiscriminator
// ADDRESS_TYPE (uint1)
// 0 - code address for regular probes (for downwards compatibility)
// - GUID of linkage name for sentinel probes
// 1 - address delta
// CODE_ADDRESS (uint64 or ULEB128)
// code address or address delta, depending on ADDRESS_TYPE
+// DISCRIMINATOR (ULEB128) if HasDiscriminator
// INLINED FUNCTION RECORDS
// A list of NUM_INLINED_FUNCTIONS entries describing each of the inlined
// callees. Each record contains:
@@ -108,6 +111,7 @@ class MCPseudoProbeBase {
protected:
uint64_t Guid;
uint64_t Index;
+ uint32_t Discriminator;
uint8_t Attributes;
uint8_t Type;
// The value should be equal to PseudoProbeReservedId::Last + 1 which is
@@ -116,8 +120,8 @@ class MCPseudoProbeBase {
const static uint32_t PseudoProbeFirstId = 1;
public:
- MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T)
- : Guid(G), Index(I), Attributes(At), Type(T) {}
+ MCPseudoProbeBase(uint64_t G, uint64_t I, uint64_t At, uint8_t T, uint32_t D)
+ : Guid(G), Index(I), Discriminator(D), Attributes(At), Type(T) {}
bool isEntry() const { return Index == PseudoProbeFirstId; }
@@ -125,6 +129,8 @@ class MCPseudoProbeBase {
uint64_t getIndex() const { return Index; }
+ uint32_t getDiscriminator() const { return Discriminator; }
+
uint8_t getAttributes() const { return Attributes; }
uint8_t getType() const { return Type; }
@@ -155,8 +161,9 @@ class MCPseudoProbe : public MCPseudoProbeBase {
public:
MCPseudoProbe(MCSymbol *Label, uint64_t Guid, uint64_t Index, uint64_t Type,
- uint64_t Attributes)
- : MCPseudoProbeBase(Guid, Index, Attributes, Type), Label(Label) {
+ uint64_t Attributes, uint32_t Discriminator)
+ : MCPseudoProbeBase(Guid, Index, Attributes, Type, Discriminator),
+ Label(Label) {
assert(Type <= 0xFF && "Probe type too big to encode, exceeding 2^8");
assert(Attributes <= 0xFF &&
"Probe attributes too big to encode, exceeding 2^16");
@@ -175,8 +182,9 @@ class MCDecodedPseudoProbe : public MCPseudoProbeBase {
public:
MCDecodedPseudoProbe(uint64_t Ad, uint64_t G, uint32_t I, PseudoProbeType K,
- uint8_t At, MCDecodedPseudoProbeInlineTree *Tree)
- : MCPseudoProbeBase(G, I, At, static_cast<uint8_t>(K)), Address(Ad),
+ uint8_t At, uint32_t D,
+ MCDecodedPseudoProbeInlineTree *Tree)
+ : MCPseudoProbeBase(G, I, At, static_cast<uint8_t>(K), D), Address(Ad),
InlineTree(Tree){};
uint64_t getAddress() const { return Address; }
diff --git a/llvm/include/llvm/MC/MCStreamer.h b/llvm/include/llvm/MC/MCStreamer.h
index f5891b24ae4b4..8b081db4dedd1 100644
--- a/llvm/include/llvm/MC/MCStreamer.h
+++ b/llvm/include/llvm/MC/MCStreamer.h
@@ -636,7 +636,7 @@ class MCStreamer {
/// \param Symbol - The function containing the trap.
/// \param Lang - The language code for the exception entry.
/// \param Reason - The reason code for the exception entry.
- virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol,
+ virtual void emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize, bool hasDebug);
@@ -1100,7 +1100,7 @@ class MCStreamer {
/// Emit the a pseudo probe into the current section.
virtual void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
- uint64_t Attr,
+ uint64_t Attr, uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym);
diff --git a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
index 0540a4c2ae02a..7f7eef9f10e2d 100644
--- a/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/PseudoProbePrinter.cpp
@@ -42,8 +42,13 @@ void PseudoProbeHandler::emitPseudoProbe(uint64_t Guid, uint64_t Index,
ReversedInlineStack.emplace_back(CallerGuid, CallerProbeId);
InlinedAt = InlinedAt->getInlinedAt();
}
-
+ uint64_t Discriminator = 0;
+ // For now only block probes have FS discriminators. See
+ // MIRFSDiscriminator.cpp for more details.
+ if (DebugLoc &&
+ !DILocation::isPseudoProbeDiscriminator(DebugLoc->getDiscriminator()))
+ Discriminator = DebugLoc->getDiscriminator();
SmallVector<InlineSite, 8> InlineStack(llvm::reverse(ReversedInlineStack));
- Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, InlineStack,
- Asm->CurrentFnSym);
+ Asm->OutStreamer->emitPseudoProbe(Guid, Index, Type, Attr, Discriminator,
+ InlineStack, Asm->CurrentFnSym);
}
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index c11d28b746f60..e3d11a2a33744 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -196,7 +196,7 @@ class MCAsmStreamer final : public MCStreamer {
void emitXCOFFRefDirective(const MCSymbol *Symbol) override;
- void emitXCOFFExceptDirective(const MCSymbol *Symbol,
+ void emitXCOFFExceptDirective(const MCSymbol *Symbol,
const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize, bool hasDebug) override;
@@ -377,8 +377,9 @@ class MCAsmStreamer final : public MCStreamer {
void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
void emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
- uint64_t Attr,
- const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) override;
+ uint64_t Attr, uint64_t Discriminator,
+ const MCPseudoProbeInlineStack &InlineStack,
+ MCSymbol *FnSym) override;
void emitBundleAlignMode(Align Alignment) override;
void emitBundleLock(bool AlignToEnd) override;
@@ -950,7 +951,7 @@ void MCAsmStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
}
void MCAsmStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol,
- const MCSymbol *Trap,
+ const MCSymbol *Trap,
unsigned Lang,
unsigned Reason,
unsigned FunctionSize,
@@ -2336,11 +2337,14 @@ void MCAsmStreamer::emitInstruction(const MCInst &Inst,
EmitEOL();
}
-void MCAsmStreamer::emitPseudoProbe(
- uint64_t Guid, uint64_t Index, uint64_t Type, uint64_t Attr,
- const MCPseudoProbeInlineStack &InlineStack, MCSymbol *FnSym) {
- OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " "
- << Attr;
+void MCAsmStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index,
+ uint64_t Type, uint64_t Attr,
+ uint64_t Discriminator,
+ const MCPseudoProbeInlineStack &InlineStack,
+ MCSymbol *FnSym) {
+ OS << "\t.pseudoprobe\t" << Guid << " " << Index << " " << Type << " " << Attr;
+ if (Discriminator)
+ OS << " " << Discriminator;
// Emit inline stack like
// @ GUIDmain:3 @ GUIDCaller:1 @ GUIDDirectCaller:11
for (const auto &Site : InlineStack)
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index c49e51332b126..472f55af84014 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -5853,24 +5853,23 @@ bool AsmParser::parseDirectivePseudoProbe() {
int64_t Index;
int64_t Type;
int64_t Attr;
+ int64_t Discriminator = 0;
- if (getLexer().is(AsmToken::Integer)) {
- if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive"))
- return true;
- }
+ if (parseIntToken(Guid, "unexpected token in '.pseudoprobe' directive"))
+ return true;
- if (getLexer().is(AsmToken::Integer)) {
- if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive"))
- return true;
- }
+ if (parseIntToken(Index, "unexpected token in '.pseudoprobe' directive"))
+ return true;
- if (getLexer().is(AsmToken::Integer)) {
- if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive"))
- return true;
- }
+ if (parseIntToken(Type, "unexpected token in '.pseudoprobe' directive"))
+ return true;
- if (getLexer().is(AsmToken::Integer)) {
- if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive"))
+ if (parseIntToken(Attr, "unexpected token in '.pseudoprobe' directive"))
+ return true;
+
+ if (hasDiscriminator(Attr)) {
+ if (parseIntToken(Discriminator,
+ "unexpected token in '.pseudoprobe' directive"))
return true;
}
@@ -5912,7 +5911,8 @@ bool AsmParser::parseDirectivePseudoProbe() {
if (parseEOL())
return true;
- getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, InlineStack, FnSym);
+ getStreamer().emitPseudoProbe(Guid, Index, Type, Attr, Discriminator,
+ InlineStack, FnSym);
return false;
}
diff --git a/llvm/lib/MC/MCPseudoProbe.cpp b/llvm/lib/MC/MCPseudoProbe.cpp
index a9460b86d22a2..35eafec721695 100644
--- a/llvm/lib/MC/MCPseudoProbe.cpp
+++ b/llvm/lib/MC/MCPseudoProbe.cpp
@@ -8,6 +8,7 @@
#include "llvm/MC/MCPseudoProbe.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/IR/PseudoProbe.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
@@ -58,10 +59,14 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
// Type (bit 0 to 3), with bit 4 to 6 for attributes.
// Flag (bit 7, 0 - code address, 1 - address delta). This indicates whether
// the following field is a symbolic code address or an address delta.
+ // Emit FS discriminator
assert(Type <= 0xF && "Probe type too big to encode, exceeding 15");
- assert(Attributes <= 0x7 &&
+ auto NewAttributes = Attributes;
+ if (Discriminator)
+ NewAttributes |= (uint32_t)PseudoProbeAttributes::HasDiscriminator;
+ assert(NewAttributes <= 0x7 &&
"Probe attributes too big to encode, exceeding 7");
- uint8_t PackedType = Type | (Attributes << 4);
+ uint8_t PackedType = Type | (NewAttributes << 4);
uint8_t Flag =
!IsSentinel ? ((int8_t)MCPseudoProbeFlag::AddressDelta << 7) : 0;
MCOS->emitInt8(Flag | PackedType);
@@ -81,6 +86,9 @@ void MCPseudoProbe::emit(MCObjectStreamer *MCOS,
MCOS->emitInt64(Guid);
}
+ if (Discriminator)
+ MCOS->emitULEB128IntValue(Discriminator);
+
LLVM_DEBUG({
dbgs().indent(MCPseudoProbeTable::DdgPrintIndent);
dbgs() << "Probe: " << Index << "\n";
@@ -222,11 +230,11 @@ void MCPseudoProbeSections::emit(MCObjectStreamer *MCOS) {
for (const auto &Inlinee : Inlinees) {
// Emit the group guarded by a sentinel probe.
- MCPseudoProbe SentinelProbe(const_cast<MCSymbol *>(FuncSym),
- MD5Hash(FuncSym->getName()),
- (uint32_t)PseudoProbeReservedId::Invalid,
- (uint32_t)PseudoProbeType::Block,
- (uint32_t)PseudoProbeAttributes::Sentinel);
+ MCPseudoProbe SentinelProbe(
+ const_cast<MCSymbol *>(FuncSym), MD5Hash(FuncSym->getName()),
+ (uint32_t)PseudoProbeReservedId::Invalid,
+ (uint32_t)PseudoProbeType::Block,
+ (uint32_t)PseudoProbeAttributes::Sentinel, 0);
const MCPseudoProbe *Probe = &SentinelProbe;
Inlinee.second->emit(MCOS, Probe);
}
@@ -310,6 +318,8 @@ void MCDecodedPseudoProbe::print(raw_ostream &OS,
OS << Guid << " ";
}
OS << "Index: " << Index << " ";
+ if (Discriminator)
+ OS << "Discriminator: " << Discriminator << " ";
OS << "Type: " << PseudoProbeTypeStr[static_cast<uint8_t>(Type)] << " ";
std::string InlineContextStr = getInlineContextStr(GUID2FuncMAP);
if (InlineContextStr.size()) {
@@ -491,11 +501,19 @@ bool MCPseudoProbeDecoder::buildAddress2ProbeMap(
}
}
+ uint32_t Discriminator = 0;
+ if (hasDiscriminator(Attr)) {
+ auto ErrorOrDiscriminator = readUnsignedNumber<uint32_t>();
+ if (!ErrorOrDiscriminator)
+ return false;
+ Discriminator = std::move(*ErrorOrDiscriminator);
+ }
+
if (Cur && !isSentinelProbe(Attr)) {
// Populate Address2ProbesMap
auto &Probes = Address2ProbesMap[Addr];
Probes.emplace_back(Addr, Cur->Guid, Index, PseudoProbeType(Kind), Attr,
- Cur);
+ Discriminator, Cur);
Cur->addProbes(&Probes.back());
}
LastAddr = Addr;
diff --git a/llvm/lib/MC/MCStreamer.cpp b/llvm/lib/MC/MCStreamer.cpp
index 8f29153783066..0cb04594522b2 100644
--- a/llvm/lib/MC/MCStreamer.cpp
+++ b/llvm/lib/MC/MCStreamer.cpp
@@ -1105,7 +1105,7 @@ void MCStreamer::emitInstruction(const MCInst &Inst, const MCSubtargetInfo &) {
}
void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
- uint64_t Attr,
+ uint64_t Attr, uint64_t Discriminator,
const MCPseudoProbeInlineStack &InlineStack,
MCSymbol *FnSym) {
auto &Context = getContext();
@@ -1117,7 +1117,7 @@ void MCStreamer::emitPseudoProbe(uint64_t Guid, uint64_t Index, uint64_t Type,
emitLabel(ProbeSym);
// Create a (local) probe entry with the symbol.
- MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr);
+ MCPseudoProbe Probe(ProbeSym, Guid, Index, Type, Attr, Discriminator);
// Add the probe entry to this section's entries.
Context.getMCPseudoProbeTable().getProbeSections().addPseudoProbe(
@@ -1196,7 +1196,7 @@ void MCStreamer::emitXCOFFRefDirective(const MCSymbol *Symbol) {
}
void MCStreamer::emitXCOFFExceptDirective(const MCSymbol *Symbol,
- const MCSymbol *Trap,
+ const MCSymbol *Trap,
unsigned Lang, unsigned Reason,
unsigned FunctionSize,
bool hasDebug) {
diff --git a/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin b/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin
new file mode 100755
index 0000000000000..c356e84ef1ae8
Binary files /dev/null and b/llvm/test/tools/llvm-profgen/Inputs/fs-discriminator-probe.perfbin
diff er
diff --git a/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test b/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test
new file mode 100644
index 0000000000000..efdc6399beb49
--- /dev/null
+++ b/llvm/test/tools/llvm-profgen/pseudoprobe-decoding-discriminator.test
@@ -0,0 +1,62 @@
+; RUN: llvm-profgen --format=text --perfscript=%s --binary=%S/Inputs/fs-discriminator-probe.perfbin --output=%t --show-pseudo-probe --show-disassembly-only | FileCheck %s
+
+; CHECK: <quick_sort>:
+; CHECK: [Probe]: FUNC: quick_sort Index: 1 Type: Block
+; CHECK: [Probe]: FUNC: quick_sort Index: 1 Discriminator: 15360 Type: Block
+; CHECK: [Probe]: FUNC: quick_sort Index: 4 Type: IndirectCall
+; CHECK: [Probe]: FUNC: quick_sort Index: 5 Type: DirectCall
+
+
+; original code:
+; clang -O3 -g -mllvm --enable-fs-discriminator -fdebug-info-for-profiling -fpseudo-probe-for-profiling qsort.c -o a.out
+#include <stdio.h>
+#include <stdlib.h>
+
+void swap(int *a, int *b) {
+ int t = *a;
+ *a = *b;
+ *b = t;
+}
+
+int partition_pivot_last(int* array, int low, int high) {
+ int pivot = array[high];
+ int i = low - 1;
+ for (int j = low; j < high; j++)
+ if (array[j] < pivot)
+ swap(&array[++i], &array[j]);
+ swap(&array[i + 1], &array[high]);
+ return (i + 1);
+}
+
+int partition_pivot_first(int* array, int low, int high) {
+ int pivot = array[low];
+ int i = low + 1;
+ for (int j = low + 1; j <= high; j++)
+ if (array[j] < pivot) { if (j != i) swap(&array[i], &array[j]); i++;}
+ swap(&array[i - 1], &array[low]);
+ return i - 1;
+}
+
+void quick_sort(int* array, int low, int high, int (*partition_func)(int *, int, int)) {
+ if (low < high) {
+ int pi = (*partition_func)(array, low, high);
+ quick_sort(array, low, pi - 1, partition_func);
+ quick_sort(array, pi + 1, high, partition_func);
+ }
+}
+
+int main() {
+ const int size = 200;
+ int sum = 0;
+ int *array = malloc(size * sizeof(int));
+ for(int i = 0; i < 100 * 1000; i++) {
+ for(int j = 0; j < size; j++)
+ array[j] = j % 10 ? rand() % size: j;
+ int (*fptr)(int *, int, int) = i % 3 ? partition_pivot_last : partition_pivot_first;
+ quick_sort(array, 0, size - 1, fptr);
+ sum += array[i % size];
+ }
+ printf("sum=%d\n", sum);
+
+ return 0;
+}
More information about the llvm-commits
mailing list