[llvm] r367328 - [Remarks] Add two serialization modes for remarks: separate and standalone
Francis Visoiu Mistrih via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 30 09:01:41 PDT 2019
Author: thegameg
Date: Tue Jul 30 09:01:40 2019
New Revision: 367328
URL: http://llvm.org/viewvc/llvm-project?rev=367328&view=rev
Log:
[Remarks] Add two serialization modes for remarks: separate and standalone
The default mode is separate, where the metadata is serialized
separately from the remarks.
Another mode is the standalone mode, where the metadata is serialized
before the remarks, on the same stream.
Modified:
llvm/trunk/docs/Remarks.rst
llvm/trunk/include/llvm/Remarks/RemarkSerializer.h
llvm/trunk/include/llvm/Remarks/YAMLRemarkSerializer.h
llvm/trunk/lib/IR/RemarkStreamer.cpp
llvm/trunk/lib/Remarks/RemarkSerializer.cpp
llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp
llvm/trunk/unittests/Remarks/YAMLRemarksSerializerTest.cpp
Modified: llvm/trunk/docs/Remarks.rst
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/docs/Remarks.rst?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/docs/Remarks.rst (original)
+++ llvm/trunk/docs/Remarks.rst Tue Jul 30 09:01:40 2019
@@ -147,6 +147,26 @@ Other tools that support remarks:
.. option:: -opt-remarks-format=<format>
.. option:: -opt-remarks-with-hotness
+Serialization modes
+===================
+
+There are two modes available for serializing remarks:
+
+``Separate``
+
+ In this mode, the remarks and the metadata are serialized separately. The
+ client is responsible for parsing the metadata first, then use the metadata
+ to correctly parse the remarks.
+
+``Standalone``
+
+ In this mode, the remarks and the metadata are serialized to the same
+ stream. The metadata will always come before the remarks.
+
+ The compiler does not support emitting standalone remarks. This mode is
+ more suited for post-processing tools like linkers, that can merge the
+ remarks for one whole project.
+
.. _yamlremarks:
YAML remarks
Modified: llvm/trunk/include/llvm/Remarks/RemarkSerializer.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Remarks/RemarkSerializer.h?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Remarks/RemarkSerializer.h (original)
+++ llvm/trunk/include/llvm/Remarks/RemarkSerializer.h Tue Jul 30 09:01:40 2019
@@ -14,12 +14,23 @@
#define LLVM_REMARKS_REMARK_SERIALIZER_H
#include "llvm/Remarks/Remark.h"
+#include "llvm/Remarks/RemarkFormat.h"
#include "llvm/Remarks/RemarkStringTable.h"
#include "llvm/Support/raw_ostream.h"
namespace llvm {
namespace remarks {
+enum class SerializerMode {
+ Separate, // A mode where the metadata is serialized separately from the
+ // remarks. Typically, this is used when the remarks need to be
+ // streamed to a side file and the metadata is embedded into the
+ // final result of the compilation.
+ Standalone // A mode where everything can be retrieved in the same
+ // file/buffer. Typically, this is used for storing remarks for
+ // later use.
+};
+
struct MetaSerializer;
/// This is the base class for a remark serializer.
@@ -27,11 +38,14 @@ struct MetaSerializer;
struct RemarkSerializer {
/// The open raw_ostream that the remark diagnostics are emitted to.
raw_ostream &OS;
+ /// The serialization mode.
+ SerializerMode Mode;
/// The string table containing all the unique strings used in the output.
/// The table can be serialized to be consumed after the compilation.
Optional<StringTable> StrTab;
- RemarkSerializer(raw_ostream &OS) : OS(OS), StrTab() {}
+ RemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+ : OS(OS), Mode(Mode), StrTab() {}
/// This is just an interface.
virtual ~RemarkSerializer() = default;
@@ -57,12 +71,12 @@ struct MetaSerializer {
/// Create a remark serializer.
Expected<std::unique_ptr<RemarkSerializer>>
-createRemarkSerializer(Format RemarksFormat, raw_ostream &OS);
+createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, raw_ostream &OS);
/// Create a remark serializer that uses a pre-filled string table.
Expected<std::unique_ptr<RemarkSerializer>>
-createRemarkSerializer(Format RemarksFormat, raw_ostream &OS,
- remarks::StringTable StrTab);
+createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+ raw_ostream &OS, remarks::StringTable StrTab);
} // end namespace remarks
} // end namespace llvm
Modified: llvm/trunk/include/llvm/Remarks/YAMLRemarkSerializer.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/Remarks/YAMLRemarkSerializer.h?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/include/llvm/Remarks/YAMLRemarkSerializer.h (original)
+++ llvm/trunk/include/llvm/Remarks/YAMLRemarkSerializer.h Tue Jul 30 09:01:40 2019
@@ -34,12 +34,15 @@ struct YAMLRemarkSerializer : public Rem
/// The YAML streamer.
yaml::Output YAMLOutput;
- YAMLRemarkSerializer(raw_ostream &OS);
+ YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode);
void emit(const Remark &Remark) override;
std::unique_ptr<MetaSerializer>
metaSerializer(raw_ostream &OS,
Optional<StringRef> ExternalFilename = None) override;
+
+protected:
+ bool DidEmitMeta = false;
};
struct YAMLMetaSerializer : public MetaSerializer {
@@ -55,12 +58,14 @@ struct YAMLMetaSerializer : public MetaS
/// like the regular YAML remark but instead of string entries it's using
/// numbers that map to an index in the string table.
struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
- YAMLStrTabRemarkSerializer(raw_ostream &OS) : YAMLRemarkSerializer(OS) {
+ YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+ : YAMLRemarkSerializer(OS, Mode) {
// Having a string table set up enables the serializer to use it.
StrTab.emplace();
}
- YAMLStrTabRemarkSerializer(raw_ostream &OS, StringTable StrTabIn)
- : YAMLRemarkSerializer(OS) {
+ YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
+ StringTable StrTabIn)
+ : YAMLRemarkSerializer(OS, Mode) {
StrTab = std::move(StrTabIn);
}
std::unique_ptr<MetaSerializer>
Modified: llvm/trunk/lib/IR/RemarkStreamer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/IR/RemarkStreamer.cpp?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/lib/IR/RemarkStreamer.cpp (original)
+++ llvm/trunk/lib/IR/RemarkStreamer.cpp Tue Jul 30 09:01:40 2019
@@ -136,7 +136,7 @@ llvm::setupOptimizationRemarks(LLVMConte
return make_error<RemarkSetupFormatError>(std::move(E));
Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer =
- remarks::createRemarkSerializer(*Format, RemarksFile->os());
+ remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, RemarksFile->os());
if (Error E = RemarkSerializer.takeError())
return make_error<RemarkSetupFormatError>(std::move(E));
Modified: llvm/trunk/lib/Remarks/RemarkSerializer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Remarks/RemarkSerializer.cpp?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/lib/Remarks/RemarkSerializer.cpp (original)
+++ llvm/trunk/lib/Remarks/RemarkSerializer.cpp Tue Jul 30 09:01:40 2019
@@ -17,22 +17,23 @@ using namespace llvm;
using namespace llvm::remarks;
Expected<std::unique_ptr<RemarkSerializer>>
-remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS) {
+remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+ raw_ostream &OS) {
switch (RemarksFormat) {
case Format::Unknown:
return createStringError(std::errc::invalid_argument,
"Unknown remark serializer format.");
case Format::YAML:
- return llvm::make_unique<YAMLRemarkSerializer>(OS);
+ return llvm::make_unique<YAMLRemarkSerializer>(OS, Mode);
case Format::YAMLStrTab:
- return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS);
+ return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode);
}
llvm_unreachable("Unknown remarks::Format enum");
}
Expected<std::unique_ptr<RemarkSerializer>>
-remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS,
- remarks::StringTable StrTab) {
+remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+ raw_ostream &OS, remarks::StringTable StrTab) {
switch (RemarksFormat) {
case Format::Unknown:
return createStringError(std::errc::invalid_argument,
@@ -42,7 +43,8 @@ remarks::createRemarkSerializer(Format R
"Unable to use a string table with the yaml "
"format. Use 'yaml-strtab' instead.");
case Format::YAMLStrTab:
- return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, std::move(StrTab));
+ return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode,
+ std::move(StrTab));
}
llvm_unreachable("Unknown remarks::Format enum");
}
Modified: llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp (original)
+++ llvm/trunk/lib/Remarks/YAMLRemarkSerializer.cpp Tue Jul 30 09:01:40 2019
@@ -149,10 +149,19 @@ template <> struct MappingTraits<Argumen
LLVM_YAML_IS_SEQUENCE_VECTOR(Argument)
-YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS)
- : RemarkSerializer(OS), YAMLOutput(OS, reinterpret_cast<void *>(this)) {}
+YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+ : RemarkSerializer(OS, Mode), YAMLOutput(OS, reinterpret_cast<void *>(this)) {}
void YAMLRemarkSerializer::emit(const Remark &Remark) {
+ // In standalone mode, emit the metadata first and set DidEmitMeta to avoid
+ // emitting it again.
+ if (Mode == SerializerMode::Standalone) {
+ std::unique_ptr<MetaSerializer> MetaSerializer =
+ metaSerializer(OS, /*ExternalFilename=*/None);
+ MetaSerializer->emit();
+ DidEmitMeta = true;
+ }
+
// Again, YAMLTraits expect a non-const object for inputting, but we're not
// using that here.
auto R = const_cast<remarks::Remark *>(&Remark);
Modified: llvm/trunk/unittests/Remarks/YAMLRemarksSerializerTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Remarks/YAMLRemarksSerializerTest.cpp?rev=367328&r1=367327&r2=367328&view=diff
==============================================================================
--- llvm/trunk/unittests/Remarks/YAMLRemarksSerializerTest.cpp (original)
+++ llvm/trunk/unittests/Remarks/YAMLRemarksSerializerTest.cpp Tue Jul 30 09:01:40 2019
@@ -21,20 +21,21 @@
using namespace llvm;
-static void check(const remarks::Remark &R, StringRef ExpectedR,
- StringRef ExpectedMeta, bool UseStrTab = false,
+static void check(remarks::SerializerMode Mode, const remarks::Remark &R,
+ StringRef ExpectedR, Optional<StringRef> ExpectedMeta,
+ bool UseStrTab = false,
Optional<remarks::StringTable> StrTab = None) {
std::string Buf;
raw_string_ostream OS(Buf);
Expected<std::unique_ptr<remarks::RemarkSerializer>> MaybeS = [&] {
if (UseStrTab) {
if (StrTab)
- return createRemarkSerializer(remarks::Format::YAMLStrTab, OS,
+ return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS,
std::move(*StrTab));
else
- return createRemarkSerializer(remarks::Format::YAMLStrTab, OS);
+ return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS);
} else
- return createRemarkSerializer(remarks::Format::YAML, OS);
+ return createRemarkSerializer(remarks::Format::YAML, Mode, OS);
}();
EXPECT_FALSE(errorToBool(MaybeS.takeError()));
std::unique_ptr<remarks::RemarkSerializer> S = std::move(*MaybeS);
@@ -42,11 +43,27 @@ static void check(const remarks::Remark
S->emit(R);
EXPECT_EQ(OS.str(), ExpectedR);
- Buf.clear();
- std::unique_ptr<remarks::MetaSerializer> MS =
- S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH));
- MS->emit();
- EXPECT_EQ(OS.str(), ExpectedMeta);
+ if (ExpectedMeta) {
+ Buf.clear();
+ std::unique_ptr<remarks::MetaSerializer> MS =
+ S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH));
+ MS->emit();
+ EXPECT_EQ(OS.str(), *ExpectedMeta);
+ }
+}
+
+static void check(const remarks::Remark &R, StringRef ExpectedR,
+ StringRef ExpectedMeta, bool UseStrTab = false,
+ Optional<remarks::StringTable> StrTab = None) {
+ return check(remarks::SerializerMode::Separate, R, ExpectedR, ExpectedMeta,
+ UseStrTab, std::move(StrTab));
+}
+
+static void checkStandalone(const remarks::Remark &R, StringRef ExpectedR,
+ Optional<remarks::StringTable> StrTab = None) {
+ bool UseStrTab = StrTab.hasValue();
+ return check(remarks::SerializerMode::Standalone, R, ExpectedR,
+ /*ExpectedMeta=*/None, UseStrTab, std::move(StrTab));
}
TEST(YAMLRemarks, SerializerRemark) {
@@ -83,6 +100,40 @@ TEST(YAMLRemarks, SerializerRemark) {
38));
}
+TEST(YAMLRemarks, SerializerRemarkStandalone) {
+ remarks::Remark R;
+ R.RemarkType = remarks::Type::Missed;
+ R.PassName = "pass";
+ R.RemarkName = "name";
+ R.FunctionName = "func";
+ R.Loc = remarks::RemarkLocation{"path", 3, 4};
+ R.Hotness = 5;
+ R.Args.emplace_back();
+ R.Args.back().Key = "key";
+ R.Args.back().Val = "value";
+ R.Args.emplace_back();
+ R.Args.back().Key = "keydebug";
+ R.Args.back().Val = "valuedebug";
+ R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
+ checkStandalone(
+ R,
+ StringRef("REMARKS\0"
+ "\0\0\0\0\0\0\0\0"
+ "\0\0\0\0\0\0\0\0"
+ "--- !Missed\n"
+ "Pass: pass\n"
+ "Name: name\n"
+ "DebugLoc: { File: path, Line: 3, Column: 4 }\n"
+ "Function: func\n"
+ "Hotness: 5\n"
+ "Args:\n"
+ " - key: value\n"
+ " - keydebug: valuedebug\n"
+ " DebugLoc: { File: argpath, Line: 6, Column: 7 }\n"
+ "...\n",
+ 301));
+}
+
TEST(YAMLRemarks, SerializerRemarkStrTab) {
remarks::Remark R;
R.RemarkType = remarks::Type::Missed;
@@ -156,3 +207,42 @@ TEST(YAMLRemarks, SerializerRemarkParsed
/*UseStrTab=*/true,
remarks::StringTable(remarks::ParsedStringTable(StrTab)));
}
+
+TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) {
+ StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
+ remarks::ParsedStringTable ParsedStrTab(StrTab);
+ remarks::StringTable PreFilledStrTab(ParsedStrTab);
+ remarks::Remark R;
+ R.RemarkType = remarks::Type::Missed;
+ R.PassName = "pass";
+ R.RemarkName = "name";
+ R.FunctionName = "func";
+ R.Loc = remarks::RemarkLocation{"path", 3, 4};
+ R.Hotness = 5;
+ R.Args.emplace_back();
+ R.Args.back().Key = "key";
+ R.Args.back().Val = "value";
+ R.Args.emplace_back();
+ R.Args.back().Key = "keydebug";
+ R.Args.back().Val = "valuedebug";
+ R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
+ checkStandalone(
+ R,
+ StringRef("REMARKS\0"
+ "\0\0\0\0\0\0\0\0"
+ "\x2d\0\0\0\0\0\0\0"
+ "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
+ "--- !Missed\n"
+ "Pass: 0\n"
+ "Name: 1\n"
+ "DebugLoc: { File: 3, Line: 3, Column: 4 }\n"
+ "Function: 2\n"
+ "Hotness: 5\n"
+ "Args:\n"
+ " - key: 4\n"
+ " - keydebug: 5\n"
+ " DebugLoc: { File: 6, Line: 6, Column: 7 }\n"
+ "...\n",
+ 315),
+ std::move(PreFilledStrTab));
+}
More information about the llvm-commits
mailing list