[llvm] Solve llvm-dwp overflow problem, skipped over 4GB '.dwo' files (PR #71902)
Jinjie Huang via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 20 21:19:06 PST 2023
https://github.com/Labman-001 updated https://github.com/llvm/llvm-project/pull/71902
>From 0271dfa2d947823b6573583d3add6d61a091fcc5 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 16:38:32 +0800
Subject: [PATCH 1/9] init skpipped over 4g dwo
---
llvm/lib/DWP/DWP.cpp | 27 +++++++++++++++++++++------
1 file changed, 21 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 89101ca7e5736ba..7a88ed0029fc8e7 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -201,7 +201,8 @@ static Error addAllTypesFromDWP(
MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
- unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow) {
+ unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
+ bool &SeeOverflowFlag) {
Out.switchSection(OutputTypes);
for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
auto *I = E.getContributions();
@@ -235,6 +236,8 @@ static Error addAllTypesFromDWP(
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
return Err;
+ SeeOverflowFlag = true;
+ return Error::success();
}
}
return Error::success();
@@ -244,7 +247,7 @@ static Error addAllTypesFromTypesSection(
MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
- bool ContinueOnCuIndexOverflow) {
+ bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
for (StringRef Types : TypesSections) {
Out.switchSection(OutputTypes);
uint64_t Offset = 0;
@@ -276,6 +279,8 @@ static Error addAllTypesFromTypesSection(
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
return Err;
+ SeeOverflowFlag = true;
+ return Error::success();
}
}
}
@@ -613,6 +618,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
uint32_t ContributionOffsets[8] = {};
uint16_t Version = 0;
uint32_t IndexVersion = 0;
+ bool SeeOverflowFlag = false;
DWPStringPool Strings(Out, StrSection);
@@ -687,12 +693,15 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
uint32_t SectionIndex = 0;
for (auto &Section : Obj.sections()) {
if (SectionIndex == Index) {
- return sectionOverflowErrorOrWarning(
+ if (Error Err sectionOverflowErrorOrWarning(
OldOffset, ContributionOffsets[Index], *Section.getName(),
- ContinueOnCuIndexOverflow);
+ ContinueOnCuIndexOverflow))
+ return Err;
}
++SectionIndex;
}
+ SeeOverflowFlag = true;
+ break;
}
}
@@ -722,6 +731,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
InfoSectionOffset, InfoSectionOffset + C.getLength32(),
"debug_info", ContinueOnCuIndexOverflow))
return Err;
+ SeeOverflowFlag = true;
+ break;
}
UnitOffset += C.getLength32();
@@ -752,6 +763,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
Info.substr(UnitOffset - C.getLength32(), C.getLength32()));
InfoSectionOffset += C.getLength32();
}
+ if (SeeOverflowFlag)
+ break;
}
if (!FoundCUUnit)
@@ -762,7 +775,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromTypesSection(
Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
- ContinueOnCuIndexOverflow))
+ ContinueOnCuIndexOverflow, SeeOverflowFlag))
return Err;
}
continue;
@@ -860,9 +873,11 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromDWP(
Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
CurEntry, ContributionOffsets[TypesContributionIndex],
- TypesContributionIndex, ContinueOnCuIndexOverflow))
+ TypesContributionIndex, ContinueOnCuIndexOverflow, SeeOverflowFlag))
return Err;
}
+ if (SeeOverflowFlag)
+ break;
}
if (Version < 5) {
>From a82d566eabd067014a60ef80b6cd12d5612b531b Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 16:43:19 +0800
Subject: [PATCH 2/9] fix compile
---
llvm/lib/DWP/DWP.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 7a88ed0029fc8e7..b128d55dc46bd4b 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -693,7 +693,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
uint32_t SectionIndex = 0;
for (auto &Section : Obj.sections()) {
if (SectionIndex == Index) {
- if (Error Err sectionOverflowErrorOrWarning(
+ if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, ContributionOffsets[Index], *Section.getName(),
ContinueOnCuIndexOverflow))
return Err;
>From 9dd98438a4ddeedc611e9c438f2f757c84d685d7 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 17:02:45 +0800
Subject: [PATCH 3/9] fix clang-format
---
llvm/lib/DWP/DWP.cpp | 20 +++++++++++---------
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index b128d55dc46bd4b..002c905c78f17c6 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -197,12 +197,13 @@ static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
return make_error<DWPError>(Msg);
}
-static Error addAllTypesFromDWP(
- MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
- const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
- const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
- unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
- bool &SeeOverflowFlag) {
+static Error
+addAllTypesFromDWP(MCStreamer &Out,
+ MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
+ const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
+ StringRef Types, const UnitIndexEntry &TUEntry,
+ uint32_t &TypesOffset, unsigned TypesContributionIndex,
+ bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
Out.switchSection(OutputTypes);
for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
auto *I = E.getContributions();
@@ -694,8 +695,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
for (auto &Section : Obj.sections()) {
if (SectionIndex == Index) {
if (Error Err = sectionOverflowErrorOrWarning(
- OldOffset, ContributionOffsets[Index], *Section.getName(),
- ContinueOnCuIndexOverflow))
+ OldOffset, ContributionOffsets[Index], *Section.getName(),
+ ContinueOnCuIndexOverflow))
return Err;
}
++SectionIndex;
@@ -873,7 +874,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromDWP(
Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
CurEntry, ContributionOffsets[TypesContributionIndex],
- TypesContributionIndex, ContinueOnCuIndexOverflow, SeeOverflowFlag))
+ TypesContributionIndex, ContinueOnCuIndexOverflow,
+ SeeOverflowFlag))
return Err;
}
if (SeeOverflowFlag)
>From 3c549f3014bd7cb18f46d09c7c3af2c7ddd946b0 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Wed, 15 Nov 2023 20:20:31 +0800
Subject: [PATCH 4/9] add StopOnCuIndexOverflow
---
llvm/lib/DWP/DWP.cpp | 31 ++++++++++++++++++++-----------
llvm/tools/llvm-dwp/llvm-dwp.cpp | 4 +++-
2 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 002c905c78f17c6..3080939133cfebc 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -203,7 +203,8 @@ addAllTypesFromDWP(MCStreamer &Out,
const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
StringRef Types, const UnitIndexEntry &TUEntry,
uint32_t &TypesOffset, unsigned TypesContributionIndex,
- bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
+ bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
+ bool StopOnCuIndexOverflow) {
Out.switchSection(OutputTypes);
for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
auto *I = E.getContributions();
@@ -237,7 +238,8 @@ addAllTypesFromDWP(MCStreamer &Out,
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
return Err;
- SeeOverflowFlag = true;
+ if (StopOnCuIndexOverflow)
+ SeeOverflowFlag = true;
return Error::success();
}
}
@@ -248,7 +250,8 @@ static Error addAllTypesFromTypesSection(
MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
- bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
+ bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
+ bool StopOnCuIndexOverflow) {
for (StringRef Types : TypesSections) {
Out.switchSection(OutputTypes);
uint64_t Offset = 0;
@@ -280,7 +283,8 @@ static Error addAllTypesFromTypesSection(
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
return Err;
- SeeOverflowFlag = true;
+ if (StopOnCuIndexOverflow)
+ SeeOverflowFlag = true;
return Error::success();
}
}
@@ -589,7 +593,8 @@ Error handleSection(
}
Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
- bool ContinueOnCuIndexOverflow) {
+ bool ContinueOnCuIndexOverflow,
+ bool StopOnCuIndexOverflow) {
const auto &MCOFI = *Out.getContext().getObjectFileInfo();
MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -701,8 +706,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
}
++SectionIndex;
}
- SeeOverflowFlag = true;
- break;
+ if (StopOnCuIndexOverflow) {
+ SeeOverflowFlag = true;
+ break;
+ }
}
}
@@ -732,8 +739,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
InfoSectionOffset, InfoSectionOffset + C.getLength32(),
"debug_info", ContinueOnCuIndexOverflow))
return Err;
- SeeOverflowFlag = true;
- break;
+ if (StopOnCuIndexOverflow) {
+ SeeOverflowFlag = true;
+ break;
+ }
}
UnitOffset += C.getLength32();
@@ -776,7 +785,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromTypesSection(
Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
- ContinueOnCuIndexOverflow, SeeOverflowFlag))
+ ContinueOnCuIndexOverflow, SeeOverflowFlag, StopOnCuIndexOverflow))
return Err;
}
continue;
@@ -875,7 +884,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
CurEntry, ContributionOffsets[TypesContributionIndex],
TypesContributionIndex, ContinueOnCuIndexOverflow,
- SeeOverflowFlag))
+ SeeOverflowFlag, StopOnCuIndexOverflow))
return Err;
}
if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index f976298dcadc2f4..ce74361804e9f7b 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -144,6 +144,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
+ StopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
ExecFilenames.emplace_back(A->getValue());
@@ -255,7 +256,8 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
if (!MS)
return error("no object streamer for target " + TripleName, Context);
- if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow)) {
+ if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
+ StopOnCuIndexOverflow)) {
logAllUnhandledErrors(std::move(Err), WithColor::error());
return 1;
}
>From c8f138bae38dd0df7d47960c9a387fd09eee1154 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:34:58 +0800
Subject: [PATCH 5/9] init
---
llvm/tools/llvm-dwp/llvm-dwp.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index ce74361804e9f7b..8383f5e77cea8b4 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -144,7 +144,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
- StopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
+ SoftStopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
ExecFilenames.emplace_back(A->getValue());
@@ -257,7 +257,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
return error("no object streamer for target " + TripleName, Context);
if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
- StopOnCuIndexOverflow)) {
+ SoftStopOnCuIndexOverflow)) {
logAllUnhandledErrors(std::move(Err), WithColor::error());
return 1;
}
>From 69a46fcce6f4d523c376e68e3f928f1005b5e357 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:44:39 +0800
Subject: [PATCH 6/9] fix clang
---
llvm/lib/DWP/DWP.cpp | 32 +++++++++++++++-----------------
llvm/tools/llvm-dwp/llvm-dwp.cpp | 2 +-
2 files changed, 16 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 3080939133cfebc..dfca16b1ba0d9d2 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -197,14 +197,12 @@ static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
return make_error<DWPError>(Msg);
}
-static Error
-addAllTypesFromDWP(MCStreamer &Out,
- MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
- const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
- StringRef Types, const UnitIndexEntry &TUEntry,
- uint32_t &TypesOffset, unsigned TypesContributionIndex,
- bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
- bool StopOnCuIndexOverflow) {
+static Error addAllTypesFromDWP(
+ MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
+ const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
+ const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
+ unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
+ bool &SeeOverflowFlag, bool SoftStopOnCuIndexOverflow) {
Out.switchSection(OutputTypes);
for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
auto *I = E.getContributions();
@@ -238,7 +236,7 @@ addAllTypesFromDWP(MCStreamer &Out,
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
return Err;
- if (StopOnCuIndexOverflow)
+ if (SoftStopOnCuIndexOverflow)
SeeOverflowFlag = true;
return Error::success();
}
@@ -251,7 +249,7 @@ static Error addAllTypesFromTypesSection(
MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
- bool StopOnCuIndexOverflow) {
+ bool SoftStopOnCuIndexOverflow) {
for (StringRef Types : TypesSections) {
Out.switchSection(OutputTypes);
uint64_t Offset = 0;
@@ -283,7 +281,7 @@ static Error addAllTypesFromTypesSection(
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
return Err;
- if (StopOnCuIndexOverflow)
+ if (SoftStopOnCuIndexOverflow)
SeeOverflowFlag = true;
return Error::success();
}
@@ -593,8 +591,7 @@ Error handleSection(
}
Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
- bool ContinueOnCuIndexOverflow,
- bool StopOnCuIndexOverflow) {
+ bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow) {
const auto &MCOFI = *Out.getContext().getObjectFileInfo();
MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -706,7 +703,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
}
++SectionIndex;
}
- if (StopOnCuIndexOverflow) {
+ if (SoftStopOnCuIndexOverflow) {
SeeOverflowFlag = true;
break;
}
@@ -739,7 +736,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
InfoSectionOffset, InfoSectionOffset + C.getLength32(),
"debug_info", ContinueOnCuIndexOverflow))
return Err;
- if (StopOnCuIndexOverflow) {
+ if (SoftStopOnCuIndexOverflow) {
SeeOverflowFlag = true;
break;
}
@@ -785,7 +782,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromTypesSection(
Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
- ContinueOnCuIndexOverflow, SeeOverflowFlag, StopOnCuIndexOverflow))
+ ContinueOnCuIndexOverflow, SeeOverflowFlag,
+ SoftStopOnCuIndexOverflow))
return Err;
}
continue;
@@ -884,7 +882,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
CurEntry, ContributionOffsets[TypesContributionIndex],
TypesContributionIndex, ContinueOnCuIndexOverflow,
- SeeOverflowFlag, StopOnCuIndexOverflow))
+ SeeOverflowFlag, SoftStopOnCuIndexOverflow))
return Err;
}
if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 8383f5e77cea8b4..7b79e7c5ad1efd1 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -257,7 +257,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
return error("no object streamer for target " + TripleName, Context);
if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
- SoftStopOnCuIndexOverflow)) {
+ SoftStopOnCuIndexOverflow)) {
logAllUnhandledErrors(std::move(Err), WithColor::error());
return 1;
}
>From 175ce76bb7d9e83d208b8498d806d0b50e250956 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:59:21 +0800
Subject: [PATCH 7/9] fix clang
---
llvm/tools/llvm-dwp/llvm-dwp.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 7b79e7c5ad1efd1..beb89741a729e9c 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -72,6 +72,7 @@ class DwpOptTable : public opt::GenericOptTable {
static std::vector<std::string> ExecFilenames;
static std::string OutputFilename;
static bool ContinueOnCuIndexOverflow;
+static bool SoftStopOnCuIndexOverflow;
static Expected<SmallVector<std::string, 16>>
getDWOFilenames(StringRef ExecFilename) {
@@ -144,7 +145,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
- SoftStopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
+ SoftStopOnCuIndexOverflow = Args.hasArg(OPT_softStopOnCuIndexOverflow);
for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
ExecFilenames.emplace_back(A->getValue());
>From 080c9800f3bd1da6a3e37daad6f18f2288adb171 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 11:37:24 +0800
Subject: [PATCH 8/9] fix opts.td
---
llvm/include/llvm/DWP/DWP.h | 2 +-
llvm/tools/llvm-dwp/Opts.td | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/DWP/DWP.h b/llvm/include/llvm/DWP/DWP.h
index e33133d38ae008e..4434feadd8857e0 100644
--- a/llvm/include/llvm/DWP/DWP.h
+++ b/llvm/include/llvm/DWP/DWP.h
@@ -61,7 +61,7 @@ struct CompileUnitIdentifiers {
};
Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
- bool ContinueOnCuIndexOverflow);
+ bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow);
unsigned getContributionIndex(DWARFSectionKind Kind, uint32_t IndexVersion);
diff --git a/llvm/tools/llvm-dwp/Opts.td b/llvm/tools/llvm-dwp/Opts.td
index c01fa4a12cbafb8..cd27de0a1f7708e 100644
--- a/llvm/tools/llvm-dwp/Opts.td
+++ b/llvm/tools/llvm-dwp/Opts.td
@@ -11,3 +11,5 @@ def execFileNames : S<"e", "Specify the executable/library files to get the list
def outputFileName : S<"o", "Specify the output file.">, MetaVarName<"<filename>">;
def continueOnCuIndexOverflow: F<"continue-on-cu-index-overflow", "This turns an error when offset for .debug_*.dwo sections "
"overfolws into a warning.">, MetaVarName<"<filename>">;
+def softStopOnCuIndexOverflow: F<"soft-stop-on-cu-index-overflow", "This turns to make a best effort dwp when offset for "
+ ".debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file
>From b23f06763e160db78b0c1d55c3f918b571a5a1d2 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Tue, 21 Nov 2023 13:18:37 +0800
Subject: [PATCH 9/9] user enum OnCuIndexOverflow to control softStop/continue
---
llvm/include/llvm/DWP/DWP.h | 6 ++++
llvm/lib/DWP/DWP.cpp | 50 +++++++++++++++-----------------
llvm/tools/llvm-dwp/Opts.td | 8 ++---
llvm/tools/llvm-dwp/llvm-dwp.cpp | 15 ++++++----
4 files changed, 42 insertions(+), 37 deletions(-)
diff --git a/llvm/include/llvm/DWP/DWP.h b/llvm/include/llvm/DWP/DWP.h
index 4434feadd8857e0..a21bfc6e567393e 100644
--- a/llvm/include/llvm/DWP/DWP.h
+++ b/llvm/include/llvm/DWP/DWP.h
@@ -15,6 +15,12 @@
#include <vector>
namespace llvm {
+enum OnCuIndexOverflow {
+ HardStop,
+ SoftStop,
+ Continue,
+};
+
struct UnitIndexEntry {
DWARFUnitIndex::Entry::SectionContribution Contributions[8];
std::string Name;
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index dfca16b1ba0d9d2..fa884a942492676 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -181,16 +181,21 @@ static StringRef getSubsection(StringRef Section,
}
static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
- uint32_t OverflowedOffset,
- StringRef SectionName,
- bool ContinueOnCuIndexOverflow) {
+ uint32_t OverflowedOffset,
+ StringRef SectionName,
+ const OnCuIndexOverflow &overflowOptValue,
+ bool &SeeOverflowFlag) {
std::string Msg =
(SectionName +
Twine(" Section Contribution Offset overflow 4G. Previous Offset ") +
Twine(PrevOffset) + Twine(", After overflow offset ") +
Twine(OverflowedOffset) + Twine("."))
.str();
- if (ContinueOnCuIndexOverflow) {
+ if (overflowOptValue == OnCuIndexOverflow::Continue) {
+ WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
+ return Error::success();
+ } else if (overflowOptValue == OnCuIndexOverflow::SoftStop) {
+ SeeOverflowFlag = true;
WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
return Error::success();
}
@@ -201,8 +206,8 @@ static Error addAllTypesFromDWP(
MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
- unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
- bool &SeeOverflowFlag, bool SoftStopOnCuIndexOverflow) {
+ unsigned TypesContributionIndex, const OnCuIndexOverflow &overflowOptValue,
+ bool &SeeOverflowFlag) {
Out.switchSection(OutputTypes);
for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
auto *I = E.getContributions();
@@ -234,10 +239,9 @@ static Error addAllTypesFromDWP(
TypesOffset += C.getLength();
if (OldOffset > TypesOffset) {
if (Error Err = sectionOverflowErrorOrWarning(
- OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
+ OldOffset, TypesOffset, "Types",
+ overflowOptValue, SeeOverflowFlag))
return Err;
- if (SoftStopOnCuIndexOverflow)
- SeeOverflowFlag = true;
return Error::success();
}
}
@@ -248,8 +252,7 @@ static Error addAllTypesFromTypesSection(
MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
- bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
- bool SoftStopOnCuIndexOverflow) {
+ const OnCuIndexOverflow &overflowOptValue, bool &SeeOverflowFlag) {
for (StringRef Types : TypesSections) {
Out.switchSection(OutputTypes);
uint64_t Offset = 0;
@@ -279,10 +282,9 @@ static Error addAllTypesFromTypesSection(
TypesOffset += C.getLength32();
if (OldOffset > TypesOffset) {
if (Error Err = sectionOverflowErrorOrWarning(
- OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
+ OldOffset, TypesOffset, "Types",
+ overflowOptValue, SeeOverflowFlag))
return Err;
- if (SoftStopOnCuIndexOverflow)
- SeeOverflowFlag = true;
return Error::success();
}
}
@@ -591,7 +593,7 @@ Error handleSection(
}
Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
- bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow) {
+ const OnCuIndexOverflow &overflowOptValue) {
const auto &MCOFI = *Out.getContext().getObjectFileInfo();
MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -698,15 +700,13 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (SectionIndex == Index) {
if (Error Err = sectionOverflowErrorOrWarning(
OldOffset, ContributionOffsets[Index], *Section.getName(),
- ContinueOnCuIndexOverflow))
+ overflowOptValue, SeeOverflowFlag))
return Err;
}
++SectionIndex;
}
- if (SoftStopOnCuIndexOverflow) {
- SeeOverflowFlag = true;
+ if (SeeOverflowFlag)
break;
- }
}
}
@@ -734,12 +734,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
C.getLength32()) {
if (Error Err = sectionOverflowErrorOrWarning(
InfoSectionOffset, InfoSectionOffset + C.getLength32(),
- "debug_info", ContinueOnCuIndexOverflow))
+ "debug_info", overflowOptValue, SeeOverflowFlag))
return Err;
- if (SoftStopOnCuIndexOverflow) {
- SeeOverflowFlag = true;
+ if (SeeOverflowFlag)
break;
- }
}
UnitOffset += C.getLength32();
@@ -782,8 +780,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromTypesSection(
Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
- ContinueOnCuIndexOverflow, SeeOverflowFlag,
- SoftStopOnCuIndexOverflow))
+ overflowOptValue, SeeOverflowFlag))
return Err;
}
continue;
@@ -881,8 +878,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
if (Error Err = addAllTypesFromDWP(
Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
CurEntry, ContributionOffsets[TypesContributionIndex],
- TypesContributionIndex, ContinueOnCuIndexOverflow,
- SeeOverflowFlag, SoftStopOnCuIndexOverflow))
+ TypesContributionIndex, overflowOptValue, SeeOverflowFlag))
return Err;
}
if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/Opts.td b/llvm/tools/llvm-dwp/Opts.td
index cd27de0a1f7708e..b183b415333434d 100644
--- a/llvm/tools/llvm-dwp/Opts.td
+++ b/llvm/tools/llvm-dwp/Opts.td
@@ -9,7 +9,7 @@ def version : F<"version", "Display the version of this program">;
def execFileNames : S<"e", "Specify the executable/library files to get the list of *.dwo from.">, MetaVarName<"<filename>">;
def outputFileName : S<"o", "Specify the output file.">, MetaVarName<"<filename>">;
-def continueOnCuIndexOverflow: F<"continue-on-cu-index-overflow", "This turns an error when offset for .debug_*.dwo sections "
- "overfolws into a warning.">, MetaVarName<"<filename>">;
-def softStopOnCuIndexOverflow: F<"soft-stop-on-cu-index-overflow", "This turns to make a best effort dwp when offset for "
- ".debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file
+def continueOnCuIndexOverflow : S<"continue-on-cu-index-overflow", " =soft-stop, This produces a truncated but valid "
+ "DWP file, discarding any DWO files that would not fit within the 32 "
+ "bit/4GB limits of the format. =continue, This turns an error when offset "
+ "for .debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index beb89741a729e9c..fc126c6282c2cfc 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -71,8 +71,7 @@ class DwpOptTable : public opt::GenericOptTable {
// Options
static std::vector<std::string> ExecFilenames;
static std::string OutputFilename;
-static bool ContinueOnCuIndexOverflow;
-static bool SoftStopOnCuIndexOverflow;
+static std::string continueOption;
static Expected<SmallVector<std::string, 16>>
getDWOFilenames(StringRef ExecFilename) {
@@ -126,6 +125,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
DwpOptTable Tbl;
llvm::BumpPtrAllocator A;
llvm::StringSaver Saver{A};
+ OnCuIndexOverflow overflowOptValue = OnCuIndexOverflow::HardStop;
opt::InputArgList Args =
Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
llvm::errs() << Msg << '\n';
@@ -144,8 +144,12 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
}
OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
- ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
- SoftStopOnCuIndexOverflow = Args.hasArg(OPT_softStopOnCuIndexOverflow);
+ continueOption = Args.getLastArgValue(OPT_continueOnCuIndexOverflow, "hard-stop");
+ if (continueOption == "soft-stop") {
+ overflowOptValue = OnCuIndexOverflow::SoftStop;
+ } else if (continueOption == "soft-stop") {
+ overflowOptValue = OnCuIndexOverflow::Continue;
+ }
for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
ExecFilenames.emplace_back(A->getValue());
@@ -257,8 +261,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
if (!MS)
return error("no object streamer for target " + TripleName, Context);
- if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
- SoftStopOnCuIndexOverflow)) {
+ if (auto Err = write(*MS, DWOFilenames, overflowOptValue)) {
logAllUnhandledErrors(std::move(Err), WithColor::error());
return 1;
}
More information about the llvm-commits
mailing list