[llvm] Solve llvm-dwp overflow problem, skipped over 4GB '.dwo' files (PR #71902)

Jinjie Huang via llvm-commits llvm-commits at lists.llvm.org
Mon Nov 20 21:19:06 PST 2023


https://github.com/Labman-001 updated https://github.com/llvm/llvm-project/pull/71902

>From 0271dfa2d947823b6573583d3add6d61a091fcc5 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 16:38:32 +0800
Subject: [PATCH 1/9] init skpipped over 4g dwo

---
 llvm/lib/DWP/DWP.cpp | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 89101ca7e5736ba..7a88ed0029fc8e7 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -201,7 +201,8 @@ static Error addAllTypesFromDWP(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
     const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
-    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow) {
+    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
+    bool &SeeOverflowFlag) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -235,6 +236,8 @@ static Error addAllTypesFromDWP(
       if (Error Err = sectionOverflowErrorOrWarning(
               OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
         return Err;
+      SeeOverflowFlag = true;
+      return Error::success();
     }
   }
   return Error::success();
@@ -244,7 +247,7 @@ static Error addAllTypesFromTypesSection(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
     const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
-    bool ContinueOnCuIndexOverflow) {
+    bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
   for (StringRef Types : TypesSections) {
     Out.switchSection(OutputTypes);
     uint64_t Offset = 0;
@@ -276,6 +279,8 @@ static Error addAllTypesFromTypesSection(
         if (Error Err = sectionOverflowErrorOrWarning(
                 OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
           return Err;
+        SeeOverflowFlag = true;
+        return Error::success();
       }
     }
   }
@@ -613,6 +618,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
   uint32_t ContributionOffsets[8] = {};
   uint16_t Version = 0;
   uint32_t IndexVersion = 0;
+  bool SeeOverflowFlag = false;
 
   DWPStringPool Strings(Out, StrSection);
 
@@ -687,12 +693,15 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         uint32_t SectionIndex = 0;
         for (auto &Section : Obj.sections()) {
           if (SectionIndex == Index) {
-            return sectionOverflowErrorOrWarning(
+            if (Error Err sectionOverflowErrorOrWarning(
                 OldOffset, ContributionOffsets[Index], *Section.getName(),
-                ContinueOnCuIndexOverflow);
+                ContinueOnCuIndexOverflow))
+              return Err;
           }
           ++SectionIndex;
         }
+        SeeOverflowFlag = true;
+        break;
       }
     }
 
@@ -722,6 +731,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
                     InfoSectionOffset, InfoSectionOffset + C.getLength32(),
                     "debug_info", ContinueOnCuIndexOverflow))
               return Err;
+            SeeOverflowFlag = true;
+            break;
           }
 
           UnitOffset += C.getLength32();
@@ -752,6 +763,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               Info.substr(UnitOffset - C.getLength32(), C.getLength32()));
           InfoSectionOffset += C.getLength32();
         }
+        if (SeeOverflowFlag)
+          break;
       }
 
       if (!FoundCUUnit)
@@ -762,7 +775,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         if (Error Err = addAllTypesFromTypesSection(
                 Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
                 ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
-                ContinueOnCuIndexOverflow))
+                ContinueOnCuIndexOverflow, SeeOverflowFlag))
           return Err;
       }
       continue;
@@ -860,9 +873,11 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
       if (Error Err = addAllTypesFromDWP(
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
-              TypesContributionIndex, ContinueOnCuIndexOverflow))
+              TypesContributionIndex, ContinueOnCuIndexOverflow, SeeOverflowFlag))
         return Err;
     }
+    if (SeeOverflowFlag)
+      break;
   }
 
   if (Version < 5) {

>From a82d566eabd067014a60ef80b6cd12d5612b531b Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 16:43:19 +0800
Subject: [PATCH 2/9] fix compile

---
 llvm/lib/DWP/DWP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 7a88ed0029fc8e7..b128d55dc46bd4b 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -693,7 +693,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         uint32_t SectionIndex = 0;
         for (auto &Section : Obj.sections()) {
           if (SectionIndex == Index) {
-            if (Error Err sectionOverflowErrorOrWarning(
+            if (Error Err = sectionOverflowErrorOrWarning(
                 OldOffset, ContributionOffsets[Index], *Section.getName(),
                 ContinueOnCuIndexOverflow))
               return Err;

>From 9dd98438a4ddeedc611e9c438f2f757c84d685d7 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Fri, 10 Nov 2023 17:02:45 +0800
Subject: [PATCH 3/9] fix clang-format

---
 llvm/lib/DWP/DWP.cpp | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index b128d55dc46bd4b..002c905c78f17c6 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -197,12 +197,13 @@ static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
   return make_error<DWPError>(Msg);
 }
 
-static Error addAllTypesFromDWP(
-    MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
-    const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
-    const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
-    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
-    bool &SeeOverflowFlag) {
+static Error
+addAllTypesFromDWP(MCStreamer &Out,
+                   MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
+                   const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
+                   StringRef Types, const UnitIndexEntry &TUEntry,
+                   uint32_t &TypesOffset, unsigned TypesContributionIndex,
+                   bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -694,8 +695,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         for (auto &Section : Obj.sections()) {
           if (SectionIndex == Index) {
             if (Error Err = sectionOverflowErrorOrWarning(
-                OldOffset, ContributionOffsets[Index], *Section.getName(),
-                ContinueOnCuIndexOverflow))
+                    OldOffset, ContributionOffsets[Index], *Section.getName(),
+                    ContinueOnCuIndexOverflow))
               return Err;
           }
           ++SectionIndex;
@@ -873,7 +874,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
       if (Error Err = addAllTypesFromDWP(
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
-              TypesContributionIndex, ContinueOnCuIndexOverflow, SeeOverflowFlag))
+              TypesContributionIndex, ContinueOnCuIndexOverflow,
+              SeeOverflowFlag))
         return Err;
     }
     if (SeeOverflowFlag)

>From 3c549f3014bd7cb18f46d09c7c3af2c7ddd946b0 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Wed, 15 Nov 2023 20:20:31 +0800
Subject: [PATCH 4/9] add StopOnCuIndexOverflow

---
 llvm/lib/DWP/DWP.cpp             | 31 ++++++++++++++++++++-----------
 llvm/tools/llvm-dwp/llvm-dwp.cpp |  4 +++-
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 002c905c78f17c6..3080939133cfebc 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -203,7 +203,8 @@ addAllTypesFromDWP(MCStreamer &Out,
                    const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
                    StringRef Types, const UnitIndexEntry &TUEntry,
                    uint32_t &TypesOffset, unsigned TypesContributionIndex,
-                   bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
+                   bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
+                   bool StopOnCuIndexOverflow) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -237,7 +238,8 @@ addAllTypesFromDWP(MCStreamer &Out,
       if (Error Err = sectionOverflowErrorOrWarning(
               OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
         return Err;
-      SeeOverflowFlag = true;
+      if (StopOnCuIndexOverflow)
+        SeeOverflowFlag = true;
       return Error::success();
     }
   }
@@ -248,7 +250,8 @@ static Error addAllTypesFromTypesSection(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
     const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
-    bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag) {
+    bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
+    bool StopOnCuIndexOverflow) {
   for (StringRef Types : TypesSections) {
     Out.switchSection(OutputTypes);
     uint64_t Offset = 0;
@@ -280,7 +283,8 @@ static Error addAllTypesFromTypesSection(
         if (Error Err = sectionOverflowErrorOrWarning(
                 OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
           return Err;
-        SeeOverflowFlag = true;
+        if (StopOnCuIndexOverflow)
+          SeeOverflowFlag = true;
         return Error::success();
       }
     }
@@ -589,7 +593,8 @@ Error handleSection(
 }
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow) {
+            bool ContinueOnCuIndexOverflow,
+            bool StopOnCuIndexOverflow) {
   const auto &MCOFI = *Out.getContext().getObjectFileInfo();
   MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
   MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -701,8 +706,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
           }
           ++SectionIndex;
         }
-        SeeOverflowFlag = true;
-        break;
+        if (StopOnCuIndexOverflow) {
+          SeeOverflowFlag = true;
+          break;
+        }
       }
     }
 
@@ -732,8 +739,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
                     InfoSectionOffset, InfoSectionOffset + C.getLength32(),
                     "debug_info", ContinueOnCuIndexOverflow))
               return Err;
-            SeeOverflowFlag = true;
-            break;
+            if (StopOnCuIndexOverflow) {
+              SeeOverflowFlag = true;
+              break;
+            }
           }
 
           UnitOffset += C.getLength32();
@@ -776,7 +785,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         if (Error Err = addAllTypesFromTypesSection(
                 Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
                 ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
-                ContinueOnCuIndexOverflow, SeeOverflowFlag))
+                ContinueOnCuIndexOverflow, SeeOverflowFlag, StopOnCuIndexOverflow))
           return Err;
       }
       continue;
@@ -875,7 +884,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
               TypesContributionIndex, ContinueOnCuIndexOverflow,
-              SeeOverflowFlag))
+              SeeOverflowFlag, StopOnCuIndexOverflow))
         return Err;
     }
     if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index f976298dcadc2f4..ce74361804e9f7b 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -144,6 +144,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
 
   OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
   ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
+  StopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
 
   for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
     ExecFilenames.emplace_back(A->getValue());
@@ -255,7 +256,8 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
 
-  if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow)) {
+  if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
+      StopOnCuIndexOverflow)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }

>From c8f138bae38dd0df7d47960c9a387fd09eee1154 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:34:58 +0800
Subject: [PATCH 5/9] init

---
 llvm/tools/llvm-dwp/llvm-dwp.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index ce74361804e9f7b..8383f5e77cea8b4 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -144,7 +144,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
 
   OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
   ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
-  StopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
+  SoftStopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
 
   for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
     ExecFilenames.emplace_back(A->getValue());
@@ -257,7 +257,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
     return error("no object streamer for target " + TripleName, Context);
 
   if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
-      StopOnCuIndexOverflow)) {
+      SoftStopOnCuIndexOverflow)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }

>From 69a46fcce6f4d523c376e68e3f928f1005b5e357 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:44:39 +0800
Subject: [PATCH 6/9] fix clang

---
 llvm/lib/DWP/DWP.cpp             | 32 +++++++++++++++-----------------
 llvm/tools/llvm-dwp/llvm-dwp.cpp |  2 +-
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 3080939133cfebc..dfca16b1ba0d9d2 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -197,14 +197,12 @@ static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
   return make_error<DWPError>(Msg);
 }
 
-static Error
-addAllTypesFromDWP(MCStreamer &Out,
-                   MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
-                   const DWARFUnitIndex &TUIndex, MCSection *OutputTypes,
-                   StringRef Types, const UnitIndexEntry &TUEntry,
-                   uint32_t &TypesOffset, unsigned TypesContributionIndex,
-                   bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
-                   bool StopOnCuIndexOverflow) {
+static Error addAllTypesFromDWP(
+    MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
+    const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
+    const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
+    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
+    bool &SeeOverflowFlag, bool SoftStopOnCuIndexOverflow) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -238,7 +236,7 @@ addAllTypesFromDWP(MCStreamer &Out,
       if (Error Err = sectionOverflowErrorOrWarning(
               OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
         return Err;
-      if (StopOnCuIndexOverflow)
+      if (SoftStopOnCuIndexOverflow)
         SeeOverflowFlag = true;
       return Error::success();
     }
@@ -251,7 +249,7 @@ static Error addAllTypesFromTypesSection(
     MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
     const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
     bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
-    bool StopOnCuIndexOverflow) {
+    bool SoftStopOnCuIndexOverflow) {
   for (StringRef Types : TypesSections) {
     Out.switchSection(OutputTypes);
     uint64_t Offset = 0;
@@ -283,7 +281,7 @@ static Error addAllTypesFromTypesSection(
         if (Error Err = sectionOverflowErrorOrWarning(
                 OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
           return Err;
-        if (StopOnCuIndexOverflow)
+        if (SoftStopOnCuIndexOverflow)
           SeeOverflowFlag = true;
         return Error::success();
       }
@@ -593,8 +591,7 @@ Error handleSection(
 }
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow,
-            bool StopOnCuIndexOverflow) {
+            bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow) {
   const auto &MCOFI = *Out.getContext().getObjectFileInfo();
   MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
   MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -706,7 +703,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
           }
           ++SectionIndex;
         }
-        if (StopOnCuIndexOverflow) {
+        if (SoftStopOnCuIndexOverflow) {
           SeeOverflowFlag = true;
           break;
         }
@@ -739,7 +736,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
                     InfoSectionOffset, InfoSectionOffset + C.getLength32(),
                     "debug_info", ContinueOnCuIndexOverflow))
               return Err;
-            if (StopOnCuIndexOverflow) {
+            if (SoftStopOnCuIndexOverflow) {
               SeeOverflowFlag = true;
               break;
             }
@@ -785,7 +782,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         if (Error Err = addAllTypesFromTypesSection(
                 Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
                 ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
-                ContinueOnCuIndexOverflow, SeeOverflowFlag, StopOnCuIndexOverflow))
+                ContinueOnCuIndexOverflow, SeeOverflowFlag,
+                SoftStopOnCuIndexOverflow))
           return Err;
       }
       continue;
@@ -884,7 +882,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
               TypesContributionIndex, ContinueOnCuIndexOverflow,
-              SeeOverflowFlag, StopOnCuIndexOverflow))
+              SeeOverflowFlag, SoftStopOnCuIndexOverflow))
         return Err;
     }
     if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 8383f5e77cea8b4..7b79e7c5ad1efd1 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -257,7 +257,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
     return error("no object streamer for target " + TripleName, Context);
 
   if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
-      SoftStopOnCuIndexOverflow)) {
+                       SoftStopOnCuIndexOverflow)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }

>From 175ce76bb7d9e83d208b8498d806d0b50e250956 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 10:59:21 +0800
Subject: [PATCH 7/9] fix clang

---
 llvm/tools/llvm-dwp/llvm-dwp.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index 7b79e7c5ad1efd1..beb89741a729e9c 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -72,6 +72,7 @@ class DwpOptTable : public opt::GenericOptTable {
 static std::vector<std::string> ExecFilenames;
 static std::string OutputFilename;
 static bool ContinueOnCuIndexOverflow;
+static bool SoftStopOnCuIndexOverflow;
 
 static Expected<SmallVector<std::string, 16>>
 getDWOFilenames(StringRef ExecFilename) {
@@ -144,7 +145,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
 
   OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
   ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
-  SoftStopOnCuIndexOverflow = Args.hasArg(OPT_stopOnCuIndexOverflow);
+  SoftStopOnCuIndexOverflow = Args.hasArg(OPT_softStopOnCuIndexOverflow);
 
   for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
     ExecFilenames.emplace_back(A->getValue());

>From 080c9800f3bd1da6a3e37daad6f18f2288adb171 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Mon, 20 Nov 2023 11:37:24 +0800
Subject: [PATCH 8/9] fix opts.td

---
 llvm/include/llvm/DWP/DWP.h | 2 +-
 llvm/tools/llvm-dwp/Opts.td | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/DWP/DWP.h b/llvm/include/llvm/DWP/DWP.h
index e33133d38ae008e..4434feadd8857e0 100644
--- a/llvm/include/llvm/DWP/DWP.h
+++ b/llvm/include/llvm/DWP/DWP.h
@@ -61,7 +61,7 @@ struct CompileUnitIdentifiers {
 };
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow);
+            bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow);
 
 unsigned getContributionIndex(DWARFSectionKind Kind, uint32_t IndexVersion);
 
diff --git a/llvm/tools/llvm-dwp/Opts.td b/llvm/tools/llvm-dwp/Opts.td
index c01fa4a12cbafb8..cd27de0a1f7708e 100644
--- a/llvm/tools/llvm-dwp/Opts.td
+++ b/llvm/tools/llvm-dwp/Opts.td
@@ -11,3 +11,5 @@ def execFileNames : S<"e", "Specify the executable/library files to get the list
 def outputFileName : S<"o", "Specify the output file.">, MetaVarName<"<filename>">;
 def continueOnCuIndexOverflow: F<"continue-on-cu-index-overflow", "This turns an error when offset for .debug_*.dwo sections "
                                          "overfolws into a warning.">, MetaVarName<"<filename>">;
+def softStopOnCuIndexOverflow: F<"soft-stop-on-cu-index-overflow", "This turns to make a best effort dwp when offset for "
+                                        ".debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file

>From b23f06763e160db78b0c1d55c3f918b571a5a1d2 Mon Sep 17 00:00:00 2001
From: huangjinjie <huangjinjie at bytedance.com>
Date: Tue, 21 Nov 2023 13:18:37 +0800
Subject: [PATCH 9/9] user enum OnCuIndexOverflow to control softStop/continue

---
 llvm/include/llvm/DWP/DWP.h      |  6 ++++
 llvm/lib/DWP/DWP.cpp             | 50 +++++++++++++++-----------------
 llvm/tools/llvm-dwp/Opts.td      |  8 ++---
 llvm/tools/llvm-dwp/llvm-dwp.cpp | 15 ++++++----
 4 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/llvm/include/llvm/DWP/DWP.h b/llvm/include/llvm/DWP/DWP.h
index 4434feadd8857e0..a21bfc6e567393e 100644
--- a/llvm/include/llvm/DWP/DWP.h
+++ b/llvm/include/llvm/DWP/DWP.h
@@ -15,6 +15,12 @@
 #include <vector>
 
 namespace llvm {
+enum OnCuIndexOverflow {
+  HardStop,
+  SoftStop,
+  Continue,
+};
+
 struct UnitIndexEntry {
   DWARFUnitIndex::Entry::SectionContribution Contributions[8];
   std::string Name;
diff --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index dfca16b1ba0d9d2..fa884a942492676 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -181,16 +181,21 @@ static StringRef getSubsection(StringRef Section,
 }
 
 static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
-                                           uint32_t OverflowedOffset,
-                                           StringRef SectionName,
-                                           bool ContinueOnCuIndexOverflow) {
+    uint32_t OverflowedOffset,
+    StringRef SectionName,
+    const OnCuIndexOverflow &overflowOptValue,
+    bool &SeeOverflowFlag) {
   std::string Msg =
       (SectionName +
        Twine(" Section Contribution Offset overflow 4G. Previous Offset ") +
        Twine(PrevOffset) + Twine(", After overflow offset ") +
        Twine(OverflowedOffset) + Twine("."))
           .str();
-  if (ContinueOnCuIndexOverflow) {
+  if (overflowOptValue == OnCuIndexOverflow::Continue) {
+    WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
+    return Error::success();
+  } else if (overflowOptValue == OnCuIndexOverflow::SoftStop) {
+    SeeOverflowFlag = true;
     WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
     return Error::success();
   }
@@ -201,8 +206,8 @@ static Error addAllTypesFromDWP(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
     const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
-    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow,
-    bool &SeeOverflowFlag, bool SoftStopOnCuIndexOverflow) {
+    unsigned TypesContributionIndex, const OnCuIndexOverflow &overflowOptValue,
+    bool &SeeOverflowFlag) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -234,10 +239,9 @@ static Error addAllTypesFromDWP(
     TypesOffset += C.getLength();
     if (OldOffset > TypesOffset) {
       if (Error Err = sectionOverflowErrorOrWarning(
-              OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
+              OldOffset, TypesOffset, "Types", 
+              overflowOptValue, SeeOverflowFlag))
         return Err;
-      if (SoftStopOnCuIndexOverflow)
-        SeeOverflowFlag = true;
       return Error::success();
     }
   }
@@ -248,8 +252,7 @@ static Error addAllTypesFromTypesSection(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
     const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
-    bool ContinueOnCuIndexOverflow, bool &SeeOverflowFlag,
-    bool SoftStopOnCuIndexOverflow) {
+    const OnCuIndexOverflow &overflowOptValue, bool &SeeOverflowFlag) {
   for (StringRef Types : TypesSections) {
     Out.switchSection(OutputTypes);
     uint64_t Offset = 0;
@@ -279,10 +282,9 @@ static Error addAllTypesFromTypesSection(
       TypesOffset += C.getLength32();
       if (OldOffset > TypesOffset) {
         if (Error Err = sectionOverflowErrorOrWarning(
-                OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
+                OldOffset, TypesOffset, "Types", 
+                overflowOptValue, SeeOverflowFlag))
           return Err;
-        if (SoftStopOnCuIndexOverflow)
-          SeeOverflowFlag = true;
         return Error::success();
       }
     }
@@ -591,7 +593,7 @@ Error handleSection(
 }
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow, bool SoftStopOnCuIndexOverflow) {
+            const OnCuIndexOverflow &overflowOptValue) {
   const auto &MCOFI = *Out.getContext().getObjectFileInfo();
   MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
   MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -698,15 +700,13 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
           if (SectionIndex == Index) {
             if (Error Err = sectionOverflowErrorOrWarning(
                     OldOffset, ContributionOffsets[Index], *Section.getName(),
-                    ContinueOnCuIndexOverflow))
+                    overflowOptValue, SeeOverflowFlag))
               return Err;
           }
           ++SectionIndex;
         }
-        if (SoftStopOnCuIndexOverflow) {
-          SeeOverflowFlag = true;
+        if (SeeOverflowFlag)
           break;
-        }
       }
     }
 
@@ -734,12 +734,10 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               C.getLength32()) {
             if (Error Err = sectionOverflowErrorOrWarning(
                     InfoSectionOffset, InfoSectionOffset + C.getLength32(),
-                    "debug_info", ContinueOnCuIndexOverflow))
+                    "debug_info", overflowOptValue, SeeOverflowFlag))
               return Err;
-            if (SoftStopOnCuIndexOverflow) {
-              SeeOverflowFlag = true;
+            if (SeeOverflowFlag)
               break;
-            }
           }
 
           UnitOffset += C.getLength32();
@@ -782,8 +780,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         if (Error Err = addAllTypesFromTypesSection(
                 Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
                 ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
-                ContinueOnCuIndexOverflow, SeeOverflowFlag,
-                SoftStopOnCuIndexOverflow))
+                overflowOptValue, SeeOverflowFlag))
           return Err;
       }
       continue;
@@ -881,8 +878,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
       if (Error Err = addAllTypesFromDWP(
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
-              TypesContributionIndex, ContinueOnCuIndexOverflow,
-              SeeOverflowFlag, SoftStopOnCuIndexOverflow))
+              TypesContributionIndex, overflowOptValue, SeeOverflowFlag))
         return Err;
     }
     if (SeeOverflowFlag)
diff --git a/llvm/tools/llvm-dwp/Opts.td b/llvm/tools/llvm-dwp/Opts.td
index cd27de0a1f7708e..b183b415333434d 100644
--- a/llvm/tools/llvm-dwp/Opts.td
+++ b/llvm/tools/llvm-dwp/Opts.td
@@ -9,7 +9,7 @@ def version : F<"version", "Display the version of this program">;
 
 def execFileNames : S<"e", "Specify the executable/library files to get the list of *.dwo from.">, MetaVarName<"<filename>">;
 def outputFileName : S<"o", "Specify the output file.">, MetaVarName<"<filename>">;
-def continueOnCuIndexOverflow: F<"continue-on-cu-index-overflow", "This turns an error when offset for .debug_*.dwo sections "
-                                         "overfolws into a warning.">, MetaVarName<"<filename>">;
-def softStopOnCuIndexOverflow: F<"soft-stop-on-cu-index-overflow", "This turns to make a best effort dwp when offset for "
-                                        ".debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file
+def continueOnCuIndexOverflow : S<"continue-on-cu-index-overflow", " =soft-stop, This produces a truncated but valid "
+                                         "DWP file, discarding any DWO files that would not fit within the 32 "
+                                         "bit/4GB limits of the format. =continue, This turns an error when offset "
+                                         "for .debug_*.dwo sections overfolws into a warning.">, MetaVarName<"<filename>">;
\ No newline at end of file
diff --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index beb89741a729e9c..fc126c6282c2cfc 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -71,8 +71,7 @@ class DwpOptTable : public opt::GenericOptTable {
 // Options
 static std::vector<std::string> ExecFilenames;
 static std::string OutputFilename;
-static bool ContinueOnCuIndexOverflow;
-static bool SoftStopOnCuIndexOverflow;
+static std::string continueOption;
 
 static Expected<SmallVector<std::string, 16>>
 getDWOFilenames(StringRef ExecFilename) {
@@ -126,6 +125,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   DwpOptTable Tbl;
   llvm::BumpPtrAllocator A;
   llvm::StringSaver Saver{A};
+  OnCuIndexOverflow overflowOptValue = OnCuIndexOverflow::HardStop;
   opt::InputArgList Args =
       Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
         llvm::errs() << Msg << '\n';
@@ -144,8 +144,12 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   }
 
   OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
-  ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
-  SoftStopOnCuIndexOverflow = Args.hasArg(OPT_softStopOnCuIndexOverflow);
+  continueOption = Args.getLastArgValue(OPT_continueOnCuIndexOverflow, "hard-stop");
+  if (continueOption == "soft-stop") {
+    overflowOptValue = OnCuIndexOverflow::SoftStop;
+  } else if (continueOption == "soft-stop") {
+    overflowOptValue = OnCuIndexOverflow::Continue;
+  }
 
   for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
     ExecFilenames.emplace_back(A->getValue());
@@ -257,8 +261,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
 
-  if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow,
-                       SoftStopOnCuIndexOverflow)) {
+  if (auto Err = write(*MS, DWOFilenames, overflowOptValue)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }



More information about the llvm-commits mailing list