[llvm] 4c44dcf - Support soft failure on DWP section overflow, producing a truncated but valid DWP(#71902)

via llvm-commits llvm-commits at lists.llvm.org
Fri Dec 1 12:01:32 PST 2023


Author: Jinjie Huang
Date: 2023-12-01T12:01:22-08:00
New Revision: 4c44dcffd5f1557bde2c21773221081437308895

URL: https://github.com/llvm/llvm-project/commit/4c44dcffd5f1557bde2c21773221081437308895
DIFF: https://github.com/llvm/llvm-project/commit/4c44dcffd5f1557bde2c21773221081437308895.diff

LOG: Support soft failure on DWP section overflow, producing a truncated but valid DWP(#71902)

When 'ContinueOnCuIndexOverflow' enables without this modification, the
forcibly generated '.dwp' won't be recognized by Debugger(gdb 10.1)
correctly.
<img width="657" alt="image"
src="https://github.com/llvm/llvm-project/assets/150100070/31732775-2548-453a-a47a-fa04c6d05fe9">
it looks like there's a problem with processing the dwarf header, which
makes debugging completely impossible. (unless the consumer walks the debug_info section to rebuild that column (if that's the only section that overflowed - if it's another section, there's no recovery))

**About patch:**
When llvm-dwp enables option '--continue-on-cu-index-overflow=soft-stop'
and recognizes the overflow problem , it will stop packing and generate
the '.dwp' file at once, discarding any DWO files that would not fit
within the 32 bit/4GB limits of the format.
<img width="625" alt="image"
src="https://github.com/llvm/llvm-project/assets/150100070/77d6be24-262b-4f4c-afc0-9a6c49e133c7">

Added: 
    

Modified: 
    llvm/include/llvm/DWP/DWP.h
    llvm/lib/DWP/DWP.cpp
    llvm/tools/llvm-dwp/Opts.td
    llvm/tools/llvm-dwp/llvm-dwp.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/DWP/DWP.h b/llvm/include/llvm/DWP/DWP.h
index e33133d38ae008e..9925deaa636b36b 100644
--- a/llvm/include/llvm/DWP/DWP.h
+++ b/llvm/include/llvm/DWP/DWP.h
@@ -15,6 +15,12 @@
 #include <vector>
 
 namespace llvm {
+enum OnCuIndexOverflow {
+  HardStop,
+  SoftStop,
+  Continue,
+};
+
 struct UnitIndexEntry {
   DWARFUnitIndex::Entry::SectionContribution Contributions[8];
   std::string Name;
@@ -61,7 +67,7 @@ struct CompileUnitIdentifiers {
 };
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow);
+            OnCuIndexOverflow OverflowOptValue);
 
 unsigned getContributionIndex(DWARFSectionKind Kind, uint32_t IndexVersion);
 

diff  --git a/llvm/lib/DWP/DWP.cpp b/llvm/lib/DWP/DWP.cpp
index 89101ca7e5736ba..77bd22d1f071c6e 100644
--- a/llvm/lib/DWP/DWP.cpp
+++ b/llvm/lib/DWP/DWP.cpp
@@ -183,14 +183,19 @@ static StringRef getSubsection(StringRef Section,
 static Error sectionOverflowErrorOrWarning(uint32_t PrevOffset,
                                            uint32_t OverflowedOffset,
                                            StringRef SectionName,
-                                           bool ContinueOnCuIndexOverflow) {
+                                           OnCuIndexOverflow OverflowOptValue,
+                                           bool &AnySectionOverflow) {
   std::string Msg =
       (SectionName +
        Twine(" Section Contribution Offset overflow 4G. Previous Offset ") +
        Twine(PrevOffset) + Twine(", After overflow offset ") +
        Twine(OverflowedOffset) + Twine("."))
           .str();
-  if (ContinueOnCuIndexOverflow) {
+  if (OverflowOptValue == OnCuIndexOverflow::Continue) {
+    WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
+    return Error::success();
+  } else if (OverflowOptValue == OnCuIndexOverflow::SoftStop) {
+    AnySectionOverflow = true;
     WithColor::defaultWarningHandler(make_error<DWPError>(Msg));
     return Error::success();
   }
@@ -201,7 +206,8 @@ static Error addAllTypesFromDWP(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     const DWARFUnitIndex &TUIndex, MCSection *OutputTypes, StringRef Types,
     const UnitIndexEntry &TUEntry, uint32_t &TypesOffset,
-    unsigned TypesContributionIndex, bool ContinueOnCuIndexOverflow) {
+    unsigned TypesContributionIndex, OnCuIndexOverflow OverflowOptValue,
+    bool &AnySectionOverflow) {
   Out.switchSection(OutputTypes);
   for (const DWARFUnitIndex::Entry &E : TUIndex.getRows()) {
     auto *I = E.getContributions();
@@ -232,9 +238,14 @@ static Error addAllTypesFromDWP(
     static_assert(sizeof(OldOffset) == sizeof(TypesOffset));
     TypesOffset += C.getLength();
     if (OldOffset > TypesOffset) {
-      if (Error Err = sectionOverflowErrorOrWarning(
-              OldOffset, TypesOffset, "Types", ContinueOnCuIndexOverflow))
+      if (Error Err = sectionOverflowErrorOrWarning(OldOffset, TypesOffset,
+                                                    "Types", OverflowOptValue,
+                                                    AnySectionOverflow))
         return Err;
+      if (AnySectionOverflow) {
+        TypesOffset = OldOffset;
+        return Error::success();
+      }
     }
   }
   return Error::success();
@@ -244,7 +255,7 @@ static Error addAllTypesFromTypesSection(
     MCStreamer &Out, MapVector<uint64_t, UnitIndexEntry> &TypeIndexEntries,
     MCSection *OutputTypes, const std::vector<StringRef> &TypesSections,
     const UnitIndexEntry &CUEntry, uint32_t &TypesOffset,
-    bool ContinueOnCuIndexOverflow) {
+    OnCuIndexOverflow OverflowOptValue, bool &AnySectionOverflow) {
   for (StringRef Types : TypesSections) {
     Out.switchSection(OutputTypes);
     uint64_t Offset = 0;
@@ -273,9 +284,14 @@ static Error addAllTypesFromTypesSection(
       uint32_t OldOffset = TypesOffset;
       TypesOffset += C.getLength32();
       if (OldOffset > TypesOffset) {
-        if (Error Err = sectionOverflowErrorOrWarning(
-                OldOffset, TypesOffset, "types", ContinueOnCuIndexOverflow))
+        if (Error Err = sectionOverflowErrorOrWarning(OldOffset, TypesOffset,
+                                                      "Types", OverflowOptValue,
+                                                      AnySectionOverflow))
           return Err;
+        if (AnySectionOverflow) {
+          TypesOffset = OldOffset;
+          return Error::success();
+        }
       }
     }
   }
@@ -583,7 +599,7 @@ Error handleSection(
 }
 
 Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
-            bool ContinueOnCuIndexOverflow) {
+            OnCuIndexOverflow OverflowOptValue) {
   const auto &MCOFI = *Out.getContext().getObjectFileInfo();
   MCSection *const StrSection = MCOFI.getDwarfStrDWOSection();
   MCSection *const StrOffsetSection = MCOFI.getDwarfStrOffDWOSection();
@@ -613,6 +629,7 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
   uint32_t ContributionOffsets[8] = {};
   uint16_t Version = 0;
   uint32_t IndexVersion = 0;
+  bool AnySectionOverflow = false;
 
   DWPStringPool Strings(Out, StrSection);
 
@@ -687,12 +704,15 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         uint32_t SectionIndex = 0;
         for (auto &Section : Obj.sections()) {
           if (SectionIndex == Index) {
-            return sectionOverflowErrorOrWarning(
-                OldOffset, ContributionOffsets[Index], *Section.getName(),
-                ContinueOnCuIndexOverflow);
+            if (Error Err = sectionOverflowErrorOrWarning(
+                    OldOffset, ContributionOffsets[Index], *Section.getName(),
+                    OverflowOptValue, AnySectionOverflow))
+              return Err;
           }
           ++SectionIndex;
         }
+        if (AnySectionOverflow)
+          break;
       }
     }
 
@@ -720,8 +740,14 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               C.getLength32()) {
             if (Error Err = sectionOverflowErrorOrWarning(
                     InfoSectionOffset, InfoSectionOffset + C.getLength32(),
-                    "debug_info", ContinueOnCuIndexOverflow))
+                    "debug_info", OverflowOptValue, AnySectionOverflow))
               return Err;
+            if (AnySectionOverflow) {
+              if (Header.Version < 5 ||
+                  Header.UnitType == dwarf::DW_UT_split_compile)
+                FoundCUUnit = true;
+              break;
+            }
           }
 
           UnitOffset += C.getLength32();
@@ -752,6 +778,8 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
               Info.substr(UnitOffset - C.getLength32(), C.getLength32()));
           InfoSectionOffset += C.getLength32();
         }
+        if (AnySectionOverflow)
+          break;
       }
 
       if (!FoundCUUnit)
@@ -762,9 +790,11 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
         if (Error Err = addAllTypesFromTypesSection(
                 Out, TypeIndexEntries, TypesSection, CurTypesSection, CurEntry,
                 ContributionOffsets[getContributionIndex(DW_SECT_EXT_TYPES, 2)],
-                ContinueOnCuIndexOverflow))
+                OverflowOptValue, AnySectionOverflow))
           return Err;
       }
+      if (AnySectionOverflow)
+        break;
       continue;
     }
 
@@ -860,9 +890,11 @@ Error write(MCStreamer &Out, ArrayRef<std::string> Inputs,
       if (Error Err = addAllTypesFromDWP(
               Out, TypeIndexEntries, TUIndex, OutSection, TypeInputSection,
               CurEntry, ContributionOffsets[TypesContributionIndex],
-              TypesContributionIndex, ContinueOnCuIndexOverflow))
+              TypesContributionIndex, OverflowOptValue, AnySectionOverflow))
         return Err;
     }
+    if (AnySectionOverflow)
+      break;
   }
 
   if (Version < 5) {

diff  --git a/llvm/tools/llvm-dwp/Opts.td b/llvm/tools/llvm-dwp/Opts.td
index c01fa4a12cbafb8..75b56fdb670cf4b 100644
--- a/llvm/tools/llvm-dwp/Opts.td
+++ b/llvm/tools/llvm-dwp/Opts.td
@@ -9,5 +9,7 @@ def version : F<"version", "Display the version of this program">;
 
 def execFileNames : S<"e", "Specify the executable/library files to get the list of *.dwo from.">, MetaVarName<"<filename>">;
 def outputFileName : S<"o", "Specify the output file.">, MetaVarName<"<filename>">;
-def continueOnCuIndexOverflow: F<"continue-on-cu-index-overflow", "This turns an error when offset for .debug_*.dwo sections "
-                                         "overfolws into a warning.">, MetaVarName<"<filename>">;
+def continueOnCuIndexOverflow : S<"continue-on-cu-index-overflow", "default = continue, This turns an error when offset "
+                                         "for .debug_*.dwo sections overfolws into a warning. = soft-stop, This produces a "
+                                         "truncated but valid DWP file, discarding any DWO files that would not fit within "
+                                         "the 32 bit/4GB limits of the format.">, MetaVarName<"<filename>">;
\ No newline at end of file

diff  --git a/llvm/tools/llvm-dwp/llvm-dwp.cpp b/llvm/tools/llvm-dwp/llvm-dwp.cpp
index f976298dcadc2f4..5cd4c0078a45866 100644
--- a/llvm/tools/llvm-dwp/llvm-dwp.cpp
+++ b/llvm/tools/llvm-dwp/llvm-dwp.cpp
@@ -71,7 +71,7 @@ class DwpOptTable : public opt::GenericOptTable {
 // Options
 static std::vector<std::string> ExecFilenames;
 static std::string OutputFilename;
-static bool ContinueOnCuIndexOverflow;
+static std::string ContinueOption;
 
 static Expected<SmallVector<std::string, 16>>
 getDWOFilenames(StringRef ExecFilename) {
@@ -125,6 +125,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   DwpOptTable Tbl;
   llvm::BumpPtrAllocator A;
   llvm::StringSaver Saver{A};
+  OnCuIndexOverflow OverflowOptValue = OnCuIndexOverflow::HardStop;
   opt::InputArgList Args =
       Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
         llvm::errs() << Msg << '\n';
@@ -143,7 +144,15 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   }
 
   OutputFilename = Args.getLastArgValue(OPT_outputFileName, "");
-  ContinueOnCuIndexOverflow = Args.hasArg(OPT_continueOnCuIndexOverflow);
+  if (Args.hasArg(OPT_continueOnCuIndexOverflow)) {
+    ContinueOption =
+        Args.getLastArgValue(OPT_continueOnCuIndexOverflow, "continue");
+    if (ContinueOption == "soft-stop") {
+      OverflowOptValue = OnCuIndexOverflow::SoftStop;
+    } else {
+      OverflowOptValue = OnCuIndexOverflow::Continue;
+    }
+  }
 
   for (const llvm::opt::Arg *A : Args.filtered(OPT_execFileNames))
     ExecFilenames.emplace_back(A->getValue());
@@ -255,7 +264,7 @@ int llvm_dwp_main(int argc, char **argv, const llvm::ToolContext &) {
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
 
-  if (auto Err = write(*MS, DWOFilenames, ContinueOnCuIndexOverflow)) {
+  if (auto Err = write(*MS, DWOFilenames, OverflowOptValue)) {
     logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }


        


More information about the llvm-commits mailing list