[lld] [lld][ELF] Add --debug-names to create merged .debug_names. (PR #86508)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 16 22:58:25 PDT 2024
================
@@ -2711,6 +2714,599 @@ static uint32_t computeGdbHash(StringRef s) {
return h;
}
+// 4-byte alignment ensures that values in the hash lookup table and the name
+// table are aligned.
+DebugNamesBaseSection::DebugNamesBaseSection()
+ : SyntheticSection(0, SHT_PROGBITS, 4, ".debug_names") {}
+
+// Get the size of the .debug_names section header in bytes for DWARF32:
+static uint32_t getDebugNamesHeaderSize(uint32_t augmentationStringSize) {
+ return /* unit length */ 4 +
+ /* version */ 2 +
+ /* padding */ 2 +
+ /* CU count */ 4 +
+ /* TU count */ 4 +
+ /* Foreign TU count */ 4 +
+ /* Bucket Count */ 4 +
+ /* Name Count */ 4 +
+ /* Abbrev table size */ 4 +
+ /* Augmentation string size */ 4 +
+ /* Augmentation string */ augmentationStringSize;
+}
+
+void DebugNamesBaseSection::parseDebugNames(
+ InputChunk &inputChunk, OutputChunk &chunk,
+ DWARFDataExtractor &namesExtractor, DataExtractor &strExtractor,
+ function_ref<SmallVector<uint32_t, 0>(
+ uint32_t numCus, const DWARFDebugNames::Header &,
+ const DWARFDebugNames::DWARFDebugNamesOffsets &)>
+ readOffsets) {
+ const LLDDWARFSection namesSec = inputChunk.section;
+ DenseMap<uint32_t, IndexEntry *> offsetMap;
+ // Number of CUs seen in previous NameIndex sections within current chunk.
+ uint32_t numCus = 0;
+ for (const DWARFDebugNames::NameIndex &ni : *inputChunk.llvmDebugNames) {
+ NameData &nd = inputChunk.nameData.emplace_back();
+ nd.hdr = ni.getHeader();
+ if (nd.hdr.Format != DwarfFormat::DWARF32) {
+ errorOrWarn(toString(namesSec.sec) +
+ Twine(": found DWARF64, which is currently unsupported"));
+ return;
+ }
+ if (nd.hdr.Version != 5) {
+ errorOrWarn(toString(namesSec.sec) + Twine(": unsupported version: ") +
+ Twine(nd.hdr.Version));
+ return;
+ }
+ const uint32_t dwarfSize =
+ dwarf::getDwarfOffsetByteSize(DwarfFormat::DWARF32);
+ DWARFDebugNames::DWARFDebugNamesOffsets locs = ni.getOffsets();
+ if (locs.EntriesBase > namesExtractor.getData().size()) {
+ errorOrWarn(toString(namesSec.sec) +
+ Twine(": entry pool start is beyond end of section"));
+ return;
+ }
+
+ SmallVector<uint32_t, 0> entryOffsets = readOffsets(numCus, nd.hdr, locs);
+
+ // Read the entry pool.
+ offsetMap.clear();
+ nd.nameEntries.resize(nd.hdr.NameCount);
+ for (auto i : seq(nd.hdr.NameCount)) {
+ NameEntry &ne = nd.nameEntries[i];
+ uint64_t strOffset = locs.StringOffsetsBase + i * dwarfSize;
+ ne.stringOffset = strOffset;
+ uint64_t strp = namesExtractor.getRelocatedValue(dwarfSize, &strOffset);
+ StringRef name = strExtractor.getCStrRef(&strp);
+ ne.name = name.data();
+ ne.hashValue = caseFoldingDjbHash(name);
+
+ // Read a series of index entries that end with abbreviation code 0.
+ const char *errMsg = nullptr;
+ uint64_t offset = locs.EntriesBase + entryOffsets[i];
+ while (offset < namesSec.Data.size() && namesSec.Data[offset] != 0) {
+ // Read & store all entries (for the same string).
+ auto ie = makeThreadLocal<IndexEntry>();
+ ie->poolOffset = offset;
+ Error err = Error::success();
+ ie->abbrevCode =
+ static_cast<uint32_t>(namesExtractor.getULEB128(&offset, &err));
+ if (err) {
+ consumeError(std::move(err));
+ errMsg = ": invalid abbrev code in entry";
+ break;
+ }
+ auto it = ni.getAbbrevs().find_as(ie->abbrevCode);
+ if (it == ni.getAbbrevs().end()) {
+ errMsg = ": invalid abbrev code in entry";
+ break;
+ }
+
+ AttrValue attr, cuAttr = {0, 0};
+ for (DWARFDebugNames::AttributeEncoding a : it->Attributes) {
+ if (a.Index == dwarf::DW_IDX_parent) {
+ if (a.Form == dwarf::DW_FORM_ref4) {
+ attr.attrValue = namesExtractor.getU32(&offset, &err);
+ attr.attrSize = 4;
+ ie->parentOffset = locs.EntriesBase + attr.attrValue;
+ } else if (a.Form != DW_FORM_flag_present) {
+ errMsg = ": invalid form for DW_IDX_parent";
+ }
+ } else {
+ switch (a.Form) {
+ case DW_FORM_data1:
+ case DW_FORM_ref1: {
+ attr.attrValue = namesExtractor.getU8(&offset, &err);
+ attr.attrSize = 1;
+ break;
+ }
+ case DW_FORM_data2:
+ case DW_FORM_ref2: {
+ attr.attrValue = namesExtractor.getU16(&offset, &err);
+ attr.attrSize = 2;
+ break;
+ }
+ case DW_FORM_data4:
+ case DW_FORM_ref4: {
+ attr.attrValue = namesExtractor.getU32(&offset, &err);
+ attr.attrSize = 4;
+ break;
+ }
+ default:
+ errorOrWarn(toString(namesSec.sec) +
+ Twine(": unrecognized form encoding ") +
+ Twine(a.Form) + Twine(" in abbrev table"));
+ return;
+ }
+ }
+ if (err) {
+ errorOrWarn(toString(namesSec.sec) +
+ Twine(": error while reading attributes: ") +
+ toString(std::move(err)));
+ return;
+ }
+ if (a.Index == DW_IDX_compile_unit)
+ cuAttr = attr;
+ else if (a.Form != DW_FORM_flag_present)
+ ie->attrValues.push_back(attr);
+ }
+
+ // Canonicalize abbrev by placing the CU/TU index at the end.
+ ie->attrValues.push_back(cuAttr);
+ ne.indexEntries.push_back(std::move(ie));
+ }
+ if (offset >= namesSec.Data.size())
+ errMsg = ": index entry is out of bounds";
+ if (errMsg)
+ errorOrWarn(toString(namesSec.sec) + Twine(errMsg));
+
+ for (IndexEntry &ie : ne.entries())
+ offsetMap[ie.poolOffset] = &ie;
+ }
+
+ // Assign parent pointers, which will be used to update DW_IDX_parent index
+ // attributes. Note: offsetMap[0] does not exist, so parentOffset == 0 will
+ // get parentEntry == null as well.
+ for (NameEntry &ne : nd.nameEntries)
+ for (IndexEntry &ie : ne.entries())
+ ie.parentEntry = offsetMap.lookup(ie.parentOffset);
+ numCus += nd.hdr.CompUnitCount;
+ }
+}
+
+// Compute the form for output DW_IDX_compile_unit attributes, similar to
+// DIEInteger::BestForm. The input form (often DW_FORM_data1) may not hold all
+// the merged CU indices.
+std::pair<uint8_t, dwarf::Form> static getMergedCuCountForm(
+ uint32_t compUnitCount) {
+ if (compUnitCount > UINT16_MAX)
+ return {4, DW_FORM_data4};
+ if (compUnitCount > UINT8_MAX)
+ return {2, DW_FORM_data2};
+ return {1, DW_FORM_data1};
+}
+
+void DebugNamesBaseSection::computeHdrAndAbbrevTable(
+ MutableArrayRef<InputChunk> inputChunks) {
+ TimeTraceScope timeScope("Merge .debug_names", "hdr and abbrev table");
+ size_t numCu = 0;
+ hdr.Format = DwarfFormat::DWARF32;
+ hdr.Version = 5;
+ hdr.CompUnitCount = 0;
+ hdr.LocalTypeUnitCount = 0;
+ hdr.ForeignTypeUnitCount = 0;
+ hdr.AugmentationStringSize = 0;
+
+ // Compute CU and TU counts.
+ for (auto i : seq(numChunks)) {
+ InputChunk &inputChunk = inputChunks[i];
+ inputChunk.baseCuIdx = numCu;
+ numCu += chunks[i].compUnits.size();
+ for (const NameData &nd : inputChunk.nameData) {
+ hdr.CompUnitCount += nd.hdr.CompUnitCount;
+ hdr.LocalTypeUnitCount += nd.hdr.LocalTypeUnitCount;
+ hdr.ForeignTypeUnitCount += nd.hdr.ForeignTypeUnitCount;
+ // If augmentation strings are not identical, use an empty string.
+ if (i == 0) {
+ hdr.AugmentationStringSize = nd.hdr.AugmentationStringSize;
+ hdr.AugmentationString = nd.hdr.AugmentationString;
+ } else if (hdr.AugmentationString != nd.hdr.AugmentationString) {
+ // There are conflicting augmentation strings, so it's best for the
+ // merged index to not use an augmentation string.
+ hdr.AugmentationStringSize = 0;
+ hdr.AugmentationString.clear();
+ }
+ }
+ }
+
+ // Create the merged abbrev table, uniquifyinng the input abbrev tables and
+ // computing mapping from old (per-cu) abbrev codes to new (merged) abbrev
+ // codes.
+ FoldingSet<Abbrev> abbrevSet;
+ // Determine the form for the DW_IDX_compile_unit attributes in the merged
+ // index. The input form may not be big enough for all CU indices.
+ const dwarf::Form cuAttrForm = getMergedCuCountForm(hdr.CompUnitCount).second;
+ for (InputChunk &inputChunk : inputChunks) {
+ for (auto [i, ni] : enumerate(*inputChunk.llvmDebugNames)) {
+ for (const DWARFDebugNames::Abbrev &oldAbbrev : ni.getAbbrevs()) {
+ // Canonicalize abbrev by placing the CU/TU index at the end,
+ // similar to 'parseDebugNames'.
+ Abbrev abbrev;
+ DWARFDebugNames::AttributeEncoding cuAttr(DW_IDX_compile_unit,
+ cuAttrForm);
+ abbrev.code = oldAbbrev.Code;
+ abbrev.tag = oldAbbrev.Tag;
+ for (DWARFDebugNames::AttributeEncoding a : oldAbbrev.Attributes) {
+ if (a.Index == DW_IDX_compile_unit)
+ cuAttr.Index = a.Index;
+ else
+ abbrev.attributes.push_back({a.Index, a.Form});
+ }
+ // Put the CU/TU index at the end of the attributes list.
+ abbrev.attributes.push_back(cuAttr);
+
+ // Profile the abbrev, get or assign a new code, then record the abbrev
+ // code mapping.
+ FoldingSetNodeID id;
+ abbrev.Profile(id);
+ uint32_t newCode;
+ void *insertPos;
+ if (Abbrev *existing = abbrevSet.FindNodeOrInsertPos(id, insertPos)) {
+ // Found it; we've already seen an identical abbreviation.
+ newCode = existing->code;
+ } else {
+ Abbrev *abbrev2 =
+ new (abbrevAlloc.Allocate()) Abbrev(std::move(abbrev));
+ abbrevSet.InsertNode(abbrev2, insertPos);
+ abbrevTable.push_back(abbrev2);
+ newCode = abbrevTable.size();
+ abbrev2->code = newCode;
+ }
+ inputChunk.nameData[i].abbrevCodeMap[oldAbbrev.Code] = newCode;
+ }
+ }
+ }
+
+ // Compute the merged abbrev table.
+ raw_svector_ostream os(abbrevTableBuf);
+ for (Abbrev *abbrev : abbrevTable) {
+ encodeULEB128(abbrev->code, os);
+ encodeULEB128(abbrev->tag, os);
+ for (DWARFDebugNames::AttributeEncoding a : abbrev->attributes) {
+ encodeULEB128(a.Index, os);
+ encodeULEB128(a.Form, os);
+ }
+ os.write("\0", 2); // attribute specification end
+ }
+ os.write(0); // abbrev table end
+ hdr.AbbrevTableSize = abbrevTableBuf.size();
+}
+
+void DebugNamesBaseSection::Abbrev::Profile(FoldingSetNodeID &id) const {
+ id.AddInteger(tag);
+ for (const DWARFDebugNames::AttributeEncoding &attr : attributes) {
+ id.AddInteger(attr.Index);
+ id.AddInteger(attr.Form);
+ }
+}
+
+std::pair<uint32_t, uint32_t> DebugNamesBaseSection::computeEntryPool(
+ MutableArrayRef<InputChunk> inputChunks) {
+ TimeTraceScope timeScope("Merge .debug_names", "entry pool");
+ // Collect and de-duplicate all the names (preserving all the entries).
+ // Speed it up using multithreading, as the number of symbols can be in the
+ // order of millions.
+ const size_t concurrency =
+ bit_floor(std::min<size_t>(config->threadCount, numShards));
+ const size_t shift = 32 - countr_zero(numShards);
+ const uint8_t cuAttrSize = getMergedCuCountForm(hdr.CompUnitCount).first;
+ DenseMap<CachedHashStringRef, size_t> maps[numShards];
+
+ parallelFor(0, concurrency, [&](size_t threadId) {
+ for (auto i : seq(numChunks)) {
+ InputChunk &inputChunk = inputChunks[i];
+ for (auto j : seq(inputChunk.nameData.size())) {
+ NameData &nd = inputChunk.nameData[j];
+ // Deduplicate the NameEntry records (based on the string/name),
+ // appending all IndexEntries from duplicate NameEntry records to
+ // the single preserved copy.
+ for (NameEntry &ne : nd.nameEntries) {
+ auto shardId = ne.hashValue >> shift;
+ if ((shardId & (concurrency - 1)) != threadId)
+ continue;
+
+ ne.chunkIdx = i;
+ for (IndexEntry &ie : ne.entries()) {
+ // Update the IndexEntry's abbrev code to match the merged
+ // abbreviations.
+ ie.abbrevCode = nd.abbrevCodeMap[ie.abbrevCode];
+ // Update the DW_IDX_compile_unit attribute (the last one after
+ // canonicalization) to have correct merged offset value and size.
+ auto &back = ie.attrValues.back();
+ back.attrValue += inputChunk.baseCuIdx + j;
+ back.attrSize = cuAttrSize;
+ }
+
+ auto &nameVec = nameVecs[shardId];
+ auto [it, inserted] = maps[shardId].try_emplace(
+ CachedHashStringRef(ne.name, ne.hashValue), nameVec.size());
+ if (inserted)
+ nameVec.push_back(std::move(ne));
+ else
+ nameVec[it->second].indexEntries.append(std::move(ne.indexEntries));
+ }
+ }
+ }
+ });
+
+ // Compute entry offsets in parallel. First, comptute offsets relative to the
+ // current shard.
+ uint32_t offsets[numShards];
+ parallelFor(0, numShards, [&](size_t shard) {
+ uint32_t offset = 0;
+ for (NameEntry &ne : nameVecs[shard]) {
+ ne.entryOffset = offset;
+ for (IndexEntry &ie : ne.entries()) {
+ ie.poolOffset = offset;
+ offset += getULEB128Size(ie.abbrevCode);
+ for (AttrValue value : ie.attrValues)
+ offset += value.attrSize;
+ }
+ ++offset; // index entry sentinel
+ }
+ offsets[shard] = offset;
+ });
+ // Then add shard offsets.
+ std::partial_sum(offsets, std::end(offsets), offsets);
+ parallelFor(1, numShards, [&](size_t shard) {
+ uint32_t offset = offsets[shard - 1];
+ for (NameEntry &ne : nameVecs[shard]) {
+ ne.entryOffset += offset;
+ for (IndexEntry &ie : ne.entries())
+ ie.poolOffset += offset;
+ }
+ });
+
+ // Update the DW_IDX_parent entries that refer to real parents (have
+ // DW_FORM_ref4).
+ parallelFor(0, numShards, [&](size_t shard) {
+ for (NameEntry &ne : nameVecs[shard]) {
+ for (IndexEntry &ie : ne.entries()) {
+ if (!ie.parentEntry)
+ continue;
+ // Abbrevs are indexed starting at 1; vector starts at 0. (abbrevCode
+ // corresponds to position in the merged table vector).
+ const Abbrev *abbrev = abbrevTable[ie.abbrevCode - 1];
+ for (auto i : seq(abbrev->attributes.size())) {
+ DWARFDebugNames::AttributeEncoding a = abbrev->attributes[i];
+ if (a.Index == DW_IDX_parent && a.Form == DW_FORM_ref4)
+ ie.attrValues[i].attrValue = ie.parentEntry->poolOffset;
+ }
----------------
cmtice wrote:
I'll make this change, and there's one other place we use 'seq' that I can clean up. But the rest of the places where we use 'seq' we need to keep it because the loop index variable is also used within the loop for non-indexing purposes.
https://github.com/llvm/llvm-project/pull/86508
More information about the llvm-commits
mailing list