[clang] [llvm] [clang][Modules][Serialization] Avoid serializing duplicate source location entries for shared input files (PR #209795)
Ayokunle Amodu via cfe-commits
cfe-commits at lists.llvm.org
Thu Sep 10 03:34:04 PDT 2026
https://github.com/ayokunle321 updated https://github.com/llvm/llvm-project/pull/209795
>From f543791803cc4473ff1f353c31f5221912412d9d Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 29 Jun 2026 19:45:46 -0400
Subject: [PATCH 01/32] add detection of duplicate files
---
clang/include/clang/Basic/SourceManager.h | 27 +
clang/include/clang/Serialization/ASTReader.h | 8 +
clang/lib/Basic/SourceManager.cpp | 5 +
clang/lib/Serialization/ASTReader.cpp | 58 ++
dedup-test/a.h | 5 +
dedup-test/b.h | 5 +
dedup-test/module.modulemap | 2 +
dedup-test/one_module_stats.txt | 352 ++++++++++++
dedup-test/shared.h | 403 +++++++++++++
dedup-test/two_modules_stats.txt | 535 ++++++++++++++++++
dedup-test/use.cpp | 3 +
dedup-test/use1.cpp | 2 +
12 files changed, 1405 insertions(+)
create mode 100644 dedup-test/a.h
create mode 100644 dedup-test/b.h
create mode 100644 dedup-test/module.modulemap
create mode 100644 dedup-test/one_module_stats.txt
create mode 100644 dedup-test/shared.h
create mode 100644 dedup-test/two_modules_stats.txt
create mode 100644 dedup-test/use.cpp
create mode 100644 dedup-test/use1.cpp
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 1939d1aa4915e..fe9dd42989e50 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,6 +755,33 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
+ /// --- Source-location de-duplication: detection (prototype, Stage 1) ---
+ /// The first global offset at which each file's SLoc entry was loaded.
+ /// Keyed by FileEntry identity (the same identity Clang uses to dedup file
+ /// content), so we recognize when a later module re-loads the *same file*
+ /// already present in the loaded address space (e.g. a shared header).
+ llvm::DenseMap<const FileEntry *, SourceLocation::UIntTy>
+ LoadedFileFirstOffset;
+ /// Number of loaded file SLoc entries that duplicated an already-loaded file.
+ unsigned NumDuplicateLoadedFiles = 0;
+ /// SLoc address-space bytes occupied by those duplicates (the reuse prize).
+ uint64_t DuplicateLoadedBytes = 0;
+
+public:
+ /// Record a loaded file SLoc entry. If the *same file* was already loaded,
+ /// count it as a duplicate and add its size to the reusable total. Detection
+ /// only -- no behavior change.
+ void noteLoadedFileSLocEntry(const FileEntry *FE,
+ SourceLocation::UIntTy Offset, uint64_t Size) {
+ if (!FE)
+ return;
+ if (!LoadedFileFirstOffset.try_emplace(FE, Offset).second) {
+ ++NumDuplicateLoadedFiles;
+ DuplicateLoadedBytes += Size;
+ }
+ }
+
+private:
/// A bitmap that indicates whether the entries of LoadedSLocEntryTable
/// have already been loaded from the external source.
///
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index d800af83d350b..2afc4c199afda 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,6 +2399,14 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
+ /// Walk module \p F's SLoc entry records (read-only). Fills \p Offsets[i]
+ /// with each entry's local offset and \p Files[i] with its FileEntry (null
+ /// for non-file entries). Returns false if the records couldn't be read.
+ /// Shared scaffolding for source-location de-duplication.
+ bool scanLoadedSLocEntries(ModuleFile &F,
+ SmallVectorImpl<uint32_t> &Offsets,
+ SmallVectorImpl<const FileEntry *> &Files);
+
/// Retrieve the module import location and module name for the
/// given source manager entry ID.
std::pair<SourceLocation, StringRef> getModuleImportLoc(int ID) override;
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 5540aade05ef5..194426ae2f145 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2152,6 +2152,11 @@ void SourceManager::PrintStats() const {
<< " bytes of capacity), "
<< MaxLoadedOffset - CurrentLoadedOffset
<< "B of SLoc address space used.\n";
+ if (NumDuplicateLoadedFiles)
+ llvm::errs() << NumDuplicateLoadedFiles
+ << " duplicate loaded file SLocEntries detected ("
+ << DuplicateLoadedBytes
+ << "B of SLoc address space reusable).\n";
unsigned NumLineNumsComputed = 0;
unsigned NumFileBytesMapped = 0;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 3455b729be696..2283a9ee90b65 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1815,6 +1815,44 @@ llvm::Error ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
}
}
+bool ASTReader::scanLoadedSLocEntries(
+ ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
+ SmallVectorImpl<const FileEntry *> &Files) {
+ unsigned N = F.LocalNumSLocEntries;
+ Offsets.assign(N, 0);
+ Files.assign(N, nullptr);
+
+ BitstreamCursor &Cursor = F.SLocEntryCursor;
+ SavedStreamPosition SavedPosition(Cursor);
+ for (unsigned I = 0; I != N; ++I) {
+ if (llvm::Error Err = Cursor.JumpToBit(F.SLocEntryOffsetsBase +
+ F.SLocEntryOffsets[I])) {
+ consumeError(std::move(Err));
+ return false;
+ }
+ Expected<llvm::BitstreamEntry> Entry = Cursor.advance();
+ if (!Entry) {
+ consumeError(Entry.takeError());
+ return false;
+ }
+ if (Entry->Kind != llvm::BitstreamEntry::Record)
+ return false;
+
+ RecordData Record;
+ StringRef Blob;
+ Expected<unsigned> Code = Cursor.readRecord(Entry->ID, Record, &Blob);
+ if (!Code) {
+ consumeError(Code.takeError());
+ return false;
+ }
+ Offsets[I] = (uint32_t)Record[0];
+ if (Code.get() == SM_SLOC_FILE_ENTRY)
+ if (OptionalFileEntryRef File = getInputFile(F, Record[4]).getFile())
+ Files[I] = &File->getFileEntry();
+ }
+ return true;
+}
+
llvm::Expected<SourceLocation::UIntTy>
ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
BitstreamCursor &Cursor = F->SLocEntryCursor;
@@ -4259,6 +4297,26 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
- SLocSpaceSize,&F));
TotalNumSLocEntries += F.LocalNumSLocEntries;
+
+ // De-duplication detection (prototype, Stage 1). Walk this module's SLoc
+ // file entries and recognize when the same file was already loaded by an
+ // earlier module, measuring the address space a future reuse (Stage 2)
+ // could reclaim. Detection only: no allocation/translation change here.
+ {
+ SmallVector<uint32_t, 64> Offsets;
+ SmallVector<const FileEntry *, 64> Files;
+ if (scanLoadedSLocEntries(F, Offsets, Files)) {
+ unsigned N = Offsets.size();
+ for (unsigned I = 0; I != N; ++I) {
+ if (!Files[I])
+ continue;
+ uint64_t Size =
+ (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
+ SourceMgr.noteLoadedFileSLocEntry(
+ Files[I], F.SLocEntryBaseOffset + Offsets[I], Size);
+ }
+ }
+ }
break;
}
diff --git a/dedup-test/a.h b/dedup-test/a.h
new file mode 100644
index 0000000000000..1fe7fc27db549
--- /dev/null
+++ b/dedup-test/a.h
@@ -0,0 +1,5 @@
+#ifndef A_H
+#define A_H
+#include "shared.h"
+inline int a_entry(int x) { return shared_fn_0(x); }
+#endif
diff --git a/dedup-test/b.h b/dedup-test/b.h
new file mode 100644
index 0000000000000..85839261fc602
--- /dev/null
+++ b/dedup-test/b.h
@@ -0,0 +1,5 @@
+#ifndef B_H
+#define B_H
+#include "shared.h"
+inline int b_entry(int x) { return shared_fn_1(x); }
+#endif
diff --git a/dedup-test/module.modulemap b/dedup-test/module.modulemap
new file mode 100644
index 0000000000000..2da1c03a22211
--- /dev/null
+++ b/dedup-test/module.modulemap
@@ -0,0 +1,2 @@
+module A { header "a.h" export * }
+module B { header "b.h" export * }
diff --git a/dedup-test/one_module_stats.txt b/dedup-test/one_module_stats.txt
new file mode 100644
index 0000000000000..373fe97b5e1d8
--- /dev/null
+++ b/dedup-test/one_module_stats.txt
@@ -0,0 +1,352 @@
+
+STATISTICS:
+
+*** Semantic Analysis Stats:
+
+Number of memory regions: 0
+Bytes used: 0
+Bytes allocated: 0
+Bytes wasted: 0 (includes alignment, etc)
+
+*** Analysis Based Warnings Stats:
+401 functions analyzed (0 w/o CFGs).
+ 1203 CFG blocks built.
+ 3 average CFG blocks per function.
+ 3 max CFG blocks per function.
+0 functions analyzed for uninitialiazed variables
+ 0 variables analyzed.
+ 0 average variables per function.
+ 0 max variables per function.
+ 0 block visits.
+ 0 average block visits per function.
+ 0 max block visits per function.
+
+*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
+
+
+
+*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
+
+
+Total missing origins: 0
+
+****************************************
+
+*** AST Context Stats:
+ 156 types total.
+ 120 Builtin types, 32 each (3840 bytes)
+ 1 FunctionProto types, 48 each (48 bytes)
+ 5 Pointer types, 48 each (240 bytes)
+ 1 Record types, 32 each (32 bytes)
+ 1 Typedef types, 32 each (32 bytes)
+ 28 Vector types, 48 each (1344 bytes)
+Total bytes = 5536
+0/0 implicit default constructors created
+0/0 implicit copy constructors created
+0/0 implicit move constructors created
+0/0 implicit copy assignment operators created
+0/0 implicit move assignment operators created
+0/0 implicit destructors created
+
+*** AST File Statistics:
+
+*** PCH/ModuleFile Remappings:
+
+*** PCH/Modules Loaded:
+
+Number of memory regions: 62
+Bytes used: 248400
+Bytes allocated: 253952
+Bytes wasted: 5552 (includes alignment, etc)
+
+*** Decl Stats:
+ 995 decls total.
+ 1 TranslationUnit decls, 104 each (104 bytes)
+ 1 ExternCContext decls, 72 each (72 bytes)
+ 401 Function decls, 168 each (67368 bytes)
+ 401 ParmVar decls, 104 each (41704 bytes)
+ 8 Field decls, 80 each (640 bytes)
+ 2 CXXRecord decls, 144 each (288 bytes)
+ 180 Typedef decls, 88 each (15840 bytes)
+ 1 Import decls, 56 each (56 bytes)
+Total bytes = 126072
+
+*** Stmt/Expr Stats:
+ 2408 stmts/exprs total.
+ 1 UnresolvedLookupExpr, 64 each (64 bytes)
+ 400 IntegerLiteral, 32 each (12800 bytes)
+ 402 DeclRefExpr, 32 each (12864 bytes)
+ 402 ImplicitCastExpr, 24 each (9648 bytes)
+ 1 CallExpr, 24 each (24 bytes)
+ 400 BinaryOperator, 32 each (12800 bytes)
+ 401 ReturnStmt, 16 each (6416 bytes)
+ 401 CompoundStmt, 16 each (6416 bytes)
+Total bytes = 61032
+
+STATISTICS FOR './module.modulemap':
+
+*** Preprocessor Stats:
+564 directives found:
+ 553 #define.
+ 0 #undef.
+ #include/#include_next/#import:
+ 4 source files entered.
+ 1 max include stack depth
+ 2 #if/#ifndef/#ifdef.
+ 0 #else/#elif/#elifdef/#elifndef.
+ 2 #endif.
+ 2 #pragma.
+0 #if/#ifndef#ifdef regions skipped
+0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
+0 token paste (##) operations performed, 0 on the fast path.
+
+Preprocessor Memory: 110103B total
+ BumpPtr: 57344
+ Macro Expanded Tokens: 384
+ Predefines Buffer: 32767
+ Macros: 16512
+ #pragma push_macro Info: 2056
+ Poison Reasons: 1032
+ Comment Handlers: 8
+
+*** Identifier Table Stats:
+# Identifiers: 13432
+# Empty Buckets: 19336
+Hash density (#identifiers per bucket): 0.409912
+Ave identifier length: 26.735631
+Max identifier length: 49
+
+Number of memory regions: 182
+Bytes used: 909825
+Bytes allocated: 966656
+Bytes wasted: 56831 (includes alignment, etc)
+
+*** HeaderSearch Stats:
+3 files tracked.
+ 0 #import/#pragma once files.
+ 2 #include/#include_next/#import.
+ 0 #includes skipped due to the multi-include optimization.
+0 framework lookups.
+0 subframework lookups.
+
+*** Source Manager Stats:
+3 files mapped, 3 mem buffers mapped.
+7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
+0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
+20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
+FileID scans: 32 linear, 0 binary.
+
+
+*** File Manager Stats:
+5 real files found, 7 real dirs found.
+0 virtual files found, 0 virtual dirs found.
+28 dir lookups, 9 dir cache misses.
+19 file lookups, 8 file cache misses.
+
+*** Virtual File System Stats:
+14 status() calls
+5 openFileForRead() calls
+0 dir_begin() calls
+1 getRealPath() calls
+0 exists() calls
+0 isLocal() calls
+
+===-------------------------------------------------------------------------===
+ ... Statistics Collected ...
+===-------------------------------------------------------------------------===
+
+ 2 file-search - Number of attempted #includes.
+43765 source-manager - Maximum number of bytes used by source locations (both loaded and local).
+
+
+STATISTICS:
+
+*** Semantic Analysis Stats:
+
+Number of memory regions: 0
+Bytes used: 0
+Bytes allocated: 0
+Bytes wasted: 0 (includes alignment, etc)
+
+*** Analysis Based Warnings Stats:
+0 functions analyzed (0 w/o CFGs).
+ 0 CFG blocks built.
+ 0 average CFG blocks per function.
+ 0 max CFG blocks per function.
+0 functions analyzed for uninitialiazed variables
+ 0 variables analyzed.
+ 0 average variables per function.
+ 0 max variables per function.
+ 0 block visits.
+ 0 average block visits per function.
+ 0 max block visits per function.
+
+*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
+
+
+
+*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
+
+
+Total missing origins: 0
+
+****************************************
+
+*** AST Context Stats:
+ 157 types total.
+ 120 Builtin types, 32 each (3840 bytes)
+ 2 FunctionProto types, 48 each (96 bytes)
+ 6 Pointer types, 48 each (288 bytes)
+ 1 Record types, 32 each (32 bytes)
+ 28 Vector types, 48 each (1344 bytes)
+Total bytes = 5600
+0/0 implicit default constructors created
+0/0 implicit copy constructors created
+0/0 implicit move constructors created
+0/0 implicit copy assignment operators created
+0/0 implicit move assignment operators created
+0/0 implicit destructors created
+
+*** AST File Statistics:
+ 0/6 source location entries read (0.000000%)
+ 2/32 types read (6.250000%)
+ 516/888 declarations read (58.108109%)
+ 7/496 identifiers read (1.411290%)
+ 0/2 macros read (0.000000%)
+ 7/2407 statements read (0.290818%)
+ 0/2 macros read (0.000000%)
+ 0/401 lexical declcontexts read (0.000000%)
+
+*** PCH/ModuleFile Remappings:
+Global bit offset map:
+ 0 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+Global source location entry map:
+ 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+Global submodule map:
+ 1 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+
+*** PCH/Modules Loaded:
+Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+ Base source location offset: 2147439884
+ Base identifier ID: 0
+ Number of identifiers: 496
+ Base macro ID: 0
+ Number of macros: 2
+ Base submodule ID: 0
+ Number of submodules: 1
+ Submodule ID local -> global map:
+ 0 -> 0
+ Base selector ID: 0
+ Number of selectors: 0
+ Base preprocessed entity ID: 0
+ Number of preprocessed entities: 0
+ Base type index: 0
+ Number of types: 32
+ Base decl index: 0
+ Number of decls: 888
+
+
+Number of memory regions: 5
+Bytes used: 18452
+Bytes allocated: 20480
+Bytes wasted: 2028 (includes alignment, etc)
+
+*** Decl Stats:
+ 1002 decls total.
+ 1 TranslationUnit decls, 104 each (104 bytes)
+ 2 ExternCContext decls, 72 each (144 bytes)
+ 404 Function decls, 168 each (67872 bytes)
+ 403 ParmVar decls, 104 each (41912 bytes)
+ 8 Field decls, 80 each (640 bytes)
+ 2 CXXRecord decls, 144 each (288 bytes)
+ 180 Typedef decls, 88 each (15840 bytes)
+ 2 Import decls, 56 each (112 bytes)
+Total bytes = 126912
+
+*** Stmt/Expr Stats:
+ 2422 stmts/exprs total.
+ 2 UnresolvedLookupExpr, 64 each (128 bytes)
+ 401 IntegerLiteral, 32 each (12832 bytes)
+ 405 DeclRefExpr, 32 each (12960 bytes)
+ 405 ImplicitCastExpr, 24 each (9720 bytes)
+ 3 CallExpr, 24 each (72 bytes)
+ 400 BinaryOperator, 32 each (12800 bytes)
+ 403 ReturnStmt, 16 each (6448 bytes)
+ 403 CompoundStmt, 16 each (6448 bytes)
+Total bytes = 61408
+
+STATISTICS FOR 'use1.cpp':
+
+*** Preprocessor Stats:
+558 directives found:
+ 552 #define.
+ 0 #undef.
+ #include/#include_next/#import:
+ 2 source files entered.
+ 0 max include stack depth
+ 0 #if/#ifndef/#ifdef.
+ 0 #else/#elif/#elifdef/#elifndef.
+ 0 #endif.
+ 2 #pragma.
+0 #if/#ifndef#ifdef regions skipped
+0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
+0 token paste (##) operations performed, 0 on the fast path.
+
+Preprocessor Memory: 106007B total
+ BumpPtr: 53248
+ Macro Expanded Tokens: 384
+ Predefines Buffer: 32767
+ Macros: 16512
+ #pragma push_macro Info: 2056
+ Poison Reasons: 1032
+ Comment Handlers: 8
+
+*** Identifier Table Stats:
+# Identifiers: 13033
+# Empty Buckets: 19735
+Hash density (#identifiers per bucket): 0.397736
+Ave identifier length: 27.165119
+Max identifier length: 49
+
+Number of memory regions: 180
+Bytes used: 888396
+Bytes allocated: 950272
+Bytes wasted: 61876 (includes alignment, etc)
+
+*** HeaderSearch Stats:
+3 files tracked.
+ 0 #import/#pragma once files.
+ 2 #include/#include_next/#import.
+ 0 #includes skipped due to the multi-include optimization.
+0 framework lookups.
+0 subframework lookups.
+
+*** Source Manager Stats:
+2 files mapped, 2 mem buffers mapped.
+5 local SLocEntries allocated (168 bytes of capacity), 23108B of SLoc address space used.
+6 loaded SLocEntries allocated (768 bytes of capacity), 43764B of SLoc address space used.
+119 bytes of files mapped, 0 files with line #'s computed, 0 files with macro args computed.
+FileID scans: 11 linear, 0 binary.
+
+
+*** File Manager Stats:
+6 real files found, 8 real dirs found.
+0 virtual files found, 0 virtual dirs found.
+31 dir lookups, 11 dir cache misses.
+27 file lookups, 9 file cache misses.
+
+*** Virtual File System Stats:
+17 status() calls
+6 openFileForRead() calls
+0 dir_begin() calls
+1 getRealPath() calls
+0 exists() calls
+0 isLocal() calls
+
+===-------------------------------------------------------------------------===
+ ... Statistics Collected ...
+===-------------------------------------------------------------------------===
+
+ 2 file-search - Number of attempted #includes.
+66872 source-manager - Maximum number of bytes used by source locations (both loaded and local).
+
diff --git a/dedup-test/shared.h b/dedup-test/shared.h
new file mode 100644
index 0000000000000..3ac57a45355f9
--- /dev/null
+++ b/dedup-test/shared.h
@@ -0,0 +1,403 @@
+#ifndef SHARED_H
+#define SHARED_H
+inline int shared_fn_0(int x) { return x + 0; }
+inline int shared_fn_1(int x) { return x + 1; }
+inline int shared_fn_2(int x) { return x + 2; }
+inline int shared_fn_3(int x) { return x + 3; }
+inline int shared_fn_4(int x) { return x + 4; }
+inline int shared_fn_5(int x) { return x + 5; }
+inline int shared_fn_6(int x) { return x + 6; }
+inline int shared_fn_7(int x) { return x + 7; }
+inline int shared_fn_8(int x) { return x + 8; }
+inline int shared_fn_9(int x) { return x + 9; }
+inline int shared_fn_10(int x) { return x + 10; }
+inline int shared_fn_11(int x) { return x + 11; }
+inline int shared_fn_12(int x) { return x + 12; }
+inline int shared_fn_13(int x) { return x + 13; }
+inline int shared_fn_14(int x) { return x + 14; }
+inline int shared_fn_15(int x) { return x + 15; }
+inline int shared_fn_16(int x) { return x + 16; }
+inline int shared_fn_17(int x) { return x + 17; }
+inline int shared_fn_18(int x) { return x + 18; }
+inline int shared_fn_19(int x) { return x + 19; }
+inline int shared_fn_20(int x) { return x + 20; }
+inline int shared_fn_21(int x) { return x + 21; }
+inline int shared_fn_22(int x) { return x + 22; }
+inline int shared_fn_23(int x) { return x + 23; }
+inline int shared_fn_24(int x) { return x + 24; }
+inline int shared_fn_25(int x) { return x + 25; }
+inline int shared_fn_26(int x) { return x + 26; }
+inline int shared_fn_27(int x) { return x + 27; }
+inline int shared_fn_28(int x) { return x + 28; }
+inline int shared_fn_29(int x) { return x + 29; }
+inline int shared_fn_30(int x) { return x + 30; }
+inline int shared_fn_31(int x) { return x + 31; }
+inline int shared_fn_32(int x) { return x + 32; }
+inline int shared_fn_33(int x) { return x + 33; }
+inline int shared_fn_34(int x) { return x + 34; }
+inline int shared_fn_35(int x) { return x + 35; }
+inline int shared_fn_36(int x) { return x + 36; }
+inline int shared_fn_37(int x) { return x + 37; }
+inline int shared_fn_38(int x) { return x + 38; }
+inline int shared_fn_39(int x) { return x + 39; }
+inline int shared_fn_40(int x) { return x + 40; }
+inline int shared_fn_41(int x) { return x + 41; }
+inline int shared_fn_42(int x) { return x + 42; }
+inline int shared_fn_43(int x) { return x + 43; }
+inline int shared_fn_44(int x) { return x + 44; }
+inline int shared_fn_45(int x) { return x + 45; }
+inline int shared_fn_46(int x) { return x + 46; }
+inline int shared_fn_47(int x) { return x + 47; }
+inline int shared_fn_48(int x) { return x + 48; }
+inline int shared_fn_49(int x) { return x + 49; }
+inline int shared_fn_50(int x) { return x + 50; }
+inline int shared_fn_51(int x) { return x + 51; }
+inline int shared_fn_52(int x) { return x + 52; }
+inline int shared_fn_53(int x) { return x + 53; }
+inline int shared_fn_54(int x) { return x + 54; }
+inline int shared_fn_55(int x) { return x + 55; }
+inline int shared_fn_56(int x) { return x + 56; }
+inline int shared_fn_57(int x) { return x + 57; }
+inline int shared_fn_58(int x) { return x + 58; }
+inline int shared_fn_59(int x) { return x + 59; }
+inline int shared_fn_60(int x) { return x + 60; }
+inline int shared_fn_61(int x) { return x + 61; }
+inline int shared_fn_62(int x) { return x + 62; }
+inline int shared_fn_63(int x) { return x + 63; }
+inline int shared_fn_64(int x) { return x + 64; }
+inline int shared_fn_65(int x) { return x + 65; }
+inline int shared_fn_66(int x) { return x + 66; }
+inline int shared_fn_67(int x) { return x + 67; }
+inline int shared_fn_68(int x) { return x + 68; }
+inline int shared_fn_69(int x) { return x + 69; }
+inline int shared_fn_70(int x) { return x + 70; }
+inline int shared_fn_71(int x) { return x + 71; }
+inline int shared_fn_72(int x) { return x + 72; }
+inline int shared_fn_73(int x) { return x + 73; }
+inline int shared_fn_74(int x) { return x + 74; }
+inline int shared_fn_75(int x) { return x + 75; }
+inline int shared_fn_76(int x) { return x + 76; }
+inline int shared_fn_77(int x) { return x + 77; }
+inline int shared_fn_78(int x) { return x + 78; }
+inline int shared_fn_79(int x) { return x + 79; }
+inline int shared_fn_80(int x) { return x + 80; }
+inline int shared_fn_81(int x) { return x + 81; }
+inline int shared_fn_82(int x) { return x + 82; }
+inline int shared_fn_83(int x) { return x + 83; }
+inline int shared_fn_84(int x) { return x + 84; }
+inline int shared_fn_85(int x) { return x + 85; }
+inline int shared_fn_86(int x) { return x + 86; }
+inline int shared_fn_87(int x) { return x + 87; }
+inline int shared_fn_88(int x) { return x + 88; }
+inline int shared_fn_89(int x) { return x + 89; }
+inline int shared_fn_90(int x) { return x + 90; }
+inline int shared_fn_91(int x) { return x + 91; }
+inline int shared_fn_92(int x) { return x + 92; }
+inline int shared_fn_93(int x) { return x + 93; }
+inline int shared_fn_94(int x) { return x + 94; }
+inline int shared_fn_95(int x) { return x + 95; }
+inline int shared_fn_96(int x) { return x + 96; }
+inline int shared_fn_97(int x) { return x + 97; }
+inline int shared_fn_98(int x) { return x + 98; }
+inline int shared_fn_99(int x) { return x + 99; }
+inline int shared_fn_100(int x) { return x + 100; }
+inline int shared_fn_101(int x) { return x + 101; }
+inline int shared_fn_102(int x) { return x + 102; }
+inline int shared_fn_103(int x) { return x + 103; }
+inline int shared_fn_104(int x) { return x + 104; }
+inline int shared_fn_105(int x) { return x + 105; }
+inline int shared_fn_106(int x) { return x + 106; }
+inline int shared_fn_107(int x) { return x + 107; }
+inline int shared_fn_108(int x) { return x + 108; }
+inline int shared_fn_109(int x) { return x + 109; }
+inline int shared_fn_110(int x) { return x + 110; }
+inline int shared_fn_111(int x) { return x + 111; }
+inline int shared_fn_112(int x) { return x + 112; }
+inline int shared_fn_113(int x) { return x + 113; }
+inline int shared_fn_114(int x) { return x + 114; }
+inline int shared_fn_115(int x) { return x + 115; }
+inline int shared_fn_116(int x) { return x + 116; }
+inline int shared_fn_117(int x) { return x + 117; }
+inline int shared_fn_118(int x) { return x + 118; }
+inline int shared_fn_119(int x) { return x + 119; }
+inline int shared_fn_120(int x) { return x + 120; }
+inline int shared_fn_121(int x) { return x + 121; }
+inline int shared_fn_122(int x) { return x + 122; }
+inline int shared_fn_123(int x) { return x + 123; }
+inline int shared_fn_124(int x) { return x + 124; }
+inline int shared_fn_125(int x) { return x + 125; }
+inline int shared_fn_126(int x) { return x + 126; }
+inline int shared_fn_127(int x) { return x + 127; }
+inline int shared_fn_128(int x) { return x + 128; }
+inline int shared_fn_129(int x) { return x + 129; }
+inline int shared_fn_130(int x) { return x + 130; }
+inline int shared_fn_131(int x) { return x + 131; }
+inline int shared_fn_132(int x) { return x + 132; }
+inline int shared_fn_133(int x) { return x + 133; }
+inline int shared_fn_134(int x) { return x + 134; }
+inline int shared_fn_135(int x) { return x + 135; }
+inline int shared_fn_136(int x) { return x + 136; }
+inline int shared_fn_137(int x) { return x + 137; }
+inline int shared_fn_138(int x) { return x + 138; }
+inline int shared_fn_139(int x) { return x + 139; }
+inline int shared_fn_140(int x) { return x + 140; }
+inline int shared_fn_141(int x) { return x + 141; }
+inline int shared_fn_142(int x) { return x + 142; }
+inline int shared_fn_143(int x) { return x + 143; }
+inline int shared_fn_144(int x) { return x + 144; }
+inline int shared_fn_145(int x) { return x + 145; }
+inline int shared_fn_146(int x) { return x + 146; }
+inline int shared_fn_147(int x) { return x + 147; }
+inline int shared_fn_148(int x) { return x + 148; }
+inline int shared_fn_149(int x) { return x + 149; }
+inline int shared_fn_150(int x) { return x + 150; }
+inline int shared_fn_151(int x) { return x + 151; }
+inline int shared_fn_152(int x) { return x + 152; }
+inline int shared_fn_153(int x) { return x + 153; }
+inline int shared_fn_154(int x) { return x + 154; }
+inline int shared_fn_155(int x) { return x + 155; }
+inline int shared_fn_156(int x) { return x + 156; }
+inline int shared_fn_157(int x) { return x + 157; }
+inline int shared_fn_158(int x) { return x + 158; }
+inline int shared_fn_159(int x) { return x + 159; }
+inline int shared_fn_160(int x) { return x + 160; }
+inline int shared_fn_161(int x) { return x + 161; }
+inline int shared_fn_162(int x) { return x + 162; }
+inline int shared_fn_163(int x) { return x + 163; }
+inline int shared_fn_164(int x) { return x + 164; }
+inline int shared_fn_165(int x) { return x + 165; }
+inline int shared_fn_166(int x) { return x + 166; }
+inline int shared_fn_167(int x) { return x + 167; }
+inline int shared_fn_168(int x) { return x + 168; }
+inline int shared_fn_169(int x) { return x + 169; }
+inline int shared_fn_170(int x) { return x + 170; }
+inline int shared_fn_171(int x) { return x + 171; }
+inline int shared_fn_172(int x) { return x + 172; }
+inline int shared_fn_173(int x) { return x + 173; }
+inline int shared_fn_174(int x) { return x + 174; }
+inline int shared_fn_175(int x) { return x + 175; }
+inline int shared_fn_176(int x) { return x + 176; }
+inline int shared_fn_177(int x) { return x + 177; }
+inline int shared_fn_178(int x) { return x + 178; }
+inline int shared_fn_179(int x) { return x + 179; }
+inline int shared_fn_180(int x) { return x + 180; }
+inline int shared_fn_181(int x) { return x + 181; }
+inline int shared_fn_182(int x) { return x + 182; }
+inline int shared_fn_183(int x) { return x + 183; }
+inline int shared_fn_184(int x) { return x + 184; }
+inline int shared_fn_185(int x) { return x + 185; }
+inline int shared_fn_186(int x) { return x + 186; }
+inline int shared_fn_187(int x) { return x + 187; }
+inline int shared_fn_188(int x) { return x + 188; }
+inline int shared_fn_189(int x) { return x + 189; }
+inline int shared_fn_190(int x) { return x + 190; }
+inline int shared_fn_191(int x) { return x + 191; }
+inline int shared_fn_192(int x) { return x + 192; }
+inline int shared_fn_193(int x) { return x + 193; }
+inline int shared_fn_194(int x) { return x + 194; }
+inline int shared_fn_195(int x) { return x + 195; }
+inline int shared_fn_196(int x) { return x + 196; }
+inline int shared_fn_197(int x) { return x + 197; }
+inline int shared_fn_198(int x) { return x + 198; }
+inline int shared_fn_199(int x) { return x + 199; }
+inline int shared_fn_200(int x) { return x + 200; }
+inline int shared_fn_201(int x) { return x + 201; }
+inline int shared_fn_202(int x) { return x + 202; }
+inline int shared_fn_203(int x) { return x + 203; }
+inline int shared_fn_204(int x) { return x + 204; }
+inline int shared_fn_205(int x) { return x + 205; }
+inline int shared_fn_206(int x) { return x + 206; }
+inline int shared_fn_207(int x) { return x + 207; }
+inline int shared_fn_208(int x) { return x + 208; }
+inline int shared_fn_209(int x) { return x + 209; }
+inline int shared_fn_210(int x) { return x + 210; }
+inline int shared_fn_211(int x) { return x + 211; }
+inline int shared_fn_212(int x) { return x + 212; }
+inline int shared_fn_213(int x) { return x + 213; }
+inline int shared_fn_214(int x) { return x + 214; }
+inline int shared_fn_215(int x) { return x + 215; }
+inline int shared_fn_216(int x) { return x + 216; }
+inline int shared_fn_217(int x) { return x + 217; }
+inline int shared_fn_218(int x) { return x + 218; }
+inline int shared_fn_219(int x) { return x + 219; }
+inline int shared_fn_220(int x) { return x + 220; }
+inline int shared_fn_221(int x) { return x + 221; }
+inline int shared_fn_222(int x) { return x + 222; }
+inline int shared_fn_223(int x) { return x + 223; }
+inline int shared_fn_224(int x) { return x + 224; }
+inline int shared_fn_225(int x) { return x + 225; }
+inline int shared_fn_226(int x) { return x + 226; }
+inline int shared_fn_227(int x) { return x + 227; }
+inline int shared_fn_228(int x) { return x + 228; }
+inline int shared_fn_229(int x) { return x + 229; }
+inline int shared_fn_230(int x) { return x + 230; }
+inline int shared_fn_231(int x) { return x + 231; }
+inline int shared_fn_232(int x) { return x + 232; }
+inline int shared_fn_233(int x) { return x + 233; }
+inline int shared_fn_234(int x) { return x + 234; }
+inline int shared_fn_235(int x) { return x + 235; }
+inline int shared_fn_236(int x) { return x + 236; }
+inline int shared_fn_237(int x) { return x + 237; }
+inline int shared_fn_238(int x) { return x + 238; }
+inline int shared_fn_239(int x) { return x + 239; }
+inline int shared_fn_240(int x) { return x + 240; }
+inline int shared_fn_241(int x) { return x + 241; }
+inline int shared_fn_242(int x) { return x + 242; }
+inline int shared_fn_243(int x) { return x + 243; }
+inline int shared_fn_244(int x) { return x + 244; }
+inline int shared_fn_245(int x) { return x + 245; }
+inline int shared_fn_246(int x) { return x + 246; }
+inline int shared_fn_247(int x) { return x + 247; }
+inline int shared_fn_248(int x) { return x + 248; }
+inline int shared_fn_249(int x) { return x + 249; }
+inline int shared_fn_250(int x) { return x + 250; }
+inline int shared_fn_251(int x) { return x + 251; }
+inline int shared_fn_252(int x) { return x + 252; }
+inline int shared_fn_253(int x) { return x + 253; }
+inline int shared_fn_254(int x) { return x + 254; }
+inline int shared_fn_255(int x) { return x + 255; }
+inline int shared_fn_256(int x) { return x + 256; }
+inline int shared_fn_257(int x) { return x + 257; }
+inline int shared_fn_258(int x) { return x + 258; }
+inline int shared_fn_259(int x) { return x + 259; }
+inline int shared_fn_260(int x) { return x + 260; }
+inline int shared_fn_261(int x) { return x + 261; }
+inline int shared_fn_262(int x) { return x + 262; }
+inline int shared_fn_263(int x) { return x + 263; }
+inline int shared_fn_264(int x) { return x + 264; }
+inline int shared_fn_265(int x) { return x + 265; }
+inline int shared_fn_266(int x) { return x + 266; }
+inline int shared_fn_267(int x) { return x + 267; }
+inline int shared_fn_268(int x) { return x + 268; }
+inline int shared_fn_269(int x) { return x + 269; }
+inline int shared_fn_270(int x) { return x + 270; }
+inline int shared_fn_271(int x) { return x + 271; }
+inline int shared_fn_272(int x) { return x + 272; }
+inline int shared_fn_273(int x) { return x + 273; }
+inline int shared_fn_274(int x) { return x + 274; }
+inline int shared_fn_275(int x) { return x + 275; }
+inline int shared_fn_276(int x) { return x + 276; }
+inline int shared_fn_277(int x) { return x + 277; }
+inline int shared_fn_278(int x) { return x + 278; }
+inline int shared_fn_279(int x) { return x + 279; }
+inline int shared_fn_280(int x) { return x + 280; }
+inline int shared_fn_281(int x) { return x + 281; }
+inline int shared_fn_282(int x) { return x + 282; }
+inline int shared_fn_283(int x) { return x + 283; }
+inline int shared_fn_284(int x) { return x + 284; }
+inline int shared_fn_285(int x) { return x + 285; }
+inline int shared_fn_286(int x) { return x + 286; }
+inline int shared_fn_287(int x) { return x + 287; }
+inline int shared_fn_288(int x) { return x + 288; }
+inline int shared_fn_289(int x) { return x + 289; }
+inline int shared_fn_290(int x) { return x + 290; }
+inline int shared_fn_291(int x) { return x + 291; }
+inline int shared_fn_292(int x) { return x + 292; }
+inline int shared_fn_293(int x) { return x + 293; }
+inline int shared_fn_294(int x) { return x + 294; }
+inline int shared_fn_295(int x) { return x + 295; }
+inline int shared_fn_296(int x) { return x + 296; }
+inline int shared_fn_297(int x) { return x + 297; }
+inline int shared_fn_298(int x) { return x + 298; }
+inline int shared_fn_299(int x) { return x + 299; }
+inline int shared_fn_300(int x) { return x + 300; }
+inline int shared_fn_301(int x) { return x + 301; }
+inline int shared_fn_302(int x) { return x + 302; }
+inline int shared_fn_303(int x) { return x + 303; }
+inline int shared_fn_304(int x) { return x + 304; }
+inline int shared_fn_305(int x) { return x + 305; }
+inline int shared_fn_306(int x) { return x + 306; }
+inline int shared_fn_307(int x) { return x + 307; }
+inline int shared_fn_308(int x) { return x + 308; }
+inline int shared_fn_309(int x) { return x + 309; }
+inline int shared_fn_310(int x) { return x + 310; }
+inline int shared_fn_311(int x) { return x + 311; }
+inline int shared_fn_312(int x) { return x + 312; }
+inline int shared_fn_313(int x) { return x + 313; }
+inline int shared_fn_314(int x) { return x + 314; }
+inline int shared_fn_315(int x) { return x + 315; }
+inline int shared_fn_316(int x) { return x + 316; }
+inline int shared_fn_317(int x) { return x + 317; }
+inline int shared_fn_318(int x) { return x + 318; }
+inline int shared_fn_319(int x) { return x + 319; }
+inline int shared_fn_320(int x) { return x + 320; }
+inline int shared_fn_321(int x) { return x + 321; }
+inline int shared_fn_322(int x) { return x + 322; }
+inline int shared_fn_323(int x) { return x + 323; }
+inline int shared_fn_324(int x) { return x + 324; }
+inline int shared_fn_325(int x) { return x + 325; }
+inline int shared_fn_326(int x) { return x + 326; }
+inline int shared_fn_327(int x) { return x + 327; }
+inline int shared_fn_328(int x) { return x + 328; }
+inline int shared_fn_329(int x) { return x + 329; }
+inline int shared_fn_330(int x) { return x + 330; }
+inline int shared_fn_331(int x) { return x + 331; }
+inline int shared_fn_332(int x) { return x + 332; }
+inline int shared_fn_333(int x) { return x + 333; }
+inline int shared_fn_334(int x) { return x + 334; }
+inline int shared_fn_335(int x) { return x + 335; }
+inline int shared_fn_336(int x) { return x + 336; }
+inline int shared_fn_337(int x) { return x + 337; }
+inline int shared_fn_338(int x) { return x + 338; }
+inline int shared_fn_339(int x) { return x + 339; }
+inline int shared_fn_340(int x) { return x + 340; }
+inline int shared_fn_341(int x) { return x + 341; }
+inline int shared_fn_342(int x) { return x + 342; }
+inline int shared_fn_343(int x) { return x + 343; }
+inline int shared_fn_344(int x) { return x + 344; }
+inline int shared_fn_345(int x) { return x + 345; }
+inline int shared_fn_346(int x) { return x + 346; }
+inline int shared_fn_347(int x) { return x + 347; }
+inline int shared_fn_348(int x) { return x + 348; }
+inline int shared_fn_349(int x) { return x + 349; }
+inline int shared_fn_350(int x) { return x + 350; }
+inline int shared_fn_351(int x) { return x + 351; }
+inline int shared_fn_352(int x) { return x + 352; }
+inline int shared_fn_353(int x) { return x + 353; }
+inline int shared_fn_354(int x) { return x + 354; }
+inline int shared_fn_355(int x) { return x + 355; }
+inline int shared_fn_356(int x) { return x + 356; }
+inline int shared_fn_357(int x) { return x + 357; }
+inline int shared_fn_358(int x) { return x + 358; }
+inline int shared_fn_359(int x) { return x + 359; }
+inline int shared_fn_360(int x) { return x + 360; }
+inline int shared_fn_361(int x) { return x + 361; }
+inline int shared_fn_362(int x) { return x + 362; }
+inline int shared_fn_363(int x) { return x + 363; }
+inline int shared_fn_364(int x) { return x + 364; }
+inline int shared_fn_365(int x) { return x + 365; }
+inline int shared_fn_366(int x) { return x + 366; }
+inline int shared_fn_367(int x) { return x + 367; }
+inline int shared_fn_368(int x) { return x + 368; }
+inline int shared_fn_369(int x) { return x + 369; }
+inline int shared_fn_370(int x) { return x + 370; }
+inline int shared_fn_371(int x) { return x + 371; }
+inline int shared_fn_372(int x) { return x + 372; }
+inline int shared_fn_373(int x) { return x + 373; }
+inline int shared_fn_374(int x) { return x + 374; }
+inline int shared_fn_375(int x) { return x + 375; }
+inline int shared_fn_376(int x) { return x + 376; }
+inline int shared_fn_377(int x) { return x + 377; }
+inline int shared_fn_378(int x) { return x + 378; }
+inline int shared_fn_379(int x) { return x + 379; }
+inline int shared_fn_380(int x) { return x + 380; }
+inline int shared_fn_381(int x) { return x + 381; }
+inline int shared_fn_382(int x) { return x + 382; }
+inline int shared_fn_383(int x) { return x + 383; }
+inline int shared_fn_384(int x) { return x + 384; }
+inline int shared_fn_385(int x) { return x + 385; }
+inline int shared_fn_386(int x) { return x + 386; }
+inline int shared_fn_387(int x) { return x + 387; }
+inline int shared_fn_388(int x) { return x + 388; }
+inline int shared_fn_389(int x) { return x + 389; }
+inline int shared_fn_390(int x) { return x + 390; }
+inline int shared_fn_391(int x) { return x + 391; }
+inline int shared_fn_392(int x) { return x + 392; }
+inline int shared_fn_393(int x) { return x + 393; }
+inline int shared_fn_394(int x) { return x + 394; }
+inline int shared_fn_395(int x) { return x + 395; }
+inline int shared_fn_396(int x) { return x + 396; }
+inline int shared_fn_397(int x) { return x + 397; }
+inline int shared_fn_398(int x) { return x + 398; }
+inline int shared_fn_399(int x) { return x + 399; }
+#endif
diff --git a/dedup-test/two_modules_stats.txt b/dedup-test/two_modules_stats.txt
new file mode 100644
index 0000000000000..dd3a1f0c2d3ac
--- /dev/null
+++ b/dedup-test/two_modules_stats.txt
@@ -0,0 +1,535 @@
+
+STATISTICS:
+
+*** Semantic Analysis Stats:
+
+Number of memory regions: 0
+Bytes used: 0
+Bytes allocated: 0
+Bytes wasted: 0 (includes alignment, etc)
+
+*** Analysis Based Warnings Stats:
+401 functions analyzed (0 w/o CFGs).
+ 1203 CFG blocks built.
+ 3 average CFG blocks per function.
+ 3 max CFG blocks per function.
+0 functions analyzed for uninitialiazed variables
+ 0 variables analyzed.
+ 0 average variables per function.
+ 0 max variables per function.
+ 0 block visits.
+ 0 average block visits per function.
+ 0 max block visits per function.
+
+*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
+
+
+
+*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
+
+
+Total missing origins: 0
+
+****************************************
+
+*** AST Context Stats:
+ 156 types total.
+ 120 Builtin types, 32 each (3840 bytes)
+ 1 FunctionProto types, 48 each (48 bytes)
+ 5 Pointer types, 48 each (240 bytes)
+ 1 Record types, 32 each (32 bytes)
+ 1 Typedef types, 32 each (32 bytes)
+ 28 Vector types, 48 each (1344 bytes)
+Total bytes = 5536
+0/0 implicit default constructors created
+0/0 implicit copy constructors created
+0/0 implicit move constructors created
+0/0 implicit copy assignment operators created
+0/0 implicit move assignment operators created
+0/0 implicit destructors created
+
+*** AST File Statistics:
+
+*** PCH/ModuleFile Remappings:
+
+*** PCH/Modules Loaded:
+
+Number of memory regions: 62
+Bytes used: 248400
+Bytes allocated: 253952
+Bytes wasted: 5552 (includes alignment, etc)
+
+*** Decl Stats:
+ 995 decls total.
+ 1 TranslationUnit decls, 104 each (104 bytes)
+ 1 ExternCContext decls, 72 each (72 bytes)
+ 401 Function decls, 168 each (67368 bytes)
+ 401 ParmVar decls, 104 each (41704 bytes)
+ 8 Field decls, 80 each (640 bytes)
+ 2 CXXRecord decls, 144 each (288 bytes)
+ 180 Typedef decls, 88 each (15840 bytes)
+ 1 Import decls, 56 each (56 bytes)
+Total bytes = 126072
+
+*** Stmt/Expr Stats:
+ 2408 stmts/exprs total.
+ 1 UnresolvedLookupExpr, 64 each (64 bytes)
+ 400 IntegerLiteral, 32 each (12800 bytes)
+ 402 DeclRefExpr, 32 each (12864 bytes)
+ 402 ImplicitCastExpr, 24 each (9648 bytes)
+ 1 CallExpr, 24 each (24 bytes)
+ 400 BinaryOperator, 32 each (12800 bytes)
+ 401 ReturnStmt, 16 each (6416 bytes)
+ 401 CompoundStmt, 16 each (6416 bytes)
+Total bytes = 61032
+
+STATISTICS FOR './module.modulemap':
+
+*** Preprocessor Stats:
+564 directives found:
+ 553 #define.
+ 0 #undef.
+ #include/#include_next/#import:
+ 4 source files entered.
+ 1 max include stack depth
+ 2 #if/#ifndef/#ifdef.
+ 0 #else/#elif/#elifdef/#elifndef.
+ 2 #endif.
+ 2 #pragma.
+0 #if/#ifndef#ifdef regions skipped
+0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
+0 token paste (##) operations performed, 0 on the fast path.
+
+Preprocessor Memory: 110103B total
+ BumpPtr: 57344
+ Macro Expanded Tokens: 384
+ Predefines Buffer: 32767
+ Macros: 16512
+ #pragma push_macro Info: 2056
+ Poison Reasons: 1032
+ Comment Handlers: 8
+
+*** Identifier Table Stats:
+# Identifiers: 13432
+# Empty Buckets: 19336
+Hash density (#identifiers per bucket): 0.409912
+Ave identifier length: 26.735631
+Max identifier length: 49
+
+Number of memory regions: 182
+Bytes used: 909825
+Bytes allocated: 966656
+Bytes wasted: 56831 (includes alignment, etc)
+
+*** HeaderSearch Stats:
+3 files tracked.
+ 0 #import/#pragma once files.
+ 2 #include/#include_next/#import.
+ 0 #includes skipped due to the multi-include optimization.
+0 framework lookups.
+0 subframework lookups.
+
+*** Source Manager Stats:
+3 files mapped, 3 mem buffers mapped.
+7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
+0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
+20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
+FileID scans: 32 linear, 0 binary.
+
+
+*** File Manager Stats:
+5 real files found, 7 real dirs found.
+0 virtual files found, 0 virtual dirs found.
+28 dir lookups, 9 dir cache misses.
+19 file lookups, 8 file cache misses.
+
+*** Virtual File System Stats:
+14 status() calls
+5 openFileForRead() calls
+0 dir_begin() calls
+1 getRealPath() calls
+0 exists() calls
+0 isLocal() calls
+
+===-------------------------------------------------------------------------===
+ ... Statistics Collected ...
+===-------------------------------------------------------------------------===
+
+ 2 file-search - Number of attempted #includes.
+43765 source-manager - Maximum number of bytes used by source locations (both loaded and local).
+
+
+STATISTICS:
+
+*** Semantic Analysis Stats:
+
+Number of memory regions: 0
+Bytes used: 0
+Bytes allocated: 0
+Bytes wasted: 0 (includes alignment, etc)
+
+*** Analysis Based Warnings Stats:
+401 functions analyzed (0 w/o CFGs).
+ 1203 CFG blocks built.
+ 3 average CFG blocks per function.
+ 3 max CFG blocks per function.
+0 functions analyzed for uninitialiazed variables
+ 0 variables analyzed.
+ 0 average variables per function.
+ 0 max variables per function.
+ 0 block visits.
+ 0 average block visits per function.
+ 0 max block visits per function.
+
+*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
+
+
+
+*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
+
+
+Total missing origins: 0
+
+****************************************
+
+*** AST Context Stats:
+ 156 types total.
+ 120 Builtin types, 32 each (3840 bytes)
+ 1 FunctionProto types, 48 each (48 bytes)
+ 5 Pointer types, 48 each (240 bytes)
+ 1 Record types, 32 each (32 bytes)
+ 1 Typedef types, 32 each (32 bytes)
+ 28 Vector types, 48 each (1344 bytes)
+Total bytes = 5536
+0/0 implicit default constructors created
+0/0 implicit copy constructors created
+0/0 implicit move constructors created
+0/0 implicit copy assignment operators created
+0/0 implicit move assignment operators created
+0/0 implicit destructors created
+
+*** AST File Statistics:
+
+*** PCH/ModuleFile Remappings:
+
+*** PCH/Modules Loaded:
+
+Number of memory regions: 62
+Bytes used: 248400
+Bytes allocated: 253952
+Bytes wasted: 5552 (includes alignment, etc)
+
+*** Decl Stats:
+ 1896 decls total.
+ 2 TranslationUnit decls, 104 each (208 bytes)
+ 2 ExternCContext decls, 72 each (144 bytes)
+ 802 Function decls, 168 each (134736 bytes)
+ 802 ParmVar decls, 104 each (83408 bytes)
+ 12 Field decls, 80 each (960 bytes)
+ 3 CXXRecord decls, 144 each (432 bytes)
+ 270 Typedef decls, 88 each (23760 bytes)
+ 3 Import decls, 56 each (168 bytes)
+Total bytes = 243816
+
+*** Stmt/Expr Stats:
+ 4816 stmts/exprs total.
+ 2 UnresolvedLookupExpr, 64 each (128 bytes)
+ 800 IntegerLiteral, 32 each (25600 bytes)
+ 804 DeclRefExpr, 32 each (25728 bytes)
+ 804 ImplicitCastExpr, 24 each (19296 bytes)
+ 2 CallExpr, 24 each (48 bytes)
+ 800 BinaryOperator, 32 each (25600 bytes)
+ 802 ReturnStmt, 16 each (12832 bytes)
+ 802 CompoundStmt, 16 each (12832 bytes)
+Total bytes = 122064
+
+STATISTICS FOR './module.modulemap':
+
+*** Preprocessor Stats:
+564 directives found:
+ 553 #define.
+ 0 #undef.
+ #include/#include_next/#import:
+ 4 source files entered.
+ 1 max include stack depth
+ 2 #if/#ifndef/#ifdef.
+ 0 #else/#elif/#elifdef/#elifndef.
+ 2 #endif.
+ 2 #pragma.
+0 #if/#ifndef#ifdef regions skipped
+0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
+0 token paste (##) operations performed, 0 on the fast path.
+
+Preprocessor Memory: 110103B total
+ BumpPtr: 57344
+ Macro Expanded Tokens: 384
+ Predefines Buffer: 32767
+ Macros: 16512
+ #pragma push_macro Info: 2056
+ Poison Reasons: 1032
+ Comment Handlers: 8
+
+*** Identifier Table Stats:
+# Identifiers: 13432
+# Empty Buckets: 19336
+Hash density (#identifiers per bucket): 0.409912
+Ave identifier length: 26.735631
+Max identifier length: 49
+
+Number of memory regions: 182
+Bytes used: 909825
+Bytes allocated: 966656
+Bytes wasted: 56831 (includes alignment, etc)
+
+*** HeaderSearch Stats:
+3 files tracked.
+ 0 #import/#pragma once files.
+ 4 #include/#include_next/#import.
+ 0 #includes skipped due to the multi-include optimization.
+0 framework lookups.
+0 subframework lookups.
+
+*** Source Manager Stats:
+3 files mapped, 3 mem buffers mapped.
+7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
+0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
+20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
+FileID scans: 32 linear, 0 binary.
+
+
+*** File Manager Stats:
+5 real files found, 7 real dirs found.
+0 virtual files found, 0 virtual dirs found.
+42 dir lookups, 10 dir cache misses.
+38 file lookups, 8 file cache misses.
+
+*** Virtual File System Stats:
+15 status() calls
+8 openFileForRead() calls
+0 dir_begin() calls
+1 getRealPath() calls
+0 exists() calls
+0 isLocal() calls
+
+===-------------------------------------------------------------------------===
+ ... Statistics Collected ...
+===-------------------------------------------------------------------------===
+
+ 4 file-search - Number of attempted #includes.
+66900 source-manager - Maximum number of bytes used by source locations (both loaded and local).
+
+
+STATISTICS:
+
+*** Semantic Analysis Stats:
+
+Number of memory regions: 0
+Bytes used: 0
+Bytes allocated: 0
+Bytes wasted: 0 (includes alignment, etc)
+
+*** Analysis Based Warnings Stats:
+0 functions analyzed (0 w/o CFGs).
+ 0 CFG blocks built.
+ 0 average CFG blocks per function.
+ 0 max CFG blocks per function.
+0 functions analyzed for uninitialiazed variables
+ 0 variables analyzed.
+ 0 average variables per function.
+ 0 max variables per function.
+ 0 block visits.
+ 0 average block visits per function.
+ 0 max block visits per function.
+
+*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
+
+
+
+*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
+
+
+Total missing origins: 0
+
+****************************************
+
+*** AST Context Stats:
+ 157 types total.
+ 120 Builtin types, 32 each (3840 bytes)
+ 2 FunctionProto types, 48 each (96 bytes)
+ 6 Pointer types, 48 each (288 bytes)
+ 1 Record types, 32 each (32 bytes)
+ 28 Vector types, 48 each (1344 bytes)
+Total bytes = 5600
+0/0 implicit default constructors created
+0/0 implicit copy constructors created
+0/0 implicit move constructors created
+0/0 implicit copy assignment operators created
+0/0 implicit move assignment operators created
+0/0 implicit destructors created
+
+*** AST File Statistics:
+ 0/12 source location entries read (0.000000%)
+ 4/64 types read (6.250000%)
+ 1032/1776 declarations read (58.108109%)
+ 14/992 identifiers read (1.411290%)
+ 0/4 macros read (0.000000%)
+ 14/4814 statements read (0.290818%)
+ 0/4 macros read (0.000000%)
+ 0/802 lexical declcontexts read (0.000000%)
+
+*** PCH/ModuleFile Remappings:
+Global bit offset map:
+ 0 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+ 1144160 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
+Global source location entry map:
+ 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+ 8 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
+Global submodule map:
+ 1 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+ 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
+
+*** PCH/Modules Loaded:
+Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
+ Base source location offset: 2147439884
+ Base identifier ID: 0
+ Number of identifiers: 496
+ Base macro ID: 0
+ Number of macros: 2
+ Base submodule ID: 0
+ Number of submodules: 1
+ Submodule ID local -> global map:
+ 0 -> 0
+ Base selector ID: 0
+ Number of selectors: 0
+ Base preprocessed entity ID: 0
+ Number of preprocessed entities: 0
+ Base type index: 0
+ Number of types: 32
+ Base decl index: 0
+ Number of decls: 888
+
+Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
+ Base source location offset: 2147396120
+ Base identifier ID: 496
+ Number of identifiers: 496
+ Base macro ID: 2
+ Number of macros: 2
+ Base submodule ID: 1
+ Number of submodules: 1
+ Submodule ID local -> global map:
+ 0 -> 1
+ Base selector ID: 0
+ Number of selectors: 0
+ Base preprocessed entity ID: 0
+ Number of preprocessed entities: 0
+ Base type index: 32
+ Number of types: 32
+ Base decl index: 888
+ Number of decls: 888
+
+
+Number of memory regions: 5
+Bytes used: 19784
+Bytes allocated: 20480
+Bytes wasted: 696 (includes alignment, etc)
+
+*** Decl Stats:
+ 1907 decls total.
+ 2 TranslationUnit decls, 104 each (208 bytes)
+ 3 ExternCContext decls, 72 each (216 bytes)
+ 807 Function decls, 168 each (135576 bytes)
+ 806 ParmVar decls, 104 each (83824 bytes)
+ 12 Field decls, 80 each (960 bytes)
+ 3 CXXRecord decls, 144 each (432 bytes)
+ 270 Typedef decls, 88 each (23760 bytes)
+ 4 Import decls, 56 each (224 bytes)
+Total bytes = 245200
+
+*** Stmt/Expr Stats:
+ 4843 stmts/exprs total.
+ 4 UnresolvedLookupExpr, 64 each (256 bytes)
+ 802 IntegerLiteral, 32 each (25664 bytes)
+ 810 DeclRefExpr, 32 each (25920 bytes)
+ 810 ImplicitCastExpr, 24 each (19440 bytes)
+ 6 CallExpr, 24 each (144 bytes)
+ 801 BinaryOperator, 32 each (25632 bytes)
+ 805 ReturnStmt, 16 each (12880 bytes)
+ 805 CompoundStmt, 16 each (12880 bytes)
+Total bytes = 122816
+
+STATISTICS FOR 'use.cpp':
+
+*** Preprocessor Stats:
+559 directives found:
+ 552 #define.
+ 0 #undef.
+ #include/#include_next/#import:
+ 2 source files entered.
+ 0 max include stack depth
+ 0 #if/#ifndef/#ifdef.
+ 0 #else/#elif/#elifdef/#elifndef.
+ 0 #endif.
+ 2 #pragma.
+0 #if/#ifndef#ifdef regions skipped
+0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
+0 token paste (##) operations performed, 0 on the fast path.
+
+Preprocessor Memory: 106007B total
+ BumpPtr: 53248
+ Macro Expanded Tokens: 384
+ Predefines Buffer: 32767
+ Macros: 16512
+ #pragma push_macro Info: 2056
+ Poison Reasons: 1032
+ Comment Handlers: 8
+
+*** Identifier Table Stats:
+# Identifiers: 13037
+# Empty Buckets: 19731
+Hash density (#identifiers per bucket): 0.397858
+Ave identifier length: 27.158472
+Max identifier length: 49
+
+Number of memory regions: 180
+Bytes used: 888582
+Bytes allocated: 950272
+Bytes wasted: 61690 (includes alignment, etc)
+
+*** HeaderSearch Stats:
+3 files tracked.
+ 0 #import/#pragma once files.
+ 4 #include/#include_next/#import.
+ 0 #includes skipped due to the multi-include optimization.
+0 framework lookups.
+0 subframework lookups.
+
+*** Source Manager Stats:
+2 files mapped, 2 mem buffers mapped.
+5 local SLocEntries allocated (168 bytes of capacity), 23136B of SLoc address space used.
+12 loaded SLocEntries allocated (768 bytes of capacity), 87528B of SLoc address space used.
+2 duplicate loaded file SLocEntries detected (20694B of SLoc address space reusable).
+147 bytes of files mapped, 0 files with line #'s computed, 0 files with macro args computed.
+FileID scans: 15 linear, 0 binary.
+
+
+*** File Manager Stats:
+7 real files found, 8 real dirs found.
+0 virtual files found, 0 virtual dirs found.
+46 dir lookups, 11 dir cache misses.
+47 file lookups, 10 file cache misses.
+
+*** Virtual File System Stats:
+18 status() calls
+10 openFileForRead() calls
+0 dir_begin() calls
+1 getRealPath() calls
+0 exists() calls
+0 isLocal() calls
+
+===-------------------------------------------------------------------------===
+ ... Statistics Collected ...
+===-------------------------------------------------------------------------===
+
+ 4 file-search - Number of attempted #includes.
+110664 source-manager - Maximum number of bytes used by source locations (both loaded and local).
+
diff --git a/dedup-test/use.cpp b/dedup-test/use.cpp
new file mode 100644
index 0000000000000..cc00041ff4cc4
--- /dev/null
+++ b/dedup-test/use.cpp
@@ -0,0 +1,3 @@
+#include "a.h"
+#include "b.h"
+int main() { return a_entry(1) + b_entry(2); }
diff --git a/dedup-test/use1.cpp b/dedup-test/use1.cpp
new file mode 100644
index 0000000000000..1d4c4250df0c7
--- /dev/null
+++ b/dedup-test/use1.cpp
@@ -0,0 +1,2 @@
+#include "a.h"
+int main() { return a_entry(1); }
>From da76943d6bc7aff8c1e4ff3cd4757e658c267f49 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 29 Jun 2026 20:13:59 -0400
Subject: [PATCH 02/32] add no-op piecewise translation
---
clang/include/clang/Serialization/ASTReader.h | 15 +++++++++++++++
clang/include/clang/Serialization/ModuleFile.h | 16 ++++++++++++++++
clang/lib/Serialization/ASTReader.cpp | 9 +++++++++
two_modules_stats.txt | 1 +
4 files changed, 41 insertions(+)
create mode 100644 two_modules_stats.txt
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 2afc4c199afda..62bc2b185d9d6 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2501,6 +2501,21 @@ class ASTReader : public ExternalPreprocessorSource,
// translated or refactor the code to make it clear that
// TranslateSourceLocation won't be called with translated source location.
+ // De-duplication (Stage 2): a module's local->global offset map may be
+ // piecewise rather than a single flat shift, so look up the segment that
+ // covers this location. The seeded identity segment makes this identical to
+ // the flat shift below; redirect segments (Stage 2b) send a duplicated
+ // file's locations into the module that first loaded it.
+ if (!ModuleFile.SLocRemap.empty()) {
+ SourceLocation::UIntTy Raw = Loc.getRawEncoding();
+ for (const auto &Seg : ModuleFile.SLocRemap)
+ if (Raw >= Seg.LocalBegin && Raw < Seg.LocalEnd)
+ return SourceLocation::getFromRawEncoding(
+ static_cast<SourceLocation::UIntTy>(
+ static_cast<int64_t>(Raw) + Seg.Delta));
+ // No segment matched (shouldn't happen): fall through to the flat shift.
+ }
+
return Loc.getLocWithOffset(ModuleFile.SLocEntryBaseOffset - 2);
}
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 6c47040fde093..6d3d46003f058 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -346,6 +346,22 @@ class ModuleFile {
/// AST file.
const uint32_t *SLocEntryOffsets = nullptr;
+ // === Source-location de-duplication: offset remap (prototype, Stage 2) ===
+
+ /// One offset-remap segment in this module's raw SourceLocation space:
+ /// a raw local location L in [LocalBegin, LocalEnd) maps to global L + Delta.
+ struct SLocRemapSegment {
+ SourceLocation::UIntTy LocalBegin;
+ SourceLocation::UIntTy LocalEnd;
+ int64_t Delta;
+ };
+
+ /// Piecewise local->global offset map for this module, sorted by LocalBegin.
+ /// Empty => fall back to the flat shift by (SLocEntryBaseOffset - 2).
+ /// Stage 2a seeds a single identity segment equivalent to the flat shift;
+ /// Stage 2b adds redirect/shift segments for de-duplicated files.
+ llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
+
// === Identifiers ===
/// The number of identifiers in this AST file.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 2283a9ee90b65..d7c5468392ddf 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -4282,6 +4282,15 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
return llvm::createStringError(std::errc::invalid_argument,
"ran out of source locations");
}
+
+ // De-duplication (Stage 2a): seed the piecewise offset map with a single
+ // identity segment spanning the whole value range, equivalent to the flat
+ // shift by (BaseOffset - 2). This routes translation through the segment
+ // list with no behavior change; Stage 2b adds redirect/shift segments for
+ // duplicated files.
+ F.SLocRemap.push_back(
+ {/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
+ /*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
// Make our entry in the range map. BaseID is negative and growing, so
// we invert it. Because we invert it, though, we need the other end of
// the range.
diff --git a/two_modules_stats.txt b/two_modules_stats.txt
new file mode 100644
index 0000000000000..d0df4d9e70b35
--- /dev/null
+++ b/two_modules_stats.txt
@@ -0,0 +1 @@
+zsh: command not found: -std=c++20
>From c9290e67e6f751fcc79f2d9a0e90c6c88227b9bb Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 1 Jul 2026 08:56:07 -0400
Subject: [PATCH 03/32] redirect duplicated files to previous allocation
---
clang/include/clang/Basic/SourceManager.h | 56 ++++--
clang/include/clang/Serialization/ASTReader.h | 28 ++-
.../include/clang/Serialization/ModuleFile.h | 11 ++
clang/lib/Basic/SourceManager.cpp | 4 +-
clang/lib/Serialization/ASTReader.cpp | 181 +++++++++++++-----
dedup-test/use_err.cpp | 14 ++
dedup-test/use_err_rev.cpp | 10 +
7 files changed, 235 insertions(+), 69 deletions(-)
create mode 100644 dedup-test/use_err.cpp
create mode 100644 dedup-test/use_err_rev.cpp
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index fe9dd42989e50..75b346018bb67 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,30 +755,48 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- /// --- Source-location de-duplication: detection (prototype, Stage 1) ---
- /// The first global offset at which each file's SLoc entry was loaded.
- /// Keyed by FileEntry identity (the same identity Clang uses to dedup file
- /// content), so we recognize when a later module re-loads the *same file*
- /// already present in the loaded address space (e.g. a shared header).
- llvm::DenseMap<const FileEntry *, SourceLocation::UIntTy>
- LoadedFileFirstOffset;
+ /// --- Source-location de-duplication (prototype, Stage 2) ---
+ /// The canonical loaded location of a file: where its SLoc entry first landed
+ /// in the global address space. Lets a later module reuse it instead of
+ /// re-allocating. Keyed by FileEntry identity (the same identity Clang uses
+ /// to dedup file content).
+public:
+ struct LoadedFileLoc {
+ SourceLocation::UIntTy Offset = 0; ///< global raw start offset
+ int ID = 0; ///< global SLoc entry ID
+ };
+
+private:
+ llvm::DenseMap<const FileEntry *, LoadedFileLoc> CanonicalLoadedFiles;
/// Number of loaded file SLoc entries that duplicated an already-loaded file.
unsigned NumDuplicateLoadedFiles = 0;
- /// SLoc address-space bytes occupied by those duplicates (the reuse prize).
+ /// SLoc address-space bytes those duplicates reused instead of allocating.
uint64_t DuplicateLoadedBytes = 0;
public:
- /// Record a loaded file SLoc entry. If the *same file* was already loaded,
- /// count it as a duplicate and add its size to the reusable total. Detection
- /// only -- no behavior change.
- void noteLoadedFileSLocEntry(const FileEntry *FE,
- SourceLocation::UIntTy Offset, uint64_t Size) {
- if (!FE)
- return;
- if (!LoadedFileFirstOffset.try_emplace(FE, Offset).second) {
- ++NumDuplicateLoadedFiles;
- DuplicateLoadedBytes += Size;
- }
+ /// Whether a file with this identity was already loaded into the address
+ /// space by an earlier module (i.e. its SLoc entry would be a duplicate).
+ bool isLoadedFileDuplicate(const FileEntry *FE) const {
+ return FE && CanonicalLoadedFiles.contains(FE);
+ }
+
+ /// The canonical loaded location of a previously-loaded file, or null.
+ const LoadedFileLoc *getCanonicalLoadedFile(const FileEntry *FE) const {
+ auto It = CanonicalLoadedFiles.find(FE);
+ return It == CanonicalLoadedFiles.end() ? nullptr : &It->second;
+ }
+
+ /// Record the canonical loaded location of a file the first time it loads.
+ void registerCanonicalLoadedFile(const FileEntry *FE,
+ SourceLocation::UIntTy Offset, int ID) {
+ if (FE)
+ CanonicalLoadedFiles.try_emplace(FE, LoadedFileLoc{Offset, ID});
+ }
+
+ /// Account for a de-duplicated file entry (for -print-stats reporting).
+ void noteDuplicateLoadedFile(uint64_t Size) {
+ ++NumDuplicateLoadedFiles;
+ DuplicateLoadedBytes += Size;
}
private:
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 62bc2b185d9d6..a063904e4567e 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2407,6 +2407,13 @@ class ASTReader : public ExternalPreprocessorSource,
SmallVectorImpl<uint32_t> &Offsets,
SmallVectorImpl<const FileEntry *> &Files);
+ /// Map a module-local SLoc entry offset (the value stored in an entry record,
+ /// i.e. Record[0]) to its global raw start offset, applying the
+ /// de-duplication remap. Equals (SLocEntryBaseOffset + LocalOffset) when the
+ /// module is not de-duplicated.
+ SourceLocation::UIntTy remapSLocEntryOffset(ModuleFile &F,
+ uint32_t LocalOffset) const;
+
/// Retrieve the module import location and module name for the
/// given source manager entry ID.
std::pair<SourceLocation, StringRef> getModuleImportLoc(int ID) override;
@@ -2506,13 +2513,20 @@ class ASTReader : public ExternalPreprocessorSource,
// covers this location. The seeded identity segment makes this identical to
// the flat shift below; redirect segments (Stage 2b) send a duplicated
// file's locations into the module that first loaded it.
+ //
+ // Segments are keyed by the *file-offset* part of the location, so strip
+ // the macro bit before matching and re-apply it to the result (a macro
+ // location encodes its position in the low bits with the high bit set).
if (!ModuleFile.SLocRemap.empty()) {
SourceLocation::UIntTy Raw = Loc.getRawEncoding();
+ SourceLocation::UIntTy MacroBit = Raw & SourceLocation::MacroIDBit;
+ SourceLocation::UIntTy Low = Raw & ~SourceLocation::MacroIDBit;
for (const auto &Seg : ModuleFile.SLocRemap)
- if (Raw >= Seg.LocalBegin && Raw < Seg.LocalEnd)
+ if (Low >= Seg.LocalBegin && Low < Seg.LocalEnd)
return SourceLocation::getFromRawEncoding(
- static_cast<SourceLocation::UIntTy>(
- static_cast<int64_t>(Raw) + Seg.Delta));
+ (static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
+ Seg.Delta)) |
+ MacroBit);
// No segment matched (shouldn't happen): fall through to the flat shift.
}
@@ -2537,6 +2551,14 @@ class ASTReader : public ExternalPreprocessorSource,
assert(FID.ID >= 0 && "Reading non-local FileID.");
if (FID.isInvalid())
return FID;
+ // De-duplication (Stage 2b): a local FileID may map to the canonical copy
+ // in an earlier module, so consult the explicit map when present. Local
+ // FileID N corresponds to local entry index N-1.
+ if (!F.LocalToGlobalID.empty()) {
+ assert((unsigned)(FID.ID - 1) < F.LocalToGlobalID.size() &&
+ "local FileID out of range");
+ return FileID::get(F.LocalToGlobalID[FID.ID - 1]);
+ }
return FileID::get(F.SLocEntryBaseID + FID.ID - 1);
}
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 6d3d46003f058..6b65089cc7e6b 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -362,6 +362,17 @@ class ModuleFile {
/// Stage 2b adds redirect/shift segments for de-duplicated files.
llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
+ /// Maps a local SLoc entry index -> its resulting global SLoc entry ID.
+ /// Kept entries map to their own newly-allocated ID; de-duplicated entries
+ /// map to the ID of the canonical copy in an earlier module. Empty => no
+ /// de-duplication for this module (global ID is SLocEntryBaseID + index).
+ std::vector<int> LocalToGlobalID;
+
+ /// For each kept global slot j (0-based, relative to SLocEntryBaseID), the
+ /// original local entry index in this module (index into SLocEntryOffsets).
+ /// Empty => no de-duplication (kept slot j == local index j).
+ std::vector<unsigned> KeptSLocLocalIndex;
+
// === Identifiers ===
/// The number of identifiers in this AST file.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 194426ae2f145..fc1d0094a84d8 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2154,9 +2154,9 @@ void SourceManager::PrintStats() const {
<< "B of SLoc address space used.\n";
if (NumDuplicateLoadedFiles)
llvm::errs() << NumDuplicateLoadedFiles
- << " duplicate loaded file SLocEntries detected ("
+ << " duplicate loaded file SLocEntries de-duplicated ("
<< DuplicateLoadedBytes
- << "B of SLoc address space reusable).\n";
+ << "B of SLoc address space reused).\n";
unsigned NumLineNumsComputed = 0;
unsigned NumFileBytesMapped = 0;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index d7c5468392ddf..d8041c06a4674 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1853,6 +1853,21 @@ bool ASTReader::scanLoadedSLocEntries(
return true;
}
+SourceLocation::UIntTy
+ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
+ // The entry's local raw start location is LocalOffset + 2 (offsets 0 and 1
+ // are reserved). Find the segment covering it and apply that segment's delta.
+ if (!F.SLocRemap.empty()) {
+ SourceLocation::UIntTy Low = LocalOffset + 2;
+ for (const auto &Seg : F.SLocRemap)
+ if (Low >= Seg.LocalBegin && Low < Seg.LocalEnd)
+ return static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
+ Seg.Delta);
+ }
+ // No remap (or no segment matched): original flat shift.
+ return F.SLocEntryBaseOffset + LocalOffset;
+}
+
llvm::Expected<SourceLocation::UIntTy>
ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
BitstreamCursor &Cursor = F->SLocEntryCursor;
@@ -1885,7 +1900,7 @@ ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
case SM_SLOC_FILE_ENTRY:
case SM_SLOC_BUFFER_ENTRY:
case SM_SLOC_EXPANSION_ENTRY:
- return F->SLocEntryBaseOffset + Record[0];
+ return remapSLocEntryOffset(*F, Record[0]);
}
}
@@ -1898,13 +1913,21 @@ int ASTReader::getSLocEntryID(SourceLocation::UIntTy SLocOffset) {
bool Invalid = false;
+ // De-duplication (Stage 2b): the global table holds only this module's kept
+ // entries, so search over kept slots and translate each slot to its on-disk
+ // local entry index when reading the offset.
+ bool Dedup = !F->KeptSLocLocalIndex.empty();
+ unsigned NumSlots =
+ Dedup ? F->KeptSLocLocalIndex.size() : F->LocalNumSLocEntries;
+
auto It = llvm::upper_bound(
- llvm::index_range(0, F->LocalNumSLocEntries), SLocOffset,
- [&](SourceLocation::UIntTy Offset, std::size_t LocalIndex) {
- int ID = F->SLocEntryBaseID + LocalIndex;
+ llvm::index_range(0, NumSlots), SLocOffset,
+ [&](SourceLocation::UIntTy Offset, std::size_t Slot) {
+ int ID = F->SLocEntryBaseID + Slot;
std::size_t Index = -ID - 2;
if (!SourceMgr.SLocEntryOffsetLoaded[Index]) {
assert(!SourceMgr.SLocEntryLoaded[Index]);
+ unsigned LocalIndex = Dedup ? F->KeptSLocLocalIndex[Slot] : Slot;
auto MaybeEntryOffset = readSLocOffset(F, LocalIndex);
if (!MaybeEntryOffset) {
Error(MaybeEntryOffset.takeError());
@@ -1986,15 +2009,23 @@ bool ASTReader::ReadSLocEntry(int ID) {
};
ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
+ // De-duplication (Stage 2b): with entries skipped, the global slot
+ // (ID - SLocEntryBaseID) is no longer the on-disk local index, so translate
+ // it through the kept-index table when present.
+ unsigned LocalIndex = ID - F->SLocEntryBaseID;
+ if (!F->KeptSLocLocalIndex.empty()) {
+ assert(LocalIndex < F->KeptSLocLocalIndex.size() && "kept slot out of range");
+ LocalIndex = F->KeptSLocLocalIndex[LocalIndex];
+ }
if (llvm::Error Err = F->SLocEntryCursor.JumpToBit(
- F->SLocEntryOffsetsBase +
- F->SLocEntryOffsets[ID - F->SLocEntryBaseID])) {
+ F->SLocEntryOffsetsBase + F->SLocEntryOffsets[LocalIndex])) {
Error(std::move(Err));
return true;
}
BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
SourceLocation::UIntTy BaseOffset = F->SLocEntryBaseOffset;
+ (void)BaseOffset;
++NumSLocEntriesRead;
Expected<llvm::BitstreamEntry> MaybeEntry = SLocEntryCursor.advance();
@@ -2044,7 +2075,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
SrcMgr::CharacteristicKind
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
- BaseOffset + Record[0]);
+ remapSLocEntryOffset(*F, Record[0]));
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
if (Record[3])
@@ -2086,7 +2117,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
if (!Buffer)
return true;
FileID FID = SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
- BaseOffset + Offset, IncludeLoc);
+ remapSLocEntryOffset(*F, Offset),
+ IncludeLoc);
if (Record[3]) {
auto &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.setHasLineDirectives();
@@ -2100,7 +2132,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
SourceLocation ExpansionEnd = ReadSourceLocation(*F, Record[3]);
SourceMgr.createExpansionLoc(SpellingLoc, ExpansionBegin, ExpansionEnd,
Record[5], Record[4], ID,
- BaseOffset + Record[0]);
+ remapSLocEntryOffset(*F, Record[0]));
break;
}
}
@@ -4273,9 +4305,36 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
F.LocalNumSLocEntries = Record[0];
SourceLocation::UIntTy SLocSpaceSize = Record[1];
F.SLocEntryOffsetsBase = Record[2] + F.SourceManagerBlockStartOffset;
+ unsigned N = F.LocalNumSLocEntries;
+
+ // De-duplication (Stage 2b): scan this module's SLoc entries *before*
+ // allocating, so we can recognize files already loaded by an earlier
+ // module and reserve less address space for them.
+ SmallVector<uint32_t, 64> Offsets;
+ SmallVector<const FileEntry *, 64> Files;
+ bool Scanned = scanLoadedSLocEntries(F, Offsets, Files);
+
+ // Classify duplicates and compute the reduced allocation request.
+ auto entrySize = [&](unsigned I) -> uint64_t {
+ return (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
+ };
+ unsigned NumDupEntries = 0;
+ uint64_t DupBytes = 0;
+ if (Scanned)
+ for (unsigned I = 0; I != N; ++I)
+ if (SourceMgr.isLoadedFileDuplicate(Files[I])) {
+ uint64_t Size = entrySize(I);
+ SourceMgr.noteDuplicateLoadedFile(Size);
+ DupBytes += Size;
+ ++NumDupEntries;
+ }
+ unsigned ReducedNumEntries = N - NumDupEntries;
+ SourceLocation::UIntTy ReducedSize = SLocSpaceSize - DupBytes;
+
+ // Reserve the reduced amount (equals the full amount when nothing is
+ // de-duplicated).
std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
- SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
- SLocSpaceSize);
+ SourceMgr.AllocateLoadedSLocEntries(ReducedNumEntries, ReducedSize);
if (!F.SLocEntryBaseID) {
Diags.Report(SourceLocation(), diag::remark_sloc_usage);
SourceMgr.noteSLocAddressSpaceUsage(Diags);
@@ -4283,49 +4342,81 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
"ran out of source locations");
}
- // De-duplication (Stage 2a): seed the piecewise offset map with a single
- // identity segment spanning the whole value range, equivalent to the flat
- // shift by (BaseOffset - 2). This routes translation through the segment
- // list with no behavior change; Stage 2b adds redirect/shift segments for
- // duplicated files.
- F.SLocRemap.push_back(
- {/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
- /*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
+ if (NumDupEntries == 0) {
+ // No de-duplication: one identity segment (== flat shift) and a linear
+ // local->global ID mapping. Register each file as the canonical copy so
+ // later modules can reuse it.
+ F.SLocRemap.push_back(
+ {/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
+ /*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
+ if (Scanned)
+ for (unsigned I = 0; I != N; ++I)
+ if (Files[I])
+ SourceMgr.registerCanonicalLoadedFile(
+ Files[I], F.SLocEntryBaseOffset + Offsets[I],
+ F.SLocEntryBaseID + (int)I);
+ } else {
+ // De-duplication: build keep/redirect offset segments and the
+ // local->global ID map. Skipped (duplicate) entries get no slot and no
+ // address space; their references are redirected to the canonical copy.
+ F.LocalToGlobalID.assign(N, 0);
+ F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
+ uint64_t DupBefore = 0;
+ unsigned KeptCount = 0;
+ for (unsigned I = 0; I != N; ++I) {
+ SourceLocation::UIntTy LowStart = Offsets[I] + 2;
+ SourceLocation::UIntTy LowEnd =
+ (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) + 2;
+ const FileEntry *FE = Files[I];
+ if (FE && SourceMgr.isLoadedFileDuplicate(FE)) {
+ // Redirect this file's locations into the module that first loaded
+ // it; reserve nothing here.
+ const SourceManager::LoadedFileLoc *Canon =
+ SourceMgr.getCanonicalLoadedFile(FE);
+ F.SLocRemap.push_back({LowStart, LowEnd,
+ static_cast<int64_t>(Canon->Offset) -
+ static_cast<int64_t>(LowStart)});
+ F.LocalToGlobalID[I] = Canon->ID;
+ DupBefore += LowEnd - LowStart;
+ } else {
+ // Keep: lands in this module's block, shifted down to close gaps
+ // left by skipped duplicates before it.
+ int GlobalID = F.SLocEntryBaseID + (int)KeptCount;
+ SourceLocation::UIntTy GlobalStart =
+ static_cast<SourceLocation::UIntTy>(F.SLocEntryBaseOffset +
+ Offsets[I] - DupBefore);
+ F.SLocRemap.push_back({LowStart, LowEnd,
+ static_cast<int64_t>(GlobalStart) -
+ static_cast<int64_t>(LowStart)});
+ F.LocalToGlobalID[I] = GlobalID;
+ F.KeptSLocLocalIndex.push_back(I);
+ if (FE)
+ SourceMgr.registerCanonicalLoadedFile(FE, GlobalStart, GlobalID);
+ ++KeptCount;
+ }
+ }
+ // Extend the first/last segments to cover the whole value range so
+ // every translated location matches a segment.
+ F.SLocRemap.front().LocalBegin = 0;
+ F.SLocRemap.back().LocalEnd = ~SourceLocation::UIntTy(0);
+ assert(KeptCount == ReducedNumEntries && "kept count mismatch");
+ }
+
// Make our entry in the range map. BaseID is negative and growing, so
// we invert it. Because we invert it, though, we need the other end of
// the range.
unsigned RangeStart =
- unsigned(-F.SLocEntryBaseID) - F.LocalNumSLocEntries + 1;
+ unsigned(-F.SLocEntryBaseID) - ReducedNumEntries + 1;
GlobalSLocEntryMap.insert(std::make_pair(RangeStart, &F));
F.FirstLoc = SourceLocation::getFromRawEncoding(F.SLocEntryBaseOffset);
// SLocEntryBaseOffset is lower than MaxLoadedOffset and decreasing.
assert((F.SLocEntryBaseOffset & SourceLocation::MacroIDBit) == 0);
- GlobalSLocOffsetMap.insert(
- std::make_pair(SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset
- - SLocSpaceSize,&F));
-
- TotalNumSLocEntries += F.LocalNumSLocEntries;
-
- // De-duplication detection (prototype, Stage 1). Walk this module's SLoc
- // file entries and recognize when the same file was already loaded by an
- // earlier module, measuring the address space a future reuse (Stage 2)
- // could reclaim. Detection only: no allocation/translation change here.
- {
- SmallVector<uint32_t, 64> Offsets;
- SmallVector<const FileEntry *, 64> Files;
- if (scanLoadedSLocEntries(F, Offsets, Files)) {
- unsigned N = Offsets.size();
- for (unsigned I = 0; I != N; ++I) {
- if (!Files[I])
- continue;
- uint64_t Size =
- (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
- SourceMgr.noteLoadedFileSLocEntry(
- Files[I], F.SLocEntryBaseOffset + Offsets[I], Size);
- }
- }
- }
+ GlobalSLocOffsetMap.insert(std::make_pair(
+ SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset - ReducedSize,
+ &F));
+
+ TotalNumSLocEntries += ReducedNumEntries;
break;
}
diff --git a/dedup-test/use_err.cpp b/dedup-test/use_err.cpp
new file mode 100644
index 0000000000000..e3cf22fe27df4
--- /dev/null
+++ b/dedup-test/use_err.cpp
@@ -0,0 +1,14 @@
+// Stage 2c: prove the redirect resolves correctly.
+// Include BOTH modules so A loads first (registers shared.h as the canonical
+// copy) and B loads second (its shared.h is de-duplicated / redirected into
+// A's copy). Then trigger a diagnostic whose location lives in shared.h.
+// The error must still point at shared.h:<line> with the right function/line,
+// which only holds if the redirected offsets resolve correctly.
+#include "a.h"
+#include "b.h"
+
+int main() {
+ // shared_fn_1 is defined in shared.h (present in both A and B); pass a bad
+ // argument so overload resolution reports the candidate in shared.h.
+ return a_entry(1) + shared_fn_1("oops");
+}
diff --git a/dedup-test/use_err_rev.cpp b/dedup-test/use_err_rev.cpp
new file mode 100644
index 0000000000000..817bfb4bc2b1e
--- /dev/null
+++ b/dedup-test/use_err_rev.cpp
@@ -0,0 +1,10 @@
+// Stage 2c: load-order independence. Same as use_err.cpp but with the include
+// order flipped so B is seen first. Whichever module loads first becomes the
+// canonical copy; the other redirects into it. Result should be identical:
+// dedup active + the diagnostic still resolves to shared.h:<line>.
+#include "b.h"
+#include "a.h"
+
+int main() {
+ return a_entry(1) + shared_fn_1("oops");
+}
>From 01ee38690d55b24e1ca5618601c14764e4f5801d Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 1 Jul 2026 09:53:16 -0400
Subject: [PATCH 04/32] clean up stats files
---
clang/include/clang/Serialization/ASTReader.h | 3 +--
two_modules_stats.txt | 1 -
2 files changed, 1 insertion(+), 3 deletions(-)
delete mode 100644 two_modules_stats.txt
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index a063904e4567e..ec4312cf1972c 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,10 +2399,9 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Walk module \p F's SLoc entry records (read-only). Fills \p Offsets[i]
+ /// Walk module F's SLoc entry records (read-only). Fills Offsets[i]
/// with each entry's local offset and \p Files[i] with its FileEntry (null
/// for non-file entries). Returns false if the records couldn't be read.
- /// Shared scaffolding for source-location de-duplication.
bool scanLoadedSLocEntries(ModuleFile &F,
SmallVectorImpl<uint32_t> &Offsets,
SmallVectorImpl<const FileEntry *> &Files);
diff --git a/two_modules_stats.txt b/two_modules_stats.txt
deleted file mode 100644
index d0df4d9e70b35..0000000000000
--- a/two_modules_stats.txt
+++ /dev/null
@@ -1 +0,0 @@
-zsh: command not found: -std=c++20
>From a72b1437714aa2a673bb25c8b3095bc9d7ba4a57 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 1 Jul 2026 09:56:21 -0400
Subject: [PATCH 05/32] clean up stats files
---
dedup-test/one_module_stats.txt | 352 --------------------
dedup-test/two_modules_stats.txt | 535 -------------------------------
2 files changed, 887 deletions(-)
delete mode 100644 dedup-test/one_module_stats.txt
delete mode 100644 dedup-test/two_modules_stats.txt
diff --git a/dedup-test/one_module_stats.txt b/dedup-test/one_module_stats.txt
deleted file mode 100644
index 373fe97b5e1d8..0000000000000
--- a/dedup-test/one_module_stats.txt
+++ /dev/null
@@ -1,352 +0,0 @@
-
-STATISTICS:
-
-*** Semantic Analysis Stats:
-
-Number of memory regions: 0
-Bytes used: 0
-Bytes allocated: 0
-Bytes wasted: 0 (includes alignment, etc)
-
-*** Analysis Based Warnings Stats:
-401 functions analyzed (0 w/o CFGs).
- 1203 CFG blocks built.
- 3 average CFG blocks per function.
- 3 max CFG blocks per function.
-0 functions analyzed for uninitialiazed variables
- 0 variables analyzed.
- 0 average variables per function.
- 0 max variables per function.
- 0 block visits.
- 0 average block visits per function.
- 0 max block visits per function.
-
-*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
-
-
-
-*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
-
-
-Total missing origins: 0
-
-****************************************
-
-*** AST Context Stats:
- 156 types total.
- 120 Builtin types, 32 each (3840 bytes)
- 1 FunctionProto types, 48 each (48 bytes)
- 5 Pointer types, 48 each (240 bytes)
- 1 Record types, 32 each (32 bytes)
- 1 Typedef types, 32 each (32 bytes)
- 28 Vector types, 48 each (1344 bytes)
-Total bytes = 5536
-0/0 implicit default constructors created
-0/0 implicit copy constructors created
-0/0 implicit move constructors created
-0/0 implicit copy assignment operators created
-0/0 implicit move assignment operators created
-0/0 implicit destructors created
-
-*** AST File Statistics:
-
-*** PCH/ModuleFile Remappings:
-
-*** PCH/Modules Loaded:
-
-Number of memory regions: 62
-Bytes used: 248400
-Bytes allocated: 253952
-Bytes wasted: 5552 (includes alignment, etc)
-
-*** Decl Stats:
- 995 decls total.
- 1 TranslationUnit decls, 104 each (104 bytes)
- 1 ExternCContext decls, 72 each (72 bytes)
- 401 Function decls, 168 each (67368 bytes)
- 401 ParmVar decls, 104 each (41704 bytes)
- 8 Field decls, 80 each (640 bytes)
- 2 CXXRecord decls, 144 each (288 bytes)
- 180 Typedef decls, 88 each (15840 bytes)
- 1 Import decls, 56 each (56 bytes)
-Total bytes = 126072
-
-*** Stmt/Expr Stats:
- 2408 stmts/exprs total.
- 1 UnresolvedLookupExpr, 64 each (64 bytes)
- 400 IntegerLiteral, 32 each (12800 bytes)
- 402 DeclRefExpr, 32 each (12864 bytes)
- 402 ImplicitCastExpr, 24 each (9648 bytes)
- 1 CallExpr, 24 each (24 bytes)
- 400 BinaryOperator, 32 each (12800 bytes)
- 401 ReturnStmt, 16 each (6416 bytes)
- 401 CompoundStmt, 16 each (6416 bytes)
-Total bytes = 61032
-
-STATISTICS FOR './module.modulemap':
-
-*** Preprocessor Stats:
-564 directives found:
- 553 #define.
- 0 #undef.
- #include/#include_next/#import:
- 4 source files entered.
- 1 max include stack depth
- 2 #if/#ifndef/#ifdef.
- 0 #else/#elif/#elifdef/#elifndef.
- 2 #endif.
- 2 #pragma.
-0 #if/#ifndef#ifdef regions skipped
-0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
-0 token paste (##) operations performed, 0 on the fast path.
-
-Preprocessor Memory: 110103B total
- BumpPtr: 57344
- Macro Expanded Tokens: 384
- Predefines Buffer: 32767
- Macros: 16512
- #pragma push_macro Info: 2056
- Poison Reasons: 1032
- Comment Handlers: 8
-
-*** Identifier Table Stats:
-# Identifiers: 13432
-# Empty Buckets: 19336
-Hash density (#identifiers per bucket): 0.409912
-Ave identifier length: 26.735631
-Max identifier length: 49
-
-Number of memory regions: 182
-Bytes used: 909825
-Bytes allocated: 966656
-Bytes wasted: 56831 (includes alignment, etc)
-
-*** HeaderSearch Stats:
-3 files tracked.
- 0 #import/#pragma once files.
- 2 #include/#include_next/#import.
- 0 #includes skipped due to the multi-include optimization.
-0 framework lookups.
-0 subframework lookups.
-
-*** Source Manager Stats:
-3 files mapped, 3 mem buffers mapped.
-7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
-0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
-20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
-FileID scans: 32 linear, 0 binary.
-
-
-*** File Manager Stats:
-5 real files found, 7 real dirs found.
-0 virtual files found, 0 virtual dirs found.
-28 dir lookups, 9 dir cache misses.
-19 file lookups, 8 file cache misses.
-
-*** Virtual File System Stats:
-14 status() calls
-5 openFileForRead() calls
-0 dir_begin() calls
-1 getRealPath() calls
-0 exists() calls
-0 isLocal() calls
-
-===-------------------------------------------------------------------------===
- ... Statistics Collected ...
-===-------------------------------------------------------------------------===
-
- 2 file-search - Number of attempted #includes.
-43765 source-manager - Maximum number of bytes used by source locations (both loaded and local).
-
-
-STATISTICS:
-
-*** Semantic Analysis Stats:
-
-Number of memory regions: 0
-Bytes used: 0
-Bytes allocated: 0
-Bytes wasted: 0 (includes alignment, etc)
-
-*** Analysis Based Warnings Stats:
-0 functions analyzed (0 w/o CFGs).
- 0 CFG blocks built.
- 0 average CFG blocks per function.
- 0 max CFG blocks per function.
-0 functions analyzed for uninitialiazed variables
- 0 variables analyzed.
- 0 average variables per function.
- 0 max variables per function.
- 0 block visits.
- 0 average block visits per function.
- 0 max block visits per function.
-
-*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
-
-
-
-*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
-
-
-Total missing origins: 0
-
-****************************************
-
-*** AST Context Stats:
- 157 types total.
- 120 Builtin types, 32 each (3840 bytes)
- 2 FunctionProto types, 48 each (96 bytes)
- 6 Pointer types, 48 each (288 bytes)
- 1 Record types, 32 each (32 bytes)
- 28 Vector types, 48 each (1344 bytes)
-Total bytes = 5600
-0/0 implicit default constructors created
-0/0 implicit copy constructors created
-0/0 implicit move constructors created
-0/0 implicit copy assignment operators created
-0/0 implicit move assignment operators created
-0/0 implicit destructors created
-
-*** AST File Statistics:
- 0/6 source location entries read (0.000000%)
- 2/32 types read (6.250000%)
- 516/888 declarations read (58.108109%)
- 7/496 identifiers read (1.411290%)
- 0/2 macros read (0.000000%)
- 7/2407 statements read (0.290818%)
- 0/2 macros read (0.000000%)
- 0/401 lexical declcontexts read (0.000000%)
-
-*** PCH/ModuleFile Remappings:
-Global bit offset map:
- 0 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
-Global source location entry map:
- 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
-Global submodule map:
- 1 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
-
-*** PCH/Modules Loaded:
-Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache1/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
- Base source location offset: 2147439884
- Base identifier ID: 0
- Number of identifiers: 496
- Base macro ID: 0
- Number of macros: 2
- Base submodule ID: 0
- Number of submodules: 1
- Submodule ID local -> global map:
- 0 -> 0
- Base selector ID: 0
- Number of selectors: 0
- Base preprocessed entity ID: 0
- Number of preprocessed entities: 0
- Base type index: 0
- Number of types: 32
- Base decl index: 0
- Number of decls: 888
-
-
-Number of memory regions: 5
-Bytes used: 18452
-Bytes allocated: 20480
-Bytes wasted: 2028 (includes alignment, etc)
-
-*** Decl Stats:
- 1002 decls total.
- 1 TranslationUnit decls, 104 each (104 bytes)
- 2 ExternCContext decls, 72 each (144 bytes)
- 404 Function decls, 168 each (67872 bytes)
- 403 ParmVar decls, 104 each (41912 bytes)
- 8 Field decls, 80 each (640 bytes)
- 2 CXXRecord decls, 144 each (288 bytes)
- 180 Typedef decls, 88 each (15840 bytes)
- 2 Import decls, 56 each (112 bytes)
-Total bytes = 126912
-
-*** Stmt/Expr Stats:
- 2422 stmts/exprs total.
- 2 UnresolvedLookupExpr, 64 each (128 bytes)
- 401 IntegerLiteral, 32 each (12832 bytes)
- 405 DeclRefExpr, 32 each (12960 bytes)
- 405 ImplicitCastExpr, 24 each (9720 bytes)
- 3 CallExpr, 24 each (72 bytes)
- 400 BinaryOperator, 32 each (12800 bytes)
- 403 ReturnStmt, 16 each (6448 bytes)
- 403 CompoundStmt, 16 each (6448 bytes)
-Total bytes = 61408
-
-STATISTICS FOR 'use1.cpp':
-
-*** Preprocessor Stats:
-558 directives found:
- 552 #define.
- 0 #undef.
- #include/#include_next/#import:
- 2 source files entered.
- 0 max include stack depth
- 0 #if/#ifndef/#ifdef.
- 0 #else/#elif/#elifdef/#elifndef.
- 0 #endif.
- 2 #pragma.
-0 #if/#ifndef#ifdef regions skipped
-0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
-0 token paste (##) operations performed, 0 on the fast path.
-
-Preprocessor Memory: 106007B total
- BumpPtr: 53248
- Macro Expanded Tokens: 384
- Predefines Buffer: 32767
- Macros: 16512
- #pragma push_macro Info: 2056
- Poison Reasons: 1032
- Comment Handlers: 8
-
-*** Identifier Table Stats:
-# Identifiers: 13033
-# Empty Buckets: 19735
-Hash density (#identifiers per bucket): 0.397736
-Ave identifier length: 27.165119
-Max identifier length: 49
-
-Number of memory regions: 180
-Bytes used: 888396
-Bytes allocated: 950272
-Bytes wasted: 61876 (includes alignment, etc)
-
-*** HeaderSearch Stats:
-3 files tracked.
- 0 #import/#pragma once files.
- 2 #include/#include_next/#import.
- 0 #includes skipped due to the multi-include optimization.
-0 framework lookups.
-0 subframework lookups.
-
-*** Source Manager Stats:
-2 files mapped, 2 mem buffers mapped.
-5 local SLocEntries allocated (168 bytes of capacity), 23108B of SLoc address space used.
-6 loaded SLocEntries allocated (768 bytes of capacity), 43764B of SLoc address space used.
-119 bytes of files mapped, 0 files with line #'s computed, 0 files with macro args computed.
-FileID scans: 11 linear, 0 binary.
-
-
-*** File Manager Stats:
-6 real files found, 8 real dirs found.
-0 virtual files found, 0 virtual dirs found.
-31 dir lookups, 11 dir cache misses.
-27 file lookups, 9 file cache misses.
-
-*** Virtual File System Stats:
-17 status() calls
-6 openFileForRead() calls
-0 dir_begin() calls
-1 getRealPath() calls
-0 exists() calls
-0 isLocal() calls
-
-===-------------------------------------------------------------------------===
- ... Statistics Collected ...
-===-------------------------------------------------------------------------===
-
- 2 file-search - Number of attempted #includes.
-66872 source-manager - Maximum number of bytes used by source locations (both loaded and local).
-
diff --git a/dedup-test/two_modules_stats.txt b/dedup-test/two_modules_stats.txt
deleted file mode 100644
index dd3a1f0c2d3ac..0000000000000
--- a/dedup-test/two_modules_stats.txt
+++ /dev/null
@@ -1,535 +0,0 @@
-
-STATISTICS:
-
-*** Semantic Analysis Stats:
-
-Number of memory regions: 0
-Bytes used: 0
-Bytes allocated: 0
-Bytes wasted: 0 (includes alignment, etc)
-
-*** Analysis Based Warnings Stats:
-401 functions analyzed (0 w/o CFGs).
- 1203 CFG blocks built.
- 3 average CFG blocks per function.
- 3 max CFG blocks per function.
-0 functions analyzed for uninitialiazed variables
- 0 variables analyzed.
- 0 average variables per function.
- 0 max variables per function.
- 0 block visits.
- 0 average block visits per function.
- 0 max block visits per function.
-
-*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
-
-
-
-*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
-
-
-Total missing origins: 0
-
-****************************************
-
-*** AST Context Stats:
- 156 types total.
- 120 Builtin types, 32 each (3840 bytes)
- 1 FunctionProto types, 48 each (48 bytes)
- 5 Pointer types, 48 each (240 bytes)
- 1 Record types, 32 each (32 bytes)
- 1 Typedef types, 32 each (32 bytes)
- 28 Vector types, 48 each (1344 bytes)
-Total bytes = 5536
-0/0 implicit default constructors created
-0/0 implicit copy constructors created
-0/0 implicit move constructors created
-0/0 implicit copy assignment operators created
-0/0 implicit move assignment operators created
-0/0 implicit destructors created
-
-*** AST File Statistics:
-
-*** PCH/ModuleFile Remappings:
-
-*** PCH/Modules Loaded:
-
-Number of memory regions: 62
-Bytes used: 248400
-Bytes allocated: 253952
-Bytes wasted: 5552 (includes alignment, etc)
-
-*** Decl Stats:
- 995 decls total.
- 1 TranslationUnit decls, 104 each (104 bytes)
- 1 ExternCContext decls, 72 each (72 bytes)
- 401 Function decls, 168 each (67368 bytes)
- 401 ParmVar decls, 104 each (41704 bytes)
- 8 Field decls, 80 each (640 bytes)
- 2 CXXRecord decls, 144 each (288 bytes)
- 180 Typedef decls, 88 each (15840 bytes)
- 1 Import decls, 56 each (56 bytes)
-Total bytes = 126072
-
-*** Stmt/Expr Stats:
- 2408 stmts/exprs total.
- 1 UnresolvedLookupExpr, 64 each (64 bytes)
- 400 IntegerLiteral, 32 each (12800 bytes)
- 402 DeclRefExpr, 32 each (12864 bytes)
- 402 ImplicitCastExpr, 24 each (9648 bytes)
- 1 CallExpr, 24 each (24 bytes)
- 400 BinaryOperator, 32 each (12800 bytes)
- 401 ReturnStmt, 16 each (6416 bytes)
- 401 CompoundStmt, 16 each (6416 bytes)
-Total bytes = 61032
-
-STATISTICS FOR './module.modulemap':
-
-*** Preprocessor Stats:
-564 directives found:
- 553 #define.
- 0 #undef.
- #include/#include_next/#import:
- 4 source files entered.
- 1 max include stack depth
- 2 #if/#ifndef/#ifdef.
- 0 #else/#elif/#elifdef/#elifndef.
- 2 #endif.
- 2 #pragma.
-0 #if/#ifndef#ifdef regions skipped
-0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
-0 token paste (##) operations performed, 0 on the fast path.
-
-Preprocessor Memory: 110103B total
- BumpPtr: 57344
- Macro Expanded Tokens: 384
- Predefines Buffer: 32767
- Macros: 16512
- #pragma push_macro Info: 2056
- Poison Reasons: 1032
- Comment Handlers: 8
-
-*** Identifier Table Stats:
-# Identifiers: 13432
-# Empty Buckets: 19336
-Hash density (#identifiers per bucket): 0.409912
-Ave identifier length: 26.735631
-Max identifier length: 49
-
-Number of memory regions: 182
-Bytes used: 909825
-Bytes allocated: 966656
-Bytes wasted: 56831 (includes alignment, etc)
-
-*** HeaderSearch Stats:
-3 files tracked.
- 0 #import/#pragma once files.
- 2 #include/#include_next/#import.
- 0 #includes skipped due to the multi-include optimization.
-0 framework lookups.
-0 subframework lookups.
-
-*** Source Manager Stats:
-3 files mapped, 3 mem buffers mapped.
-7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
-0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
-20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
-FileID scans: 32 linear, 0 binary.
-
-
-*** File Manager Stats:
-5 real files found, 7 real dirs found.
-0 virtual files found, 0 virtual dirs found.
-28 dir lookups, 9 dir cache misses.
-19 file lookups, 8 file cache misses.
-
-*** Virtual File System Stats:
-14 status() calls
-5 openFileForRead() calls
-0 dir_begin() calls
-1 getRealPath() calls
-0 exists() calls
-0 isLocal() calls
-
-===-------------------------------------------------------------------------===
- ... Statistics Collected ...
-===-------------------------------------------------------------------------===
-
- 2 file-search - Number of attempted #includes.
-43765 source-manager - Maximum number of bytes used by source locations (both loaded and local).
-
-
-STATISTICS:
-
-*** Semantic Analysis Stats:
-
-Number of memory regions: 0
-Bytes used: 0
-Bytes allocated: 0
-Bytes wasted: 0 (includes alignment, etc)
-
-*** Analysis Based Warnings Stats:
-401 functions analyzed (0 w/o CFGs).
- 1203 CFG blocks built.
- 3 average CFG blocks per function.
- 3 max CFG blocks per function.
-0 functions analyzed for uninitialiazed variables
- 0 variables analyzed.
- 0 average variables per function.
- 0 max variables per function.
- 0 block visits.
- 0 average block visits per function.
- 0 max block visits per function.
-
-*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
-
-
-
-*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
-
-
-Total missing origins: 0
-
-****************************************
-
-*** AST Context Stats:
- 156 types total.
- 120 Builtin types, 32 each (3840 bytes)
- 1 FunctionProto types, 48 each (48 bytes)
- 5 Pointer types, 48 each (240 bytes)
- 1 Record types, 32 each (32 bytes)
- 1 Typedef types, 32 each (32 bytes)
- 28 Vector types, 48 each (1344 bytes)
-Total bytes = 5536
-0/0 implicit default constructors created
-0/0 implicit copy constructors created
-0/0 implicit move constructors created
-0/0 implicit copy assignment operators created
-0/0 implicit move assignment operators created
-0/0 implicit destructors created
-
-*** AST File Statistics:
-
-*** PCH/ModuleFile Remappings:
-
-*** PCH/Modules Loaded:
-
-Number of memory regions: 62
-Bytes used: 248400
-Bytes allocated: 253952
-Bytes wasted: 5552 (includes alignment, etc)
-
-*** Decl Stats:
- 1896 decls total.
- 2 TranslationUnit decls, 104 each (208 bytes)
- 2 ExternCContext decls, 72 each (144 bytes)
- 802 Function decls, 168 each (134736 bytes)
- 802 ParmVar decls, 104 each (83408 bytes)
- 12 Field decls, 80 each (960 bytes)
- 3 CXXRecord decls, 144 each (432 bytes)
- 270 Typedef decls, 88 each (23760 bytes)
- 3 Import decls, 56 each (168 bytes)
-Total bytes = 243816
-
-*** Stmt/Expr Stats:
- 4816 stmts/exprs total.
- 2 UnresolvedLookupExpr, 64 each (128 bytes)
- 800 IntegerLiteral, 32 each (25600 bytes)
- 804 DeclRefExpr, 32 each (25728 bytes)
- 804 ImplicitCastExpr, 24 each (19296 bytes)
- 2 CallExpr, 24 each (48 bytes)
- 800 BinaryOperator, 32 each (25600 bytes)
- 802 ReturnStmt, 16 each (12832 bytes)
- 802 CompoundStmt, 16 each (12832 bytes)
-Total bytes = 122064
-
-STATISTICS FOR './module.modulemap':
-
-*** Preprocessor Stats:
-564 directives found:
- 553 #define.
- 0 #undef.
- #include/#include_next/#import:
- 4 source files entered.
- 1 max include stack depth
- 2 #if/#ifndef/#ifdef.
- 0 #else/#elif/#elifdef/#elifndef.
- 2 #endif.
- 2 #pragma.
-0 #if/#ifndef#ifdef regions skipped
-0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
-0 token paste (##) operations performed, 0 on the fast path.
-
-Preprocessor Memory: 110103B total
- BumpPtr: 57344
- Macro Expanded Tokens: 384
- Predefines Buffer: 32767
- Macros: 16512
- #pragma push_macro Info: 2056
- Poison Reasons: 1032
- Comment Handlers: 8
-
-*** Identifier Table Stats:
-# Identifiers: 13432
-# Empty Buckets: 19336
-Hash density (#identifiers per bucket): 0.409912
-Ave identifier length: 26.735631
-Max identifier length: 49
-
-Number of memory regions: 182
-Bytes used: 909825
-Bytes allocated: 966656
-Bytes wasted: 56831 (includes alignment, etc)
-
-*** HeaderSearch Stats:
-3 files tracked.
- 0 #import/#pragma once files.
- 4 #include/#include_next/#import.
- 0 #includes skipped due to the multi-include optimization.
-0 framework lookups.
-0 subframework lookups.
-
-*** Source Manager Stats:
-3 files mapped, 3 mem buffers mapped.
-7 local SLocEntries allocated (168 bytes of capacity), 43765B of SLoc address space used.
-0 loaded SLocEntries allocated (0 bytes of capacity), 0B of SLoc address space used.
-20795 bytes of files mapped, 2 files with line #'s computed, 0 files with macro args computed.
-FileID scans: 32 linear, 0 binary.
-
-
-*** File Manager Stats:
-5 real files found, 7 real dirs found.
-0 virtual files found, 0 virtual dirs found.
-42 dir lookups, 10 dir cache misses.
-38 file lookups, 8 file cache misses.
-
-*** Virtual File System Stats:
-15 status() calls
-8 openFileForRead() calls
-0 dir_begin() calls
-1 getRealPath() calls
-0 exists() calls
-0 isLocal() calls
-
-===-------------------------------------------------------------------------===
- ... Statistics Collected ...
-===-------------------------------------------------------------------------===
-
- 4 file-search - Number of attempted #includes.
-66900 source-manager - Maximum number of bytes used by source locations (both loaded and local).
-
-
-STATISTICS:
-
-*** Semantic Analysis Stats:
-
-Number of memory regions: 0
-Bytes used: 0
-Bytes allocated: 0
-Bytes wasted: 0 (includes alignment, etc)
-
-*** Analysis Based Warnings Stats:
-0 functions analyzed (0 w/o CFGs).
- 0 CFG blocks built.
- 0 average CFG blocks per function.
- 0 max CFG blocks per function.
-0 functions analyzed for uninitialiazed variables
- 0 variables analyzed.
- 0 average variables per function.
- 0 max variables per function.
- 0 block visits.
- 0 average block visits per function.
- 0 max block visits per function.
-
-*** LifetimeSafety Missing Origin per QualType: (QualType : count) :
-
-
-
-*** LifetimeSafety Missing Origin per StmtClassName: (StmtClassName : count) :
-
-
-Total missing origins: 0
-
-****************************************
-
-*** AST Context Stats:
- 157 types total.
- 120 Builtin types, 32 each (3840 bytes)
- 2 FunctionProto types, 48 each (96 bytes)
- 6 Pointer types, 48 each (288 bytes)
- 1 Record types, 32 each (32 bytes)
- 28 Vector types, 48 each (1344 bytes)
-Total bytes = 5600
-0/0 implicit default constructors created
-0/0 implicit copy constructors created
-0/0 implicit move constructors created
-0/0 implicit copy assignment operators created
-0/0 implicit move assignment operators created
-0/0 implicit destructors created
-
-*** AST File Statistics:
- 0/12 source location entries read (0.000000%)
- 4/64 types read (6.250000%)
- 1032/1776 declarations read (58.108109%)
- 14/992 identifiers read (1.411290%)
- 0/4 macros read (0.000000%)
- 14/4814 statements read (0.290818%)
- 0/4 macros read (0.000000%)
- 0/802 lexical declcontexts read (0.000000%)
-
-*** PCH/ModuleFile Remappings:
-Global bit offset map:
- 0 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
- 1144160 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
-Global source location entry map:
- 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
- 8 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
-Global submodule map:
- 1 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
- 2 -> /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
-
-*** PCH/Modules Loaded:
-Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/A-CAAI8SHN1D31.pcm
- Base source location offset: 2147439884
- Base identifier ID: 0
- Number of identifiers: 496
- Base macro ID: 0
- Number of macros: 2
- Base submodule ID: 0
- Number of submodules: 1
- Submodule ID local -> global map:
- 0 -> 0
- Base selector ID: 0
- Number of selectors: 0
- Base preprocessed entity ID: 0
- Number of preprocessed entities: 0
- Base type index: 0
- Number of types: 32
- Base decl index: 0
- Number of decls: 888
-
-Module: /Users/ayokunleamodu/Repos/llvm-project/dedup-test/cache/71GTEYCXW85I/B-CAAI8SHN1D31.pcm
- Base source location offset: 2147396120
- Base identifier ID: 496
- Number of identifiers: 496
- Base macro ID: 2
- Number of macros: 2
- Base submodule ID: 1
- Number of submodules: 1
- Submodule ID local -> global map:
- 0 -> 1
- Base selector ID: 0
- Number of selectors: 0
- Base preprocessed entity ID: 0
- Number of preprocessed entities: 0
- Base type index: 32
- Number of types: 32
- Base decl index: 888
- Number of decls: 888
-
-
-Number of memory regions: 5
-Bytes used: 19784
-Bytes allocated: 20480
-Bytes wasted: 696 (includes alignment, etc)
-
-*** Decl Stats:
- 1907 decls total.
- 2 TranslationUnit decls, 104 each (208 bytes)
- 3 ExternCContext decls, 72 each (216 bytes)
- 807 Function decls, 168 each (135576 bytes)
- 806 ParmVar decls, 104 each (83824 bytes)
- 12 Field decls, 80 each (960 bytes)
- 3 CXXRecord decls, 144 each (432 bytes)
- 270 Typedef decls, 88 each (23760 bytes)
- 4 Import decls, 56 each (224 bytes)
-Total bytes = 245200
-
-*** Stmt/Expr Stats:
- 4843 stmts/exprs total.
- 4 UnresolvedLookupExpr, 64 each (256 bytes)
- 802 IntegerLiteral, 32 each (25664 bytes)
- 810 DeclRefExpr, 32 each (25920 bytes)
- 810 ImplicitCastExpr, 24 each (19440 bytes)
- 6 CallExpr, 24 each (144 bytes)
- 801 BinaryOperator, 32 each (25632 bytes)
- 805 ReturnStmt, 16 each (12880 bytes)
- 805 CompoundStmt, 16 each (12880 bytes)
-Total bytes = 122816
-
-STATISTICS FOR 'use.cpp':
-
-*** Preprocessor Stats:
-559 directives found:
- 552 #define.
- 0 #undef.
- #include/#include_next/#import:
- 2 source files entered.
- 0 max include stack depth
- 0 #if/#ifndef/#ifdef.
- 0 #else/#elif/#elifdef/#elifndef.
- 0 #endif.
- 2 #pragma.
-0 #if/#ifndef#ifdef regions skipped
-0/0/0 obj/fn/builtin macros expanded, 0 on the fast path.
-0 token paste (##) operations performed, 0 on the fast path.
-
-Preprocessor Memory: 106007B total
- BumpPtr: 53248
- Macro Expanded Tokens: 384
- Predefines Buffer: 32767
- Macros: 16512
- #pragma push_macro Info: 2056
- Poison Reasons: 1032
- Comment Handlers: 8
-
-*** Identifier Table Stats:
-# Identifiers: 13037
-# Empty Buckets: 19731
-Hash density (#identifiers per bucket): 0.397858
-Ave identifier length: 27.158472
-Max identifier length: 49
-
-Number of memory regions: 180
-Bytes used: 888582
-Bytes allocated: 950272
-Bytes wasted: 61690 (includes alignment, etc)
-
-*** HeaderSearch Stats:
-3 files tracked.
- 0 #import/#pragma once files.
- 4 #include/#include_next/#import.
- 0 #includes skipped due to the multi-include optimization.
-0 framework lookups.
-0 subframework lookups.
-
-*** Source Manager Stats:
-2 files mapped, 2 mem buffers mapped.
-5 local SLocEntries allocated (168 bytes of capacity), 23136B of SLoc address space used.
-12 loaded SLocEntries allocated (768 bytes of capacity), 87528B of SLoc address space used.
-2 duplicate loaded file SLocEntries detected (20694B of SLoc address space reusable).
-147 bytes of files mapped, 0 files with line #'s computed, 0 files with macro args computed.
-FileID scans: 15 linear, 0 binary.
-
-
-*** File Manager Stats:
-7 real files found, 8 real dirs found.
-0 virtual files found, 0 virtual dirs found.
-46 dir lookups, 11 dir cache misses.
-47 file lookups, 10 file cache misses.
-
-*** Virtual File System Stats:
-18 status() calls
-10 openFileForRead() calls
-0 dir_begin() calls
-1 getRealPath() calls
-0 exists() calls
-0 isLocal() calls
-
-===-------------------------------------------------------------------------===
- ... Statistics Collected ...
-===-------------------------------------------------------------------------===
-
- 4 file-search - Number of attempted #includes.
-110664 source-manager - Maximum number of bytes used by source locations (both loaded and local).
-
>From 7ebb518c2a1f575ba5774ca2e4fd72f16089237c Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 6 Jul 2026 14:37:08 -0400
Subject: [PATCH 06/32] fix SLoc de-dup miscount for files repeated within a
module
---
clang/lib/Serialization/ASTReader.cpp | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index d8041c06a4674..872c5bd458fac 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -4314,15 +4314,20 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
SmallVector<const FileEntry *, 64> Files;
bool Scanned = scanLoadedSLocEntries(F, Offsets, Files);
- // Classify duplicates and compute the reduced allocation request.
+ // Classify each entry against the canonical map as it stands now, before
+ // this module registers any of its own files, and store the decision per
+ // entry. The build pass reads these flags, so a file that appears
+ // multiple times in this module is classified consistently.
auto entrySize = [&](unsigned I) -> uint64_t {
return (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
};
+ SmallVector<bool, 64> IsDup(N, false);
unsigned NumDupEntries = 0;
uint64_t DupBytes = 0;
if (Scanned)
for (unsigned I = 0; I != N; ++I)
if (SourceMgr.isLoadedFileDuplicate(Files[I])) {
+ IsDup[I] = true;
uint64_t Size = entrySize(I);
SourceMgr.noteDuplicateLoadedFile(Size);
DupBytes += Size;
@@ -4368,7 +4373,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
SourceLocation::UIntTy LowEnd =
(I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) + 2;
const FileEntry *FE = Files[I];
- if (FE && SourceMgr.isLoadedFileDuplicate(FE)) {
+ if (IsDup[I]) {
// Redirect this file's locations into the module that first loaded
// it; reserve nothing here.
const SourceManager::LoadedFileLoc *Canon =
>From 8cee1dfcd4b3ba2a0e794c0b0cafb3745d299d0d Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 6 Jul 2026 14:54:07 -0400
Subject: [PATCH 07/32] clean up comments
---
clang/include/clang/Basic/SourceManager.h | 19 ++++-----
clang/include/clang/Serialization/ASTReader.h | 32 ++++++---------
.../include/clang/Serialization/ModuleFile.h | 30 +++++++-------
clang/lib/Serialization/ASTReader.cpp | 40 ++++++++-----------
4 files changed, 56 insertions(+), 65 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 75b346018bb67..4809a9dcbbd46 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,22 +755,23 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- /// --- Source-location de-duplication (prototype, Stage 2) ---
- /// The canonical loaded location of a file: where its SLoc entry first landed
- /// in the global address space. Lets a later module reuse it instead of
- /// re-allocating. Keyed by FileEntry identity (the same identity Clang uses
- /// to dedup file content).
+ // === Source location de-duplication ===
public:
+ /// Where a file's SLoc entry first landed in the loaded address space. A file
+ /// shared by several modules is loaded once; later modules reuse this rather
+ /// than allocating their own.
struct LoadedFileLoc {
- SourceLocation::UIntTy Offset = 0; ///< global raw start offset
+ SourceLocation::UIntTy Offset = 0; ///< global start offset
int ID = 0; ///< global SLoc entry ID
};
private:
+ /// The first loaded location of each file, keyed by the FileEntry identity
+ /// Clang already uses to share file contents.
llvm::DenseMap<const FileEntry *, LoadedFileLoc> CanonicalLoadedFiles;
- /// Number of loaded file SLoc entries that duplicated an already-loaded file.
+ /// Number of loaded file entries reused from an earlier module.
unsigned NumDuplicateLoadedFiles = 0;
- /// SLoc address-space bytes those duplicates reused instead of allocating.
+ /// Address-space bytes reused instead of allocated, for -print-stats.
uint64_t DuplicateLoadedBytes = 0;
public:
@@ -793,7 +794,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
CanonicalLoadedFiles.try_emplace(FE, LoadedFileLoc{Offset, ID});
}
- /// Account for a de-duplicated file entry (for -print-stats reporting).
+ /// Record that a loaded file entry was reused from an earlier module.
void noteDuplicateLoadedFile(uint64_t Size) {
++NumDuplicateLoadedFiles;
DuplicateLoadedBytes += Size;
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index ec4312cf1972c..a9991091fc659 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,17 +2399,16 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Walk module F's SLoc entry records (read-only). Fills Offsets[i]
- /// with each entry's local offset and \p Files[i] with its FileEntry (null
- /// for non-file entries). Returns false if the records couldn't be read.
+ /// Read \p F's SLoc entry records without materializing them, filling
+ /// \p Offsets[i] with each entry's local offset and \p Files[i] with its
+ /// FileEntry (null for non-file entries). Returns false on a malformed record.
bool scanLoadedSLocEntries(ModuleFile &F,
SmallVectorImpl<uint32_t> &Offsets,
SmallVectorImpl<const FileEntry *> &Files);
- /// Map a module-local SLoc entry offset (the value stored in an entry record,
- /// i.e. Record[0]) to its global raw start offset, applying the
- /// de-duplication remap. Equals (SLocEntryBaseOffset + LocalOffset) when the
- /// module is not de-duplicated.
+ /// Map a local SLoc entry offset (as stored in the entry record) to its
+ /// global start offset. This is (SLocEntryBaseOffset + LocalOffset) unless a
+ /// file in this module was reused from an earlier one.
SourceLocation::UIntTy remapSLocEntryOffset(ModuleFile &F,
uint32_t LocalOffset) const;
@@ -2507,15 +2506,10 @@ class ASTReader : public ExternalPreprocessorSource,
// translated or refactor the code to make it clear that
// TranslateSourceLocation won't be called with translated source location.
- // De-duplication (Stage 2): a module's local->global offset map may be
- // piecewise rather than a single flat shift, so look up the segment that
- // covers this location. The seeded identity segment makes this identical to
- // the flat shift below; redirect segments (Stage 2b) send a duplicated
- // file's locations into the module that first loaded it.
- //
- // Segments are keyed by the *file-offset* part of the location, so strip
- // the macro bit before matching and re-apply it to the result (a macro
- // location encodes its position in the low bits with the high bit set).
+ // When a file in this module was reused from an earlier one, the map is
+ // piecewise rather than a single shift, so find the segment covering this
+ // location. A macro location keeps its offset in the low bits with the high
+ // bit set, so match on the offset part and re-apply the bit to the result.
if (!ModuleFile.SLocRemap.empty()) {
SourceLocation::UIntTy Raw = Loc.getRawEncoding();
SourceLocation::UIntTy MacroBit = Raw & SourceLocation::MacroIDBit;
@@ -2550,9 +2544,9 @@ class ASTReader : public ExternalPreprocessorSource,
assert(FID.ID >= 0 && "Reading non-local FileID.");
if (FID.isInvalid())
return FID;
- // De-duplication (Stage 2b): a local FileID may map to the canonical copy
- // in an earlier module, so consult the explicit map when present. Local
- // FileID N corresponds to local entry index N-1.
+ // When a file was reused from an earlier module, its local FileID maps to
+ // that module's copy, so use the explicit map. Local FileID N is local
+ // entry index N-1.
if (!F.LocalToGlobalID.empty()) {
assert((unsigned)(FID.ID - 1) < F.LocalToGlobalID.size() &&
"local FileID out of range");
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 6b65089cc7e6b..a985c17904aa6 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -346,31 +346,33 @@ class ModuleFile {
/// AST file.
const uint32_t *SLocEntryOffsets = nullptr;
- // === Source-location de-duplication: offset remap (prototype, Stage 2) ===
+ // === Source location de-duplication ===
- /// One offset-remap segment in this module's raw SourceLocation space:
- /// a raw local location L in [LocalBegin, LocalEnd) maps to global L + Delta.
+ /// One segment of the local-to-global source location map: a local raw
+ /// location L in [LocalBegin, LocalEnd) maps to the global location L + Delta.
struct SLocRemapSegment {
SourceLocation::UIntTy LocalBegin;
SourceLocation::UIntTy LocalEnd;
int64_t Delta;
};
- /// Piecewise local->global offset map for this module, sorted by LocalBegin.
- /// Empty => fall back to the flat shift by (SLocEntryBaseOffset - 2).
- /// Stage 2a seeds a single identity segment equivalent to the flat shift;
- /// Stage 2b adds redirect/shift segments for de-duplicated files.
+ /// The local-to-global source location map for this module, sorted by
+ /// LocalBegin. A module whose files are all distinct has a single segment
+ /// equivalent to the flat shift by (SLocEntryBaseOffset - 2); when a file is
+ /// reused from an earlier module, extra segments redirect that file's
+ /// locations into the earlier module. Empty for a module that is not loaded
+ /// through this path, in which case the flat shift is used directly.
llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
- /// Maps a local SLoc entry index -> its resulting global SLoc entry ID.
- /// Kept entries map to their own newly-allocated ID; de-duplicated entries
- /// map to the ID of the canonical copy in an earlier module. Empty => no
- /// de-duplication for this module (global ID is SLocEntryBaseID + index).
+ /// Maps a local SLoc entry index to its global SLoc entry ID. A kept entry
+ /// maps to its own ID; a file reused from an earlier module maps to that
+ /// module's copy. Empty when no file was reused (the global ID is then
+ /// SLocEntryBaseID + index).
std::vector<int> LocalToGlobalID;
- /// For each kept global slot j (0-based, relative to SLocEntryBaseID), the
- /// original local entry index in this module (index into SLocEntryOffsets).
- /// Empty => no de-duplication (kept slot j == local index j).
+ /// For each kept entry, in order, its original local index (into
+ /// SLocEntryOffsets). Reused entries have no slot, so this skips them. Empty
+ /// when no file was reused (kept slot j is then local index j).
std::vector<unsigned> KeptSLocLocalIndex;
// === Identifiers ===
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 872c5bd458fac..f2e69bd42d1cb 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1913,9 +1913,8 @@ int ASTReader::getSLocEntryID(SourceLocation::UIntTy SLocOffset) {
bool Invalid = false;
- // De-duplication (Stage 2b): the global table holds only this module's kept
- // entries, so search over kept slots and translate each slot to its on-disk
- // local entry index when reading the offset.
+ // The table holds only this module's kept entries, so search over kept slots
+ // and map each back to its on-disk local index when reading the offset.
bool Dedup = !F->KeptSLocLocalIndex.empty();
unsigned NumSlots =
Dedup ? F->KeptSLocLocalIndex.size() : F->LocalNumSLocEntries;
@@ -2009,9 +2008,9 @@ bool ASTReader::ReadSLocEntry(int ID) {
};
ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
- // De-duplication (Stage 2b): with entries skipped, the global slot
- // (ID - SLocEntryBaseID) is no longer the on-disk local index, so translate
- // it through the kept-index table when present.
+ // When entries have been skipped, the global slot (ID - SLocEntryBaseID) is
+ // no longer the on-disk local index, so map it back through the kept-index
+ // table.
unsigned LocalIndex = ID - F->SLocEntryBaseID;
if (!F->KeptSLocLocalIndex.empty()) {
assert(LocalIndex < F->KeptSLocLocalIndex.size() && "kept slot out of range");
@@ -2024,8 +2023,6 @@ bool ASTReader::ReadSLocEntry(int ID) {
}
BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
- SourceLocation::UIntTy BaseOffset = F->SLocEntryBaseOffset;
- (void)BaseOffset;
++NumSLocEntriesRead;
Expected<llvm::BitstreamEntry> MaybeEntry = SLocEntryCursor.advance();
@@ -4307,17 +4304,16 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
F.SLocEntryOffsetsBase = Record[2] + F.SourceManagerBlockStartOffset;
unsigned N = F.LocalNumSLocEntries;
- // De-duplication (Stage 2b): scan this module's SLoc entries *before*
- // allocating, so we can recognize files already loaded by an earlier
- // module and reserve less address space for them.
+ // Scan this module's SLoc entries before allocating, so files already
+ // loaded by an earlier module can be recognized and reserve no space here.
SmallVector<uint32_t, 64> Offsets;
SmallVector<const FileEntry *, 64> Files;
bool Scanned = scanLoadedSLocEntries(F, Offsets, Files);
- // Classify each entry against the canonical map as it stands now, before
- // this module registers any of its own files, and store the decision per
- // entry. The build pass reads these flags, so a file that appears
- // multiple times in this module is classified consistently.
+ // Decide duplicates here, up front. The loop below registers each kept
+ // file as it goes, so checking against the map there would treat a file
+ // that appears more than once in this module as a duplicate of its own
+ // first occurrence.
auto entrySize = [&](unsigned I) -> uint64_t {
return (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
};
@@ -4336,8 +4332,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
unsigned ReducedNumEntries = N - NumDupEntries;
SourceLocation::UIntTy ReducedSize = SLocSpaceSize - DupBytes;
- // Reserve the reduced amount (equals the full amount when nothing is
- // de-duplicated).
+ // Reserve the reduced amount (equal to the full amount when no file is
+ // reused).
std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
SourceMgr.AllocateLoadedSLocEntries(ReducedNumEntries, ReducedSize);
if (!F.SLocEntryBaseID) {
@@ -4348,9 +4344,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
}
if (NumDupEntries == 0) {
- // No de-duplication: one identity segment (== flat shift) and a linear
- // local->global ID mapping. Register each file as the canonical copy so
- // later modules can reuse it.
+ // Nothing reused: a single segment equal to the flat shift, and a
+ // linear ID mapping. Record each file so later modules can reuse it.
F.SLocRemap.push_back(
{/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
/*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
@@ -4361,9 +4356,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
Files[I], F.SLocEntryBaseOffset + Offsets[I],
F.SLocEntryBaseID + (int)I);
} else {
- // De-duplication: build keep/redirect offset segments and the
- // local->global ID map. Skipped (duplicate) entries get no slot and no
- // address space; their references are redirected to the canonical copy.
+ // Build the offset segments and the ID map. A reused file gets no slot
+ // and no address space; its references point at the earlier copy.
F.LocalToGlobalID.assign(N, 0);
F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
uint64_t DupBefore = 0;
>From 1c39cde258648bb88f9e964b5a75cfd3e3820b05 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 6 Jul 2026 19:31:03 -0400
Subject: [PATCH 08/32] add bench scripts
---
dedup-test/bench/bench.sh | 124 +++++++++++++++++++++++++++++++++++
dedup-test/bench/timewrap.sh | 24 +++++++
2 files changed, 148 insertions(+)
create mode 100755 dedup-test/bench/bench.sh
create mode 100755 dedup-test/bench/timewrap.sh
diff --git a/dedup-test/bench/bench.sh b/dedup-test/bench/bench.sh
new file mode 100755
index 0000000000000..68a9667545866
--- /dev/null
+++ b/dedup-test/bench/bench.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+# Compare baseline (pre-patch) and patched clang on a modules-enabled build,
+# reporting peak RSS, CPU time, and loaded SourceLocation usage.
+#
+# The current checkout is the patched tree; the baseline comes from a detached
+# worktree at $BASELINE_REF, so the branch and index are left untouched. Both
+# clangs are built Release/no-asserts with the same flags.
+#
+# macOS (/usr/bin/time -l via timewrap.sh).
+set -euo pipefail
+
+REPO="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)"
+BASELINE_REF="${BASELINE_REF:-main}"
+BASELINE_TREE="${BASELINE_TREE:-$REPO/../llvm-bench-baseline}"
+JOBS="${JOBS:-$(sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
+RUNS="${RUNS:-5}"
+WORKLOAD_TARGET="${WORKLOAD_TARGET:-clangBasic}"
+OUT="${OUT:-$REPO/dedup-test/bench/out}"
+WRAP="$REPO/dedup-test/bench/timewrap.sh"
+chmod +x "$WRAP"
+mkdir -p "$OUT"
+
+echo "baseline=$BASELINE_REF jobs=$JOBS runs=$RUNS workload=$WORKLOAD_TARGET"
+
+# Baseline tree.
+if [ ! -d "$BASELINE_TREE" ]; then
+ git -C "$REPO" worktree add --detach "$BASELINE_TREE" "$BASELINE_REF"
+fi
+
+# Build a Release clang from a source tree.
+build_clang() { # <src> <tag>
+ local src="$1" tag="$2" bdir="$1/build-bench"
+ echo "building $tag clang"
+ cmake -G Ninja -S "$src/llvm" -B "$bdir" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_PROJECTS=clang \
+ -DLLVM_ENABLE_ASSERTIONS=OFF \
+ -DLLVM_TARGETS_TO_BUILD=Native >/dev/null
+ # Building 'clang' also produces the clang++/clang-cl symlinks.
+ ninja -C "$bdir" clang
+}
+build_clang "$REPO" patched
+build_clang "$BASELINE_TREE" baseline
+
+# Compile the workload with one clang, timing every invocation. The first build
+# populates the module cache; only subsequent clean rebuilds are recorded, so
+# the numbers reflect module loading rather than first-time module building.
+run_workload() { # <clang-tree> <tag>
+ local ctree="$1" tag="$2"
+ local cxx="$ctree/build-bench/bin/clang++" cc="$ctree/build-bench/bin/clang"
+ local wdir="$OUT/work-$tag" logdir="$OUT/log-$tag"
+ rm -rf "$wdir" "$logdir"; mkdir -p "$logdir"; : > "$logdir/all.txt"
+
+ cmake -G Ninja -S "$REPO/llvm" -B "$wdir" \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_ENABLE_PROJECTS=clang \
+ -DLLVM_ENABLE_MODULES=ON \
+ -DLLVM_TARGETS_TO_BUILD=Native \
+ -DCMAKE_C_COMPILER="$cc" -DCMAKE_CXX_COMPILER="$cxx" \
+ -DCMAKE_C_COMPILER_LAUNCHER="$WRAP" \
+ -DCMAKE_CXX_COMPILER_LAUNCHER="$WRAP" >/dev/null
+
+ # Warm the module cache. Output streams to the terminal (and to warm.log);
+ # pipefail makes a compile failure here abort the run.
+ echo "[$tag] warming module cache"
+ ninja -C "$wdir" "$WORKLOAD_TARGET" 2>&1 | tee "$logdir/warm.log"
+
+ for r in $(seq 1 "$RUNS"); do
+ echo "[$tag] measured run $r/$RUNS"
+ ninja -C "$wdir" -t clean >/dev/null 2>&1
+ rm -f "$logdir"/*.m
+ BENCH_LOG_DIR="$logdir" \
+ ninja -C "$wdir" -j "$JOBS" "$WORKLOAD_TARGET" 2>&1 | tee "$logdir/run$r.log"
+ cat "$logdir"/*.m >> "$logdir/all.txt"
+ done
+}
+run_workload "$REPO" patched
+run_workload "$BASELINE_TREE" baseline
+PATCHED_LOG="$OUT/log-patched/all.txt"
+BASELINE_LOG="$OUT/log-baseline/all.txt"
+
+# Loaded SourceLocation usage on one representative TU, for each clang.
+sloc_stats() { # <tag>
+ local wdir="$OUT/work-$1" cmd
+ cmd=$(ninja -C "$wdir" -t commands "$WORKLOAD_TARGET" 2>/dev/null \
+ | grep -m1 -E 'clang\+\+.*\.cpp\.o') || return 0
+ eval "$cmd -Xclang -print-stats" 2>&1 \
+ | awk -v t="$1" '/loaded SLocEntries|de-duplicated/{print " ["t"] "$0}'
+}
+
+python3 - "$BASELINE_LOG" "$PATCHED_LOG" <<'PY'
+import sys, math, statistics as st
+def load(p):
+ rss=[]; cpu=[]
+ for ln in open(p):
+ a=ln.split()
+ if len(a)==3:
+ rss.append(float(a[0])/1048576.0) # bytes -> MB
+ cpu.append(float(a[1])+float(a[2])) # user+sys
+ return rss,cpu
+gm=lambda xs: math.exp(sum(map(math.log,xs))/len(xs)) if xs else 0.0
+br,bc=load(sys.argv[1]); pr,pc=load(sys.argv[2])
+def row(n,b,p,f):
+ d=(p-b)/b*100 if b else 0.0
+ print(f"{n:<32}{f(b):>12}{f(p):>12}{d:>+9.1f}%")
+mb=lambda x:f"{x:,.0f}"; s=lambda x:f"{x:.3f}"
+print(f"\n{'':<32}{'baseline':>12}{'patched':>12}{'delta':>10}")
+print("-"*66)
+print(f"compiles: {len(bc)} baseline / {len(pc)} patched")
+row("peak RSS, max cc1 (MB)", max(br), max(pr), mb)
+row("peak RSS, geomean/TU (MB)", gm(br), gm(pr), mb)
+row("CPU time, total (s)", sum(bc), sum(pc), s)
+row("CPU time, geomean/TU (s)", gm(bc), gm(pc), s)
+row("CPU time, median/TU (s)", st.median(bc), st.median(pc), s)
+PY
+
+echo
+echo "loaded SourceLocation usage (one TU):"
+sloc_stats baseline || true
+sloc_stats patched || true
+
+echo
+echo "logs: $OUT/log-{baseline,patched}/all.txt"
+echo "drop the baseline tree with: git worktree remove $BASELINE_TREE"
diff --git a/dedup-test/bench/timewrap.sh b/dedup-test/bench/timewrap.sh
new file mode 100755
index 0000000000000..02fa917f6d3b4
--- /dev/null
+++ b/dedup-test/bench/timewrap.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Compiler launcher. Runs the compile under `time -l` and, when BENCH_LOG_DIR
+# is set, records peak RSS (bytes) and user+sys time (seconds) for it there.
+# With BENCH_LOG_DIR unset (e.g. the warm build) it just runs the compile.
+set -u
+
+t=$(mktemp)
+/usr/bin/time -l "$@" 2>"$t"
+rc=$?
+
+rss=$(awk '/maximum resident set size/{print $1}' "$t")
+read -r usr sys < <(awk '/real/&&/user/&&/sys/{print $3, $5}' "$t")
+
+# Record only measured builds, and only for real compiles (link/archive steps
+# have no RSS line).
+if [ -n "${BENCH_LOG_DIR:-}" ] && [ -n "${rss:-}" ]; then
+ printf '%s %s %s\n' "$rss" "${usr:-0}" "${sys:-0}" \
+ > "$BENCH_LOG_DIR/$(date +%s)-$$-$RANDOM.m"
+fi
+
+# Forward the compiler's own stderr; drop the trailing resource block.
+awk '/real/&&/user/&&/sys/{stop=1} !stop{print}' "$t" >&2
+rm -f "$t"
+exit $rc
>From 35bddf7f9f965be389f73025ccaf4a62c2a1623e Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 13:14:02 +0000
Subject: [PATCH 09/32] cleaned up comments and variable names
---
.gitignore | 3 +
clang/include/clang/Basic/SourceManager.h | 37 +-----
clang/include/clang/Serialization/ASTReader.h | 62 ++++++++--
.../include/clang/Serialization/ModuleFile.h | 16 +--
clang/lib/Serialization/ASTReader.cpp | 109 ++++++++++++------
5 files changed, 145 insertions(+), 82 deletions(-)
diff --git a/.gitignore b/.gitignore
index a4382c9ea7390..1bceeef94e7e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,9 @@
# Nested build directory
/build*
+/ Test directory
+/dedup-test
+
#==============================================================================#
# Explicit files to ignore (only matches one).
#==============================================================================#
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 4809a9dcbbd46..424de78725492 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,45 +755,18 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- // === Source location de-duplication ===
-public:
- /// Where a file's SLoc entry first landed in the loaded address space. A file
- /// shared by several modules is loaded once; later modules reuse this rather
- /// than allocating their own.
- struct LoadedFileLoc {
- SourceLocation::UIntTy Offset = 0; ///< global start offset
- int ID = 0; ///< global SLoc entry ID
- };
-
+ // Source location de-duplication. A file included into many modules is
+ // serialized into each of their PCMs. ASTReader keeps the first loaded copy
+ // and redirects later modules' references to it instead of allocating a
+ // duplicate SLoc range. These counters record the address space that reuse
+ // saved, for -print-stats.
private:
- /// The first loaded location of each file, keyed by the FileEntry identity
- /// Clang already uses to share file contents.
- llvm::DenseMap<const FileEntry *, LoadedFileLoc> CanonicalLoadedFiles;
/// Number of loaded file entries reused from an earlier module.
unsigned NumDuplicateLoadedFiles = 0;
/// Address-space bytes reused instead of allocated, for -print-stats.
uint64_t DuplicateLoadedBytes = 0;
public:
- /// Whether a file with this identity was already loaded into the address
- /// space by an earlier module (i.e. its SLoc entry would be a duplicate).
- bool isLoadedFileDuplicate(const FileEntry *FE) const {
- return FE && CanonicalLoadedFiles.contains(FE);
- }
-
- /// The canonical loaded location of a previously-loaded file, or null.
- const LoadedFileLoc *getCanonicalLoadedFile(const FileEntry *FE) const {
- auto It = CanonicalLoadedFiles.find(FE);
- return It == CanonicalLoadedFiles.end() ? nullptr : &It->second;
- }
-
- /// Record the canonical loaded location of a file the first time it loads.
- void registerCanonicalLoadedFile(const FileEntry *FE,
- SourceLocation::UIntTy Offset, int ID) {
- if (FE)
- CanonicalLoadedFiles.try_emplace(FE, LoadedFileLoc{Offset, ID});
- }
-
/// Record that a loaded file entry was reused from an earlier module.
void noteDuplicateLoadedFile(uint64_t Size) {
++NumDuplicateLoadedFiles;
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index a9991091fc659..e1020e64fcca8 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,12 +2399,49 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
+ /// Identity of an input file, taken from serialized metadata without touching
+ /// the filesystem. It uses the stored name, size, and modification time that
+ /// Clang's own input-file staleness check relies on. An empty Name marks a
+ /// non-file entry (a buffer or expansion), which never deduplicates.
+ struct SLocFileIdentity {
+ StringRef Name;
+ off_t Size = 0;
+ time_t Time = 0;
+ };
+
+ /// Where a file's SLoc entry first landed in the loaded address space, kept
+ /// with the identity fields used to confirm a later module's same-named entry
+ /// really is the same file.
+ struct PrimaryLoadedFileLoc {
+ SourceLocation::UIntTy Offset = 0; ///< global start offset
+ int ID = 0; ///< global SLoc entry ID
+ off_t Size = 0;
+ time_t Time = 0;
+ };
+
+ /// Files already loaded into the loaded SLoc address space, keyed by stored
+ /// name. Lets a later module reuse an earlier module's copy instead of
+ /// allocating a duplicate range. The key is serialized metadata, so no input
+ /// file is resolved or stat'd at load time.
+ llvm::StringMap<PrimaryLoadedFileLoc> PrimaryLoadedFiles;
+
+ /// The location of a previously-loaded file matching \p Id (same name, size,
+ /// and time), or null if none has been loaded.
+ const PrimaryLoadedFileLoc *
+ getPrimaryLoadedFile(const SLocFileIdentity &Id) const;
+
+ /// Record the location of a file the first time it loads. \p Id must name a
+ /// file entry (non-empty Name). Later duplicates are ignored.
+ void registerPrimaryLoadedFile(const SLocFileIdentity &Id,
+ SourceLocation::UIntTy Offset, int ID);
+
/// Read \p F's SLoc entry records without materializing them, filling
/// \p Offsets[i] with each entry's local offset and \p Files[i] with its
- /// FileEntry (null for non-file entries). Returns false on a malformed record.
- bool scanLoadedSLocEntries(ModuleFile &F,
- SmallVectorImpl<uint32_t> &Offsets,
- SmallVectorImpl<const FileEntry *> &Files);
+ /// file identity (empty Name for non-file entries). File identity comes from
+ /// serialized metadata only, with no input file resolved on disk. Returns
+ /// false on a malformed record.
+ bool scanLoadedSLocEntries(ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
+ SmallVectorImpl<SLocFileIdentity> &Files);
/// Map a local SLoc entry offset (as stored in the entry record) to its
/// global start offset. This is (SLocEntryBaseOffset + LocalOffset) unless a
@@ -2514,13 +2551,24 @@ class ASTReader : public ExternalPreprocessorSource,
SourceLocation::UIntTy Raw = Loc.getRawEncoding();
SourceLocation::UIntTy MacroBit = Raw & SourceLocation::MacroIDBit;
SourceLocation::UIntTy Low = Raw & ~SourceLocation::MacroIDBit;
- for (const auto &Seg : ModuleFile.SLocRemap)
- if (Low >= Seg.LocalBegin && Low < Seg.LocalEnd)
+ // The list is sorted by LocalBegin and its segments are contiguous, so
+ // the covering segment is the last one whose LocalBegin is <= Low.
+ auto It = llvm::upper_bound(
+ ModuleFile.SLocRemap, Low,
+ [](SourceLocation::UIntTy V,
+ const serialization::ModuleFile::SLocRemapSegment &S) {
+ return V < S.LocalBegin;
+ });
+ if (It != ModuleFile.SLocRemap.begin()) {
+ const auto &Seg = *std::prev(It);
+ if (Low < Seg.LocalEnd)
return SourceLocation::getFromRawEncoding(
(static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
Seg.Delta)) |
MacroBit);
- // No segment matched (shouldn't happen): fall through to the flat shift.
+ }
+ // No segment matched. This shouldn't happen, so fall through to the
+ // flat shift below.
}
return Loc.getLocWithOffset(ModuleFile.SLocEntryBaseOffset - 2);
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index a985c17904aa6..76cefc2280936 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -348,8 +348,8 @@ class ModuleFile {
// === Source location de-duplication ===
- /// One segment of the local-to-global source location map: a local raw
- /// location L in [LocalBegin, LocalEnd) maps to the global location L + Delta.
+ /// One segment of the local-to-global source location map. A local raw
+ /// location L in [LocalBegin, LocalEnd) maps to global location L + Delta.
struct SLocRemapSegment {
SourceLocation::UIntTy LocalBegin;
SourceLocation::UIntTy LocalEnd;
@@ -357,15 +357,15 @@ class ModuleFile {
};
/// The local-to-global source location map for this module, sorted by
- /// LocalBegin. A module whose files are all distinct has a single segment
- /// equivalent to the flat shift by (SLocEntryBaseOffset - 2); when a file is
- /// reused from an earlier module, extra segments redirect that file's
- /// locations into the earlier module. Empty for a module that is not loaded
- /// through this path, in which case the flat shift is used directly.
+ /// LocalBegin. When a module's files are all distinct it holds a single
+ /// segment equivalent to the flat shift by (SLocEntryBaseOffset - 2). When a
+ /// file is reused from an earlier module, extra segments redirect that file's
+ /// locations into the earlier module. Empty when this path did not run, in
+ /// which case the flat shift is used directly.
llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
/// Maps a local SLoc entry index to its global SLoc entry ID. A kept entry
- /// maps to its own ID; a file reused from an earlier module maps to that
+ /// maps to its own ID. A file reused from an earlier module maps to that
/// module's copy. Empty when no file was reused (the global ID is then
/// SLocEntryBaseID + index).
std::vector<int> LocalToGlobalID;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f2e69bd42d1cb..5df770034e7db 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1815,18 +1815,40 @@ llvm::Error ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
}
}
+const ASTReader::PrimaryLoadedFileLoc *
+ASTReader::getPrimaryLoadedFile(const SLocFileIdentity &Id) const {
+ assert(!Id.Name.empty() && "querying a non-file entry");
+ auto It = PrimaryLoadedFiles.find(Id.Name);
+ if (It == PrimaryLoadedFiles.end())
+ return nullptr;
+ const PrimaryLoadedFileLoc &Primary = It->second;
+ // A matching name but different size or time is a different file, e.g. two
+ // modules built against different versions of the same path. Don't merge.
+ if (Primary.Size != Id.Size || Primary.Time != Id.Time)
+ return nullptr;
+ return &Primary;
+}
+
+void ASTReader::registerPrimaryLoadedFile(const SLocFileIdentity &Id,
+ SourceLocation::UIntTy Offset,
+ int ID) {
+ assert(!Id.Name.empty() && "registering a non-file entry");
+ PrimaryLoadedFiles.try_emplace(
+ Id.Name, PrimaryLoadedFileLoc{Offset, ID, Id.Size, Id.Time});
+}
+
bool ASTReader::scanLoadedSLocEntries(
ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
- SmallVectorImpl<const FileEntry *> &Files) {
+ SmallVectorImpl<SLocFileIdentity> &Files) {
unsigned N = F.LocalNumSLocEntries;
Offsets.assign(N, 0);
- Files.assign(N, nullptr);
+ Files.assign(N, SLocFileIdentity{});
BitstreamCursor &Cursor = F.SLocEntryCursor;
SavedStreamPosition SavedPosition(Cursor);
for (unsigned I = 0; I != N; ++I) {
- if (llvm::Error Err = Cursor.JumpToBit(F.SLocEntryOffsetsBase +
- F.SLocEntryOffsets[I])) {
+ if (llvm::Error Err =
+ Cursor.JumpToBit(F.SLocEntryOffsetsBase + F.SLocEntryOffsets[I])) {
consumeError(std::move(Err));
return false;
}
@@ -1839,16 +1861,22 @@ bool ASTReader::scanLoadedSLocEntries(
return false;
RecordData Record;
- StringRef Blob;
- Expected<unsigned> Code = Cursor.readRecord(Entry->ID, Record, &Blob);
+ Expected<unsigned> Code = Cursor.readRecord(Entry->ID, Record);
if (!Code) {
consumeError(Code.takeError());
return false;
}
Offsets[I] = (uint32_t)Record[0];
- if (Code.get() == SM_SLOC_FILE_ENTRY)
- if (OptionalFileEntryRef File = getInputFile(F, Record[4]).getFile())
- Files[I] = &File->getFileEntry();
+ if (Code.get() == SM_SLOC_FILE_ENTRY) {
+ // Identify the file from serialized metadata only. Resolving it on disk
+ // here (getInputFile) would stat and open every input file of every
+ // module at load, which is prohibitive. The stored name, size, and time
+ // are the identity Clang's own staleness check already uses.
+ InputFileInfo IFI = getInputFileInfo(F, Record[4]);
+ if (IFI.isValid())
+ Files[I] = SLocFileIdentity{IFI.UnresolvedImportedFilename,
+ IFI.StoredSize, IFI.StoredTime};
+ }
}
return true;
}
@@ -1859,12 +1887,22 @@ ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
// are reserved). Find the segment covering it and apply that segment's delta.
if (!F.SLocRemap.empty()) {
SourceLocation::UIntTy Low = LocalOffset + 2;
- for (const auto &Seg : F.SLocRemap)
- if (Low >= Seg.LocalBegin && Low < Seg.LocalEnd)
+ // The list is sorted by LocalBegin and its segments are contiguous, so the
+ // covering segment is the last one whose LocalBegin is <= Low.
+ auto It = llvm::upper_bound(
+ F.SLocRemap, Low,
+ [](SourceLocation::UIntTy V,
+ const serialization::ModuleFile::SLocRemapSegment &S) {
+ return V < S.LocalBegin;
+ });
+ if (It != F.SLocRemap.begin()) {
+ const auto &Seg = *std::prev(It);
+ if (Low < Seg.LocalEnd)
return static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
Seg.Delta);
+ }
}
- // No remap (or no segment matched): original flat shift.
+ // No remap, or no segment matched. Use the original flat shift.
return F.SLocEntryBaseOffset + LocalOffset;
}
@@ -2013,7 +2051,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
// table.
unsigned LocalIndex = ID - F->SLocEntryBaseID;
if (!F->KeptSLocLocalIndex.empty()) {
- assert(LocalIndex < F->KeptSLocLocalIndex.size() && "kept slot out of range");
+ assert(LocalIndex < F->KeptSLocLocalIndex.size() &&
+ "kept slot out of range");
LocalIndex = F->KeptSLocLocalIndex[LocalIndex];
}
if (llvm::Error Err = F->SLocEntryCursor.JumpToBit(
@@ -2113,9 +2152,9 @@ bool ASTReader::ReadSLocEntry(int ID) {
auto Buffer = ReadBuffer(SLocEntryCursor, Name);
if (!Buffer)
return true;
- FileID FID = SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
- remapSLocEntryOffset(*F, Offset),
- IncludeLoc);
+ FileID FID =
+ SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
+ remapSLocEntryOffset(*F, Offset), IncludeLoc);
if (Record[3]) {
auto &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.setHasLineDirectives();
@@ -4305,9 +4344,10 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
unsigned N = F.LocalNumSLocEntries;
// Scan this module's SLoc entries before allocating, so files already
- // loaded by an earlier module can be recognized and reserve no space here.
+ // loaded by an earlier module can be recognized and reserve no space
+ // here.
SmallVector<uint32_t, 64> Offsets;
- SmallVector<const FileEntry *, 64> Files;
+ SmallVector<SLocFileIdentity, 64> Files;
bool Scanned = scanLoadedSLocEntries(F, Offsets, Files);
// Decide duplicates here, up front. The loop below registers each kept
@@ -4322,7 +4362,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
uint64_t DupBytes = 0;
if (Scanned)
for (unsigned I = 0; I != N; ++I)
- if (SourceMgr.isLoadedFileDuplicate(Files[I])) {
+ if (!Files[I].Name.empty() && getPrimaryLoadedFile(Files[I])) {
IsDup[I] = true;
uint64_t Size = entrySize(I);
SourceMgr.noteDuplicateLoadedFile(Size);
@@ -4344,20 +4384,20 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
}
if (NumDupEntries == 0) {
- // Nothing reused: a single segment equal to the flat shift, and a
+ // Nothing reused. A single segment equal to the flat shift, plus a
// linear ID mapping. Record each file so later modules can reuse it.
F.SLocRemap.push_back(
{/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
/*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
if (Scanned)
for (unsigned I = 0; I != N; ++I)
- if (Files[I])
- SourceMgr.registerCanonicalLoadedFile(
- Files[I], F.SLocEntryBaseOffset + Offsets[I],
- F.SLocEntryBaseID + (int)I);
+ if (!Files[I].Name.empty())
+ registerPrimaryLoadedFile(Files[I],
+ F.SLocEntryBaseOffset + Offsets[I],
+ F.SLocEntryBaseID + (int)I);
} else {
// Build the offset segments and the ID map. A reused file gets no slot
- // and no address space; its references point at the earlier copy.
+ // and no address space, and its references point at the earlier copy.
F.LocalToGlobalID.assign(N, 0);
F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
uint64_t DupBefore = 0;
@@ -4366,20 +4406,19 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
SourceLocation::UIntTy LowStart = Offsets[I] + 2;
SourceLocation::UIntTy LowEnd =
(I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) + 2;
- const FileEntry *FE = Files[I];
+ const SLocFileIdentity &Id = Files[I];
if (IsDup[I]) {
// Redirect this file's locations into the module that first loaded
- // it; reserve nothing here.
- const SourceManager::LoadedFileLoc *Canon =
- SourceMgr.getCanonicalLoadedFile(FE);
+ // it and reserve nothing here.
+ const PrimaryLoadedFileLoc *Primary = getPrimaryLoadedFile(Id);
F.SLocRemap.push_back({LowStart, LowEnd,
- static_cast<int64_t>(Canon->Offset) -
+ static_cast<int64_t>(Primary->Offset) -
static_cast<int64_t>(LowStart)});
- F.LocalToGlobalID[I] = Canon->ID;
+ F.LocalToGlobalID[I] = Primary->ID;
DupBefore += LowEnd - LowStart;
} else {
- // Keep: lands in this module's block, shifted down to close gaps
- // left by skipped duplicates before it.
+ // Kept entry. It lands in this module's block, shifted down to
+ // close gaps left by skipped duplicates before it.
int GlobalID = F.SLocEntryBaseID + (int)KeptCount;
SourceLocation::UIntTy GlobalStart =
static_cast<SourceLocation::UIntTy>(F.SLocEntryBaseOffset +
@@ -4389,8 +4428,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
static_cast<int64_t>(LowStart)});
F.LocalToGlobalID[I] = GlobalID;
F.KeptSLocLocalIndex.push_back(I);
- if (FE)
- SourceMgr.registerCanonicalLoadedFile(FE, GlobalStart, GlobalID);
+ if (!Id.Name.empty())
+ registerPrimaryLoadedFile(Id, GlobalStart, GlobalID);
++KeptCount;
}
}
>From f2be131ccd936ad0a211722f3884fe4dc8a116fc Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 13:16:09 +0000
Subject: [PATCH 10/32] Stop tracking dedup-test/
---
dedup-test/a.h | 5 -
dedup-test/b.h | 5 -
dedup-test/bench/bench.sh | 124 -----------
dedup-test/bench/timewrap.sh | 24 ---
dedup-test/module.modulemap | 2 -
dedup-test/shared.h | 403 -----------------------------------
dedup-test/use.cpp | 3 -
dedup-test/use1.cpp | 2 -
dedup-test/use_err.cpp | 14 --
dedup-test/use_err_rev.cpp | 10 -
10 files changed, 592 deletions(-)
delete mode 100644 dedup-test/a.h
delete mode 100644 dedup-test/b.h
delete mode 100755 dedup-test/bench/bench.sh
delete mode 100755 dedup-test/bench/timewrap.sh
delete mode 100644 dedup-test/module.modulemap
delete mode 100644 dedup-test/shared.h
delete mode 100644 dedup-test/use.cpp
delete mode 100644 dedup-test/use1.cpp
delete mode 100644 dedup-test/use_err.cpp
delete mode 100644 dedup-test/use_err_rev.cpp
diff --git a/dedup-test/a.h b/dedup-test/a.h
deleted file mode 100644
index 1fe7fc27db549..0000000000000
--- a/dedup-test/a.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#ifndef A_H
-#define A_H
-#include "shared.h"
-inline int a_entry(int x) { return shared_fn_0(x); }
-#endif
diff --git a/dedup-test/b.h b/dedup-test/b.h
deleted file mode 100644
index 85839261fc602..0000000000000
--- a/dedup-test/b.h
+++ /dev/null
@@ -1,5 +0,0 @@
-#ifndef B_H
-#define B_H
-#include "shared.h"
-inline int b_entry(int x) { return shared_fn_1(x); }
-#endif
diff --git a/dedup-test/bench/bench.sh b/dedup-test/bench/bench.sh
deleted file mode 100755
index 68a9667545866..0000000000000
--- a/dedup-test/bench/bench.sh
+++ /dev/null
@@ -1,124 +0,0 @@
-#!/usr/bin/env bash
-# Compare baseline (pre-patch) and patched clang on a modules-enabled build,
-# reporting peak RSS, CPU time, and loaded SourceLocation usage.
-#
-# The current checkout is the patched tree; the baseline comes from a detached
-# worktree at $BASELINE_REF, so the branch and index are left untouched. Both
-# clangs are built Release/no-asserts with the same flags.
-#
-# macOS (/usr/bin/time -l via timewrap.sh).
-set -euo pipefail
-
-REPO="$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel)"
-BASELINE_REF="${BASELINE_REF:-main}"
-BASELINE_TREE="${BASELINE_TREE:-$REPO/../llvm-bench-baseline}"
-JOBS="${JOBS:-$(sysctl -n hw.ncpu 2>/dev/null || echo 4)}"
-RUNS="${RUNS:-5}"
-WORKLOAD_TARGET="${WORKLOAD_TARGET:-clangBasic}"
-OUT="${OUT:-$REPO/dedup-test/bench/out}"
-WRAP="$REPO/dedup-test/bench/timewrap.sh"
-chmod +x "$WRAP"
-mkdir -p "$OUT"
-
-echo "baseline=$BASELINE_REF jobs=$JOBS runs=$RUNS workload=$WORKLOAD_TARGET"
-
-# Baseline tree.
-if [ ! -d "$BASELINE_TREE" ]; then
- git -C "$REPO" worktree add --detach "$BASELINE_TREE" "$BASELINE_REF"
-fi
-
-# Build a Release clang from a source tree.
-build_clang() { # <src> <tag>
- local src="$1" tag="$2" bdir="$1/build-bench"
- echo "building $tag clang"
- cmake -G Ninja -S "$src/llvm" -B "$bdir" \
- -DCMAKE_BUILD_TYPE=Release \
- -DLLVM_ENABLE_PROJECTS=clang \
- -DLLVM_ENABLE_ASSERTIONS=OFF \
- -DLLVM_TARGETS_TO_BUILD=Native >/dev/null
- # Building 'clang' also produces the clang++/clang-cl symlinks.
- ninja -C "$bdir" clang
-}
-build_clang "$REPO" patched
-build_clang "$BASELINE_TREE" baseline
-
-# Compile the workload with one clang, timing every invocation. The first build
-# populates the module cache; only subsequent clean rebuilds are recorded, so
-# the numbers reflect module loading rather than first-time module building.
-run_workload() { # <clang-tree> <tag>
- local ctree="$1" tag="$2"
- local cxx="$ctree/build-bench/bin/clang++" cc="$ctree/build-bench/bin/clang"
- local wdir="$OUT/work-$tag" logdir="$OUT/log-$tag"
- rm -rf "$wdir" "$logdir"; mkdir -p "$logdir"; : > "$logdir/all.txt"
-
- cmake -G Ninja -S "$REPO/llvm" -B "$wdir" \
- -DCMAKE_BUILD_TYPE=Release \
- -DLLVM_ENABLE_PROJECTS=clang \
- -DLLVM_ENABLE_MODULES=ON \
- -DLLVM_TARGETS_TO_BUILD=Native \
- -DCMAKE_C_COMPILER="$cc" -DCMAKE_CXX_COMPILER="$cxx" \
- -DCMAKE_C_COMPILER_LAUNCHER="$WRAP" \
- -DCMAKE_CXX_COMPILER_LAUNCHER="$WRAP" >/dev/null
-
- # Warm the module cache. Output streams to the terminal (and to warm.log);
- # pipefail makes a compile failure here abort the run.
- echo "[$tag] warming module cache"
- ninja -C "$wdir" "$WORKLOAD_TARGET" 2>&1 | tee "$logdir/warm.log"
-
- for r in $(seq 1 "$RUNS"); do
- echo "[$tag] measured run $r/$RUNS"
- ninja -C "$wdir" -t clean >/dev/null 2>&1
- rm -f "$logdir"/*.m
- BENCH_LOG_DIR="$logdir" \
- ninja -C "$wdir" -j "$JOBS" "$WORKLOAD_TARGET" 2>&1 | tee "$logdir/run$r.log"
- cat "$logdir"/*.m >> "$logdir/all.txt"
- done
-}
-run_workload "$REPO" patched
-run_workload "$BASELINE_TREE" baseline
-PATCHED_LOG="$OUT/log-patched/all.txt"
-BASELINE_LOG="$OUT/log-baseline/all.txt"
-
-# Loaded SourceLocation usage on one representative TU, for each clang.
-sloc_stats() { # <tag>
- local wdir="$OUT/work-$1" cmd
- cmd=$(ninja -C "$wdir" -t commands "$WORKLOAD_TARGET" 2>/dev/null \
- | grep -m1 -E 'clang\+\+.*\.cpp\.o') || return 0
- eval "$cmd -Xclang -print-stats" 2>&1 \
- | awk -v t="$1" '/loaded SLocEntries|de-duplicated/{print " ["t"] "$0}'
-}
-
-python3 - "$BASELINE_LOG" "$PATCHED_LOG" <<'PY'
-import sys, math, statistics as st
-def load(p):
- rss=[]; cpu=[]
- for ln in open(p):
- a=ln.split()
- if len(a)==3:
- rss.append(float(a[0])/1048576.0) # bytes -> MB
- cpu.append(float(a[1])+float(a[2])) # user+sys
- return rss,cpu
-gm=lambda xs: math.exp(sum(map(math.log,xs))/len(xs)) if xs else 0.0
-br,bc=load(sys.argv[1]); pr,pc=load(sys.argv[2])
-def row(n,b,p,f):
- d=(p-b)/b*100 if b else 0.0
- print(f"{n:<32}{f(b):>12}{f(p):>12}{d:>+9.1f}%")
-mb=lambda x:f"{x:,.0f}"; s=lambda x:f"{x:.3f}"
-print(f"\n{'':<32}{'baseline':>12}{'patched':>12}{'delta':>10}")
-print("-"*66)
-print(f"compiles: {len(bc)} baseline / {len(pc)} patched")
-row("peak RSS, max cc1 (MB)", max(br), max(pr), mb)
-row("peak RSS, geomean/TU (MB)", gm(br), gm(pr), mb)
-row("CPU time, total (s)", sum(bc), sum(pc), s)
-row("CPU time, geomean/TU (s)", gm(bc), gm(pc), s)
-row("CPU time, median/TU (s)", st.median(bc), st.median(pc), s)
-PY
-
-echo
-echo "loaded SourceLocation usage (one TU):"
-sloc_stats baseline || true
-sloc_stats patched || true
-
-echo
-echo "logs: $OUT/log-{baseline,patched}/all.txt"
-echo "drop the baseline tree with: git worktree remove $BASELINE_TREE"
diff --git a/dedup-test/bench/timewrap.sh b/dedup-test/bench/timewrap.sh
deleted file mode 100755
index 02fa917f6d3b4..0000000000000
--- a/dedup-test/bench/timewrap.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Compiler launcher. Runs the compile under `time -l` and, when BENCH_LOG_DIR
-# is set, records peak RSS (bytes) and user+sys time (seconds) for it there.
-# With BENCH_LOG_DIR unset (e.g. the warm build) it just runs the compile.
-set -u
-
-t=$(mktemp)
-/usr/bin/time -l "$@" 2>"$t"
-rc=$?
-
-rss=$(awk '/maximum resident set size/{print $1}' "$t")
-read -r usr sys < <(awk '/real/&&/user/&&/sys/{print $3, $5}' "$t")
-
-# Record only measured builds, and only for real compiles (link/archive steps
-# have no RSS line).
-if [ -n "${BENCH_LOG_DIR:-}" ] && [ -n "${rss:-}" ]; then
- printf '%s %s %s\n' "$rss" "${usr:-0}" "${sys:-0}" \
- > "$BENCH_LOG_DIR/$(date +%s)-$$-$RANDOM.m"
-fi
-
-# Forward the compiler's own stderr; drop the trailing resource block.
-awk '/real/&&/user/&&/sys/{stop=1} !stop{print}' "$t" >&2
-rm -f "$t"
-exit $rc
diff --git a/dedup-test/module.modulemap b/dedup-test/module.modulemap
deleted file mode 100644
index 2da1c03a22211..0000000000000
--- a/dedup-test/module.modulemap
+++ /dev/null
@@ -1,2 +0,0 @@
-module A { header "a.h" export * }
-module B { header "b.h" export * }
diff --git a/dedup-test/shared.h b/dedup-test/shared.h
deleted file mode 100644
index 3ac57a45355f9..0000000000000
--- a/dedup-test/shared.h
+++ /dev/null
@@ -1,403 +0,0 @@
-#ifndef SHARED_H
-#define SHARED_H
-inline int shared_fn_0(int x) { return x + 0; }
-inline int shared_fn_1(int x) { return x + 1; }
-inline int shared_fn_2(int x) { return x + 2; }
-inline int shared_fn_3(int x) { return x + 3; }
-inline int shared_fn_4(int x) { return x + 4; }
-inline int shared_fn_5(int x) { return x + 5; }
-inline int shared_fn_6(int x) { return x + 6; }
-inline int shared_fn_7(int x) { return x + 7; }
-inline int shared_fn_8(int x) { return x + 8; }
-inline int shared_fn_9(int x) { return x + 9; }
-inline int shared_fn_10(int x) { return x + 10; }
-inline int shared_fn_11(int x) { return x + 11; }
-inline int shared_fn_12(int x) { return x + 12; }
-inline int shared_fn_13(int x) { return x + 13; }
-inline int shared_fn_14(int x) { return x + 14; }
-inline int shared_fn_15(int x) { return x + 15; }
-inline int shared_fn_16(int x) { return x + 16; }
-inline int shared_fn_17(int x) { return x + 17; }
-inline int shared_fn_18(int x) { return x + 18; }
-inline int shared_fn_19(int x) { return x + 19; }
-inline int shared_fn_20(int x) { return x + 20; }
-inline int shared_fn_21(int x) { return x + 21; }
-inline int shared_fn_22(int x) { return x + 22; }
-inline int shared_fn_23(int x) { return x + 23; }
-inline int shared_fn_24(int x) { return x + 24; }
-inline int shared_fn_25(int x) { return x + 25; }
-inline int shared_fn_26(int x) { return x + 26; }
-inline int shared_fn_27(int x) { return x + 27; }
-inline int shared_fn_28(int x) { return x + 28; }
-inline int shared_fn_29(int x) { return x + 29; }
-inline int shared_fn_30(int x) { return x + 30; }
-inline int shared_fn_31(int x) { return x + 31; }
-inline int shared_fn_32(int x) { return x + 32; }
-inline int shared_fn_33(int x) { return x + 33; }
-inline int shared_fn_34(int x) { return x + 34; }
-inline int shared_fn_35(int x) { return x + 35; }
-inline int shared_fn_36(int x) { return x + 36; }
-inline int shared_fn_37(int x) { return x + 37; }
-inline int shared_fn_38(int x) { return x + 38; }
-inline int shared_fn_39(int x) { return x + 39; }
-inline int shared_fn_40(int x) { return x + 40; }
-inline int shared_fn_41(int x) { return x + 41; }
-inline int shared_fn_42(int x) { return x + 42; }
-inline int shared_fn_43(int x) { return x + 43; }
-inline int shared_fn_44(int x) { return x + 44; }
-inline int shared_fn_45(int x) { return x + 45; }
-inline int shared_fn_46(int x) { return x + 46; }
-inline int shared_fn_47(int x) { return x + 47; }
-inline int shared_fn_48(int x) { return x + 48; }
-inline int shared_fn_49(int x) { return x + 49; }
-inline int shared_fn_50(int x) { return x + 50; }
-inline int shared_fn_51(int x) { return x + 51; }
-inline int shared_fn_52(int x) { return x + 52; }
-inline int shared_fn_53(int x) { return x + 53; }
-inline int shared_fn_54(int x) { return x + 54; }
-inline int shared_fn_55(int x) { return x + 55; }
-inline int shared_fn_56(int x) { return x + 56; }
-inline int shared_fn_57(int x) { return x + 57; }
-inline int shared_fn_58(int x) { return x + 58; }
-inline int shared_fn_59(int x) { return x + 59; }
-inline int shared_fn_60(int x) { return x + 60; }
-inline int shared_fn_61(int x) { return x + 61; }
-inline int shared_fn_62(int x) { return x + 62; }
-inline int shared_fn_63(int x) { return x + 63; }
-inline int shared_fn_64(int x) { return x + 64; }
-inline int shared_fn_65(int x) { return x + 65; }
-inline int shared_fn_66(int x) { return x + 66; }
-inline int shared_fn_67(int x) { return x + 67; }
-inline int shared_fn_68(int x) { return x + 68; }
-inline int shared_fn_69(int x) { return x + 69; }
-inline int shared_fn_70(int x) { return x + 70; }
-inline int shared_fn_71(int x) { return x + 71; }
-inline int shared_fn_72(int x) { return x + 72; }
-inline int shared_fn_73(int x) { return x + 73; }
-inline int shared_fn_74(int x) { return x + 74; }
-inline int shared_fn_75(int x) { return x + 75; }
-inline int shared_fn_76(int x) { return x + 76; }
-inline int shared_fn_77(int x) { return x + 77; }
-inline int shared_fn_78(int x) { return x + 78; }
-inline int shared_fn_79(int x) { return x + 79; }
-inline int shared_fn_80(int x) { return x + 80; }
-inline int shared_fn_81(int x) { return x + 81; }
-inline int shared_fn_82(int x) { return x + 82; }
-inline int shared_fn_83(int x) { return x + 83; }
-inline int shared_fn_84(int x) { return x + 84; }
-inline int shared_fn_85(int x) { return x + 85; }
-inline int shared_fn_86(int x) { return x + 86; }
-inline int shared_fn_87(int x) { return x + 87; }
-inline int shared_fn_88(int x) { return x + 88; }
-inline int shared_fn_89(int x) { return x + 89; }
-inline int shared_fn_90(int x) { return x + 90; }
-inline int shared_fn_91(int x) { return x + 91; }
-inline int shared_fn_92(int x) { return x + 92; }
-inline int shared_fn_93(int x) { return x + 93; }
-inline int shared_fn_94(int x) { return x + 94; }
-inline int shared_fn_95(int x) { return x + 95; }
-inline int shared_fn_96(int x) { return x + 96; }
-inline int shared_fn_97(int x) { return x + 97; }
-inline int shared_fn_98(int x) { return x + 98; }
-inline int shared_fn_99(int x) { return x + 99; }
-inline int shared_fn_100(int x) { return x + 100; }
-inline int shared_fn_101(int x) { return x + 101; }
-inline int shared_fn_102(int x) { return x + 102; }
-inline int shared_fn_103(int x) { return x + 103; }
-inline int shared_fn_104(int x) { return x + 104; }
-inline int shared_fn_105(int x) { return x + 105; }
-inline int shared_fn_106(int x) { return x + 106; }
-inline int shared_fn_107(int x) { return x + 107; }
-inline int shared_fn_108(int x) { return x + 108; }
-inline int shared_fn_109(int x) { return x + 109; }
-inline int shared_fn_110(int x) { return x + 110; }
-inline int shared_fn_111(int x) { return x + 111; }
-inline int shared_fn_112(int x) { return x + 112; }
-inline int shared_fn_113(int x) { return x + 113; }
-inline int shared_fn_114(int x) { return x + 114; }
-inline int shared_fn_115(int x) { return x + 115; }
-inline int shared_fn_116(int x) { return x + 116; }
-inline int shared_fn_117(int x) { return x + 117; }
-inline int shared_fn_118(int x) { return x + 118; }
-inline int shared_fn_119(int x) { return x + 119; }
-inline int shared_fn_120(int x) { return x + 120; }
-inline int shared_fn_121(int x) { return x + 121; }
-inline int shared_fn_122(int x) { return x + 122; }
-inline int shared_fn_123(int x) { return x + 123; }
-inline int shared_fn_124(int x) { return x + 124; }
-inline int shared_fn_125(int x) { return x + 125; }
-inline int shared_fn_126(int x) { return x + 126; }
-inline int shared_fn_127(int x) { return x + 127; }
-inline int shared_fn_128(int x) { return x + 128; }
-inline int shared_fn_129(int x) { return x + 129; }
-inline int shared_fn_130(int x) { return x + 130; }
-inline int shared_fn_131(int x) { return x + 131; }
-inline int shared_fn_132(int x) { return x + 132; }
-inline int shared_fn_133(int x) { return x + 133; }
-inline int shared_fn_134(int x) { return x + 134; }
-inline int shared_fn_135(int x) { return x + 135; }
-inline int shared_fn_136(int x) { return x + 136; }
-inline int shared_fn_137(int x) { return x + 137; }
-inline int shared_fn_138(int x) { return x + 138; }
-inline int shared_fn_139(int x) { return x + 139; }
-inline int shared_fn_140(int x) { return x + 140; }
-inline int shared_fn_141(int x) { return x + 141; }
-inline int shared_fn_142(int x) { return x + 142; }
-inline int shared_fn_143(int x) { return x + 143; }
-inline int shared_fn_144(int x) { return x + 144; }
-inline int shared_fn_145(int x) { return x + 145; }
-inline int shared_fn_146(int x) { return x + 146; }
-inline int shared_fn_147(int x) { return x + 147; }
-inline int shared_fn_148(int x) { return x + 148; }
-inline int shared_fn_149(int x) { return x + 149; }
-inline int shared_fn_150(int x) { return x + 150; }
-inline int shared_fn_151(int x) { return x + 151; }
-inline int shared_fn_152(int x) { return x + 152; }
-inline int shared_fn_153(int x) { return x + 153; }
-inline int shared_fn_154(int x) { return x + 154; }
-inline int shared_fn_155(int x) { return x + 155; }
-inline int shared_fn_156(int x) { return x + 156; }
-inline int shared_fn_157(int x) { return x + 157; }
-inline int shared_fn_158(int x) { return x + 158; }
-inline int shared_fn_159(int x) { return x + 159; }
-inline int shared_fn_160(int x) { return x + 160; }
-inline int shared_fn_161(int x) { return x + 161; }
-inline int shared_fn_162(int x) { return x + 162; }
-inline int shared_fn_163(int x) { return x + 163; }
-inline int shared_fn_164(int x) { return x + 164; }
-inline int shared_fn_165(int x) { return x + 165; }
-inline int shared_fn_166(int x) { return x + 166; }
-inline int shared_fn_167(int x) { return x + 167; }
-inline int shared_fn_168(int x) { return x + 168; }
-inline int shared_fn_169(int x) { return x + 169; }
-inline int shared_fn_170(int x) { return x + 170; }
-inline int shared_fn_171(int x) { return x + 171; }
-inline int shared_fn_172(int x) { return x + 172; }
-inline int shared_fn_173(int x) { return x + 173; }
-inline int shared_fn_174(int x) { return x + 174; }
-inline int shared_fn_175(int x) { return x + 175; }
-inline int shared_fn_176(int x) { return x + 176; }
-inline int shared_fn_177(int x) { return x + 177; }
-inline int shared_fn_178(int x) { return x + 178; }
-inline int shared_fn_179(int x) { return x + 179; }
-inline int shared_fn_180(int x) { return x + 180; }
-inline int shared_fn_181(int x) { return x + 181; }
-inline int shared_fn_182(int x) { return x + 182; }
-inline int shared_fn_183(int x) { return x + 183; }
-inline int shared_fn_184(int x) { return x + 184; }
-inline int shared_fn_185(int x) { return x + 185; }
-inline int shared_fn_186(int x) { return x + 186; }
-inline int shared_fn_187(int x) { return x + 187; }
-inline int shared_fn_188(int x) { return x + 188; }
-inline int shared_fn_189(int x) { return x + 189; }
-inline int shared_fn_190(int x) { return x + 190; }
-inline int shared_fn_191(int x) { return x + 191; }
-inline int shared_fn_192(int x) { return x + 192; }
-inline int shared_fn_193(int x) { return x + 193; }
-inline int shared_fn_194(int x) { return x + 194; }
-inline int shared_fn_195(int x) { return x + 195; }
-inline int shared_fn_196(int x) { return x + 196; }
-inline int shared_fn_197(int x) { return x + 197; }
-inline int shared_fn_198(int x) { return x + 198; }
-inline int shared_fn_199(int x) { return x + 199; }
-inline int shared_fn_200(int x) { return x + 200; }
-inline int shared_fn_201(int x) { return x + 201; }
-inline int shared_fn_202(int x) { return x + 202; }
-inline int shared_fn_203(int x) { return x + 203; }
-inline int shared_fn_204(int x) { return x + 204; }
-inline int shared_fn_205(int x) { return x + 205; }
-inline int shared_fn_206(int x) { return x + 206; }
-inline int shared_fn_207(int x) { return x + 207; }
-inline int shared_fn_208(int x) { return x + 208; }
-inline int shared_fn_209(int x) { return x + 209; }
-inline int shared_fn_210(int x) { return x + 210; }
-inline int shared_fn_211(int x) { return x + 211; }
-inline int shared_fn_212(int x) { return x + 212; }
-inline int shared_fn_213(int x) { return x + 213; }
-inline int shared_fn_214(int x) { return x + 214; }
-inline int shared_fn_215(int x) { return x + 215; }
-inline int shared_fn_216(int x) { return x + 216; }
-inline int shared_fn_217(int x) { return x + 217; }
-inline int shared_fn_218(int x) { return x + 218; }
-inline int shared_fn_219(int x) { return x + 219; }
-inline int shared_fn_220(int x) { return x + 220; }
-inline int shared_fn_221(int x) { return x + 221; }
-inline int shared_fn_222(int x) { return x + 222; }
-inline int shared_fn_223(int x) { return x + 223; }
-inline int shared_fn_224(int x) { return x + 224; }
-inline int shared_fn_225(int x) { return x + 225; }
-inline int shared_fn_226(int x) { return x + 226; }
-inline int shared_fn_227(int x) { return x + 227; }
-inline int shared_fn_228(int x) { return x + 228; }
-inline int shared_fn_229(int x) { return x + 229; }
-inline int shared_fn_230(int x) { return x + 230; }
-inline int shared_fn_231(int x) { return x + 231; }
-inline int shared_fn_232(int x) { return x + 232; }
-inline int shared_fn_233(int x) { return x + 233; }
-inline int shared_fn_234(int x) { return x + 234; }
-inline int shared_fn_235(int x) { return x + 235; }
-inline int shared_fn_236(int x) { return x + 236; }
-inline int shared_fn_237(int x) { return x + 237; }
-inline int shared_fn_238(int x) { return x + 238; }
-inline int shared_fn_239(int x) { return x + 239; }
-inline int shared_fn_240(int x) { return x + 240; }
-inline int shared_fn_241(int x) { return x + 241; }
-inline int shared_fn_242(int x) { return x + 242; }
-inline int shared_fn_243(int x) { return x + 243; }
-inline int shared_fn_244(int x) { return x + 244; }
-inline int shared_fn_245(int x) { return x + 245; }
-inline int shared_fn_246(int x) { return x + 246; }
-inline int shared_fn_247(int x) { return x + 247; }
-inline int shared_fn_248(int x) { return x + 248; }
-inline int shared_fn_249(int x) { return x + 249; }
-inline int shared_fn_250(int x) { return x + 250; }
-inline int shared_fn_251(int x) { return x + 251; }
-inline int shared_fn_252(int x) { return x + 252; }
-inline int shared_fn_253(int x) { return x + 253; }
-inline int shared_fn_254(int x) { return x + 254; }
-inline int shared_fn_255(int x) { return x + 255; }
-inline int shared_fn_256(int x) { return x + 256; }
-inline int shared_fn_257(int x) { return x + 257; }
-inline int shared_fn_258(int x) { return x + 258; }
-inline int shared_fn_259(int x) { return x + 259; }
-inline int shared_fn_260(int x) { return x + 260; }
-inline int shared_fn_261(int x) { return x + 261; }
-inline int shared_fn_262(int x) { return x + 262; }
-inline int shared_fn_263(int x) { return x + 263; }
-inline int shared_fn_264(int x) { return x + 264; }
-inline int shared_fn_265(int x) { return x + 265; }
-inline int shared_fn_266(int x) { return x + 266; }
-inline int shared_fn_267(int x) { return x + 267; }
-inline int shared_fn_268(int x) { return x + 268; }
-inline int shared_fn_269(int x) { return x + 269; }
-inline int shared_fn_270(int x) { return x + 270; }
-inline int shared_fn_271(int x) { return x + 271; }
-inline int shared_fn_272(int x) { return x + 272; }
-inline int shared_fn_273(int x) { return x + 273; }
-inline int shared_fn_274(int x) { return x + 274; }
-inline int shared_fn_275(int x) { return x + 275; }
-inline int shared_fn_276(int x) { return x + 276; }
-inline int shared_fn_277(int x) { return x + 277; }
-inline int shared_fn_278(int x) { return x + 278; }
-inline int shared_fn_279(int x) { return x + 279; }
-inline int shared_fn_280(int x) { return x + 280; }
-inline int shared_fn_281(int x) { return x + 281; }
-inline int shared_fn_282(int x) { return x + 282; }
-inline int shared_fn_283(int x) { return x + 283; }
-inline int shared_fn_284(int x) { return x + 284; }
-inline int shared_fn_285(int x) { return x + 285; }
-inline int shared_fn_286(int x) { return x + 286; }
-inline int shared_fn_287(int x) { return x + 287; }
-inline int shared_fn_288(int x) { return x + 288; }
-inline int shared_fn_289(int x) { return x + 289; }
-inline int shared_fn_290(int x) { return x + 290; }
-inline int shared_fn_291(int x) { return x + 291; }
-inline int shared_fn_292(int x) { return x + 292; }
-inline int shared_fn_293(int x) { return x + 293; }
-inline int shared_fn_294(int x) { return x + 294; }
-inline int shared_fn_295(int x) { return x + 295; }
-inline int shared_fn_296(int x) { return x + 296; }
-inline int shared_fn_297(int x) { return x + 297; }
-inline int shared_fn_298(int x) { return x + 298; }
-inline int shared_fn_299(int x) { return x + 299; }
-inline int shared_fn_300(int x) { return x + 300; }
-inline int shared_fn_301(int x) { return x + 301; }
-inline int shared_fn_302(int x) { return x + 302; }
-inline int shared_fn_303(int x) { return x + 303; }
-inline int shared_fn_304(int x) { return x + 304; }
-inline int shared_fn_305(int x) { return x + 305; }
-inline int shared_fn_306(int x) { return x + 306; }
-inline int shared_fn_307(int x) { return x + 307; }
-inline int shared_fn_308(int x) { return x + 308; }
-inline int shared_fn_309(int x) { return x + 309; }
-inline int shared_fn_310(int x) { return x + 310; }
-inline int shared_fn_311(int x) { return x + 311; }
-inline int shared_fn_312(int x) { return x + 312; }
-inline int shared_fn_313(int x) { return x + 313; }
-inline int shared_fn_314(int x) { return x + 314; }
-inline int shared_fn_315(int x) { return x + 315; }
-inline int shared_fn_316(int x) { return x + 316; }
-inline int shared_fn_317(int x) { return x + 317; }
-inline int shared_fn_318(int x) { return x + 318; }
-inline int shared_fn_319(int x) { return x + 319; }
-inline int shared_fn_320(int x) { return x + 320; }
-inline int shared_fn_321(int x) { return x + 321; }
-inline int shared_fn_322(int x) { return x + 322; }
-inline int shared_fn_323(int x) { return x + 323; }
-inline int shared_fn_324(int x) { return x + 324; }
-inline int shared_fn_325(int x) { return x + 325; }
-inline int shared_fn_326(int x) { return x + 326; }
-inline int shared_fn_327(int x) { return x + 327; }
-inline int shared_fn_328(int x) { return x + 328; }
-inline int shared_fn_329(int x) { return x + 329; }
-inline int shared_fn_330(int x) { return x + 330; }
-inline int shared_fn_331(int x) { return x + 331; }
-inline int shared_fn_332(int x) { return x + 332; }
-inline int shared_fn_333(int x) { return x + 333; }
-inline int shared_fn_334(int x) { return x + 334; }
-inline int shared_fn_335(int x) { return x + 335; }
-inline int shared_fn_336(int x) { return x + 336; }
-inline int shared_fn_337(int x) { return x + 337; }
-inline int shared_fn_338(int x) { return x + 338; }
-inline int shared_fn_339(int x) { return x + 339; }
-inline int shared_fn_340(int x) { return x + 340; }
-inline int shared_fn_341(int x) { return x + 341; }
-inline int shared_fn_342(int x) { return x + 342; }
-inline int shared_fn_343(int x) { return x + 343; }
-inline int shared_fn_344(int x) { return x + 344; }
-inline int shared_fn_345(int x) { return x + 345; }
-inline int shared_fn_346(int x) { return x + 346; }
-inline int shared_fn_347(int x) { return x + 347; }
-inline int shared_fn_348(int x) { return x + 348; }
-inline int shared_fn_349(int x) { return x + 349; }
-inline int shared_fn_350(int x) { return x + 350; }
-inline int shared_fn_351(int x) { return x + 351; }
-inline int shared_fn_352(int x) { return x + 352; }
-inline int shared_fn_353(int x) { return x + 353; }
-inline int shared_fn_354(int x) { return x + 354; }
-inline int shared_fn_355(int x) { return x + 355; }
-inline int shared_fn_356(int x) { return x + 356; }
-inline int shared_fn_357(int x) { return x + 357; }
-inline int shared_fn_358(int x) { return x + 358; }
-inline int shared_fn_359(int x) { return x + 359; }
-inline int shared_fn_360(int x) { return x + 360; }
-inline int shared_fn_361(int x) { return x + 361; }
-inline int shared_fn_362(int x) { return x + 362; }
-inline int shared_fn_363(int x) { return x + 363; }
-inline int shared_fn_364(int x) { return x + 364; }
-inline int shared_fn_365(int x) { return x + 365; }
-inline int shared_fn_366(int x) { return x + 366; }
-inline int shared_fn_367(int x) { return x + 367; }
-inline int shared_fn_368(int x) { return x + 368; }
-inline int shared_fn_369(int x) { return x + 369; }
-inline int shared_fn_370(int x) { return x + 370; }
-inline int shared_fn_371(int x) { return x + 371; }
-inline int shared_fn_372(int x) { return x + 372; }
-inline int shared_fn_373(int x) { return x + 373; }
-inline int shared_fn_374(int x) { return x + 374; }
-inline int shared_fn_375(int x) { return x + 375; }
-inline int shared_fn_376(int x) { return x + 376; }
-inline int shared_fn_377(int x) { return x + 377; }
-inline int shared_fn_378(int x) { return x + 378; }
-inline int shared_fn_379(int x) { return x + 379; }
-inline int shared_fn_380(int x) { return x + 380; }
-inline int shared_fn_381(int x) { return x + 381; }
-inline int shared_fn_382(int x) { return x + 382; }
-inline int shared_fn_383(int x) { return x + 383; }
-inline int shared_fn_384(int x) { return x + 384; }
-inline int shared_fn_385(int x) { return x + 385; }
-inline int shared_fn_386(int x) { return x + 386; }
-inline int shared_fn_387(int x) { return x + 387; }
-inline int shared_fn_388(int x) { return x + 388; }
-inline int shared_fn_389(int x) { return x + 389; }
-inline int shared_fn_390(int x) { return x + 390; }
-inline int shared_fn_391(int x) { return x + 391; }
-inline int shared_fn_392(int x) { return x + 392; }
-inline int shared_fn_393(int x) { return x + 393; }
-inline int shared_fn_394(int x) { return x + 394; }
-inline int shared_fn_395(int x) { return x + 395; }
-inline int shared_fn_396(int x) { return x + 396; }
-inline int shared_fn_397(int x) { return x + 397; }
-inline int shared_fn_398(int x) { return x + 398; }
-inline int shared_fn_399(int x) { return x + 399; }
-#endif
diff --git a/dedup-test/use.cpp b/dedup-test/use.cpp
deleted file mode 100644
index cc00041ff4cc4..0000000000000
--- a/dedup-test/use.cpp
+++ /dev/null
@@ -1,3 +0,0 @@
-#include "a.h"
-#include "b.h"
-int main() { return a_entry(1) + b_entry(2); }
diff --git a/dedup-test/use1.cpp b/dedup-test/use1.cpp
deleted file mode 100644
index 1d4c4250df0c7..0000000000000
--- a/dedup-test/use1.cpp
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "a.h"
-int main() { return a_entry(1); }
diff --git a/dedup-test/use_err.cpp b/dedup-test/use_err.cpp
deleted file mode 100644
index e3cf22fe27df4..0000000000000
--- a/dedup-test/use_err.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-// Stage 2c: prove the redirect resolves correctly.
-// Include BOTH modules so A loads first (registers shared.h as the canonical
-// copy) and B loads second (its shared.h is de-duplicated / redirected into
-// A's copy). Then trigger a diagnostic whose location lives in shared.h.
-// The error must still point at shared.h:<line> with the right function/line,
-// which only holds if the redirected offsets resolve correctly.
-#include "a.h"
-#include "b.h"
-
-int main() {
- // shared_fn_1 is defined in shared.h (present in both A and B); pass a bad
- // argument so overload resolution reports the candidate in shared.h.
- return a_entry(1) + shared_fn_1("oops");
-}
diff --git a/dedup-test/use_err_rev.cpp b/dedup-test/use_err_rev.cpp
deleted file mode 100644
index 817bfb4bc2b1e..0000000000000
--- a/dedup-test/use_err_rev.cpp
+++ /dev/null
@@ -1,10 +0,0 @@
-// Stage 2c: load-order independence. Same as use_err.cpp but with the include
-// order flipped so B is seen first. Whichever module loads first becomes the
-// canonical copy; the other redirects into it. Result should be identical:
-// dedup active + the diagnostic still resolves to shared.h:<line>.
-#include "b.h"
-#include "a.h"
-
-int main() {
- return a_entry(1) + shared_fn_1("oops");
-}
>From 88086bdb3afc929a57f225057f2a7442d1f332cc Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 13:40:40 +0000
Subject: [PATCH 11/32] clean up comments
---
clang/include/clang/Basic/SourceManager.h | 2 +-
clang/include/clang/Serialization/ASTReader.h | 21 +++++++++----------
.../include/clang/Serialization/ModuleFile.h | 2 +-
clang/lib/Basic/SourceManager.cpp | 2 +-
clang/lib/Serialization/ASTReader.cpp | 5 ++++-
5 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 424de78725492..b776d37b6ba1a 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,7 +755,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- // Source location de-duplication. A file included into many modules is
+ // Source location deduplication. A file included into many modules is
// serialized into each of their PCMs. ASTReader keeps the first loaded copy
// and redirects later modules' references to it instead of allocating a
// duplicate SLoc range. These counters record the address space that reuse
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e1020e64fcca8..429d7fc3e653f 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,10 +2399,11 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Identity of an input file, taken from serialized metadata without touching
- /// the filesystem. It uses the stored name, size, and modification time that
- /// Clang's own input-file staleness check relies on. An empty Name marks a
- /// non-file entry (a buffer or expansion), which never deduplicates.
+ /// Identity of an input file, from serialized metadata (no filesystem
+ /// access). Uses the stored name, size, and modification time that Clang's
+ /// own input-file staleness check relies on. ContentHash is not used because
+ /// it is zero unless -fvalidate-ast-input-files-content is set. An empty Name
+ /// marks a non-file entry (a buffer or expansion), which never deduplicates.
struct SLocFileIdentity {
StringRef Name;
off_t Size = 0;
@@ -2421,8 +2422,7 @@ class ASTReader : public ExternalPreprocessorSource,
/// Files already loaded into the loaded SLoc address space, keyed by stored
/// name. Lets a later module reuse an earlier module's copy instead of
- /// allocating a duplicate range. The key is serialized metadata, so no input
- /// file is resolved or stat'd at load time.
+ /// allocating a duplicate range.
llvm::StringMap<PrimaryLoadedFileLoc> PrimaryLoadedFiles;
/// The location of a previously-loaded file matching \p Id (same name, size,
@@ -2437,9 +2437,8 @@ class ASTReader : public ExternalPreprocessorSource,
/// Read \p F's SLoc entry records without materializing them, filling
/// \p Offsets[i] with each entry's local offset and \p Files[i] with its
- /// file identity (empty Name for non-file entries). File identity comes from
- /// serialized metadata only, with no input file resolved on disk. Returns
- /// false on a malformed record.
+ /// file identity (empty Name for non-file entries). Returns false on a
+ /// malformed record.
bool scanLoadedSLocEntries(ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
SmallVectorImpl<SLocFileIdentity> &Files);
@@ -2567,8 +2566,8 @@ class ASTReader : public ExternalPreprocessorSource,
Seg.Delta)) |
MacroBit);
}
- // No segment matched. This shouldn't happen, so fall through to the
- // flat shift below.
+ // The segments cover the whole value range, so one always matches.
+ llvm_unreachable("SLocRemap segments must cover every source location");
}
return Loc.getLocWithOffset(ModuleFile.SLocEntryBaseOffset - 2);
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 76cefc2280936..3d2ef6f2124ac 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -346,7 +346,7 @@ class ModuleFile {
/// AST file.
const uint32_t *SLocEntryOffsets = nullptr;
- // === Source location de-duplication ===
+ // === Source location deduplication ===
/// One segment of the local-to-global source location map. A local raw
/// location L in [LocalBegin, LocalEnd) maps to global location L + Delta.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index fc1d0094a84d8..ab20a0914a856 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2154,7 +2154,7 @@ void SourceManager::PrintStats() const {
<< "B of SLoc address space used.\n";
if (NumDuplicateLoadedFiles)
llvm::errs() << NumDuplicateLoadedFiles
- << " duplicate loaded file SLocEntries de-duplicated ("
+ << " duplicate loaded file SLocEntries deduplicated ("
<< DuplicateLoadedBytes
<< "B of SLoc address space reused).\n";
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 5df770034e7db..3cf5f3d1a5dcc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1901,8 +1901,10 @@ ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
return static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
Seg.Delta);
}
+ // The segments cover the whole value range, so one always matches.
+ llvm_unreachable("SLocRemap segments must cover every source location");
}
- // No remap, or no segment matched. Use the original flat shift.
+ // No remap for this module. Use the flat shift.
return F.SLocEntryBaseOffset + LocalOffset;
}
@@ -4354,6 +4356,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
// file as it goes, so checking against the map there would treat a file
// that appears more than once in this module as a duplicate of its own
// first occurrence.
+ // Entry I's size is the gap to the next offset (block end for the last).
auto entrySize = [&](unsigned I) -> uint64_t {
return (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
};
>From ed105725188726a79994b13fc7cdc945f9ad043f Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 14:19:29 +0000
Subject: [PATCH 12/32] clean up git ignore
---
.gitignore | 3 ---
1 file changed, 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index 1bceeef94e7e4..a4382c9ea7390 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,9 +31,6 @@
# Nested build directory
/build*
-/ Test directory
-/dedup-test
-
#==============================================================================#
# Explicit files to ignore (only matches one).
#==============================================================================#
>From bfb8c50239c99ec9b24aa53e8140556ff02c7367 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 14:35:13 +0000
Subject: [PATCH 13/32] clean up comments
---
clang/include/clang/Serialization/ASTReader.h | 8 +++-----
clang/lib/Serialization/ASTReader.cpp | 6 ++----
2 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 429d7fc3e653f..ac54818e850b7 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,11 +2399,9 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Identity of an input file, from serialized metadata (no filesystem
- /// access). Uses the stored name, size, and modification time that Clang's
- /// own input-file staleness check relies on. ContentHash is not used because
- /// it is zero unless -fvalidate-ast-input-files-content is set. An empty Name
- /// marks a non-file entry (a buffer or expansion), which never deduplicates.
+ /// Identity of an input file, built from serialized metadata so we touch no
+ /// files at load. An empty Name means a non-file entry like a buffer or
+ /// expansion, and never deduplicates.
struct SLocFileIdentity {
StringRef Name;
off_t Size = 0;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 3cf5f3d1a5dcc..e51a56e3de0da 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1868,10 +1868,8 @@ bool ASTReader::scanLoadedSLocEntries(
}
Offsets[I] = (uint32_t)Record[0];
if (Code.get() == SM_SLOC_FILE_ENTRY) {
- // Identify the file from serialized metadata only. Resolving it on disk
- // here (getInputFile) would stat and open every input file of every
- // module at load, which is prohibitive. The stored name, size, and time
- // are the identity Clang's own staleness check already uses.
+ // File identity comes from serialized metadata, so no input file is
+ // touched on disk at load.
InputFileInfo IFI = getInputFileInfo(F, Record[4]);
if (IFI.isValid())
Files[I] = SLocFileIdentity{IFI.UnresolvedImportedFilename,
>From 8445c987e53899bf1efc59bec461ad96d1c481dc Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 15 Jul 2026 11:43:19 -0400
Subject: [PATCH 14/32] run clang format
---
clang/include/clang/Serialization/ASTReader.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index ac54818e850b7..a2d673beb90c5 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2400,7 +2400,7 @@ class ASTReader : public ExternalPreprocessorSource,
unsigned Index);
/// Identity of an input file, built from serialized metadata so we touch no
- /// files at load. An empty Name means a non-file entry like a buffer or
+ /// files at load. An empty Name means a non-file entry like a buffer or
/// expansion, and never deduplicates.
struct SLocFileIdentity {
StringRef Name;
>From 1f075e3292f5d7751935a39947473480363feac1 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 22 Jul 2026 09:54:53 -0400
Subject: [PATCH 15/32] refactor scanning details
---
clang/include/clang/Serialization/ASTReader.h | 17 ++
clang/lib/Serialization/ASTReader.cpp | 185 ++++++++++--------
2 files changed, 118 insertions(+), 84 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index a2d673beb90c5..8be83d9e4ec4e 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2440,6 +2440,23 @@ class ASTReader : public ExternalPreprocessorSource,
bool scanLoadedSLocEntries(ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
SmallVectorImpl<SLocFileIdentity> &Files);
+ /// Mark the scanned entries that duplicate an already-loaded file. Returns
+ /// the number of duplicates and the space they would otherwise occupy.
+ std::pair<unsigned, SourceLocation::UIntTy>
+ classifyDuplicateSLocEntries(ArrayRef<uint32_t> Offsets,
+ ArrayRef<SLocFileIdentity> Files,
+ SourceLocation::UIntTy SLocSpaceSize,
+ SmallVectorImpl<bool> &IsDup);
+
+ /// Build \p F's local-to-global SLoc remapping and register its files. Run
+ /// after AllocateLoadedSLocEntries has assigned \p F's base ID and offset.
+ void buildLoadedSLocRemapping(ModuleFile &F, ArrayRef<uint32_t> Offsets,
+ ArrayRef<SLocFileIdentity> Files,
+ ArrayRef<bool> IsDup,
+ SourceLocation::UIntTy SLocSpaceSize,
+ unsigned NumDupEntries,
+ unsigned ReducedNumEntries);
+
/// Map a local SLoc entry offset (as stored in the entry record) to its
/// global start offset. This is (SLocEntryBaseOffset + LocalOffset) unless a
/// file in this module was reused from an earlier one.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index e51a56e3de0da..4f9e34867a041 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1879,6 +1879,96 @@ bool ASTReader::scanLoadedSLocEntries(
return true;
}
+/// Bytes that scanned entry \p I occupies on the loaded number line.
+static SourceLocation::UIntTy
+slocEntrySize(ArrayRef<uint32_t> Offsets, unsigned I,
+ SourceLocation::UIntTy SLocSpaceSize) {
+ return (I + 1 < Offsets.size() ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
+}
+
+std::pair<unsigned, SourceLocation::UIntTy>
+ASTReader::classifyDuplicateSLocEntries(ArrayRef<uint32_t> Offsets,
+ ArrayRef<SLocFileIdentity> Files,
+ SourceLocation::UIntTy SLocSpaceSize,
+ SmallVectorImpl<bool> &IsDup) {
+ unsigned N = Offsets.size();
+ IsDup.assign(N, false);
+
+ // Decide duplicates here. buildLoadedSLocRemapping registers this module's
+ // own files as it goes, so a file seen twice in one module must not be
+ // treated as a duplicate of its own first copy.
+ unsigned NumDup = 0;
+ SourceLocation::UIntTy DupBytes = 0;
+ for (unsigned I = 0; I != N; ++I) {
+ if (Files[I].Name.empty() || !getPrimaryLoadedFile(Files[I]))
+ continue;
+ SourceLocation::UIntTy Size = slocEntrySize(Offsets, I, SLocSpaceSize);
+ IsDup[I] = true;
+ SourceMgr.noteDuplicateLoadedFile(Size);
+ DupBytes += Size;
+ ++NumDup;
+ }
+ return {NumDup, DupBytes};
+}
+
+void ASTReader::buildLoadedSLocRemapping(ModuleFile &F,
+ ArrayRef<uint32_t> Offsets,
+ ArrayRef<SLocFileIdentity> Files,
+ ArrayRef<bool> IsDup,
+ SourceLocation::UIntTy SLocSpaceSize,
+ unsigned NumDupEntries,
+ unsigned ReducedNumEntries) {
+ unsigned N = Files.size();
+
+ // With no duplicates a single identity segment reproduces the flat shift and
+ // the local-to-global ID mapping stays linear. Record each file so a later
+ // module can reuse it.
+ if (NumDupEntries == 0) {
+ F.SLocRemap.push_back({0, ~SourceLocation::UIntTy(0),
+ static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
+ for (unsigned I = 0; I != N; ++I)
+ if (!Files[I].Name.empty())
+ registerPrimaryLoadedFile(Files[I], F.SLocEntryBaseOffset + Offsets[I],
+ F.SLocEntryBaseID + (int)I);
+ return;
+ }
+
+ F.LocalToGlobalID.assign(N, 0);
+ F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
+ SourceLocation::UIntTy DupBefore = 0;
+ unsigned KeptCount = 0;
+ for (unsigned I = 0; I != N; ++I) {
+ SourceLocation::UIntTy LowStart = Offsets[I] + 2;
+ SourceLocation::UIntTy LowEnd =
+ LowStart + slocEntrySize(Offsets, I, SLocSpaceSize);
+ if (IsDup[I]) {
+ // Redirect into the module that first loaded the file and reserve nothing.
+ const PrimaryLoadedFileLoc *Primary = getPrimaryLoadedFile(Files[I]);
+ F.SLocRemap.push_back({LowStart, LowEnd,
+ static_cast<int64_t>(Primary->Offset) -
+ static_cast<int64_t>(LowStart)});
+ F.LocalToGlobalID[I] = Primary->ID;
+ DupBefore += LowEnd - LowStart;
+ } else {
+ // Keep the entry, shifted down past the duplicates skipped before it.
+ int GlobalID = F.SLocEntryBaseID + (int)KeptCount++;
+ SourceLocation::UIntTy GlobalStart =
+ F.SLocEntryBaseOffset + Offsets[I] - DupBefore;
+ F.SLocRemap.push_back({LowStart, LowEnd,
+ static_cast<int64_t>(GlobalStart) -
+ static_cast<int64_t>(LowStart)});
+ F.LocalToGlobalID[I] = GlobalID;
+ F.KeptSLocLocalIndex.push_back(I);
+ if (!Files[I].Name.empty())
+ registerPrimaryLoadedFile(Files[I], GlobalStart, GlobalID);
+ }
+ }
+ // Cover the whole value range so every location maps to a segment.
+ F.SLocRemap.front().LocalBegin = 0;
+ F.SLocRemap.back().LocalEnd = ~SourceLocation::UIntTy(0);
+ assert(KeptCount == ReducedNumEntries && "kept count mismatch");
+}
+
SourceLocation::UIntTy
ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
// The entry's local raw start location is LocalOffset + 2 (offsets 0 and 1
@@ -4341,40 +4431,21 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
F.LocalNumSLocEntries = Record[0];
SourceLocation::UIntTy SLocSpaceSize = Record[1];
F.SLocEntryOffsetsBase = Record[2] + F.SourceManagerBlockStartOffset;
- unsigned N = F.LocalNumSLocEntries;
- // Scan this module's SLoc entries before allocating, so files already
- // loaded by an earlier module can be recognized and reserve no space
- // here.
+ // Scan the entries and reserve no space for files an earlier module
+ // already loaded.
SmallVector<uint32_t, 64> Offsets;
SmallVector<SLocFileIdentity, 64> Files;
- bool Scanned = scanLoadedSLocEntries(F, Offsets, Files);
-
- // Decide duplicates here, up front. The loop below registers each kept
- // file as it goes, so checking against the map there would treat a file
- // that appears more than once in this module as a duplicate of its own
- // first occurrence.
- // Entry I's size is the gap to the next offset (block end for the last).
- auto entrySize = [&](unsigned I) -> uint64_t {
- return (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
- };
- SmallVector<bool, 64> IsDup(N, false);
+ SmallVector<bool, 64> IsDup;
unsigned NumDupEntries = 0;
- uint64_t DupBytes = 0;
- if (Scanned)
- for (unsigned I = 0; I != N; ++I)
- if (!Files[I].Name.empty() && getPrimaryLoadedFile(Files[I])) {
- IsDup[I] = true;
- uint64_t Size = entrySize(I);
- SourceMgr.noteDuplicateLoadedFile(Size);
- DupBytes += Size;
- ++NumDupEntries;
- }
- unsigned ReducedNumEntries = N - NumDupEntries;
+ SourceLocation::UIntTy DupBytes = 0;
+ if (scanLoadedSLocEntries(F, Offsets, Files))
+ std::tie(NumDupEntries, DupBytes) =
+ classifyDuplicateSLocEntries(Offsets, Files, SLocSpaceSize, IsDup);
+
+ unsigned ReducedNumEntries = F.LocalNumSLocEntries - NumDupEntries;
SourceLocation::UIntTy ReducedSize = SLocSpaceSize - DupBytes;
- // Reserve the reduced amount (equal to the full amount when no file is
- // reused).
std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
SourceMgr.AllocateLoadedSLocEntries(ReducedNumEntries, ReducedSize);
if (!F.SLocEntryBaseID) {
@@ -4384,62 +4455,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
"ran out of source locations");
}
- if (NumDupEntries == 0) {
- // Nothing reused. A single segment equal to the flat shift, plus a
- // linear ID mapping. Record each file so later modules can reuse it.
- F.SLocRemap.push_back(
- {/*LocalBegin=*/0, /*LocalEnd=*/~SourceLocation::UIntTy(0),
- /*Delta=*/static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
- if (Scanned)
- for (unsigned I = 0; I != N; ++I)
- if (!Files[I].Name.empty())
- registerPrimaryLoadedFile(Files[I],
- F.SLocEntryBaseOffset + Offsets[I],
- F.SLocEntryBaseID + (int)I);
- } else {
- // Build the offset segments and the ID map. A reused file gets no slot
- // and no address space, and its references point at the earlier copy.
- F.LocalToGlobalID.assign(N, 0);
- F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
- uint64_t DupBefore = 0;
- unsigned KeptCount = 0;
- for (unsigned I = 0; I != N; ++I) {
- SourceLocation::UIntTy LowStart = Offsets[I] + 2;
- SourceLocation::UIntTy LowEnd =
- (I + 1 < N ? Offsets[I + 1] : SLocSpaceSize) + 2;
- const SLocFileIdentity &Id = Files[I];
- if (IsDup[I]) {
- // Redirect this file's locations into the module that first loaded
- // it and reserve nothing here.
- const PrimaryLoadedFileLoc *Primary = getPrimaryLoadedFile(Id);
- F.SLocRemap.push_back({LowStart, LowEnd,
- static_cast<int64_t>(Primary->Offset) -
- static_cast<int64_t>(LowStart)});
- F.LocalToGlobalID[I] = Primary->ID;
- DupBefore += LowEnd - LowStart;
- } else {
- // Kept entry. It lands in this module's block, shifted down to
- // close gaps left by skipped duplicates before it.
- int GlobalID = F.SLocEntryBaseID + (int)KeptCount;
- SourceLocation::UIntTy GlobalStart =
- static_cast<SourceLocation::UIntTy>(F.SLocEntryBaseOffset +
- Offsets[I] - DupBefore);
- F.SLocRemap.push_back({LowStart, LowEnd,
- static_cast<int64_t>(GlobalStart) -
- static_cast<int64_t>(LowStart)});
- F.LocalToGlobalID[I] = GlobalID;
- F.KeptSLocLocalIndex.push_back(I);
- if (!Id.Name.empty())
- registerPrimaryLoadedFile(Id, GlobalStart, GlobalID);
- ++KeptCount;
- }
- }
- // Extend the first/last segments to cover the whole value range so
- // every translated location matches a segment.
- F.SLocRemap.front().LocalBegin = 0;
- F.SLocRemap.back().LocalEnd = ~SourceLocation::UIntTy(0);
- assert(KeptCount == ReducedNumEntries && "kept count mismatch");
- }
+ buildLoadedSLocRemapping(F, Offsets, Files, IsDup, SLocSpaceSize,
+ NumDupEntries, ReducedNumEntries);
// Make our entry in the range map. BaseID is negative and growing, so
// we invert it. Because we invert it, though, we need the other end of
>From a90403e4c6b35d35316d02e0f3cb94d82f80655a Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Wed, 22 Jul 2026 15:32:25 -0400
Subject: [PATCH 16/32] used resolved file name identity instead of just raw
name
---
clang/include/clang/Serialization/ASTReader.h | 7 ++++---
clang/lib/Serialization/ASTReader.cpp | 12 ++++++++----
2 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 8be83d9e4ec4e..5012a6ec6d8cc 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2400,10 +2400,11 @@ class ASTReader : public ExternalPreprocessorSource,
unsigned Index);
/// Identity of an input file, built from serialized metadata so we touch no
- /// files at load. An empty Name means a non-file entry like a buffer or
- /// expansion, and never deduplicates.
+ /// files at load. Name is the resolved path, so two files with the same
+ /// spelling in different directories stay distinct. An empty Name means a
+ /// non-file entry like a buffer or expansion, and never deduplicates.
struct SLocFileIdentity {
- StringRef Name;
+ std::string Name;
off_t Size = 0;
time_t Time = 0;
};
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 4f9e34867a041..42c35ba5b7ce3 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1844,6 +1844,8 @@ bool ASTReader::scanLoadedSLocEntries(
Offsets.assign(N, 0);
Files.assign(N, SLocFileIdentity{});
+ SmallString<0> PathBuf;
+ PathBuf.reserve(256);
BitstreamCursor &Cursor = F.SLocEntryCursor;
SavedStreamPosition SavedPosition(Cursor);
for (unsigned I = 0; I != N; ++I) {
@@ -1868,12 +1870,14 @@ bool ASTReader::scanLoadedSLocEntries(
}
Offsets[I] = (uint32_t)Record[0];
if (Code.get() == SM_SLOC_FILE_ENTRY) {
- // File identity comes from serialized metadata, so no input file is
- // touched on disk at load.
+ // Identity comes from serialized metadata, so no input file is touched on
+ // disk. Resolve the stored name to a path so two same-named files in
+ // different directories are not treated as one; this is string work only.
InputFileInfo IFI = getInputFileInfo(F, Record[4]);
if (IFI.isValid())
- Files[I] = SLocFileIdentity{IFI.UnresolvedImportedFilename,
- IFI.StoredSize, IFI.StoredTime};
+ Files[I] = SLocFileIdentity{
+ ResolveImportedPathAndAllocate(PathBuf, IFI.UnresolvedImportedFilename, F),
+ IFI.StoredSize, IFI.StoredTime};
}
}
return true;
>From ead40d17f964791b14994f9db1e0bffd142814f3 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Fri, 24 Jul 2026 15:21:36 -0400
Subject: [PATCH 17/32] drop time from file key
---
clang/include/clang/Serialization/ASTReader.h | 2 --
clang/lib/Serialization/ASTReader.cpp | 10 +++++-----
2 files changed, 5 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 5012a6ec6d8cc..664a30866d9e4 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2406,7 +2406,6 @@ class ASTReader : public ExternalPreprocessorSource,
struct SLocFileIdentity {
std::string Name;
off_t Size = 0;
- time_t Time = 0;
};
/// Where a file's SLoc entry first landed in the loaded address space, kept
@@ -2416,7 +2415,6 @@ class ASTReader : public ExternalPreprocessorSource,
SourceLocation::UIntTy Offset = 0; ///< global start offset
int ID = 0; ///< global SLoc entry ID
off_t Size = 0;
- time_t Time = 0;
};
/// Files already loaded into the loaded SLoc address space, keyed by stored
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 42c35ba5b7ce3..b91b7a152911d 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1822,9 +1822,9 @@ ASTReader::getPrimaryLoadedFile(const SLocFileIdentity &Id) const {
if (It == PrimaryLoadedFiles.end())
return nullptr;
const PrimaryLoadedFileLoc &Primary = It->second;
- // A matching name but different size or time is a different file, e.g. two
- // modules built against different versions of the same path. Don't merge.
- if (Primary.Size != Id.Size || Primary.Time != Id.Time)
+ // A matching name but different size is a different file, e.g. two modules
+ // built against different versions of the same path. Don't merge.
+ if (Primary.Size != Id.Size)
return nullptr;
return &Primary;
}
@@ -1834,7 +1834,7 @@ void ASTReader::registerPrimaryLoadedFile(const SLocFileIdentity &Id,
int ID) {
assert(!Id.Name.empty() && "registering a non-file entry");
PrimaryLoadedFiles.try_emplace(
- Id.Name, PrimaryLoadedFileLoc{Offset, ID, Id.Size, Id.Time});
+ Id.Name, PrimaryLoadedFileLoc{Offset, ID, Id.Size});
}
bool ASTReader::scanLoadedSLocEntries(
@@ -1877,7 +1877,7 @@ bool ASTReader::scanLoadedSLocEntries(
if (IFI.isValid())
Files[I] = SLocFileIdentity{
ResolveImportedPathAndAllocate(PathBuf, IFI.UnresolvedImportedFilename, F),
- IFI.StoredSize, IFI.StoredTime};
+ IFI.StoredSize};
}
}
return true;
>From 26a1cf01e886606d2ffd6129ca6e481bdf2f4432 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Fri, 24 Jul 2026 15:24:26 -0400
Subject: [PATCH 18/32] format style
---
clang/include/clang/Serialization/ASTReader.h | 8 ++---
clang/lib/Serialization/ASTReader.cpp | 31 +++++++++----------
2 files changed, 18 insertions(+), 21 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 664a30866d9e4..fc4f475fc85f7 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2441,11 +2441,9 @@ class ASTReader : public ExternalPreprocessorSource,
/// Mark the scanned entries that duplicate an already-loaded file. Returns
/// the number of duplicates and the space they would otherwise occupy.
- std::pair<unsigned, SourceLocation::UIntTy>
- classifyDuplicateSLocEntries(ArrayRef<uint32_t> Offsets,
- ArrayRef<SLocFileIdentity> Files,
- SourceLocation::UIntTy SLocSpaceSize,
- SmallVectorImpl<bool> &IsDup);
+ std::pair<unsigned, SourceLocation::UIntTy> classifyDuplicateSLocEntries(
+ ArrayRef<uint32_t> Offsets, ArrayRef<SLocFileIdentity> Files,
+ SourceLocation::UIntTy SLocSpaceSize, SmallVectorImpl<bool> &IsDup);
/// Build \p F's local-to-global SLoc remapping and register its files. Run
/// after AllocateLoadedSLocEntries has assigned \p F's base ID and offset.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index b91b7a152911d..6096182c27fdc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1833,8 +1833,8 @@ void ASTReader::registerPrimaryLoadedFile(const SLocFileIdentity &Id,
SourceLocation::UIntTy Offset,
int ID) {
assert(!Id.Name.empty() && "registering a non-file entry");
- PrimaryLoadedFiles.try_emplace(
- Id.Name, PrimaryLoadedFileLoc{Offset, ID, Id.Size});
+ PrimaryLoadedFiles.try_emplace(Id.Name,
+ PrimaryLoadedFileLoc{Offset, ID, Id.Size});
}
bool ASTReader::scanLoadedSLocEntries(
@@ -1875,9 +1875,10 @@ bool ASTReader::scanLoadedSLocEntries(
// different directories are not treated as one; this is string work only.
InputFileInfo IFI = getInputFileInfo(F, Record[4]);
if (IFI.isValid())
- Files[I] = SLocFileIdentity{
- ResolveImportedPathAndAllocate(PathBuf, IFI.UnresolvedImportedFilename, F),
- IFI.StoredSize};
+ Files[I] =
+ SLocFileIdentity{ResolveImportedPathAndAllocate(
+ PathBuf, IFI.UnresolvedImportedFilename, F),
+ IFI.StoredSize};
}
}
return true;
@@ -1915,13 +1916,10 @@ ASTReader::classifyDuplicateSLocEntries(ArrayRef<uint32_t> Offsets,
return {NumDup, DupBytes};
}
-void ASTReader::buildLoadedSLocRemapping(ModuleFile &F,
- ArrayRef<uint32_t> Offsets,
- ArrayRef<SLocFileIdentity> Files,
- ArrayRef<bool> IsDup,
- SourceLocation::UIntTy SLocSpaceSize,
- unsigned NumDupEntries,
- unsigned ReducedNumEntries) {
+void ASTReader::buildLoadedSLocRemapping(
+ ModuleFile &F, ArrayRef<uint32_t> Offsets, ArrayRef<SLocFileIdentity> Files,
+ ArrayRef<bool> IsDup, SourceLocation::UIntTy SLocSpaceSize,
+ unsigned NumDupEntries, unsigned ReducedNumEntries) {
unsigned N = Files.size();
// With no duplicates a single identity segment reproduces the flat shift and
@@ -1946,7 +1944,8 @@ void ASTReader::buildLoadedSLocRemapping(ModuleFile &F,
SourceLocation::UIntTy LowEnd =
LowStart + slocEntrySize(Offsets, I, SLocSpaceSize);
if (IsDup[I]) {
- // Redirect into the module that first loaded the file and reserve nothing.
+ // Redirect into the module that first loaded the file and reserve
+ // nothing.
const PrimaryLoadedFileLoc *Primary = getPrimaryLoadedFile(Files[I]);
F.SLocRemap.push_back({LowStart, LowEnd,
static_cast<int64_t>(Primary->Offset) -
@@ -1958,9 +1957,9 @@ void ASTReader::buildLoadedSLocRemapping(ModuleFile &F,
int GlobalID = F.SLocEntryBaseID + (int)KeptCount++;
SourceLocation::UIntTy GlobalStart =
F.SLocEntryBaseOffset + Offsets[I] - DupBefore;
- F.SLocRemap.push_back({LowStart, LowEnd,
- static_cast<int64_t>(GlobalStart) -
- static_cast<int64_t>(LowStart)});
+ F.SLocRemap.push_back(
+ {LowStart, LowEnd,
+ static_cast<int64_t>(GlobalStart) - static_cast<int64_t>(LowStart)});
F.LocalToGlobalID[I] = GlobalID;
F.KeptSLocLocalIndex.push_back(I);
if (!Files[I].Name.empty())
>From 32e36f88fca3b8b29fb9ba68b9869289007c74c8 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Tue, 1 Sep 2026 12:30:34 +0200
Subject: [PATCH 19/32] add inverse remapping for ASTWriter
---
clang/include/clang/Serialization/ASTReader.h | 5 +++
.../include/clang/Serialization/ModuleFile.h | 5 +++
clang/lib/Serialization/ASTReader.cpp | 35 +++++++++++++++++--
clang/lib/Serialization/ASTWriter.cpp | 25 +++++++++++--
4 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index fc4f475fc85f7..cd755ab1f928e 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2460,6 +2460,11 @@ class ASTReader : public ExternalPreprocessorSource,
SourceLocation::UIntTy remapSLocEntryOffset(ModuleFile &F,
uint32_t LocalOffset) const;
+ /// The delta that produced global offset \p G in \p F, to be subtracted to
+ /// recover \p F's local location. \p G must lie in \p F's own range, as it
+ /// does when \p F was found through GlobalSLocOffsetMap.
+ int64_t getSLocInverseDelta(ModuleFile &F, SourceLocation::UIntTy G) const;
+
/// Retrieve the module import location and module name for the
/// given source manager entry ID.
std::pair<SourceLocation, StringRef> getModuleImportLoc(int ID) override;
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 3d2ef6f2124ac..a74d3a9542bae 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -364,6 +364,11 @@ class ModuleFile {
/// which case the flat shift is used directly.
llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
+ /// The inverse of SLocRemap, sorted by global start, where LocalBegin and
+ /// LocalEnd hold global bounds and a global location G maps to G - Delta.
+ /// Holds only the entries this module kept, which tile its own range.
+ llvm::SmallVector<SLocRemapSegment, 4> SLocRemapGlobal;
+
/// Maps a local SLoc entry index to its global SLoc entry ID. A kept entry
/// maps to its own ID. A file reused from an earlier module maps to that
/// module's copy. Empty when no file was reused (the global ID is then
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 6096182c27fdc..84303eb520a82 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1928,6 +1928,10 @@ void ASTReader::buildLoadedSLocRemapping(
if (NumDupEntries == 0) {
F.SLocRemap.push_back({0, ~SourceLocation::UIntTy(0),
static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
+ // The inverse is the same flat shift over this module's whole range.
+ F.SLocRemapGlobal.push_back(
+ {F.SLocEntryBaseOffset, F.SLocEntryBaseOffset + SLocSpaceSize,
+ static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
for (unsigned I = 0; I != N; ++I)
if (!Files[I].Name.empty())
registerPrimaryLoadedFile(Files[I], F.SLocEntryBaseOffset + Offsets[I],
@@ -1957,9 +1961,13 @@ void ASTReader::buildLoadedSLocRemapping(
int GlobalID = F.SLocEntryBaseID + (int)KeptCount++;
SourceLocation::UIntTy GlobalStart =
F.SLocEntryBaseOffset + Offsets[I] - DupBefore;
- F.SLocRemap.push_back(
- {LowStart, LowEnd,
- static_cast<int64_t>(GlobalStart) - static_cast<int64_t>(LowStart)});
+ int64_t Delta =
+ static_cast<int64_t>(GlobalStart) - static_cast<int64_t>(LowStart);
+ F.SLocRemap.push_back({LowStart, LowEnd, Delta});
+ // Kept entries are visited in increasing global order, so appending
+ // keeps SLocRemapGlobal sorted.
+ F.SLocRemapGlobal.push_back(
+ {GlobalStart, GlobalStart + (LowEnd - LowStart), Delta});
F.LocalToGlobalID[I] = GlobalID;
F.KeptSLocLocalIndex.push_back(I);
if (!Files[I].Name.empty())
@@ -1999,6 +2007,27 @@ ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
return F.SLocEntryBaseOffset + LocalOffset;
}
+int64_t ASTReader::getSLocInverseDelta(ModuleFile &F,
+ SourceLocation::UIntTy G) const {
+ // The list is sorted by global start and its segments tile this module's
+ // range, so the covering segment is the last one whose start is <= G.
+ if (!F.SLocRemapGlobal.empty()) {
+ auto It = llvm::upper_bound(
+ F.SLocRemapGlobal, G,
+ [](SourceLocation::UIntTy V,
+ const serialization::ModuleFile::SLocRemapSegment &S) {
+ return V < S.LocalBegin;
+ });
+ if (It != F.SLocRemapGlobal.begin()) {
+ const auto &Seg = *std::prev(It);
+ if (G < Seg.LocalEnd)
+ return Seg.Delta;
+ }
+ }
+ // No inverse for this module. Use the flat shift.
+ return static_cast<int64_t>(F.SLocEntryBaseOffset) - 2;
+}
+
llvm::Expected<SourceLocation::UIntTy>
ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
BitstreamCursor &Cursor = F->SLocEntryCursor;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 9d362b1eed920..c303e2e91eb68 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6836,6 +6836,7 @@ SourceLocationEncoding::RawLocEncoding
ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
SourceLocation::UIntTy BaseOffset = 0;
unsigned ModuleFileIndex = 0;
+ [[maybe_unused]] ModuleFile *OwningModuleFile = nullptr;
// See SourceLocationEncoding.h for the encoding details.
if (PP->getSourceManager().isLoadedSourceLocation(Loc) && Loc.isValid()) {
@@ -6845,14 +6846,34 @@ ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
assert(SLocMapI != getChain()->GlobalSLocOffsetMap.end() &&
"Corrupted global sloc offset map");
ModuleFile *F = SLocMapI->second;
- BaseOffset = F->SLocEntryBaseOffset - 2;
+ OwningModuleFile = F;
+ // The reader's local-to-global map is piecewise once a file has been
+ // reused from an earlier module, so SLocEntryBaseOffset - 2 does not invert
+ // it. Recover the delta that produced this location.
+ BaseOffset = static_cast<SourceLocation::UIntTy>(
+ getChain()->getSLocInverseDelta(*F, Loc.getOffset()));
// 0 means the location is not loaded. So we need to add 1 to the index to
// make it clear.
ModuleFileIndex = F->Index + 1;
assert(&getChain()->getModuleManager()[F->Index] == F);
}
- return SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
+ SourceLocationEncoding::RawLocEncoding Encoded =
+ SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
+
+#ifndef NDEBUG
+ // The reader must recover Loc from what we wrote, which holds only if
+ // BaseOffset inverts its local-to-global map. A stale inverse otherwise goes
+ // unnoticed until the location resolves into an unrelated file.
+ if (OwningModuleFile) {
+ SourceLocation Decoded = SourceLocationEncoding::decode(Encoded).first;
+ assert(getChain()->TranslateSourceLocation(*OwningModuleFile, Decoded) ==
+ Loc &&
+ "loaded source location did not round-trip");
+ }
+#endif
+
+ return Encoded;
}
void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record) {
>From ba5720b280e06e3908d4ff011b39ef069a85dabb Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Tue, 1 Sep 2026 12:38:17 +0200
Subject: [PATCH 20/32] clean up comments
---
clang/include/clang/Basic/SourceManager.h | 6 +-----
clang/include/clang/Serialization/ASTReader.h | 19 ++++++-------------
clang/lib/Serialization/ASTReader.cpp | 8 +++-----
clang/lib/Serialization/ASTWriter.cpp | 5 ++---
4 files changed, 12 insertions(+), 26 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index b776d37b6ba1a..1c085a0097364 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,11 +755,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- // Source location deduplication. A file included into many modules is
- // serialized into each of their PCMs. ASTReader keeps the first loaded copy
- // and redirects later modules' references to it instead of allocating a
- // duplicate SLoc range. These counters record the address space that reuse
- // saved, for -print-stats.
+ // Source location deduplication statistics.
private:
/// Number of loaded file entries reused from an earlier module.
unsigned NumDuplicateLoadedFiles = 0;
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index cd755ab1f928e..8b8427c1190f1 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,18 +2399,14 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Identity of an input file, built from serialized metadata so we touch no
- /// files at load. Name is the resolved path, so two files with the same
- /// spelling in different directories stay distinct. An empty Name means a
- /// non-file entry like a buffer or expansion, and never deduplicates.
+ /// Identity of an input file used for source location deduplication.
+ /// Name is the resolved path; an empty Name denotes a non-file entry.
struct SLocFileIdentity {
std::string Name;
off_t Size = 0;
};
- /// Where a file's SLoc entry first landed in the loaded address space, kept
- /// with the identity fields used to confirm a later module's same-named entry
- /// really is the same file.
+ /// Location and identity of the first loaded copy of a file.
struct PrimaryLoadedFileLoc {
SourceLocation::UIntTy Offset = 0; ///< global start offset
int ID = 0; ///< global SLoc entry ID
@@ -2559,16 +2555,13 @@ class ASTReader : public ExternalPreprocessorSource,
// translated or refactor the code to make it clear that
// TranslateSourceLocation won't be called with translated source location.
- // When a file in this module was reused from an earlier one, the map is
- // piecewise rather than a single shift, so find the segment covering this
- // location. A macro location keeps its offset in the low bits with the high
- // bit set, so match on the offset part and re-apply the bit to the result.
+ // The remap is piecewise when files are deduplicated. SourceLocation's
+ // MacroID bit is not part of the offset used to select a segment.
if (!ModuleFile.SLocRemap.empty()) {
SourceLocation::UIntTy Raw = Loc.getRawEncoding();
SourceLocation::UIntTy MacroBit = Raw & SourceLocation::MacroIDBit;
SourceLocation::UIntTy Low = Raw & ~SourceLocation::MacroIDBit;
- // The list is sorted by LocalBegin and its segments are contiguous, so
- // the covering segment is the last one whose LocalBegin is <= Low.
+ // Find the segment containing Low.
auto It = llvm::upper_bound(
ModuleFile.SLocRemap, Low,
[](SourceLocation::UIntTy V,
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 84303eb520a82..7e5a7bf0c04ff 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1922,9 +1922,7 @@ void ASTReader::buildLoadedSLocRemapping(
unsigned NumDupEntries, unsigned ReducedNumEntries) {
unsigned N = Files.size();
- // With no duplicates a single identity segment reproduces the flat shift and
- // the local-to-global ID mapping stays linear. Record each file so a later
- // module can reuse it.
+ // Without duplicates, the remap is the original flat shift.
if (NumDupEntries == 0) {
F.SLocRemap.push_back({0, ~SourceLocation::UIntTy(0),
static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
@@ -1932,6 +1930,7 @@ void ASTReader::buildLoadedSLocRemapping(
F.SLocRemapGlobal.push_back(
{F.SLocEntryBaseOffset, F.SLocEntryBaseOffset + SLocSpaceSize,
static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
+ // Record each file for deduplication by later modules.
for (unsigned I = 0; I != N; ++I)
if (!Files[I].Name.empty())
registerPrimaryLoadedFile(Files[I], F.SLocEntryBaseOffset + Offsets[I],
@@ -2009,8 +2008,7 @@ ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
int64_t ASTReader::getSLocInverseDelta(ModuleFile &F,
SourceLocation::UIntTy G) const {
- // The list is sorted by global start and its segments tile this module's
- // range, so the covering segment is the last one whose start is <= G.
+ // SLocRemapGlobal is sorted by global start; find the segment containing G.
if (!F.SLocRemapGlobal.empty()) {
auto It = llvm::upper_bound(
F.SLocRemapGlobal, G,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c303e2e91eb68..af1426b7a5b68 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6862,9 +6862,8 @@ ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
#ifndef NDEBUG
- // The reader must recover Loc from what we wrote, which holds only if
- // BaseOffset inverts its local-to-global map. A stale inverse otherwise goes
- // unnoticed until the location resolves into an unrelated file.
+ // Verify that serialization and deserialization round-trip loaded locations.
+ // A stale inverse may otherwise resolve the location to an unrelated file.
if (OwningModuleFile) {
SourceLocation Decoded = SourceLocationEncoding::decode(Encoded).first;
assert(getChain()->TranslateSourceLocation(*OwningModuleFile, Decoded) ==
>From 703e8ecb818b3e90725e0fe3e31a08648625d8a9 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Tue, 1 Sep 2026 14:27:29 +0200
Subject: [PATCH 21/32] add regression test for wrong inverse mapping
---
.../sloc-round-trip-deduplicated-file.cpp | 47 +++++++++++++++++++
1 file changed, 47 insertions(+)
create mode 100644 clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
diff --git a/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp b/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
new file mode 100644
index 0000000000000..9f7c13210fad3
--- /dev/null
+++ b/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
@@ -0,0 +1,47 @@
+// Locations in a file reused from an earlier module must survive a
+// serialize/deserialize round trip. dup.h is written into q.pcm and reused when
+// a.pcm loads it, so DECL(Box) gives Box macro expansion locations inside that
+// reused file. Writing b.pcm re-encodes those loaded locations, which requires
+// inverting the reader's local-to-global map. While that inverse assumed a flat
+// shift it did not invert the piecewise map, and writing b.pcm asserted while
+// serializing 'Box<int>'.
+//
+// Assertions are required: with the inverse wrong but assertions off, every
+// step below still succeeds.
+
+// REQUIRES: asserts
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/q.h \
+// RUN: -I%t -Wno-experimental-header-units -o %t/q.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/a.h \
+// RUN: -I%t -fmodule-file=%t/q.pcm -Wno-experimental-header-units -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/b.h \
+// RUN: -I%t -fmodule-file=%t/a.pcm -Wno-experimental-header-units -o %t/b.pcm
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/use.cpp -I%t \
+// RUN: -fmodule-file=%t/a.pcm -fmodule-file=%t/b.pcm \
+// RUN: -Wno-experimental-header-units
+
+//--- dup.h
+#define DECL(name) template <class T> struct name { T value; };
+
+//--- q.h
+#include "dup.h"
+
+//--- a.h
+import "q.h";
+#include "dup.h"
+DECL(Box)
+
+//--- b.h
+import "a.h";
+using Alias = Box<int>;
+
+//--- use.cpp
+import "a.h";
+import "b.h";
+Alias value;
+int main() { return value.value; }
>From 55e6b9a3c89a7591c066bc1c2fc3fde3c33cdea2 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Sat, 5 Sep 2026 01:02:47 +0200
Subject: [PATCH 22/32] catch duplicate files when writing instead of reading
---
clang/include/clang/Basic/SourceManager.h | 15 -
clang/include/clang/Serialization/ASTReader.h | 95 ------
clang/include/clang/Serialization/ASTWriter.h | 23 ++
.../include/clang/Serialization/ModuleFile.h | 34 ---
clang/lib/Basic/SourceManager.cpp | 5 -
clang/lib/Serialization/ASTReader.cpp | 280 ++----------------
clang/lib/Serialization/ASTWriter.cpp | 180 +++++++----
.../sloc-round-trip-deduplicated-file.cpp | 47 ---
8 files changed, 168 insertions(+), 511 deletions(-)
delete mode 100644 clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 1c085a0097364..1939d1aa4915e 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -755,21 +755,6 @@ class SourceManager : public RefCountedBase<SourceManager> {
static const SourceLocation::UIntTy MaxLoadedOffset =
1ULL << (8 * sizeof(SourceLocation::UIntTy) - 1);
- // Source location deduplication statistics.
-private:
- /// Number of loaded file entries reused from an earlier module.
- unsigned NumDuplicateLoadedFiles = 0;
- /// Address-space bytes reused instead of allocated, for -print-stats.
- uint64_t DuplicateLoadedBytes = 0;
-
-public:
- /// Record that a loaded file entry was reused from an earlier module.
- void noteDuplicateLoadedFile(uint64_t Size) {
- ++NumDuplicateLoadedFiles;
- DuplicateLoadedBytes += Size;
- }
-
-private:
/// A bitmap that indicates whether the entries of LoadedSLocEntryTable
/// have already been loaded from the external source.
///
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 8b8427c1190f1..d800af83d350b 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2399,68 +2399,6 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::Expected<SourceLocation::UIntTy> readSLocOffset(ModuleFile *F,
unsigned Index);
- /// Identity of an input file used for source location deduplication.
- /// Name is the resolved path; an empty Name denotes a non-file entry.
- struct SLocFileIdentity {
- std::string Name;
- off_t Size = 0;
- };
-
- /// Location and identity of the first loaded copy of a file.
- struct PrimaryLoadedFileLoc {
- SourceLocation::UIntTy Offset = 0; ///< global start offset
- int ID = 0; ///< global SLoc entry ID
- off_t Size = 0;
- };
-
- /// Files already loaded into the loaded SLoc address space, keyed by stored
- /// name. Lets a later module reuse an earlier module's copy instead of
- /// allocating a duplicate range.
- llvm::StringMap<PrimaryLoadedFileLoc> PrimaryLoadedFiles;
-
- /// The location of a previously-loaded file matching \p Id (same name, size,
- /// and time), or null if none has been loaded.
- const PrimaryLoadedFileLoc *
- getPrimaryLoadedFile(const SLocFileIdentity &Id) const;
-
- /// Record the location of a file the first time it loads. \p Id must name a
- /// file entry (non-empty Name). Later duplicates are ignored.
- void registerPrimaryLoadedFile(const SLocFileIdentity &Id,
- SourceLocation::UIntTy Offset, int ID);
-
- /// Read \p F's SLoc entry records without materializing them, filling
- /// \p Offsets[i] with each entry's local offset and \p Files[i] with its
- /// file identity (empty Name for non-file entries). Returns false on a
- /// malformed record.
- bool scanLoadedSLocEntries(ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
- SmallVectorImpl<SLocFileIdentity> &Files);
-
- /// Mark the scanned entries that duplicate an already-loaded file. Returns
- /// the number of duplicates and the space they would otherwise occupy.
- std::pair<unsigned, SourceLocation::UIntTy> classifyDuplicateSLocEntries(
- ArrayRef<uint32_t> Offsets, ArrayRef<SLocFileIdentity> Files,
- SourceLocation::UIntTy SLocSpaceSize, SmallVectorImpl<bool> &IsDup);
-
- /// Build \p F's local-to-global SLoc remapping and register its files. Run
- /// after AllocateLoadedSLocEntries has assigned \p F's base ID and offset.
- void buildLoadedSLocRemapping(ModuleFile &F, ArrayRef<uint32_t> Offsets,
- ArrayRef<SLocFileIdentity> Files,
- ArrayRef<bool> IsDup,
- SourceLocation::UIntTy SLocSpaceSize,
- unsigned NumDupEntries,
- unsigned ReducedNumEntries);
-
- /// Map a local SLoc entry offset (as stored in the entry record) to its
- /// global start offset. This is (SLocEntryBaseOffset + LocalOffset) unless a
- /// file in this module was reused from an earlier one.
- SourceLocation::UIntTy remapSLocEntryOffset(ModuleFile &F,
- uint32_t LocalOffset) const;
-
- /// The delta that produced global offset \p G in \p F, to be subtracted to
- /// recover \p F's local location. \p G must lie in \p F's own range, as it
- /// does when \p F was found through GlobalSLocOffsetMap.
- int64_t getSLocInverseDelta(ModuleFile &F, SourceLocation::UIntTy G) const;
-
/// Retrieve the module import location and module name for the
/// given source manager entry ID.
std::pair<SourceLocation, StringRef> getModuleImportLoc(int ID) override;
@@ -2555,31 +2493,6 @@ class ASTReader : public ExternalPreprocessorSource,
// translated or refactor the code to make it clear that
// TranslateSourceLocation won't be called with translated source location.
- // The remap is piecewise when files are deduplicated. SourceLocation's
- // MacroID bit is not part of the offset used to select a segment.
- if (!ModuleFile.SLocRemap.empty()) {
- SourceLocation::UIntTy Raw = Loc.getRawEncoding();
- SourceLocation::UIntTy MacroBit = Raw & SourceLocation::MacroIDBit;
- SourceLocation::UIntTy Low = Raw & ~SourceLocation::MacroIDBit;
- // Find the segment containing Low.
- auto It = llvm::upper_bound(
- ModuleFile.SLocRemap, Low,
- [](SourceLocation::UIntTy V,
- const serialization::ModuleFile::SLocRemapSegment &S) {
- return V < S.LocalBegin;
- });
- if (It != ModuleFile.SLocRemap.begin()) {
- const auto &Seg = *std::prev(It);
- if (Low < Seg.LocalEnd)
- return SourceLocation::getFromRawEncoding(
- (static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
- Seg.Delta)) |
- MacroBit);
- }
- // The segments cover the whole value range, so one always matches.
- llvm_unreachable("SLocRemap segments must cover every source location");
- }
-
return Loc.getLocWithOffset(ModuleFile.SLocEntryBaseOffset - 2);
}
@@ -2601,14 +2514,6 @@ class ASTReader : public ExternalPreprocessorSource,
assert(FID.ID >= 0 && "Reading non-local FileID.");
if (FID.isInvalid())
return FID;
- // When a file was reused from an earlier module, its local FileID maps to
- // that module's copy, so use the explicit map. Local FileID N is local
- // entry index N-1.
- if (!F.LocalToGlobalID.empty()) {
- assert((unsigned)(FID.ID - 1) < F.LocalToGlobalID.size() &&
- "local FileID out of range");
- return FileID::get(F.LocalToGlobalID[FID.ID - 1]);
- }
return FileID::get(F.SLocEntryBaseID + FID.ID - 1);
}
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 95ae8a6ba8c74..b74a9058f4b16 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -542,6 +542,22 @@ class ASTWriter : public ASTDeserializationListener,
std::vector<SourceRange> NonAffectingRanges;
std::vector<SourceLocation::UIntTy> NonAffectingOffsetAdjustments;
+ /// Mapping from a range to the amount an offset within it must move to reach
+ /// the loaded copy we point at. This is \c Begin-LoadedBase, always negative
+ /// since loaded offsets sit above local ones, and zero for a range with no
+ /// copy.
+ std::vector<int64_t> NonAffectingRedirectAdjustments;
+
+ /// Mapping from an input file to the \c FileID a loaded module already uses
+ /// for it. Filled on first use, so only a module write pays for the walk.
+ llvm::DenseMap<const FileEntry *, FileID> LoadedCopyFileIDs;
+ bool LoadedCopyFileIDsBuilt = false;
+
+ /// Whether the control block has been written. It records import locations,
+ /// which must stay local to this module file, so we rewrite nothing before
+ /// then.
+ bool ControlBlockWritten = false;
+
/// A list of classes in named modules which need to emit the VTable in
/// the corresponding object file.
llvm::SmallVector<CXXRecordDecl *> PendingEmittingVTables;
@@ -556,6 +572,13 @@ class ASTWriter : public ASTDeserializationListener,
SourceLocation getAffectingIncludeLoc(const SourceManager &SourceMgr,
const SrcMgr::FileInfo &File);
+ /// The \c FileID a loaded module uses for \p FE, if one of them has it.
+ FileID getLoadedCopyFileID(const FileEntry *FE);
+
+ /// Returns \p Loc translated into the module file that already has its file,
+ /// or an invalid location if we kept the file.
+ SourceLocation getRedirectedLocation(SourceLocation Loc) const;
+
/// Returns an adjusted \c FileID, accounting for any non-affecting input
/// files.
FileID getAdjustedFileID(FileID FID) const;
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index a74d3a9542bae..6c47040fde093 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -346,40 +346,6 @@ class ModuleFile {
/// AST file.
const uint32_t *SLocEntryOffsets = nullptr;
- // === Source location deduplication ===
-
- /// One segment of the local-to-global source location map. A local raw
- /// location L in [LocalBegin, LocalEnd) maps to global location L + Delta.
- struct SLocRemapSegment {
- SourceLocation::UIntTy LocalBegin;
- SourceLocation::UIntTy LocalEnd;
- int64_t Delta;
- };
-
- /// The local-to-global source location map for this module, sorted by
- /// LocalBegin. When a module's files are all distinct it holds a single
- /// segment equivalent to the flat shift by (SLocEntryBaseOffset - 2). When a
- /// file is reused from an earlier module, extra segments redirect that file's
- /// locations into the earlier module. Empty when this path did not run, in
- /// which case the flat shift is used directly.
- llvm::SmallVector<SLocRemapSegment, 4> SLocRemap;
-
- /// The inverse of SLocRemap, sorted by global start, where LocalBegin and
- /// LocalEnd hold global bounds and a global location G maps to G - Delta.
- /// Holds only the entries this module kept, which tile its own range.
- llvm::SmallVector<SLocRemapSegment, 4> SLocRemapGlobal;
-
- /// Maps a local SLoc entry index to its global SLoc entry ID. A kept entry
- /// maps to its own ID. A file reused from an earlier module maps to that
- /// module's copy. Empty when no file was reused (the global ID is then
- /// SLocEntryBaseID + index).
- std::vector<int> LocalToGlobalID;
-
- /// For each kept entry, in order, its original local index (into
- /// SLocEntryOffsets). Reused entries have no slot, so this skips them. Empty
- /// when no file was reused (kept slot j is then local index j).
- std::vector<unsigned> KeptSLocLocalIndex;
-
// === Identifiers ===
/// The number of identifiers in this AST file.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index ab20a0914a856..5540aade05ef5 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -2152,11 +2152,6 @@ void SourceManager::PrintStats() const {
<< " bytes of capacity), "
<< MaxLoadedOffset - CurrentLoadedOffset
<< "B of SLoc address space used.\n";
- if (NumDuplicateLoadedFiles)
- llvm::errs() << NumDuplicateLoadedFiles
- << " duplicate loaded file SLocEntries deduplicated ("
- << DuplicateLoadedBytes
- << "B of SLoc address space reused).\n";
unsigned NumLineNumsComputed = 0;
unsigned NumFileBytesMapped = 0;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7e5a7bf0c04ff..3455b729be696 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1815,217 +1815,6 @@ llvm::Error ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
}
}
-const ASTReader::PrimaryLoadedFileLoc *
-ASTReader::getPrimaryLoadedFile(const SLocFileIdentity &Id) const {
- assert(!Id.Name.empty() && "querying a non-file entry");
- auto It = PrimaryLoadedFiles.find(Id.Name);
- if (It == PrimaryLoadedFiles.end())
- return nullptr;
- const PrimaryLoadedFileLoc &Primary = It->second;
- // A matching name but different size is a different file, e.g. two modules
- // built against different versions of the same path. Don't merge.
- if (Primary.Size != Id.Size)
- return nullptr;
- return &Primary;
-}
-
-void ASTReader::registerPrimaryLoadedFile(const SLocFileIdentity &Id,
- SourceLocation::UIntTy Offset,
- int ID) {
- assert(!Id.Name.empty() && "registering a non-file entry");
- PrimaryLoadedFiles.try_emplace(Id.Name,
- PrimaryLoadedFileLoc{Offset, ID, Id.Size});
-}
-
-bool ASTReader::scanLoadedSLocEntries(
- ModuleFile &F, SmallVectorImpl<uint32_t> &Offsets,
- SmallVectorImpl<SLocFileIdentity> &Files) {
- unsigned N = F.LocalNumSLocEntries;
- Offsets.assign(N, 0);
- Files.assign(N, SLocFileIdentity{});
-
- SmallString<0> PathBuf;
- PathBuf.reserve(256);
- BitstreamCursor &Cursor = F.SLocEntryCursor;
- SavedStreamPosition SavedPosition(Cursor);
- for (unsigned I = 0; I != N; ++I) {
- if (llvm::Error Err =
- Cursor.JumpToBit(F.SLocEntryOffsetsBase + F.SLocEntryOffsets[I])) {
- consumeError(std::move(Err));
- return false;
- }
- Expected<llvm::BitstreamEntry> Entry = Cursor.advance();
- if (!Entry) {
- consumeError(Entry.takeError());
- return false;
- }
- if (Entry->Kind != llvm::BitstreamEntry::Record)
- return false;
-
- RecordData Record;
- Expected<unsigned> Code = Cursor.readRecord(Entry->ID, Record);
- if (!Code) {
- consumeError(Code.takeError());
- return false;
- }
- Offsets[I] = (uint32_t)Record[0];
- if (Code.get() == SM_SLOC_FILE_ENTRY) {
- // Identity comes from serialized metadata, so no input file is touched on
- // disk. Resolve the stored name to a path so two same-named files in
- // different directories are not treated as one; this is string work only.
- InputFileInfo IFI = getInputFileInfo(F, Record[4]);
- if (IFI.isValid())
- Files[I] =
- SLocFileIdentity{ResolveImportedPathAndAllocate(
- PathBuf, IFI.UnresolvedImportedFilename, F),
- IFI.StoredSize};
- }
- }
- return true;
-}
-
-/// Bytes that scanned entry \p I occupies on the loaded number line.
-static SourceLocation::UIntTy
-slocEntrySize(ArrayRef<uint32_t> Offsets, unsigned I,
- SourceLocation::UIntTy SLocSpaceSize) {
- return (I + 1 < Offsets.size() ? Offsets[I + 1] : SLocSpaceSize) - Offsets[I];
-}
-
-std::pair<unsigned, SourceLocation::UIntTy>
-ASTReader::classifyDuplicateSLocEntries(ArrayRef<uint32_t> Offsets,
- ArrayRef<SLocFileIdentity> Files,
- SourceLocation::UIntTy SLocSpaceSize,
- SmallVectorImpl<bool> &IsDup) {
- unsigned N = Offsets.size();
- IsDup.assign(N, false);
-
- // Decide duplicates here. buildLoadedSLocRemapping registers this module's
- // own files as it goes, so a file seen twice in one module must not be
- // treated as a duplicate of its own first copy.
- unsigned NumDup = 0;
- SourceLocation::UIntTy DupBytes = 0;
- for (unsigned I = 0; I != N; ++I) {
- if (Files[I].Name.empty() || !getPrimaryLoadedFile(Files[I]))
- continue;
- SourceLocation::UIntTy Size = slocEntrySize(Offsets, I, SLocSpaceSize);
- IsDup[I] = true;
- SourceMgr.noteDuplicateLoadedFile(Size);
- DupBytes += Size;
- ++NumDup;
- }
- return {NumDup, DupBytes};
-}
-
-void ASTReader::buildLoadedSLocRemapping(
- ModuleFile &F, ArrayRef<uint32_t> Offsets, ArrayRef<SLocFileIdentity> Files,
- ArrayRef<bool> IsDup, SourceLocation::UIntTy SLocSpaceSize,
- unsigned NumDupEntries, unsigned ReducedNumEntries) {
- unsigned N = Files.size();
-
- // Without duplicates, the remap is the original flat shift.
- if (NumDupEntries == 0) {
- F.SLocRemap.push_back({0, ~SourceLocation::UIntTy(0),
- static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
- // The inverse is the same flat shift over this module's whole range.
- F.SLocRemapGlobal.push_back(
- {F.SLocEntryBaseOffset, F.SLocEntryBaseOffset + SLocSpaceSize,
- static_cast<int64_t>(F.SLocEntryBaseOffset) - 2});
- // Record each file for deduplication by later modules.
- for (unsigned I = 0; I != N; ++I)
- if (!Files[I].Name.empty())
- registerPrimaryLoadedFile(Files[I], F.SLocEntryBaseOffset + Offsets[I],
- F.SLocEntryBaseID + (int)I);
- return;
- }
-
- F.LocalToGlobalID.assign(N, 0);
- F.KeptSLocLocalIndex.reserve(ReducedNumEntries);
- SourceLocation::UIntTy DupBefore = 0;
- unsigned KeptCount = 0;
- for (unsigned I = 0; I != N; ++I) {
- SourceLocation::UIntTy LowStart = Offsets[I] + 2;
- SourceLocation::UIntTy LowEnd =
- LowStart + slocEntrySize(Offsets, I, SLocSpaceSize);
- if (IsDup[I]) {
- // Redirect into the module that first loaded the file and reserve
- // nothing.
- const PrimaryLoadedFileLoc *Primary = getPrimaryLoadedFile(Files[I]);
- F.SLocRemap.push_back({LowStart, LowEnd,
- static_cast<int64_t>(Primary->Offset) -
- static_cast<int64_t>(LowStart)});
- F.LocalToGlobalID[I] = Primary->ID;
- DupBefore += LowEnd - LowStart;
- } else {
- // Keep the entry, shifted down past the duplicates skipped before it.
- int GlobalID = F.SLocEntryBaseID + (int)KeptCount++;
- SourceLocation::UIntTy GlobalStart =
- F.SLocEntryBaseOffset + Offsets[I] - DupBefore;
- int64_t Delta =
- static_cast<int64_t>(GlobalStart) - static_cast<int64_t>(LowStart);
- F.SLocRemap.push_back({LowStart, LowEnd, Delta});
- // Kept entries are visited in increasing global order, so appending
- // keeps SLocRemapGlobal sorted.
- F.SLocRemapGlobal.push_back(
- {GlobalStart, GlobalStart + (LowEnd - LowStart), Delta});
- F.LocalToGlobalID[I] = GlobalID;
- F.KeptSLocLocalIndex.push_back(I);
- if (!Files[I].Name.empty())
- registerPrimaryLoadedFile(Files[I], GlobalStart, GlobalID);
- }
- }
- // Cover the whole value range so every location maps to a segment.
- F.SLocRemap.front().LocalBegin = 0;
- F.SLocRemap.back().LocalEnd = ~SourceLocation::UIntTy(0);
- assert(KeptCount == ReducedNumEntries && "kept count mismatch");
-}
-
-SourceLocation::UIntTy
-ASTReader::remapSLocEntryOffset(ModuleFile &F, uint32_t LocalOffset) const {
- // The entry's local raw start location is LocalOffset + 2 (offsets 0 and 1
- // are reserved). Find the segment covering it and apply that segment's delta.
- if (!F.SLocRemap.empty()) {
- SourceLocation::UIntTy Low = LocalOffset + 2;
- // The list is sorted by LocalBegin and its segments are contiguous, so the
- // covering segment is the last one whose LocalBegin is <= Low.
- auto It = llvm::upper_bound(
- F.SLocRemap, Low,
- [](SourceLocation::UIntTy V,
- const serialization::ModuleFile::SLocRemapSegment &S) {
- return V < S.LocalBegin;
- });
- if (It != F.SLocRemap.begin()) {
- const auto &Seg = *std::prev(It);
- if (Low < Seg.LocalEnd)
- return static_cast<SourceLocation::UIntTy>(static_cast<int64_t>(Low) +
- Seg.Delta);
- }
- // The segments cover the whole value range, so one always matches.
- llvm_unreachable("SLocRemap segments must cover every source location");
- }
- // No remap for this module. Use the flat shift.
- return F.SLocEntryBaseOffset + LocalOffset;
-}
-
-int64_t ASTReader::getSLocInverseDelta(ModuleFile &F,
- SourceLocation::UIntTy G) const {
- // SLocRemapGlobal is sorted by global start; find the segment containing G.
- if (!F.SLocRemapGlobal.empty()) {
- auto It = llvm::upper_bound(
- F.SLocRemapGlobal, G,
- [](SourceLocation::UIntTy V,
- const serialization::ModuleFile::SLocRemapSegment &S) {
- return V < S.LocalBegin;
- });
- if (It != F.SLocRemapGlobal.begin()) {
- const auto &Seg = *std::prev(It);
- if (G < Seg.LocalEnd)
- return Seg.Delta;
- }
- }
- // No inverse for this module. Use the flat shift.
- return static_cast<int64_t>(F.SLocEntryBaseOffset) - 2;
-}
-
llvm::Expected<SourceLocation::UIntTy>
ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
BitstreamCursor &Cursor = F->SLocEntryCursor;
@@ -2058,7 +1847,7 @@ ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
case SM_SLOC_FILE_ENTRY:
case SM_SLOC_BUFFER_ENTRY:
case SM_SLOC_EXPANSION_ENTRY:
- return remapSLocEntryOffset(*F, Record[0]);
+ return F->SLocEntryBaseOffset + Record[0];
}
}
@@ -2071,20 +1860,13 @@ int ASTReader::getSLocEntryID(SourceLocation::UIntTy SLocOffset) {
bool Invalid = false;
- // The table holds only this module's kept entries, so search over kept slots
- // and map each back to its on-disk local index when reading the offset.
- bool Dedup = !F->KeptSLocLocalIndex.empty();
- unsigned NumSlots =
- Dedup ? F->KeptSLocLocalIndex.size() : F->LocalNumSLocEntries;
-
auto It = llvm::upper_bound(
- llvm::index_range(0, NumSlots), SLocOffset,
- [&](SourceLocation::UIntTy Offset, std::size_t Slot) {
- int ID = F->SLocEntryBaseID + Slot;
+ llvm::index_range(0, F->LocalNumSLocEntries), SLocOffset,
+ [&](SourceLocation::UIntTy Offset, std::size_t LocalIndex) {
+ int ID = F->SLocEntryBaseID + LocalIndex;
std::size_t Index = -ID - 2;
if (!SourceMgr.SLocEntryOffsetLoaded[Index]) {
assert(!SourceMgr.SLocEntryLoaded[Index]);
- unsigned LocalIndex = Dedup ? F->KeptSLocLocalIndex[Slot] : Slot;
auto MaybeEntryOffset = readSLocOffset(F, LocalIndex);
if (!MaybeEntryOffset) {
Error(MaybeEntryOffset.takeError());
@@ -2166,22 +1948,15 @@ bool ASTReader::ReadSLocEntry(int ID) {
};
ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
- // When entries have been skipped, the global slot (ID - SLocEntryBaseID) is
- // no longer the on-disk local index, so map it back through the kept-index
- // table.
- unsigned LocalIndex = ID - F->SLocEntryBaseID;
- if (!F->KeptSLocLocalIndex.empty()) {
- assert(LocalIndex < F->KeptSLocLocalIndex.size() &&
- "kept slot out of range");
- LocalIndex = F->KeptSLocLocalIndex[LocalIndex];
- }
if (llvm::Error Err = F->SLocEntryCursor.JumpToBit(
- F->SLocEntryOffsetsBase + F->SLocEntryOffsets[LocalIndex])) {
+ F->SLocEntryOffsetsBase +
+ F->SLocEntryOffsets[ID - F->SLocEntryBaseID])) {
Error(std::move(Err));
return true;
}
BitstreamCursor &SLocEntryCursor = F->SLocEntryCursor;
+ SourceLocation::UIntTy BaseOffset = F->SLocEntryBaseOffset;
++NumSLocEntriesRead;
Expected<llvm::BitstreamEntry> MaybeEntry = SLocEntryCursor.advance();
@@ -2231,7 +2006,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
SrcMgr::CharacteristicKind
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
- remapSLocEntryOffset(*F, Record[0]));
+ BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
if (Record[3])
@@ -2272,9 +2047,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
auto Buffer = ReadBuffer(SLocEntryCursor, Name);
if (!Buffer)
return true;
- FileID FID =
- SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
- remapSLocEntryOffset(*F, Offset), IncludeLoc);
+ FileID FID = SourceMgr.createFileID(std::move(Buffer), FileCharacter, ID,
+ BaseOffset + Offset, IncludeLoc);
if (Record[3]) {
auto &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.setHasLineDirectives();
@@ -2288,7 +2062,7 @@ bool ASTReader::ReadSLocEntry(int ID) {
SourceLocation ExpansionEnd = ReadSourceLocation(*F, Record[3]);
SourceMgr.createExpansionLoc(SpellingLoc, ExpansionBegin, ExpansionEnd,
Record[5], Record[4], ID,
- remapSLocEntryOffset(*F, Record[0]));
+ BaseOffset + Record[0]);
break;
}
}
@@ -4461,48 +4235,30 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
F.LocalNumSLocEntries = Record[0];
SourceLocation::UIntTy SLocSpaceSize = Record[1];
F.SLocEntryOffsetsBase = Record[2] + F.SourceManagerBlockStartOffset;
-
- // Scan the entries and reserve no space for files an earlier module
- // already loaded.
- SmallVector<uint32_t, 64> Offsets;
- SmallVector<SLocFileIdentity, 64> Files;
- SmallVector<bool, 64> IsDup;
- unsigned NumDupEntries = 0;
- SourceLocation::UIntTy DupBytes = 0;
- if (scanLoadedSLocEntries(F, Offsets, Files))
- std::tie(NumDupEntries, DupBytes) =
- classifyDuplicateSLocEntries(Offsets, Files, SLocSpaceSize, IsDup);
-
- unsigned ReducedNumEntries = F.LocalNumSLocEntries - NumDupEntries;
- SourceLocation::UIntTy ReducedSize = SLocSpaceSize - DupBytes;
-
std::tie(F.SLocEntryBaseID, F.SLocEntryBaseOffset) =
- SourceMgr.AllocateLoadedSLocEntries(ReducedNumEntries, ReducedSize);
+ SourceMgr.AllocateLoadedSLocEntries(F.LocalNumSLocEntries,
+ SLocSpaceSize);
if (!F.SLocEntryBaseID) {
Diags.Report(SourceLocation(), diag::remark_sloc_usage);
SourceMgr.noteSLocAddressSpaceUsage(Diags);
return llvm::createStringError(std::errc::invalid_argument,
"ran out of source locations");
}
-
- buildLoadedSLocRemapping(F, Offsets, Files, IsDup, SLocSpaceSize,
- NumDupEntries, ReducedNumEntries);
-
// Make our entry in the range map. BaseID is negative and growing, so
// we invert it. Because we invert it, though, we need the other end of
// the range.
unsigned RangeStart =
- unsigned(-F.SLocEntryBaseID) - ReducedNumEntries + 1;
+ unsigned(-F.SLocEntryBaseID) - F.LocalNumSLocEntries + 1;
GlobalSLocEntryMap.insert(std::make_pair(RangeStart, &F));
F.FirstLoc = SourceLocation::getFromRawEncoding(F.SLocEntryBaseOffset);
// SLocEntryBaseOffset is lower than MaxLoadedOffset and decreasing.
assert((F.SLocEntryBaseOffset & SourceLocation::MacroIDBit) == 0);
- GlobalSLocOffsetMap.insert(std::make_pair(
- SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset - ReducedSize,
- &F));
+ GlobalSLocOffsetMap.insert(
+ std::make_pair(SourceManager::MaxLoadedOffset - F.SLocEntryBaseOffset
+ - SLocSpaceSize,&F));
- TotalNumSLocEntries += ReducedNumEntries;
+ TotalNumSLocEntries += F.LocalNumSLocEntries;
break;
}
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index af1426b7a5b68..f1dbff8790515 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5560,6 +5560,26 @@ static void AddLazyVectorEmiitedDecls(ASTWriter &Writer, Vector &Vec,
}
}
+FileID ASTWriter::getLoadedCopyFileID(const FileEntry *FE) {
+ const SourceManager &SM = PP->getSourceManager();
+ if (!LoadedCopyFileIDsBuilt) {
+ // Keying on the FileEntry keeps two files with the same name and size
+ // distinct.
+ LoadedCopyFileIDsBuilt = true;
+ for (unsigned I = 0, N = SM.loaded_sloc_entry_size(); I != N; ++I) {
+ bool Invalid = false;
+ const SrcMgr::SLocEntry &E = SM.getLoadedSLocEntry(I, &Invalid);
+ if (Invalid || !E.isFile())
+ continue;
+ if (OptionalFileEntryRef OE = E.getFile().getContentCache().OrigEntry)
+ LoadedCopyFileIDs.try_emplace(&OE->getFileEntry(),
+ FileID::get(-int(I) - 2));
+ }
+ }
+ auto It = LoadedCopyFileIDs.find(FE);
+ return It == LoadedCopyFileIDs.end() ? FileID() : It->second;
+}
+
void ASTWriter::computeNonAffectingInputFiles() {
SourceManager &SrcMgr = PP->getSourceManager();
unsigned N = SrcMgr.local_sloc_entry_size();
@@ -5581,6 +5601,34 @@ void ASTWriter::computeNonAffectingInputFiles() {
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
+ // Leaves \p FID out of this module file. A nonzero \p RedirectAdjustment
+ // points its locations at a loaded copy instead.
+ auto MarkNonAffecting = [&](FileID FID, int64_t RedirectAdjustment) {
+ FileIDAdjustment += 1;
+ // Even empty files take up one element in the offset table.
+ OffsetAdjustment += SrcMgr.getFileIDSize(FID) + 1;
+
+ // If the previous file was non-affecting as well, just extend its entry
+ // with our information. Files that point at different copies must stay in
+ // separate ranges, since we keep one redirect per range.
+ if (!NonAffectingFileIDs.empty() &&
+ NonAffectingFileIDs.back().ID == FID.ID - 1 &&
+ NonAffectingRedirectAdjustments.back() == RedirectAdjustment) {
+ NonAffectingFileIDs.back() = FID;
+ NonAffectingRanges.back().setEnd(SrcMgr.getLocForEndOfFile(FID));
+ NonAffectingFileIDAdjustments.back() = FileIDAdjustment;
+ NonAffectingOffsetAdjustments.back() = OffsetAdjustment;
+ return;
+ }
+
+ NonAffectingFileIDs.push_back(FID);
+ NonAffectingRanges.emplace_back(SrcMgr.getLocForStartOfFile(FID),
+ SrcMgr.getLocForEndOfFile(FID));
+ NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
+ NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
+ NonAffectingRedirectAdjustments.push_back(RedirectAdjustment);
+ };
+
for (unsigned I = 1; I != N; ++I) {
const SrcMgr::SLocEntry *SLoc = &SrcMgr.getLocalSLocEntry(I);
FileID FID = FileID::get(I);
@@ -5593,42 +5641,45 @@ void ASTWriter::computeNonAffectingInputFiles() {
if (!Cache->OrigEntry)
continue;
- // Don't prune anything other than module maps.
- if (!isModuleMap(File.getFileCharacteristic()))
- continue;
-
- // Don't prune module maps if all are guaranteed to be affecting.
- if (!AffectingModuleMaps)
- continue;
+ if (isModuleMap(File.getFileCharacteristic())) {
+ // Don't prune module maps if all are guaranteed to be affecting.
+ if (!AffectingModuleMaps)
+ continue;
- // Don't prune module maps that are affecting.
- if (AffectingModuleMaps->DefinitionFileIDs.contains(FID))
- continue;
+ // A module map nothing points into can be left out along with its
+ // locations.
+ if (!AffectingModuleMaps->DefinitionFileIDs.contains(FID)) {
+ IsSLocAffecting[I] = false;
+ IsSLocFileEntryAffecting[I] =
+ AffectingModuleMaps->DefinitionFiles.contains(*Cache->OrigEntry);
+ MarkNonAffecting(FID, 0);
+ continue;
+ }
- IsSLocAffecting[I] = false;
- IsSLocFileEntryAffecting[I] =
- AffectingModuleMaps->DefinitionFiles.contains(*Cache->OrigEntry);
+ // An affecting module map falls through. We can still leave it out if
+ // a module we import has it, since its locations then have a copy to
+ // point at.
+ }
- FileIDAdjustment += 1;
- // Even empty files take up one element in the offset table.
- OffsetAdjustment += SrcMgr.getFileIDSize(FID) + 1;
+ // Keep the main file. We write its FileID as the original file of this
+ // module, and an adjusted FileID only names a file whose entries we
+ // wrote.
+ if (FID == SrcMgr.getMainFileID())
+ continue;
- // If the previous file was non-affecting as well, just extend its entry
- // with our information.
- if (!NonAffectingFileIDs.empty() &&
- NonAffectingFileIDs.back().ID == FID.ID - 1) {
- NonAffectingFileIDs.back() = FID;
- NonAffectingRanges.back().setEnd(SrcMgr.getLocForEndOfFile(FID));
- NonAffectingFileIDAdjustments.back() = FileIDAdjustment;
- NonAffectingOffsetAdjustments.back() = OffsetAdjustment;
+ // A module we import may already have this input file. If it does, we
+ // point our locations at its copy instead of writing a second set of
+ // entries for the same text. The input file record is still written, so
+ // validation continues to work.
+ FileID LoadedFID = getLoadedCopyFileID(&Cache->OrigEntry->getFileEntry());
+ if (!LoadedFID.isValid())
continue;
- }
- NonAffectingFileIDs.push_back(FID);
- NonAffectingRanges.emplace_back(SrcMgr.getLocForStartOfFile(FID),
- SrcMgr.getLocForEndOfFile(FID));
- NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
- NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
+ IsSLocAffecting[I] = false;
+ IsSLocFileEntryAffecting[I] = true;
+ MarkNonAffecting(FID,
+ int64_t(SLoc->getOffset()) -
+ int64_t(SrcMgr.getSLocEntry(LoadedFID).getOffset()));
}
if (!PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage)
@@ -6152,6 +6203,9 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema *SemaPtr, StringRef isysroot,
// Write the control block
WriteControlBlock(*PP, isysroot);
+ // The import locations in the control block had to stay local. Now that it
+ // has been written, we can start rewriting.
+ ControlBlockWritten = true;
// Write the remaining AST contents.
Stream.FlushToWord();
@@ -6789,9 +6843,46 @@ unsigned ASTWriter::getAdjustedNumCreatedFIDs(FileID FID) const {
return AdjustedNumCreatedFIDs;
}
+SourceLocation ASTWriter::getRedirectedLocation(SourceLocation Loc) const {
+ if (NonAffectingRedirectAdjustments.empty())
+ return SourceLocation();
+
+ SourceLocation::UIntTy Offset = Loc.getOffset();
+ if (PP->getSourceManager().isLoadedOffset(Offset))
+ return SourceLocation();
+
+ // The same search getAdjustment does.
+ auto Contains = [](const SourceRange &Range, SourceLocation::UIntTy Offset) {
+ return Range.getEnd().getOffset() < Offset;
+ };
+ auto It = llvm::lower_bound(NonAffectingRanges, Offset, Contains);
+ if (It == NonAffectingRanges.end())
+ return SourceLocation();
+
+ // lower_bound also lands here for an offset before the range, so check that
+ // the offset really is inside it.
+ if (Offset < It->getBegin().getOffset())
+ return SourceLocation();
+
+ unsigned Idx = std::distance(NonAffectingRanges.begin(), It);
+ int64_t Adjustment = NonAffectingRedirectAdjustments[Idx];
+ if (!Adjustment)
+ return SourceLocation();
+ return SourceLocation::getFromRawEncoding(
+ static_cast<SourceLocation::UIntTy>(int64_t(Offset) - Adjustment));
+}
+
SourceLocation ASTWriter::getAdjustedLocation(SourceLocation Loc) const {
if (Loc.isInvalid())
return Loc;
+ // A location in a file we left out must move to the loaded copy first, since
+ // the shift below only handles locations that are still local. This stays out
+ // of getAdjustment because getAdjustedOffset shares it, and we call that on
+ // file sizes and on the next local offset as well.
+ if (ControlBlockWritten && !Loc.isMacroID())
+ if (SourceLocation Redirected = getRedirectedLocation(Loc);
+ Redirected.isValid())
+ return Redirected;
return Loc.getLocWithOffset(-getAdjustment(Loc.getOffset()));
}
@@ -6836,7 +6927,6 @@ SourceLocationEncoding::RawLocEncoding
ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
SourceLocation::UIntTy BaseOffset = 0;
unsigned ModuleFileIndex = 0;
- [[maybe_unused]] ModuleFile *OwningModuleFile = nullptr;
// See SourceLocationEncoding.h for the encoding details.
if (PP->getSourceManager().isLoadedSourceLocation(Loc) && Loc.isValid()) {
@@ -6846,33 +6936,14 @@ ASTWriter::getRawSourceLocationEncoding(SourceLocation Loc) {
assert(SLocMapI != getChain()->GlobalSLocOffsetMap.end() &&
"Corrupted global sloc offset map");
ModuleFile *F = SLocMapI->second;
- OwningModuleFile = F;
- // The reader's local-to-global map is piecewise once a file has been
- // reused from an earlier module, so SLocEntryBaseOffset - 2 does not invert
- // it. Recover the delta that produced this location.
- BaseOffset = static_cast<SourceLocation::UIntTy>(
- getChain()->getSLocInverseDelta(*F, Loc.getOffset()));
+ BaseOffset = F->SLocEntryBaseOffset - 2;
// 0 means the location is not loaded. So we need to add 1 to the index to
// make it clear.
ModuleFileIndex = F->Index + 1;
assert(&getChain()->getModuleManager()[F->Index] == F);
}
- SourceLocationEncoding::RawLocEncoding Encoded =
- SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
-
-#ifndef NDEBUG
- // Verify that serialization and deserialization round-trip loaded locations.
- // A stale inverse may otherwise resolve the location to an unrelated file.
- if (OwningModuleFile) {
- SourceLocation Decoded = SourceLocationEncoding::decode(Encoded).first;
- assert(getChain()->TranslateSourceLocation(*OwningModuleFile, Decoded) ==
- Loc &&
- "loaded source location did not round-trip");
- }
-#endif
-
- return Encoded;
+ return SourceLocationEncoding::encode(Loc, BaseOffset, ModuleFileIndex);
}
void ASTWriter::AddSourceLocation(SourceLocation Loc, RecordDataImpl &Record) {
@@ -7206,7 +7277,10 @@ void ASTWriter::associateDeclWithFile(const Decl *D, LocalDeclID ID) {
if (FID.isInvalid())
return;
assert(SM.getSLocEntry(FID).isFile());
- assert(IsSLocAffecting[FID.ID]);
+ // We don't build a per-file declaration table for a file we left out. The
+ // module that has the file already built one.
+ if (!IsSLocAffecting[FID.ID])
+ return;
std::unique_ptr<DeclIDInFileInfo> &Info = FileDeclIDs[FID];
if (!Info)
diff --git a/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp b/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
deleted file mode 100644
index 9f7c13210fad3..0000000000000
--- a/clang/test/Modules/sloc-round-trip-deduplicated-file.cpp
+++ /dev/null
@@ -1,47 +0,0 @@
-// Locations in a file reused from an earlier module must survive a
-// serialize/deserialize round trip. dup.h is written into q.pcm and reused when
-// a.pcm loads it, so DECL(Box) gives Box macro expansion locations inside that
-// reused file. Writing b.pcm re-encodes those loaded locations, which requires
-// inverting the reader's local-to-global map. While that inverse assumed a flat
-// shift it did not invert the piecewise map, and writing b.pcm asserted while
-// serializing 'Box<int>'.
-//
-// Assertions are required: with the inverse wrong but assertions off, every
-// step below still succeeds.
-
-// REQUIRES: asserts
-
-// RUN: rm -rf %t
-// RUN: mkdir -p %t
-// RUN: split-file %s %t
-//
-// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/q.h \
-// RUN: -I%t -Wno-experimental-header-units -o %t/q.pcm
-// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/a.h \
-// RUN: -I%t -fmodule-file=%t/q.pcm -Wno-experimental-header-units -o %t/a.pcm
-// RUN: %clang_cc1 -std=c++20 -emit-header-unit -xc++-user-header %t/b.h \
-// RUN: -I%t -fmodule-file=%t/a.pcm -Wno-experimental-header-units -o %t/b.pcm
-// RUN: %clang_cc1 -std=c++20 -fsyntax-only %t/use.cpp -I%t \
-// RUN: -fmodule-file=%t/a.pcm -fmodule-file=%t/b.pcm \
-// RUN: -Wno-experimental-header-units
-
-//--- dup.h
-#define DECL(name) template <class T> struct name { T value; };
-
-//--- q.h
-#include "dup.h"
-
-//--- a.h
-import "q.h";
-#include "dup.h"
-DECL(Box)
-
-//--- b.h
-import "a.h";
-using Alias = Box<int>;
-
-//--- use.cpp
-import "a.h";
-import "b.h";
-Alias value;
-int main() { return value.value; }
>From 913f30e15c82a690249c794b2151118abc4114a2 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Sun, 6 Sep 2026 18:28:46 -0400
Subject: [PATCH 23/32] fix FileID redirect and skipped-range adjustment for
dropped files
---
clang/lib/Serialization/ASTWriter.cpp | 49 +++++++++++++++++----------
1 file changed, 31 insertions(+), 18 deletions(-)
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index f1dbff8790515..c0001680bbb34 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -2919,10 +2919,12 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec,
if (SkippedRanges.size() > 0) {
std::vector<PPSkippedRange> SerializedSkippedRanges;
SerializedSkippedRanges.reserve(SkippedRanges.size());
- for (auto const& Range : SkippedRanges)
+ for (auto const &Range : SkippedRanges) {
+ SourceRange R = getAdjustedRange(Range);
SerializedSkippedRanges.emplace_back(
- getRawSourceLocationEncoding(Range.getBegin()),
- getRawSourceLocationEncoding(Range.getEnd()));
+ getRawSourceLocationEncoding(R.getBegin()),
+ getRawSourceLocationEncoding(R.getEnd()));
+ }
using namespace llvm;
auto Abbrev = std::make_shared<BitCodeAbbrev>();
@@ -5592,6 +5594,17 @@ void ASTWriter::computeNonAffectingInputFiles() {
auto AffectingModuleMaps = GetAffectingModuleMaps(*PP, WritingModule);
+ // Unlike a SourceLocation, a FileID is written as an index into our own SLoc
+ // table, so it cannot name a file we leave out. Collect the files something
+ // still refers to by FileID.
+ llvm::DenseSet<FileID> NamedFileIDs;
+ NamedFileIDs.insert(SrcMgr.getMainFileID());
+ if (SrcMgr.hasLineTable())
+ for (const auto &L : SrcMgr.getLineTable())
+ NamedFileIDs.insert(L.first);
+ for (const auto &F : PP->getDiagnostics().DiagStatesByLoc.Files)
+ NamedFileIDs.insert(F.first);
+
unsigned FileIDAdjustment = 0;
unsigned OffsetAdjustment = 0;
@@ -5646,25 +5659,22 @@ void ASTWriter::computeNonAffectingInputFiles() {
if (!AffectingModuleMaps)
continue;
- // A module map nothing points into can be left out along with its
- // locations.
- if (!AffectingModuleMaps->DefinitionFileIDs.contains(FID)) {
- IsSLocAffecting[I] = false;
- IsSLocFileEntryAffecting[I] =
- AffectingModuleMaps->DefinitionFiles.contains(*Cache->OrigEntry);
- MarkNonAffecting(FID, 0);
+ // Don't prune module maps that are affecting. The submodule block names
+ // them by FileID when an inferred module was uniqued by one, so they
+ // cannot be redirected either.
+ if (AffectingModuleMaps->DefinitionFileIDs.contains(FID))
continue;
- }
- // An affecting module map falls through. We can still leave it out if
- // a module we import has it, since its locations then have a copy to
- // point at.
+ // A module map nothing points into can be left out along with its
+ // locations.
+ IsSLocAffecting[I] = false;
+ IsSLocFileEntryAffecting[I] =
+ AffectingModuleMaps->DefinitionFiles.contains(*Cache->OrigEntry);
+ MarkNonAffecting(FID, 0);
+ continue;
}
- // Keep the main file. We write its FileID as the original file of this
- // module, and an adjusted FileID only names a file whose entries we
- // wrote.
- if (FID == SrcMgr.getMainFileID())
+ if (NamedFileIDs.contains(FID))
continue;
// A module we import may already have this input file. If it does, we
@@ -6824,6 +6834,9 @@ FileID ASTWriter::getAdjustedFileID(FileID FID) const {
if (FID.isInvalid() || PP->getSourceManager().isLoadedFileID(FID) ||
NonAffectingFileIDs.empty())
return FID;
+ assert(getRedirectedLocation(PP->getSourceManager().getLocForStartOfFile(FID))
+ .isInvalid() &&
+ "Cannot name a redirected file by FileID");
auto It = llvm::lower_bound(NonAffectingFileIDs, FID);
unsigned Idx = std::distance(NonAffectingFileIDs.begin(), It);
unsigned Offset = NonAffectingFileIDAdjustments[Idx];
>From 8463bdf5dc01b75be87eed40175a15e18fe5d19f Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 02:04:12 +0200
Subject: [PATCH 24/32] narrow FileID redirect guard to files with local diag
transitions
---
clang/lib/Serialization/ASTWriter.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c0001680bbb34..fc5051ecccaa1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5603,7 +5603,8 @@ void ASTWriter::computeNonAffectingInputFiles() {
for (const auto &L : SrcMgr.getLineTable())
NamedFileIDs.insert(L.first);
for (const auto &F : PP->getDiagnostics().DiagStatesByLoc.Files)
- NamedFileIDs.insert(F.first);
+ if (F.second.HasLocalTransitions)
+ NamedFileIDs.insert(F.first);
unsigned FileIDAdjustment = 0;
unsigned OffsetAdjustment = 0;
>From 8fbcb520288d6a68d29aab26f572d1e8592a8ce7 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 02:56:13 +0200
Subject: [PATCH 25/32] filter invalid FileIDs from NamedFileIDs to match
writing loops
---
clang/lib/Serialization/ASTWriter.cpp | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index fc5051ecccaa1..d4143488aff0d 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5596,14 +5596,17 @@ void ASTWriter::computeNonAffectingInputFiles() {
// Unlike a SourceLocation, a FileID is written as an index into our own SLoc
// table, so it cannot name a file we leave out. Collect the files something
- // still refers to by FileID.
+ // still refers to by FileID. Only a local file can be named that way, so we
+ // keep the same checks the loops that write these tables make.
llvm::DenseSet<FileID> NamedFileIDs;
- NamedFileIDs.insert(SrcMgr.getMainFileID());
+ if (SrcMgr.getMainFileID().isValid())
+ NamedFileIDs.insert(SrcMgr.getMainFileID());
if (SrcMgr.hasLineTable())
for (const auto &L : SrcMgr.getLineTable())
- NamedFileIDs.insert(L.first);
+ if (L.first.ID > 0)
+ NamedFileIDs.insert(L.first);
for (const auto &F : PP->getDiagnostics().DiagStatesByLoc.Files)
- if (F.second.HasLocalTransitions)
+ if (F.second.HasLocalTransitions && F.first.isValid())
NamedFileIDs.insert(F.first);
unsigned FileIDAdjustment = 0;
>From 5c32c646ec437a676d6c9842aa23f3de1dbc6ff8 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 14:23:53 +0200
Subject: [PATCH 26/32] replace loaded-entry scan with partial read for
input-file identity
---
clang/include/clang/Serialization/ASTReader.h | 41 +++++++
clang/include/clang/Serialization/ASTWriter.h | 8 --
clang/lib/Serialization/ASTReader.cpp | 114 ++++++++++++++++++
clang/lib/Serialization/ASTWriter.cpp | 35 ++----
4 files changed, 164 insertions(+), 34 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index d800af83d350b..8036dee6fa438 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1448,6 +1448,47 @@ class ASTReader : public ExternalPreprocessorSource,
const StringRef &operator*() && = delete;
};
+public:
+ /// Where a loaded module keeps its copy of a file: the FileID naming it and
+ /// the offset its locations start at. The FileID is invalid when no loaded
+ /// module has the file.
+ struct LoadedFileLoc {
+ FileID FID;
+ SourceLocation::UIntTy Offset = 0;
+ };
+
+ /// Where a loaded module keeps the input file with resolved path \p Path and
+ /// size \p Size.
+ LoadedFileLoc getLoadedFileLoc(StringRef Path, off_t Size);
+
+private:
+ /// An input file of a loaded module, as its own serialized data describes
+ /// it.
+ struct LoadedInputFile {
+ off_t Size;
+ ModuleFile *F;
+ unsigned InputID;
+ };
+
+ /// The input files of every loaded module, keyed by resolved path, in module
+ /// index order. Filled on first use.
+ llvm::StringMap<SmallVector<LoadedInputFile, 1>> LoadedInputFiles;
+ bool LoadedInputFilesBuilt = false;
+
+ /// For each module we have walked, where it keeps each of its input files.
+ /// We walk a module only once something asks about a file it has.
+ llvm::DenseMap<ModuleFile *, llvm::DenseMap<unsigned, LoadedFileLoc>>
+ LoadedInputFileLocs;
+
+ void canonicalizePathForIdentity(SmallVectorImpl<char> &Path) const;
+ void buildLoadedInputFiles();
+ LoadedFileLoc getLoadedInputFileLoc(ModuleFile &F, unsigned InputID);
+
+ /// Read the offset and input file index out of the file entry at local index
+ /// \p Index in \p F. The index is zero for an entry that is not a file.
+ llvm::Expected<std::pair<SourceLocation::UIntTy, unsigned>>
+ readSLocFileEntry(ModuleFile *F, unsigned Index);
+
public:
/// Get the buffer for resolving paths.
SmallString<0> &getPathBuf() { return PathBuf; }
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index b74a9058f4b16..c591025507467 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -548,11 +548,6 @@ class ASTWriter : public ASTDeserializationListener,
/// copy.
std::vector<int64_t> NonAffectingRedirectAdjustments;
- /// Mapping from an input file to the \c FileID a loaded module already uses
- /// for it. Filled on first use, so only a module write pays for the walk.
- llvm::DenseMap<const FileEntry *, FileID> LoadedCopyFileIDs;
- bool LoadedCopyFileIDsBuilt = false;
-
/// Whether the control block has been written. It records import locations,
/// which must stay local to this module file, so we rewrite nothing before
/// then.
@@ -572,9 +567,6 @@ class ASTWriter : public ASTDeserializationListener,
SourceLocation getAffectingIncludeLoc(const SourceManager &SourceMgr,
const SrcMgr::FileInfo &File);
- /// The \c FileID a loaded module uses for \p FE, if one of them has it.
- FileID getLoadedCopyFileID(const FileEntry *FE);
-
/// Returns \p Loc translated into the module file that already has its file,
/// or an invalid location if we kept the file.
SourceLocation getRedirectedLocation(SourceLocation Loc) const;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 3455b729be696..9ab86d04949d6 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1851,6 +1851,120 @@ ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
}
}
+llvm::Expected<std::pair<SourceLocation::UIntTy, unsigned>>
+ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
+ BitstreamCursor &Cursor = F->SLocEntryCursor;
+ SavedStreamPosition SavedPosition(Cursor);
+ if (llvm::Error Err = Cursor.JumpToBit(F->SLocEntryOffsetsBase +
+ F->SLocEntryOffsets[Index]))
+ return std::move(Err);
+
+ Expected<llvm::BitstreamEntry> MaybeEntry = Cursor.advance();
+ if (!MaybeEntry)
+ return MaybeEntry.takeError();
+
+ llvm::BitstreamEntry Entry = MaybeEntry.get();
+ if (Entry.Kind != llvm::BitstreamEntry::Record)
+ return llvm::createStringError(
+ std::errc::illegal_byte_sequence,
+ "incorrectly-formatted source location entry in AST file");
+
+ RecordData Record;
+ StringRef Blob;
+ Expected<unsigned> MaybeSLOC = Cursor.readRecord(Entry.ID, Record, &Blob);
+ if (!MaybeSLOC)
+ return MaybeSLOC.takeError();
+
+ switch (MaybeSLOC.get()) {
+ default:
+ return llvm::createStringError(
+ std::errc::illegal_byte_sequence,
+ "incorrectly-formatted source location entry in AST file");
+ case SM_SLOC_FILE_ENTRY:
+ return std::make_pair(F->SLocEntryBaseOffset + Record[0],
+ unsigned(Record[4]));
+ case SM_SLOC_BUFFER_ENTRY:
+ case SM_SLOC_EXPANSION_ENTRY:
+ return std::make_pair(F->SLocEntryBaseOffset + Record[0], 0u);
+ }
+}
+
+void ASTReader::canonicalizePathForIdentity(SmallVectorImpl<char> &Path) const {
+ FileMgr.makeAbsolutePath(Path, /*Canonicalize=*/true);
+}
+
+void ASTReader::buildLoadedInputFiles() {
+ LoadedInputFilesBuilt = true;
+ // ModuleManager hands modules out in index order, so the copy we settle on
+ // for a file does not depend on the order things happened to be loaded in.
+ for (ModuleFile &F : ModuleMgr) {
+ for (unsigned I = 0, N = F.InputFilesLoaded.size(); I != N; ++I) {
+ InputFileInfo FI = getInputFileInfo(F, I + 1);
+ if (FI.UnresolvedImportedFilename.empty())
+ continue;
+ // An overridden input holds a buffer rather than the contents of the
+ // path it names, so its path and size describe nothing we can match on.
+ if (FI.Overridden)
+ continue;
+ auto Filename =
+ ResolveImportedPath(PathBuf, FI.UnresolvedImportedFilename, F);
+ SmallString<128> Key(*Filename);
+ canonicalizePathForIdentity(Key);
+ LoadedInputFiles[Key].push_back({FI.StoredSize, &F, I + 1});
+ }
+ }
+}
+
+ASTReader::LoadedFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F,
+ unsigned InputID) {
+ auto Known = LoadedInputFileLocs.find(&F);
+ if (Known == LoadedInputFileLocs.end()) {
+ llvm::DenseMap<unsigned, LoadedFileLoc> Locs;
+ for (unsigned I = 0; I != F.LocalNumSLocEntries; ++I) {
+ auto MaybeEntry = readSLocFileEntry(&F, I);
+ if (!MaybeEntry) {
+ consumeError(MaybeEntry.takeError());
+ continue;
+ }
+ auto [Offset, ID] = *MaybeEntry;
+ if (!ID)
+ continue;
+ // A module writes its entries in order, so the first entry naming an
+ // input file is the one we want.
+ Locs.try_emplace(
+ ID, LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), Offset});
+ }
+ Known = LoadedInputFileLocs.try_emplace(&F, std::move(Locs)).first;
+ }
+
+ auto It = Known->second.find(InputID);
+ return It == Known->second.end() ? LoadedFileLoc() : It->second;
+}
+
+ASTReader::LoadedFileLoc ASTReader::getLoadedFileLoc(StringRef Path,
+ off_t Size) {
+ if (!LoadedInputFilesBuilt)
+ buildLoadedInputFiles();
+
+ SmallString<128> Key(Path);
+ canonicalizePathForIdentity(Key);
+ auto Known = LoadedInputFiles.find(Key);
+ if (Known == LoadedInputFiles.end())
+ return LoadedFileLoc();
+
+ for (const LoadedInputFile &In : Known->second) {
+ if (In.Size != Size)
+ continue;
+ // A module that has the file as an input may still have left its source
+ // location entries out, in which case it has no copy to point at and we
+ // keep looking.
+ LoadedFileLoc Loc = getLoadedInputFileLoc(*In.F, In.InputID);
+ if (Loc.FID.isValid())
+ return Loc;
+ }
+ return LoadedFileLoc();
+}
+
int ASTReader::getSLocEntryID(SourceLocation::UIntTy SLocOffset) {
auto SLocMapI =
GlobalSLocOffsetMap.find(SourceManager::MaxLoadedOffset - SLocOffset - 1);
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index d4143488aff0d..99f34843541c3 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5562,26 +5562,6 @@ static void AddLazyVectorEmiitedDecls(ASTWriter &Writer, Vector &Vec,
}
}
-FileID ASTWriter::getLoadedCopyFileID(const FileEntry *FE) {
- const SourceManager &SM = PP->getSourceManager();
- if (!LoadedCopyFileIDsBuilt) {
- // Keying on the FileEntry keeps two files with the same name and size
- // distinct.
- LoadedCopyFileIDsBuilt = true;
- for (unsigned I = 0, N = SM.loaded_sloc_entry_size(); I != N; ++I) {
- bool Invalid = false;
- const SrcMgr::SLocEntry &E = SM.getLoadedSLocEntry(I, &Invalid);
- if (Invalid || !E.isFile())
- continue;
- if (OptionalFileEntryRef OE = E.getFile().getContentCache().OrigEntry)
- LoadedCopyFileIDs.try_emplace(&OE->getFileEntry(),
- FileID::get(-int(I) - 2));
- }
- }
- auto It = LoadedCopyFileIDs.find(FE);
- return It == LoadedCopyFileIDs.end() ? FileID() : It->second;
-}
-
void ASTWriter::computeNonAffectingInputFiles() {
SourceManager &SrcMgr = PP->getSourceManager();
unsigned N = SrcMgr.local_sloc_entry_size();
@@ -5684,16 +5664,19 @@ void ASTWriter::computeNonAffectingInputFiles() {
// A module we import may already have this input file. If it does, we
// point our locations at its copy instead of writing a second set of
// entries for the same text. The input file record is still written, so
- // validation continues to work.
- FileID LoadedFID = getLoadedCopyFileID(&Cache->OrigEntry->getFileEntry());
- if (!LoadedFID.isValid())
+ // validation continues to work. We ask by path and size, which a module
+ // records for every input it has, so the answer comes out of what it
+ // wrote and its own entries stay untouched.
+ if (!hasChain())
+ continue;
+ ASTReader::LoadedFileLoc Loaded = getChain()->getLoadedFileLoc(
+ Cache->OrigEntry->getName(), Cache->OrigEntry->getSize());
+ if (Loaded.FID.isInvalid())
continue;
IsSLocAffecting[I] = false;
IsSLocFileEntryAffecting[I] = true;
- MarkNonAffecting(FID,
- int64_t(SLoc->getOffset()) -
- int64_t(SrcMgr.getSLocEntry(LoadedFID).getOffset()));
+ MarkNonAffecting(FID, int64_t(SLoc->getOffset()) - int64_t(Loaded.Offset));
}
if (!PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage)
>From 7aa0b06eb165451374cf36267f869653e2ed84d5 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 14:50:24 +0200
Subject: [PATCH 27/32] add tests
---
.../Modules/prune-duplicate-input-file.cpp | 84 +++++++++++++++++++
1 file changed, 84 insertions(+)
create mode 100644 clang/test/Modules/prune-duplicate-input-file.cpp
diff --git a/clang/test/Modules/prune-duplicate-input-file.cpp b/clang/test/Modules/prune-duplicate-input-file.cpp
new file mode 100644
index 0000000000000..7e4e6c5b9645f
--- /dev/null
+++ b/clang/test/Modules/prune-duplicate-input-file.cpp
@@ -0,0 +1,84 @@
+// Check that a file two modules both include textually only takes up source
+// location space once. A module that already has the file lends its copy to
+// the next one, which points its own locations at that copy instead of
+// writing a second set of entries for the same text.
+//
+// A file that something still names by FileID has to keep its own entries,
+// since a FileID only means anything in the module file that wrote it. Those
+// are the files with a line table entry or with diagnostic state of their own.
+
+// RUN: rm -rf %t && mkdir %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN: -fmodule-map-file=%t/mods.map \
+// RUN: -fmodule-name=mod1 -emit-module %t/mods.map -o %t/mod1.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN: -fmodule-map-file=%t/mods.map -fmodule-file=%t/mod1.pcm \
+// RUN: -fmodule-name=mod2 -emit-module %t/mods.map -o %t/mod2.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN: -fmodule-map-file=%t/mods.map -fmodule-file=%t/mod2.pcm \
+// RUN: -fsyntax-only -verify %t/check_slocs.cc
+
+//--- mods.map
+module mod1 { header "mod1.h" export * }
+module mod2 { header "mod2.h" export * }
+
+//--- check_slocs.cc
+#include "mod2.h"
+#pragma clang __debug sloc_usage // expected-remark {{source manager location address space usage}}
+// expected-note@* {{% of available space}}
+
+// Both modules include this textually, and nothing names it by FileID, so
+// mod2 points at mod1's copy and the file is entered once.
+
+// expected-note at shared.h:1 {{file entered 1 time}}
+
+// Both modules include these textually as well, but each is still named by
+// FileID somewhere, so each module keeps its own entries for them.
+
+// expected-note at lines.h:1 {{file entered 2 times}}
+// expected-note at diags.h:1 {{file entered 2 times}}
+
+// expected-note@* + {{file entered}}
+
+//--- shared.h
+#ifndef SHARED_H
+#define SHARED_H
+struct Shared {
+ int a;
+};
+int shared_fn(void);
+#endif
+
+//--- lines.h
+#ifndef LINES_H
+#define LINES_H
+int lines_fn(void);
+// A line directive puts this file in the line table, which the module file
+// records by FileID.
+#line 500 "somewhere-else.h"
+#endif
+
+//--- diags.h
+#ifndef DIAGS_H
+#define DIAGS_H
+int diags_fn(void);
+// A diagnostic pragma gives this file diagnostic state of its own, which the
+// module file also records by FileID.
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunused-variable"
+#pragma clang diagnostic pop
+#endif
+
+//--- mod1.h
+#include "shared.h"
+#include "lines.h"
+#include "diags.h"
+int mod1_fn(void);
+
+//--- mod2.h
+#include "shared.h"
+#include "lines.h"
+#include "diags.h"
+int mod2_fn(void);
>From 9fb6e36df0e2f28adcc144224945fbb1bf5e89f9 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 22:57:40 +0200
Subject: [PATCH 28/32] refactored readSLocOffset and other code
---
clang/include/clang/Serialization/ASTReader.h | 15 ++++--
clang/include/clang/Serialization/ASTWriter.h | 8 +++
clang/lib/Serialization/ASTReader.cpp | 51 +++++--------------
clang/lib/Serialization/ASTWriter.cpp | 46 +++++++++--------
4 files changed, 56 insertions(+), 64 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 8036dee6fa438..e7a65339326d1 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1484,10 +1484,17 @@ class ASTReader : public ExternalPreprocessorSource,
void buildLoadedInputFiles();
LoadedFileLoc getLoadedInputFileLoc(ModuleFile &F, unsigned InputID);
- /// Read the offset and input file index out of the file entry at local index
- /// \p Index in \p F. The index is zero for an entry that is not a file.
- llvm::Expected<std::pair<SourceLocation::UIntTy, unsigned>>
- readSLocFileEntry(ModuleFile *F, unsigned Index);
+ /// The offset an SLoc entry's locations start at, and the index of the input
+ /// file it names. \c InputID is zero for an entry that is not a file.
+ struct SLocEntryInfo {
+ SourceLocation::UIntTy Offset = 0;
+ unsigned InputID = 0;
+ };
+
+ /// Read the offset and input file index out of the SLoc entry at local index
+ /// \p Index in \p F.
+ llvm::Expected<SLocEntryInfo> readSLocFileEntry(ModuleFile *F,
+ unsigned Index);
public:
/// Get the buffer for resolving paths.
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index c591025507467..8b270384cfde9 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -546,6 +546,10 @@ class ASTWriter : public ASTDeserializationListener,
/// the loaded copy we point at. This is \c Begin-LoadedBase, always negative
/// since loaded offsets sit above local ones, and zero for a range with no
/// copy.
+ ///
+ /// Unlike the two vectors above, this one is indexed by range, so entry
+ /// \c I belongs to \c NonAffectingRanges[I]. Those two carry a leading zero
+ /// and hold the adjustment that applies before the range of the same index.
std::vector<int64_t> NonAffectingRedirectAdjustments;
/// Whether the control block has been written. It records import locations,
@@ -571,6 +575,10 @@ class ASTWriter : public ASTDeserializationListener,
/// or an invalid location if we kept the file.
SourceLocation getRedirectedLocation(SourceLocation Loc) const;
+ /// Returns the index of the first non-affecting range that does not end
+ /// before \p Offset, or \c NonAffectingRanges.size() if every range does.
+ unsigned getNonAffectingRangeLowerBound(SourceLocation::UIntTy Offset) const;
+
/// Returns an adjusted \c FileID, accounting for any non-affecting input
/// files.
FileID getAdjustedFileID(FileID FID) const;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 9ab86d04949d6..b6afd469a21a8 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1817,41 +1817,13 @@ llvm::Error ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
llvm::Expected<SourceLocation::UIntTy>
ASTReader::readSLocOffset(ModuleFile *F, unsigned Index) {
- BitstreamCursor &Cursor = F->SLocEntryCursor;
- SavedStreamPosition SavedPosition(Cursor);
- if (llvm::Error Err = Cursor.JumpToBit(F->SLocEntryOffsetsBase +
- F->SLocEntryOffsets[Index]))
- return std::move(Err);
-
- Expected<llvm::BitstreamEntry> MaybeEntry = Cursor.advance();
- if (!MaybeEntry)
- return MaybeEntry.takeError();
-
- llvm::BitstreamEntry Entry = MaybeEntry.get();
- if (Entry.Kind != llvm::BitstreamEntry::Record)
- return llvm::createStringError(
- std::errc::illegal_byte_sequence,
- "incorrectly-formatted source location entry in AST file");
-
- RecordData Record;
- StringRef Blob;
- Expected<unsigned> MaybeSLOC = Cursor.readRecord(Entry.ID, Record, &Blob);
- if (!MaybeSLOC)
- return MaybeSLOC.takeError();
-
- switch (MaybeSLOC.get()) {
- default:
- return llvm::createStringError(
- std::errc::illegal_byte_sequence,
- "incorrectly-formatted source location entry in AST file");
- case SM_SLOC_FILE_ENTRY:
- case SM_SLOC_BUFFER_ENTRY:
- case SM_SLOC_EXPANSION_ENTRY:
- return F->SLocEntryBaseOffset + Record[0];
- }
+ Expected<SLocEntryInfo> MaybeInfo = readSLocFileEntry(F, Index);
+ if (!MaybeInfo)
+ return MaybeInfo.takeError();
+ return MaybeInfo->Offset;
}
-llvm::Expected<std::pair<SourceLocation::UIntTy, unsigned>>
+llvm::Expected<ASTReader::SLocEntryInfo>
ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
BitstreamCursor &Cursor = F->SLocEntryCursor;
SavedStreamPosition SavedPosition(Cursor);
@@ -1881,11 +1853,11 @@ ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
std::errc::illegal_byte_sequence,
"incorrectly-formatted source location entry in AST file");
case SM_SLOC_FILE_ENTRY:
- return std::make_pair(F->SLocEntryBaseOffset + Record[0],
- unsigned(Record[4]));
+ return SLocEntryInfo{F->SLocEntryBaseOffset + Record[0],
+ static_cast<unsigned>(Record[4])};
case SM_SLOC_BUFFER_ENTRY:
case SM_SLOC_EXPANSION_ENTRY:
- return std::make_pair(F->SLocEntryBaseOffset + Record[0], 0u);
+ return SLocEntryInfo{F->SLocEntryBaseOffset + Record[0], 0};
}
}
@@ -1926,13 +1898,14 @@ ASTReader::LoadedFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F,
consumeError(MaybeEntry.takeError());
continue;
}
- auto [Offset, ID] = *MaybeEntry;
- if (!ID)
+ const SLocEntryInfo &Info = *MaybeEntry;
+ if (!Info.InputID)
continue;
// A module writes its entries in order, so the first entry naming an
// input file is the one we want.
Locs.try_emplace(
- ID, LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), Offset});
+ Info.InputID,
+ LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), Info.Offset});
}
Known = LoadedInputFileLocs.try_emplace(&F, std::move(Locs)).first;
}
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 99f34843541c3..5cb5175ff7ee7 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5576,8 +5576,9 @@ void ASTWriter::computeNonAffectingInputFiles() {
// Unlike a SourceLocation, a FileID is written as an index into our own SLoc
// table, so it cannot name a file we leave out. Collect the files something
- // still refers to by FileID. Only a local file can be named that way, so we
- // keep the same checks the loops that write these tables make.
+ // still refers to by FileID, mirroring the conditions under which the loops
+ // that write these tables emit one. Only local files can be named that way,
+ // and we skip the invalid FileID so it never becomes a key here.
llvm::DenseSet<FileID> NamedFileIDs;
if (SrcMgr.getMainFileID().isValid())
NamedFileIDs.insert(SrcMgr.getMainFileID());
@@ -5586,7 +5587,7 @@ void ASTWriter::computeNonAffectingInputFiles() {
if (L.first.ID > 0)
NamedFileIDs.insert(L.first);
for (const auto &F : PP->getDiagnostics().DiagStatesByLoc.Files)
- if (F.second.HasLocalTransitions && F.first.isValid())
+ if (F.first.isValid() && F.second.HasLocalTransitions)
NamedFileIDs.insert(F.first);
unsigned FileIDAdjustment = 0;
@@ -5676,9 +5677,13 @@ void ASTWriter::computeNonAffectingInputFiles() {
IsSLocAffecting[I] = false;
IsSLocFileEntryAffecting[I] = true;
- MarkNonAffecting(FID, int64_t(SLoc->getOffset()) - int64_t(Loaded.Offset));
+ MarkNonAffecting(FID, static_cast<int64_t>(SLoc->getOffset()) -
+ static_cast<int64_t>(Loaded.Offset));
}
+ assert(NonAffectingRedirectAdjustments.size() == NonAffectingRanges.size() &&
+ "Every non-affecting range needs a redirect adjustment");
+
if (!PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage)
return;
@@ -6851,25 +6856,20 @@ SourceLocation ASTWriter::getRedirectedLocation(SourceLocation Loc) const {
if (PP->getSourceManager().isLoadedOffset(Offset))
return SourceLocation();
- // The same search getAdjustment does.
- auto Contains = [](const SourceRange &Range, SourceLocation::UIntTy Offset) {
- return Range.getEnd().getOffset() < Offset;
- };
- auto It = llvm::lower_bound(NonAffectingRanges, Offset, Contains);
- if (It == NonAffectingRanges.end())
+ unsigned Idx = getNonAffectingRangeLowerBound(Offset);
+ if (Idx == NonAffectingRanges.size())
return SourceLocation();
- // lower_bound also lands here for an offset before the range, so check that
- // the offset really is inside it.
- if (Offset < It->getBegin().getOffset())
+ // The search only rules out ranges ending before the offset, so check that
+ // the offset really is inside the one we landed on.
+ if (Offset < NonAffectingRanges[Idx].getBegin().getOffset())
return SourceLocation();
- unsigned Idx = std::distance(NonAffectingRanges.begin(), It);
int64_t Adjustment = NonAffectingRedirectAdjustments[Idx];
if (!Adjustment)
return SourceLocation();
- return SourceLocation::getFromRawEncoding(
- static_cast<SourceLocation::UIntTy>(int64_t(Offset) - Adjustment));
+ return SourceLocation::getFileLoc(static_cast<SourceLocation::UIntTy>(
+ static_cast<int64_t>(Offset) - Adjustment));
}
SourceLocation ASTWriter::getAdjustedLocation(SourceLocation Loc) const {
@@ -6910,13 +6910,17 @@ ASTWriter::getAdjustment(SourceLocation::UIntTy Offset) const {
if (Offset < NonAffectingRanges.front().getBegin().getOffset())
return 0;
- auto Contains = [](const SourceRange &Range, SourceLocation::UIntTy Offset) {
+ return NonAffectingOffsetAdjustments[getNonAffectingRangeLowerBound(Offset)];
+}
+
+unsigned
+ASTWriter::getNonAffectingRangeLowerBound(SourceLocation::UIntTy Offset) const {
+ auto EndsBefore = [](const SourceRange &Range,
+ SourceLocation::UIntTy Offset) {
return Range.getEnd().getOffset() < Offset;
};
-
- auto It = llvm::lower_bound(NonAffectingRanges, Offset, Contains);
- unsigned Idx = std::distance(NonAffectingRanges.begin(), It);
- return NonAffectingOffsetAdjustments[Idx];
+ auto It = llvm::lower_bound(NonAffectingRanges, Offset, EndsBefore);
+ return std::distance(NonAffectingRanges.begin(), It);
}
void ASTWriter::AddFileID(FileID FID, RecordDataImpl &Record) {
>From 3526aad761d3ca28bfdfd43a20ef652ebc52c4ab Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Mon, 7 Sep 2026 23:33:47 +0200
Subject: [PATCH 29/32] fix sloc casts in reasSlocFileEntry
---
clang/lib/Serialization/ASTReader.cpp | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index b6afd469a21a8..10c89cdfaa310 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1853,11 +1853,14 @@ ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
std::errc::illegal_byte_sequence,
"incorrectly-formatted source location entry in AST file");
case SM_SLOC_FILE_ENTRY:
- return SLocEntryInfo{F->SLocEntryBaseOffset + Record[0],
- static_cast<unsigned>(Record[4])};
+ return SLocEntryInfo{
+ static_cast<SourceLocation::UIntTy>(F->SLocEntryBaseOffset + Record[0]),
+ static_cast<unsigned>(Record[4])};
case SM_SLOC_BUFFER_ENTRY:
case SM_SLOC_EXPANSION_ENTRY:
- return SLocEntryInfo{F->SLocEntryBaseOffset + Record[0], 0};
+ return SLocEntryInfo{
+ static_cast<SourceLocation::UIntTy>(F->SLocEntryBaseOffset + Record[0]),
+ 0};
}
}
>From 113418078c32dc4cb71e2841490a46301379accd Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Tue, 8 Sep 2026 13:33:29 +0200
Subject: [PATCH 30/32] some refactoring
---
clang/include/clang/Serialization/ASTReader.h | 11 +++---
clang/lib/Serialization/ASTReader.cpp | 34 +++++++++----------
2 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e7a65339326d1..2d274d663b095 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1449,16 +1449,16 @@ class ASTReader : public ExternalPreprocessorSource,
};
public:
- /// Where a loaded module keeps its copy of a file: the FileID naming it and
- /// the offset its locations start at. The FileID is invalid when no loaded
- /// module has the file.
+ /// Records where a loaded module keeps its copy of a file. \c FID names the
+ /// copy and \c Offset is where its locations start. \c FID is invalid when
+ /// no loaded module has the file.
struct LoadedFileLoc {
FileID FID;
SourceLocation::UIntTy Offset = 0;
};
- /// Where a loaded module keeps the input file with resolved path \p Path and
- /// size \p Size.
+ /// Returns where a loaded module keeps the input file with resolved path
+ /// \p Path and size \p Size.
LoadedFileLoc getLoadedFileLoc(StringRef Path, off_t Size);
private:
@@ -1480,7 +1480,6 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::DenseMap<ModuleFile *, llvm::DenseMap<unsigned, LoadedFileLoc>>
LoadedInputFileLocs;
- void canonicalizePathForIdentity(SmallVectorImpl<char> &Path) const;
void buildLoadedInputFiles();
LoadedFileLoc getLoadedInputFileLoc(ModuleFile &F, unsigned InputID);
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 10c89cdfaa310..011888b1fd763 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1864,10 +1864,6 @@ ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
}
}
-void ASTReader::canonicalizePathForIdentity(SmallVectorImpl<char> &Path) const {
- FileMgr.makeAbsolutePath(Path, /*Canonicalize=*/true);
-}
-
void ASTReader::buildLoadedInputFiles() {
LoadedInputFilesBuilt = true;
// ModuleManager hands modules out in index order, so the copy we settle on
@@ -1883,8 +1879,11 @@ void ASTReader::buildLoadedInputFiles() {
continue;
auto Filename =
ResolveImportedPath(PathBuf, FI.UnresolvedImportedFilename, F);
+ // Both sides of a comparison have to spell a path the same way, so make
+ // it absolute and drop any dot segments. This works on the string alone
+ // and reads nothing from the file system.
SmallString<128> Key(*Filename);
- canonicalizePathForIdentity(Key);
+ FileMgr.makeAbsolutePath(Key, /*Canonicalize=*/true);
LoadedInputFiles[Key].push_back({FI.StoredSize, &F, I + 1});
}
}
@@ -1892,25 +1891,24 @@ void ASTReader::buildLoadedInputFiles() {
ASTReader::LoadedFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F,
unsigned InputID) {
- auto Known = LoadedInputFileLocs.find(&F);
- if (Known == LoadedInputFileLocs.end()) {
- llvm::DenseMap<unsigned, LoadedFileLoc> Locs;
+ auto [Known, Inserted] = LoadedInputFileLocs.try_emplace(&F);
+ if (Inserted) {
for (unsigned I = 0; I != F.LocalNumSLocEntries; ++I) {
- auto MaybeEntry = readSLocFileEntry(&F, I);
- if (!MaybeEntry) {
- consumeError(MaybeEntry.takeError());
+ Expected<SLocEntryInfo> MaybeInfo = readSLocFileEntry(&F, I);
+ if (!MaybeInfo) {
+ // Losing an entry only costs us a redirect, so leave the file to the
+ // module that is writing it rather than failing the write.
+ consumeError(MaybeInfo.takeError());
continue;
}
- const SLocEntryInfo &Info = *MaybeEntry;
- if (!Info.InputID)
+ if (!MaybeInfo->InputID)
continue;
// A module writes its entries in order, so the first entry naming an
// input file is the one we want.
- Locs.try_emplace(
- Info.InputID,
- LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), Info.Offset});
+ Known->second.try_emplace(
+ MaybeInfo->InputID,
+ LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), MaybeInfo->Offset});
}
- Known = LoadedInputFileLocs.try_emplace(&F, std::move(Locs)).first;
}
auto It = Known->second.find(InputID);
@@ -1923,7 +1921,7 @@ ASTReader::LoadedFileLoc ASTReader::getLoadedFileLoc(StringRef Path,
buildLoadedInputFiles();
SmallString<128> Key(Path);
- canonicalizePathForIdentity(Key);
+ FileMgr.makeAbsolutePath(Key, /*Canonicalize=*/true);
auto Known = LoadedInputFiles.find(Key);
if (Known == LoadedInputFiles.end())
return LoadedFileLoc();
>From 8ae1b97c7685ce6e23799bbe27cd632f1e3a829a Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Tue, 8 Sep 2026 19:31:44 +0200
Subject: [PATCH 31/32] move per-module map to ModuleFile
---
clang/include/clang/Serialization/ASTReader.h | 21 +++------
.../include/clang/Serialization/ModuleFile.h | 14 ++++++
clang/lib/Serialization/ASTReader.cpp | 31 ++++++-------
clang/lib/Serialization/ASTWriter.cpp | 2 +-
...ile.cpp => reuse-duplicate-input-file.cpp} | 44 +++++++++++--------
5 files changed, 61 insertions(+), 51 deletions(-)
rename clang/test/Modules/{prune-duplicate-input-file.cpp => reuse-duplicate-input-file.cpp} (56%)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 2d274d663b095..c5af416aafc30 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1449,17 +1449,10 @@ class ASTReader : public ExternalPreprocessorSource,
};
public:
- /// Records where a loaded module keeps its copy of a file. \c FID names the
- /// copy and \c Offset is where its locations start. \c FID is invalid when
- /// no loaded module has the file.
- struct LoadedFileLoc {
- FileID FID;
- SourceLocation::UIntTy Offset = 0;
- };
-
/// Returns where a loaded module keeps the input file with resolved path
- /// \p Path and size \p Size.
- LoadedFileLoc getLoadedFileLoc(StringRef Path, off_t Size);
+ /// \p Path and size \p Size. The returned \c FID is invalid when no loaded
+ /// module has the file.
+ serialization::InputFileLoc getLoadedFileLoc(StringRef Path, off_t Size);
private:
/// An input file of a loaded module, as its own serialized data describes
@@ -1475,13 +1468,9 @@ class ASTReader : public ExternalPreprocessorSource,
llvm::StringMap<SmallVector<LoadedInputFile, 1>> LoadedInputFiles;
bool LoadedInputFilesBuilt = false;
- /// For each module we have walked, where it keeps each of its input files.
- /// We walk a module only once something asks about a file it has.
- llvm::DenseMap<ModuleFile *, llvm::DenseMap<unsigned, LoadedFileLoc>>
- LoadedInputFileLocs;
-
void buildLoadedInputFiles();
- LoadedFileLoc getLoadedInputFileLoc(ModuleFile &F, unsigned InputID);
+ serialization::InputFileLoc getLoadedInputFileLoc(ModuleFile &F,
+ unsigned InputID);
/// The offset an SLoc entry's locations start at, and the index of the input
/// file it names. \c InputID is zero for an entry that is not a file.
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index 6c47040fde093..e8a22422a81bb 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -79,6 +79,14 @@ struct InputFileInfo {
}
};
+/// Where a module file keeps its own copy of an input file. \c FID names the
+/// copy and \c Offset is where its locations start. \c FID is invalid for an
+/// input file the module file wrote no source location entries for.
+struct InputFileLoc {
+ FileID FID;
+ SourceLocation::UIntTy Offset = 0;
+};
+
/// The input file that has been loaded from this AST file, along with
/// bools indicating whether this was an overridden buffer or if it was
/// out-of-date or not-found.
@@ -304,6 +312,12 @@ class ModuleFile {
/// The input file infos that have been loaded from this AST file.
std::vector<InputFileInfo> InputFileInfosLoaded;
+ /// Where this module file keeps each of its input files. We read this out of
+ /// the source location entries the first time something asks, since only a
+ /// module write needs it.
+ std::vector<InputFileLoc> InputFileLocsLoaded;
+ bool InputFileLocsLoadedBuilt = false;
+
// All user input files reside at the index range [0, NumUserInputFiles), and
// system input files reside at [NumUserInputFiles, InputFilesLoaded.size()).
unsigned NumUserInputFiles = 0;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 011888b1fd763..3b092dd9561fb 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1889,10 +1889,10 @@ void ASTReader::buildLoadedInputFiles() {
}
}
-ASTReader::LoadedFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F,
- unsigned InputID) {
- auto [Known, Inserted] = LoadedInputFileLocs.try_emplace(&F);
- if (Inserted) {
+InputFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F, unsigned InputID) {
+ if (!F.InputFileLocsLoadedBuilt) {
+ F.InputFileLocsLoadedBuilt = true;
+ F.InputFileLocsLoaded.resize(F.InputFilesLoaded.size());
for (unsigned I = 0; I != F.LocalNumSLocEntries; ++I) {
Expected<SLocEntryInfo> MaybeInfo = readSLocFileEntry(&F, I);
if (!MaybeInfo) {
@@ -1901,22 +1901,23 @@ ASTReader::LoadedFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F,
consumeError(MaybeInfo.takeError());
continue;
}
- if (!MaybeInfo->InputID)
+ if (!MaybeInfo->InputID ||
+ MaybeInfo->InputID > F.InputFileLocsLoaded.size())
continue;
// A module writes its entries in order, so the first entry naming an
// input file is the one we want.
- Known->second.try_emplace(
- MaybeInfo->InputID,
- LoadedFileLoc{FileID::get(F.SLocEntryBaseID + I), MaybeInfo->Offset});
+ InputFileLoc &Loc = F.InputFileLocsLoaded[MaybeInfo->InputID - 1];
+ if (Loc.FID.isInvalid())
+ Loc = {FileID::get(F.SLocEntryBaseID + I), MaybeInfo->Offset};
}
}
- auto It = Known->second.find(InputID);
- return It == Known->second.end() ? LoadedFileLoc() : It->second;
+ if (InputID == 0 || InputID > F.InputFileLocsLoaded.size())
+ return InputFileLoc();
+ return F.InputFileLocsLoaded[InputID - 1];
}
-ASTReader::LoadedFileLoc ASTReader::getLoadedFileLoc(StringRef Path,
- off_t Size) {
+InputFileLoc ASTReader::getLoadedFileLoc(StringRef Path, off_t Size) {
if (!LoadedInputFilesBuilt)
buildLoadedInputFiles();
@@ -1924,7 +1925,7 @@ ASTReader::LoadedFileLoc ASTReader::getLoadedFileLoc(StringRef Path,
FileMgr.makeAbsolutePath(Key, /*Canonicalize=*/true);
auto Known = LoadedInputFiles.find(Key);
if (Known == LoadedInputFiles.end())
- return LoadedFileLoc();
+ return InputFileLoc();
for (const LoadedInputFile &In : Known->second) {
if (In.Size != Size)
@@ -1932,11 +1933,11 @@ ASTReader::LoadedFileLoc ASTReader::getLoadedFileLoc(StringRef Path,
// A module that has the file as an input may still have left its source
// location entries out, in which case it has no copy to point at and we
// keep looking.
- LoadedFileLoc Loc = getLoadedInputFileLoc(*In.F, In.InputID);
+ InputFileLoc Loc = getLoadedInputFileLoc(*In.F, In.InputID);
if (Loc.FID.isValid())
return Loc;
}
- return LoadedFileLoc();
+ return InputFileLoc();
}
int ASTReader::getSLocEntryID(SourceLocation::UIntTy SLocOffset) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 5cb5175ff7ee7..4965e4810e45a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5670,7 +5670,7 @@ void ASTWriter::computeNonAffectingInputFiles() {
// wrote and its own entries stay untouched.
if (!hasChain())
continue;
- ASTReader::LoadedFileLoc Loaded = getChain()->getLoadedFileLoc(
+ serialization::InputFileLoc Loaded = getChain()->getLoadedFileLoc(
Cache->OrigEntry->getName(), Cache->OrigEntry->getSize());
if (Loaded.FID.isInvalid())
continue;
diff --git a/clang/test/Modules/prune-duplicate-input-file.cpp b/clang/test/Modules/reuse-duplicate-input-file.cpp
similarity index 56%
rename from clang/test/Modules/prune-duplicate-input-file.cpp
rename to clang/test/Modules/reuse-duplicate-input-file.cpp
index 7e4e6c5b9645f..bd2caff9be1a1 100644
--- a/clang/test/Modules/prune-duplicate-input-file.cpp
+++ b/clang/test/Modules/reuse-duplicate-input-file.cpp
@@ -1,11 +1,7 @@
-// Check that a file two modules both include textually only takes up source
-// location space once. A module that already has the file lends its copy to
-// the next one, which points its own locations at that copy instead of
-// writing a second set of entries for the same text.
-//
-// A file that something still names by FileID has to keep its own entries,
-// since a FileID only means anything in the module file that wrote it. Those
-// are the files with a line table entry or with diagnostic state of their own.
+// Check that a header included textually by several modules does not allocate
+// extra source location space, and that a header still named by FileID does.
+// This optimization is important for large codebases to avoid running out of
+// source location space.
// RUN: rm -rf %t && mkdir %t
// RUN: split-file %s %t
@@ -18,27 +14,35 @@
// RUN: -fmodule-name=mod2 -emit-module %t/mods.map -o %t/mod2.pcm
// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
// RUN: -fmodule-map-file=%t/mods.map -fmodule-file=%t/mod2.pcm \
+// RUN: -fmodule-name=mod3 -emit-module %t/mods.map -o %t/mod3.pcm
+// RUN: %clang_cc1 -xc++ -fmodules -fno-implicit-modules \
+// RUN: -fmodule-map-file=%t/mods.map -fmodule-file=%t/mod3.pcm \
// RUN: -fsyntax-only -verify %t/check_slocs.cc
+// The modules are siblings chained only through -fmodule-file. Including one
+// from the next would carry the include guards along and nothing would be
+// entered textually at all.
+
//--- mods.map
module mod1 { header "mod1.h" export * }
module mod2 { header "mod2.h" export * }
+module mod3 { header "mod3.h" export * }
//--- check_slocs.cc
-#include "mod2.h"
+#include "mod3.h"
#pragma clang __debug sloc_usage // expected-remark {{source manager location address space usage}}
// expected-note@* {{% of available space}}
-// Both modules include this textually, and nothing names it by FileID, so
-// mod2 points at mod1's copy and the file is entered once.
+// shared.h must be entered once for the whole chain. mod2 points at mod1's copy
+// and mod3 must look past mod2, which kept no entries of its own, to find it.
// expected-note at shared.h:1 {{file entered 1 time}}
-// Both modules include these textually as well, but each is still named by
-// FileID somewhere, so each module keeps its own entries for them.
+// lines.h and diags.h are named by FileID through the line table and through
+// diagnostic state, so each module must keep its own entries for them.
-// expected-note at lines.h:1 {{file entered 2 times}}
-// expected-note at diags.h:1 {{file entered 2 times}}
+// expected-note at lines.h:1 {{file entered 3 times}}
+// expected-note at diags.h:1 {{file entered 3 times}}
// expected-note@* + {{file entered}}
@@ -55,8 +59,6 @@ int shared_fn(void);
#ifndef LINES_H
#define LINES_H
int lines_fn(void);
-// A line directive puts this file in the line table, which the module file
-// records by FileID.
#line 500 "somewhere-else.h"
#endif
@@ -64,8 +66,6 @@ int lines_fn(void);
#ifndef DIAGS_H
#define DIAGS_H
int diags_fn(void);
-// A diagnostic pragma gives this file diagnostic state of its own, which the
-// module file also records by FileID.
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-variable"
#pragma clang diagnostic pop
@@ -82,3 +82,9 @@ int mod1_fn(void);
#include "lines.h"
#include "diags.h"
int mod2_fn(void);
+
+//--- mod3.h
+#include "shared.h"
+#include "lines.h"
+#include "diags.h"
+int mod3_fn(void);
>From 1a7dfbcd05f305edf20d41a0c46be0bc77f61b44 Mon Sep 17 00:00:00 2001
From: Ayokunle Amodu <ayokunle321 at gmail.com>
Date: Thu, 10 Sep 2026 12:33:28 +0200
Subject: [PATCH 32/32] adjust comments
---
clang/include/clang/Serialization/ASTReader.h | 16 +++----
clang/include/clang/Serialization/ASTWriter.h | 23 ++++------
.../include/clang/Serialization/ModuleFile.h | 11 ++---
clang/lib/Serialization/ASTReader.cpp | 21 ++++-----
clang/lib/Serialization/ASTWriter.cpp | 46 +++++++------------
.../Modules/reuse-duplicate-input-file.cpp | 19 ++++----
6 files changed, 54 insertions(+), 82 deletions(-)
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index c5af416aafc30..4e63200f09b67 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -1449,22 +1449,18 @@ class ASTReader : public ExternalPreprocessorSource,
};
public:
- /// Returns where a loaded module keeps the input file with resolved path
- /// \p Path and size \p Size. The returned \c FID is invalid when no loaded
- /// module has the file.
+ /// Returns where a loaded module keeps the input file with path \p Path and
+ /// size \p Size, or an invalid \c FID if no loaded module has the file.
serialization::InputFileLoc getLoadedFileLoc(StringRef Path, off_t Size);
private:
- /// An input file of a loaded module, as its own serialized data describes
- /// it.
struct LoadedInputFile {
off_t Size;
ModuleFile *F;
unsigned InputID;
};
- /// The input files of every loaded module, keyed by resolved path, in module
- /// index order. Filled on first use.
+ /// Input files of loaded modules, keyed by resolved path. Built on first use.
llvm::StringMap<SmallVector<LoadedInputFile, 1>> LoadedInputFiles;
bool LoadedInputFilesBuilt = false;
@@ -1472,14 +1468,14 @@ class ASTReader : public ExternalPreprocessorSource,
serialization::InputFileLoc getLoadedInputFileLoc(ModuleFile &F,
unsigned InputID);
- /// The offset an SLoc entry's locations start at, and the index of the input
- /// file it names. \c InputID is zero for an entry that is not a file.
+ /// The offset of an SLoc entry and the input file it names. \c InputID is
+ /// zero for entries that are not files.
struct SLocEntryInfo {
SourceLocation::UIntTy Offset = 0;
unsigned InputID = 0;
};
- /// Read the offset and input file index out of the SLoc entry at local index
+ /// Reads the offset and input file index from the SLoc entry at local index
/// \p Index in \p F.
llvm::Expected<SLocEntryInfo> readSLocFileEntry(ModuleFile *F,
unsigned Index);
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 8b270384cfde9..6bbfaad86cf51 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -542,19 +542,15 @@ class ASTWriter : public ASTDeserializationListener,
std::vector<SourceRange> NonAffectingRanges;
std::vector<SourceLocation::UIntTy> NonAffectingOffsetAdjustments;
- /// Mapping from a range to the amount an offset within it must move to reach
- /// the loaded copy we point at. This is \c Begin-LoadedBase, always negative
- /// since loaded offsets sit above local ones, and zero for a range with no
- /// copy.
+ /// Adjustment from a local range to the corresponding loaded range. Zero
+ /// means the range has no loaded copy.
///
- /// Unlike the two vectors above, this one is indexed by range, so entry
- /// \c I belongs to \c NonAffectingRanges[I]. Those two carry a leading zero
- /// and hold the adjustment that applies before the range of the same index.
+ /// Unlike the adjustment vectors above, this vector is indexed by range, so
+ /// entry \c I corresponds to \c NonAffectingRanges[I].
std::vector<int64_t> NonAffectingRedirectAdjustments;
- /// Whether the control block has been written. It records import locations,
- /// which must stay local to this module file, so we rewrite nothing before
- /// then.
+ /// Whether the control block has been written. Import locations in the
+ /// control block must remain local.
bool ControlBlockWritten = false;
/// A list of classes in named modules which need to emit the VTable in
@@ -571,12 +567,11 @@ class ASTWriter : public ASTDeserializationListener,
SourceLocation getAffectingIncludeLoc(const SourceManager &SourceMgr,
const SrcMgr::FileInfo &File);
- /// Returns \p Loc translated into the module file that already has its file,
- /// or an invalid location if we kept the file.
+ /// Returns \p Loc in a loaded copy of its file, or an invalid location if the
+ /// file is kept locally.
SourceLocation getRedirectedLocation(SourceLocation Loc) const;
- /// Returns the index of the first non-affecting range that does not end
- /// before \p Offset, or \c NonAffectingRanges.size() if every range does.
+ /// Returns the first non-affecting range whose end is not before \p Offset.
unsigned getNonAffectingRangeLowerBound(SourceLocation::UIntTy Offset) const;
/// Returns an adjusted \c FileID, accounting for any non-affecting input
diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h
index e8a22422a81bb..c05f7c8fb25e2 100644
--- a/clang/include/clang/Serialization/ModuleFile.h
+++ b/clang/include/clang/Serialization/ModuleFile.h
@@ -79,9 +79,9 @@ struct InputFileInfo {
}
};
-/// Where a module file keeps its own copy of an input file. \c FID names the
-/// copy and \c Offset is where its locations start. \c FID is invalid for an
-/// input file the module file wrote no source location entries for.
+/// Where a module file keeps an input file. \c FID names the file and
+/// \c Offset is where its locations start. \c FID is invalid if the module
+/// file wrote no source location entries for the input file.
struct InputFileLoc {
FileID FID;
SourceLocation::UIntTy Offset = 0;
@@ -312,9 +312,8 @@ class ModuleFile {
/// The input file infos that have been loaded from this AST file.
std::vector<InputFileInfo> InputFileInfosLoaded;
- /// Where this module file keeps each of its input files. We read this out of
- /// the source location entries the first time something asks, since only a
- /// module write needs it.
+ /// Where this module file keeps each input file. Built from source location
+ /// entries on first use.
std::vector<InputFileLoc> InputFileLocsLoaded;
bool InputFileLocsLoadedBuilt = false;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 3b092dd9561fb..208a5f88f9f93 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1866,22 +1866,20 @@ ASTReader::readSLocFileEntry(ModuleFile *F, unsigned Index) {
void ASTReader::buildLoadedInputFiles() {
LoadedInputFilesBuilt = true;
- // ModuleManager hands modules out in index order, so the copy we settle on
- // for a file does not depend on the order things happened to be loaded in.
+ // ModuleManager iterates modules in index order, so the copy chosen for a
+ // file does not depend on module load order.
for (ModuleFile &F : ModuleMgr) {
for (unsigned I = 0, N = F.InputFilesLoaded.size(); I != N; ++I) {
InputFileInfo FI = getInputFileInfo(F, I + 1);
if (FI.UnresolvedImportedFilename.empty())
continue;
- // An overridden input holds a buffer rather than the contents of the
- // path it names, so its path and size describe nothing we can match on.
+ // An overridden input holds a buffer rather than the file named by its
+ // path, so its path and size cannot identify matching contents.
if (FI.Overridden)
continue;
auto Filename =
ResolveImportedPath(PathBuf, FI.UnresolvedImportedFilename, F);
- // Both sides of a comparison have to spell a path the same way, so make
- // it absolute and drop any dot segments. This works on the string alone
- // and reads nothing from the file system.
+ // Make both paths absolute and remove dot segments before comparing them.
SmallString<128> Key(*Filename);
FileMgr.makeAbsolutePath(Key, /*Canonicalize=*/true);
LoadedInputFiles[Key].push_back({FI.StoredSize, &F, I + 1});
@@ -1896,8 +1894,8 @@ InputFileLoc ASTReader::getLoadedInputFileLoc(ModuleFile &F, unsigned InputID) {
for (unsigned I = 0; I != F.LocalNumSLocEntries; ++I) {
Expected<SLocEntryInfo> MaybeInfo = readSLocFileEntry(&F, I);
if (!MaybeInfo) {
- // Losing an entry only costs us a redirect, so leave the file to the
- // module that is writing it rather than failing the write.
+ // Failing to find an entry only prevents a redirect, so leave the file
+ // local rather than failing the write.
consumeError(MaybeInfo.takeError());
continue;
}
@@ -1930,9 +1928,8 @@ InputFileLoc ASTReader::getLoadedFileLoc(StringRef Path, off_t Size) {
for (const LoadedInputFile &In : Known->second) {
if (In.Size != Size)
continue;
- // A module that has the file as an input may still have left its source
- // location entries out, in which case it has no copy to point at and we
- // keep looking.
+ // An input file may have no source location entries, leaving no copy to
+ // redirect to.
InputFileLoc Loc = getLoadedInputFileLoc(*In.F, In.InputID);
if (Loc.FID.isValid())
return Loc;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 4965e4810e45a..620ee0913136d 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -5574,11 +5574,9 @@ void ASTWriter::computeNonAffectingInputFiles() {
auto AffectingModuleMaps = GetAffectingModuleMaps(*PP, WritingModule);
- // Unlike a SourceLocation, a FileID is written as an index into our own SLoc
- // table, so it cannot name a file we leave out. Collect the files something
- // still refers to by FileID, mirroring the conditions under which the loops
- // that write these tables emit one. Only local files can be named that way,
- // and we skip the invalid FileID so it never becomes a key here.
+ // A FileID is serialized as an index into this module's SLoc table, so
+ // collect the local files named by records written below. The invalid FileID
+ // cannot be stored in a DenseSet, so skip it.
llvm::DenseSet<FileID> NamedFileIDs;
if (SrcMgr.getMainFileID().isValid())
NamedFileIDs.insert(SrcMgr.getMainFileID());
@@ -5599,16 +5597,15 @@ void ASTWriter::computeNonAffectingInputFiles() {
NonAffectingFileIDAdjustments.push_back(FileIDAdjustment);
NonAffectingOffsetAdjustments.push_back(OffsetAdjustment);
- // Leaves \p FID out of this module file. A nonzero \p RedirectAdjustment
- // points its locations at a loaded copy instead.
+ // Leaves \p FID out of this module. A nonzero \p RedirectAdjustment redirects
+ // its locations to a loaded copy.
auto MarkNonAffecting = [&](FileID FID, int64_t RedirectAdjustment) {
FileIDAdjustment += 1;
// Even empty files take up one element in the offset table.
OffsetAdjustment += SrcMgr.getFileIDSize(FID) + 1;
- // If the previous file was non-affecting as well, just extend its entry
- // with our information. Files that point at different copies must stay in
- // separate ranges, since we keep one redirect per range.
+ // Adjacent files with the same redirect can share a range. Files redirected
+ // to different copies need separate ranges.
if (!NonAffectingFileIDs.empty() &&
NonAffectingFileIDs.back().ID == FID.ID - 1 &&
NonAffectingRedirectAdjustments.back() == RedirectAdjustment) {
@@ -5644,14 +5641,12 @@ void ASTWriter::computeNonAffectingInputFiles() {
if (!AffectingModuleMaps)
continue;
- // Don't prune module maps that are affecting. The submodule block names
- // them by FileID when an inferred module was uniqued by one, so they
- // cannot be redirected either.
+ // Affecting module maps may be named by FileID in the submodule block, so
+ // they cannot be redirected.
if (AffectingModuleMaps->DefinitionFileIDs.contains(FID))
continue;
- // A module map nothing points into can be left out along with its
- // locations.
+ // A module map with no affecting locations can be left out.
IsSLocAffecting[I] = false;
IsSLocFileEntryAffecting[I] =
AffectingModuleMaps->DefinitionFiles.contains(*Cache->OrigEntry);
@@ -5662,12 +5657,8 @@ void ASTWriter::computeNonAffectingInputFiles() {
if (NamedFileIDs.contains(FID))
continue;
- // A module we import may already have this input file. If it does, we
- // point our locations at its copy instead of writing a second set of
- // entries for the same text. The input file record is still written, so
- // validation continues to work. We ask by path and size, which a module
- // records for every input it has, so the answer comes out of what it
- // wrote and its own entries stay untouched.
+ // Reuse the source location entries of a loaded module that already has
+ // this input file.
if (!hasChain())
continue;
serialization::InputFileLoc Loaded = getChain()->getLoadedFileLoc(
@@ -6205,8 +6196,8 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema *SemaPtr, StringRef isysroot,
// Write the control block
WriteControlBlock(*PP, isysroot);
- // The import locations in the control block had to stay local. Now that it
- // has been written, we can start rewriting.
+ // Import locations in the control block must remain local, so start rewriting
+ // only after it has been written.
ControlBlockWritten = true;
// Write the remaining AST contents.
@@ -6875,10 +6866,8 @@ SourceLocation ASTWriter::getRedirectedLocation(SourceLocation Loc) const {
SourceLocation ASTWriter::getAdjustedLocation(SourceLocation Loc) const {
if (Loc.isInvalid())
return Loc;
- // A location in a file we left out must move to the loaded copy first, since
- // the shift below only handles locations that are still local. This stays out
- // of getAdjustment because getAdjustedOffset shares it, and we call that on
- // file sizes and on the next local offset as well.
+ // Redirect locations in omitted files before adjusting local offsets.
+ // getAdjustment() is also used for values that are not source locations.
if (ControlBlockWritten && !Loc.isMacroID())
if (SourceLocation Redirected = getRedirectedLocation(Loc);
Redirected.isValid())
@@ -7281,8 +7270,7 @@ void ASTWriter::associateDeclWithFile(const Decl *D, LocalDeclID ID) {
if (FID.isInvalid())
return;
assert(SM.getSLocEntry(FID).isFile());
- // We don't build a per-file declaration table for a file we left out. The
- // module that has the file already built one.
+ // A redirected file already has its declaration table in the loaded module.
if (!IsSLocAffecting[FID.ID])
return;
diff --git a/clang/test/Modules/reuse-duplicate-input-file.cpp b/clang/test/Modules/reuse-duplicate-input-file.cpp
index bd2caff9be1a1..bacb5077f00d8 100644
--- a/clang/test/Modules/reuse-duplicate-input-file.cpp
+++ b/clang/test/Modules/reuse-duplicate-input-file.cpp
@@ -1,7 +1,5 @@
-// Check that a header included textually by several modules does not allocate
-// extra source location space, and that a header still named by FileID does.
-// This optimization is important for large codebases to avoid running out of
-// source location space.
+// Check that a header included textually by several modules reuses source
+// location entries, while headers named by FileID do not.
// RUN: rm -rf %t && mkdir %t
// RUN: split-file %s %t
@@ -19,9 +17,8 @@
// RUN: -fmodule-map-file=%t/mods.map -fmodule-file=%t/mod3.pcm \
// RUN: -fsyntax-only -verify %t/check_slocs.cc
-// The modules are siblings chained only through -fmodule-file. Including one
-// from the next would carry the include guards along and nothing would be
-// entered textually at all.
+// The modules are chained through -fmodule-file rather than including one
+// another, which would carry include guards along and avoid textual entry.
//--- mods.map
module mod1 { header "mod1.h" export * }
@@ -33,13 +30,13 @@ module mod3 { header "mod3.h" export * }
#pragma clang __debug sloc_usage // expected-remark {{source manager location address space usage}}
// expected-note@* {{% of available space}}
-// shared.h must be entered once for the whole chain. mod2 points at mod1's copy
-// and mod3 must look past mod2, which kept no entries of its own, to find it.
+// shared.h is entered once. mod2 redirects to mod1's copy, and mod3 finds
+// that copy through mod2.
// expected-note at shared.h:1 {{file entered 1 time}}
-// lines.h and diags.h are named by FileID through the line table and through
-// diagnostic state, so each module must keep its own entries for them.
+// lines.h and diags.h are named by FileID through the line table and
+// diagnostic state, so they cannot be redirected.
// expected-note at lines.h:1 {{file entered 3 times}}
// expected-note at diags.h:1 {{file entered 3 times}}
More information about the cfe-commits
mailing list