r322503 - [PCH] Serialize skipped preprocessor ranges
Cameron Desrochers via cfe-commits
cfe-commits at lists.llvm.org
Mon Jan 15 11:14:17 PST 2018
Author: cameron314
Date: Mon Jan 15 11:14:16 2018
New Revision: 322503
URL: http://llvm.org/viewvc/llvm-project?rev=322503&view=rev
Log:
[PCH] Serialize skipped preprocessor ranges
The skipped preprocessor ranges are now serialized in the AST PCH file. This fixes, for example, libclang's clang_getSkippedRanges() returning zero ranges after reparsing a translation unit.
Differential Revision: https://reviews.llvm.org/D20124
Modified:
cfe/trunk/include/clang/Lex/PreprocessingRecord.h
cfe/trunk/include/clang/Serialization/ASTBitCodes.h
cfe/trunk/include/clang/Serialization/ASTReader.h
cfe/trunk/include/clang/Serialization/Module.h
cfe/trunk/lib/Lex/PPDirectives.cpp
cfe/trunk/lib/Lex/PreprocessingRecord.cpp
cfe/trunk/lib/Serialization/ASTReader.cpp
cfe/trunk/lib/Serialization/ASTWriter.cpp
cfe/trunk/tools/libclang/CIndex.cpp
cfe/trunk/unittests/libclang/LibclangTest.cpp
Modified: cfe/trunk/include/clang/Lex/PreprocessingRecord.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessingRecord.h?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessingRecord.h (original)
+++ cfe/trunk/include/clang/Lex/PreprocessingRecord.h Mon Jan 15 11:14:16 2018
@@ -297,6 +297,9 @@ class Token;
FileID FID) {
return None;
}
+
+ /// \brief Read a preallocated skipped range from the external source.
+ virtual SourceRange ReadSkippedRange(unsigned Index) = 0;
};
/// \brief A record of the steps taken while preprocessing a source file,
@@ -322,6 +325,8 @@ class Token;
/// \brief The set of ranges that were skipped by the preprocessor,
std::vector<SourceRange> SkippedRanges;
+ bool SkippedRangesAllLoaded = true;
+
/// \brief Global (loaded or local) ID for a preprocessed entity.
/// Negative values are used to indicate preprocessed entities
/// loaded from the external source while non-negative values are used to
@@ -377,6 +382,16 @@ class Token;
/// corresponds to the first newly-allocated entity.
unsigned allocateLoadedEntities(unsigned NumEntities);
+ /// \brief Allocate space for a new set of loaded preprocessed skipped
+ /// ranges.
+ ///
+ /// \returns The index into the set of loaded preprocessed ranges, which
+ /// corresponds to the first newly-allocated range.
+ unsigned allocateSkippedRanges(unsigned NumRanges);
+
+ /// \brief Ensures that all external skipped ranges have been loaded.
+ void ensureSkippedRangesLoaded();
+
/// \brief Register a new macro definition.
void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinitionRecord *Def);
@@ -499,7 +514,8 @@ class Token;
MacroDefinitionRecord *findMacroDefinition(const MacroInfo *MI);
/// \brief Retrieve all ranges that got skipped while preprocessing.
- const std::vector<SourceRange> &getSkippedRanges() const {
+ const std::vector<SourceRange> &getSkippedRanges() {
+ ensureSkippedRangesLoaded();
return SkippedRanges;
}
Modified: cfe/trunk/include/clang/Serialization/ASTBitCodes.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTBitCodes.h?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/ASTBitCodes.h (original)
+++ cfe/trunk/include/clang/Serialization/ASTBitCodes.h Mon Jan 15 11:14:16 2018
@@ -198,6 +198,25 @@ namespace serialization {
}
};
+ /// \brief Source range of a skipped preprocessor region
+ struct PPSkippedRange {
+ /// \brief Raw source location of beginning of range.
+ unsigned Begin;
+ /// \brief Raw source location of end of range.
+ unsigned End;
+
+ PPSkippedRange(SourceRange R)
+ : Begin(R.getBegin().getRawEncoding()),
+ End(R.getEnd().getRawEncoding()) { }
+
+ SourceLocation getBegin() const {
+ return SourceLocation::getFromRawEncoding(Begin);
+ }
+ SourceLocation getEnd() const {
+ return SourceLocation::getFromRawEncoding(End);
+ }
+ };
+
/// \brief Source range/offset of a preprocessed entity.
struct DeclOffset {
/// \brief Raw source location.
@@ -627,6 +646,9 @@ namespace serialization {
/// \brief The stack of open #ifs/#ifdefs recorded in a preamble.
PP_CONDITIONAL_STACK = 62,
+
+ /// \brief A table of skipped ranges within the preprocessing record.
+ PPD_SKIPPED_RANGES = 63
};
/// \brief Record types used within a source manager block.
Modified: cfe/trunk/include/clang/Serialization/ASTReader.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/ASTReader.h?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/ASTReader.h (original)
+++ cfe/trunk/include/clang/Serialization/ASTReader.h Mon Jan 15 11:14:16 2018
@@ -753,6 +753,13 @@ private:
/// added to the global preprocessing entity ID to produce a local ID.
GlobalPreprocessedEntityMapType GlobalPreprocessedEntityMap;
+ typedef ContinuousRangeMap<unsigned, ModuleFile *, 4>
+ GlobalSkippedRangeMapType;
+
+ /// \brief Mapping from global skipped range base IDs to the module in which
+ /// the skipped ranges reside.
+ GlobalSkippedRangeMapType GlobalSkippedRangeMap;
+
/// \name CodeGen-relevant special data
/// \brief Fields containing data that is relevant to CodeGen.
//@{
@@ -1693,6 +1700,9 @@ public:
Optional<bool> isPreprocessedEntityInFileID(unsigned Index,
FileID FID) override;
+ /// \brief Read a preallocated skipped range from the external source.
+ SourceRange ReadSkippedRange(unsigned Index) override;
+
/// \brief Read the header file information for the given file entry.
HeaderFileInfo GetHeaderFileInfo(const FileEntry *FE) override;
Modified: cfe/trunk/include/clang/Serialization/Module.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Serialization/Module.h?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/include/clang/Serialization/Module.h (original)
+++ cfe/trunk/include/clang/Serialization/Module.h Mon Jan 15 11:14:16 2018
@@ -324,6 +324,12 @@ public:
const PPEntityOffset *PreprocessedEntityOffsets = nullptr;
unsigned NumPreprocessedEntities = 0;
+ /// \brief Base ID for preprocessed skipped ranges local to this module.
+ unsigned BasePreprocessedSkippedRangeID = 0;
+
+ const PPSkippedRange *PreprocessedSkippedRangeOffsets = nullptr;
+ unsigned NumPreprocessedSkippedRanges = 0;
+
// === Header search information ===
/// \brief The number of local HeaderFileInfo structures.
Modified: cfe/trunk/lib/Lex/PPDirectives.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPDirectives.cpp (original)
+++ cfe/trunk/lib/Lex/PPDirectives.cpp Mon Jan 15 11:14:16 2018
@@ -558,7 +558,9 @@ void Preprocessor::SkipExcludedCondition
// the #if block.
CurPPLexer->LexingRawMode = false;
- if (Callbacks)
+ // The last skipped range isn't actually skipped yet if it's truncated
+ // by the end of the preamble; we'll resume parsing after the preamble.
+ if (Callbacks && (Tok.isNot(tok::eof) || !isRecordingPreamble()))
Callbacks->SourceRangeSkipped(
SourceRange(HashTokenLoc, CurPPLexer->getSourceLocation()),
Tok.getLocation());
Modified: cfe/trunk/lib/Lex/PreprocessingRecord.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PreprocessingRecord.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PreprocessingRecord.cpp (original)
+++ cfe/trunk/lib/Lex/PreprocessingRecord.cpp Mon Jan 15 11:14:16 2018
@@ -329,6 +329,23 @@ unsigned PreprocessingRecord::allocateLo
return Result;
}
+unsigned PreprocessingRecord::allocateSkippedRanges(unsigned NumRanges) {
+ unsigned Result = SkippedRanges.size();
+ SkippedRanges.resize(SkippedRanges.size() + NumRanges);
+ SkippedRangesAllLoaded = false;
+ return Result;
+}
+
+void PreprocessingRecord::ensureSkippedRangesLoaded() {
+ if (SkippedRangesAllLoaded || !ExternalSource)
+ return;
+ for (unsigned Index = 0; Index != SkippedRanges.size(); ++Index) {
+ if (SkippedRanges[Index].isInvalid())
+ SkippedRanges[Index] = ExternalSource->ReadSkippedRange(Index);
+ }
+ SkippedRangesAllLoaded = true;
+}
+
void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro,
MacroDefinitionRecord *Def) {
MacroDefinitions[Macro] = Def;
@@ -418,6 +435,7 @@ void PreprocessingRecord::Defined(const
void PreprocessingRecord::SourceRangeSkipped(SourceRange Range,
SourceLocation EndifLoc) {
+ assert(Range.isValid());
SkippedRanges.emplace_back(Range.getBegin(), EndifLoc);
}
@@ -497,5 +515,6 @@ size_t PreprocessingRecord::getTotalMemo
return BumpAlloc.getTotalMemory()
+ llvm::capacity_in_bytes(MacroDefinitions)
+ llvm::capacity_in_bytes(PreprocessedEntities)
- + llvm::capacity_in_bytes(LoadedPreprocessedEntities);
+ + llvm::capacity_in_bytes(LoadedPreprocessedEntities)
+ + llvm::capacity_in_bytes(SkippedRanges);
}
Modified: cfe/trunk/lib/Serialization/ASTReader.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/ASTReader.cpp (original)
+++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Jan 15 11:14:16 2018
@@ -3215,6 +3215,24 @@ ASTReader::ReadASTBlock(ModuleFile &F, u
break;
}
+ case PPD_SKIPPED_RANGES: {
+ F.PreprocessedSkippedRangeOffsets = (const PPSkippedRange*)Blob.data();
+ assert(Blob.size() % sizeof(PPSkippedRange) == 0);
+ F.NumPreprocessedSkippedRanges = Blob.size() / sizeof(PPSkippedRange);
+
+ if (!PP.getPreprocessingRecord())
+ PP.createPreprocessingRecord();
+ if (!PP.getPreprocessingRecord()->getExternalSource())
+ PP.getPreprocessingRecord()->SetExternalSource(*this);
+ F.BasePreprocessedSkippedRangeID = PP.getPreprocessingRecord()
+ ->allocateSkippedRanges(F.NumPreprocessedSkippedRanges);
+
+ if (F.NumPreprocessedSkippedRanges > 0)
+ GlobalSkippedRangeMap.insert(
+ std::make_pair(F.BasePreprocessedSkippedRangeID, &F));
+ break;
+ }
+
case DECL_UPDATE_OFFSETS:
if (Record.size() % 2 != 0) {
Error("invalid DECL_UPDATE_OFFSETS block in AST file");
@@ -5387,6 +5405,20 @@ ASTReader::getModuleFileLevelDecls(Modul
Mod.FileSortedDecls + Mod.NumFileSortedDecls));
}
+SourceRange ASTReader::ReadSkippedRange(unsigned GlobalIndex) {
+ auto I = GlobalSkippedRangeMap.find(GlobalIndex);
+ assert(I != GlobalSkippedRangeMap.end() &&
+ "Corrupted global skipped range map");
+ ModuleFile *M = I->second;
+ unsigned LocalIndex = GlobalIndex - M->BasePreprocessedSkippedRangeID;
+ assert(LocalIndex < M->NumPreprocessedSkippedRanges);
+ PPSkippedRange RawRange = M->PreprocessedSkippedRangeOffsets[LocalIndex];
+ SourceRange Range(TranslateSourceLocation(*M, RawRange.getBegin()),
+ TranslateSourceLocation(*M, RawRange.getEnd()));
+ assert(Range.isValid());
+ return Range;
+}
+
PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
PreprocessedEntityID PPID = Index+1;
std::pair<ModuleFile *, unsigned> PPInfo = getModulePreprocessedEntity(Index);
Modified: cfe/trunk/lib/Serialization/ASTWriter.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTWriter.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/lib/Serialization/ASTWriter.cpp (original)
+++ cfe/trunk/lib/Serialization/ASTWriter.cpp Mon Jan 15 11:14:16 2018
@@ -1104,6 +1104,7 @@ void ASTWriter::WriteBlockInfoBlock() {
RECORD(UNUSED_FILESCOPED_DECLS);
RECORD(PPD_ENTITIES_OFFSETS);
RECORD(VTABLE_USES);
+ RECORD(PPD_SKIPPED_RANGES);
RECORD(REFERENCED_SELECTOR_POOL);
RECORD(TU_UPDATE_LEXICAL);
RECORD(SEMA_DECL_REFS);
@@ -2726,6 +2727,26 @@ void ASTWriter::WritePreprocessorDetail(
Stream.EmitRecordWithBlob(PPEOffsetAbbrev, Record,
bytes(PreprocessedEntityOffsets));
}
+
+ // Write the skipped region table for the preprocessing record.
+ ArrayRef<SourceRange> SkippedRanges = PPRec.getSkippedRanges();
+ if (SkippedRanges.size() > 0) {
+ std::vector<PPSkippedRange> SerializedSkippedRanges;
+ SerializedSkippedRanges.reserve(SkippedRanges.size());
+ for (auto const& Range : SkippedRanges)
+ SerializedSkippedRanges.emplace_back(Range);
+
+ using namespace llvm;
+ auto Abbrev = std::make_shared<BitCodeAbbrev>();
+ Abbrev->Add(BitCodeAbbrevOp(PPD_SKIPPED_RANGES));
+ Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+ unsigned PPESkippedRangeAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
+
+ Record.clear();
+ Record.push_back(PPD_SKIPPED_RANGES);
+ Stream.EmitRecordWithBlob(PPESkippedRangeAbbrev, Record,
+ bytes(SerializedSkippedRanges));
+ }
}
unsigned ASTWriter::getLocalOrImportedSubmoduleID(Module *Mod) {
Modified: cfe/trunk/tools/libclang/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndex.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndex.cpp (original)
+++ cfe/trunk/tools/libclang/CIndex.cpp Mon Jan 15 11:14:16 2018
@@ -8160,6 +8160,7 @@ CXSourceRangeList *clang_getSkippedRange
SourceManager &sm = Ctx.getSourceManager();
FileEntry *fileEntry = static_cast<FileEntry *>(file);
FileID wantedFileID = sm.translateFile(fileEntry);
+ bool isMainFile = wantedFileID == sm.getMainFileID();
const std::vector<SourceRange> &SkippedRanges = ppRec->getSkippedRanges();
std::vector<SourceRange> wantedRanges;
@@ -8167,6 +8168,8 @@ CXSourceRangeList *clang_getSkippedRange
i != ei; ++i) {
if (sm.getFileID(i->getBegin()) == wantedFileID || sm.getFileID(i->getEnd()) == wantedFileID)
wantedRanges.push_back(*i);
+ else if (isMainFile && (astUnit->isInPreambleFileID(i->getBegin()) || astUnit->isInPreambleFileID(i->getEnd())))
+ wantedRanges.push_back(*i);
}
skipped->count = wantedRanges.size();
Modified: cfe/trunk/unittests/libclang/LibclangTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/libclang/LibclangTest.cpp?rev=322503&r1=322502&r2=322503&view=diff
==============================================================================
--- cfe/trunk/unittests/libclang/LibclangTest.cpp (original)
+++ cfe/trunk/unittests/libclang/LibclangTest.cpp Mon Jan 15 11:14:16 2018
@@ -572,3 +572,81 @@ TEST_F(LibclangReparseTest, clang_parseT
EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU));
DisplayDiagnostics();
}
+
+TEST_F(LibclangReparseTest, PreprocessorSkippedRanges) {
+ std::string Header = "header.h", Main = "main.cpp";
+ WriteFile(Header,
+ "#ifdef MANGOS\n"
+ "printf(\"mmm\");\n"
+ "#endif");
+ WriteFile(Main,
+ "#include \"header.h\"\n"
+ "#ifdef GUAVA\n"
+ "#endif\n"
+ "#ifdef KIWIS\n"
+ "printf(\"mmm!!\");\n"
+ "#endif");
+
+ for (int i = 0; i != 3; ++i) {
+ unsigned flags = TUFlags | CXTranslationUnit_PrecompiledPreamble;
+ if (i == 2)
+ flags |= CXTranslationUnit_CreatePreambleOnFirstParse;
+
+ // parse once
+ ClangTU = clang_parseTranslationUnit(Index, Main.c_str(), nullptr, 0,
+ nullptr, 0, flags);
+ if (i != 0) {
+ // reparse
+ ASSERT_TRUE(ReparseTU(0, nullptr /* No unsaved files. */));
+ }
+
+ // Check all ranges are there
+ CXSourceRangeList *Ranges = clang_getAllSkippedRanges(ClangTU);
+ EXPECT_EQ(3U, Ranges->count);
+
+ CXSourceLocation cxl;
+ unsigned line;
+ cxl = clang_getRangeStart(Ranges->ranges[0]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(1U, line);
+ cxl = clang_getRangeEnd(Ranges->ranges[0]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(3U, line);
+
+ cxl = clang_getRangeStart(Ranges->ranges[1]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(2U, line);
+ cxl = clang_getRangeEnd(Ranges->ranges[1]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(3U, line);
+
+ cxl = clang_getRangeStart(Ranges->ranges[2]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(4U, line);
+ cxl = clang_getRangeEnd(Ranges->ranges[2]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(6U, line);
+
+ clang_disposeSourceRangeList(Ranges);
+
+ // Check obtaining ranges by each file works
+ CXFile cxf = clang_getFile(ClangTU, Header.c_str());
+ Ranges = clang_getSkippedRanges(ClangTU, cxf);
+ EXPECT_EQ(1U, Ranges->count);
+ cxl = clang_getRangeStart(Ranges->ranges[0]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(1U, line);
+ clang_disposeSourceRangeList(Ranges);
+
+ cxf = clang_getFile(ClangTU, Main.c_str());
+ Ranges = clang_getSkippedRanges(ClangTU, cxf);
+ EXPECT_EQ(2U, Ranges->count);
+ cxl = clang_getRangeStart(Ranges->ranges[0]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(2U, line);
+ cxl = clang_getRangeStart(Ranges->ranges[1]);
+ clang_getSpellingLocation(cxl, nullptr, &line, nullptr, nullptr);
+ EXPECT_EQ(4U, line);
+ clang_disposeSourceRangeList(Ranges);
+ }
+}
More information about the cfe-commits
mailing list