r371656 - [clang-scan-deps] add skip excluded conditional preprocessor block preprocessing optimization
Alex Lorenz via cfe-commits
cfe-commits at lists.llvm.org
Wed Sep 11 13:40:31 PDT 2019
Author: arphaman
Date: Wed Sep 11 13:40:31 2019
New Revision: 371656
URL: http://llvm.org/viewvc/llvm-project?rev=371656&view=rev
Log:
[clang-scan-deps] add skip excluded conditional preprocessor block preprocessing optimization
This commit adds an optimization to clang-scan-deps and clang's preprocessor that skips excluded preprocessor
blocks by bumping the lexer pointer, and not lexing the tokens until reaching appropriate #else/#endif directive.
The skip positions and lexer offsets are computed when the file is minimized, directly from the minimized tokens.
On an 18-core iMacPro with macOS Catalina Beta I got 10-15% speed-up from this optimization when running clang-scan-deps on
the compilation database for a recent LLVM and Clang (3511 files).
Differential Revision: https://reviews.llvm.org/D67127
Added:
cfe/trunk/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h
Modified:
cfe/trunk/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
cfe/trunk/include/clang/Lex/Lexer.h
cfe/trunk/include/clang/Lex/Preprocessor.h
cfe/trunk/include/clang/Lex/PreprocessorOptions.h
cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
cfe/trunk/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
cfe/trunk/lib/Lex/Lexer.cpp
cfe/trunk/lib/Lex/PPDirectives.cpp
cfe/trunk/lib/Lex/Preprocessor.cpp
cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
cfe/trunk/test/ClangScanDeps/regular_cdb.cpp
cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp
cfe/trunk/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
Modified: cfe/trunk/include/clang/Lex/DependencyDirectivesSourceMinimizer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/DependencyDirectivesSourceMinimizer.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/DependencyDirectivesSourceMinimizer.h (original)
+++ cfe/trunk/include/clang/Lex/DependencyDirectivesSourceMinimizer.h Wed Sep 11 13:40:31 2019
@@ -66,6 +66,24 @@ struct Token {
Token(TokenKind K, int Offset) : K(K), Offset(Offset) {}
};
+/// Simplified token range to track the range of a potentially skippable PP
+/// directive.
+struct SkippedRange {
+ /// Offset into the output byte stream of where the skipped directive begins.
+ int Offset;
+
+ /// The number of bytes that can be skipped before the preprocessing must
+ /// resume.
+ int Length;
+};
+
+/// Computes the potential source ranges that can be skipped by the preprocessor
+/// when skipping a directive like #if, #ifdef or #elsif.
+///
+/// \returns false on success, true on error.
+bool computeSkippedRanges(ArrayRef<Token> Input,
+ llvm::SmallVectorImpl<SkippedRange> &Range);
+
} // end namespace minimize_source_to_dependency_directives
/// Minimize the input down to the preprocessor directives that might have
Modified: cfe/trunk/include/clang/Lex/Lexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Lexer.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Lexer.h (original)
+++ cfe/trunk/include/clang/Lex/Lexer.h Wed Sep 11 13:40:31 2019
@@ -265,6 +265,21 @@ public:
/// Return the current location in the buffer.
const char *getBufferLocation() const { return BufferPtr; }
+ /// Returns the current lexing offset.
+ unsigned getCurrentBufferOffset() {
+ assert(BufferPtr >= BufferStart && "Invalid buffer state");
+ return BufferPtr - BufferStart;
+ }
+
+ /// Skip over \p NumBytes bytes.
+ ///
+ /// If the skip is successful, the next token will be lexed from the new
+ /// offset. The lexer also assumes that we skipped to the start of the line.
+ ///
+ /// \returns true if the skip failed (new offset would have been past the
+ /// end of the buffer), false otherwise.
+ bool skipOver(unsigned NumBytes);
+
/// Stringify - Convert the specified string into a C string by i) escaping
/// '\\' and " characters and ii) replacing newline character(s) with "\\n".
/// If Charify is true, this escapes the ' character instead of ".
Modified: cfe/trunk/include/clang/Lex/Preprocessor.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/Preprocessor.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/Preprocessor.h (original)
+++ cfe/trunk/include/clang/Lex/Preprocessor.h Wed Sep 11 13:40:31 2019
@@ -28,6 +28,7 @@
#include "clang/Lex/ModuleLoader.h"
#include "clang/Lex/ModuleMap.h"
#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
#include "clang/Lex/Token.h"
#include "clang/Lex/TokenLexer.h"
#include "llvm/ADT/ArrayRef.h"
@@ -2320,6 +2321,15 @@ public:
/// A macro is used, update information about macros that need unused
/// warnings.
void markMacroAsUsed(MacroInfo *MI);
+
+private:
+ Optional<unsigned>
+ getSkippedRangeForExcludedConditionalBlock(SourceLocation HashLoc);
+
+ /// Contains the currently active skipped range mappings for skipping excluded
+ /// conditional directives.
+ ExcludedPreprocessorDirectiveSkipMapping
+ *ExcludedConditionalDirectiveSkipMappings;
};
/// Abstract base class that describes a handler that will receive
Added: cfe/trunk/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h?rev=371656&view=auto
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h (added)
+++ cfe/trunk/include/clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h Wed Sep 11 13:40:31 2019
@@ -0,0 +1,31 @@
+//===- PreprocessorExcludedConditionalDirectiveSkipMapping.h - --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_PREPROCESSOR_EXCLUDED_COND_DIRECTIVE_SKIP_MAPPING_H
+#define LLVM_CLANG_LEX_PREPROCESSOR_EXCLUDED_COND_DIRECTIVE_SKIP_MAPPING_H
+
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace clang {
+
+/// A mapping from an offset into a buffer to the number of bytes that can be
+/// skipped by the preprocessor when skipping over excluded conditional
+/// directive ranges.
+using PreprocessorSkippedRangeMapping = llvm::DenseMap<unsigned, unsigned>;
+
+/// The datastructure that holds the mapping between the active memory buffers
+/// and the individual skip mappings.
+using ExcludedPreprocessorDirectiveSkipMapping =
+ llvm::DenseMap<const llvm::MemoryBuffer *,
+ const PreprocessorSkippedRangeMapping *>;
+
+} // end namespace clang
+
+#endif // LLVM_CLANG_LEX_PREPROCESSOR_EXCLUDED_COND_DIRECTIVE_SKIP_MAPPING_H
Modified: cfe/trunk/include/clang/Lex/PreprocessorOptions.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Lex/PreprocessorOptions.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Lex/PreprocessorOptions.h (original)
+++ cfe/trunk/include/clang/Lex/PreprocessorOptions.h Wed Sep 11 13:40:31 2019
@@ -10,6 +10,7 @@
#define LLVM_CLANG_LEX_PREPROCESSOROPTIONS_H_
#include "clang/Basic/LLVM.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSet.h"
#include <memory>
@@ -172,6 +173,14 @@ public:
/// build it again.
std::shared_ptr<FailedModulesSet> FailedModules;
+ /// Contains the currently active skipped range mappings for skipping excluded
+ /// conditional directives.
+ ///
+ /// The pointer is passed to the Preprocessor when it's constructed. The
+ /// pointer is unowned, the client is responsible for its lifetime.
+ ExcludedPreprocessorDirectiveSkipMapping
+ *ExcludedConditionalDirectiveSkipMappings = nullptr;
+
public:
PreprocessorOptions() : PrecompiledPreambleBytes(0, false) {}
Modified: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h (original)
+++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h Wed Sep 11 13:40:31 2019
@@ -10,6 +10,7 @@
#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
#include "clang/Basic/LLVM.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/Support/Allocator.h"
@@ -76,6 +77,12 @@ public:
return MaybeStat->getName();
}
+ /// Return the mapping between location -> distance that is used to speed up
+ /// the block skipping in the preprocessor.
+ const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
+ return PPSkippedRangeMapping;
+ }
+
CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
@@ -89,6 +96,7 @@ private:
// Note: small size of 1 allows us to store an empty string with an implicit
// null terminator without any allocations.
llvm::SmallString<1> Contents;
+ PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
};
/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
@@ -133,8 +141,10 @@ class DependencyScanningWorkerFilesystem
public:
DependencyScanningWorkerFilesystem(
DependencyScanningFilesystemSharedCache &SharedCache,
- IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
- : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {}
+ IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+ ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
+ : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
+ PPSkipMappings(PPSkipMappings) {}
llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
@@ -159,6 +169,10 @@ private:
/// The local cache is used by the worker thread to cache file system queries
/// locally instead of querying the global cache every time.
llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+ /// The optional mapping structure which records information about the
+ /// excluded conditional directive skip mappings that are used by the
+ /// currently active preprocessor.
+ ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
};
} // end namespace dependencies
Modified: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h (original)
+++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningService.h Wed Sep 11 13:40:31 2019
@@ -34,12 +34,15 @@ enum class ScanningMode {
/// the invidual dependency scanning workers.
class DependencyScanningService {
public:
- DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true);
+ DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true,
+ bool SkipExcludedPPRanges = true);
ScanningMode getMode() const { return Mode; }
bool canReuseFileManager() const { return ReuseFileManager; }
+ bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
+
DependencyScanningFilesystemSharedCache &getSharedCache() {
return SharedCache;
}
@@ -47,6 +50,10 @@ public:
private:
const ScanningMode Mode;
const bool ReuseFileManager;
+ /// Set to true to use the preprocessor optimization that skips excluded PP
+ /// ranges by bumping the buffer pointer in the lexer instead of lexing the
+ /// tokens in the range until reaching the corresponding directive.
+ const bool SkipExcludedPPRanges;
/// The global file system cache.
DependencyScanningFilesystemSharedCache SharedCache;
};
Modified: cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h (original)
+++ cfe/trunk/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h Wed Sep 11 13:40:31 2019
@@ -13,6 +13,7 @@
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LLVM.h"
#include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
#include "clang/Tooling/CompilationDatabase.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
@@ -62,6 +63,7 @@ public:
private:
IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+ std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
/// The file system that is used by each worker when scanning for
Modified: cfe/trunk/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/DependencyDirectivesSourceMinimizer.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/DependencyDirectivesSourceMinimizer.cpp (original)
+++ cfe/trunk/lib/Lex/DependencyDirectivesSourceMinimizer.cpp Wed Sep 11 13:40:31 2019
@@ -865,6 +865,54 @@ bool Minimizer::minimize() {
return Error;
}
+bool clang::minimize_source_to_dependency_directives::computeSkippedRanges(
+ ArrayRef<Token> Input, llvm::SmallVectorImpl<SkippedRange> &Range) {
+ struct Directive {
+ enum DirectiveKind {
+ If, // if/ifdef/ifndef
+ Else // elif,else
+ };
+ int Offset;
+ DirectiveKind Kind;
+ };
+ llvm::SmallVector<Directive, 32> Offsets;
+ for (const Token &T : Input) {
+ switch (T.K) {
+ case pp_if:
+ case pp_ifdef:
+ case pp_ifndef:
+ Offsets.push_back({T.Offset, Directive::If});
+ break;
+
+ case pp_elif:
+ case pp_else: {
+ if (Offsets.empty())
+ return true;
+ int PreviousOffset = Offsets.back().Offset;
+ Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
+ Offsets.push_back({T.Offset, Directive::Else});
+ break;
+ }
+
+ case pp_endif: {
+ if (Offsets.empty())
+ return true;
+ int PreviousOffset = Offsets.back().Offset;
+ Range.push_back({PreviousOffset, T.Offset - PreviousOffset});
+ do {
+ Directive::DirectiveKind Kind = Offsets.pop_back_val().Kind;
+ if (Kind == Directive::If)
+ break;
+ } while (!Offsets.empty());
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ return false;
+}
+
bool clang::minimizeSourceToDependencyDirectives(
StringRef Input, SmallVectorImpl<char> &Output,
SmallVectorImpl<Token> &Tokens, DiagnosticsEngine *Diags,
Modified: cfe/trunk/lib/Lex/Lexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Lexer.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Lexer.cpp (original)
+++ cfe/trunk/lib/Lex/Lexer.cpp Wed Sep 11 13:40:31 2019
@@ -218,6 +218,15 @@ Lexer *Lexer::Create_PragmaLexer(SourceL
return L;
}
+bool Lexer::skipOver(unsigned NumBytes) {
+ IsAtPhysicalStartOfLine = true;
+ IsAtStartOfLine = true;
+ if ((BufferPtr + NumBytes) > BufferEnd)
+ return true;
+ BufferPtr += NumBytes;
+ return false;
+}
+
template <typename T> static void StringifyImpl(T &Str, char Quote) {
typename T::size_type i = 0, e = Str.size();
while (i < e) {
Modified: cfe/trunk/lib/Lex/PPDirectives.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/PPDirectives.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/PPDirectives.cpp (original)
+++ cfe/trunk/lib/Lex/PPDirectives.cpp Wed Sep 11 13:40:31 2019
@@ -370,6 +370,37 @@ SourceLocation Preprocessor::CheckEndOfD
return DiscardUntilEndOfDirective().getEnd();
}
+Optional<unsigned> Preprocessor::getSkippedRangeForExcludedConditionalBlock(
+ SourceLocation HashLoc) {
+ if (!ExcludedConditionalDirectiveSkipMappings)
+ return None;
+ if (!HashLoc.isFileID())
+ return None;
+
+ std::pair<FileID, unsigned> HashFileOffset =
+ SourceMgr.getDecomposedLoc(HashLoc);
+ const llvm::MemoryBuffer *Buf = SourceMgr.getBuffer(HashFileOffset.first);
+ auto It = ExcludedConditionalDirectiveSkipMappings->find(Buf);
+ if (It == ExcludedConditionalDirectiveSkipMappings->end())
+ return None;
+
+ const PreprocessorSkippedRangeMapping &SkippedRanges = *It->getSecond();
+ // Check if the offset of '#' is mapped in the skipped ranges.
+ auto MappingIt = SkippedRanges.find(HashFileOffset.second);
+ if (MappingIt == SkippedRanges.end())
+ return None;
+
+ unsigned BytesToSkip = MappingIt->getSecond();
+ unsigned CurLexerBufferOffset = CurLexer->getCurrentBufferOffset();
+ assert(CurLexerBufferOffset >= HashFileOffset.second &&
+ "lexer is before the hash?");
+ // Take into account the fact that the lexer has already advanced, so the
+ // number of bytes to skip must be adjusted.
+ unsigned LengthDiff = CurLexerBufferOffset - HashFileOffset.second;
+ assert(BytesToSkip >= LengthDiff && "lexer is after the skipped range?");
+ return BytesToSkip - LengthDiff;
+}
+
/// SkipExcludedConditionalBlock - We just read a \#if or related directive and
/// decided that the subsequent tokens are in the \#if'd out portion of the
/// file. Lex the rest of the file, until we see an \#endif. If
@@ -396,6 +427,11 @@ void Preprocessor::SkipExcludedCondition
// disabling warnings, etc.
CurPPLexer->LexingRawMode = true;
Token Tok;
+ if (auto SkipLength =
+ getSkippedRangeForExcludedConditionalBlock(HashTokenLoc)) {
+ // Skip to the next '#endif' / '#else' / '#elif'.
+ CurLexer->skipOver(*SkipLength);
+ }
while (true) {
CurLexer->Lex(Tok);
Modified: cfe/trunk/lib/Lex/Preprocessor.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Lex/Preprocessor.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Lex/Preprocessor.cpp (original)
+++ cfe/trunk/lib/Lex/Preprocessor.cpp Wed Sep 11 13:40:31 2019
@@ -158,6 +158,11 @@ Preprocessor::Preprocessor(std::shared_p
if (this->PPOpts->GeneratePreamble)
PreambleConditionalStack.startRecording();
+
+ ExcludedConditionalDirectiveSkipMappings =
+ this->PPOpts->ExcludedConditionalDirectiveSkipMappings;
+ if (ExcludedConditionalDirectiveSkipMappings)
+ ExcludedConditionalDirectiveSkipMappings->clear();
}
Preprocessor::~Preprocessor() {
Modified: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp (original)
+++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp Wed Sep 11 13:40:31 2019
@@ -69,6 +69,25 @@ CachedFileSystemEntry CachedFileSystemEn
// Now make the null terminator implicit again, so that Clang's lexer can find
// it right where the buffer ends.
Result.Contents.pop_back();
+
+ // Compute the skipped PP ranges that speedup skipping over inactive
+ // preprocessor blocks.
+ llvm::SmallVector<minimize_source_to_dependency_directives::SkippedRange, 32>
+ SkippedRanges;
+ minimize_source_to_dependency_directives::computeSkippedRanges(Tokens,
+ SkippedRanges);
+ PreprocessorSkippedRangeMapping Mapping;
+ for (const auto &Range : SkippedRanges) {
+ if (Range.Length < 16) {
+ // Ignore small ranges as non-profitable.
+ // FIXME: This is a heuristic, its worth investigating the tradeoffs
+ // when it should be applied.
+ continue;
+ }
+ Mapping[Range.Offset] = Range.Length;
+ }
+ Result.PPSkippedRangeMapping = std::move(Mapping);
+
return Result;
}
@@ -172,14 +191,19 @@ private:
};
llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
-createFile(const CachedFileSystemEntry *Entry) {
+createFile(const CachedFileSystemEntry *Entry,
+ ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
llvm::ErrorOr<StringRef> Contents = Entry->getContents();
if (!Contents)
return Contents.getError();
- return std::make_unique<MinimizedVFSFile>(
+ auto Result = std::make_unique<MinimizedVFSFile>(
llvm::MemoryBuffer::getMemBuffer(*Contents, Entry->getName(),
/*RequiresNullTerminator=*/false),
*Entry->getStatus());
+ if (!Entry->getPPSkippedRangeMapping().empty() && PPSkipMappings)
+ (*PPSkipMappings)[Result->getBufferPtr()] =
+ &Entry->getPPSkippedRangeMapping();
+ return Result;
}
} // end anonymous namespace
@@ -191,7 +215,7 @@ DependencyScanningWorkerFilesystem::open
// Check the local cache first.
if (const CachedFileSystemEntry *Entry = getCachedEntry(Filename))
- return createFile(Entry);
+ return createFile(Entry, PPSkipMappings);
// FIXME: Handle PCM/PCH files.
// FIXME: Handle module map files.
@@ -214,5 +238,5 @@ DependencyScanningWorkerFilesystem::open
// Store the result in the local cache.
setCachedEntry(Filename, Result);
- return createFile(Result);
+ return createFile(Result, PPSkipMappings);
}
Modified: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp (original)
+++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningService.cpp Wed Sep 11 13:40:31 2019
@@ -13,5 +13,7 @@ using namespace tooling;
using namespace dependencies;
DependencyScanningService::DependencyScanningService(ScanningMode Mode,
- bool ReuseFileManager)
- : Mode(Mode), ReuseFileManager(ReuseFileManager) {}
+ bool ReuseFileManager,
+ bool SkipExcludedPPRanges)
+ : Mode(Mode), ReuseFileManager(ReuseFileManager),
+ SkipExcludedPPRanges(SkipExcludedPPRanges) {}
Modified: cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp (original)
+++ cfe/trunk/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp Wed Sep 11 13:40:31 2019
@@ -12,6 +12,7 @@
#include "clang/Frontend/FrontendActions.h"
#include "clang/Frontend/TextDiagnosticPrinter.h"
#include "clang/Frontend/Utils.h"
+#include "clang/Lex/PreprocessorOptions.h"
#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
#include "clang/Tooling/Tooling.h"
@@ -66,9 +67,10 @@ class DependencyScanningAction : public
public:
DependencyScanningAction(
StringRef WorkingDirectory, DependencyConsumer &Consumer,
- llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS)
+ llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
+ ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
: WorkingDirectory(WorkingDirectory), Consumer(Consumer),
- DepFS(std::move(DepFS)) {}
+ DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *FileMgr,
@@ -101,6 +103,12 @@ public:
// filesystem.
FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
CI, Compiler.getDiagnostics(), DepFS));
+
+ // Pass the skip mappings which should speed up excluded conditional block
+ // skipping in the preprocessor.
+ if (PPSkipMappings)
+ Compiler.getPreprocessorOpts()
+ .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
}
FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory;
@@ -134,6 +142,7 @@ private:
StringRef WorkingDirectory;
DependencyConsumer &Consumer;
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
+ ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
};
} // end anonymous namespace
@@ -143,9 +152,12 @@ DependencyScanningWorker::DependencyScan
DiagOpts = new DiagnosticOptions();
PCHContainerOps = std::make_shared<PCHContainerOperations>();
RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
+ if (Service.canSkipExcludedPPRanges())
+ PPSkipMappings =
+ std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
- DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
- RealFS);
+ DepFS = new DependencyScanningWorkerFilesystem(
+ Service.getSharedCache(), RealFS, PPSkipMappings.get());
if (Service.canReuseFileManager())
Files = new FileManager(FileSystemOptions(), RealFS);
}
@@ -178,7 +190,8 @@ llvm::Error DependencyScanningWorker::co
Tool.setRestoreWorkingDir(false);
Tool.setPrintErrorMessage(false);
Tool.setDiagnosticConsumer(&DC);
- DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS);
+ DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
+ PPSkipMappings.get());
return !Tool.run(&Action);
});
}
Modified: cfe/trunk/test/ClangScanDeps/regular_cdb.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/regular_cdb.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/test/ClangScanDeps/regular_cdb.cpp (original)
+++ cfe/trunk/test/ClangScanDeps/regular_cdb.cpp Wed Sep 11 13:40:31 2019
@@ -12,6 +12,8 @@
// RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess | \
// RUN: FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -mode preprocess-minimized-sources \
+// RUN: -skip-excluded-pp-ranges=0 | FileCheck --check-prefixes=CHECK1,CHECK2,CHECK2NO %s
//
// Make sure we didn't produce any dependency files!
// RUN: not cat %t.dir/regular_cdb.d
Modified: cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp (original)
+++ cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp Wed Sep 11 13:40:31 2019
@@ -168,6 +168,14 @@ llvm::cl::opt<bool> ReuseFileManager(
llvm::cl::desc("Reuse the file manager and its cache between invocations."),
llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+llvm::cl::opt<bool> SkipExcludedPPRanges(
+ "skip-excluded-pp-ranges",
+ llvm::cl::desc(
+ "Use the preprocessor optimization that skips excluded conditionals by "
+ "bumping the buffer pointer in the lexer instead of lexing the tokens "
+ "until reaching the end directive."),
+ llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+
} // end anonymous namespace
int main(int argc, const char **argv) {
@@ -214,7 +222,8 @@ int main(int argc, const char **argv) {
// Print out the dependency results to STDOUT by default.
SharedStream DependencyOS(llvm::outs());
- DependencyScanningService Service(ScanMode, ReuseFileManager);
+ DependencyScanningService Service(ScanMode, ReuseFileManager,
+ SkipExcludedPPRanges);
#if LLVM_ENABLE_THREADS
unsigned NumWorkers =
NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
Modified: cfe/trunk/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp?rev=371656&r1=371655&r2=371656&view=diff
==============================================================================
--- cfe/trunk/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp (original)
+++ cfe/trunk/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp Wed Sep 11 13:40:31 2019
@@ -617,4 +617,46 @@ ort \
minimize_source_to_dependency_directives::cxx_module_decl);
}
+TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesBasic) {
+ SmallString<128> Out;
+ SmallVector<Token, 32> Toks;
+ StringRef Source = "#ifndef GUARD\n"
+ "#define GUARD\n"
+ "void foo();\n"
+ "#endif\n";
+ ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks));
+ SmallVector<SkippedRange, 4> Ranges;
+ ASSERT_FALSE(computeSkippedRanges(Toks, Ranges));
+ EXPECT_EQ(Ranges.size(), 1u);
+ EXPECT_EQ(Ranges[0].Offset, 0);
+ EXPECT_EQ(Ranges[0].Length, (int)Out.find("#endif"));
+}
+
+TEST(MinimizeSourceToDependencyDirectivesTest, SkippedPPRangesNested) {
+ SmallString<128> Out;
+ SmallVector<Token, 32> Toks;
+ StringRef Source = "#ifndef GUARD\n"
+ "#define GUARD\n"
+ "#if FOO\n"
+ "#include hello\n"
+ "#elif BAR\n"
+ "#include bye\n"
+ "#endif\n"
+ "#else\n"
+ "#include nothing\n"
+ "#endif\n";
+ ASSERT_FALSE(minimizeSourceToDependencyDirectives(Source, Out, Toks));
+ SmallVector<SkippedRange, 4> Ranges;
+ ASSERT_FALSE(computeSkippedRanges(Toks, Ranges));
+ EXPECT_EQ(Ranges.size(), 4u);
+ EXPECT_EQ(Ranges[0].Offset, (int)Out.find("#if FOO"));
+ EXPECT_EQ(Ranges[0].Offset + Ranges[0].Length, (int)Out.find("#elif"));
+ EXPECT_EQ(Ranges[1].Offset, (int)Out.find("#elif BAR"));
+ EXPECT_EQ(Ranges[1].Offset + Ranges[1].Length, (int)Out.find("#endif"));
+ EXPECT_EQ(Ranges[2].Offset, 0);
+ EXPECT_EQ(Ranges[2].Length, (int)Out.find("#else"));
+ EXPECT_EQ(Ranges[3].Offset, (int)Out.find("#else"));
+ EXPECT_EQ(Ranges[3].Offset + Ranges[3].Length, (int)Out.rfind("#endif"));
+}
+
} // end anonymous namespace
More information about the cfe-commits
mailing list