r304726 - Rather than rejecting attempts to run preprocessor-only actions on AST files,
Evgenii Stepanov via cfe-commits
cfe-commits at lists.llvm.org
Mon Jun 5 13:14:32 PDT 2017
This change leaks memory:
http://lab.llvm.org:8011/builders/sanitizer-x86_64-linux-fast/builds/5453/steps/check-clang%20asan/logs/stdio
On Mon, Jun 5, 2017 at 11:10 AM, Richard Smith via cfe-commits
<cfe-commits at lists.llvm.org> wrote:
> Author: rsmith
> Date: Mon Jun 5 13:10:11 2017
> New Revision: 304726
>
> URL: http://llvm.org/viewvc/llvm-project?rev=304726&view=rev
> Log:
> Rather than rejecting attempts to run preprocessor-only actions on AST files,
> replay the steps taken to create the AST file with the preprocessor-only action
> installed to produce preprocessed output.
>
> This can be used to produce the preprocessed text for an existing .pch or .pcm
> file.
>
> Modified:
> cfe/trunk/include/clang/Basic/LangOptions.h
> cfe/trunk/include/clang/Basic/SourceManager.h
> cfe/trunk/include/clang/Frontend/ASTUnit.h
> cfe/trunk/include/clang/Frontend/FrontendAction.h
> cfe/trunk/lib/Basic/SourceManager.cpp
> cfe/trunk/lib/Frontend/ASTUnit.cpp
> cfe/trunk/lib/Frontend/FrontendAction.cpp
> cfe/trunk/lib/Serialization/ASTReader.cpp
> cfe/trunk/test/Modules/preprocess-module.cpp
>
> Modified: cfe/trunk/include/clang/Basic/LangOptions.h
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/LangOptions.h?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/include/clang/Basic/LangOptions.h (original)
> +++ cfe/trunk/include/clang/Basic/LangOptions.h Mon Jun 5 13:10:11 2017
> @@ -58,6 +58,7 @@ public:
> SOB_Trapping // -ftrapv
> };
>
> + // FIXME: Unify with TUKind.
> enum CompilingModuleKind {
> CMK_None, ///< Not compiling a module interface at all.
> CMK_ModuleMap, ///< Compiling a module from a module map.
>
> Modified: cfe/trunk/include/clang/Basic/SourceManager.h
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/SourceManager.h?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/include/clang/Basic/SourceManager.h (original)
> +++ cfe/trunk/include/clang/Basic/SourceManager.h Mon Jun 5 13:10:11 2017
> @@ -722,6 +722,10 @@ public:
>
> void clearIDTables();
>
> + /// Initialize this source manager suitably to replay the compilation
> + /// described by \p Old. Requires that \p Old outlive \p *this.
> + void initializeForReplay(const SourceManager &Old);
> +
> DiagnosticsEngine &getDiagnostics() const { return Diag; }
>
> FileManager &getFileManager() const { return FileMgr; }
>
> Modified: cfe/trunk/include/clang/Frontend/ASTUnit.h
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/ASTUnit.h?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/include/clang/Frontend/ASTUnit.h (original)
> +++ cfe/trunk/include/clang/Frontend/ASTUnit.h Mon Jun 5 13:10:11 2017
> @@ -51,6 +51,7 @@ class DiagnosticsEngine;
> class FileEntry;
> class FileManager;
> class HeaderSearch;
> +class InputKind;
> class MemoryBufferCache;
> class Preprocessor;
> class PCHContainerOperations;
> @@ -305,9 +306,6 @@ private:
> /// (likely to change while trying to use them).
> bool UserFilesAreVolatile : 1;
>
> - /// \brief The language options used when we load an AST file.
> - LangOptions ASTFileLangOpts;
> -
> static void ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
> ASTUnit &AST, bool CaptureDiagnostics);
>
> @@ -702,6 +700,9 @@ public:
> /// \brief Determine what kind of translation unit this AST represents.
> TranslationUnitKind getTranslationUnitKind() const { return TUKind; }
>
> + /// \brief Determine the input kind this AST unit represents.
> + InputKind getInputKind() const;
> +
> /// \brief A mapping from a file name to the memory buffer that stores the
> /// remapped contents of that file.
> typedef std::pair<std::string, llvm::MemoryBuffer *> RemappedFile;
>
> Modified: cfe/trunk/include/clang/Frontend/FrontendAction.h
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/FrontendAction.h?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/include/clang/Frontend/FrontendAction.h (original)
> +++ cfe/trunk/include/clang/Frontend/FrontendAction.h Mon Jun 5 13:10:11 2017
> @@ -176,10 +176,10 @@ public:
> virtual TranslationUnitKind getTranslationUnitKind() { return TU_Complete; }
>
> /// \brief Does this action support use with PCH?
> - virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); }
> + virtual bool hasPCHSupport() const { return true; }
>
> /// \brief Does this action support use with AST files?
> - virtual bool hasASTFileSupport() const { return !usesPreprocessorOnly(); }
> + virtual bool hasASTFileSupport() const { return true; }
>
> /// \brief Does this action support use with IR files?
> virtual bool hasIRSupport() const { return false; }
>
> Modified: cfe/trunk/lib/Basic/SourceManager.cpp
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/SourceManager.cpp?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/lib/Basic/SourceManager.cpp (original)
> +++ cfe/trunk/lib/Basic/SourceManager.cpp Mon Jun 5 13:10:11 2017
> @@ -345,6 +345,41 @@ void SourceManager::clearIDTables() {
> createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(), 1);
> }
>
> +void SourceManager::initializeForReplay(const SourceManager &Old) {
> + assert(MainFileID.isInvalid() && "expected uninitialized SourceManager");
> +
> + auto CloneContentCache = [&](const ContentCache *Cache) -> ContentCache * {
> + auto *Clone = new (ContentCacheAlloc.Allocate<ContentCache>()) ContentCache;
> + Clone->OrigEntry = Cache->OrigEntry;
> + Clone->ContentsEntry = Cache->ContentsEntry;
> + Clone->BufferOverridden = Cache->BufferOverridden;
> + Clone->IsSystemFile = Cache->IsSystemFile;
> + Clone->IsTransient = Cache->IsTransient;
> + Clone->replaceBuffer(Cache->getRawBuffer(), /*DoNotFree*/true);
> + return Clone;
> + };
> +
> + // Set up our main file ID as a copy of the old source manager's main file.
> + const SLocEntry &OldMainFile = Old.getSLocEntry(Old.getMainFileID());
> + assert(OldMainFile.isFile() && "main file is macro expansion?");
> + setMainFileID(createFileID(
> + CloneContentCache(OldMainFile.getFile().getContentCache()),
> + SourceLocation(), OldMainFile.getFile().getFileCharacteristic(), 0, 0));
> +
> + // Ensure all SLocEntries are loaded from the external source.
> + for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N; ++I)
> + if (!Old.SLocEntryLoaded[I])
> + Old.loadSLocEntry(I, nullptr);
> +
> + // Inherit any content cache data from the old source manager.
> + for (auto &FileInfo : Old.FileInfos) {
> + SrcMgr::ContentCache *&Slot = FileInfos[FileInfo.first];
> + if (Slot)
> + continue;
> + Slot = CloneContentCache(FileInfo.second);
> + }
> +}
> +
> /// getOrCreateContentCache - Create or return a cached ContentCache for the
> /// specified file.
> const ContentCache *
>
> Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ASTUnit.cpp?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/lib/Frontend/ASTUnit.cpp (original)
> +++ cfe/trunk/lib/Frontend/ASTUnit.cpp Mon Jun 5 13:10:11 2017
> @@ -667,6 +667,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
>
> ConfigureDiags(Diags, *AST, CaptureDiagnostics);
>
> + AST->LangOpts = std::make_shared<LangOptions>();
> AST->OnlyLocalDecls = OnlyLocalDecls;
> AST->CaptureDiagnostics = CaptureDiagnostics;
> AST->Diagnostics = Diags;
> @@ -682,7 +683,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
> AST->getSourceManager(),
> AST->getDiagnostics(),
> - AST->ASTFileLangOpts,
> + AST->getLangOpts(),
> /*Target=*/nullptr));
>
> auto PPOpts = std::make_shared<PreprocessorOptions>();
> @@ -696,13 +697,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> unsigned Counter;
>
> AST->PP = std::make_shared<Preprocessor>(
> - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts,
> + std::move(PPOpts), AST->getDiagnostics(), *AST->LangOpts,
> AST->getSourceManager(), *AST->PCMCache, HeaderInfo, *AST,
> /*IILookup=*/nullptr,
> /*OwnsHeaderSearch=*/false);
> Preprocessor &PP = *AST->PP;
>
> - AST->Ctx = new ASTContext(AST->ASTFileLangOpts, AST->getSourceManager(),
> + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(),
> PP.getIdentifierTable(), PP.getSelectorTable(),
> PP.getBuiltinInfo());
> ASTContext &Context = *AST->Ctx;
> @@ -716,7 +717,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> AllowPCHWithCompilerErrors);
>
> AST->Reader->setListener(llvm::make_unique<ASTInfoCollector>(
> - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts, AST->Target,
> + *AST->PP, Context, *AST->LangOpts, AST->TargetOpts, AST->Target,
> Counter));
>
> // Attach the AST reader to the AST context as an external AST
> @@ -2879,7 +2880,32 @@ const FileEntry *ASTUnit::getPCHFile() {
> }
>
> bool ASTUnit::isModuleFile() {
> - return isMainFileAST() && ASTFileLangOpts.isCompilingModule();
> + return isMainFileAST() && getLangOpts().isCompilingModule();
> +}
> +
> +InputKind ASTUnit::getInputKind() const {
> + auto &LangOpts = getLangOpts();
> +
> + InputKind::Language Lang;
> + if (LangOpts.OpenCL)
> + Lang = InputKind::OpenCL;
> + else if (LangOpts.CUDA)
> + Lang = InputKind::CUDA;
> + else if (LangOpts.RenderScript)
> + Lang = InputKind::RenderScript;
> + else if (LangOpts.CPlusPlus)
> + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX;
> + else
> + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C;
> +
> + InputKind::Format Fmt = InputKind::Source;
> + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap)
> + Fmt = InputKind::ModuleMap;
> +
> + // We don't know if input was preprocessed. Assume not.
> + bool PP = false;
> +
> + return InputKind(Lang, Fmt, PP);
> }
>
> void ASTUnit::PreambleData::countLines() const {
>
> Modified: cfe/trunk/lib/Frontend/FrontendAction.cpp
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/FrontendAction.cpp?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/lib/Frontend/FrontendAction.cpp (original)
> +++ cfe/trunk/lib/Frontend/FrontendAction.cpp Mon Jun 5 13:10:11 2017
> @@ -387,8 +387,7 @@ static std::error_code collectModuleHead
> return std::error_code();
> }
>
> -static bool loadModuleMapForModuleBuild(CompilerInstance &CI,
> - StringRef Filename, bool IsSystem,
> +static bool loadModuleMapForModuleBuild(CompilerInstance &CI, bool IsSystem,
> bool IsPreprocessed,
> std::string &PresumedModuleMapFile,
> unsigned &Offset) {
> @@ -523,7 +522,8 @@ getInputBufferForModule(CompilerInstance
> }
>
> bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
> - const FrontendInputFile &Input) {
> + const FrontendInputFile &RealInput) {
> + FrontendInputFile Input(RealInput);
> assert(!Instance && "Already processing a source file!");
> assert(!Input.isEmpty() && "Unexpected empty filename!");
> setCurrentInput(Input);
> @@ -531,15 +531,69 @@ bool FrontendAction::BeginSourceFile(Com
>
> StringRef InputFile = Input.getFile();
> bool HasBegunSourceFile = false;
> + bool ReplayASTFile = Input.getKind().getFormat() == InputKind::Precompiled &&
> + usesPreprocessorOnly();
> if (!BeginInvocation(CI))
> goto failure;
>
> + // If we're replaying the build of an AST file, import it and set up
> + // the initial state from its build.
> + if (ReplayASTFile) {
> + IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics());
> +
> + // The AST unit populates its own diagnostics engine rather than ours.
> + IntrusiveRefCntPtr<DiagnosticsEngine> ASTDiags(
> + new DiagnosticsEngine(Diags->getDiagnosticIDs(),
> + &Diags->getDiagnosticOptions()));
> + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false);
> +
> + std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromASTFile(
> + InputFile, CI.getPCHContainerReader(), ASTDiags, CI.getFileSystemOpts(),
> + CI.getCodeGenOpts().DebugTypeExtRefs);
> + if (!AST)
> + goto failure;
> +
> + // Options relating to how we treat the input (but not what we do with it)
> + // are inherited from the AST unit.
> + CI.getLangOpts() = AST->getLangOpts();
> +
> + // Preload all the module files loaded transitively by the AST unit.
> + if (auto ASTReader = AST->getASTReader()) {
> + auto &MM = ASTReader->getModuleManager();
> + for (ModuleFile &MF : MM)
> + if (&MF != &MM.getPrimaryModule())
> + CI.getFrontendOpts().ModuleFiles.push_back(MF.FileName);
> + }
> +
> + // Set the shared objects, these are reset when we finish processing the
> + // file, otherwise the CompilerInstance will happily destroy them.
> + CI.setFileManager(&AST->getFileManager());
> + CI.createSourceManager(CI.getFileManager());
> + CI.getSourceManager().initializeForReplay(AST->getSourceManager());
> + CI.createPreprocessor(getTranslationUnitKind());
> +
> + // Set up the input file for replay purposes.
> + auto Kind = AST->getInputKind();
> + if (Kind.getFormat() == InputKind::ModuleMap) {
> + Module *ASTModule =
> + AST->getPreprocessor().getHeaderSearchInfo().lookupModule(
> + AST->getLangOpts().CurrentModule, /*AllowSearch*/ false);
> + Input = FrontendInputFile(ASTModule->PresumedModuleMapFile, Kind);
> + } else {
> + auto &SM = CI.getSourceManager();
> + FileID ID = SM.getMainFileID();
> + if (auto *File = SM.getFileEntryForID(ID))
> + Input = FrontendInputFile(File->getName(), Kind);
> + else
> + Input = FrontendInputFile(SM.getBuffer(ID), Kind);
> + }
> + setCurrentInput(Input, std::move(AST));
> + }
> +
> // AST files follow a very different path, since they share objects via the
> // AST unit.
> if (Input.getKind().getFormat() == InputKind::Precompiled) {
> - // FIXME: We should not be asserting on bad command-line arguments.
> - assert(!usesPreprocessorOnly() &&
> - "Attempt to pass AST file to preprocessor only action!");
> + assert(!usesPreprocessorOnly() && "this case was handled above");
> assert(hasASTFileSupport() &&
> "This action does not have AST file support!");
>
> @@ -680,7 +734,7 @@ bool FrontendAction::BeginSourceFile(Com
>
> std::string PresumedModuleMapFile;
> unsigned OffsetToContents;
> - if (loadModuleMapForModuleBuild(CI, Input.getFile(), Input.isSystem(),
> + if (loadModuleMapForModuleBuild(CI, Input.isSystem(),
> Input.isPreprocessed(),
> PresumedModuleMapFile, OffsetToContents))
> goto failure;
> @@ -829,14 +883,7 @@ bool FrontendAction::BeginSourceFile(Com
>
> // If we failed, reset state since the client will not end up calling the
> // matching EndSourceFile().
> - failure:
> - if (isCurrentFileAST()) {
> - CI.setASTContext(nullptr);
> - CI.setPreprocessor(nullptr);
> - CI.setSourceManager(nullptr);
> - CI.setFileManager(nullptr);
> - }
> -
> +failure:
> if (HasBegunSourceFile)
> CI.getDiagnosticClient().EndSourceFile();
> CI.clearOutputFiles(/*EraseFiles=*/true);
> @@ -914,6 +961,7 @@ void FrontendAction::EndSourceFile() {
> CI.resetAndLeakPreprocessor();
> CI.resetAndLeakSourceManager();
> CI.resetAndLeakFileManager();
> + BuryPointer(CurrentASTUnit.release());
> } else {
> CI.setPreprocessor(nullptr);
> CI.setSourceManager(nullptr);
>
> Modified: cfe/trunk/lib/Serialization/ASTReader.cpp
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Serialization/ASTReader.cpp?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/lib/Serialization/ASTReader.cpp (original)
> +++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Jun 5 13:10:11 2017
> @@ -4918,6 +4918,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile
> }
>
> CurrentModule->setASTFile(F.File);
> + CurrentModule->PresumedModuleMapFile = F.ModuleMapPath;
> }
>
> CurrentModule->Kind = ModuleKind;
>
> Modified: cfe/trunk/test/Modules/preprocess-module.cpp
> URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/preprocess-module.cpp?rev=304726&r1=304725&r2=304726&view=diff
> ==============================================================================
> --- cfe/trunk/test/Modules/preprocess-module.cpp (original)
> +++ cfe/trunk/test/Modules/preprocess-module.cpp Mon Jun 5 13:10:11 2017
> @@ -34,10 +34,15 @@
> // RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap
> // RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -x c++-module-map-cpp-output %t/copy.ii -emit-module -o %t/copy.pcm
>
> -// Finally, check that our module contains correct mapping information for the headers.
> +// Check that our module contains correct mapping information for the headers.
> // RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t
> // RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify -fno-modules-error-recovery -DCOPY -DINCLUDE
>
> +// Check that we can preprocess from a .pcm file and that we get the same result as preprocessing from the original sources.
> +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess -x c++-module-map %S/Inputs/preprocess/module.modulemap -emit-module -o %t/file.pcm
> +// RUN: %clang_cc1 -fmodules -fmodule-name=file -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess %t/file.pcm -E -frewrite-includes -o %t/file.rewrite.ii
> +// RUN: cmp %t/rewrite.ii %t/file.rewrite.ii
> +
> // == module map
> // CHECK: # 1 "{{.*}}module.modulemap"
> // CHECK: module file {
>
>
> _______________________________________________
> cfe-commits mailing list
> cfe-commits at lists.llvm.org
> http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
More information about the cfe-commits
mailing list