r304726 - Rather than rejecting attempts to run preprocessor-only actions on AST files,
Richard Smith via cfe-commits
cfe-commits at lists.llvm.org
Mon Jun 5 14:09:27 PDT 2017
Thanks, investigating.
On 5 June 2017 at 13:14, Evgenii Stepanov <eugeni.stepanov at gmail.com> wrote:
> This change leaks memory:
> http://lab.llvm.org:8011/builders/sanitizer-x86_64-
> linux-fast/builds/5453/steps/check-clang%20asan/logs/stdio
>
> On Mon, Jun 5, 2017 at 11:10 AM, Richard Smith via cfe-commits
> <cfe-commits at lists.llvm.org> wrote:
> > Author: rsmith
> > Date: Mon Jun 5 13:10:11 2017
> > New Revision: 304726
> >
> > URL: http://llvm.org/viewvc/llvm-project?rev=304726&view=rev
> > Log:
> > Rather than rejecting attempts to run preprocessor-only actions on AST
> files,
> > replay the steps taken to create the AST file with the preprocessor-only
> action
> > installed to produce preprocessed output.
> >
> > This can be used to produce the preprocessed text for an existing .pch
> or .pcm
> > file.
> >
> > Modified:
> > cfe/trunk/include/clang/Basic/LangOptions.h
> > cfe/trunk/include/clang/Basic/SourceManager.h
> > cfe/trunk/include/clang/Frontend/ASTUnit.h
> > cfe/trunk/include/clang/Frontend/FrontendAction.h
> > cfe/trunk/lib/Basic/SourceManager.cpp
> > cfe/trunk/lib/Frontend/ASTUnit.cpp
> > cfe/trunk/lib/Frontend/FrontendAction.cpp
> > cfe/trunk/lib/Serialization/ASTReader.cpp
> > cfe/trunk/test/Modules/preprocess-module.cpp
> >
> > Modified: cfe/trunk/include/clang/Basic/LangOptions.h
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/
> clang/Basic/LangOptions.h?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/include/clang/Basic/LangOptions.h (original)
> > +++ cfe/trunk/include/clang/Basic/LangOptions.h Mon Jun 5 13:10:11 2017
> > @@ -58,6 +58,7 @@ public:
> > SOB_Trapping // -ftrapv
> > };
> >
> > + // FIXME: Unify with TUKind.
> > enum CompilingModuleKind {
> > CMK_None, ///< Not compiling a module interface at all.
> > CMK_ModuleMap, ///< Compiling a module from a module map.
> >
> > Modified: cfe/trunk/include/clang/Basic/SourceManager.h
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/
> clang/Basic/SourceManager.h?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/include/clang/Basic/SourceManager.h (original)
> > +++ cfe/trunk/include/clang/Basic/SourceManager.h Mon Jun 5 13:10:11
> 2017
> > @@ -722,6 +722,10 @@ public:
> >
> > void clearIDTables();
> >
> > + /// Initialize this source manager suitably to replay the compilation
> > + /// described by \p Old. Requires that \p Old outlive \p *this.
> > + void initializeForReplay(const SourceManager &Old);
> > +
> > DiagnosticsEngine &getDiagnostics() const { return Diag; }
> >
> > FileManager &getFileManager() const { return FileMgr; }
> >
> > Modified: cfe/trunk/include/clang/Frontend/ASTUnit.h
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/
> clang/Frontend/ASTUnit.h?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/include/clang/Frontend/ASTUnit.h (original)
> > +++ cfe/trunk/include/clang/Frontend/ASTUnit.h Mon Jun 5 13:10:11 2017
> > @@ -51,6 +51,7 @@ class DiagnosticsEngine;
> > class FileEntry;
> > class FileManager;
> > class HeaderSearch;
> > +class InputKind;
> > class MemoryBufferCache;
> > class Preprocessor;
> > class PCHContainerOperations;
> > @@ -305,9 +306,6 @@ private:
> > /// (likely to change while trying to use them).
> > bool UserFilesAreVolatile : 1;
> >
> > - /// \brief The language options used when we load an AST file.
> > - LangOptions ASTFileLangOpts;
> > -
> > static void ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine>
> Diags,
> > ASTUnit &AST, bool CaptureDiagnostics);
> >
> > @@ -702,6 +700,9 @@ public:
> > /// \brief Determine what kind of translation unit this AST
> represents.
> > TranslationUnitKind getTranslationUnitKind() const { return TUKind; }
> >
> > + /// \brief Determine the input kind this AST unit represents.
> > + InputKind getInputKind() const;
> > +
> > /// \brief A mapping from a file name to the memory buffer that
> stores the
> > /// remapped contents of that file.
> > typedef std::pair<std::string, llvm::MemoryBuffer *> RemappedFile;
> >
> > Modified: cfe/trunk/include/clang/Frontend/FrontendAction.h
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/
> clang/Frontend/FrontendAction.h?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/include/clang/Frontend/FrontendAction.h (original)
> > +++ cfe/trunk/include/clang/Frontend/FrontendAction.h Mon Jun 5
> 13:10:11 2017
> > @@ -176,10 +176,10 @@ public:
> > virtual TranslationUnitKind getTranslationUnitKind() { return
> TU_Complete; }
> >
> > /// \brief Does this action support use with PCH?
> > - virtual bool hasPCHSupport() const { return !usesPreprocessorOnly(); }
> > + virtual bool hasPCHSupport() const { return true; }
> >
> > /// \brief Does this action support use with AST files?
> > - virtual bool hasASTFileSupport() const { return
> !usesPreprocessorOnly(); }
> > + virtual bool hasASTFileSupport() const { return true; }
> >
> > /// \brief Does this action support use with IR files?
> > virtual bool hasIRSupport() const { return false; }
> >
> > Modified: cfe/trunk/lib/Basic/SourceManager.cpp
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/
> SourceManager.cpp?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/lib/Basic/SourceManager.cpp (original)
> > +++ cfe/trunk/lib/Basic/SourceManager.cpp Mon Jun 5 13:10:11 2017
> > @@ -345,6 +345,41 @@ void SourceManager::clearIDTables() {
> > createExpansionLoc(SourceLocation(),SourceLocation(),SourceLocation(),
> 1);
> > }
> >
> > +void SourceManager::initializeForReplay(const SourceManager &Old) {
> > + assert(MainFileID.isInvalid() && "expected uninitialized
> SourceManager");
> > +
> > + auto CloneContentCache = [&](const ContentCache *Cache) ->
> ContentCache * {
> > + auto *Clone = new (ContentCacheAlloc.Allocate<ContentCache>())
> ContentCache;
> > + Clone->OrigEntry = Cache->OrigEntry;
> > + Clone->ContentsEntry = Cache->ContentsEntry;
> > + Clone->BufferOverridden = Cache->BufferOverridden;
> > + Clone->IsSystemFile = Cache->IsSystemFile;
> > + Clone->IsTransient = Cache->IsTransient;
> > + Clone->replaceBuffer(Cache->getRawBuffer(), /*DoNotFree*/true);
> > + return Clone;
> > + };
> > +
> > + // Set up our main file ID as a copy of the old source manager's main
> file.
> > + const SLocEntry &OldMainFile = Old.getSLocEntry(Old.getMainFileID());
> > + assert(OldMainFile.isFile() && "main file is macro expansion?");
> > + setMainFileID(createFileID(
> > + CloneContentCache(OldMainFile.getFile().getContentCache()),
> > + SourceLocation(), OldMainFile.getFile().getFileCharacteristic(),
> 0, 0));
> > +
> > + // Ensure all SLocEntries are loaded from the external source.
> > + for (unsigned I = 0, N = Old.LoadedSLocEntryTable.size(); I != N;
> ++I)
> > + if (!Old.SLocEntryLoaded[I])
> > + Old.loadSLocEntry(I, nullptr);
> > +
> > + // Inherit any content cache data from the old source manager.
> > + for (auto &FileInfo : Old.FileInfos) {
> > + SrcMgr::ContentCache *&Slot = FileInfos[FileInfo.first];
> > + if (Slot)
> > + continue;
> > + Slot = CloneContentCache(FileInfo.second);
> > + }
> > +}
> > +
> > /// getOrCreateContentCache - Create or return a cached ContentCache
> for the
> > /// specified file.
> > const ContentCache *
> >
> > Modified: cfe/trunk/lib/Frontend/ASTUnit.cpp
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/
> Frontend/ASTUnit.cpp?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/lib/Frontend/ASTUnit.cpp (original)
> > +++ cfe/trunk/lib/Frontend/ASTUnit.cpp Mon Jun 5 13:10:11 2017
> > @@ -667,6 +667,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> >
> > ConfigureDiags(Diags, *AST, CaptureDiagnostics);
> >
> > + AST->LangOpts = std::make_shared<LangOptions>();
> > AST->OnlyLocalDecls = OnlyLocalDecls;
> > AST->CaptureDiagnostics = CaptureDiagnostics;
> > AST->Diagnostics = Diags;
> > @@ -682,7 +683,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> > AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
> > AST->getSourceManager(),
> > AST->getDiagnostics(),
> > - AST->ASTFileLangOpts,
> > + AST->getLangOpts(),
> > /*Target=*/nullptr));
> >
> > auto PPOpts = std::make_shared<PreprocessorOptions>();
> > @@ -696,13 +697,13 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> > unsigned Counter;
> >
> > AST->PP = std::make_shared<Preprocessor>(
> > - std::move(PPOpts), AST->getDiagnostics(), AST->ASTFileLangOpts,
> > + std::move(PPOpts), AST->getDiagnostics(), *AST->LangOpts,
> > AST->getSourceManager(), *AST->PCMCache, HeaderInfo, *AST,
> > /*IILookup=*/nullptr,
> > /*OwnsHeaderSearch=*/false);
> > Preprocessor &PP = *AST->PP;
> >
> > - AST->Ctx = new ASTContext(AST->ASTFileLangOpts,
> AST->getSourceManager(),
> > + AST->Ctx = new ASTContext(*AST->LangOpts, AST->getSourceManager(),
> > PP.getIdentifierTable(),
> PP.getSelectorTable(),
> > PP.getBuiltinInfo());
> > ASTContext &Context = *AST->Ctx;
> > @@ -716,7 +717,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFr
> > AllowPCHWithCompilerErrors);
> >
> > AST->Reader->setListener(llvm::make_unique<ASTInfoCollector>(
> > - *AST->PP, Context, AST->ASTFileLangOpts, AST->TargetOpts,
> AST->Target,
> > + *AST->PP, Context, *AST->LangOpts, AST->TargetOpts, AST->Target,
> > Counter));
> >
> > // Attach the AST reader to the AST context as an external AST
> > @@ -2879,7 +2880,32 @@ const FileEntry *ASTUnit::getPCHFile() {
> > }
> >
> > bool ASTUnit::isModuleFile() {
> > - return isMainFileAST() && ASTFileLangOpts.isCompilingModule();
> > + return isMainFileAST() && getLangOpts().isCompilingModule();
> > +}
> > +
> > +InputKind ASTUnit::getInputKind() const {
> > + auto &LangOpts = getLangOpts();
> > +
> > + InputKind::Language Lang;
> > + if (LangOpts.OpenCL)
> > + Lang = InputKind::OpenCL;
> > + else if (LangOpts.CUDA)
> > + Lang = InputKind::CUDA;
> > + else if (LangOpts.RenderScript)
> > + Lang = InputKind::RenderScript;
> > + else if (LangOpts.CPlusPlus)
> > + Lang = LangOpts.ObjC1 ? InputKind::ObjCXX : InputKind::CXX;
> > + else
> > + Lang = LangOpts.ObjC1 ? InputKind::ObjC : InputKind::C;
> > +
> > + InputKind::Format Fmt = InputKind::Source;
> > + if (LangOpts.getCompilingModule() == LangOptions::CMK_ModuleMap)
> > + Fmt = InputKind::ModuleMap;
> > +
> > + // We don't know if input was preprocessed. Assume not.
> > + bool PP = false;
> > +
> > + return InputKind(Lang, Fmt, PP);
> > }
> >
> > void ASTUnit::PreambleData::countLines() const {
> >
> > Modified: cfe/trunk/lib/Frontend/FrontendAction.cpp
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/
> Frontend/FrontendAction.cpp?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/lib/Frontend/FrontendAction.cpp (original)
> > +++ cfe/trunk/lib/Frontend/FrontendAction.cpp Mon Jun 5 13:10:11 2017
> > @@ -387,8 +387,7 @@ static std::error_code collectModuleHead
> > return std::error_code();
> > }
> >
> > -static bool loadModuleMapForModuleBuild(CompilerInstance &CI,
> > - StringRef Filename, bool
> IsSystem,
> > +static bool loadModuleMapForModuleBuild(CompilerInstance &CI, bool
> IsSystem,
> > bool IsPreprocessed,
> > std::string
> &PresumedModuleMapFile,
> > unsigned &Offset) {
> > @@ -523,7 +522,8 @@ getInputBufferForModule(CompilerInstance
> > }
> >
> > bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
> > - const FrontendInputFile &Input) {
> > + const FrontendInputFile
> &RealInput) {
> > + FrontendInputFile Input(RealInput);
> > assert(!Instance && "Already processing a source file!");
> > assert(!Input.isEmpty() && "Unexpected empty filename!");
> > setCurrentInput(Input);
> > @@ -531,15 +531,69 @@ bool FrontendAction::BeginSourceFile(Com
> >
> > StringRef InputFile = Input.getFile();
> > bool HasBegunSourceFile = false;
> > + bool ReplayASTFile = Input.getKind().getFormat() ==
> InputKind::Precompiled &&
> > + usesPreprocessorOnly();
> > if (!BeginInvocation(CI))
> > goto failure;
> >
> > + // If we're replaying the build of an AST file, import it and set up
> > + // the initial state from its build.
> > + if (ReplayASTFile) {
> > + IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics());
> > +
> > + // The AST unit populates its own diagnostics engine rather than
> ours.
> > + IntrusiveRefCntPtr<DiagnosticsEngine> ASTDiags(
> > + new DiagnosticsEngine(Diags->getDiagnosticIDs(),
> > + &Diags->getDiagnosticOptions()));
> > + ASTDiags->setClient(Diags->getClient(), /*OwnsClient*/false);
> > +
> > + std::unique_ptr<ASTUnit> AST = ASTUnit::LoadFromASTFile(
> > + InputFile, CI.getPCHContainerReader(), ASTDiags,
> CI.getFileSystemOpts(),
> > + CI.getCodeGenOpts().DebugTypeExtRefs);
> > + if (!AST)
> > + goto failure;
> > +
> > + // Options relating to how we treat the input (but not what we do
> with it)
> > + // are inherited from the AST unit.
> > + CI.getLangOpts() = AST->getLangOpts();
> > +
> > + // Preload all the module files loaded transitively by the AST unit.
> > + if (auto ASTReader = AST->getASTReader()) {
> > + auto &MM = ASTReader->getModuleManager();
> > + for (ModuleFile &MF : MM)
> > + if (&MF != &MM.getPrimaryModule())
> > + CI.getFrontendOpts().ModuleFiles.push_back(MF.FileName);
> > + }
> > +
> > + // Set the shared objects, these are reset when we finish
> processing the
> > + // file, otherwise the CompilerInstance will happily destroy them.
> > + CI.setFileManager(&AST->getFileManager());
> > + CI.createSourceManager(CI.getFileManager());
> > + CI.getSourceManager().initializeForReplay(AST->getSourceManager());
> > + CI.createPreprocessor(getTranslationUnitKind());
> > +
> > + // Set up the input file for replay purposes.
> > + auto Kind = AST->getInputKind();
> > + if (Kind.getFormat() == InputKind::ModuleMap) {
> > + Module *ASTModule =
> > + AST->getPreprocessor().getHeaderSearchInfo().lookupModule(
> > + AST->getLangOpts().CurrentModule, /*AllowSearch*/ false);
> > + Input = FrontendInputFile(ASTModule->PresumedModuleMapFile,
> Kind);
> > + } else {
> > + auto &SM = CI.getSourceManager();
> > + FileID ID = SM.getMainFileID();
> > + if (auto *File = SM.getFileEntryForID(ID))
> > + Input = FrontendInputFile(File->getName(), Kind);
> > + else
> > + Input = FrontendInputFile(SM.getBuffer(ID), Kind);
> > + }
> > + setCurrentInput(Input, std::move(AST));
> > + }
> > +
> > // AST files follow a very different path, since they share objects
> via the
> > // AST unit.
> > if (Input.getKind().getFormat() == InputKind::Precompiled) {
> > - // FIXME: We should not be asserting on bad command-line arguments.
> > - assert(!usesPreprocessorOnly() &&
> > - "Attempt to pass AST file to preprocessor only action!");
> > + assert(!usesPreprocessorOnly() && "this case was handled above");
> > assert(hasASTFileSupport() &&
> > "This action does not have AST file support!");
> >
> > @@ -680,7 +734,7 @@ bool FrontendAction::BeginSourceFile(Com
> >
> > std::string PresumedModuleMapFile;
> > unsigned OffsetToContents;
> > - if (loadModuleMapForModuleBuild(CI, Input.getFile(),
> Input.isSystem(),
> > + if (loadModuleMapForModuleBuild(CI, Input.isSystem(),
> > Input.isPreprocessed(),
> > PresumedModuleMapFile,
> OffsetToContents))
> > goto failure;
> > @@ -829,14 +883,7 @@ bool FrontendAction::BeginSourceFile(Com
> >
> > // If we failed, reset state since the client will not end up calling
> the
> > // matching EndSourceFile().
> > - failure:
> > - if (isCurrentFileAST()) {
> > - CI.setASTContext(nullptr);
> > - CI.setPreprocessor(nullptr);
> > - CI.setSourceManager(nullptr);
> > - CI.setFileManager(nullptr);
> > - }
> > -
> > +failure:
> > if (HasBegunSourceFile)
> > CI.getDiagnosticClient().EndSourceFile();
> > CI.clearOutputFiles(/*EraseFiles=*/true);
> > @@ -914,6 +961,7 @@ void FrontendAction::EndSourceFile() {
> > CI.resetAndLeakPreprocessor();
> > CI.resetAndLeakSourceManager();
> > CI.resetAndLeakFileManager();
> > + BuryPointer(CurrentASTUnit.release());
> > } else {
> > CI.setPreprocessor(nullptr);
> > CI.setSourceManager(nullptr);
> >
> > Modified: cfe/trunk/lib/Serialization/ASTReader.cpp
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/
> Serialization/ASTReader.cpp?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/lib/Serialization/ASTReader.cpp (original)
> > +++ cfe/trunk/lib/Serialization/ASTReader.cpp Mon Jun 5 13:10:11 2017
> > @@ -4918,6 +4918,7 @@ ASTReader::ReadSubmoduleBlock(ModuleFile
> > }
> >
> > CurrentModule->setASTFile(F.File);
> > + CurrentModule->PresumedModuleMapFile = F.ModuleMapPath;
> > }
> >
> > CurrentModule->Kind = ModuleKind;
> >
> > Modified: cfe/trunk/test/Modules/preprocess-module.cpp
> > URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/
> Modules/preprocess-module.cpp?rev=304726&r1=304725&r2=304726&view=diff
> > ============================================================
> ==================
> > --- cfe/trunk/test/Modules/preprocess-module.cpp (original)
> > +++ cfe/trunk/test/Modules/preprocess-module.cpp Mon Jun 5 13:10:11
> 2017
> > @@ -34,10 +34,15 @@
> > // RUN: rm %t/fwd.h %t/file.h %t/file2.h %t/module.modulemap
> > // RUN: %clang_cc1 -fmodules -fmodule-name=file
> -fmodule-file=%t/fwd.pcm -x c++-module-map-cpp-output %t/copy.ii
> -emit-module -o %t/copy.pcm
> >
> > -// Finally, check that our module contains correct mapping information
> for the headers.
> > +// Check that our module contains correct mapping information for the
> headers.
> > // RUN: cp %S/Inputs/preprocess/fwd.h %S/Inputs/preprocess/file.h
> %S/Inputs/preprocess/file2.h %S/Inputs/preprocess/module.modulemap %t
> > // RUN: %clang_cc1 -fmodules -fmodule-file=%t/copy.pcm %s -I%t -verify
> -fno-modules-error-recovery -DCOPY -DINCLUDE
> >
> > +// Check that we can preprocess from a .pcm file and that we get the
> same result as preprocessing from the original sources.
> > +// RUN: %clang_cc1 -fmodules -fmodule-name=file
> -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess -x c++-module-map
> %S/Inputs/preprocess/module.modulemap -emit-module -o %t/file.pcm
> > +// RUN: %clang_cc1 -fmodules -fmodule-name=file
> -fmodule-file=%t/fwd.pcm -I%S/Inputs/preprocess %t/file.pcm -E
> -frewrite-includes -o %t/file.rewrite.ii
> > +// RUN: cmp %t/rewrite.ii %t/file.rewrite.ii
> > +
> > // == module map
> > // CHECK: # 1 "{{.*}}module.modulemap"
> > // CHECK: module file {
> >
> >
> > _______________________________________________
> > cfe-commits mailing list
> > cfe-commits at lists.llvm.org
> > http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20170605/c33a27d7/attachment-0001.html>
More information about the cfe-commits
mailing list