r290326 - [CrashReproducer] Add support for merging -ivfsoverlay

Bruno Cardoso Lopes via cfe-commits cfe-commits at lists.llvm.org
Wed Dec 21 23:06:03 PST 2016


Author: bruno
Date: Thu Dec 22 01:06:03 2016
New Revision: 290326

URL: http://llvm.org/viewvc/llvm-project?rev=290326&view=rev
Log:
[CrashReproducer] Add support for merging -ivfsoverlay

Merge all VFS mapped files inside -ivfsoverlay inputs into the vfs
overlay provided by the crash reproducer. This is the last missing piece
to allow crash reproducers to fully work with user frameworks; when
combined with headermaps, it allows clang to find additional frameworks.

rdar://problem/27913709

Added:
    cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m
Modified:
    cfe/trunk/include/clang/Basic/VirtualFileSystem.h
    cfe/trunk/include/clang/Frontend/Utils.h
    cfe/trunk/lib/Basic/VirtualFileSystem.cpp
    cfe/trunk/lib/Frontend/CompilerInstance.cpp
    cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp

Modified: cfe/trunk/include/clang/Basic/VirtualFileSystem.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Basic/VirtualFileSystem.h?rev=290326&r1=290325&r2=290326&view=diff
==============================================================================
--- cfe/trunk/include/clang/Basic/VirtualFileSystem.h (original)
+++ cfe/trunk/include/clang/Basic/VirtualFileSystem.h Thu Dec 22 01:06:03 2016
@@ -362,6 +362,16 @@ struct YAMLVFSEntry {
   std::string RPath;
 };
 
+/// \brief Collect all pairs of <virtual path, real path> entries from the
+/// \p YAMLFilePath. This is used by the module dependency collector to forward
+/// the entries into the reproducer output VFS YAML file.
+void collectVFSFromYAML(
+    std::unique_ptr<llvm::MemoryBuffer> Buffer,
+    llvm::SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
+    SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
+    void *DiagContext = nullptr,
+    IntrusiveRefCntPtr<FileSystem> ExternalFS = getRealFileSystem());
+
 class YAMLVFSWriter {
   std::vector<YAMLVFSEntry> Mappings;
   Optional<bool> IsCaseSensitive;

Modified: cfe/trunk/include/clang/Frontend/Utils.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Frontend/Utils.h?rev=290326&r1=290325&r2=290326&view=diff
==============================================================================
--- cfe/trunk/include/clang/Frontend/Utils.h (original)
+++ cfe/trunk/include/clang/Frontend/Utils.h Thu Dec 22 01:06:03 2016
@@ -128,11 +128,11 @@ class ModuleDependencyCollector : public
   llvm::StringMap<std::string> SymLinkMap;
 
   bool getRealPath(StringRef SrcPath, SmallVectorImpl<char> &Result);
-  std::error_code copyToRoot(StringRef Src);
+  std::error_code copyToRoot(StringRef Src, StringRef Dst = "");
 public:
   StringRef getDest() { return DestDir; }
   bool insertSeen(StringRef Filename) { return Seen.insert(Filename).second; }
-  void addFile(StringRef Filename);
+  void addFile(StringRef Filename, StringRef FileDst = "");
   void addFileMapping(StringRef VPath, StringRef RPath) {
     VFSWriter.addFileMapping(VPath, RPath);
   }

Modified: cfe/trunk/lib/Basic/VirtualFileSystem.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Basic/VirtualFileSystem.cpp?rev=290326&r1=290325&r2=290326&view=diff
==============================================================================
--- cfe/trunk/lib/Basic/VirtualFileSystem.cpp (original)
+++ cfe/trunk/lib/Basic/VirtualFileSystem.cpp Thu Dec 22 01:06:03 2016
@@ -887,9 +887,6 @@ private:
   RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)
       : ExternalFS(std::move(ExternalFS)) {}
 
-  /// \brief Looks up \p Path in \c Roots.
-  ErrorOr<Entry *> lookupPath(const Twine &Path);
-
   /// \brief Looks up the path <tt>[Start, End)</tt> in \p From, possibly
   /// recursing into the contents of \p From if it is a directory.
   ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
@@ -899,6 +896,9 @@ private:
   ErrorOr<Status> status(const Twine &Path, Entry *E);
 
 public:
+  /// \brief Looks up \p Path in \c Roots.
+  ErrorOr<Entry *> lookupPath(const Twine &Path);
+
   /// \brief Parses \p Buffer, which is expected to be in YAML format and
   /// returns a virtual file system representing its contents.
   static RedirectingFileSystem *
@@ -1606,6 +1606,47 @@ vfs::getVFSFromYAML(std::unique_ptr<Memo
                                        std::move(ExternalFS));
 }
 
+static void getVFSEntries(Entry *SrcE, SmallVectorImpl<StringRef> &Path,
+                          SmallVectorImpl<YAMLVFSEntry> &Entries) {
+  auto Kind = SrcE->getKind();
+  if (Kind == EK_Directory) {
+    auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE);
+    assert(DE && "Must be a directory");
+    for (std::unique_ptr<Entry> &SubEntry :
+         llvm::make_range(DE->contents_begin(), DE->contents_end())) {
+      Path.push_back(SubEntry->getName());
+      getVFSEntries(SubEntry.get(), Path, Entries);
+      Path.pop_back();
+    }
+    return;
+  }
+
+  assert(Kind == EK_File && "Must be a EK_File");
+  auto *FE = dyn_cast<RedirectingFileEntry>(SrcE);
+  assert(FE && "Must be a file");
+  SmallString<128> VPath;
+  for (auto &Comp : Path)
+    llvm::sys::path::append(VPath, Comp);
+  Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath()));
+}
+
+void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
+                             SourceMgr::DiagHandlerTy DiagHandler,
+                             StringRef YAMLFilePath,
+                             SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
+                             void *DiagContext,
+                             IntrusiveRefCntPtr<FileSystem> ExternalFS) {
+  RedirectingFileSystem *VFS = RedirectingFileSystem::create(
+      std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
+      std::move(ExternalFS));
+  ErrorOr<Entry *> RootE = VFS->lookupPath("/");
+  if (!RootE)
+    return;
+  SmallVector<StringRef, 8> Components;
+  Components.push_back("/");
+  getVFSEntries(*RootE, Components, CollectedEntries);
+}
+
 UniqueID vfs::getNextVirtualUniqueID() {
   static std::atomic<unsigned> UID;
   unsigned ID = ++UID;

Modified: cfe/trunk/lib/Frontend/CompilerInstance.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/CompilerInstance.cpp?rev=290326&r1=290325&r2=290326&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/CompilerInstance.cpp (original)
+++ cfe/trunk/lib/Frontend/CompilerInstance.cpp Thu Dec 22 01:06:03 2016
@@ -181,6 +181,26 @@ static void collectIncludePCH(CompilerIn
   }
 }
 
+static void collectVFSEntries(CompilerInstance &CI,
+                              std::shared_ptr<ModuleDependencyCollector> MDC) {
+  if (CI.getHeaderSearchOpts().VFSOverlayFiles.empty())
+    return;
+
+  // Collect all VFS found.
+  SmallVector<vfs::YAMLVFSEntry, 16> VFSEntries;
+  for (const std::string &VFSFile : CI.getHeaderSearchOpts().VFSOverlayFiles) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
+        llvm::MemoryBuffer::getFile(VFSFile);
+    if (!Buffer)
+      return;
+    vfs::collectVFSFromYAML(std::move(Buffer.get()), /*DiagHandler*/ nullptr,
+                            VFSFile, VFSEntries);
+  }
+
+  for (auto &E : VFSEntries)
+    MDC->addFile(E.VPath, E.RPath);
+}
+
 // Diagnostics
 static void SetUpDiagnosticLog(DiagnosticOptions *DiagOpts,
                                const CodeGenOptions *CodeGenOpts,
@@ -412,6 +432,7 @@ void CompilerInstance::createPreprocesso
     addDependencyCollector(ModuleDepCollector);
     collectHeaderMaps(PP->getHeaderSearchInfo(), ModuleDepCollector);
     collectIncludePCH(*this, ModuleDepCollector);
+    collectVFSEntries(*this, ModuleDepCollector);
   }
 
   for (auto &Listener : DependencyCollectors)

Modified: cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp?rev=290326&r1=290325&r2=290326&view=diff
==============================================================================
--- cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp (original)
+++ cfe/trunk/lib/Frontend/ModuleDependencyCollector.cpp Thu Dec 22 01:06:03 2016
@@ -201,7 +201,8 @@ bool ModuleDependencyCollector::getRealP
   return true;
 }
 
-std::error_code ModuleDependencyCollector::copyToRoot(StringRef Src) {
+std::error_code ModuleDependencyCollector::copyToRoot(StringRef Src,
+                                                      StringRef Dst) {
   using namespace llvm::sys;
 
   // We need an absolute src path to append to the root.
@@ -213,23 +214,35 @@ std::error_code ModuleDependencyCollecto
   AbsoluteSrc = path::remove_leading_dotslash(AbsoluteSrc);
 
   // Canonicalize the source path by removing "..", "." components.
-  SmallString<256> CanonicalPath = AbsoluteSrc;
-  path::remove_dots(CanonicalPath, /*remove_dot_dot=*/true);
+  SmallString<256> VirtualPath = AbsoluteSrc;
+  path::remove_dots(VirtualPath, /*remove_dot_dot=*/true);
 
   // If a ".." component is present after a symlink component, remove_dots may
   // lead to the wrong real destination path. Let the source be canonicalized
   // like that but make sure we always use the real path for the destination.
-  SmallString<256> RealPath;
-  if (!getRealPath(AbsoluteSrc, RealPath))
-    RealPath = CanonicalPath;
-  SmallString<256> Dest = getDest();
-  path::append(Dest, path::relative_path(RealPath));
+  SmallString<256> CopyFrom;
+  if (!getRealPath(AbsoluteSrc, CopyFrom))
+    CopyFrom = VirtualPath;
+  SmallString<256> CacheDst = getDest();
+
+  if (Dst.empty()) {
+    // The common case is to map the virtual path to the same path inside the
+    // cache.
+    path::append(CacheDst, path::relative_path(CopyFrom));
+  } else {
+    // When collecting entries from input vfsoverlays, copy the external
+    // contents into the cache but still map from the source.
+    if (!fs::exists(Dst))
+      return std::error_code();
+    path::append(CacheDst, Dst);
+    CopyFrom = Dst;
+  }
 
   // Copy the file into place.
-  if (std::error_code EC = fs::create_directories(path::parent_path(Dest),
-                                                   /*IgnoreExisting=*/true))
+  if (std::error_code EC = fs::create_directories(path::parent_path(CacheDst),
+                                                  /*IgnoreExisting=*/true))
     return EC;
-  if (std::error_code EC = fs::copy_file(RealPath, Dest))
+  if (std::error_code EC = fs::copy_file(CopyFrom, CacheDst))
     return EC;
 
   // Always map a canonical src path to its real path into the YAML, by doing
@@ -237,12 +250,12 @@ std::error_code ModuleDependencyCollecto
   // overlay, which is a way to emulate symlink inside the VFS; this is also
   // needed for correctness, not doing that can lead to module redifinition
   // errors.
-  addFileMapping(CanonicalPath, Dest);
+  addFileMapping(VirtualPath, CacheDst);
   return std::error_code();
 }
 
-void ModuleDependencyCollector::addFile(StringRef Filename) {
+void ModuleDependencyCollector::addFile(StringRef Filename, StringRef FileDst) {
   if (insertSeen(Filename))
-    if (copyToRoot(Filename))
+    if (copyToRoot(Filename, FileDst))
       HasErrors = true;
 }

Added: cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m?rev=290326&view=auto
==============================================================================
--- cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m (added)
+++ cfe/trunk/test/Modules/crash-vfs-ivfsoverlay.m Thu Dec 22 01:06:03 2016
@@ -0,0 +1,45 @@
+// REQUIRES: crash-recovery, shell, system-darwin
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/m
+// RUN: cp %S/../VFS/Inputs/actual_module2.map %t/actual_module2.map
+// RUN: sed -e "s:INPUT_DIR:%t:g" -e "s:OUT_DIR:%t/example:g" \
+// RUN:   %S/../VFS/Inputs/vfsoverlay2.yaml > %t/srcvfs.yaml
+
+// RUN: not env FORCE_CLANG_DIAGNOSTICS_CRASH= TMPDIR=%t TEMP=%t TMP=%t \
+// RUN: %clang -fsyntax-only -nostdinc %s \
+// RUN:     -I %S/Inputs/crash-recovery/usr/include \
+// RUN:     -ivfsoverlay %t/srcvfs.yaml \
+// RUN:     -fmodules -fmodules-cache-path=%t/m/ 2>&1 | FileCheck %s
+
+// RUN: FileCheck --check-prefix=CHECKSH %s -input-file %t/crash-vfs-*.sh
+// RUN: FileCheck --check-prefix=CHECKYAML %s -input-file \
+// RUN:   %t/crash-vfs-*.cache/vfs/vfs.yaml
+// RUN: find %t/crash-vfs-*.cache/vfs | \
+// RUN:   grep "%t/actual_module2.map" | count 1
+
+#include <stdio.h>
+
+// CHECK: Preprocessed source(s) and associated run script(s) are located at:
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.m
+// CHECK-NEXT: note: diagnostic msg: {{.*}}.cache
+
+// CHECKSH: # Crash reproducer
+// CHECKSH-NEXT: # Driver args: "-fsyntax-only"
+// CHECKSH-NEXT: # Original command: {{.*$}}
+// CHECKSH-NEXT: "-cc1"
+// CHECKSH: "crash-vfs-{{[^ ]*}}.m"
+// CHECKSH: "-ivfsoverlay" "crash-vfs-{{[^ ]*}}.cache/vfs/vfs.yaml"
+// CHECKSH: "-fmodules-cache-path=crash-vfs-{{[^ ]*}}.cache/repro-modules"
+
+// CHECKYAML: 'case-sensitive':
+// CHECKYAML-NEXT: 'use-external-names': 'false',
+// CHECKYAML-NEXT: 'overlay-relative': 'true',
+// CHECKYAML-NEXT: 'ignore-non-existent-contents': 'false'
+// CHECKYAML: 'type': 'directory'
+// CHECKYAML: 'name': "/[[PATH:.*]]/example"
+// CHECKYAML: 'contents': [
+// CHECKYAML-NEXT:   {
+// CHECKYAML-NEXT:     'type': 'file',
+// CHECKYAML-NEXT:     'name': "module.map",
+// CHECKYAML-NEXT:     'external-contents': "/[[OTHERPATH:.*]]/actual_module2.map"




More information about the cfe-commits mailing list