r319729 - [libclang] Store unsaved file hashes when recording parsing invocations

[Alex Lorenz via cfe-commits]

Author: arphaman
Date: Mon Dec  4 18:30:43 2017
New Revision: 319729

URL: http://llvm.org/viewvc/llvm-project?rev=319729&view=rev
Log:
[libclang] Store unsaved file hashes when recording parsing invocations

Storing the contents of unsaved files is too expensive.
Instead a hash is stored with a record invocation. When a reproducer is
generated, Clang will compare the stored hashes to the new hashes to determine
if the contents of a file has changed. This way we'll know when a reproducer was
generated for a different source to the one that triggered the original crash.

rdar://35322543

Added:
    cfe/trunk/test/Index/Inputs/record-parsing-invocation-remap.c
Modified:
    cfe/trunk/test/Index/record-parsing-invocation.c
    cfe/trunk/tools/libclang/CIndex.cpp
    cfe/trunk/tools/libclang/CIndexer.cpp
    cfe/trunk/tools/libclang/CIndexer.h

Added: cfe/trunk/test/Index/Inputs/record-parsing-invocation-remap.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/Inputs/record-parsing-invocation-remap.c?rev=319729&view=auto
==============================================================================
--- cfe/trunk/test/Index/Inputs/record-parsing-invocation-remap.c (added)
+++ cfe/trunk/test/Index/Inputs/record-parsing-invocation-remap.c Mon Dec  4 18:30:43 2017
@@ -0,0 +1,2 @@
+
+#pragma clang __debug parser_crash

Modified: cfe/trunk/test/Index/record-parsing-invocation.c
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Index/record-parsing-invocation.c?rev=319729&r1=319728&r2=319729&view=diff
==============================================================================
--- cfe/trunk/test/Index/record-parsing-invocation.c (original)
+++ cfe/trunk/test/Index/record-parsing-invocation.c Mon Dec  4 18:30:43 2017
@@ -14,8 +14,15 @@
 // RUN: env CINDEXTEST_INVOCATION_EMISSION_PATH=%t c-index-test -test-load-source all %s -DAVOID_CRASH
 // RUN: ls %t | count 0
 
+// Make sure we record the unsaved file hash.
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: not env CINDEXTEST_INVOCATION_EMISSION_PATH=%t c-index-test -test-load-source all "-remap-file=%s,%S/Inputs/record-parsing-invocation-remap.c" %s
+// RUN: cat %t/libclang-* | FileCheck --check-prefix=CHECK-UNSAVED %s
+
 #ifndef AVOID_CRASH
 #  pragma clang __debug parser_crash
 #endif
 
 // CHECK: {"toolchain":"{{.*}}","libclang.operation":"parse","libclang.opts":1,"args":["clang","-fno-spell-checking","{{.*}}record-parsing-invocation.c","-Xclang","-detailed-preprocessing-record","-fallow-editor-placeholders"]}
+// CHECK-UNSAVED: {"toolchain":"{{.*}}","libclang.operation":"parse","libclang.opts":1,"args":["clang","-fno-spell-checking","{{.*}}record-parsing-invocation.c","-Xclang","-detailed-preprocessing-record","-fallow-editor-placeholders"],"unsaved_file_hashes":[{"name":"{{.*}}record-parsing-invocation.c","md5":"aee23773de90e665992b48209351d70e"}]}

Modified: cfe/trunk/tools/libclang/CIndex.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndex.cpp?rev=319729&r1=319728&r2=319729&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndex.cpp (original)
+++ cfe/trunk/tools/libclang/CIndex.cpp Mon Dec  4 18:30:43 2017
@@ -3438,10 +3438,9 @@ clang_parseTranslationUnit_Impl(CXIndex
   unsigned PrecompilePreambleAfterNParses =
       !PrecompilePreamble ? 0 : 2 - CreatePreambleOnFirstParse;
 
-  // FIXME: Record the hash of the unsaved files.
   LibclangInvocationReporter InvocationReporter(
       *CXXIdx, LibclangInvocationReporter::OperationKind::ParseOperation,
-      options, llvm::makeArrayRef(*Args));
+      options, llvm::makeArrayRef(*Args), unsaved_files);
   std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCommandLine(
       Args->data(), Args->data() + Args->size(),
       CXXIdx->getPCHContainerOperations(), Diags,

Modified: cfe/trunk/tools/libclang/CIndexer.cpp
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndexer.cpp?rev=319729&r1=319728&r2=319729&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndexer.cpp (original)
+++ cfe/trunk/tools/libclang/CIndexer.cpp Mon Dec  4 18:30:43 2017
@@ -12,11 +12,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "CIndexer.h"
+#include "CXString.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Version.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Support/MD5.h"
 #include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
@@ -90,7 +92,8 @@ StringRef CIndexer::getClangToolchainPat
 
 LibclangInvocationReporter::LibclangInvocationReporter(
     CIndexer &Idx, OperationKind Op, unsigned ParseOptions,
-    llvm::ArrayRef<const char *> Args) {
+    llvm::ArrayRef<const char *> Args,
+    llvm::ArrayRef<CXUnsavedFile> UnsavedFiles) {
   StringRef Path = Idx.getInvocationEmissionPath();
   if (Path.empty())
     return;
@@ -124,6 +127,23 @@ LibclangInvocationReporter::LibclangInvo
       OS << ',';
     OS << '"' << I.value() << '"';
   }
+  if (!UnsavedFiles.empty()) {
+    OS << R"(],"unsaved_file_hashes":[)";
+    for (const auto &UF : llvm::enumerate(UnsavedFiles)) {
+      if (UF.index())
+        OS << ',';
+      OS << '{';
+      WriteStringKey("name", UF.value().Filename);
+      OS << ',';
+      llvm::MD5 Hash;
+      Hash.update(getContents(UF.value()));
+      llvm::MD5::MD5Result Result;
+      Hash.final(Result);
+      SmallString<32> Digest = Result.digest();
+      WriteStringKey("md5", Digest);
+      OS << '}';
+    }
+  }
   OS << "]}";
 }
 

Modified: cfe/trunk/tools/libclang/CIndexer.h
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/libclang/CIndexer.h?rev=319729&r1=319728&r2=319729&view=diff
==============================================================================
--- cfe/trunk/tools/libclang/CIndexer.h (original)
+++ cfe/trunk/tools/libclang/CIndexer.h Mon Dec  4 18:30:43 2017
@@ -94,7 +94,8 @@ public:
 
   LibclangInvocationReporter(CIndexer &Idx, OperationKind Op,
                              unsigned ParseOptions,
-                             llvm::ArrayRef<const char *> Args);
+                             llvm::ArrayRef<const char *> Args,
+                             llvm::ArrayRef<CXUnsavedFile> UnsavedFiles);
   ~LibclangInvocationReporter();
 
 private: