[clang-tools-extra] [clang-doc] integrate JSON generator with Mustache templates (PR #149006)

[Erick Velez via cfe-commits]

================
@@ -132,404 +135,74 @@ Error MustacheHTMLGenerator::generateDocs(
       return Err;
   }
 
-  // Track which directories we already tried to create.
-  StringSet<> CreatedDirs;
-  // Collect all output by file name and create the necessary directories.
-  StringMap<std::vector<doc::Info *>> FileToInfos;
-  for (const auto &Group : Infos) {
-    llvm::TimeTraceScope TS("setup directories");
-    doc::Info *Info = Group.getValue().get();
-
-    SmallString<128> Path;
-    sys::path::native(RootDir, Path);
-    sys::path::append(Path, Info->getRelativeFilePath(""));
-    if (!CreatedDirs.contains(Path)) {
-      if (std::error_code EC = sys::fs::create_directories(Path))
-        return createStringError(EC, "failed to create directory '%s'.",
-                                 Path.c_str());
-      CreatedDirs.insert(Path);
-    }
+  {
+    llvm::TimeTraceScope TS("Generate JSON for Mustache");
+    if (auto JSONGenerator = findGeneratorByName("json")) {
+      if (Error Err = JSONGenerator.get()->generateDocs(
+              RootDir, std::move(Infos), CDCtx))
+        return Err;
+    } else
+      return JSONGenerator.takeError();
+  }
 
-    sys::path::append(Path, Info->getFileBaseName() + ".html");
-    FileToInfos[Path].push_back(Info);
+  StringMap<json::Value> JSONFileMap;
+  {
+    llvm::TimeTraceScope TS("Iterate JSON files");
+    std::error_code EC;
+    sys::fs::directory_iterator JSONIter(RootDir, EC);
+    std::vector<json::Value> JSONFiles;
+    JSONFiles.reserve(Infos.size());
+    if (EC)
+      return createStringError("Failed to create directory iterator.");
+
+    while (JSONIter != sys::fs::directory_iterator()) {
+      if (EC)
+        return createStringError(EC, "Failed to iterate directory");
+
+      auto Path = StringRef(JSONIter->path());
+      if (!Path.ends_with(".json")) {
+        JSONIter.increment(EC);
+        continue;
+      }
+
+      auto File = MemoryBuffer::getFile(Path);
+      if ((EC = File.getError()))
+        continue;
+
+      auto Parsed = json::parse((*File)->getBuffer());
+      if (!Parsed)
+        return Parsed.takeError();
+      JSONFileMap.try_emplace(Path, *Parsed);
----------------
evelez7 wrote:

This patch introduces a noticeable performance impact, at least for me anecdotally. I cant run this on clang anymore since my machine OOMs. I'm guessing the JSON files are much larger than I thought they would be, since I've tried running with the Infos being deleted after the JSON generator is done and I still OOM. I'm wondering if something like discarding files after they've been converted is worth exploring.

https://github.com/llvm/llvm-project/pull/149006