[lld] [llvm] [DTLTO] Overlap temporary file removal (PR #209423)

[Andrew Ng via llvm-commits]

================
@@ -27,27 +27,61 @@
 #include "llvm/Support/raw_ostream.h"
 
 #include <string>
+#include <system_error>
+#include <utility>
+#include <vector>
 
 using namespace llvm;
 
+// Experimentation showed that serial deletion is most efficient, hence
+// a single thread.
+lto::DTLTO::BackgroundDeletion::BackgroundDeletion()
+    : DefaultThreadPool(hardware_concurrency(1)) {}
+
+lto::DTLTO::BackgroundDeletion::~BackgroundDeletion() { waitForTasks(); }
+
+void lto::DTLTO::BackgroundDeletion::waitForTasks() {
+  wait();
+  for (const std::string &Warning : Warnings)
+    errs() << "warning: could not remove the file " << Warning << "\n";
+  Warnings.clear();
+}
+
+void lto::DTLTO::BackgroundDeletion::remove(std::vector<std::string> &&Files,
+                                            const Config &Conf) {
+  if (Files.empty())
+    return;
+
+  async([this, Files = std::move(Files), TTE = Conf.TimeTraceEnabled,
+         TTG = Conf.TimeTraceGranularity] {
+    if (LLVM_ENABLE_THREADS && TTE)
+      timeTraceProfilerInitialize(TTG, "Remove DTLTO temporary files");
+    {
+      TimeTraceScope TimeScope("Remove DTLTO temporary files");
+      for (const std::string &Path : Files) {
+        std::error_code EC = sys::fs::remove(Path, true);
+        if (!EC ||
+            EC == std::make_error_code(std::errc::no_such_file_or_directory))
+          continue;
+
+        Warnings.emplace_back("'" + Path + "': " + EC.message());
+      }
+    }
+    if (LLVM_ENABLE_THREADS && TTE)
+      timeTraceProfilerFinishThread();
+  });
+}
+
+void lto::DTLTO::cleanupAfterRun() { BackgroundDeleter.waitForTasks(); }
+
 // Remove temporary files created to enable distribution.
 void lto::DTLTO::cleanup() {
-  if (!SaveTemps) {
-    // Remove one file, report error if any.
-    auto removeFile = [](StringRef FileName) -> void {
-      std::error_code EC = sys::fs::remove(FileName, true);
-      if (EC &&
-          EC != std::make_error_code(std::errc::no_such_file_or_directory))
-        errs() << "warning: could not remove the file '" << FileName
-               << "': " << EC.message() << "\n";
-    };
-
-    TimeTraceScope JobScope("Remove DTLTO temporary files");
-    for (const auto &Name : CleanupList)
-      removeFile(Name);
-    // Clean the CleanupList for safety.
-    CleanupList.clear();
-  }
+  if (SaveTemps)
+    return;
+
+  BackgroundDeleter.remove(std::move(CleanupList), Conf);
+  // Clean the CleanupList for safety.
+  CleanupList.clear();
----------------
nga888 wrote:

Doesn't the `std::move(CleanupList)` above make the `clear()` redundant? Also at first glance, I read this as "remove" `CleanupList` from the `BackgroundDeleter`, it's a little bit confusing but probably OK.

https://github.com/llvm/llvm-project/pull/209423