[llvm] Support: Add vfs::OutputBackend and OutputFile to virtualize compiler outputs (PR #113363)

[Steven Wu via llvm-commits]

================
@@ -0,0 +1,598 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the VirtualOutputBackend types, including:
+/// * NullOutputBackend: Outputs to NullOutputBackend are discarded.
+/// * FilteringOutputBackend: Filter paths from output.
+/// * MirroringOutputBackend: Mirror the output into two different backend.
+/// * OnDiskOutputBackend: Write output files to disk.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/VirtualOutputError.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+void ProxyOutputBackend::anchor() {}
+void OnDiskOutputBackend::anchor() {}
+
+IntrusiveRefCntPtr<OutputBackend> vfs::makeNullOutputBackend() {
+  struct NullOutputBackend : public OutputBackend {
+    IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+      return const_cast<NullOutputBackend *>(this);
+    }
+    Expected<std::unique_ptr<OutputFileImpl>>
+    createFileImpl(StringRef Path, std::optional<OutputConfig>) override {
+      return std::make_unique<NullOutputFileImpl>();
+    }
+  };
+
+  return makeIntrusiveRefCnt<NullOutputBackend>();
+}
+
+IntrusiveRefCntPtr<OutputBackend> vfs::makeFilteringOutputBackend(
+    IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+    std::function<bool(StringRef, std::optional<OutputConfig>)> Filter) {
+  struct FilteringOutputBackend : public ProxyOutputBackend {
+    Expected<std::unique_ptr<OutputFileImpl>>
+    createFileImpl(StringRef Path,
+                   std::optional<OutputConfig> Config) override {
+      if (Filter(Path, Config))
+        return ProxyOutputBackend::createFileImpl(Path, Config);
+      return std::make_unique<NullOutputFileImpl>();
+    }
+
+    IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+      return makeIntrusiveRefCnt<FilteringOutputBackend>(
+          getUnderlyingBackend().clone(), Filter);
+    }
+
+    FilteringOutputBackend(
+        IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+        std::function<bool(StringRef, std::optional<OutputConfig>)> Filter)
+        : ProxyOutputBackend(std::move(UnderlyingBackend)),
+          Filter(std::move(Filter)) {
+      assert(this->Filter && "Expected a non-null function");
+    }
+    std::function<bool(StringRef, std::optional<OutputConfig>)> Filter;
+  };
+
+  return makeIntrusiveRefCnt<FilteringOutputBackend>(
+      std::move(UnderlyingBackend), std::move(Filter));
+}
+
+IntrusiveRefCntPtr<OutputBackend>
+vfs::makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+                                IntrusiveRefCntPtr<OutputBackend> Backend2) {
+  struct ProxyOutputBackend1 : public ProxyOutputBackend {
+    using ProxyOutputBackend::ProxyOutputBackend;
+  };
+  struct ProxyOutputBackend2 : public ProxyOutputBackend {
+    using ProxyOutputBackend::ProxyOutputBackend;
+  };
+  struct MirroringOutput final : public OutputFileImpl, raw_pwrite_stream {
+    Error keep() final {
+      flush();
+      return joinErrors(F1->keep(), F2->keep());
+    }
+    Error discard() final {
+      flush();
+      return joinErrors(F1->discard(), F2->discard());
+    }
+    raw_pwrite_stream &getOS() final { return *this; }
+
+    void write_impl(const char *Ptr, size_t Size) override {
+      F1->getOS().write(Ptr, Size);
+      F2->getOS().write(Ptr, Size);
+    }
+    void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+      this->flush();
+      F1->getOS().pwrite(Ptr, Size, Offset);
+      F2->getOS().pwrite(Ptr, Size, Offset);
+    }
+    uint64_t current_pos() const override { return F1->getOS().tell(); }
+    size_t preferred_buffer_size() const override {
+      return PreferredBufferSize;
+    }
+    void reserveExtraSpace(uint64_t ExtraSize) override {
+      F1->getOS().reserveExtraSpace(ExtraSize);
+      F2->getOS().reserveExtraSpace(ExtraSize);
+    }
+    bool is_displayed() const override {
+      return F1->getOS().is_displayed() && F2->getOS().is_displayed();
+    }
+    bool has_colors() const override {
+      return F1->getOS().has_colors() && F2->getOS().has_colors();
+    }
+    void enable_colors(bool enable) override {
+      raw_pwrite_stream::enable_colors(enable);
+      F1->getOS().enable_colors(enable);
+      F2->getOS().enable_colors(enable);
+    }
+
+    MirroringOutput(std::unique_ptr<OutputFileImpl> F1,
+                    std::unique_ptr<OutputFileImpl> F2)
+        : PreferredBufferSize(std::max(F1->getOS().GetBufferSize(),
+                                       F1->getOS().GetBufferSize())),
+          F1(std::move(F1)), F2(std::move(F2)) {
+      // Don't double buffer.
+      this->F1->getOS().SetUnbuffered();
+      this->F2->getOS().SetUnbuffered();
+    }
+    size_t PreferredBufferSize;
+    std::unique_ptr<OutputFileImpl> F1;
+    std::unique_ptr<OutputFileImpl> F2;
+  };
+  struct MirroringOutputBackend : public ProxyOutputBackend1,
+                                  public ProxyOutputBackend2 {
+    Expected<std::unique_ptr<OutputFileImpl>>
+    createFileImpl(StringRef Path,
+                   std::optional<OutputConfig> Config) override {
+      std::unique_ptr<OutputFileImpl> File1;
+      std::unique_ptr<OutputFileImpl> File2;
+      if (Error E =
+              ProxyOutputBackend1::createFileImpl(Path, Config).moveInto(File1))
+        return std::move(E);
+      if (Error E =
+              ProxyOutputBackend2::createFileImpl(Path, Config).moveInto(File2))
+        return joinErrors(std::move(E), File1->discard());
+
+      // Skip the extra indirection if one of these is a null output.
+      if (isa<NullOutputFileImpl>(*File1)) {
+        consumeError(File1->discard());
+        return std::move(File2);
+      }
+      if (isa<NullOutputFileImpl>(*File2)) {
+        consumeError(File2->discard());
+        return std::move(File1);
+      }
+      return std::make_unique<MirroringOutput>(std::move(File1),
+                                               std::move(File2));
+    }
+
+    IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+      return IntrusiveRefCntPtr<ProxyOutputBackend1>(
+          makeIntrusiveRefCnt<MirroringOutputBackend>(
+              ProxyOutputBackend1::getUnderlyingBackend().clone(),
+              ProxyOutputBackend2::getUnderlyingBackend().clone()));
+    }
+    void Retain() const { ProxyOutputBackend1::Retain(); }
+    void Release() const { ProxyOutputBackend1::Release(); }
+
+    MirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+                           IntrusiveRefCntPtr<OutputBackend> Backend2)
+        : ProxyOutputBackend1(std::move(Backend1)),
+          ProxyOutputBackend2(std::move(Backend2)) {}
+  };
+
+  assert(Backend1 && "Expected actual backend");
+  assert(Backend2 && "Expected actual backend");
+  return IntrusiveRefCntPtr<ProxyOutputBackend1>(
+      makeIntrusiveRefCnt<MirroringOutputBackend>(std::move(Backend1),
+                                                  std::move(Backend2)));
+}
+
+static OutputConfig
+applySettings(std::optional<OutputConfig> &&Config,
+              const OnDiskOutputBackend::OutputSettings &Settings) {
+  if (!Config)
+    Config = Settings.DefaultConfig;
+  if (Settings.DisableTemporaries)
+    Config->setNoAtomicWrite();
+  if (Settings.DisableRemoveOnSignal)
+    Config->setNoDiscardOnSignal();
+  return *Config;
+}
+
+namespace {
+class OnDiskOutputFile final : public OutputFileImpl {
+public:
+  Error keep() override;
+  Error discard() override;
+  raw_pwrite_stream &getOS() override {
+    assert(FileOS && "Expected valid file");
+    if (BufferOS)
+      return *BufferOS;
+    return *FileOS;
+  }
+
+  /// Attempt to open a temporary file for \p OutputPath.
+  ///
+  /// This tries to open a uniquely-named temporary file for \p OutputPath,
+  /// possibly also creating any missing directories if \a
+  /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a
+  /// Config.
+  ///
+  /// \post FD and \a TempPath are initialized if this is successful.
+  Error tryToCreateTemporary(std::optional<int> &FD);
+
+  Error initializeFD(std::optional<int> &FD);
+  Error initializeStream();
+  Error reset();
+
+  OnDiskOutputFile(StringRef OutputPath, std::optional<OutputConfig> Config,
+                   const OnDiskOutputBackend::OutputSettings &Settings)
+      : Config(applySettings(std::move(Config), Settings)),
+        OutputPath(OutputPath.str()) {}
+
+  OutputConfig Config;
+  const std::string OutputPath;
+  std::optional<std::string> TempPath;
+  std::optional<raw_fd_ostream> FileOS;
+  std::optional<buffer_ostream> BufferOS;
+};
+} // end namespace
+
+static Error createDirectoriesOnDemand(StringRef OutputPath,
+                                       OutputConfig Config,
+                                       llvm::function_ref<Error()> CreateFile) {
+  return handleErrors(CreateFile(), [&](std::unique_ptr<ECError> EC) {
+    if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory ||
+        Config.getNoImplyCreateDirectories())
+      return Error(std::move(EC));
+
+    StringRef ParentPath = sys::path::parent_path(OutputPath);
+    if (std::error_code EC = sys::fs::create_directories(ParentPath))
+      return make_error<OutputError>(ParentPath, EC);
+    return CreateFile();
+  });
+}
+
+Error OnDiskOutputFile::tryToCreateTemporary(std::optional<int> &FD) {
+  // Create a temporary file.
+  // Insert -%%%%%%%% before the extension (if any), and because some tools
+  // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build
+  // artifacts, also append .tmp.
+  StringRef OutputExtension = sys::path::extension(OutputPath);
+  SmallString<128> ModelPath =
+      StringRef(OutputPath).drop_back(OutputExtension.size());
+  ModelPath += "-%%%%%%%%";
+  ModelPath += OutputExtension;
+  ModelPath += ".tmp";
+
+  return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error {
+    int NewFD;
+    SmallString<128> UniquePath;
+    if (std::error_code EC =
+            sys::fs::createUniqueFile(ModelPath, NewFD, UniquePath))
+      return make_error<TempFileOutputError>(ModelPath, OutputPath, EC);
+
+    if (Config.getDiscardOnSignal())
+      sys::RemoveFileOnSignal(UniquePath);
+
+    TempPath = UniquePath.str().str();
+    FD.emplace(NewFD);
+    return Error::success();
+  });
+}
+
+Error OnDiskOutputFile::initializeFD(std::optional<int> &FD) {
----------------
cachemeifyoucan wrote:

Rename to `initializeFile` but keep the variable name since we still convert fileHandle to `FD` in LLVM sys::fs APIs. 

https://github.com/llvm/llvm-project/pull/113363