[clang] [Support] Add VirtualOutputBackends to virtualize the output from tools (PR #68447)
Steven Wu via cfe-commits
cfe-commits at lists.llvm.org
Fri Oct 6 13:55:55 PDT 2023
https://github.com/cachemeifyoucan created https://github.com/llvm/llvm-project/pull/68447
Add output backend that can be used to virtualize outputs from all LLVM based tools. This can easily allow functions like:
* redirect the compiler output
* duplicating the compiler output
* output to memory instead of to file system
* set the output to NULL
* etc.
This change also makes creating different kind of output from LLVM tools easier, and also provides correct error handling from the process.
>From f3abfdc41bf1da9173d8c4613274c239f6683502 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Thu, 5 Oct 2023 13:05:49 -0700
Subject: [PATCH 1/3] Support: Add proxies for raw_ostream and
raw_pwrite_stream
Add proxies classes for `raw_ostream` and `raw_pwrite_stream` called
`raw_ostream_proxy` and `raw_pwrite_stream_proxy`. Add adaptor classes,
`raw_ostream_proxy_adaptor<>` and `raw_pwrite_stream_proxy_adaptor<>`,
to allow subclasses to use a different parent class than `raw_ostream`
or `raw_pwrite_stream`.
The adaptors are used by a future patch to help a subclass of
`llvm::vfs::OutputFile`, an abstract subclass of `raw_pwrite_stream`, to
proxy a `raw_fd_ostream`.
Patched by dexonsmith.
Differential Revision: https://reviews.llvm.org/D133503
---
llvm/include/llvm/Support/raw_ostream_proxy.h | 158 +++++++++++++
llvm/lib/Support/CMakeLists.txt | 1 +
llvm/lib/Support/raw_ostream_proxy.cpp | 15 ++
llvm/unittests/Support/CMakeLists.txt | 1 +
.../Support/raw_ostream_proxy_test.cpp | 219 ++++++++++++++++++
5 files changed, 394 insertions(+)
create mode 100644 llvm/include/llvm/Support/raw_ostream_proxy.h
create mode 100644 llvm/lib/Support/raw_ostream_proxy.cpp
create mode 100644 llvm/unittests/Support/raw_ostream_proxy_test.cpp
diff --git a/llvm/include/llvm/Support/raw_ostream_proxy.h b/llvm/include/llvm/Support/raw_ostream_proxy.h
new file mode 100644
index 000000000000000..093d0a927833f1d
--- /dev/null
+++ b/llvm/include/llvm/Support/raw_ostream_proxy.h
@@ -0,0 +1,158 @@
+//===- raw_ostream_proxy.h - Proxies for raw output streams -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_RAW_OSTREAM_PROXY_H
+#define LLVM_SUPPORT_RAW_OSTREAM_PROXY_H
+
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm {
+
+/// Common bits for \a raw_ostream_proxy_adaptor<>, split out to dedup in
+/// template instantions.
+class raw_ostream_proxy_adaptor_base {
+protected:
+ raw_ostream_proxy_adaptor_base() = delete;
+ raw_ostream_proxy_adaptor_base(const raw_ostream_proxy_adaptor_base &) =
+ delete;
+
+ explicit raw_ostream_proxy_adaptor_base(raw_ostream &OS)
+ : OS(&OS), PreferredBufferSize(OS.GetBufferSize()) {
+ // Drop OS's buffer to make this->flush() forward. This proxy will add a
+ // buffer in its place.
+ OS.SetUnbuffered();
+ }
+
+ ~raw_ostream_proxy_adaptor_base() {
+ assert(!OS && "Derived objects should call resetProxiedOS()");
+ }
+
+ /// Stop proxying the stream, taking the derived object by reference as \p
+ /// ThisProxyOS. Updates \p ThisProxyOS to stop buffering before setting \a
+ /// OS to \c nullptr, ensuring that future writes crash immediately.
+ void resetProxiedOS(raw_ostream &ThisProxyOS) {
+ ThisProxyOS.SetUnbuffered();
+ OS = nullptr;
+ }
+
+ bool hasProxiedOS() const { return OS; }
+ raw_ostream &getProxiedOS() const {
+ assert(OS && "raw_ostream_proxy_adaptor use after reset");
+ return *OS;
+ }
+ size_t getPreferredBufferSize() const { return PreferredBufferSize; }
+
+private:
+ raw_ostream *OS;
+
+ /// Caches the value of OS->GetBufferSize() at construction time.
+ size_t PreferredBufferSize;
+};
+
+/// Adaptor to create a stream class that proxies another \a raw_ostream.
+///
+/// Use \a raw_ostream_proxy_adaptor<> directly to implement an abstract
+/// derived class of \a raw_ostream as a proxy. Otherwise use \a
+/// raw_ostream_proxy.
+///
+/// Most operations are forwarded to the proxied stream.
+///
+/// If the proxied stream is buffered, the buffer is dropped and moved to this
+/// stream. This allows \a flush() to work correctly, flushing immediately from
+/// the proxy through to the final stream, and avoids any wasteful
+/// double-buffering.
+///
+/// \a enable_colors() changes both the proxied stream and the proxy itself.
+/// \a is_displayed() and \a has_colors() are forwarded to the proxy. \a
+/// changeColor(), resetColor(), and \a reverseColor() are not forwarded, since
+/// they need to call \a flush() and the buffer lives in the proxy.
+template <class RawOstreamT = raw_ostream>
+class raw_ostream_proxy_adaptor : public RawOstreamT,
+ public raw_ostream_proxy_adaptor_base {
+ void write_impl(const char *Ptr, size_t Size) override {
+ getProxiedOS().write(Ptr, Size);
+ }
+ uint64_t current_pos() const override { return getProxiedOS().tell(); }
+ size_t preferred_buffer_size() const override {
+ return getPreferredBufferSize();
+ }
+
+public:
+ void reserveExtraSpace(uint64_t ExtraSize) override {
+ getProxiedOS().reserveExtraSpace(ExtraSize);
+ }
+ bool is_displayed() const override { return getProxiedOS().is_displayed(); }
+ bool has_colors() const override { return getProxiedOS().has_colors(); }
+ void enable_colors(bool enable) override {
+ RawOstreamT::enable_colors(enable);
+ getProxiedOS().enable_colors(enable);
+ }
+
+ ~raw_ostream_proxy_adaptor() override { resetProxiedOS(); }
+
+protected:
+ template <class... ArgsT>
+ explicit raw_ostream_proxy_adaptor(raw_ostream &OS, ArgsT &&...Args)
+ : RawOstreamT(std::forward<ArgsT>(Args)...),
+ raw_ostream_proxy_adaptor_base(OS) {}
+
+ /// Stop proxying the stream. Flush and set up a crash for future writes.
+ ///
+ /// For example, this can simplify logic when a subclass might have a longer
+ /// lifetime than the stream it proxies.
+ void resetProxiedOS() {
+ raw_ostream_proxy_adaptor_base::resetProxiedOS(*this);
+ }
+ void resetProxiedOS(raw_ostream &) = delete;
+};
+
+/// Adaptor for creating a stream that proxies a \a raw_pwrite_stream.
+template <class RawPwriteStreamT = raw_pwrite_stream>
+class raw_pwrite_stream_proxy_adaptor
+ : public raw_ostream_proxy_adaptor<RawPwriteStreamT> {
+ using RawOstreamAdaptorT = raw_ostream_proxy_adaptor<RawPwriteStreamT>;
+
+ void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+ this->flush();
+ getProxiedOS().pwrite(Ptr, Size, Offset);
+ }
+
+protected:
+ raw_pwrite_stream_proxy_adaptor() = default;
+ template <class... ArgsT>
+ explicit raw_pwrite_stream_proxy_adaptor(raw_pwrite_stream &OS,
+ ArgsT &&...Args)
+ : RawOstreamAdaptorT(OS, std::forward<ArgsT>(Args)...) {}
+
+ raw_pwrite_stream &getProxiedOS() const {
+ return static_cast<raw_pwrite_stream &>(RawOstreamAdaptorT::getProxiedOS());
+ }
+};
+
+/// Non-owning proxy for a \a raw_ostream. Enables passing a stream into of an
+/// API that takes ownership.
+class raw_ostream_proxy : public raw_ostream_proxy_adaptor<> {
+ void anchor() override;
+
+public:
+ raw_ostream_proxy(raw_ostream &OS) : raw_ostream_proxy_adaptor<>(OS) {}
+};
+
+/// Non-owning proxy for a \a raw_pwrite_stream. Enables passing a stream
+/// into of an API that takes ownership.
+class raw_pwrite_stream_proxy : public raw_pwrite_stream_proxy_adaptor<> {
+ void anchor() override;
+
+public:
+ raw_pwrite_stream_proxy(raw_pwrite_stream &OS)
+ : raw_pwrite_stream_proxy_adaptor<>(OS) {}
+};
+
+} // end namespace llvm
+
+#endif // LLVM_SUPPORT_RAW_OSTREAM_PROXY_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index b96d62c7a6224d6..2a1903a1a960682 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -252,6 +252,7 @@ add_llvm_component_library(LLVMSupport
YAMLTraits.cpp
raw_os_ostream.cpp
raw_ostream.cpp
+ raw_ostream_proxy.cpp
regcomp.c
regerror.c
regexec.c
diff --git a/llvm/lib/Support/raw_ostream_proxy.cpp b/llvm/lib/Support/raw_ostream_proxy.cpp
new file mode 100644
index 000000000000000..2bbaa82f4afa7a4
--- /dev/null
+++ b/llvm/lib/Support/raw_ostream_proxy.cpp
@@ -0,0 +1,15 @@
+//===- raw_ostream_proxy.cpp - Implement the raw_ostream proxies ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/raw_ostream_proxy.h"
+
+using namespace llvm;
+
+void raw_ostream_proxy::anchor() {}
+
+void raw_pwrite_stream_proxy::anchor() {}
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 12f2e9959326045..1307907e7640af1 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -99,6 +99,7 @@ add_llvm_unittest(SupportTests
formatted_raw_ostream_test.cpp
raw_fd_stream_test.cpp
raw_ostream_test.cpp
+ raw_ostream_proxy_test.cpp
raw_pwrite_stream_test.cpp
raw_sha1_ostream_test.cpp
xxhashTest.cpp
diff --git a/llvm/unittests/Support/raw_ostream_proxy_test.cpp b/llvm/unittests/Support/raw_ostream_proxy_test.cpp
new file mode 100644
index 000000000000000..ee97fe65b660038
--- /dev/null
+++ b/llvm/unittests/Support/raw_ostream_proxy_test.cpp
@@ -0,0 +1,219 @@
+//===- raw_ostream_proxy_test.cpp - Tests for raw ostream proxies ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/raw_ostream_proxy.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+/// Naive version of raw_svector_ostream that is buffered (by default) and
+/// doesn't support pwrite.
+class BufferedNoPwriteSmallVectorStream : public raw_ostream {
+public:
+ // Choose a strange buffer size to ensure it doesn't collide with the default
+ // on \a raw_ostream.
+ constexpr static const size_t PreferredBufferSize = 63;
+
+ size_t preferred_buffer_size() const override { return PreferredBufferSize; }
+ uint64_t current_pos() const override { return Vector.size(); }
+ void write_impl(const char *Ptr, size_t Size) override {
+ Vector.append(Ptr, Ptr + Size);
+ }
+
+ bool is_displayed() const override { return IsDisplayed; }
+
+ explicit BufferedNoPwriteSmallVectorStream(SmallVectorImpl<char> &Vector)
+ : Vector(Vector) {}
+ ~BufferedNoPwriteSmallVectorStream() override { flush(); }
+
+ SmallVectorImpl<char> &Vector;
+ bool IsDisplayed = false;
+};
+
+constexpr const size_t BufferedNoPwriteSmallVectorStream::PreferredBufferSize;
+
+TEST(raw_ostream_proxyTest, write) {
+ // Besides confirming that "write" works, this test confirms that the proxy
+ // takes on the buffer from the stream it's proxying, such that writes to the
+ // proxy are flushed to the underlying stream as if no proxy were present.
+ SmallString<128> Dest;
+ {
+ // Confirm that BufferedNoPwriteSmallVectorStream is buffered by default,
+ // and that setting up a proxy effectively transfers a buffer of the same
+ // size to the proxy.
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ EXPECT_EQ(BufferedNoPwriteSmallVectorStream::PreferredBufferSize,
+ DestOS.GetBufferSize());
+ raw_ostream_proxy ProxyOS(DestOS);
+ EXPECT_EQ(0u, DestOS.GetBufferSize());
+ EXPECT_EQ(BufferedNoPwriteSmallVectorStream::PreferredBufferSize,
+ ProxyOS.GetBufferSize());
+
+ // Flushing should send through to Dest.
+ ProxyOS << "abcd";
+ EXPECT_EQ("", Dest);
+ ProxyOS.flush();
+ EXPECT_EQ("abcd", Dest);
+
+ // Buffer should still work.
+ ProxyOS << "e";
+ EXPECT_EQ("abcd", Dest);
+ }
+
+ // Destructing ProxyOS should flush (and not crash).
+ EXPECT_EQ("abcde", Dest);
+
+ {
+ // Set up another stream, this time unbuffered.
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ DestOS.SetUnbuffered();
+ EXPECT_EQ(0u, DestOS.GetBufferSize());
+ raw_ostream_proxy ProxyOS(DestOS);
+ EXPECT_EQ(0u, DestOS.GetBufferSize());
+ EXPECT_EQ(0u, ProxyOS.GetBufferSize());
+
+ // Flushing should not be required.
+ ProxyOS << "f";
+ EXPECT_EQ("abcdef", Dest);
+ }
+ EXPECT_EQ("abcdef", Dest);
+}
+
+TEST(raw_ostream_proxyTest, pwrite) {
+ // This test confirms that the proxy takes on the buffer from the stream it's
+ // proxying, such that writes to the proxy are flushed to the underlying
+ // stream as if no proxy were present.
+ SmallString<128> Dest;
+ raw_svector_ostream DestOS(Dest);
+ raw_pwrite_stream_proxy ProxyOS(DestOS);
+ EXPECT_EQ(0u, ProxyOS.GetBufferSize());
+
+ // Get some initial data.
+ ProxyOS << "abcd";
+ EXPECT_EQ("abcd", Dest);
+
+ // Confirm that pwrite works.
+ ProxyOS.pwrite("BC", 2, 1);
+ EXPECT_EQ("aBCd", Dest);
+}
+
+TEST(raw_ostream_proxyTest, pwriteWithBuffer) {
+ // This test confirms that when a buffer is configured, pwrite still works.
+ SmallString<128> Dest;
+ raw_svector_ostream DestOS(Dest);
+ DestOS.SetBufferSize(256);
+ EXPECT_EQ(256u, DestOS.GetBufferSize());
+
+ // Confirm that the proxy steals the buffer.
+ raw_pwrite_stream_proxy ProxyOS(DestOS);
+ EXPECT_EQ(0u, DestOS.GetBufferSize());
+ EXPECT_EQ(256u, ProxyOS.GetBufferSize());
+
+ // Check that the buffer is working.
+ ProxyOS << "abcd";
+ EXPECT_EQ("", Dest);
+
+ // Confirm that pwrite flushes.
+ ProxyOS.pwrite("BC", 2, 1);
+ EXPECT_EQ("aBCd", Dest);
+}
+
+class ProxyWithReset : public raw_ostream_proxy_adaptor<> {
+public:
+ ProxyWithReset(raw_ostream &OS) : raw_ostream_proxy_adaptor<>(OS) {}
+
+ // Allow this to be called outside the class.
+ using raw_ostream_proxy_adaptor<>::hasProxiedOS;
+ using raw_ostream_proxy_adaptor<>::getProxiedOS;
+ using raw_ostream_proxy_adaptor<>::resetProxiedOS;
+};
+
+TEST(raw_ostream_proxyTest, resetProxiedOS) {
+ // Confirm that base classes can drop the proxied OS before destruction and
+ // get consistent crashes.
+ SmallString<128> Dest;
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ ProxyWithReset ProxyOS(DestOS);
+ EXPECT_TRUE(ProxyOS.hasProxiedOS());
+ EXPECT_EQ(&DestOS, &ProxyOS.getProxiedOS());
+
+ // Write some data.
+ ProxyOS << "abcd";
+ EXPECT_EQ("", Dest);
+
+ // Reset the underlying stream.
+ ProxyOS.resetProxiedOS();
+ EXPECT_EQ("abcd", Dest);
+ EXPECT_EQ(0u, ProxyOS.GetBufferSize());
+ EXPECT_FALSE(ProxyOS.hasProxiedOS());
+
+#if GTEST_HAS_DEATH_TEST
+ EXPECT_DEATH(ProxyOS << "e", "use after reset");
+ EXPECT_DEATH(ProxyOS.getProxiedOS(), "use after reset");
+#endif
+}
+
+TEST(raw_ostream_proxyTest, ColorMode) {
+ {
+ SmallString<128> Dest;
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ raw_ostream_proxy ProxyOS(DestOS);
+ ProxyOS.enable_colors(true);
+
+ WithColor(ProxyOS, HighlightColor::Error, ColorMode::Disable) << "test";
+ EXPECT_EQ("", Dest);
+ ProxyOS.flush();
+ EXPECT_EQ("test", Dest);
+ }
+
+ {
+ SmallString<128> Dest;
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ raw_ostream_proxy ProxyOS(DestOS);
+ ProxyOS.enable_colors(true);
+
+ WithColor(ProxyOS, HighlightColor::Error, ColorMode::Auto) << "test";
+ EXPECT_EQ("", Dest);
+ ProxyOS.flush();
+ EXPECT_EQ("test", Dest);
+ }
+
+#ifdef LLVM_ON_UNIX
+ {
+ SmallString<128> Dest;
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ raw_ostream_proxy ProxyOS(DestOS);
+ ProxyOS.enable_colors(true);
+
+ WithColor(ProxyOS, HighlightColor::Error, ColorMode::Enable) << "test";
+ EXPECT_EQ("", Dest);
+ ProxyOS.flush();
+ EXPECT_EQ("\x1B[0;1;31mtest\x1B[0m", Dest);
+ }
+
+ {
+ SmallString<128> Dest;
+ BufferedNoPwriteSmallVectorStream DestOS(Dest);
+ DestOS.IsDisplayed = true;
+ raw_ostream_proxy ProxyOS(DestOS);
+ ProxyOS.enable_colors(true);
+
+ WithColor(ProxyOS, HighlightColor::Error, ColorMode::Auto) << "test";
+ EXPECT_EQ("", Dest);
+ ProxyOS.flush();
+ EXPECT_EQ("\x1B[0;1;31mtest\x1B[0m", Dest);
+ }
+#endif
+}
+
+} // end namespace
>From 28c9f9c6b72feeb55de9637e458b80f34936d9ea Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Thu, 5 Oct 2023 13:06:30 -0700
Subject: [PATCH 2/3] Support: Add vfs::OutputBackend and OutputFile to
virtualize compiler outputs
Add OutputBackend and OutputFile to the `llvm::vfs` namespace for
virtualizing compiler outputs. This is intended for use in Clang,
The headers are:
- llvm/Support/VirtualOutputConfig.h
- llvm/Support/VirtualOutputError.h
- llvm/Support/VirtualOutputFile.h
- llvm/Support/VirtualOutputBackend.h
OutputFile is moveable and owns an OutputFileImpl, which is provided by
the derived OutputBackend.
- OutputFileImpl::keep() and OutputFileImpl::discard() should keep or
discard the output. OutputFile guarantees that exactly one of these
will be called before destruction.
- OutputFile::keep() and OutputFile::discard() wrap OutputFileImpl
and catch usage errors such as double-close.
- OutputFile::discardOnDestroy() installs an error handler for the
destructor to use if the file is still open. The handler will be
called if discard() fails.
- OutputFile::~OutputFile() calls report_fatal_error() if none of
keep(), discard(), or discardOnDestroy() has been called. It still
calls OutputFileImpl::discard().
- getOS() returns the wrapped raw_pwrite_stream. For convenience,
OutputFile has an implicit conversion to `raw_ostream` and
`raw_ostream &operator<<(OutputFile&, T&&)`.
OutputBackend can be stored in IntrusiveRefCntPtr.
- Most operations are thread-safe.
- clone() returns a backend that targets the same destination.
All operations are thread-safe when done on different clones.
- createFile() takes a path and an OutputConfig (see below) and returns
an OutputFile. Backends implement createFileImpl().
OutputConfig has flags to configure the output. Backends may ignore or
override flags that aren't relevant or implementable.
- The initial flags are:
- AtomicWrite: whether the output should appear atomically (e.g., by
using a temporary file and renaming it).
- CrashCleanup: whether the output should be cleaned up if there's a
crash (e.g., with RemoveFileOnSignal).
- ImplyCreateDirectories: whether to implicitly create missing
directories in the path to the file.
- Text: matches sys::fs::OF_Text.
- CRLF: matches sys::fs::OF_CRLF.
- Append: matches sys::fs::OF_Append and can use with AtomicWrite
for atomic append.
- Each "Flag" has `setFlag(bool)` and `bool getFlag()` and shortcuts
`setFlag()` and `setNoFlag()`. The setters are `constexpr` and return
`OutputConfig&` to make it easy to declare a default value for a filed
in a class or struct.
- Setters and getters for Binary and TextWithCRLF are derived from Text
and CRLF. For convenience, sys::fs::OpenFlags can be passed
directly to setOpenFlags().
This patch intentionally lacks a number of important features that have
been left for follow-ups:
- Set a (virtual) current working directory.
- Create a directory.
- Create a file or directory with a unique name (avoiding collisions
with existing filenames).
Patch originally by dexonsmith
---
.../llvm/Support/HashingOutputBackend.h | 112 +++
.../llvm/Support/VirtualOutputBackend.h | 62 ++
.../llvm/Support/VirtualOutputBackends.h | 110 +++
.../llvm/Support/VirtualOutputConfig.def | 26 +
.../llvm/Support/VirtualOutputConfig.h | 91 ++
.../include/llvm/Support/VirtualOutputError.h | 134 +++
llvm/include/llvm/Support/VirtualOutputFile.h | 162 ++++
llvm/lib/Support/CMakeLists.txt | 5 +
llvm/lib/Support/VirtualOutputBackend.cpp | 38 +
llvm/lib/Support/VirtualOutputBackends.cpp | 594 ++++++++++++
llvm/lib/Support/VirtualOutputConfig.cpp | 50 +
llvm/lib/Support/VirtualOutputError.cpp | 53 ++
llvm/lib/Support/VirtualOutputFile.cpp | 106 +++
llvm/unittests/Support/CMakeLists.txt | 4 +
.../Support/VirtualOutputBackendTest.cpp | 147 +++
.../Support/VirtualOutputBackendsTest.cpp | 886 ++++++++++++++++++
.../Support/VirtualOutputConfigTest.cpp | 152 +++
.../Support/VirtualOutputFileTest.cpp | 342 +++++++
18 files changed, 3074 insertions(+)
create mode 100644 llvm/include/llvm/Support/HashingOutputBackend.h
create mode 100644 llvm/include/llvm/Support/VirtualOutputBackend.h
create mode 100644 llvm/include/llvm/Support/VirtualOutputBackends.h
create mode 100644 llvm/include/llvm/Support/VirtualOutputConfig.def
create mode 100644 llvm/include/llvm/Support/VirtualOutputConfig.h
create mode 100644 llvm/include/llvm/Support/VirtualOutputError.h
create mode 100644 llvm/include/llvm/Support/VirtualOutputFile.h
create mode 100644 llvm/lib/Support/VirtualOutputBackend.cpp
create mode 100644 llvm/lib/Support/VirtualOutputBackends.cpp
create mode 100644 llvm/lib/Support/VirtualOutputConfig.cpp
create mode 100644 llvm/lib/Support/VirtualOutputError.cpp
create mode 100644 llvm/lib/Support/VirtualOutputFile.cpp
create mode 100644 llvm/unittests/Support/VirtualOutputBackendTest.cpp
create mode 100644 llvm/unittests/Support/VirtualOutputBackendsTest.cpp
create mode 100644 llvm/unittests/Support/VirtualOutputConfigTest.cpp
create mode 100644 llvm/unittests/Support/VirtualOutputFileTest.cpp
diff --git a/llvm/include/llvm/Support/HashingOutputBackend.h b/llvm/include/llvm/Support/HashingOutputBackend.h
new file mode 100644
index 000000000000000..d2e79663f552666
--- /dev/null
+++ b/llvm/include/llvm/Support/HashingOutputBackend.h
@@ -0,0 +1,112 @@
+//===- HashingOutputBackends.h - Hashing output backends --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/HashBuilder.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm::vfs {
+
+/// raw_pwrite_stream that writes to a hasher.
+template <typename HasherT>
+class HashingStream : public llvm::raw_pwrite_stream {
+private:
+ SmallVector<char> Buffer;
+ raw_svector_ostream OS;
+
+ using HashBuilderT = HashBuilder<HasherT, support::endianness::native>;
+ HashBuilderT Builder;
+
+ void write_impl(const char *Ptr, size_t Size) override {
+ OS.write(Ptr, Size);
+ }
+
+ void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+ OS.pwrite(Ptr, Size, Offset);
+ }
+
+ uint64_t current_pos() const override { return OS.str().size(); }
+
+public:
+ HashingStream() : OS(Buffer) { SetUnbuffered(); }
+
+ auto final() {
+ Builder.update(OS.str());
+ return Builder.final();
+ }
+};
+
+template <typename HasherT> class HashingOutputFile;
+
+/// An output backend that only generates the hash for outputs.
+template <typename HasherT> class HashingOutputBackend : public OutputBackend {
+private:
+ friend class HashingOutputFile<HasherT>;
+ void addOutputFile(StringRef Path, StringRef Hash) {
+ OutputHashes[Path] = std::string(Hash);
+ }
+
+protected:
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ return const_cast<HashingOutputBackend<HasherT> *>(this);
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+ return std::make_unique<HashingOutputFile<HasherT>>(Path, *this);
+ }
+
+public:
+ /// Iterator for all the output file names.
+ auto outputFiles() const { return OutputHashes.keys(); }
+
+ /// Get hash value for the output files in hex representation.
+ /// Return None if the requested path is not generated.
+ std::optional<std::string> getHashValueForFile(StringRef Path) {
+ auto F = OutputHashes.find(Path);
+ if (F == OutputHashes.end())
+ return std::nullopt;
+ return toHex(F->second);
+ }
+
+private:
+ StringMap<std::string> OutputHashes;
+};
+
+/// HashingOutputFile.
+template <typename HasherT>
+class HashingOutputFile final : public OutputFileImpl {
+public:
+ Error keep() override {
+ auto Result = OS.final();
+ Backend.addOutputFile(OutputPath, toStringRef(Result));
+ return Error::success();
+ }
+ Error discard() override { return Error::success(); }
+ raw_pwrite_stream &getOS() override { return OS; }
+
+ HashingOutputFile(StringRef OutputPath,
+ HashingOutputBackend<HasherT> &Backend)
+ : OutputPath(OutputPath.str()), Backend(Backend) {}
+
+private:
+ const std::string OutputPath;
+ HashingStream<HasherT> OS;
+ HashingOutputBackend<HasherT> &Backend;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
diff --git a/llvm/include/llvm/Support/VirtualOutputBackend.h b/llvm/include/llvm/Support/VirtualOutputBackend.h
new file mode 100644
index 000000000000000..2328252c7054fcb
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputBackend.h
@@ -0,0 +1,62 @@
+//===- VirtualOutputBackend.h - Output virtualization -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/VirtualOutputFile.h"
+
+namespace llvm::vfs {
+
+/// Interface for virtualized outputs.
+///
+/// If virtual functions are added here, also add them to \a
+/// ProxyOutputBackend.
+class OutputBackend : public RefCountedBase<OutputBackend> {
+ virtual void anchor();
+
+public:
+ /// Get a backend that points to the same destination as this one but that
+ /// has independent settings.
+ ///
+ /// Not thread-safe, but all operations are thread-safe when performed on
+ /// separate clones of the same backend.
+ IntrusiveRefCntPtr<OutputBackend> clone() const { return cloneImpl(); }
+
+ /// Create a file. If \p Config is \c std::nullopt, uses the backend's default
+ /// OutputConfig (may match \a OutputConfig::OutputConfig(), or may
+ /// have been customized).
+ ///
+ /// Thread-safe.
+ Expected<OutputFile>
+ createFile(const Twine &Path,
+ std::optional<OutputConfig> Config = std::nullopt);
+
+protected:
+ /// Must be thread-safe. Virtual function has a different name than \a
+ /// clone() so that implementations can override the return value.
+ virtual IntrusiveRefCntPtr<OutputBackend> cloneImpl() const = 0;
+
+ /// Create a file for \p Path. Must be thread-safe.
+ ///
+ /// \pre \p Config is valid or std::nullopt.
+ virtual Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) = 0;
+
+ OutputBackend() = default;
+
+public:
+ virtual ~OutputBackend() = default;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
diff --git a/llvm/include/llvm/Support/VirtualOutputBackends.h b/llvm/include/llvm/Support/VirtualOutputBackends.h
new file mode 100644
index 000000000000000..6f702000d77b3e4
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputBackends.h
@@ -0,0 +1,110 @@
+//===- VirtualOutputBackends.h - Virtual output backends --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+
+namespace llvm::vfs {
+
+/// Create a backend that ignores all output.
+IntrusiveRefCntPtr<OutputBackend> makeNullOutputBackend();
+
+/// Make a backend where \a OutputBackend::createFile() forwards to
+/// \p UnderlyingBackend when \p Filter is true, and otherwise returns a
+/// \a NullOutput.
+IntrusiveRefCntPtr<OutputBackend> makeFilteringOutputBackend(
+ IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+ std::function<bool(StringRef, std::optional<OutputConfig>)> Filter);
+
+/// Create a backend that forwards \a OutputBackend::createFile() to both \p
+/// Backend1 and \p Backend2 and sends content to both places.
+IntrusiveRefCntPtr<OutputBackend>
+makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+ IntrusiveRefCntPtr<OutputBackend> Backend2);
+
+/// A helper class for proxying another backend, with the default
+/// implementation to forward to the underlying backend.
+class ProxyOutputBackend : public OutputBackend {
+ void anchor() override;
+
+protected:
+ // Require subclass to implement cloneImpl().
+ //
+ // IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override;
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+ OutputFile File;
+ if (Error E = UnderlyingBackend->createFile(Path, Config).moveInto(File))
+ return std::move(E);
+ return File.takeImpl();
+ }
+
+ OutputBackend &getUnderlyingBackend() const { return *UnderlyingBackend; }
+
+public:
+ ProxyOutputBackend(IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend)
+ : UnderlyingBackend(std::move(UnderlyingBackend)) {
+ assert(this->UnderlyingBackend && "Expected non-null backend");
+ }
+
+private:
+ IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend;
+};
+
+/// An output backend that creates files on disk, wrapping APIs in sys::fs.
+class OnDiskOutputBackend : public OutputBackend {
+ void anchor() override;
+
+protected:
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ return clone();
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override;
+
+public:
+ /// Resolve an absolute path.
+ Error makeAbsolute(SmallVectorImpl<char> &Path) const;
+
+ /// On disk output settings.
+ struct OutputSettings {
+ /// Register output files to be deleted if a signal is received. Also
+ /// disabled for outputs with \a OutputConfig::getNoDiscardOnSignal().
+ bool DisableRemoveOnSignal = false;
+
+ /// Disable temporary files. Also disabled for outputs with \a
+ /// OutputConfig::getNoAtomicWrite().
+ bool DisableTemporaries = false;
+
+ // Default configuration for this backend.
+ OutputConfig DefaultConfig;
+ };
+
+ IntrusiveRefCntPtr<OnDiskOutputBackend> clone() const {
+ auto Clone = makeIntrusiveRefCnt<OnDiskOutputBackend>();
+ Clone->Settings = Settings;
+ return Clone;
+ }
+
+ OnDiskOutputBackend() = default;
+
+ /// Settings for this backend.
+ ///
+ /// Access is not thread-safe.
+ OutputSettings Settings;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.def b/llvm/include/llvm/Support/VirtualOutputConfig.def
new file mode 100644
index 000000000000000..0b6a765cbd4c00c
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputConfig.def
@@ -0,0 +1,26 @@
+//===- VirtualOutputConfig.def - Virtual output config defs -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HANDLE_OUTPUT_CONFIG_FLAG
+#error "Missing macro definition of HANDLE_OUTPUT_CONFIG_FLAG"
+#endif
+
+// Define HANDLE_OUTPUT_CONFIG_FLAG before including.
+//
+// #define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT)
+
+HANDLE_OUTPUT_CONFIG_FLAG(Text, false) // OF_Text.
+HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF.
+HANDLE_OUTPUT_CONFIG_FLAG(Append, false) // OF_Append.
+HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal.
+HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries.
+HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true)
+// Skip atomic write if existing file content is the same
+HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false)
+
+#undef HANDLE_OUTPUT_CONFIG_FLAG
diff --git a/llvm/include/llvm/Support/VirtualOutputConfig.h b/llvm/include/llvm/Support/VirtualOutputConfig.h
new file mode 100644
index 000000000000000..d93bbf5ca63a034
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputConfig.h
@@ -0,0 +1,91 @@
+//===- VirtualOutputConfig.h - Virtual output configuration -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H
+
+#include "llvm/Support/Compiler.h"
+#include <initializer_list>
+
+namespace llvm {
+
+class raw_ostream;
+
+namespace sys {
+namespace fs {
+enum OpenFlags : unsigned;
+} // end namespace fs
+} // end namespace sys
+
+namespace vfs {
+
+namespace detail {
+/// Unused and empty base class to allow OutputConfig constructor to be
+/// constexpr, with commas before every field's initializer.
+struct EmptyBaseClass {};
+} // namespace detail
+
+/// Full configuration for an output for use by the \a OutputBackend. Each
+/// configuration flag is either \c true or \c false.
+struct OutputConfig : detail::EmptyBaseClass {
+public:
+ void print(raw_ostream &OS) const;
+ void dump() const;
+
+#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \
+ constexpr bool get##NAME() const { return NAME; } \
+ constexpr bool getNo##NAME() const { return !NAME; } \
+ constexpr OutputConfig &set##NAME(bool Value) { \
+ NAME = Value; \
+ return *this; \
+ } \
+ constexpr OutputConfig &set##NAME() { return set##NAME(true); } \
+ constexpr OutputConfig &setNo##NAME() { return set##NAME(false); }
+#include "llvm/Support/VirtualOutputConfig.def"
+
+ constexpr OutputConfig &setBinary() { return setNoText().setNoCRLF(); }
+ constexpr OutputConfig &setTextWithCRLF() { return setText().setCRLF(); }
+ constexpr OutputConfig &setTextWithCRLF(bool Value) {
+ return Value ? setText().setCRLF() : setBinary();
+ }
+ constexpr bool getTextWithCRLF() const { return getText() && getCRLF(); }
+ constexpr bool getBinary() const { return !getText(); }
+
+ /// Updates Text and CRLF flags based on \a sys::fs::OF_Text and \a
+ /// sys::fs::OF_CRLF in \p Flags. Rejects CRLF without Text (calling
+ /// \a setBinary()).
+ OutputConfig &setOpenFlags(const sys::fs::OpenFlags &Flags);
+
+ constexpr OutputConfig()
+ : EmptyBaseClass()
+#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) , NAME(DEFAULT)
+#include "llvm/Support/VirtualOutputConfig.def"
+ {
+ }
+
+ constexpr bool operator==(OutputConfig RHS) const {
+#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \
+ if (NAME != RHS.NAME) \
+ return false;
+#include "llvm/Support/VirtualOutputConfig.def"
+ return true;
+ }
+ constexpr bool operator!=(OutputConfig RHS) const { return !operator==(RHS); }
+
+private:
+#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) bool NAME : 1;
+#include "llvm/Support/VirtualOutputConfig.def"
+};
+
+} // namespace vfs
+
+raw_ostream &operator<<(raw_ostream &OS, vfs::OutputConfig Config);
+
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTCONFIG_H
diff --git a/llvm/include/llvm/Support/VirtualOutputError.h b/llvm/include/llvm/Support/VirtualOutputError.h
new file mode 100644
index 000000000000000..5459fae4552d51f
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputError.h
@@ -0,0 +1,134 @@
+//===- VirtualOutputError.h - Errors for output virtualization --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTERROR_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTERROR_H
+
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+
+namespace llvm::vfs {
+
+const std::error_category &output_category();
+
+enum class OutputErrorCode {
+ // Error code 0 is absent. Use std::error_code() instead.
+ not_closed = 1,
+ invalid_config,
+ already_closed,
+ has_open_proxy,
+};
+
+inline std::error_code make_error_code(OutputErrorCode EV) {
+ return std::error_code(static_cast<int>(EV), output_category());
+}
+
+/// Error related to an \a OutputFile. Derives from \a ECError and adds \a
+/// getOutputPath().
+class OutputError : public ErrorInfo<OutputError, ECError> {
+ void anchor() override;
+
+public:
+ StringRef getOutputPath() const { return OutputPath; }
+ void log(raw_ostream &OS) const override {
+ OS << getOutputPath() << ": ";
+ ECError::log(OS);
+ }
+
+ // Used by ErrorInfo::classID.
+ static char ID;
+
+ OutputError(const Twine &OutputPath, std::error_code EC)
+ : ErrorInfo<OutputError, ECError>(EC), OutputPath(OutputPath.str()) {
+ assert(EC && "Cannot create OutputError from success EC");
+ }
+
+ OutputError(const Twine &OutputPath, OutputErrorCode EV)
+ : ErrorInfo<OutputError, ECError>(make_error_code(EV)),
+ OutputPath(OutputPath.str()) {
+ assert(EC && "Cannot create OutputError from success EC");
+ }
+
+private:
+ std::string OutputPath;
+};
+
+/// Return \a Error::success() or use \p OutputPath to create an \a
+/// OutputError, depending on \p EC.
+inline Error convertToOutputError(const Twine &OutputPath, std::error_code EC) {
+ if (EC)
+ return make_error<OutputError>(OutputPath, EC);
+ return Error::success();
+}
+
+/// Error related to an OutputConfig for an \a OutputFile. Derives from \a
+/// OutputError and adds \a getConfig().
+class OutputConfigError : public ErrorInfo<OutputConfigError, OutputError> {
+ void anchor() override;
+
+public:
+ OutputConfig getConfig() const { return Config; }
+ void log(raw_ostream &OS) const override {
+ OutputError::log(OS);
+ OS << ": " << Config;
+ }
+
+ // Used by ErrorInfo::classID.
+ static char ID;
+
+ OutputConfigError(OutputConfig Config, const Twine &OutputPath)
+ : ErrorInfo<OutputConfigError, OutputError>(
+ OutputPath, OutputErrorCode::invalid_config),
+ Config(Config) {}
+
+private:
+ OutputConfig Config;
+};
+
+/// Error related to a temporary file for an \a OutputFile. Derives from \a
+/// OutputError and adds \a getTempPath().
+class TempFileOutputError : public ErrorInfo<TempFileOutputError, OutputError> {
+ void anchor() override;
+
+public:
+ StringRef getTempPath() const { return TempPath; }
+ void log(raw_ostream &OS) const override {
+ OS << getTempPath() << " => ";
+ OutputError::log(OS);
+ }
+
+ // Used by ErrorInfo::classID.
+ static char ID;
+
+ TempFileOutputError(const Twine &TempPath, const Twine &OutputPath,
+ std::error_code EC)
+ : ErrorInfo<TempFileOutputError, OutputError>(OutputPath, EC),
+ TempPath(TempPath.str()) {}
+
+ TempFileOutputError(const Twine &TempPath, const Twine &OutputPath,
+ OutputErrorCode EV)
+ : ErrorInfo<TempFileOutputError, OutputError>(OutputPath, EV),
+ TempPath(TempPath.str()) {}
+
+private:
+ std::string TempPath;
+};
+
+/// Return \a Error::success() or use \p TempPath and \p OutputPath to create a
+/// \a TempFileOutputError, depending on \p EC.
+inline Error convertToTempFileOutputError(const Twine &TempPath,
+ const Twine &OutputPath,
+ std::error_code EC) {
+ if (EC)
+ return make_error<TempFileOutputError>(TempPath, OutputPath, EC);
+ return Error::success();
+}
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTERROR_H
diff --git a/llvm/include/llvm/Support/VirtualOutputFile.h b/llvm/include/llvm/Support/VirtualOutputFile.h
new file mode 100644
index 000000000000000..0bf6c58f30484da
--- /dev/null
+++ b/llvm/include/llvm/Support/VirtualOutputFile.h
@@ -0,0 +1,162 @@
+//===- VirtualOutputFile.h - Output file virtualization ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTFILE_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTFILE_H
+
+#include "llvm/ADT/FunctionExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ExtensibleRTTI.h"
+#include "llvm/Support/VirtualOutputError.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace llvm::vfs {
+
+class OutputFileImpl : public RTTIExtends<OutputFileImpl, RTTIRoot> {
+ void anchor() override;
+
+public:
+ static char ID;
+ virtual ~OutputFileImpl() = default;
+
+ virtual Error keep() = 0;
+ virtual Error discard() = 0;
+ virtual raw_pwrite_stream &getOS() = 0;
+};
+
+class NullOutputFileImpl final
+ : public RTTIExtends<NullOutputFileImpl, OutputFileImpl> {
+ void anchor() override;
+
+public:
+ static char ID;
+ Error keep() final { return Error::success(); }
+ Error discard() final { return Error::success(); }
+ raw_pwrite_stream &getOS() final { return OS; }
+
+private:
+ raw_null_ostream OS;
+};
+
+/// A virtualized output file that writes to a specific backend.
+///
+/// One of \a keep(), \a discard(), or \a discardOnDestroy() must be called
+/// before destruction.
+class OutputFile {
+public:
+ StringRef getPath() const { return Path; }
+
+ /// Check if \a keep() or \a discard() has already been called.
+ bool isOpen() const { return bool(Impl); }
+
+ explicit operator bool() const { return isOpen(); }
+
+ raw_pwrite_stream &getOS() {
+ assert(isOpen() && "Expected open output stream");
+ return Impl->getOS();
+ }
+ operator raw_pwrite_stream &() { return getOS(); }
+ template <class T> raw_ostream &operator<<(T &&V) {
+ return getOS() << std::forward<T>(V);
+ }
+
+ /// Keep an output. Errors if this fails.
+ ///
+ /// If it has already been closed, calls \a report_fatal_error().
+ ///
+ /// If there's an open proxy from \a createProxy(), calls \a discard() to
+ /// clean up temporaries followed by \a report_fatal_error().
+ Error keep();
+
+ /// Discard an output, cleaning up any temporary state. Errors if clean-up
+ /// fails.
+ ///
+ /// If it has already been closed, calls \a report_fatal_error().
+ Error discard();
+
+ /// Discard the output when destroying it if it's still open, sending the
+ /// result to \a Handler.
+ void discardOnDestroy(unique_function<void(Error E)> Handler) {
+ DiscardOnDestroyHandler = std::move(Handler);
+ }
+
+ /// Create a proxy stream for clients that need to pass an owned stream to a
+ /// producer. Errors if there's already a proxy. The proxy must be deleted
+ /// before calling \a keep(). The proxy will crash if it's written to after
+ /// calling \a discard().
+ Expected<std::unique_ptr<raw_pwrite_stream>> createProxy();
+
+ bool hasOpenProxy() const { return OpenProxy; }
+
+ /// Take the implementation.
+ ///
+ /// \pre \a hasOpenProxy() is false.
+ /// \pre \a discardOnDestroy() has not been called.
+ std::unique_ptr<OutputFileImpl> takeImpl() {
+ assert(!hasOpenProxy() && "Unexpected open proxy");
+ assert(!DiscardOnDestroyHandler && "Unexpected discard handler");
+ return std::move(Impl);
+ }
+
+ /// Check whether this is a null output file.
+ bool isNull() const { return Impl && isa<NullOutputFileImpl>(*Impl); }
+
+ OutputFile() = default;
+
+ explicit OutputFile(const Twine &Path, std::unique_ptr<OutputFileImpl> Impl)
+ : Path(Path.str()), Impl(std::move(Impl)) {
+ assert(this->Impl && "Expected open output file");
+ }
+
+ ~OutputFile() { destroy(); }
+ OutputFile(OutputFile &&O) { moveFrom(O); }
+ OutputFile &operator=(OutputFile &&O) {
+ destroy();
+ return moveFrom(O);
+ }
+
+private:
+ /// Destroy \a Impl. Reports fatal error if the file is open and there's no
+ /// handler from \a discardOnDestroy().
+ void destroy();
+ OutputFile &moveFrom(OutputFile &O) {
+ Path = std::move(O.Path);
+ Impl = std::move(O.Impl);
+ DiscardOnDestroyHandler = std::move(O.DiscardOnDestroyHandler);
+ OpenProxy = O.OpenProxy;
+ O.OpenProxy = nullptr;
+ return *this;
+ }
+
+ std::string Path;
+ std::unique_ptr<OutputFileImpl> Impl;
+ unique_function<void(Error E)> DiscardOnDestroyHandler;
+
+ class TrackedProxy;
+ TrackedProxy *OpenProxy = nullptr;
+};
+
+/// Update \p File to silently discard itself if it's still open when it's
+/// destroyed.
+inline void consumeDiscardOnDestroy(OutputFile &File) {
+ File.discardOnDestroy(consumeError);
+}
+
+/// Update \p File to silently discard itself if it's still open when it's
+/// destroyed.
+inline Expected<OutputFile> consumeDiscardOnDestroy(Expected<OutputFile> File) {
+ if (File)
+ consumeDiscardOnDestroy(*File);
+ return File;
+}
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTFILE_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 2a1903a1a960682..2ff976882e9c190 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -247,6 +247,11 @@ add_llvm_component_library(LLVMSupport
UnicodeNameToCodepointGenerated.cpp
VersionTuple.cpp
VirtualFileSystem.cpp
+ VirtualOutputBackend.cpp
+ VirtualOutputBackends.cpp
+ VirtualOutputConfig.cpp
+ VirtualOutputError.cpp
+ VirtualOutputFile.cpp
WithColor.cpp
YAMLParser.cpp
YAMLTraits.cpp
diff --git a/llvm/lib/Support/VirtualOutputBackend.cpp b/llvm/lib/Support/VirtualOutputBackend.cpp
new file mode 100644
index 000000000000000..bf50c66b0bf0c2a
--- /dev/null
+++ b/llvm/lib/Support/VirtualOutputBackend.cpp
@@ -0,0 +1,38 @@
+//===- VirtualOutputBackend.cpp - Virtualize compiler outputs -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements vfs::OutputBackend.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/ADT/StringExtras.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+void OutputBackend::anchor() {}
+
+Expected<OutputFile>
+OutputBackend::createFile(const Twine &Path_,
+ std::optional<OutputConfig> Config) {
+ SmallString<128> Path;
+ Path_.toVector(Path);
+
+ if (Config) {
+ // Check for invalid configs.
+ if (!Config->getText() && Config->getCRLF())
+ return make_error<OutputConfigError>(*Config, Path);
+ }
+
+ std::unique_ptr<OutputFileImpl> Impl;
+ if (Error E = createFileImpl(Path, Config).moveInto(Impl))
+ return std::move(E);
+ assert(Impl && "Expected valid Impl or Error");
+ return OutputFile(Path, std::move(Impl));
+}
diff --git a/llvm/lib/Support/VirtualOutputBackends.cpp b/llvm/lib/Support/VirtualOutputBackends.cpp
new file mode 100644
index 000000000000000..4b8b35b688266d4
--- /dev/null
+++ b/llvm/lib/Support/VirtualOutputBackends.cpp
@@ -0,0 +1,594 @@
+//===- VirtualOutputBackends.cpp - Virtual output backends ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements vfs::OutputBackend.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LockFileManager.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/Signals.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+void ProxyOutputBackend::anchor() {}
+void OnDiskOutputBackend::anchor() {}
+
+IntrusiveRefCntPtr<OutputBackend> vfs::makeNullOutputBackend() {
+ struct NullOutputBackend : public OutputBackend {
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ return const_cast<NullOutputBackend *>(this);
+ }
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig>) override {
+ return std::make_unique<NullOutputFileImpl>();
+ }
+ };
+
+ return makeIntrusiveRefCnt<NullOutputBackend>();
+}
+
+IntrusiveRefCntPtr<OutputBackend> vfs::makeFilteringOutputBackend(
+ IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+ std::function<bool(StringRef, std::optional<OutputConfig>)> Filter) {
+ struct FilteringOutputBackend : public ProxyOutputBackend {
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path,
+ std::optional<OutputConfig> Config) override {
+ if (Filter(Path, Config))
+ return ProxyOutputBackend::createFileImpl(Path, Config);
+ return std::make_unique<NullOutputFileImpl>();
+ }
+
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ return makeIntrusiveRefCnt<FilteringOutputBackend>(
+ getUnderlyingBackend().clone(), Filter);
+ }
+
+ FilteringOutputBackend(
+ IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+ std::function<bool(StringRef, std::optional<OutputConfig>)> Filter)
+ : ProxyOutputBackend(std::move(UnderlyingBackend)),
+ Filter(std::move(Filter)) {
+ assert(this->Filter && "Expected a non-null function");
+ }
+ std::function<bool(StringRef, std::optional<OutputConfig>)> Filter;
+ };
+
+ return makeIntrusiveRefCnt<FilteringOutputBackend>(
+ std::move(UnderlyingBackend), std::move(Filter));
+}
+
+IntrusiveRefCntPtr<OutputBackend>
+vfs::makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+ IntrusiveRefCntPtr<OutputBackend> Backend2) {
+ struct ProxyOutputBackend1 : public ProxyOutputBackend {
+ using ProxyOutputBackend::ProxyOutputBackend;
+ };
+ struct ProxyOutputBackend2 : public ProxyOutputBackend {
+ using ProxyOutputBackend::ProxyOutputBackend;
+ };
+ struct MirroringOutput final : public OutputFileImpl, raw_pwrite_stream {
+ Error keep() final {
+ flush();
+ return joinErrors(F1->keep(), F2->keep());
+ }
+ Error discard() final {
+ flush();
+ return joinErrors(F1->discard(), F2->discard());
+ }
+ raw_pwrite_stream &getOS() final { return *this; }
+
+ void write_impl(const char *Ptr, size_t Size) override {
+ F1->getOS().write(Ptr, Size);
+ F2->getOS().write(Ptr, Size);
+ }
+ void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+ this->flush();
+ F1->getOS().pwrite(Ptr, Size, Offset);
+ F2->getOS().pwrite(Ptr, Size, Offset);
+ }
+ uint64_t current_pos() const override { return F1->getOS().tell(); }
+ size_t preferred_buffer_size() const override {
+ return PreferredBufferSize;
+ }
+ void reserveExtraSpace(uint64_t ExtraSize) override {
+ F1->getOS().reserveExtraSpace(ExtraSize);
+ F2->getOS().reserveExtraSpace(ExtraSize);
+ }
+ bool is_displayed() const override {
+ return F1->getOS().is_displayed() && F2->getOS().is_displayed();
+ }
+ bool has_colors() const override {
+ return F1->getOS().has_colors() && F2->getOS().has_colors();
+ }
+ void enable_colors(bool enable) override {
+ raw_pwrite_stream::enable_colors(enable);
+ F1->getOS().enable_colors(enable);
+ F2->getOS().enable_colors(enable);
+ }
+
+ MirroringOutput(std::unique_ptr<OutputFileImpl> F1,
+ std::unique_ptr<OutputFileImpl> F2)
+ : PreferredBufferSize(std::max(F1->getOS().GetBufferSize(),
+ F1->getOS().GetBufferSize())),
+ F1(std::move(F1)), F2(std::move(F2)) {
+ // Don't double buffer.
+ this->F1->getOS().SetUnbuffered();
+ this->F2->getOS().SetUnbuffered();
+ }
+ size_t PreferredBufferSize;
+ std::unique_ptr<OutputFileImpl> F1;
+ std::unique_ptr<OutputFileImpl> F2;
+ };
+ struct MirroringOutputBackend : public ProxyOutputBackend1,
+ public ProxyOutputBackend2 {
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path,
+ std::optional<OutputConfig> Config) override {
+ std::unique_ptr<OutputFileImpl> File1;
+ std::unique_ptr<OutputFileImpl> File2;
+ if (Error E =
+ ProxyOutputBackend1::createFileImpl(Path, Config).moveInto(File1))
+ return std::move(E);
+ if (Error E =
+ ProxyOutputBackend2::createFileImpl(Path, Config).moveInto(File2))
+ return joinErrors(std::move(E), File1->discard());
+
+ // Skip the extra indirection if one of these is a null output.
+ if (isa<NullOutputFileImpl>(*File1)) {
+ consumeError(File1->discard());
+ return std::move(File2);
+ }
+ if (isa<NullOutputFileImpl>(*File2)) {
+ consumeError(File2->discard());
+ return std::move(File1);
+ }
+ return std::make_unique<MirroringOutput>(std::move(File1),
+ std::move(File2));
+ }
+
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ return IntrusiveRefCntPtr<ProxyOutputBackend1>(
+ makeIntrusiveRefCnt<MirroringOutputBackend>(
+ ProxyOutputBackend1::getUnderlyingBackend().clone(),
+ ProxyOutputBackend2::getUnderlyingBackend().clone()));
+ }
+ void Retain() const { ProxyOutputBackend1::Retain(); }
+ void Release() const { ProxyOutputBackend1::Release(); }
+
+ MirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+ IntrusiveRefCntPtr<OutputBackend> Backend2)
+ : ProxyOutputBackend1(std::move(Backend1)),
+ ProxyOutputBackend2(std::move(Backend2)) {}
+ };
+
+ assert(Backend1 && "Expected actual backend");
+ assert(Backend2 && "Expected actual backend");
+ return IntrusiveRefCntPtr<ProxyOutputBackend1>(
+ makeIntrusiveRefCnt<MirroringOutputBackend>(std::move(Backend1),
+ std::move(Backend2)));
+}
+
+static OutputConfig
+applySettings(std::optional<OutputConfig> &&Config,
+ const OnDiskOutputBackend::OutputSettings &Settings) {
+ if (!Config)
+ Config = Settings.DefaultConfig;
+ if (Settings.DisableTemporaries)
+ Config->setNoAtomicWrite();
+ if (Settings.DisableRemoveOnSignal)
+ Config->setNoDiscardOnSignal();
+ return *Config;
+}
+
+namespace {
+class OnDiskOutputFile final : public OutputFileImpl {
+public:
+ Error keep() override;
+ Error discard() override;
+ raw_pwrite_stream &getOS() override {
+ assert(FileOS && "Expected valid file");
+ if (BufferOS)
+ return *BufferOS;
+ return *FileOS;
+ }
+
+ /// Attempt to open a temporary file for \p OutputPath.
+ ///
+ /// This tries to open a uniquely-named temporary file for \p OutputPath,
+ /// possibly also creating any missing directories if \a
+ /// OnDiskOutputConfig::UseTemporaryCreateMissingDirectories is set in \a
+ /// Config.
+ ///
+ /// \post FD and \a TempPath are initialized if this is successful.
+ Error tryToCreateTemporary(std::optional<int> &FD);
+
+ Error initializeFD(std::optional<int> &FD);
+ Error initializeStream();
+ Error reset();
+
+ OnDiskOutputFile(StringRef OutputPath, std::optional<OutputConfig> Config,
+ const OnDiskOutputBackend::OutputSettings &Settings)
+ : Config(applySettings(std::move(Config), Settings)),
+ OutputPath(OutputPath.str()) {}
+
+ OutputConfig Config;
+ const std::string OutputPath;
+ std::optional<std::string> TempPath;
+ std::optional<raw_fd_ostream> FileOS;
+ std::optional<buffer_ostream> BufferOS;
+};
+} // end namespace
+
+static Error createDirectoriesOnDemand(StringRef OutputPath,
+ OutputConfig Config,
+ llvm::function_ref<Error()> CreateFile) {
+ return handleErrors(CreateFile(), [&](std::unique_ptr<ECError> EC) {
+ if (EC->convertToErrorCode() != std::errc::no_such_file_or_directory ||
+ Config.getNoImplyCreateDirectories())
+ return Error(std::move(EC));
+
+ StringRef ParentPath = sys::path::parent_path(OutputPath);
+ if (std::error_code EC = sys::fs::create_directories(ParentPath))
+ return make_error<OutputError>(ParentPath, EC);
+ return CreateFile();
+ });
+}
+
+Error OnDiskOutputFile::tryToCreateTemporary(std::optional<int> &FD) {
+ // Create a temporary file.
+ // Insert -%%%%%%%% before the extension (if any), and because some tools
+ // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build
+ // artifacts, also append .tmp.
+ StringRef OutputExtension = sys::path::extension(OutputPath);
+ SmallString<128> ModelPath =
+ StringRef(OutputPath).drop_back(OutputExtension.size());
+ ModelPath += "-%%%%%%%%";
+ ModelPath += OutputExtension;
+ ModelPath += ".tmp";
+
+ return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error {
+ int NewFD;
+ SmallString<128> UniquePath;
+ if (std::error_code EC =
+ sys::fs::createUniqueFile(ModelPath, NewFD, UniquePath))
+ return make_error<TempFileOutputError>(ModelPath, OutputPath, EC);
+
+ if (Config.getDiscardOnSignal())
+ sys::RemoveFileOnSignal(UniquePath);
+
+ TempPath = UniquePath.str().str();
+ FD.emplace(NewFD);
+ return Error::success();
+ });
+}
+
+Error OnDiskOutputFile::initializeFD(std::optional<int> &FD) {
+ assert(OutputPath != "-" && "Unexpected request for FD of stdout");
+
+ // Disable temporary file for other non-regular files, and if we get a status
+ // object, also check if we can write and disable write-through buffers if
+ // appropriate.
+ if (Config.getAtomicWrite()) {
+ sys::fs::file_status Status;
+ sys::fs::status(OutputPath, Status);
+ if (sys::fs::exists(Status)) {
+ if (!sys::fs::is_regular_file(Status))
+ Config.setNoAtomicWrite();
+
+ // Fail now if we can't write to the final destination.
+ if (!sys::fs::can_write(OutputPath))
+ return make_error<OutputError>(
+ OutputPath,
+ std::make_error_code(std::errc::operation_not_permitted));
+ }
+ }
+
+ // If (still) using a temporary file, try to create it (and return success if
+ // that works).
+ if (Config.getAtomicWrite())
+ if (!errorToBool(tryToCreateTemporary(FD)))
+ return Error::success();
+
+ // Not using a temporary file. Open the final output file.
+ return createDirectoriesOnDemand(OutputPath, Config, [&]() -> Error {
+ int NewFD;
+ sys::fs::OpenFlags OF = sys::fs::OF_None;
+ if (Config.getTextWithCRLF())
+ OF |= sys::fs::OF_TextWithCRLF;
+ else if (Config.getText())
+ OF |= sys::fs::OF_Text;
+ if (Config.getAppend())
+ OF |= sys::fs::OF_Append;
+ if (std::error_code EC = sys::fs::openFileForWrite(
+ OutputPath, NewFD, sys::fs::CD_CreateAlways, OF))
+ return convertToOutputError(OutputPath, EC);
+ FD.emplace(NewFD);
+
+ if (Config.getDiscardOnSignal())
+ sys::RemoveFileOnSignal(OutputPath);
+ return Error::success();
+ });
+}
+
+Error OnDiskOutputFile::initializeStream() {
+ // Open the file stream.
+ if (OutputPath == "-") {
+ std::error_code EC;
+ FileOS.emplace(OutputPath, EC);
+ if (EC)
+ return make_error<OutputError>(OutputPath, EC);
+ } else {
+ std::optional<int> FD;
+ if (Error E = initializeFD(FD))
+ return E;
+ FileOS.emplace(*FD, /*shouldClose=*/true);
+ }
+
+ // Buffer the stream if necessary.
+ if (!FileOS->supportsSeeking() && !Config.getText())
+ BufferOS.emplace(*FileOS);
+
+ return Error::success();
+}
+
+namespace {
+class OpenFileRAII {
+ static const int InvalidFd = -1;
+
+public:
+ int Fd = InvalidFd;
+
+ ~OpenFileRAII() {
+ if (Fd != InvalidFd)
+ llvm::sys::Process::SafelyCloseFileDescriptor(Fd);
+ }
+};
+
+enum class FileDifference : uint8_t {
+ /// The source and destination paths refer to the exact same file.
+ IdenticalFile,
+ /// The source and destination paths refer to separate files with identical
+ /// contents.
+ SameContents,
+ /// The source and destination paths refer to separate files with different
+ /// contents.
+ DifferentContents
+};
+} // end anonymous namespace
+
+static Expected<FileDifference>
+areFilesDifferent(const llvm::Twine &Source, const llvm::Twine &Destination) {
+ if (sys::fs::equivalent(Source, Destination))
+ return FileDifference::IdenticalFile;
+
+ OpenFileRAII SourceFile;
+ sys::fs::file_status SourceStatus;
+ // If we can't open the source file, fail.
+ if (std::error_code EC = sys::fs::openFileForRead(Source, SourceFile.Fd))
+ return convertToOutputError(Source, EC);
+
+ // If we can't stat the source file, fail.
+ if (std::error_code EC = sys::fs::status(SourceFile.Fd, SourceStatus))
+ return convertToOutputError(Source, EC);
+
+ OpenFileRAII DestFile;
+ sys::fs::file_status DestStatus;
+ // If we can't open the destination file, report different.
+ if (std::error_code Error =
+ sys::fs::openFileForRead(Destination, DestFile.Fd))
+ return FileDifference::DifferentContents;
+
+ // If we can't open the destination file, report different.
+ if (std::error_code Error = sys::fs::status(DestFile.Fd, DestStatus))
+ return FileDifference::DifferentContents;
+
+ // If the files are different sizes, they must be different.
+ uint64_t Size = SourceStatus.getSize();
+ if (Size != DestStatus.getSize())
+ return FileDifference::DifferentContents;
+
+ // If both files are zero size, they must be the same.
+ if (Size == 0)
+ return FileDifference::SameContents;
+
+ // The two files match in size, so we have to compare the bytes to determine
+ // if they're the same.
+ std::error_code SourceRegionErr;
+ sys::fs::mapped_file_region SourceRegion(
+ sys::fs::convertFDToNativeFile(SourceFile.Fd),
+ sys::fs::mapped_file_region::readonly, Size, 0, SourceRegionErr);
+ if (SourceRegionErr)
+ return convertToOutputError(Source, SourceRegionErr);
+
+ std::error_code DestRegionErr;
+ sys::fs::mapped_file_region DestRegion(
+ sys::fs::convertFDToNativeFile(DestFile.Fd),
+ sys::fs::mapped_file_region::readonly, Size, 0, DestRegionErr);
+
+ if (DestRegionErr)
+ return FileDifference::DifferentContents;
+
+ if (memcmp(SourceRegion.const_data(), DestRegion.const_data(), Size) != 0)
+ return FileDifference::DifferentContents;
+
+ return FileDifference::SameContents;
+}
+
+Error OnDiskOutputFile::reset() {
+ // Destroy the streams to flush them.
+ BufferOS.reset();
+ if (!FileOS)
+ return Error::success();
+
+ // Remember the error in raw_fd_ostream to be reported later.
+ std::error_code EC = FileOS->error();
+ // Clear the error to avoid fatal error when reset.
+ FileOS->clear_error();
+ FileOS.reset();
+ return errorCodeToError(EC);
+}
+
+Error OnDiskOutputFile::keep() {
+ if (auto E = reset())
+ return E;
+
+ // Close the file descriptor and remove crash cleanup before exit.
+ auto RemoveDiscardOnSignal = make_scope_exit([&]() {
+ if (Config.getDiscardOnSignal())
+ sys::DontRemoveFileOnSignal(TempPath ? *TempPath : OutputPath);
+ });
+
+ if (!TempPath)
+ return Error::success();
+
+ // See if we should append instead of move.
+ if (Config.getAppend() && OutputPath != "-") {
+ // Read TempFile for the content to append.
+ auto Content = MemoryBuffer::getFile(*TempPath);
+ if (!Content)
+ return convertToTempFileOutputError(*TempPath, OutputPath,
+ Content.getError());
+ while (1) {
+ // Attempt to lock the output file.
+ // Only one process is allowed to append to this file at a time.
+ llvm::LockFileManager Locked(OutputPath);
+ switch (Locked) {
+ case llvm::LockFileManager::LFS_Error: {
+ // If we error acquiring a lock, we cannot ensure appends
+ // to the trace file are atomic - cannot ensure output correctness.
+ Locked.unsafeRemoveLockFile();
+ return convertToOutputError(
+ OutputPath, std::make_error_code(std::errc::no_lock_available));
+ }
+ case llvm::LockFileManager::LFS_Owned: {
+ // Lock acquired, perform the write and release the lock.
+ std::error_code EC;
+ llvm::raw_fd_ostream Out(OutputPath, EC, llvm::sys::fs::OF_Append);
+ if (EC)
+ return convertToOutputError(OutputPath, EC);
+ Out << (*Content)->getBuffer();
+ Out.close();
+ Locked.unsafeRemoveLockFile();
+ if (Out.has_error())
+ return convertToOutputError(OutputPath, Out.error());
+ // Remove temp file and done.
+ (void)sys::fs::remove(*TempPath);
+ return Error::success();
+ }
+ case llvm::LockFileManager::LFS_Shared: {
+ // Someone else owns the lock on this file, wait.
+ switch (Locked.waitForUnlock(256)) {
+ case llvm::LockFileManager::Res_Success:
+ LLVM_FALLTHROUGH;
+ case llvm::LockFileManager::Res_OwnerDied: {
+ continue; // try again to get the lock.
+ }
+ case llvm::LockFileManager::Res_Timeout: {
+ // We could error on timeout to avoid potentially hanging forever, but
+ // it may be more likely that an interrupted process failed to clear
+ // the lock, causing other waiting processes to time-out. Let's clear
+ // the lock and try again right away. If we do start seeing compiler
+ // hangs in this location, we will need to re-consider.
+ Locked.unsafeRemoveLockFile();
+ continue;
+ }
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ if (Config.getOnlyIfDifferent()) {
+ auto Result = areFilesDifferent(*TempPath, OutputPath);
+ if (!Result)
+ return Result.takeError();
+ switch (*Result) {
+ case FileDifference::IdenticalFile:
+ // Do nothing for a self-move.
+ return Error::success();
+
+ case FileDifference::SameContents:
+ // Files are identical; remove the source file.
+ (void) sys::fs::remove(*TempPath);
+ return Error::success();
+
+ case FileDifference::DifferentContents:
+ break; // Rename the file.
+ }
+ }
+
+ // Move temporary to the final output path and remove it if that fails.
+ std::error_code RenameEC = sys::fs::rename(*TempPath, OutputPath);
+ if (!RenameEC)
+ return Error::success();
+
+ // FIXME: TempPath should be in the same directory as OutputPath but try to
+ // copy the output to see if makes any difference. If this path is used,
+ // investigate why we need to copy.
+ RenameEC = sys::fs::copy_file(*TempPath, OutputPath);
+ (void)sys::fs::remove(*TempPath);
+
+ if (!RenameEC)
+ return Error::success();
+
+ return make_error<TempFileOutputError>(*TempPath, OutputPath, RenameEC);
+}
+
+Error OnDiskOutputFile::discard() {
+ // Destroy the streams to flush them.
+ if (auto E = reset())
+ return E;
+
+ // Nothing on the filesystem to remove for stdout.
+ if (OutputPath == "-")
+ return Error::success();
+
+ auto discardPath = [&](StringRef Path) {
+ std::error_code EC = sys::fs::remove(Path);
+ sys::DontRemoveFileOnSignal(Path);
+ return EC;
+ };
+
+ // Clean up the file that's in-progress.
+ if (!TempPath)
+ return convertToOutputError(OutputPath, discardPath(OutputPath));
+ return convertToTempFileOutputError(*TempPath, OutputPath,
+ discardPath(*TempPath));
+}
+
+Error OnDiskOutputBackend::makeAbsolute(SmallVectorImpl<char> &Path) const {
+ return convertToOutputError(StringRef(Path.data(), Path.size()),
+ sys::fs::make_absolute(Path));
+}
+
+Expected<std::unique_ptr<OutputFileImpl>>
+OnDiskOutputBackend::createFileImpl(StringRef Path,
+ std::optional<OutputConfig> Config) {
+ SmallString<256> AbsPath;
+ if (Path != "-") {
+ AbsPath = Path;
+ if (Error E = makeAbsolute(AbsPath))
+ return std::move(E);
+ Path = AbsPath;
+ }
+
+ auto File = std::make_unique<OnDiskOutputFile>(Path, Config, Settings);
+ if (Error E = File->initializeStream())
+ return std::move(E);
+
+ return std::move(File);
+}
diff --git a/llvm/lib/Support/VirtualOutputConfig.cpp b/llvm/lib/Support/VirtualOutputConfig.cpp
new file mode 100644
index 000000000000000..f1d3c0fe3c6e714
--- /dev/null
+++ b/llvm/lib/Support/VirtualOutputConfig.cpp
@@ -0,0 +1,50 @@
+//===- VirtualOutputConfig.cpp - Virtual output configuration -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+OutputConfig &OutputConfig::setOpenFlags(const sys::fs::OpenFlags &Flags) {
+ // Ignore CRLF on its own as invalid.
+ using namespace llvm::sys::fs;
+ return Flags & OF_Text
+ ? setText().setCRLF(Flags & OF_CRLF).setAppend(Flags & OF_Append)
+ : setBinary().setAppend(Flags & OF_Append);
+}
+
+void OutputConfig::print(raw_ostream &OS) const {
+ OS << "{";
+ bool IsFirst = true;
+ auto printFlag = [&](StringRef FlagName, bool Value) {
+ if (IsFirst)
+ IsFirst = false;
+ else
+ OS << ",";
+ if (!Value)
+ OS << "No";
+ OS << FlagName;
+ };
+
+#define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT) \
+ if (get##NAME() != DEFAULT) \
+ printFlag(#NAME, get##NAME());
+#include "llvm/Support/VirtualOutputConfig.def"
+ OS << "}";
+}
+
+LLVM_DUMP_METHOD void OutputConfig::dump() const { print(dbgs()); }
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, OutputConfig Config) {
+ Config.print(OS);
+ return OS;
+}
diff --git a/llvm/lib/Support/VirtualOutputError.cpp b/llvm/lib/Support/VirtualOutputError.cpp
new file mode 100644
index 000000000000000..74fa5e3fa056881
--- /dev/null
+++ b/llvm/lib/Support/VirtualOutputError.cpp
@@ -0,0 +1,53 @@
+//===- VirtualOutputError.cpp - Errors for output virtualization ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputError.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+void OutputError::anchor() {}
+void OutputConfigError::anchor() {}
+void TempFileOutputError::anchor() {}
+
+char OutputError::ID = 0;
+char OutputConfigError::ID = 0;
+char TempFileOutputError::ID = 0;
+
+namespace {
+class OutputErrorCategory : public std::error_category {
+public:
+ const char *name() const noexcept override;
+ std::string message(int EV) const override;
+};
+} // end namespace
+
+const std::error_category &vfs::output_category() {
+ static OutputErrorCategory ErrorCategory;
+ return ErrorCategory;
+}
+
+const char *OutputErrorCategory::name() const noexcept {
+ return "llvm.vfs.output";
+}
+
+std::string OutputErrorCategory::message(int EV) const {
+ OutputErrorCode E = static_cast<OutputErrorCode>(EV);
+ switch (E) {
+ case OutputErrorCode::invalid_config:
+ return "invalid config";
+ case OutputErrorCode::not_closed:
+ return "output not closed";
+ case OutputErrorCode::already_closed:
+ return "output already closed";
+ case OutputErrorCode::has_open_proxy:
+ return "output has open proxy";
+ }
+ llvm_unreachable(
+ "An enumerator of OutputErrorCode does not have a message defined.");
+}
diff --git a/llvm/lib/Support/VirtualOutputFile.cpp b/llvm/lib/Support/VirtualOutputFile.cpp
new file mode 100644
index 000000000000000..c043c0b455c5c58
--- /dev/null
+++ b/llvm/lib/Support/VirtualOutputFile.cpp
@@ -0,0 +1,106 @@
+//===- VirtualOutputFile.cpp - Output file virtualization -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputFile.h"
+#include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/Support/VirtualOutputError.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/raw_ostream_proxy.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+char OutputFileImpl::ID = 0;
+char NullOutputFileImpl::ID = 0;
+
+void OutputFileImpl::anchor() {}
+void NullOutputFileImpl::anchor() {}
+
+class OutputFile::TrackedProxy : public raw_pwrite_stream_proxy {
+public:
+ void resetProxy() {
+ TrackingPointer = nullptr;
+ resetProxiedOS();
+ }
+
+ explicit TrackedProxy(TrackedProxy *&TrackingPointer, raw_pwrite_stream &OS)
+ : raw_pwrite_stream_proxy(OS), TrackingPointer(TrackingPointer) {
+ assert(!TrackingPointer && "Expected to add a proxy");
+ TrackingPointer = this;
+ }
+
+ ~TrackedProxy() override { resetProxy(); }
+
+ TrackedProxy *&TrackingPointer;
+};
+
+Expected<std::unique_ptr<raw_pwrite_stream>> OutputFile::createProxy() {
+ if (OpenProxy)
+ return make_error<OutputError>(getPath(), OutputErrorCode::has_open_proxy);
+
+ return std::make_unique<TrackedProxy>(OpenProxy, getOS());
+}
+
+Error OutputFile::keep() {
+ // Catch double-closing logic bugs.
+ if (LLVM_UNLIKELY(!Impl))
+ report_fatal_error(
+ make_error<OutputError>(getPath(), OutputErrorCode::already_closed));
+
+ // Report a fatal error if there's an open proxy and the file is being kept.
+ // This is safer than relying on clients to remember to flush(). Also call
+ // OutputFile::discard() to give the backend a chance to clean up any
+ // side effects (such as temporaries).
+ if (LLVM_UNLIKELY(OpenProxy))
+ report_fatal_error(joinErrors(
+ make_error<OutputError>(getPath(), OutputErrorCode::has_open_proxy),
+ discard()));
+
+ Error E = Impl->keep();
+ Impl = nullptr;
+ DiscardOnDestroyHandler = nullptr;
+ return E;
+}
+
+Error OutputFile::discard() {
+ // Catch double-closing logic bugs.
+ if (LLVM_UNLIKELY(!Impl))
+ report_fatal_error(
+ make_error<OutputError>(getPath(), OutputErrorCode::already_closed));
+
+ // Be lenient about open proxies since client teardown paths won't
+ // necessarily clean up in the right order. Reset the proxy to flush any
+ // current content; if there is another write, there should be quick crash on
+ // null dereference.
+ if (OpenProxy)
+ OpenProxy->resetProxy();
+
+ Error E = Impl->discard();
+ Impl = nullptr;
+ DiscardOnDestroyHandler = nullptr;
+ return E;
+}
+
+void OutputFile::destroy() {
+ if (!Impl)
+ return;
+
+ // Clean up the file. Move the discard handler into a local since discard
+ // will reset it.
+ auto DiscardHandler = std::move(DiscardOnDestroyHandler);
+ Error E = discard();
+ assert(!Impl && "Expected discard to destroy Impl");
+
+ // If there's no handler, report a fatal error.
+ if (LLVM_UNLIKELY(!DiscardHandler))
+ llvm::report_fatal_error(joinErrors(
+ make_error<OutputError>(getPath(), OutputErrorCode::not_closed),
+ std::move(E)));
+ else if (E)
+ DiscardHandler(std::move(E));
+}
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 1307907e7640af1..c7af26030fe6e46 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -92,6 +92,10 @@ add_llvm_unittest(SupportTests
UnicodeTest.cpp
VersionTupleTest.cpp
VirtualFileSystemTest.cpp
+ VirtualOutputBackendTest.cpp
+ VirtualOutputBackendsTest.cpp
+ VirtualOutputConfigTest.cpp
+ VirtualOutputFileTest.cpp
WithColorTest.cpp
YAMLIOTest.cpp
YAMLParserTest.cpp
diff --git a/llvm/unittests/Support/VirtualOutputBackendTest.cpp b/llvm/unittests/Support/VirtualOutputBackendTest.cpp
new file mode 100644
index 000000000000000..3adb671d977742b
--- /dev/null
+++ b/llvm/unittests/Support/VirtualOutputBackendTest.cpp
@@ -0,0 +1,147 @@
+//===- VirtualOutputBackendTest.cpp - Tests for vfs::OutputBackend --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+namespace {
+
+struct MockOutputBackendData {
+ int Cloned = 0;
+ int FilesCreated = 0;
+ std::optional<OutputConfig> LastConfig;
+ unique_function<Error()> FileCreator;
+};
+
+struct MockOutputBackend final : public OutputBackend {
+ struct MockFile final : public OutputFileImpl {
+ Error keep() override { return Error::success(); }
+ Error discard() override { return Error::success(); }
+ raw_pwrite_stream &getOS() override { return OS; }
+ raw_null_ostream OS;
+ };
+
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ ++Data.Cloned;
+ return const_cast<MockOutputBackend *>(this);
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef, std::optional<OutputConfig> Config) override {
+ ++Data.FilesCreated;
+ Data.LastConfig = Config;
+ if (Data.FileCreator)
+ return Data.FileCreator();
+ return std::make_unique<MockFile>();
+ }
+
+ Expected<OutputFile>
+ createAutoDiscardFile(const Twine &OutputPath,
+ std::optional<OutputConfig> Config = std::nullopt) {
+ return consumeDiscardOnDestroy(createFile(OutputPath, Config));
+ }
+
+ MockOutputBackend(MockOutputBackendData &Data) : Data(Data) {}
+ MockOutputBackendData &Data;
+};
+
+static IntrusiveRefCntPtr<MockOutputBackend>
+createMockBackend(MockOutputBackendData &Data) {
+ return makeIntrusiveRefCnt<MockOutputBackend>(Data);
+}
+
+static Error createCustomError() {
+ return createStringError(inconvertibleErrorCode(), "custom error");
+}
+
+TEST(VirtualOutputBackendTest, construct) {
+ MockOutputBackendData Data;
+ auto B = createMockBackend(Data);
+ EXPECT_EQ(0, Data.Cloned);
+ EXPECT_EQ(0, Data.FilesCreated);
+}
+
+TEST(VirtualOutputBackendTest, clone) {
+ MockOutputBackendData Data;
+ auto Backend = createMockBackend(Data);
+ auto Clone = Backend->clone();
+ EXPECT_EQ(1, Data.Cloned);
+
+ // Confirm the clone matches what the mock's cloneImpl() does.
+ EXPECT_EQ(Backend.get(), Clone.get());
+
+ // Make another clone.
+ Backend->clone();
+ EXPECT_EQ(2, Data.Cloned);
+}
+
+TEST(VirtualOutputBackendTest, createFile) {
+ MockOutputBackendData Data;
+ auto Backend = createMockBackend(Data);
+
+ StringRef FilePath = "dir/file";
+ OutputFile F;
+ EXPECT_THAT_ERROR(Backend->createFile(Twine(FilePath)).moveInto(F),
+ Succeeded());
+ EXPECT_EQ(1, Data.FilesCreated);
+ EXPECT_EQ(FilePath, F.getPath());
+ EXPECT_EQ(std::nullopt, Data.LastConfig);
+
+ // Confirm OutputBackend has not installed a discard handler.
+#if GTEST_HAS_DEATH_TEST
+ EXPECT_DEATH(F = OutputFile(), "output not closed");
+#endif
+ consumeError(F.discard());
+
+ // Create more files and specify configs.
+ for (OutputConfig Config : {
+ OutputConfig(),
+ OutputConfig().setNoAtomicWrite().setDiscardOnSignal(),
+ OutputConfig().setAtomicWrite().setNoDiscardOnSignal(),
+ OutputConfig().setText(),
+ OutputConfig().setTextWithCRLF(),
+ }) {
+ int CreatedAlready = Data.FilesCreated;
+ EXPECT_THAT_ERROR(
+ Backend->createAutoDiscardFile(Twine(FilePath), Config).takeError(),
+ Succeeded());
+ EXPECT_EQ(Config, Data.LastConfig);
+ EXPECT_EQ(1 + CreatedAlready, Data.FilesCreated);
+ }
+}
+
+TEST(VirtualOutputBackendTest, createFileInvalidConfigCRLF) {
+ MockOutputBackendData Data;
+ auto Backend = createMockBackend(Data);
+
+ // Check that invalid configs don't make it to the backend.
+ EXPECT_THAT_ERROR(
+ Backend
+ ->createAutoDiscardFile(Twine("dir/file"), OutputConfig().setCRLF())
+ .takeError(),
+ FailedWithMessage("dir/file: invalid config: {CRLF}"));
+ EXPECT_EQ(0, Data.FilesCreated);
+}
+
+TEST(VirtualOutputBackendTest, createFileError) {
+ MockOutputBackendData Data;
+ Data.FileCreator = createCustomError;
+ auto Backend = createMockBackend(Data);
+
+ // Check that invalid configs don't make it to the backend.
+ EXPECT_THAT_ERROR(
+ Backend->createAutoDiscardFile(Twine("dir/file")).takeError(),
+ FailedWithMessage("custom error"));
+ EXPECT_EQ(1, Data.FilesCreated);
+}
+
+} // end namespace
diff --git a/llvm/unittests/Support/VirtualOutputBackendsTest.cpp b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp
new file mode 100644
index 000000000000000..630402ed354a4ff
--- /dev/null
+++ b/llvm/unittests/Support/VirtualOutputBackendsTest.cpp
@@ -0,0 +1,886 @@
+//===- VirtualOutputBackendsTest.cpp - Tests for vfs::OutputBackend impls -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/HashingOutputBackend.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+namespace {
+
+class OutputBackendProvider {
+public:
+ virtual bool rejectsMissingDirectories() = 0;
+
+ virtual IntrusiveRefCntPtr<OutputBackend> createBackend() = 0;
+ virtual std::string getFilePathToCreate() = 0;
+ virtual std::string getFilePathToCreateUnder(StringRef Parent1,
+ StringRef Parent2 = "") = 0;
+ virtual Error checkCreated(StringRef FilePath,
+ OutputConfig Config = OutputConfig()) = 0;
+ virtual Error checkWrote(StringRef FilePath, StringRef Data) = 0;
+ virtual Error checkFlushed(StringRef FilePath, StringRef Data) = 0;
+ virtual Error checkKept(StringRef FilePath, StringRef Data) = 0;
+ virtual Error checkDiscarded(StringRef FilePath) = 0;
+
+ virtual ~OutputBackendProvider() = default;
+
+ struct Generator {
+ std::string Name;
+ std::function<std::unique_ptr<OutputBackendProvider>()> Generate;
+
+ std::unique_ptr<OutputBackendProvider> operator()() const {
+ return Generate();
+ }
+ };
+};
+
+struct BackendTest
+ : public ::testing::TestWithParam<OutputBackendProvider::Generator> {
+ std::unique_ptr<OutputBackendProvider> Provider;
+
+ void SetUp() override { Provider = GetParam()(); }
+ void TearDown() override { Provider = nullptr; }
+
+ IntrusiveRefCntPtr<OutputBackend> createBackend() {
+ return Provider->createBackend();
+ }
+};
+
+TEST_P(BackendTest, Discard) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(O.discard(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded());
+ EXPECT_FALSE(O.isOpen());
+}
+
+TEST_P(BackendTest, DiscardNoAtomicWrite) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputConfig Config = OutputConfig().setNoAtomicWrite();
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),
+ Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(O.discard(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkDiscarded(FilePath), Succeeded());
+ EXPECT_FALSE(O.isOpen());
+}
+
+TEST_P(BackendTest, Keep) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+ ASSERT_TRUE(O.isOpen());
+
+ O << Data;
+ EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());
+
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+ EXPECT_FALSE(O.isOpen());
+}
+
+TEST_P(BackendTest, KeepFlush) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());
+
+ O.getOS().flush();
+ EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded());
+
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+}
+
+TEST_P(BackendTest, KeepFlushProxy) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+ {
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(O.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Data;
+ EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());
+
+ Proxy->flush();
+ EXPECT_THAT_ERROR(Provider->checkFlushed(FilePath, Data), Succeeded());
+ }
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+}
+
+TEST_P(BackendTest, KeepEmpty) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, ""), Succeeded());
+}
+
+TEST_P(BackendTest, KeepMissingDirectory) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreateUnder("missing");
+ StringRef Data = "some data";
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+}
+
+TEST_P(BackendTest, KeepMissingDirectoryNested) {
+ auto Backend = createBackend();
+ std::string FilePath =
+ Provider->getFilePathToCreateUnder("missing", "nested");
+ StringRef Data = "some data";
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath).moveInto(O), Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+}
+
+TEST_P(BackendTest, KeepNoAtomicWrite) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputConfig Config = OutputConfig().setNoAtomicWrite();
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),
+ Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());
+ O << Data;
+ EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());
+
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+ EXPECT_FALSE(O.isOpen());
+}
+
+TEST_P(BackendTest, KeepNoAtomicWriteMissingDirectory) {
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputConfig Config = OutputConfig().setNoAtomicWrite();
+
+ OutputFile O;
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O),
+ Succeeded());
+ consumeDiscardOnDestroy(O);
+ ASSERT_THAT_ERROR(Provider->checkCreated(FilePath, Config), Succeeded());
+
+ O << Data;
+ EXPECT_THAT_ERROR(Provider->checkWrote(FilePath, Data), Succeeded());
+
+ EXPECT_THAT_ERROR(O.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Provider->checkKept(FilePath, Data), Succeeded());
+ EXPECT_FALSE(O.isOpen());
+}
+
+TEST_P(BackendTest, KeepMissingDirectoryNoImply) {
+ // Skip this test if the backend doesn't have a concept of missing
+ // directories.
+ if (!Provider->rejectsMissingDirectories())
+ return;
+
+ auto Backend = createBackend();
+ std::string FilePath = Provider->getFilePathToCreateUnder("missing");
+ std::error_code EC = errorToErrorCode(
+ consumeDiscardOnDestroy(
+ Backend->createFile(FilePath,
+ OutputConfig().setNoImplyCreateDirectories()))
+ .takeError());
+ EXPECT_EQ(int(std::errc::no_such_file_or_directory), EC.value());
+}
+
+class NullOutputBackendProvider : public OutputBackendProvider {
+public:
+ bool rejectsMissingDirectories() override { return false; }
+
+ IntrusiveRefCntPtr<OutputBackend> createBackend() override {
+ return makeNullOutputBackend();
+ }
+ std::string getFilePathToCreate() override { return "ignored.data"; }
+ std::string getFilePathToCreateUnder(StringRef Parent1,
+ StringRef Parent2) override {
+ SmallString<128> Path;
+ sys::path::append(Path, Parent1, Parent2, getFilePathToCreate());
+ return Path.str().str();
+ }
+ Error checkCreated(StringRef, OutputConfig) override {
+ return Error::success();
+ }
+ Error checkWrote(StringRef, StringRef) override { return Error::success(); }
+ Error checkFlushed(StringRef, StringRef) override { return Error::success(); }
+ Error checkKept(StringRef, StringRef) override { return Error::success(); }
+ Error checkDiscarded(StringRef) override { return Error::success(); }
+};
+
+struct OnDiskFile {
+ const unittest::TempDir &D;
+ SmallString<128> Path;
+ StringRef ParentPath;
+ StringRef Filename;
+ StringRef Stem;
+ StringRef Extension;
+ std::unique_ptr<MemoryBuffer> LastBuffer;
+
+ OnDiskFile(const unittest::TempDir &D, const Twine &InputPath) : D(D) {
+ if (sys::path::is_absolute(InputPath))
+ InputPath.toVector(Path);
+ else
+ sys::path::append(Path, D.path(), InputPath);
+ ParentPath = sys::path::parent_path(Path);
+ Filename = sys::path::filename(Path);
+ Stem = sys::path::stem(Filename);
+ Extension = sys::path::extension(Filename);
+ }
+
+ std::optional<OnDiskFile> findTemp() const;
+
+ std::optional<sys::fs::UniqueID> getCurrentUniqueID();
+
+ bool hasUniqueID(sys::fs::UniqueID ID) {
+ auto CurrentID = getCurrentUniqueID();
+ if (!CurrentID)
+ return false;
+ return *CurrentID == ID;
+ }
+
+ std::optional<StringRef> getCurrentContent() {
+ auto OnDiskOrErr = MemoryBuffer::getFile(Path);
+ if (!OnDiskOrErr)
+ return std::nullopt;
+ LastBuffer = std::move(*OnDiskOrErr);
+ return LastBuffer->getBuffer();
+ }
+
+ bool equalsCurrentContent(StringRef Data) {
+ auto CurrentContent = getCurrentContent();
+ if (!CurrentContent)
+ return false;
+ return *CurrentContent == Data;
+ }
+
+ bool equalsCurrentContent(std::nullopt_t) {
+ return getCurrentContent() == std::nullopt;
+ }
+};
+
+class OnDiskOutputBackendProvider : public OutputBackendProvider {
+public:
+ bool rejectsMissingDirectories() override { return true; }
+
+ std::optional<unittest::TempDir> D;
+
+ IntrusiveRefCntPtr<OutputBackend> createBackend() override {
+ auto Backend = makeIntrusiveRefCnt<OnDiskOutputBackend>();
+ Backend->Settings = Settings;
+ return Backend;
+ }
+ void init() {
+ if (!D)
+ D.emplace("OutputBackendTest.d", /*Unique=*/true);
+ }
+ std::string getFilePathToCreate() override {
+ init();
+ return OnDiskFile(*D, "file.data").Path.str().str();
+ }
+ std::string getFilePathToCreateUnder(StringRef Parent1,
+ StringRef Parent2) override {
+ init();
+ SmallString<128> Path;
+ sys::path::append(Path, D->path(), Parent1, Parent2, getFilePathToCreate());
+ return Path.str().str();
+ }
+
+ Error checkCreated(StringRef FilePath, OutputConfig Config) override;
+ Error checkWrote(StringRef FilePath, StringRef Data) override;
+ Error checkFlushed(StringRef FilePath, StringRef Data) override;
+ Error checkKept(StringRef FilePath, StringRef Data) override;
+ Error checkDiscarded(StringRef FilePath) override;
+
+ struct FileInfo {
+ OutputConfig Config;
+ std::optional<OnDiskFile> F;
+ std::optional<OnDiskFile> Temp;
+ std::optional<sys::fs::UniqueID> UID;
+ std::optional<sys::fs::UniqueID> TempUID;
+ };
+ Error checkOpen(FileInfo &Info);
+ bool shouldUseTemporaries(const FileInfo &Info) const;
+
+ OnDiskOutputBackendProvider() = default;
+ explicit OnDiskOutputBackendProvider(
+ const OnDiskOutputBackend::OutputSettings &Settings)
+ : Settings(Settings) {}
+ OnDiskOutputBackend::OutputSettings Settings;
+
+ StringMap<FileInfo> Files;
+ Error lookupFileInfo(StringRef FilePath, FileInfo *&Info);
+};
+
+bool OnDiskOutputBackendProvider::shouldUseTemporaries(
+ const FileInfo &Info) const {
+ return Info.Config.getAtomicWrite() && !Settings.DisableTemporaries;
+}
+
+struct ProviderGeneratorList {
+ std::vector<OutputBackendProvider::Generator> Generators;
+ ProviderGeneratorList(
+ std::initializer_list<OutputBackendProvider::Generator> IL)
+ : Generators(IL) {}
+
+ std::string operator()(
+ const ::testing::TestParamInfo<OutputBackendProvider::Generator> &Info) {
+ return Info.param.Name;
+ }
+};
+
+ProviderGeneratorList BackendGenerators = {
+ {"Null", []() { return std::make_unique<NullOutputBackendProvider>(); }},
+ {"OnDisk",
+ []() { return std::make_unique<OnDiskOutputBackendProvider>(); }},
+ {"OnDisk_DisableRemoveOnSignal",
+ []() {
+ OnDiskOutputBackend::OutputSettings Settings;
+ Settings.DisableRemoveOnSignal = true;
+ return std::make_unique<OnDiskOutputBackendProvider>(Settings);
+ }},
+ {"OnDisk_DisableTemporaries",
+ []() {
+ OnDiskOutputBackend::OutputSettings Settings;
+ Settings.DisableTemporaries = true;
+ return std::make_unique<OnDiskOutputBackendProvider>(Settings);
+ }},
+};
+
+INSTANTIATE_TEST_SUITE_P(VirtualOutput, BackendTest,
+ ::testing::ValuesIn(BackendGenerators.Generators),
+ BackendGenerators);
+
+std::optional<sys::fs::UniqueID> OnDiskFile::getCurrentUniqueID() {
+ sys::fs::file_status Status;
+ sys::fs::status(Path, Status, /*follow=*/false);
+ if (!sys::fs::is_regular_file(Status))
+ return std::nullopt;
+ return Status.getUniqueID();
+}
+
+std::optional<OnDiskFile> OnDiskFile::findTemp() const {
+ std::error_code EC;
+ for (sys::fs::directory_iterator I(ParentPath, EC), E; !EC && I != E;
+ I.increment(EC)) {
+ StringRef TempPath = I->path();
+ if (!TempPath.startswith(D.path()))
+ continue;
+
+ // Look for "<stem>-*.<extension>.tmp".
+ if (sys::path::extension(TempPath) != ".tmp")
+ continue;
+
+ // Drop the ".tmp" and check the extension and stem.
+ StringRef TempStem = sys::path::stem(TempPath);
+ if (sys::path::extension(TempStem) != Extension)
+ continue;
+ StringRef OriginalStem = sys::path::stem(TempStem);
+ if (!OriginalStem.startswith(Stem))
+ continue;
+ if (!OriginalStem.drop_front(Stem.size()).startswith("-"))
+ continue;
+
+ // Found it.
+ return OnDiskFile(D, TempPath.drop_front(D.path().size() + 1));
+ }
+ return std::nullopt;
+}
+
+Error OnDiskOutputBackendProvider::lookupFileInfo(StringRef FilePath,
+ FileInfo *&Info) {
+ auto I = Files.find(FilePath);
+ if (Files.find(FilePath) == Files.end())
+ return createStringError(inconvertibleErrorCode(),
+ "Missing call to checkCreated()");
+ Info = &I->second;
+ assert(Info->F && "Expected OnDiskFile to be initialized");
+ return Error::success();
+}
+
+Error OnDiskOutputBackendProvider::checkOpen(FileInfo &Info) {
+ // Collect info about filesystem state.
+ assert(Info.F);
+ std::optional<sys::fs::UniqueID> UID = Info.F->getCurrentUniqueID();
+ std::optional<OnDiskFile> Temp = Info.F->findTemp();
+ std::optional<sys::fs::UniqueID> TempUID;
+ if (Temp)
+ TempUID = Temp->getCurrentUniqueID();
+
+ // Check if it's correct.
+ if (shouldUseTemporaries(Info)) {
+ if (!Temp)
+ return createStringError(inconvertibleErrorCode(),
+ "Missing temporary file");
+ if (!TempUID)
+ return createStringError(inconvertibleErrorCode(),
+ "Missing UID for temporary");
+ if (UID)
+ return createStringError(
+ inconvertibleErrorCode(),
+ "Unexpected final UID when temporaries should be used");
+
+ // Check previous data.
+ if (Info.Temp)
+ if (Temp->Path != Info.Temp->Path)
+ return createStringError(inconvertibleErrorCode(),
+ "Temporary path changed");
+ if (Info.TempUID)
+ if (*TempUID != *Info.TempUID)
+ return createStringError(inconvertibleErrorCode(),
+ "Temporary UID changed");
+ } else {
+ if (Temp)
+ return createStringError(inconvertibleErrorCode(),
+ "Unexpected temporary file");
+ if (!UID)
+ return createStringError(inconvertibleErrorCode(),
+ "Missing UID for temporary");
+
+ // Check previous data.
+ if (Info.UID)
+ if (*UID != *Info.UID)
+ return createStringError(inconvertibleErrorCode(), "UID changed");
+ }
+
+ Info.UID = UID;
+ if (Temp)
+ Info.Temp.emplace(*D, Temp->Path);
+ else
+ Info.Temp.reset();
+ Info.TempUID = TempUID;
+ return Error::success();
+}
+
+Error OnDiskOutputBackendProvider::checkCreated(StringRef FilePath,
+ OutputConfig Config) {
+ auto &Info = Files[FilePath];
+ if (Info.F) {
+ assert(OnDiskFile(*D, FilePath).Path == Info.F->Path);
+ Info.UID = std::nullopt;
+ Info.Temp.reset();
+ Info.TempUID = std::nullopt;
+ } else {
+ Info.F.emplace(*D, FilePath);
+ }
+ Info.Config = Config;
+ return checkOpen(Info);
+}
+
+Error OnDiskOutputBackendProvider::checkWrote(StringRef FilePath,
+ StringRef Data) {
+ FileInfo *Info = nullptr;
+ if (Error E = lookupFileInfo(FilePath, Info))
+ return E;
+ return checkOpen(*Info);
+}
+
+Error OnDiskOutputBackendProvider::checkFlushed(StringRef FilePath,
+ StringRef Data) {
+ FileInfo *Info = nullptr;
+ if (Error E = lookupFileInfo(FilePath, Info))
+ return E;
+ if (Error E = checkOpen(*Info))
+ return E;
+
+ OnDiskFile &F = shouldUseTemporaries(*Info) ? *Info->Temp : *Info->F;
+ if (!F.equalsCurrentContent(Data))
+ return createStringError(inconvertibleErrorCode(), "content not flushed");
+ return Error::success();
+}
+
+Error OnDiskOutputBackendProvider::checkKept(StringRef FilePath,
+ StringRef Data) {
+ FileInfo *Info = nullptr;
+ if (Error E = lookupFileInfo(FilePath, Info))
+ return E;
+
+ sys::fs::UniqueID UID =
+ shouldUseTemporaries(*Info) ? *Info->TempUID : *Info->UID;
+ if (!Info->F->hasUniqueID(UID))
+ return createStringError(inconvertibleErrorCode(),
+ "File not created by keep or changed UID");
+
+ if (std::optional<OnDiskFile> Temp = Info->F->findTemp())
+ return createStringError(inconvertibleErrorCode(),
+ "Temporary not removed by keep");
+
+ return Error::success();
+}
+
+Error OnDiskOutputBackendProvider::checkDiscarded(StringRef FilePath) {
+ FileInfo *Info = nullptr;
+ if (Error E = lookupFileInfo(FilePath, Info))
+ return E;
+
+ if (std::optional<sys::fs::UniqueID> UID = Info->F->getCurrentUniqueID())
+ return createStringError(inconvertibleErrorCode(),
+ "File not removed by discard");
+
+ if (std::optional<OnDiskFile> Temp = Info->F->findTemp())
+ return createStringError(inconvertibleErrorCode(),
+ "Temporary not removed by discard");
+
+ return Error::success();
+}
+
+TEST(VirtualOutputBackendAdaptors, makeFilteringOutputBackend) {
+ bool ShouldCreate = false;
+ auto Backend = makeFilteringOutputBackend(
+ makeIntrusiveRefCnt<OnDiskOutputBackend>(),
+ [&ShouldCreate](StringRef, std::optional<OutputConfig>) {
+ return ShouldCreate;
+ });
+
+ int Count = 0;
+ unittest::TempDir D("FilteringOutputBackendTest.d", /*Unique=*/true);
+ for (bool ShouldCreateVal : {false, true, true, false}) {
+ ShouldCreate = ShouldCreateVal;
+ OnDiskFile OnDisk(D, "file." + Twine(Count++) + "." + Twine(ShouldCreate));
+ OutputFile Output;
+ ASSERT_THAT_ERROR(consumeDiscardOnDestroy(Backend->createFile(OnDisk.Path))
+ .moveInto(Output),
+ Succeeded());
+ EXPECT_NE(ShouldCreate, Output.isNull());
+ Output << "content";
+ EXPECT_THAT_ERROR(Output.keep(), Succeeded());
+
+ if (ShouldCreate) {
+ EXPECT_EQ(StringRef("content"), OnDisk.getCurrentContent());
+ } else {
+ EXPECT_FALSE(OnDisk.getCurrentUniqueID());
+ }
+ }
+ SmallString<128> Path;
+}
+
+class AbsolutePathBackend : public ProxyOutputBackend {
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ llvm_unreachable("unimplemented");
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+ assert(!sys::path::is_absolute(Path) &&
+ "Expected tests to pass all relative paths");
+ SmallString<256> AbsPath;
+ sys::path::append(AbsPath, CWD, Path);
+ return ProxyOutputBackend::createFileImpl(AbsPath, Config);
+ }
+
+public:
+ AbsolutePathBackend(const Twine &CWD,
+ IntrusiveRefCntPtr<OutputBackend> Backend)
+ : ProxyOutputBackend(std::move(Backend)), CWD(CWD.str()) {
+ assert(sys::path::is_absolute(this->CWD) &&
+ "Expected tests to pass a relative path");
+ }
+
+private:
+ std::string CWD;
+};
+
+TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackend) {
+ unittest::TempDir D1("MirroringOutputBackendTest.1.d", /*Unique=*/true);
+ unittest::TempDir D2("MirroringOutputBackendTest.2.d", /*Unique=*/true);
+
+ IntrusiveRefCntPtr<OutputBackend> Backend;
+ {
+ auto OnDisk = makeIntrusiveRefCnt<OnDiskOutputBackend>();
+ Backend = makeMirroringOutputBackend(
+ makeIntrusiveRefCnt<AbsolutePathBackend>(D1.path(), OnDisk),
+ makeIntrusiveRefCnt<AbsolutePathBackend>(D2.path(), OnDisk));
+ }
+
+ OnDiskFile OnDisk1(D1, "file");
+ OnDiskFile OnDisk2(D2, "file");
+ OutputFile Output;
+ ASSERT_THAT_ERROR(
+ consumeDiscardOnDestroy(Backend->createFile("file")).moveInto(Output),
+ Succeeded());
+ EXPECT_TRUE(OnDisk1.findTemp());
+ EXPECT_TRUE(OnDisk2.findTemp());
+
+ Output << "content";
+ Output.getOS().pwrite("ON", /*Size=*/2, /*Offset=*/1);
+ EXPECT_THAT_ERROR(Output.keep(), Succeeded());
+ EXPECT_EQ(StringRef("cONtent"), OnDisk1.getCurrentContent());
+ EXPECT_EQ(StringRef("cONtent"), OnDisk2.getCurrentContent());
+ EXPECT_NE(OnDisk1.getCurrentUniqueID(), OnDisk2.getCurrentUniqueID());
+}
+
+/// Behaves like NullOutputFileImpl, but doesn't match the RTTI (so OutputFile
+/// cannot tell).
+class LikeNullOutputFile final : public OutputFileImpl {
+ Error keep() final { return Error::success(); }
+ Error discard() final { return Error::success(); }
+ raw_pwrite_stream &getOS() final { return OS; }
+
+public:
+ LikeNullOutputFile(raw_null_ostream &OS) : OS(OS) {}
+ raw_null_ostream &OS;
+};
+class LikeNullOutputBackend final : public OutputBackend {
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ llvm_unreachable("not implemented");
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+ return std::make_unique<LikeNullOutputFile>(OS);
+ }
+
+public:
+ raw_null_ostream OS;
+};
+
+TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendNull) {
+ // Check that null outputs are skipped by seeing that LikeNull->OS is passed
+ // through directly (without a mirroring proxy stream) to Output.
+ auto LikeNull = makeIntrusiveRefCnt<LikeNullOutputBackend>();
+ auto Null1 = makeNullOutputBackend();
+ auto Mirror = makeMirroringOutputBackend(Null1, LikeNull);
+ OutputFile Output;
+ ASSERT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),
+ Succeeded());
+ EXPECT_TRUE(!Output.isNull());
+ EXPECT_EQ(&Output.getOS(), &LikeNull->OS);
+
+ // Check the other direction.
+ Mirror = makeMirroringOutputBackend(LikeNull, Null1);
+ ASSERT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),
+ Succeeded());
+ EXPECT_TRUE(!Output.isNull());
+ EXPECT_EQ(&Output.getOS(), &LikeNull->OS);
+
+ // Same null backend, twice.
+ Mirror = makeMirroringOutputBackend(Null1, Null1);
+ ASSERT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),
+ Succeeded());
+ EXPECT_TRUE(Output.isNull());
+
+ // Two null backends.
+ auto Null2 = makeNullOutputBackend();
+ Mirror = makeMirroringOutputBackend(Null1, Null2);
+ ASSERT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).moveInto(Output),
+ Succeeded());
+ EXPECT_TRUE(Output.isNull());
+}
+
+class StringErrorBackend final : public OutputBackend {
+ IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+ llvm_unreachable("not implemented");
+ }
+
+ Expected<std::unique_ptr<OutputFileImpl>>
+ createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+ return createStringError(inconvertibleErrorCode(), Msg);
+ }
+
+public:
+ StringErrorBackend(const Twine &Msg) : Msg(Msg.str()) {}
+ std::string Msg;
+};
+
+TEST(VirtualOutputBackendAdaptors, makeMirroringOutputBackendCreateError) {
+ auto Error1 = makeIntrusiveRefCnt<StringErrorBackend>("error-backend-1");
+ auto Null = makeNullOutputBackend();
+
+ auto Mirror = makeMirroringOutputBackend(Null, Error1);
+ EXPECT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),
+ FailedWithMessage(Error1->Msg));
+
+ Mirror = makeMirroringOutputBackend(Error1, Null);
+ EXPECT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),
+ FailedWithMessage(Error1->Msg));
+
+ auto Error2 = makeIntrusiveRefCnt<StringErrorBackend>("error-backend-2");
+ Mirror = makeMirroringOutputBackend(Error1, Error2);
+ EXPECT_THAT_ERROR(
+ consumeDiscardOnDestroy(Mirror->createFile("file")).takeError(),
+ FailedWithMessage(Error1->Msg));
+}
+
+TEST(OnDiskBackendTest, OnlyIfDifferent) {
+ OnDiskOutputBackendProvider Provider;
+ auto Backend = Provider.createBackend();
+ std::string FilePath = Provider.getFilePathToCreate();
+ StringRef Data = "some data";
+ OutputConfig Config = OutputConfig().setOnlyIfDifferent();
+
+ OutputFile O1, O2, O3;
+ sys::fs::file_status Status1, Status2, Status3;
+ // Write first file.
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),
+ Succeeded());
+ O1 << Data;
+ EXPECT_THAT_ERROR(O1.keep(), Succeeded());
+ EXPECT_FALSE(O1.isOpen());
+ EXPECT_FALSE(sys::fs::status(FilePath, Status1, /*follow=*/false));
+
+ // Write second with same content.
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),
+ Succeeded());
+ O2 << Data;
+ EXPECT_THAT_ERROR(O2.keep(), Succeeded());
+ EXPECT_FALSE(O2.isOpen());
+ EXPECT_FALSE(sys::fs::status(FilePath, Status2, /*follow=*/false));
+
+ // Make sure the output path file is not modified with same content.
+ EXPECT_EQ(Status1.getUniqueID(), Status2.getUniqueID());
+
+ // Write third with different content.
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O3),
+ Succeeded());
+ O3 << Data << "\n";
+ EXPECT_THAT_ERROR(O3.keep(), Succeeded());
+ EXPECT_FALSE(O3.isOpen());
+ EXPECT_FALSE(sys::fs::status(FilePath, Status3, /*follow=*/false));
+
+ // This should overwrite the file and create a different UniqueID.
+ EXPECT_NE(Status1.getUniqueID(), Status3.getUniqueID());
+}
+
+TEST(OnDiskBackendTest, Append) {
+ OnDiskOutputBackendProvider Provider;
+ auto Backend = Provider.createBackend();
+ std::string FilePath = Provider.getFilePathToCreate();
+ OutputConfig Config = OutputConfig().setAppend();
+
+ OutputFile O1, O2, O3;
+ // Write first file.
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O1),
+ Succeeded());
+ O1 << "some data\n";
+ EXPECT_THAT_ERROR(O1.keep(), Succeeded());
+ EXPECT_FALSE(O1.isOpen());
+
+ OnDiskFile File1(*Provider.D, FilePath);
+ EXPECT_TRUE(File1.equalsCurrentContent("some data\n"));
+
+ // Append same data.
+ EXPECT_THAT_ERROR(Backend->createFile(FilePath, Config).moveInto(O2),
+ Succeeded());
+ O2 << "more data\n";
+ EXPECT_THAT_ERROR(O2.keep(), Succeeded());
+ EXPECT_FALSE(O2.isOpen());
+
+ // Check data is appended.
+ OnDiskFile File2(*Provider.D, FilePath);
+ EXPECT_TRUE(File2.equalsCurrentContent("some data\nmore data\n"));
+
+ // Non atomic append.
+ EXPECT_THAT_ERROR(
+ Backend->createFile(FilePath, Config.setNoAtomicWrite()).moveInto(O3),
+ Succeeded());
+ O3 << "more more\n";
+ EXPECT_THAT_ERROR(O3.keep(), Succeeded());
+ EXPECT_FALSE(O3.isOpen());
+
+ // Check data is appended.
+ OnDiskFile File3(*Provider.D, FilePath);
+ EXPECT_TRUE(File3.equalsCurrentContent("some data\nmore data\nmore more\n"));
+}
+
+TEST(HashingBackendTest, HashOutput) {
+ HashingOutputBackend<BLAKE3> Backend;
+ OutputFile O1, O2, O3, O4, O5;
+ EXPECT_THAT_ERROR(Backend.createFile("file1").moveInto(O1), Succeeded());
+ O1 << "some data";
+ EXPECT_THAT_ERROR(O1.keep(), Succeeded());
+ EXPECT_THAT_ERROR(Backend.createFile("file2").moveInto(O2), Succeeded());
+ O2 << "some data";
+ EXPECT_THAT_ERROR(O2.keep(), Succeeded());
+ EXPECT_EQ(Backend.getHashValueForFile("file1"),
+ Backend.getHashValueForFile("file2"));
+
+ EXPECT_THAT_ERROR(Backend.createFile("file3").moveInto(O3), Succeeded());
+ O3 << "some ";
+ O3 << "data";
+ EXPECT_THAT_ERROR(O3.keep(), Succeeded());
+ EXPECT_EQ(Backend.getHashValueForFile("file1"),
+ Backend.getHashValueForFile("file3"));
+
+ EXPECT_THAT_ERROR(Backend.createFile("file4").moveInto(O4), Succeeded());
+ O4 << "same data";
+ O4.getOS().pwrite("o", 1, 1);
+ EXPECT_THAT_ERROR(O4.keep(), Succeeded());
+ EXPECT_EQ(Backend.getHashValueForFile("file1"),
+ Backend.getHashValueForFile("file4"));
+
+ EXPECT_THAT_ERROR(Backend.createFile("file5").moveInto(O5), Succeeded());
+ O5 << "different data";
+ EXPECT_THAT_ERROR(O5.keep(), Succeeded());
+ EXPECT_NE(Backend.getHashValueForFile("file1"),
+ Backend.getHashValueForFile("file5"));
+}
+
+} // end namespace
diff --git a/llvm/unittests/Support/VirtualOutputConfigTest.cpp b/llvm/unittests/Support/VirtualOutputConfigTest.cpp
new file mode 100644
index 000000000000000..cf6cd19b619aa9b
--- /dev/null
+++ b/llvm/unittests/Support/VirtualOutputConfigTest.cpp
@@ -0,0 +1,152 @@
+//===- VirtualOutputConfigTest.cpp - vfs::OutputConfig tests --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/FileSystem.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+namespace {
+
+TEST(VirtualOutputConfigTest, construct) {
+ // Test defaults.
+ EXPECT_FALSE(OutputConfig().getText());
+ EXPECT_FALSE(OutputConfig().getCRLF());
+ EXPECT_TRUE(OutputConfig().getDiscardOnSignal());
+ EXPECT_TRUE(OutputConfig().getAtomicWrite());
+ EXPECT_TRUE(OutputConfig().getImplyCreateDirectories());
+ EXPECT_FALSE(OutputConfig().getOnlyIfDifferent());
+ EXPECT_FALSE(OutputConfig().getAppend());
+
+ // Test inverted defaults.
+ EXPECT_TRUE(OutputConfig().getNoText());
+ EXPECT_TRUE(OutputConfig().getNoCRLF());
+ EXPECT_FALSE(OutputConfig().getNoDiscardOnSignal());
+ EXPECT_FALSE(OutputConfig().getNoAtomicWrite());
+ EXPECT_FALSE(OutputConfig().getNoImplyCreateDirectories());
+ EXPECT_TRUE(OutputConfig().getNoOnlyIfDifferent());
+ EXPECT_TRUE(OutputConfig().getNoAppend());
+}
+
+TEST(VirtualOutputConfigTest, set) {
+ // Check a flag that defaults to false. Try both 'get's, all three 'set's,
+ // and turning back off after turning it on.
+ ASSERT_TRUE(OutputConfig().getNoText());
+ EXPECT_TRUE(OutputConfig().setText().getText());
+ EXPECT_FALSE(OutputConfig().setText().getNoText());
+ EXPECT_TRUE(OutputConfig().setText(true).getText());
+ EXPECT_FALSE(OutputConfig().setText().setNoText().getText());
+ EXPECT_FALSE(OutputConfig().setText().setText(false).getText());
+
+ // Check a flag that defaults to true. Try both 'get's, all three 'set's, and
+ // turning back on after turning it off.
+ ASSERT_TRUE(OutputConfig().getDiscardOnSignal());
+ EXPECT_FALSE(OutputConfig().setNoDiscardOnSignal().getDiscardOnSignal());
+ EXPECT_TRUE(OutputConfig().setNoDiscardOnSignal().getNoDiscardOnSignal());
+ EXPECT_FALSE(OutputConfig().setDiscardOnSignal(false).getDiscardOnSignal());
+ EXPECT_TRUE(OutputConfig()
+ .setNoDiscardOnSignal()
+ .setDiscardOnSignal()
+ .getDiscardOnSignal());
+ EXPECT_TRUE(OutputConfig()
+ .setNoDiscardOnSignal()
+ .setDiscardOnSignal(true)
+ .getDiscardOnSignal());
+
+ // Set multiple flags.
+ OutputConfig Config;
+ Config.setText().setNoDiscardOnSignal().setNoImplyCreateDirectories();
+ EXPECT_TRUE(Config.getText());
+ EXPECT_TRUE(Config.getNoDiscardOnSignal());
+ EXPECT_TRUE(Config.getNoImplyCreateDirectories());
+}
+
+TEST(VirtualOutputConfigTest, equals) {
+ EXPECT_TRUE(OutputConfig() == OutputConfig());
+ EXPECT_FALSE(OutputConfig() != OutputConfig());
+ EXPECT_EQ(OutputConfig().setAtomicWrite(), OutputConfig().setAtomicWrite());
+ EXPECT_NE(OutputConfig().setAtomicWrite(), OutputConfig().setNoAtomicWrite());
+}
+
+static std::string toString(OutputConfig Config) {
+ std::string Printed;
+ raw_string_ostream OS(Printed);
+ Config.print(OS);
+ return Printed;
+}
+
+TEST(VirtualOutputConfigTest, print) {
+ EXPECT_EQ("{}", toString(OutputConfig()));
+ EXPECT_EQ("{Text}", toString(OutputConfig().setText()));
+ EXPECT_EQ("{Text,NoDiscardOnSignal}",
+ toString(OutputConfig().setText().setNoDiscardOnSignal()));
+ EXPECT_EQ("{Text,NoDiscardOnSignal}",
+ toString(OutputConfig().setNoDiscardOnSignal().setText()));
+}
+
+TEST(VirtualOutputConfigTest, BinaryAndTextWithCRLF) {
+ // Test defaults.
+ EXPECT_TRUE(OutputConfig().getBinary());
+ EXPECT_FALSE(OutputConfig().getTextWithCRLF());
+ EXPECT_FALSE(OutputConfig().getText());
+ EXPECT_FALSE(OutputConfig().getCRLF());
+
+ // Test setting.
+ EXPECT_TRUE(OutputConfig().setTextWithCRLF().getTextWithCRLF());
+ EXPECT_TRUE(OutputConfig().setTextWithCRLF().getText());
+ EXPECT_TRUE(OutputConfig().setTextWithCRLF().getCRLF());
+ EXPECT_TRUE(OutputConfig().setText().setCRLF().getTextWithCRLF());
+ EXPECT_FALSE(OutputConfig().setText().getBinary());
+ EXPECT_FALSE(OutputConfig().setTextWithCRLF().getBinary());
+ EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getText());
+ EXPECT_FALSE(OutputConfig().setTextWithCRLF().setBinary().getCRLF());
+
+ // Test setTextWithCRLF(bool).
+ EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getText());
+ EXPECT_TRUE(OutputConfig().setBinary().setTextWithCRLF(true).getCRLF());
+ EXPECT_TRUE(
+ OutputConfig().setTextWithCRLF().setTextWithCRLF(false).getBinary());
+
+ // Test printing.
+ EXPECT_EQ("{Text,CRLF}", toString(OutputConfig().setTextWithCRLF()));
+}
+
+TEST(VirtualOutputConfigTest, OpenFlags) {
+ using namespace llvm::sys::fs;
+
+ // Confirm the default is binary.
+ ASSERT_EQ(OutputConfig().setBinary(), OutputConfig());
+
+ // Most flags are not supported / have no effect.
+ EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_None));
+ EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_Delete));
+ EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_ChildInherit));
+ EXPECT_EQ(OutputConfig(), OutputConfig().setOpenFlags(OF_UpdateAtime));
+
+ // Check setting OF_Text and OF_CRLF.
+ for (OutputConfig Init : {
+ OutputConfig(),
+ OutputConfig().setText(),
+ OutputConfig().setTextWithCRLF(),
+ OutputConfig().setAppend(),
+
+ // Should be overridden despite being invalid.
+ OutputConfig().setCRLF(),
+ }) {
+ EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_None));
+ EXPECT_EQ(OutputConfig(), Init.setOpenFlags(OF_CRLF));
+ EXPECT_EQ(OutputConfig().setText(), Init.setOpenFlags(OF_Text));
+ EXPECT_EQ(OutputConfig().setTextWithCRLF(),
+ Init.setOpenFlags(OF_TextWithCRLF));
+ EXPECT_EQ(OutputConfig().setAppend(), Init.setOpenFlags(OF_Append));
+ }
+}
+
+} // anonymous namespace
diff --git a/llvm/unittests/Support/VirtualOutputFileTest.cpp b/llvm/unittests/Support/VirtualOutputFileTest.cpp
new file mode 100644
index 000000000000000..486218513f3da50
--- /dev/null
+++ b/llvm/unittests/Support/VirtualOutputFileTest.cpp
@@ -0,0 +1,342 @@
+//===- VirtualOutputFileTest.cpp - vfs::OutputFile tests ------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/VirtualOutputFile.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::vfs;
+
+namespace {
+
+struct MockOutputFileData {
+ int Kept = 0;
+ int Discarded = 0;
+ int Handled = 0;
+ unique_function<Error()> Keeper;
+ unique_function<Error()> Discarder;
+
+ void handler(Error E) {
+ consumeError(std::move(E));
+ ++Handled;
+ }
+ unique_function<void(Error)> getHandler() {
+ return [this](Error E) { handler(std::move(E)); };
+ }
+
+ SmallString<128> V;
+ std::optional<raw_svector_ostream> VOS;
+ raw_pwrite_stream *OS = nullptr;
+
+ MockOutputFileData() : VOS(std::in_place, V), OS(&*VOS) {}
+ MockOutputFileData(raw_pwrite_stream &OS) : OS(&OS) {}
+};
+
+struct MockOutputFile final : public OutputFileImpl {
+ Error keep() override {
+ ++Data.Kept;
+ if (Data.Keeper)
+ return Data.Keeper();
+ return Error::success();
+ }
+
+ Error discard() override {
+ ++Data.Discarded;
+ if (Data.Discarder)
+ return Data.Discarder();
+ return Error::success();
+ }
+
+ raw_pwrite_stream &getOS() override {
+ if (!Data.OS)
+ report_fatal_error("missing stream in MockOutputFile::getOS");
+ return *Data.OS;
+ }
+
+ MockOutputFile(MockOutputFileData &Data) : Data(Data) {}
+ MockOutputFileData &Data;
+};
+
+static std::unique_ptr<MockOutputFile>
+createMockOutput(MockOutputFileData &Data) {
+ return std::make_unique<MockOutputFile>(Data);
+}
+
+static Error createCustomError() {
+ return createStringError(inconvertibleErrorCode(), "custom error");
+}
+
+TEST(VirtualOutputFileTest, construct) {
+ OutputFile F;
+ EXPECT_EQ("", F.getPath());
+ EXPECT_FALSE(F);
+ EXPECT_FALSE(F.isOpen());
+
+#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG)
+ EXPECT_DEATH(F.getOS(), "Expected open output stream");
+#endif
+}
+
+#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG)
+TEST(VirtualOutputFileTest, constructNull) {
+ EXPECT_DEATH(OutputFile("some/file/path", nullptr),
+ "Expected open output file");
+}
+#endif
+
+TEST(VirtualOutputFileTest, destroy) {
+ MockOutputFileData Data;
+ StringRef FilePath = "some/file/path";
+
+ // Check behaviour when destroying, first without a handler and then with
+ // one. The handler shouldn't be called.
+ std::optional<OutputFile> F(std::in_place, FilePath, createMockOutput(Data));
+ EXPECT_TRUE(F->isOpen());
+ EXPECT_EQ(FilePath, F->getPath());
+ EXPECT_EQ(Data.OS, &F->getOS());
+#if GTEST_HAS_DEATH_TEST
+ EXPECT_DEATH(F.reset(), "output not closed");
+#endif
+ F->discardOnDestroy(Data.getHandler());
+ EXPECT_EQ(0, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+ F.reset();
+ EXPECT_EQ(1, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+
+ // Try again when discard returns an error. This time the handler should be
+ // called.
+ Data.Discarder = createCustomError;
+ F.emplace("some/file/path", createMockOutput(Data));
+ F->discardOnDestroy(Data.getHandler());
+ F.reset();
+ EXPECT_EQ(2, Data.Discarded);
+ EXPECT_EQ(1, Data.Handled);
+}
+
+TEST(VirtualOutputFileTest, destroyProxy) {
+ MockOutputFileData Data;
+
+ std::optional<OutputFile> F(std::in_place, "some/file/path",
+ createMockOutput(Data));
+ F->discardOnDestroy(Data.getHandler());
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F->createProxy().moveInto(Proxy), Succeeded());
+ F.reset();
+#if GTEST_HAS_DEATH_TEST && !defined(NDEBUG)
+ EXPECT_DEATH(*Proxy << "data", "use after reset");
+#endif
+ Proxy.reset();
+}
+
+TEST(VirtualOutputFileTest, discard) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ {
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F << Content;
+ EXPECT_EQ(Content, Data.V);
+
+ EXPECT_THAT_ERROR(F.discard(), Succeeded());
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+
+#if GTEST_HAS_DEATH_TEST
+ EXPECT_DEATH(
+ consumeError(F.keep()), "some/file/path: output already closed");
+ EXPECT_DEATH(
+ consumeError(F.discard()), "some/file/path: output already closed");
+#endif
+ }
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+}
+
+TEST(VirtualOutputFileTest, discardError) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ Data.Discarder = createCustomError;
+ {
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F << Content;
+ EXPECT_EQ(Content, Data.V);
+ EXPECT_THAT_ERROR(F.discard(), FailedWithMessage("custom error"));
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+ }
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+}
+
+TEST(VirtualOutputFileTest, discardProxy) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ(Content, Data.V);
+
+ EXPECT_THAT_ERROR(F.discard(), Succeeded());
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+}
+
+TEST(VirtualOutputFileTest, discardProxyFlush) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F.getOS().SetBufferSize(Content.size() * 2);
+
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ("", Data.V);
+ EXPECT_THAT_ERROR(F.discard(), Succeeded());
+ EXPECT_EQ(Content, Data.V);
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(1, Data.Discarded);
+}
+
+TEST(VirtualOutputFileTest, keep) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ {
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F << Content;
+ EXPECT_EQ(Content, Data.V);
+
+ EXPECT_THAT_ERROR(F.keep(), Succeeded());
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+
+#if GTEST_HAS_DEATH_TEST
+ EXPECT_DEATH(
+ consumeError(F.keep()), "some/file/path: output already closed");
+ EXPECT_DEATH(
+ consumeError(F.discard()), "some/file/path: output already closed");
+#endif
+ }
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+}
+
+TEST(VirtualOutputFileTest, keepError) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ Data.Keeper = createCustomError;
+ {
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F << Content;
+ EXPECT_EQ(Content, Data.V);
+
+ EXPECT_THAT_ERROR(F.keep(), FailedWithMessage("custom error"));
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+ }
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+ EXPECT_EQ(0, Data.Handled);
+}
+
+TEST(VirtualOutputFileTest, keepProxy) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ(Content, Data.V);
+ Proxy.reset();
+ EXPECT_THAT_ERROR(F.keep(), Succeeded());
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+}
+
+#if GTEST_HAS_DEATH_TEST
+TEST(VirtualOutputFileTest, keepProxyStillOpen) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ(Content, Data.V);
+ EXPECT_DEATH(consumeError(F.keep()), "some/file/path: output has open proxy");
+}
+#endif
+
+TEST(VirtualOutputFileTest, keepProxyFlush) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+ F.getOS().SetBufferSize(Content.size() * 2);
+
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ("", Data.V);
+ Proxy.reset();
+ EXPECT_THAT_ERROR(F.keep(), Succeeded());
+ EXPECT_EQ(Content, Data.V);
+ EXPECT_FALSE(F.isOpen());
+ EXPECT_EQ(1, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+}
+
+TEST(VirtualOutputFileTest, TwoProxies) {
+ StringRef Content = "some data";
+ MockOutputFileData Data;
+
+ OutputFile F("some/file/path", createMockOutput(Data));
+ F.discardOnDestroy(Data.getHandler());
+
+ // Can't have two open proxies at once.
+ {
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ EXPECT_THAT_ERROR(
+ F.createProxy().takeError(),
+ FailedWithMessage("some/file/path: output has open proxy"));
+ }
+ EXPECT_EQ(0, Data.Kept);
+ EXPECT_EQ(0, Data.Discarded);
+
+ // A second proxy after the first closes should work...
+ {
+ std::unique_ptr<raw_pwrite_stream> Proxy;
+ EXPECT_THAT_ERROR(F.createProxy().moveInto(Proxy), Succeeded());
+ *Proxy << Content;
+ EXPECT_EQ(Content, Data.V);
+ }
+}
+
+} // end namespace
>From 58763b6dcc5682636073a22a574f85b19fce9e70 Mon Sep 17 00:00:00 2001
From: Steven Wu <stevenwu at apple.com>
Date: Thu, 5 Oct 2023 16:04:33 -0700
Subject: [PATCH 3/3] Frontend: Adopt llvm::vfs::OutputBackend in
CompilerInstance
Adopt new virtual output backend in CompilerInstance.
---
.../include/clang/Frontend/CompilerInstance.h | 36 ++--
clang/lib/Frontend/CompilerInstance.cpp | 173 ++++++------------
2 files changed, 76 insertions(+), 133 deletions(-)
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index d26a452cf94cc3b..9295b2bee115cc0 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -24,6 +24,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/BuryPointer.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/VirtualOutputBackend.h"
#include <cassert>
#include <list>
#include <memory>
@@ -92,6 +93,9 @@ class CompilerInstance : public ModuleLoader {
/// The file manager.
IntrusiveRefCntPtr<FileManager> FileMgr;
+ /// The output context.
+ IntrusiveRefCntPtr<llvm::vfs::OutputBackend> TheOutputBackend;
+
/// The source manager.
IntrusiveRefCntPtr<SourceManager> SourceMgr;
@@ -164,22 +168,8 @@ class CompilerInstance : public ModuleLoader {
/// The stream for verbose output.
raw_ostream *VerboseOutputStream = &llvm::errs();
- /// Holds information about the output file.
- ///
- /// If TempFilename is not empty we must rename it to Filename at the end.
- /// TempFilename may be empty and Filename non-empty if creating the temporary
- /// failed.
- struct OutputFile {
- std::string Filename;
- std::optional<llvm::sys::fs::TempFile> File;
-
- OutputFile(std::string filename,
- std::optional<llvm::sys::fs::TempFile> file)
- : Filename(std::move(filename)), File(std::move(file)) {}
- };
-
/// The list of active output files.
- std::list<OutputFile> OutputFiles;
+ std::list<llvm::vfs::OutputFile> OutputFiles;
/// Force an output buffer.
std::unique_ptr<llvm::raw_pwrite_stream> OutputStream;
@@ -430,6 +420,22 @@ class CompilerInstance : public ModuleLoader {
/// Replace the current file manager and virtual file system.
void setFileManager(FileManager *Value);
+ /// @name Output Backend.
+ /// {
+
+ /// Set the output backend.
+ void
+ setOutputBackend(IntrusiveRefCntPtr<llvm::vfs::OutputBackend> NewOutputs);
+
+ /// Create an output manager.
+ void createOutputBackend();
+
+ bool hasOutputBackend() const { return bool(TheOutputBackend); }
+
+ llvm::vfs::OutputBackend &getOutputBackend();
+ llvm::vfs::OutputBackend &getOrCreateOutputBackend();
+
+ /// }
/// @}
/// @name Source Manager
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index d18371f21a9d86e..f00eb145d8d2e4f 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -54,6 +54,8 @@
#include "llvm/Support/Signals.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/Timer.h"
+#include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/Support/VirtualOutputError.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TargetParser/Host.h"
#include <optional>
@@ -516,6 +518,10 @@ void CompilerInstance::createPreprocessor(TranslationUnitKind TUKind) {
collectVFSEntries(*this, ModuleDepCollector);
}
+ // Modules need an output manager.
+ if (!hasOutputBackend())
+ createOutputBackend();
+
for (auto &Listener : DependencyCollectors)
Listener->attachToPreprocessor(*PP);
@@ -759,32 +765,19 @@ void CompilerInstance::createSema(TranslationUnitKind TUKind,
void CompilerInstance::clearOutputFiles(bool EraseFiles) {
// The ASTConsumer can own streams that write to the output files.
assert(!hasASTConsumer() && "ASTConsumer should be reset");
- // Ignore errors that occur when trying to discard the temp file.
- for (OutputFile &OF : OutputFiles) {
- if (EraseFiles) {
- if (OF.File)
- consumeError(OF.File->discard());
- if (!OF.Filename.empty())
- llvm::sys::fs::remove(OF.Filename);
- continue;
- }
-
- if (!OF.File)
- continue;
-
- if (OF.File->TmpName.empty()) {
- consumeError(OF.File->discard());
- continue;
- }
-
- llvm::Error E = OF.File->keep(OF.Filename);
- if (!E)
- continue;
-
- getDiagnostics().Report(diag::err_unable_to_rename_temp)
- << OF.File->TmpName << OF.Filename << std::move(E);
-
- llvm::sys::fs::remove(OF.File->TmpName);
+ if (!EraseFiles) {
+ for (auto &O : OutputFiles)
+ llvm::handleAllErrors(
+ O.keep(),
+ [&](const llvm::vfs::TempFileOutputError &E) {
+ getDiagnostics().Report(diag::err_unable_to_rename_temp)
+ << E.getTempPath() << E.getOutputPath()
+ << E.convertToErrorCode().message();
+ },
+ [&](const llvm::vfs::OutputError &E) {
+ getDiagnostics().Report(diag::err_fe_unable_to_open_output)
+ << E.getOutputPath() << E.convertToErrorCode().message();
+ });
}
OutputFiles.clear();
if (DeleteBuiltModules) {
@@ -818,6 +811,28 @@ std::unique_ptr<raw_pwrite_stream> CompilerInstance::createNullOutputFile() {
return std::make_unique<llvm::raw_null_ostream>();
}
+void CompilerInstance::setOutputBackend(
+ IntrusiveRefCntPtr<llvm::vfs::OutputBackend> NewOutputs) {
+ assert(!TheOutputBackend && "Already has an output manager");
+ TheOutputBackend = std::move(NewOutputs);
+}
+
+void CompilerInstance::createOutputBackend() {
+ assert(!TheOutputBackend && "Already has an output manager");
+ TheOutputBackend = llvm::makeIntrusiveRefCnt<llvm::vfs::OnDiskOutputBackend>();
+}
+
+llvm::vfs::OutputBackend &CompilerInstance::getOutputBackend() {
+ assert(TheOutputBackend);
+ return *TheOutputBackend;
+}
+
+llvm::vfs::OutputBackend &CompilerInstance::getOrCreateOutputBackend() {
+ if (!hasOutputBackend())
+ createOutputBackend();
+ return getOutputBackend();
+}
+
std::unique_ptr<raw_pwrite_stream>
CompilerInstance::createOutputFile(StringRef OutputPath, bool Binary,
bool RemoveFileOnSignal, bool UseTemporary,
@@ -852,98 +867,20 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
OutputPath = *AbsPath;
}
- std::unique_ptr<llvm::raw_fd_ostream> OS;
- std::optional<StringRef> OSFile;
-
- if (UseTemporary) {
- if (OutputPath == "-")
- UseTemporary = false;
- else {
- llvm::sys::fs::file_status Status;
- llvm::sys::fs::status(OutputPath, Status);
- if (llvm::sys::fs::exists(Status)) {
- // Fail early if we can't write to the final destination.
- if (!llvm::sys::fs::can_write(OutputPath))
- return llvm::errorCodeToError(
- make_error_code(llvm::errc::operation_not_permitted));
-
- // Don't use a temporary if the output is a special file. This handles
- // things like '-o /dev/null'
- if (!llvm::sys::fs::is_regular_file(Status))
- UseTemporary = false;
- }
- }
- }
-
- std::optional<llvm::sys::fs::TempFile> Temp;
- if (UseTemporary) {
- // Create a temporary file.
- // Insert -%%%%%%%% before the extension (if any), and because some tools
- // (noticeable, clang's own GlobalModuleIndex.cpp) glob for build
- // artifacts, also append .tmp.
- StringRef OutputExtension = llvm::sys::path::extension(OutputPath);
- SmallString<128> TempPath =
- StringRef(OutputPath).drop_back(OutputExtension.size());
- TempPath += "-%%%%%%%%";
- TempPath += OutputExtension;
- TempPath += ".tmp";
- llvm::sys::fs::OpenFlags BinaryFlags =
- Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_Text;
- Expected<llvm::sys::fs::TempFile> ExpectedFile =
- llvm::sys::fs::TempFile::create(
- TempPath, llvm::sys::fs::all_read | llvm::sys::fs::all_write,
- BinaryFlags);
-
- llvm::Error E = handleErrors(
- ExpectedFile.takeError(), [&](const llvm::ECError &E) -> llvm::Error {
- std::error_code EC = E.convertToErrorCode();
- if (CreateMissingDirectories &&
- EC == llvm::errc::no_such_file_or_directory) {
- StringRef Parent = llvm::sys::path::parent_path(OutputPath);
- EC = llvm::sys::fs::create_directories(Parent);
- if (!EC) {
- ExpectedFile = llvm::sys::fs::TempFile::create(
- TempPath, llvm::sys::fs::all_read | llvm::sys::fs::all_write,
- BinaryFlags);
- if (!ExpectedFile)
- return llvm::errorCodeToError(
- llvm::errc::no_such_file_or_directory);
- }
- }
- return llvm::errorCodeToError(EC);
- });
-
- if (E) {
- consumeError(std::move(E));
- } else {
- Temp = std::move(ExpectedFile.get());
- OS.reset(new llvm::raw_fd_ostream(Temp->FD, /*shouldClose=*/false));
- OSFile = Temp->TmpName;
- }
- // If we failed to create the temporary, fallback to writing to the file
- // directly. This handles the corner case where we cannot write to the
- // directory, but can write to the file.
- }
-
- if (!OS) {
- OSFile = OutputPath;
- std::error_code EC;
- OS.reset(new llvm::raw_fd_ostream(
- *OSFile, EC,
- (Binary ? llvm::sys::fs::OF_None : llvm::sys::fs::OF_TextWithCRLF)));
- if (EC)
- return llvm::errorCodeToError(EC);
- }
-
- // Add the output file -- but don't try to remove "-", since this means we are
- // using stdin.
- OutputFiles.emplace_back(((OutputPath != "-") ? OutputPath : "").str(),
- std::move(Temp));
-
- if (!Binary || OS->supportsSeeking())
- return std::move(OS);
-
- return std::make_unique<llvm::buffer_unique_ostream>(std::move(OS));
+ using namespace llvm::vfs;
+ Expected<OutputFile> O = getOrCreateOutputBackend().createFile(
+ OutputPath,
+ OutputConfig()
+ .setTextWithCRLF(!Binary)
+ .setDiscardOnSignal(RemoveFileOnSignal)
+ .setAtomicWrite(UseTemporary)
+ .setImplyCreateDirectories(UseTemporary && CreateMissingDirectories));
+ if (!O)
+ return O.takeError();
+
+ O->discardOnDestroy([](llvm::Error E) { consumeError(std::move(E)); });
+ OutputFiles.push_back(std::move(*O));
+ return OutputFiles.back().createProxy();
}
// Initialization Utilities
More information about the cfe-commits
mailing list