[llvm] [Support] Inline some raw_ostream methods (PR #97703)
Alexis Engelke via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 4 02:39:59 PDT 2024
https://github.com/aengelke created https://github.com/llvm/llvm-project/pull/97703
Inline the fast path of write, which is frequently called, as well as flush_nonempty and SetBufferAndMode, the latter being called for every raw_svector_ostream. This avoids several out-of-line calls and improves performance.
---
c-t-t: [inline of write](http://llvm-compile-time-tracker.com/compare.php?from=ac3b7ff60a2ce4026a3a687b9220ddea9eb58d7c&to=4b21a283314f893ba319ef0a112c865a6f58ce4c&stat=instructions:u)
c-t-t: [inline of the other two methods](http://llvm-compile-time-tracker.com/compare.php?from=4b21a283314f893ba319ef0a112c865a6f58ce4c&to=23d0e1a2c1b9cd6b48866ca9c18f8ed09423502e&stat=instructions:u)
c-t-t: [both (this PR)](http://llvm-compile-time-tracker.com/compare.php?from=ac3b7ff60a2ce4026a3a687b9220ddea9eb58d7c&to=23d0e1a2c1b9cd6b48866ca9c18f8ed09423502e&stat=instructions%3Au)
This does cause a slight file size increase, but it also improves performance. Inlining these functions also allows for subsequent improvements to svector_ostream to reduce the number of function calls.
>From 784ae4bf2ad079bc667130b2c0293475779db0a0 Mon Sep 17 00:00:00 2001
From: Alexis Engelke <engelke at in.tum.de>
Date: Tue, 2 Jul 2024 16:09:31 +0200
Subject: [PATCH] [Support] Inline some raw_ostream methods
Inline the fast path of write, which is frequently called, as well as
flush and SetBufferAndMode, the latter being called for every
raw_svector_ostream. This avoids several out-of-line calls and improves
performance.
---
llvm/include/llvm/Support/raw_ostream.h | 72 ++++++++++++++++++-------
llvm/lib/Support/raw_ostream.cpp | 52 +++++-------------
2 files changed, 68 insertions(+), 56 deletions(-)
diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h
index df9ee2e5a7858..22d22373a66d8 100644
--- a/llvm/include/llvm/Support/raw_ostream.h
+++ b/llvm/include/llvm/Support/raw_ostream.h
@@ -78,7 +78,8 @@ class raw_ostream {
///
/// If a subclass installs an external buffer using SetBuffer then it can wait
/// for a \see write_impl() call to handle the data which has been put into
- /// this buffer.
+ /// this buffer. If a subclass uses an external buffer, it is not possible
+ /// to switch to an internal buffer or an unbuffered mode.
char *OutBufStart, *OutBufEnd, *OutBufCur;
bool ColorEnabled = false;
@@ -165,11 +166,13 @@ class raw_ostream {
/// Set the stream to be buffered, using the specified buffer size.
void SetBufferSize(size_t Size) {
+ assert(BufferMode != BufferKind::ExternalBuffer);
flush();
SetBufferAndMode(new char[Size], Size, BufferKind::InternalBuffer);
}
size_t GetBufferSize() const {
+ assert(BufferMode != BufferKind::ExternalBuffer);
// If we're supposed to be buffered but haven't actually gotten around
// to allocating the buffer yet, return the value that would be used.
if (BufferMode != BufferKind::Unbuffered && OutBufStart == nullptr)
@@ -183,6 +186,7 @@ class raw_ostream {
/// after every write. This routine will also flush the buffer immediately
/// when the stream is being set to unbuffered.
void SetUnbuffered() {
+ assert(BufferMode != BufferKind::ExternalBuffer);
flush();
SetBufferAndMode(nullptr, 0, BufferKind::Unbuffered);
}
@@ -222,18 +226,7 @@ class raw_ostream {
}
raw_ostream &operator<<(StringRef Str) {
- // Inline fast path, particularly for strings with a known length.
- size_t Size = Str.size();
-
- // Make sure we can use the fast path.
- if (Size > (size_t)(OutBufEnd - OutBufCur))
- return write(Str.data(), Size);
-
- if (Size) {
- memcpy(OutBufCur, Str.data(), Size);
- OutBufCur += Size;
- }
- return *this;
+ return write(Str.data(), Str.size());
}
#if defined(__cpp_char8_t)
@@ -258,7 +251,6 @@ class raw_ostream {
}
raw_ostream &operator<<(const std::string &Str) {
- // Avoid the fast path, it would only increase code size for a marginal win.
return write(Str.data(), Str.length());
}
@@ -300,8 +292,25 @@ class raw_ostream {
/// satisfy llvm::isPrint into an escape sequence.
raw_ostream &write_escaped(StringRef Str, bool UseHexEscapes = false);
- raw_ostream &write(unsigned char C);
- raw_ostream &write(const char *Ptr, size_t Size);
+ raw_ostream &write(unsigned char C) {
+ if (OutBufCur >= OutBufEnd)
+ writeSlow(C);
+ else
+ *OutBufCur++ = C;
+ return *this;
+ }
+
+ raw_ostream &write(const char *Ptr, size_t Size) {
+ // Inline fast path
+ if (LLVM_UNLIKELY(size_t(OutBufEnd - OutBufCur) < Size)) {
+ writeSlow(Ptr, Size);
+ return *this;
+ }
+ if (Size)
+ memcpy(OutBufCur, Ptr, Size);
+ OutBufCur += Size;
+ return *this;
+ }
// Formatted output, see the format() function in Support/Format.h.
raw_ostream &operator<<(const format_object_base &Fmt);
@@ -385,6 +394,8 @@ class raw_ostream {
/// use only by subclasses which can arrange for the output to go directly
/// into the desired output buffer, instead of being copied on each flush.
void SetBuffer(char *BufferStart, size_t Size) {
+ assert((!OutBufStart && BufferMode == BufferKind::InternalBuffer) ||
+ BufferMode == BufferKind::ExternalBuffer);
SetBufferAndMode(BufferStart, Size, BufferKind::ExternalBuffer);
}
@@ -400,11 +411,36 @@ class raw_ostream {
//===--------------------------------------------------------------------===//
private:
/// Install the given buffer and mode.
- void SetBufferAndMode(char *BufferStart, size_t Size, BufferKind Mode);
+ void SetBufferAndMode(char *BufferStart, size_t Size, BufferKind Mode) {
+ assert(((Mode == BufferKind::Unbuffered && !BufferStart && Size == 0) ||
+ (Mode != BufferKind::Unbuffered && BufferStart && Size != 0)) &&
+ "stream must be unbuffered or have at least one byte");
+ // Make sure the current buffer is free of content (we can't flush here; the
+ // child buffer management logic will be in write_impl).
+ assert(GetNumBytesInBuffer() == 0 && "Current buffer is non-empty!");
+
+ if (BufferMode == BufferKind::InternalBuffer)
+ delete[] OutBufStart;
+ OutBufStart = BufferStart;
+ OutBufEnd = OutBufStart + Size;
+ OutBufCur = OutBufStart;
+ BufferMode = Mode;
+
+ assert(OutBufStart <= OutBufEnd && "Invalid size!");
+ }
/// Flush the current buffer, which is known to be non-empty. This outputs the
/// currently buffered data and resets the buffer to empty.
- void flush_nonempty();
+ void flush_nonempty() {
+ assert(OutBufCur > OutBufStart && "Invalid call to flush_nonempty.");
+ size_t Length = OutBufCur - OutBufStart;
+ OutBufCur = OutBufStart;
+ write_impl(OutBufStart, Length);
+ }
+
+ /// Slow path for writing when buffer is too small.
+ void writeSlow(unsigned char C);
+ void writeSlow(const char *Ptr, size_t Size);
/// Copy data into the buffer. Size must not be greater than the number of
/// unused bytes in the buffer.
diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp
index 2ce54faa9857e..48323d5011018 100644
--- a/llvm/lib/Support/raw_ostream.cpp
+++ b/llvm/lib/Support/raw_ostream.cpp
@@ -97,6 +97,7 @@ size_t raw_ostream::preferred_buffer_size() const {
}
void raw_ostream::SetBuffered() {
+ assert(BufferMode != BufferKind::ExternalBuffer);
// Ask the subclass to determine an appropriate buffer size.
if (size_t Size = preferred_buffer_size())
SetBufferSize(Size);
@@ -105,25 +106,6 @@ void raw_ostream::SetBuffered() {
SetUnbuffered();
}
-void raw_ostream::SetBufferAndMode(char *BufferStart, size_t Size,
- BufferKind Mode) {
- assert(((Mode == BufferKind::Unbuffered && !BufferStart && Size == 0) ||
- (Mode != BufferKind::Unbuffered && BufferStart && Size != 0)) &&
- "stream must be unbuffered or have at least one byte");
- // Make sure the current buffer is free of content (we can't flush here; the
- // child buffer management logic will be in write_impl).
- assert(GetNumBytesInBuffer() == 0 && "Current buffer is non-empty!");
-
- if (BufferMode == BufferKind::InternalBuffer)
- delete [] OutBufStart;
- OutBufStart = BufferStart;
- OutBufEnd = OutBufStart+Size;
- OutBufCur = OutBufStart;
- BufferMode = Mode;
-
- assert(OutBufStart <= OutBufEnd && "Invalid size!");
-}
-
raw_ostream &raw_ostream::operator<<(unsigned long N) {
write_integer(*this, static_cast<uint64_t>(N), 0, IntegerStyle::Integer);
return *this;
@@ -217,44 +199,38 @@ raw_ostream &raw_ostream::operator<<(double N) {
return *this;
}
-void raw_ostream::flush_nonempty() {
- assert(OutBufCur > OutBufStart && "Invalid call to flush_nonempty.");
- size_t Length = OutBufCur - OutBufStart;
- OutBufCur = OutBufStart;
- write_impl(OutBufStart, Length);
-}
-
-raw_ostream &raw_ostream::write(unsigned char C) {
+void raw_ostream::writeSlow(unsigned char C) {
// Group exceptional cases into a single branch.
if (LLVM_UNLIKELY(OutBufCur >= OutBufEnd)) {
if (LLVM_UNLIKELY(!OutBufStart)) {
if (BufferMode == BufferKind::Unbuffered) {
write_impl(reinterpret_cast<char *>(&C), 1);
- return *this;
+ return;
}
// Set up a buffer and start over.
SetBuffered();
- return write(C);
+ write(C);
+ return;
}
flush_nonempty();
}
*OutBufCur++ = C;
- return *this;
}
-raw_ostream &raw_ostream::write(const char *Ptr, size_t Size) {
+void raw_ostream::writeSlow(const char *Ptr, size_t Size) {
// Group exceptional cases into a single branch.
if (LLVM_UNLIKELY(size_t(OutBufEnd - OutBufCur) < Size)) {
if (LLVM_UNLIKELY(!OutBufStart)) {
if (BufferMode == BufferKind::Unbuffered) {
write_impl(Ptr, Size);
- return *this;
+ return;
}
// Set up a buffer and start over.
SetBuffered();
- return write(Ptr, Size);
+ write(Ptr, Size);
+ return;
}
size_t NumBytes = OutBufEnd - OutBufCur;
@@ -269,22 +245,22 @@ raw_ostream &raw_ostream::write(const char *Ptr, size_t Size) {
size_t BytesRemaining = Size - BytesToWrite;
if (BytesRemaining > size_t(OutBufEnd - OutBufCur)) {
// Too much left over to copy into our buffer.
- return write(Ptr + BytesToWrite, BytesRemaining);
+ write(Ptr + BytesToWrite, BytesRemaining);
+ return;
}
copy_to_buffer(Ptr + BytesToWrite, BytesRemaining);
- return *this;
+ return;
}
// We don't have enough space in the buffer to fit the string in. Insert as
// much as possible, flush and start over with the remainder.
copy_to_buffer(Ptr, NumBytes);
flush_nonempty();
- return write(Ptr + NumBytes, Size - NumBytes);
+ write(Ptr + NumBytes, Size - NumBytes);
+ return;
}
copy_to_buffer(Ptr, Size);
-
- return *this;
}
void raw_ostream::copy_to_buffer(const char *Ptr, size_t Size) {
More information about the llvm-commits
mailing list