[libc-commits] [libc] [libc] Support dynamically-grown lines in FlatFileDatabase (PR #223811)
Jeff Bailey via libc-commits
libc-commits at lists.llvm.org
Wed Sep 16 11:49:29 PDT 2026
https://github.com/kaladron updated https://github.com/llvm/llvm-project/pull/223811
>From 09173b049210d972f44b5dab22c9bed09aae270c Mon Sep 17 00:00:00 2001
From: Jeff Bailey <jbailey at raspberryginger.com>
Date: Mon, 14 Sep 2026 07:59:29 +0100
Subject: [PATCH 1/4] [libc] Support dynamically-grown lines in
FlatFileDatabase
Add pwd::DynamicBuffer and overloads of FlatFileDatabase::getnext and
lookup that read into a growable buffer rather than a fixed span.
parse_line now returns ErrorOr<void> and accepts the buffer and line
length so parsers can report ERANGE or EINVAL directly.
Fixed-buffer getnext keeps its signature and never allocates.
Fixed-buffer lookup skips unrelated oversized preceding records using a
temporary ScopedDynamicBuffer and reports ERANGE only when the matched
record does not fit in the caller's buffer.
DynamicBuffer is trivially destructible by design so that it can be a
constant-initialised process global. Storage is released explicitly by
whoever owns it.
* Add pwd::DynamicBuffer in src/__support/pwd/dynamic_buffer.h
* Add growable overloads to FlatFileDatabase
* Update parse_line<struct passwd> signature
* Add unit tests for dynamically-grown lines in FlatFileDatabase
Assisted-by: Automated tooling, human reviewed.
---
libc/src/__support/pwd/CMakeLists.txt | 19 +-
libc/src/__support/pwd/dynamic_buffer.h | 104 +++++++++
libc/src/__support/pwd/flat_file_db.h | 206 ++++++++++++++++--
libc/src/pwd/pwd_utils.cpp | 6 +-
libc/src/pwd/pwd_utils.h | 41 ++--
libc/test/src/__support/pwd/CMakeLists.txt | 4 +
.../src/__support/pwd/flat_file_db_test.cpp | 150 ++++++++++---
7 files changed, 463 insertions(+), 67 deletions(-)
create mode 100644 libc/src/__support/pwd/dynamic_buffer.h
diff --git a/libc/src/__support/pwd/CMakeLists.txt b/libc/src/__support/pwd/CMakeLists.txt
index 5ea67f910784e..a7c17936afb4b 100644
--- a/libc/src/__support/pwd/CMakeLists.txt
+++ b/libc/src/__support/pwd/CMakeLists.txt
@@ -10,16 +10,33 @@ add_header_library(
libc.src.__support.macros.config
)
+add_header_library(
+ dynamic_buffer
+ HDRS
+ dynamic_buffer.h
+ DEPENDS
+ libc.hdr.func.free
+ libc.hdr.func.realloc
+ libc.hdr.types.size_t
+ libc.src.__support.CPP.limits
+ libc.src.__support.CPP.span
+ libc.src.__support.CPP.type_traits
+ libc.src.__support.macros.attributes
+ libc.src.__support.macros.config
+)
+
add_header_library(
flat_file_db
HDRS
flat_file_db.h
DEPENDS
- .field_tokenizer
+ .dynamic_buffer
libc.hdr.errno_macros
libc.hdr.stdio_macros
+ libc.hdr.types.off_t
libc.hdr.types.size_t
libc.src.__support.CPP.functional
+ libc.src.__support.CPP.limits
libc.src.__support.CPP.span
libc.src.__support.File.file
libc.src.__support.error_or
diff --git a/libc/src/__support/pwd/dynamic_buffer.h b/libc/src/__support/pwd/dynamic_buffer.h
new file mode 100644
index 0000000000000..59f205d4161e4
--- /dev/null
+++ b/libc/src/__support/pwd/dynamic_buffer.h
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Growable byte buffer with explicit lifetime management.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_PWD_DYNAMIC_BUFFER_H
+#define LLVM_LIBC_SRC___SUPPORT_PWD_DYNAMIC_BUFFER_H
+
+#include "hdr/func/free.h"
+#include "hdr/func/realloc.h"
+#include "hdr/types/size_t.h"
+#include "src/__support/CPP/limits.h"
+#include "src/__support/CPP/span.h"
+#include "src/__support/CPP/type_traits/is_trivially_destructible.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace pwd {
+
+// A heap-backed, growable byte buffer for the flat-file database engine.
+//
+// This type is trivially destructible by design, so an instance can be a
+// constant-initialised process global. The owner releases the storage
+// explicitly when done.
+class DynamicBuffer {
+ static constexpr size_t INITIAL_CAPACITY = 256;
+
+ char *ptr = nullptr;
+ size_t cap = 0;
+
+public:
+ LIBC_INLINE constexpr DynamicBuffer() = default;
+
+ DynamicBuffer(const DynamicBuffer &) = delete;
+ DynamicBuffer &operator=(const DynamicBuffer &) = delete;
+
+ // Grows the buffer to hold at least new_capacity bytes, preserving the
+ // existing contents. Capacity doubles so that repeated growth stays linear
+ // in the number of bytes read. Returns false if allocation failed, in which
+ // case the buffer is left untouched.
+ [[nodiscard]] LIBC_INLINE bool reserve(size_t new_capacity) {
+ if (new_capacity <= cap)
+ return true;
+
+ size_t next = cap == 0 ? INITIAL_CAPACITY : cap;
+ while (next < new_capacity) {
+ if (next > cpp::numeric_limits<size_t>::max() / 2)
+ return false;
+ next *= 2;
+ }
+
+ void *new_ptr = ::realloc(ptr, next);
+ if (new_ptr == nullptr)
+ return false;
+
+ ptr = static_cast<char *>(new_ptr);
+ cap = next;
+ return true;
+ }
+
+ // Doubles the current capacity, or allocates the initial capacity when the
+ // buffer is empty. Returns false if allocation failed.
+ [[nodiscard]] LIBC_INLINE bool grow() {
+ if (cap == cpp::numeric_limits<size_t>::max())
+ return false;
+ return reserve(cap == 0 ? INITIAL_CAPACITY : cap + 1);
+ }
+
+ // Frees the storage and returns the buffer to its empty state. Safe to call
+ // more than once.
+ LIBC_INLINE void release() {
+ ::free(ptr);
+ ptr = nullptr;
+ cap = 0;
+ }
+
+ [[nodiscard]] LIBC_INLINE cpp::span<char> span() { return {ptr, cap}; }
+ [[nodiscard]] LIBC_INLINE size_t capacity() const { return cap; }
+};
+
+static_assert(cpp::is_trivially_destructible<DynamicBuffer>::value,
+ "DynamicBuffer must be trivially destructible");
+
+// RAII wrapper around DynamicBuffer for stack-local buffers.
+class ScopedDynamicBuffer : public DynamicBuffer {
+public:
+ using DynamicBuffer::DynamicBuffer;
+
+ LIBC_INLINE ~ScopedDynamicBuffer() { this->release(); }
+};
+
+} // namespace pwd
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_PWD_DYNAMIC_BUFFER_H
diff --git a/libc/src/__support/pwd/flat_file_db.h b/libc/src/__support/pwd/flat_file_db.h
index df0b168d17556..318faf0cf5fac 100644
--- a/libc/src/__support/pwd/flat_file_db.h
+++ b/libc/src/__support/pwd/flat_file_db.h
@@ -16,27 +16,37 @@
#include "hdr/errno_macros.h"
#include "hdr/stdio_macros.h"
+#include "hdr/types/off_t.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/functional.h"
+#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/span.h"
#include "src/__support/File/file.h"
#include "src/__support/error_or.h"
#include "src/__support/macros/attributes.h"
#include "src/__support/macros/config.h"
+#include "src/__support/pwd/dynamic_buffer.h"
namespace LIBC_NAMESPACE_DECL {
namespace pwd {
+// Struct to hold the result of a line read operation.
struct ReadLineResult {
size_t bytes_read;
+ size_t raw_bytes_consumed;
bool truncated;
- // True only when no data was read because the file stream reached EOF.
+ // True only when zero bytes were read because the stream was already at EOF.
+ // A final line without a trailing newline returns bytes_read > 0 and
+ // eof == false; the following call returns bytes_read == 0 and eof == true.
bool eof;
};
-// Forward declaration of record parser for flat database files.
+// Parses a record in place and fills entry.
+// If the buffer is too small for auxiliary structures (such as pointer arrays),
+// specializations must return Error(ERANGE) prior to modifying the buffer.
template <typename EntryType>
-bool parse_line(cpp::span<char> line, EntryType *entry);
+ErrorOr<void> parse_line(cpp::span<char> buffer, size_t line_len,
+ EntryType *entry);
// Generic flat colon-delimited database engine.
template <typename EntryType> class FlatFileDatabase {
@@ -46,12 +56,18 @@ template <typename EntryType> class FlatFileDatabase {
private:
const char *file_path;
File *file = nullptr;
+ off_t current_offset = 0;
+ off_t last_line_start = 0;
// Reads a single line from the given file into the provided buffer, stripping
- // any trailing '\n' and ensuring the result is null-terminated.
+ // any trailing '\n' and ensuring the result is null-terminated. A line too
+ // long for the buffer is reported as truncated.
+ //
// Note: POSIX getline/getdelim cannot be used here because user database
- // lookups (including reentrant _r variants) must operate in-place within a
- // fixed, bounded buffer without dynamic heap allocations or realloc.
+ // iteration and lookups must operate in-place within a fixed, bounded buffer
+ // without dynamic heap allocations during getnext. See read_line_growing for
+ // the variant used by the non-reentrant interfaces, which own their buffer
+ // and may grow it.
LIBC_INLINE static ErrorOr<ReadLineResult> read_line(File *f,
cpp::span<char> buf) {
if (!f)
@@ -61,6 +77,7 @@ template <typename EntryType> class FlatFileDatabase {
File::FileLock lock(f);
size_t bytes_read = 0;
+ size_t raw_bytes_consumed = 0;
FileIOResult result(0);
bool truncated = false;
@@ -71,6 +88,7 @@ template <typename EntryType> class FlatFileDatabase {
if (result.value != 1)
break;
++bytes_read;
+ ++raw_bytes_consumed;
if (ch == '\n')
break;
}
@@ -79,14 +97,17 @@ template <typename EntryType> class FlatFileDatabase {
auto read_span = buf.first(bytes_read);
if (result.value == 1 && !read_span.empty() && read_span.back() != '\n') {
- truncated = true;
char c = '\0';
while (true) {
result = f->read_unlocked(&c, 1);
if (result.has_error())
return Error(result.error);
- if (result.value != 1 || c == '\n')
+ if (result.value != 1)
break;
+ ++raw_bytes_consumed;
+ if (c == '\n')
+ break;
+ truncated = true;
}
}
@@ -98,13 +119,67 @@ template <typename EntryType> class FlatFileDatabase {
--bytes_read;
buf[bytes_read] = '\0';
- return ReadLineResult{bytes_read, truncated, eof};
+ return ReadLineResult{bytes_read, raw_bytes_consumed, truncated, eof};
+ }
+
+ // Reads a single line into a caller-owned buffer, growing it as needed so
+ // that arbitrarily long records can be read. Otherwise behaves as read_line;
+ // the result is never truncated.
+ LIBC_INLINE static ErrorOr<ReadLineResult>
+ read_line_growing(File *f, DynamicBuffer &buf) {
+ if (!f)
+ return Error(EINVAL);
+
+ File::FileLock lock(f);
+ size_t bytes_read = 0;
+
+ while (true) {
+ // One byte for the character about to be read, one for the terminator.
+ if (bytes_read > cpp::numeric_limits<size_t>::max() - 2 ||
+ (bytes_read + 2 > buf.capacity() && !buf.reserve(bytes_read + 2))) {
+ char c = '\0';
+ while (true) {
+ FileIOResult drain = f->read_unlocked(&c, 1);
+ if (drain.has_error() || drain.value != 1 || c == '\n')
+ break;
+ }
+ return Error(ENOMEM);
+ }
+
+ char ch = '\0';
+ FileIOResult result = f->read_unlocked(&ch, 1);
+ if (result.has_error())
+ return Error(result.error);
+ if (result.value != 1)
+ break;
+
+ buf.span()[bytes_read++] = ch;
+ if (ch == '\n')
+ break;
+ }
+
+ if (f->error_unlocked())
+ return Error(EIO);
+
+ bool eof = (bytes_read == 0);
+ size_t raw_bytes_consumed = bytes_read;
+
+ // If the line ended with a newline, strip it.
+ if (bytes_read > 0 && buf.span()[bytes_read - 1] == '\n')
+ --bytes_read;
+
+ buf.span()[bytes_read] = '\0';
+ return ReadLineResult{bytes_read, raw_bytes_consumed, /*truncated=*/false,
+ eof};
}
public:
LIBC_INLINE constexpr explicit FlatFileDatabase(const char *path)
: file_path(path) {}
+ FlatFileDatabase(const FlatFileDatabase &) = delete;
+ FlatFileDatabase &operator=(const FlatFileDatabase &) = delete;
+
// Sets or overrides the file path for database operations.
LIBC_INLINE void set_path(const char *path) {
if (!path)
@@ -114,10 +189,14 @@ template <typename EntryType> class FlatFileDatabase {
file = nullptr;
}
file_path = path;
+ current_offset = 0;
+ last_line_start = 0;
}
// Opens or rewinds the database file stream.
LIBC_INLINE ErrorOr<void> setdb() {
+ current_offset = 0;
+ last_line_start = 0;
if (!file) {
auto result = openfile(file_path, "r");
if (!result.has_value())
@@ -128,11 +207,14 @@ template <typename EntryType> class FlatFileDatabase {
auto result = file->seek(0, SEEK_SET);
if (!result.has_value())
return Error(result.error());
+ file->clearerr();
return {};
}
// Closes the database file stream.
LIBC_INLINE ErrorOr<void> enddb() {
+ current_offset = 0;
+ last_line_start = 0;
if (file) {
int result = file->close();
file = nullptr;
@@ -142,9 +224,10 @@ template <typename EntryType> class FlatFileDatabase {
return {};
}
- // Reads and parses the next record from the database. Returns true if an
- // entry was read, false if EOF was reached, or an Error on failure. Blank
- // lines are skipped.
+ // Reads and parses the next record from the database into a fixed buffer.
+ // Returns true if an entry was read, false if EOF was reached, or an Error on
+ // failure. Blank lines are skipped. A record that does not fit in the buffer
+ // is reported as ERANGE.
LIBC_INLINE ErrorOr<bool> getnext(EntryType *entry, cpp::span<char> buffer) {
if (!entry)
return Error(EINVAL);
@@ -156,11 +239,13 @@ template <typename EntryType> class FlatFileDatabase {
}
while (true) {
+ last_line_start = current_offset;
auto result = read_line(file, buffer);
if (!result.has_value())
return Error(result.error());
ReadLineResult res = result.value();
+ current_offset += static_cast<off_t>(res.raw_bytes_consumed);
if (res.eof)
return false; // EOF
@@ -171,16 +256,61 @@ template <typename EntryType> class FlatFileDatabase {
if (res.truncated)
return Error(ERANGE);
- if (parse_line(buffer.first(res.bytes_read + 1), entry))
- return true;
+ auto parse_res = parse_line<EntryType>(buffer, res.bytes_read, entry);
+ if (!parse_res.has_value())
+ return Error(parse_res.error());
+ return true;
+ }
+ }
+ // Reads and parses the next record from the database into a caller-owned
+ // buffer, growing it as needed. Behaves as the fixed-buffer overload except
+ // that a long record grows the buffer rather than producing ERANGE.
+ LIBC_INLINE ErrorOr<bool> getnext(EntryType *entry, DynamicBuffer &buffer) {
+ if (!entry)
return Error(EINVAL);
+
+ if (!file) {
+ auto res = setdb();
+ if (!res.has_value())
+ return Error(res.error());
+ }
+
+ while (true) {
+ last_line_start = current_offset;
+ auto result = read_line_growing(file, buffer);
+ if (!result.has_value())
+ return Error(result.error());
+
+ ReadLineResult res = result.value();
+ current_offset += static_cast<off_t>(res.raw_bytes_consumed);
+ if (res.eof)
+ return false; // EOF
+
+ // Skip blank lines.
+ if (res.bytes_read == 0)
+ continue;
+
+ while (true) {
+ auto parse_res =
+ parse_line<EntryType>(buffer.span(), res.bytes_read, entry);
+ if (parse_res.has_value())
+ return true;
+ if (parse_res.error() != ERANGE)
+ return Error(parse_res.error());
+ if (!buffer.grow())
+ return Error(ENOMEM);
+ }
}
}
// Searches for a record matching a given predicate. Returns true if the
// entry was found, false if it's missing, or an Error if lookup failed.
- LIBC_INLINE ErrorOr<bool> lookup(Matcher matcher, EntryType *entry,
+ //
+ // Per POSIX, ERANGE is reported only if the matched entry does not fit in the
+ // caller's buffer; unrelated preceding records larger than buffer are
+ // skipped.
+ LIBC_INLINE ErrorOr<bool> lookup(const Matcher &matcher, EntryType *entry,
cpp::span<char> buffer) {
if (!entry)
return Error(EINVAL);
@@ -189,6 +319,52 @@ template <typename EntryType> class FlatFileDatabase {
if (!res.has_value())
return Error(res.error());
+ ScopedDynamicBuffer scratch_buf;
+ while (true) {
+ auto next_res = getnext(entry, buffer);
+ if (next_res.has_value()) {
+ if (!next_res.value())
+ return false; // EOF without match
+ if (matcher(*entry))
+ return true;
+ continue;
+ }
+
+ if (next_res.error() != ERANGE)
+ return Error(next_res.error());
+
+ // The record at last_line_start exceeded buffer. Check whether it is
+ // actually the target entry before reporting ERANGE. Use a stack-local
+ // scratch_entry so we do not leave dangling pointers in the caller's
+ // *entry when scratch_buf goes out of scope.
+ auto seek_res = file->seek(last_line_start, SEEK_SET);
+ if (!seek_res.has_value())
+ return Error(seek_res.error());
+ file->clearerr();
+ current_offset = last_line_start;
+
+ EntryType scratch_entry{};
+ auto dyn_res = getnext(&scratch_entry, scratch_buf);
+ if (!dyn_res.has_value())
+ return Error(dyn_res.error());
+ if (!dyn_res.value())
+ return false;
+ if (matcher(scratch_entry))
+ return Error(ERANGE);
+ }
+ }
+
+ // As above, but reads into a caller-owned buffer that grows to fit long
+ // records instead of reporting ERANGE.
+ LIBC_INLINE ErrorOr<bool> lookup(const Matcher &matcher, EntryType *entry,
+ DynamicBuffer &buffer) {
+ if (!entry)
+ return Error(EINVAL);
+
+ auto res = setdb();
+ if (!res.has_value())
+ return Error(res.error());
+
while (true) {
auto next_res = getnext(entry, buffer);
if (!next_res.has_value())
diff --git a/libc/src/pwd/pwd_utils.cpp b/libc/src/pwd/pwd_utils.cpp
index fb4e75f9544c9..55052c8c39232 100644
--- a/libc/src/pwd/pwd_utils.cpp
+++ b/libc/src/pwd/pwd_utils.cpp
@@ -13,6 +13,7 @@
#include "src/pwd/pwd_utils.h"
#include "hdr/errno_macros.h"
+#include "hdr/types/size_t.h"
#include "hdr/types/struct_passwd.h"
#include "src/__support/CPP/span.h"
#include "src/__support/CPP/string_view.h"
@@ -33,8 +34,9 @@ ErrorOr<struct passwd> parse_passwd_line(char *line) {
struct passwd pwd;
size_t len = internal::string_length(line);
- if (!parse_line(cpp::span<char>(line, len + 1), &pwd))
- return Error(EINVAL);
+ auto res = parse_line(cpp::span<char>(line, len + 1), len, &pwd);
+ if (!res.has_value())
+ return Error(res.error());
return pwd;
}
diff --git a/libc/src/pwd/pwd_utils.h b/libc/src/pwd/pwd_utils.h
index 1af49d2019cf7..36a98c7bc41d7 100644
--- a/libc/src/pwd/pwd_utils.h
+++ b/libc/src/pwd/pwd_utils.h
@@ -16,6 +16,7 @@
#include "hdr/errno_macros.h"
#include "hdr/types/gid_t.h"
+#include "hdr/types/size_t.h"
#include "hdr/types/struct_passwd.h"
#include "hdr/types/uid_t.h"
#include "src/__support/CPP/span.h"
@@ -34,59 +35,63 @@ namespace pwd {
// Parses a colon-separated line in-place into a struct passwd.
template <>
-LIBC_INLINE bool parse_line<struct passwd>(cpp::span<char> line,
- struct passwd *pwd) {
- if (line.empty() || !pwd)
- return false;
+LIBC_INLINE ErrorOr<void> parse_line<struct passwd>(cpp::span<char> buffer,
+ size_t line_len,
+ struct passwd *pwd) {
+ if (!pwd || line_len == 0 || line_len >= buffer.size())
+ return Error(EINVAL);
- FieldTokenizer tokenizer(line);
+ FieldTokenizer tokenizer(buffer.first(line_len + 1));
auto name = tokenizer.next_field();
- if (!name)
- return false;
+ if (!name || name->empty() || name->front() == '\0')
+ return Error(EINVAL);
pwd->pw_name = name->data();
auto passwd = tokenizer.next_field();
if (!passwd)
- return false;
+ return Error(EINVAL);
pwd->pw_passwd = passwd->data();
auto uid_str = tokenizer.next_field();
if (!uid_str || uid_str->empty() || !internal::isdigit(uid_str->front()))
- return false;
+ return Error(EINVAL);
auto uid_res = internal::strtointeger<uid_t>(uid_str->data(), 10);
if (uid_res.has_error() || uid_res.parsed_len <= 0 ||
- static_cast<size_t>(uid_res.parsed_len) >= uid_str->size() ||
+ static_cast<size_t>(uid_res.parsed_len) + 1 != uid_str->size() ||
(*uid_str)[uid_res.parsed_len] != '\0')
- return false;
+ return Error(EINVAL);
pwd->pw_uid = uid_res.value;
auto gid_str = tokenizer.next_field();
if (!gid_str || gid_str->empty() || !internal::isdigit(gid_str->front()))
- return false;
+ return Error(EINVAL);
auto gid_res = internal::strtointeger<gid_t>(gid_str->data(), 10);
if (gid_res.has_error() || gid_res.parsed_len <= 0 ||
- static_cast<size_t>(gid_res.parsed_len) >= gid_str->size() ||
+ static_cast<size_t>(gid_res.parsed_len) + 1 != gid_str->size() ||
(*gid_str)[gid_res.parsed_len] != '\0')
- return false;
+ return Error(EINVAL);
pwd->pw_gid = gid_res.value;
auto gecos = tokenizer.next_field();
if (!gecos)
- return false;
+ return Error(EINVAL);
pwd->pw_gecos = gecos->data();
auto dir = tokenizer.next_field();
if (!dir)
- return false;
+ return Error(EINVAL);
pwd->pw_dir = dir->data();
auto shell = tokenizer.next_field();
if (!shell)
- return false;
+ return Error(EINVAL);
pwd->pw_shell = shell->data();
- return true;
+ if (tokenizer.next_field())
+ return Error(EINVAL);
+
+ return {};
}
// Parses a colon-separated password database line into a struct passwd.
diff --git a/libc/test/src/__support/pwd/CMakeLists.txt b/libc/test/src/__support/pwd/CMakeLists.txt
index c6ec4ae517d83..a63fe2ecb010d 100644
--- a/libc/test/src/__support/pwd/CMakeLists.txt
+++ b/libc/test/src/__support/pwd/CMakeLists.txt
@@ -27,7 +27,11 @@ add_libc_test(
libc.src.__support.CPP.string_view
libc.src.__support.File.file
libc.src.__support.File.platform_file
+ libc.src.__support.error_or
+ libc.src.__support.pwd.dynamic_buffer
+ libc.src.__support.pwd.field_tokenizer
libc.src.__support.pwd.flat_file_db
libc.src.stdio.remove
libc.src.string.string_utils
+ libc.test.UnitTest.ErrnoCheckingTest
)
diff --git a/libc/test/src/__support/pwd/flat_file_db_test.cpp b/libc/test/src/__support/pwd/flat_file_db_test.cpp
index 00c709b484767..2bcb141fff8ff 100644
--- a/libc/test/src/__support/pwd/flat_file_db_test.cpp
+++ b/libc/test/src/__support/pwd/flat_file_db_test.cpp
@@ -12,14 +12,17 @@
//===----------------------------------------------------------------------===//
#include "hdr/errno_macros.h"
+#include "hdr/types/size_t.h"
#include "src/__support/CPP/span.h"
+#include "src/__support/CPP/string_view.h"
#include "src/__support/File/file.h"
+#include "src/__support/error_or.h"
+#include "src/__support/pwd/dynamic_buffer.h"
#include "src/__support/pwd/field_tokenizer.h"
#include "src/__support/pwd/flat_file_db.h"
#include "src/stdio/remove.h"
#include "src/string/string_utils.h"
#include "test/UnitTest/ErrnoCheckingTest.h"
-#include "test/UnitTest/ErrnoSetterMatcher.h"
#include "test/UnitTest/Test.h"
namespace {
@@ -59,22 +62,24 @@ namespace LIBC_NAMESPACE_DECL {
namespace pwd {
template <>
-inline bool parse_line<SimpleTestEntry>(cpp::span<char> line,
- SimpleTestEntry *entry) {
- if (line.empty() || !entry)
- return false;
- FieldTokenizer tokenizer(line);
+inline ErrorOr<void> parse_line<SimpleTestEntry>(cpp::span<char> buffer,
+ size_t line_len,
+ SimpleTestEntry *entry) {
+ if (!entry || line_len == 0 || line_len >= buffer.size())
+ return Error(EINVAL);
+
+ FieldTokenizer tokenizer(buffer.first(line_len + 1));
auto k = tokenizer.next_field();
if (!k)
- return false;
+ return Error(EINVAL);
entry->key = k->data();
auto v = tokenizer.next_field();
if (!v)
- return false;
+ return Error(EINVAL);
entry->val = v->data();
- return true;
+ return {};
}
} // namespace pwd
@@ -85,7 +90,7 @@ TEST_F(LlvmLibcFlatFileDbTest, GetNextAndLookup) {
HermeticFile test_file(libc_make_test_file_path("flat_db_test.test"),
content);
- LIBC_NAMESPACE::pwd::FlatFileDatabase<SimpleTestEntry> db(
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
test_file.get_path());
char buffer[128];
SimpleTestEntry entry;
@@ -94,20 +99,20 @@ TEST_F(LlvmLibcFlatFileDbTest, GetNextAndLookup) {
auto r1 = db.getnext(&entry, buffer);
ASSERT_TRUE(r1.has_value());
ASSERT_TRUE(r1.value());
- ASSERT_STREQ(entry.key, "user1");
- ASSERT_STREQ(entry.val, "secret1");
+ EXPECT_STREQ(entry.key, "user1");
+ EXPECT_STREQ(entry.val, "secret1");
// Second record
auto r2 = db.getnext(&entry, buffer);
ASSERT_TRUE(r2.has_value());
ASSERT_TRUE(r2.value());
- ASSERT_STREQ(entry.key, "user2");
- ASSERT_STREQ(entry.val, "secret2");
+ EXPECT_STREQ(entry.key, "user2");
+ EXPECT_STREQ(entry.val, "secret2");
// EOF
auto r3 = db.getnext(&entry, buffer);
ASSERT_TRUE(r3.has_value());
- ASSERT_FALSE(r3.value());
+ EXPECT_FALSE(r3.value());
// Rewind and lookup
db.setdb();
@@ -117,10 +122,8 @@ TEST_F(LlvmLibcFlatFileDbTest, GetNextAndLookup) {
auto lookup_res = db.lookup(matcher, &entry, buffer);
ASSERT_TRUE(lookup_res.has_value());
ASSERT_TRUE(lookup_res.value());
- ASSERT_STREQ(entry.key, "user2");
- ASSERT_STREQ(entry.val, "secret2");
-
- db.enddb();
+ EXPECT_STREQ(entry.key, "user2");
+ EXPECT_STREQ(entry.val, "secret2");
}
TEST_F(LlvmLibcFlatFileDbTest, LookupNotFound) {
@@ -128,7 +131,7 @@ TEST_F(LlvmLibcFlatFileDbTest, LookupNotFound) {
HermeticFile test_file(libc_make_test_file_path("flat_db_not_found.test"),
content);
- LIBC_NAMESPACE::pwd::FlatFileDatabase<SimpleTestEntry> db(
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
test_file.get_path());
char buffer[128];
SimpleTestEntry entry;
@@ -138,9 +141,7 @@ TEST_F(LlvmLibcFlatFileDbTest, LookupNotFound) {
};
auto lookup_res = db.lookup(matcher, &entry, buffer);
ASSERT_TRUE(lookup_res.has_value());
- ASSERT_FALSE(lookup_res.value());
-
- db.enddb();
+ EXPECT_FALSE(lookup_res.value());
}
TEST_F(LlvmLibcFlatFileDbTest, TruncatedLineReturnsErange) {
@@ -148,16 +149,14 @@ TEST_F(LlvmLibcFlatFileDbTest, TruncatedLineReturnsErange) {
HermeticFile test_file(libc_make_test_file_path("flat_db_trunc.test"),
content);
- LIBC_NAMESPACE::pwd::FlatFileDatabase<SimpleTestEntry> db(
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
test_file.get_path());
char small_buffer[8];
SimpleTestEntry entry;
auto res = db.getnext(&entry, small_buffer);
ASSERT_FALSE(res.has_value());
- ASSERT_EQ(res.error(), ERANGE);
-
- db.enddb();
+ EXPECT_EQ(res.error(), ERANGE);
}
TEST_F(LlvmLibcFlatFileDbTest, MalformedLineReturnsEinval) {
@@ -165,16 +164,14 @@ TEST_F(LlvmLibcFlatFileDbTest, MalformedLineReturnsEinval) {
HermeticFile test_file(libc_make_test_file_path("flat_db_malformed.test"),
content);
- LIBC_NAMESPACE::pwd::FlatFileDatabase<SimpleTestEntry> db(
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
test_file.get_path());
char buffer[128];
SimpleTestEntry entry;
auto res = db.getnext(&entry, buffer);
ASSERT_FALSE(res.has_value());
- ASSERT_EQ(res.error(), EINVAL);
-
- db.enddb();
+ EXPECT_EQ(res.error(), EINVAL);
}
TEST_F(LlvmLibcFlatFileDbTest, BlankLinesSkipped) {
@@ -217,3 +214,94 @@ TEST_F(LlvmLibcFlatFileDbTest, BlankLinesSkipped) {
ASSERT_STREQ(entry.key, "user2");
ASSERT_STREQ(entry.val, "secret2");
}
+
+TEST_F(LlvmLibcFlatFileDbTest, DynamicBufferReadsArbitrarilyLongLines) {
+ // Two records, each far beyond the buffer's initial capacity, so that
+ // iteration exercises repeated growth.
+ constexpr size_t RECORD_COUNT = 2;
+ constexpr size_t VALUE_LENGTH = 4000;
+ constexpr size_t RECORD_OVERHEAD = 32;
+ char content[RECORD_COUNT * (VALUE_LENGTH + RECORD_OVERHEAD)];
+
+ size_t pos = 0;
+ for (size_t record = 0; record < RECORD_COUNT; ++record) {
+ const char *key = record == 0 ? "key0:" : "key1:";
+ for (const char *p = key; *p != '\0'; ++p)
+ content[pos++] = *p;
+ for (size_t i = 0; i < VALUE_LENGTH; ++i)
+ content[pos++] = 'v';
+ content[pos++] = '\n';
+ }
+ content[pos] = '\0';
+
+ HermeticFile test_file(libc_make_test_file_path("flat_db_longline.test"),
+ content);
+
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
+ test_file.get_path());
+ LIBC_NAMESPACE::pwd::ScopedDynamicBuffer buffer;
+ SimpleTestEntry entry;
+
+ auto r1 = db.getnext(&entry, buffer);
+ ASSERT_TRUE(r1.has_value());
+ ASSERT_TRUE(r1.value());
+ ASSERT_STREQ(entry.key, "key0");
+ ASSERT_EQ(LIBC_NAMESPACE::internal::string_length(entry.val), VALUE_LENGTH);
+
+ auto r2 = db.getnext(&entry, buffer);
+ ASSERT_TRUE(r2.has_value());
+ ASSERT_TRUE(r2.value());
+ ASSERT_STREQ(entry.key, "key1");
+
+ auto r3 = db.getnext(&entry, buffer);
+ ASSERT_TRUE(r3.has_value());
+ ASSERT_FALSE(r3.value());
+}
+
+TEST_F(LlvmLibcFlatFileDbTest, PrecedingLongRecordsSkippedDuringLookup) {
+ const char *content =
+ "huge_unrelated_key:012345678901234567890123456789012345\n"
+ "target:short\n";
+ HermeticFile test_file(libc_make_test_file_path("flat_db_longskip.test"),
+ content);
+
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
+ test_file.get_path());
+ // 24 bytes is large enough for "target:short" (12 chars + '\0') but smaller
+ // than "huge_unrelated_key:...". Fixed-buffer lookup must proceed past
+ // unrelated long records without falsely returning ERANGE.
+ constexpr size_t SMALL_BUFFER_SIZE = 24;
+ char buffer[SMALL_BUFFER_SIZE];
+ SimpleTestEntry entry;
+
+ auto res = db.lookup(
+ [](const SimpleTestEntry &e) {
+ return LIBC_NAMESPACE::cpp::string_view(e.key) == "target";
+ },
+ &entry, buffer);
+ ASSERT_TRUE(res.has_value());
+ ASSERT_TRUE(res.value());
+ ASSERT_STREQ(entry.key, "target");
+ ASSERT_STREQ(entry.val, "short");
+}
+
+TEST_F(LlvmLibcFlatFileDbTest, DynamicBufferReserveGrowRelease) {
+ LIBC_NAMESPACE::pwd::ScopedDynamicBuffer buffer;
+ ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+
+ ASSERT_TRUE(buffer.grow());
+ size_t initial = buffer.capacity();
+ ASSERT_GT(initial, static_cast<size_t>(0));
+
+ ASSERT_TRUE(buffer.grow());
+ ASSERT_EQ(buffer.capacity(), initial * 2);
+
+ buffer.release();
+ ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+ buffer.release();
+ ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+
+ constexpr size_t TARGET_RESERVE_CAPACITY = 1024;
+ ASSERT_TRUE(buffer.reserve(TARGET_RESERVE_CAPACITY));
+ ASSERT_GE(buffer.capacity(), TARGET_RESERVE_CAPACITY);
+}
>From 8603f1a49a77dca7cc25170bd380d5c8a73069e5 Mon Sep 17 00:00:00 2001
From: Jeff Bailey <jbailey at raspberryginger.com>
Date: Wed, 16 Sep 2026 15:44:36 +0100
Subject: [PATCH 2/4] [libc] Address PR 223811 review feedback in
FlatFileDatabase
* Set DynamicBuffer::INITIAL_CAPACITY to 1024 to match
_SC_GETGR_R_SIZE_MAX
* Mark DynamicBuffer::span() const
* Clarify ReadLineResult::eof comment for blank and newline-only lines
* Drop redundant buf.capacity() check before buf.reserve() in
read_line_growing
* Propagate drain.error from the line-drain loop in read_line_growing
* Extract clear_file_stream() helper shared by set_path() and enddb()
Assisted-by: Automated tooling, human reviewed.
---
libc/src/__support/pwd/dynamic_buffer.h | 4 +-
libc/src/__support/pwd/flat_file_db.h | 50 +++++++++++++------------
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/libc/src/__support/pwd/dynamic_buffer.h b/libc/src/__support/pwd/dynamic_buffer.h
index 59f205d4161e4..44d269071c173 100644
--- a/libc/src/__support/pwd/dynamic_buffer.h
+++ b/libc/src/__support/pwd/dynamic_buffer.h
@@ -32,7 +32,7 @@ namespace pwd {
// constant-initialised process global. The owner releases the storage
// explicitly when done.
class DynamicBuffer {
- static constexpr size_t INITIAL_CAPACITY = 256;
+ static constexpr size_t INITIAL_CAPACITY = 1024;
char *ptr = nullptr;
size_t cap = 0;
@@ -83,7 +83,7 @@ class DynamicBuffer {
cap = 0;
}
- [[nodiscard]] LIBC_INLINE cpp::span<char> span() { return {ptr, cap}; }
+ [[nodiscard]] LIBC_INLINE cpp::span<char> span() const { return {ptr, cap}; }
[[nodiscard]] LIBC_INLINE size_t capacity() const { return cap; }
};
diff --git a/libc/src/__support/pwd/flat_file_db.h b/libc/src/__support/pwd/flat_file_db.h
index 318faf0cf5fac..49524e40a8948 100644
--- a/libc/src/__support/pwd/flat_file_db.h
+++ b/libc/src/__support/pwd/flat_file_db.h
@@ -35,9 +35,11 @@ struct ReadLineResult {
size_t bytes_read;
size_t raw_bytes_consumed;
bool truncated;
- // True only when zero bytes were read because the stream was already at EOF.
- // A final line without a trailing newline returns bytes_read > 0 and
- // eof == false; the following call returns bytes_read == 0 and eof == true.
+ // True only when no raw bytes were read (raw_bytes_consumed == 0) because the
+ // stream was already at EOF. A blank line ("\n") or a final line without a
+ // trailing newline consumes at least one raw byte and returns eof == false
+ // (with bytes_read == 0 for "\n"); the following call returns
+ // raw_bytes_consumed == 0 and eof == true.
bool eof;
};
@@ -59,6 +61,19 @@ template <typename EntryType> class FlatFileDatabase {
off_t current_offset = 0;
off_t last_line_start = 0;
+ // Closes the file stream if open and resets stream position tracking.
+ LIBC_INLINE ErrorOr<void> clear_file_stream() {
+ current_offset = 0;
+ last_line_start = 0;
+ if (file) {
+ int result = file->close();
+ file = nullptr;
+ if (result != 0)
+ return Error(result);
+ }
+ return {};
+ }
+
// Reads a single line from the given file into the provided buffer, stripping
// any trailing '\n' and ensuring the result is null-terminated. A line too
// long for the buffer is reported as truncated.
@@ -93,7 +108,7 @@ template <typename EntryType> class FlatFileDatabase {
break;
}
- bool eof = (bytes_read == 0);
+ bool eof = (raw_bytes_consumed == 0);
auto read_span = buf.first(bytes_read);
if (result.value == 1 && !read_span.empty() && read_span.back() != '\n') {
@@ -136,11 +151,13 @@ template <typename EntryType> class FlatFileDatabase {
while (true) {
// One byte for the character about to be read, one for the terminator.
if (bytes_read > cpp::numeric_limits<size_t>::max() - 2 ||
- (bytes_read + 2 > buf.capacity() && !buf.reserve(bytes_read + 2))) {
+ !buf.reserve(bytes_read + 2)) {
char c = '\0';
while (true) {
FileIOResult drain = f->read_unlocked(&c, 1);
- if (drain.has_error() || drain.value != 1 || c == '\n')
+ if (drain.has_error())
+ return Error(drain.error);
+ if (drain.value != 1 || c == '\n')
break;
}
return Error(ENOMEM);
@@ -161,8 +178,8 @@ template <typename EntryType> class FlatFileDatabase {
if (f->error_unlocked())
return Error(EIO);
- bool eof = (bytes_read == 0);
size_t raw_bytes_consumed = bytes_read;
+ bool eof = (raw_bytes_consumed == 0);
// If the line ended with a newline, strip it.
if (bytes_read > 0 && buf.span()[bytes_read - 1] == '\n')
@@ -184,13 +201,8 @@ template <typename EntryType> class FlatFileDatabase {
LIBC_INLINE void set_path(const char *path) {
if (!path)
return;
- if (file) {
- file->close();
- file = nullptr;
- }
+ clear_file_stream();
file_path = path;
- current_offset = 0;
- last_line_start = 0;
}
// Opens or rewinds the database file stream.
@@ -212,17 +224,7 @@ template <typename EntryType> class FlatFileDatabase {
}
// Closes the database file stream.
- LIBC_INLINE ErrorOr<void> enddb() {
- current_offset = 0;
- last_line_start = 0;
- if (file) {
- int result = file->close();
- file = nullptr;
- if (result != 0)
- return Error(result);
- }
- return {};
- }
+ LIBC_INLINE ErrorOr<void> enddb() { return clear_file_stream(); }
// Reads and parses the next record from the database into a fixed buffer.
// Returns true if an entry was read, false if EOF was reached, or an Error on
>From 74c48849c936cf95c45558e260c9989aa1a805f9 Mon Sep 17 00:00:00 2001
From: Jeff Bailey <jbailey at raspberryginger.com>
Date: Wed, 16 Sep 2026 16:00:29 +0100
Subject: [PATCH 3/4] [libc] Template FlatFileDatabase::lookup over BufferType
Switch fixed-buffer lookup to return ERANGE whenever getnext encounters
a record that exceeds the caller's buffer, matching glibc and FreeBSD
and keeping reentrant lookups strictly zero-allocation.
* Unify fixed-buffer and DynamicBuffer lookup into a single template
over BufferType
* Remove ScopedDynamicBuffer fallback, seek rewind, and offset tracking
from FlatFileDatabase
* Update unit test to verify fixed-buffer lookup returns ERANGE while
DynamicBuffer lookup grows across oversized preceding records
Assisted-by: Automated tooling, human reviewed.
---
libc/src/__support/pwd/CMakeLists.txt | 1 -
libc/src/__support/pwd/flat_file_db.h | 95 +++----------------
.../src/__support/pwd/flat_file_db_test.cpp | 55 ++++++-----
3 files changed, 45 insertions(+), 106 deletions(-)
diff --git a/libc/src/__support/pwd/CMakeLists.txt b/libc/src/__support/pwd/CMakeLists.txt
index a7c17936afb4b..ccf32531ab69d 100644
--- a/libc/src/__support/pwd/CMakeLists.txt
+++ b/libc/src/__support/pwd/CMakeLists.txt
@@ -33,7 +33,6 @@ add_header_library(
.dynamic_buffer
libc.hdr.errno_macros
libc.hdr.stdio_macros
- libc.hdr.types.off_t
libc.hdr.types.size_t
libc.src.__support.CPP.functional
libc.src.__support.CPP.limits
diff --git a/libc/src/__support/pwd/flat_file_db.h b/libc/src/__support/pwd/flat_file_db.h
index 49524e40a8948..1ffc1decbb1f2 100644
--- a/libc/src/__support/pwd/flat_file_db.h
+++ b/libc/src/__support/pwd/flat_file_db.h
@@ -16,7 +16,6 @@
#include "hdr/errno_macros.h"
#include "hdr/stdio_macros.h"
-#include "hdr/types/off_t.h"
#include "hdr/types/size_t.h"
#include "src/__support/CPP/functional.h"
#include "src/__support/CPP/limits.h"
@@ -33,13 +32,11 @@ namespace pwd {
// Struct to hold the result of a line read operation.
struct ReadLineResult {
size_t bytes_read;
- size_t raw_bytes_consumed;
bool truncated;
- // True only when no raw bytes were read (raw_bytes_consumed == 0) because the
- // stream was already at EOF. A blank line ("\n") or a final line without a
- // trailing newline consumes at least one raw byte and returns eof == false
- // (with bytes_read == 0 for "\n"); the following call returns
- // raw_bytes_consumed == 0 and eof == true.
+ // True only when zero raw bytes were read because the stream was already at
+ // EOF. A blank line ("\n") or a final line without a trailing newline
+ // consumes at least one raw byte and returns eof == false (with
+ // bytes_read == 0 for "\n"); the following call returns eof == true.
bool eof;
};
@@ -58,13 +55,9 @@ template <typename EntryType> class FlatFileDatabase {
private:
const char *file_path;
File *file = nullptr;
- off_t current_offset = 0;
- off_t last_line_start = 0;
- // Closes the file stream if open and resets stream position tracking.
+ // Closes the file stream if open.
LIBC_INLINE ErrorOr<void> clear_file_stream() {
- current_offset = 0;
- last_line_start = 0;
if (file) {
int result = file->close();
file = nullptr;
@@ -92,7 +85,6 @@ template <typename EntryType> class FlatFileDatabase {
File::FileLock lock(f);
size_t bytes_read = 0;
- size_t raw_bytes_consumed = 0;
FileIOResult result(0);
bool truncated = false;
@@ -103,12 +95,11 @@ template <typename EntryType> class FlatFileDatabase {
if (result.value != 1)
break;
++bytes_read;
- ++raw_bytes_consumed;
if (ch == '\n')
break;
}
- bool eof = (raw_bytes_consumed == 0);
+ bool eof = (bytes_read == 0);
auto read_span = buf.first(bytes_read);
if (result.value == 1 && !read_span.empty() && read_span.back() != '\n') {
@@ -117,10 +108,7 @@ template <typename EntryType> class FlatFileDatabase {
result = f->read_unlocked(&c, 1);
if (result.has_error())
return Error(result.error);
- if (result.value != 1)
- break;
- ++raw_bytes_consumed;
- if (c == '\n')
+ if (result.value != 1 || c == '\n')
break;
truncated = true;
}
@@ -134,7 +122,7 @@ template <typename EntryType> class FlatFileDatabase {
--bytes_read;
buf[bytes_read] = '\0';
- return ReadLineResult{bytes_read, raw_bytes_consumed, truncated, eof};
+ return ReadLineResult{bytes_read, truncated, eof};
}
// Reads a single line into a caller-owned buffer, growing it as needed so
@@ -178,16 +166,14 @@ template <typename EntryType> class FlatFileDatabase {
if (f->error_unlocked())
return Error(EIO);
- size_t raw_bytes_consumed = bytes_read;
- bool eof = (raw_bytes_consumed == 0);
+ bool eof = (bytes_read == 0);
// If the line ended with a newline, strip it.
if (bytes_read > 0 && buf.span()[bytes_read - 1] == '\n')
--bytes_read;
buf.span()[bytes_read] = '\0';
- return ReadLineResult{bytes_read, raw_bytes_consumed, /*truncated=*/false,
- eof};
+ return ReadLineResult{bytes_read, /*truncated=*/false, eof};
}
public:
@@ -207,8 +193,6 @@ template <typename EntryType> class FlatFileDatabase {
// Opens or rewinds the database file stream.
LIBC_INLINE ErrorOr<void> setdb() {
- current_offset = 0;
- last_line_start = 0;
if (!file) {
auto result = openfile(file_path, "r");
if (!result.has_value())
@@ -241,13 +225,11 @@ template <typename EntryType> class FlatFileDatabase {
}
while (true) {
- last_line_start = current_offset;
auto result = read_line(file, buffer);
if (!result.has_value())
return Error(result.error());
ReadLineResult res = result.value();
- current_offset += static_cast<off_t>(res.raw_bytes_consumed);
if (res.eof)
return false; // EOF
@@ -279,13 +261,11 @@ template <typename EntryType> class FlatFileDatabase {
}
while (true) {
- last_line_start = current_offset;
auto result = read_line_growing(file, buffer);
if (!result.has_value())
return Error(result.error());
ReadLineResult res = result.value();
- current_offset += static_cast<off_t>(res.raw_bytes_consumed);
if (res.eof)
return false; // EOF
@@ -309,57 +289,12 @@ template <typename EntryType> class FlatFileDatabase {
// Searches for a record matching a given predicate. Returns true if the
// entry was found, false if it's missing, or an Error if lookup failed.
//
- // Per POSIX, ERANGE is reported only if the matched entry does not fit in the
- // caller's buffer; unrelated preceding records larger than buffer are
- // skipped.
- LIBC_INLINE ErrorOr<bool> lookup(const Matcher &matcher, EntryType *entry,
- cpp::span<char> buffer) {
- if (!entry)
- return Error(EINVAL);
-
- auto res = setdb();
- if (!res.has_value())
- return Error(res.error());
-
- ScopedDynamicBuffer scratch_buf;
- while (true) {
- auto next_res = getnext(entry, buffer);
- if (next_res.has_value()) {
- if (!next_res.value())
- return false; // EOF without match
- if (matcher(*entry))
- return true;
- continue;
- }
-
- if (next_res.error() != ERANGE)
- return Error(next_res.error());
-
- // The record at last_line_start exceeded buffer. Check whether it is
- // actually the target entry before reporting ERANGE. Use a stack-local
- // scratch_entry so we do not leave dangling pointers in the caller's
- // *entry when scratch_buf goes out of scope.
- auto seek_res = file->seek(last_line_start, SEEK_SET);
- if (!seek_res.has_value())
- return Error(seek_res.error());
- file->clearerr();
- current_offset = last_line_start;
-
- EntryType scratch_entry{};
- auto dyn_res = getnext(&scratch_entry, scratch_buf);
- if (!dyn_res.has_value())
- return Error(dyn_res.error());
- if (!dyn_res.value())
- return false;
- if (matcher(scratch_entry))
- return Error(ERANGE);
- }
- }
-
- // As above, but reads into a caller-owned buffer that grows to fit long
- // records instead of reporting ERANGE.
+ // When BufferType is cpp::span<char>, lookup never allocates and returns
+ // ERANGE if any record encountered exceeds buffer (matching glibc and
+ // FreeBSD). When BufferType is DynamicBuffer, the buffer grows as needed.
+ template <typename BufferType>
LIBC_INLINE ErrorOr<bool> lookup(const Matcher &matcher, EntryType *entry,
- DynamicBuffer &buffer) {
+ BufferType &buffer) {
if (!entry)
return Error(EINVAL);
diff --git a/libc/test/src/__support/pwd/flat_file_db_test.cpp b/libc/test/src/__support/pwd/flat_file_db_test.cpp
index 2bcb141fff8ff..9050dab80cb4b 100644
--- a/libc/test/src/__support/pwd/flat_file_db_test.cpp
+++ b/libc/test/src/__support/pwd/flat_file_db_test.cpp
@@ -245,20 +245,20 @@ TEST_F(LlvmLibcFlatFileDbTest, DynamicBufferReadsArbitrarilyLongLines) {
auto r1 = db.getnext(&entry, buffer);
ASSERT_TRUE(r1.has_value());
ASSERT_TRUE(r1.value());
- ASSERT_STREQ(entry.key, "key0");
- ASSERT_EQ(LIBC_NAMESPACE::internal::string_length(entry.val), VALUE_LENGTH);
+ EXPECT_STREQ(entry.key, "key0");
+ EXPECT_EQ(LIBC_NAMESPACE::internal::string_length(entry.val), VALUE_LENGTH);
auto r2 = db.getnext(&entry, buffer);
ASSERT_TRUE(r2.has_value());
ASSERT_TRUE(r2.value());
- ASSERT_STREQ(entry.key, "key1");
+ EXPECT_STREQ(entry.key, "key1");
auto r3 = db.getnext(&entry, buffer);
ASSERT_TRUE(r3.has_value());
- ASSERT_FALSE(r3.value());
+ EXPECT_FALSE(r3.value());
}
-TEST_F(LlvmLibcFlatFileDbTest, PrecedingLongRecordsSkippedDuringLookup) {
+TEST_F(LlvmLibcFlatFileDbTest, LookupOversizedPrecedingRecord) {
const char *content =
"huge_unrelated_key:012345678901234567890123456789012345\n"
"target:short\n";
@@ -267,41 +267,46 @@ TEST_F(LlvmLibcFlatFileDbTest, PrecedingLongRecordsSkippedDuringLookup) {
LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<SimpleTestEntry> db(
test_file.get_path());
- // 24 bytes is large enough for "target:short" (12 chars + '\0') but smaller
- // than "huge_unrelated_key:...". Fixed-buffer lookup must proceed past
- // unrelated long records without falsely returning ERANGE.
+ auto matcher = [](const SimpleTestEntry &e) {
+ return LIBC_NAMESPACE::cpp::string_view(e.key) == "target";
+ };
+
+ // Fixed-buffer lookup never allocates and returns ERANGE when any record
+ // encountered exceeds the caller's buffer (matching glibc and FreeBSD).
constexpr size_t SMALL_BUFFER_SIZE = 24;
- char buffer[SMALL_BUFFER_SIZE];
+ char small_buffer[SMALL_BUFFER_SIZE];
SimpleTestEntry entry;
-
- auto res = db.lookup(
- [](const SimpleTestEntry &e) {
- return LIBC_NAMESPACE::cpp::string_view(e.key) == "target";
- },
- &entry, buffer);
- ASSERT_TRUE(res.has_value());
- ASSERT_TRUE(res.value());
- ASSERT_STREQ(entry.key, "target");
- ASSERT_STREQ(entry.val, "short");
+ auto fixed_res = db.lookup(matcher, &entry, small_buffer);
+ ASSERT_FALSE(fixed_res.has_value());
+ EXPECT_EQ(fixed_res.error(), ERANGE);
+
+ // DynamicBuffer lookup grows the buffer across oversized preceding records
+ // and locates the target entry.
+ LIBC_NAMESPACE::pwd::ScopedDynamicBuffer dyn_buffer;
+ auto dyn_res = db.lookup(matcher, &entry, dyn_buffer);
+ ASSERT_TRUE(dyn_res.has_value());
+ ASSERT_TRUE(dyn_res.value());
+ EXPECT_STREQ(entry.key, "target");
+ EXPECT_STREQ(entry.val, "short");
}
TEST_F(LlvmLibcFlatFileDbTest, DynamicBufferReserveGrowRelease) {
LIBC_NAMESPACE::pwd::ScopedDynamicBuffer buffer;
- ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+ EXPECT_EQ(buffer.capacity(), static_cast<size_t>(0));
ASSERT_TRUE(buffer.grow());
size_t initial = buffer.capacity();
- ASSERT_GT(initial, static_cast<size_t>(0));
+ EXPECT_GT(initial, static_cast<size_t>(0));
ASSERT_TRUE(buffer.grow());
- ASSERT_EQ(buffer.capacity(), initial * 2);
+ EXPECT_EQ(buffer.capacity(), initial * 2);
buffer.release();
- ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+ EXPECT_EQ(buffer.capacity(), static_cast<size_t>(0));
buffer.release();
- ASSERT_EQ(buffer.capacity(), static_cast<size_t>(0));
+ EXPECT_EQ(buffer.capacity(), static_cast<size_t>(0));
constexpr size_t TARGET_RESERVE_CAPACITY = 1024;
ASSERT_TRUE(buffer.reserve(TARGET_RESERVE_CAPACITY));
- ASSERT_GE(buffer.capacity(), TARGET_RESERVE_CAPACITY);
+ EXPECT_GE(buffer.capacity(), TARGET_RESERVE_CAPACITY);
}
>From 7f5898081ec812c9fa8d5aeab1128cc1fb629e83 Mon Sep 17 00:00:00 2001
From: Jeff Bailey <jbailey at raspberryginger.com>
Date: Wed, 16 Sep 2026 18:54:03 +0100
Subject: [PATCH 4/4] [libc] Pass line and scratch spans to parse_line
Refactor the parse_line interface from (cpp::span<char> buffer,
size_t line_len, EntryType *entry) to (cpp::span<char> line,
cpp::span<char> scratch, EntryType *entry).
The previous signature implicitly encoded three distinct regions in
a single buffer and length pair: the record text, the null-terminated
string, and the trailing scratch space. Callers now slice the buffer
explicitly at the terminating null byte:
* line covers the record bytes including the terminating null byte.
* scratch covers the remainder of the buffer for auxiliary structures
(such as pointer arrays). Specialisations requiring scratch space
must return Error(ERANGE) if scratch is insufficient.
In FlatFileDatabase::getnext(DynamicBuffer &), insufficient scratch
storage returned as ERANGE triggers buffer growth and retry.
Add unit tests in flat_file_db_test verifying fixed-buffer ERANGE
reporting and DynamicBuffer growth when scratch storage is required.
Assisted-by: Automated tooling, human reviewed.
---
libc/src/__support/pwd/flat_file_db.h | 23 +++++--
libc/src/pwd/pwd_utils.cpp | 2 +-
libc/src/pwd/pwd_utils.h | 8 +--
.../src/__support/pwd/flat_file_db_test.cpp | 69 +++++++++++++++++--
4 files changed, 87 insertions(+), 15 deletions(-)
diff --git a/libc/src/__support/pwd/flat_file_db.h b/libc/src/__support/pwd/flat_file_db.h
index 1ffc1decbb1f2..319197872cd3b 100644
--- a/libc/src/__support/pwd/flat_file_db.h
+++ b/libc/src/__support/pwd/flat_file_db.h
@@ -41,10 +41,13 @@ struct ReadLineResult {
};
// Parses a record in place and fills entry.
-// If the buffer is too small for auxiliary structures (such as pointer arrays),
-// specializations must return Error(ERANGE) prior to modifying the buffer.
+// line is the record bytes including the terminating NUL (line.back() == '\0').
+// scratch is spare writable storage for auxiliary structures (such as pointer
+// arrays) and may be empty. Specialisations that require scratch storage must
+// return Error(ERANGE) prior to modifying the buffer if scratch is
+// insufficient.
template <typename EntryType>
-ErrorOr<void> parse_line(cpp::span<char> buffer, size_t line_len,
+ErrorOr<void> parse_line(cpp::span<char> line, cpp::span<char> scratch,
EntryType *entry);
// Generic flat colon-delimited database engine.
@@ -240,7 +243,11 @@ template <typename EntryType> class FlatFileDatabase {
if (res.truncated)
return Error(ERANGE);
- auto parse_res = parse_line<EntryType>(buffer, res.bytes_read, entry);
+ // Slicing at bytes_read + 1 includes the terminating null byte written
+ // by read_line; the remainder of the buffer serves as scratch space.
+ auto parse_res =
+ parse_line<EntryType>(buffer.first(res.bytes_read + 1),
+ buffer.subspan(res.bytes_read + 1), entry);
if (!parse_res.has_value())
return Error(parse_res.error());
return true;
@@ -274,8 +281,12 @@ template <typename EntryType> class FlatFileDatabase {
continue;
while (true) {
- auto parse_res =
- parse_line<EntryType>(buffer.span(), res.bytes_read, entry);
+ // Slicing at bytes_read + 1 includes the terminating null byte written
+ // by read_line_growing; the remainder of the buffer serves as scratch
+ // space.
+ auto parse_res = parse_line<EntryType>(
+ buffer.span().first(res.bytes_read + 1),
+ buffer.span().subspan(res.bytes_read + 1), entry);
if (parse_res.has_value())
return true;
if (parse_res.error() != ERANGE)
diff --git a/libc/src/pwd/pwd_utils.cpp b/libc/src/pwd/pwd_utils.cpp
index 55052c8c39232..79bdd1895b311 100644
--- a/libc/src/pwd/pwd_utils.cpp
+++ b/libc/src/pwd/pwd_utils.cpp
@@ -34,7 +34,7 @@ ErrorOr<struct passwd> parse_passwd_line(char *line) {
struct passwd pwd;
size_t len = internal::string_length(line);
- auto res = parse_line(cpp::span<char>(line, len + 1), len, &pwd);
+ auto res = parse_line(cpp::span<char>(line, len + 1), {}, &pwd);
if (!res.has_value())
return Error(res.error());
diff --git a/libc/src/pwd/pwd_utils.h b/libc/src/pwd/pwd_utils.h
index 36a98c7bc41d7..b3d16f49893b5 100644
--- a/libc/src/pwd/pwd_utils.h
+++ b/libc/src/pwd/pwd_utils.h
@@ -35,13 +35,13 @@ namespace pwd {
// Parses a colon-separated line in-place into a struct passwd.
template <>
-LIBC_INLINE ErrorOr<void> parse_line<struct passwd>(cpp::span<char> buffer,
- size_t line_len,
+LIBC_INLINE ErrorOr<void> parse_line<struct passwd>(cpp::span<char> line,
+ cpp::span<char> /*scratch*/,
struct passwd *pwd) {
- if (!pwd || line_len == 0 || line_len >= buffer.size())
+ if (!pwd || line.empty() || line.back() != '\0')
return Error(EINVAL);
- FieldTokenizer tokenizer(buffer.first(line_len + 1));
+ FieldTokenizer tokenizer(line);
auto name = tokenizer.next_field();
if (!name || name->empty() || name->front() == '\0')
diff --git a/libc/test/src/__support/pwd/flat_file_db_test.cpp b/libc/test/src/__support/pwd/flat_file_db_test.cpp
index 9050dab80cb4b..69f1370cb1421 100644
--- a/libc/test/src/__support/pwd/flat_file_db_test.cpp
+++ b/libc/test/src/__support/pwd/flat_file_db_test.cpp
@@ -32,6 +32,12 @@ struct SimpleTestEntry {
const char *val;
};
+struct ScratchRequiringEntry {
+ const char *key = nullptr;
+ const char *val = nullptr;
+ LIBC_NAMESPACE::cpp::span<char> extra_storage;
+};
+
class HermeticFile {
char path[256];
@@ -62,13 +68,38 @@ namespace LIBC_NAMESPACE_DECL {
namespace pwd {
template <>
-inline ErrorOr<void> parse_line<SimpleTestEntry>(cpp::span<char> buffer,
- size_t line_len,
+inline ErrorOr<void> parse_line<SimpleTestEntry>(cpp::span<char> line,
+ cpp::span<char> /*scratch*/,
SimpleTestEntry *entry) {
- if (!entry || line_len == 0 || line_len >= buffer.size())
+ if (!entry || line.empty() || line.back() != '\0')
+ return Error(EINVAL);
+
+ FieldTokenizer tokenizer(line);
+ auto k = tokenizer.next_field();
+ if (!k)
+ return Error(EINVAL);
+ entry->key = k->data();
+
+ auto v = tokenizer.next_field();
+ if (!v)
+ return Error(EINVAL);
+ entry->val = v->data();
+
+ return {};
+}
+
+template <>
+inline ErrorOr<void>
+parse_line<ScratchRequiringEntry>(cpp::span<char> line, cpp::span<char> scratch,
+ ScratchRequiringEntry *entry) {
+ if (!entry || line.empty() || line.back() != '\0')
return Error(EINVAL);
- FieldTokenizer tokenizer(buffer.first(line_len + 1));
+ constexpr size_t REQUIRED_SCRATCH = 2048;
+ if (scratch.size() < REQUIRED_SCRATCH)
+ return Error(ERANGE);
+
+ FieldTokenizer tokenizer(line);
auto k = tokenizer.next_field();
if (!k)
return Error(EINVAL);
@@ -79,6 +110,7 @@ inline ErrorOr<void> parse_line<SimpleTestEntry>(cpp::span<char> buffer,
return Error(EINVAL);
entry->val = v->data();
+ entry->extra_storage = scratch;
return {};
}
@@ -310,3 +342,32 @@ TEST_F(LlvmLibcFlatFileDbTest, DynamicBufferReserveGrowRelease) {
ASSERT_TRUE(buffer.reserve(TARGET_RESERVE_CAPACITY));
EXPECT_GE(buffer.capacity(), TARGET_RESERVE_CAPACITY);
}
+
+TEST_F(LlvmLibcFlatFileDbTest,
+ ScratchInsufficientReturnsErangeAndDynamicBufferGrows) {
+ const char *content = "user1:secret1\n";
+ HermeticFile test_file(libc_make_test_file_path("flat_db_scratch.test"),
+ content);
+
+ // Fixed buffer with insufficient scratch returns ERANGE.
+ constexpr size_t SMALL_BUFFER_SIZE = 64;
+ char small_buffer[SMALL_BUFFER_SIZE];
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<ScratchRequiringEntry> db_fixed(
+ test_file.get_path());
+ ScratchRequiringEntry fixed_entry;
+ auto fixed_res = db_fixed.getnext(&fixed_entry, small_buffer);
+ ASSERT_FALSE(fixed_res.has_value());
+ EXPECT_EQ(fixed_res.error(), ERANGE);
+
+ // DynamicBuffer grows until scratch requirement (2048 bytes) is satisfied.
+ LIBC_NAMESPACE::pwd::ScopedFlatFileDatabase<ScratchRequiringEntry> db_dyn(
+ test_file.get_path());
+ LIBC_NAMESPACE::pwd::ScopedDynamicBuffer dyn_buffer;
+ ScratchRequiringEntry dyn_entry;
+ auto dyn_res = db_dyn.getnext(&dyn_entry, dyn_buffer);
+ ASSERT_TRUE(dyn_res.has_value());
+ ASSERT_TRUE(dyn_res.value());
+ EXPECT_STREQ(dyn_entry.key, "user1");
+ EXPECT_STREQ(dyn_entry.val, "secret1");
+ EXPECT_GE(dyn_entry.extra_storage.size(), static_cast<size_t>(2048));
+}
More information about the libc-commits
mailing list