[libc-commits] [libc] 4c407e8 - [libc][regex] Add 256-bit ByteSet and '.' wildcard (#233561)
via libc-commits
libc-commits at lists.llvm.org
Mon Oct 19 23:52:43 PDT 2026
Author: Jeff Bailey
Date: 2026-10-20T06:52:37Z
New Revision: 4c407e843c975298b6892d50afd9e72cfb0cbe46
URL: https://github.com/llvm/llvm-project/commit/4c407e843c975298b6892d50afd9e72cfb0cbe46
DIFF: https://github.com/llvm/llvm-project/commit/4c407e843c975298b6892d50afd9e72cfb0cbe46.diff
LOG: [libc][regex] Add 256-bit ByteSet and '.' wildcard (#233561)
Representing wildcards and bracket expressions as a 256-way alternation
of Literal nodes would exhaust ExprPool and blow up derivative step
time. At the same time, storing a 32-byte cpp::bitset<256> inline inside
Expr would increase every AST node from 16 bytes to 40 bytes.
Add ExprKind::ByteSet backed by a deduplicated cpp::span<ByteSet>
side-table in ExprPool, keeping sizeof(Expr) at 16 bytes. Normalise
0-bit and 1-bit sets in ExprPool::make_byte_set to EmptySet and Literal
so single-byte character classes share the same canonical AST node as
plain literals, and wire the '.' wildcard in parse_ere and derivative.
Also simplify BucketProber and the ExprPool lazy-initialisation guards.
Against the external compliance suites in REG_NOSUB mode, this passes
21/456 tests (+1) in the Henry Spencer suite and 25/202 ERE tests (+3)
in Glenn Fowler's AT&T testregex basic.dat suite.
Assisted-by: Automated tooling, human reviewed.
Added:
Modified:
libc/src/__support/regex/CMakeLists.txt
libc/src/__support/regex/regex_ast.h
libc/src/__support/regex/regex_expr_pool.cpp
libc/src/__support/regex/regex_expr_pool.h
libc/src/__support/regex/regex_matcher.cpp
libc/src/__support/regex/regex_parser.cpp
libc/test/src/__support/regex/expr_test.cpp
libc/test/src/__support/regex/matcher_test.cpp
libc/test/src/regex/regex_basic_test.cpp
Removed:
################################################################################
diff --git a/libc/src/__support/regex/CMakeLists.txt b/libc/src/__support/regex/CMakeLists.txt
index 7b6ed6becca37..b4be2eb3a211b 100644
--- a/libc/src/__support/regex/CMakeLists.txt
+++ b/libc/src/__support/regex/CMakeLists.txt
@@ -6,11 +6,14 @@ add_object_library(
regex_expr_pool.h
regex_ast.h
DEPENDS
+ libc.hdr.limits_macros
libc.hdr.regex_macros
libc.hdr.stdint_proxy
libc.src.__support.alloc_checker
libc.src.__support.CPP.array
+ libc.src.__support.CPP.bitset
libc.src.__support.CPP.new
+ libc.src.__support.CPP.scope
libc.src.__support.CPP.span
libc.src.__support.CPP.utility
libc.src.__support.error_or
diff --git a/libc/src/__support/regex/regex_ast.h b/libc/src/__support/regex/regex_ast.h
index f7cf838a0ffba..c3e6b52716695 100644
--- a/libc/src/__support/regex/regex_ast.h
+++ b/libc/src/__support/regex/regex_ast.h
@@ -14,7 +14,9 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_REGEX_REGEX_AST_H
#define LLVM_LIBC_SRC___SUPPORT_REGEX_REGEX_AST_H
+#include "hdr/limits_macros.h"
#include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/bitset.h"
#include "src/__support/macros/config.h"
namespace LIBC_NAMESPACE_DECL {
@@ -26,6 +28,11 @@ inline constexpr ExprId INVALID_EXPR_ID = 0;
inline constexpr ExprId EMPTY_SET_ID = 1;
inline constexpr ExprId EMPTY_STR_ID = 2;
+/// One bit per possible byte value (2^CHAR_BIT = 256), allowing O(1)
+/// membership tests for wildcards and bracket expressions.
+inline constexpr size_t BYTE_SET_BITS = size_t(1) << CHAR_BIT;
+using ByteSet = cpp::bitset<BYTE_SET_BITS>;
+
/// Enumeration of Regular Expression AST node types.
enum class ExprKind : uint8_t {
/// Represents the empty set (matches nothing).
@@ -34,6 +41,8 @@ enum class ExprKind : uint8_t {
EmptyStr,
/// A literal character match.
Literal,
+ /// A 256-bit character class or wildcard match.
+ ByteSet,
/// Concatenation of two expressions (left followed by right).
Concat,
/// Alternation between two expressions (left or right).
@@ -54,11 +63,15 @@ struct Expr {
char ch = '\0';
/// Precomputed 32-bit structural hash for O(1) lookup and rehashing.
uint32_t hash = 0;
- /// Sub-expressions for Concat and Alt nodes.
- struct {
- ExprId left;
- ExprId right;
- } bin = {INVALID_EXPR_ID, INVALID_EXPR_ID};
+ union {
+ /// Sub-expressions for Concat and Alt nodes.
+ struct {
+ ExprId left;
+ ExprId right;
+ } bin = {INVALID_EXPR_ID, INVALID_EXPR_ID};
+ /// Index into ExprPool::byte_sets for ByteSet nodes.
+ uint32_t byte_set_id;
+ };
/// Default constructor creates an EmptySet node.
/// Public to allow array allocation in ExprPool::Block.
@@ -69,6 +82,9 @@ struct Expr {
constexpr Expr(ExprKind k, bool is_null) : kind(k), nullable(is_null) {}
/// Create a Literal node.
constexpr Expr(char c) : kind(ExprKind::Literal), ch(c) {}
+ /// Create a ByteSet node.
+ constexpr Expr(uint32_t bs_id)
+ : kind(ExprKind::ByteSet), byte_set_id(bs_id) {}
/// Create a binary node (Concat or Alt).
constexpr Expr(ExprKind k, bool is_null, ExprId l, ExprId r)
: kind(k), nullable(is_null), bin{l, r} {}
@@ -81,6 +97,7 @@ struct Expr {
return Expr(ExprKind::EmptyStr, true);
}
static constexpr Expr make_literal(char c) { return Expr(c); }
+ static constexpr Expr make_byte_set(uint32_t bs_id) { return Expr(bs_id); }
static constexpr Expr make_concat(ExprId l, ExprId r, bool is_null = false) {
return Expr(ExprKind::Concat, is_null, l, r);
}
@@ -98,11 +115,13 @@ struct Expr {
return true;
case ExprKind::Literal:
return ch == other.ch;
+ case ExprKind::ByteSet:
+ return byte_set_id == other.byte_set_id;
case ExprKind::Concat:
case ExprKind::Alt:
return bin.left == other.bin.left && bin.right == other.bin.right;
}
- return false;
+ __builtin_unreachable();
}
};
static_assert(sizeof(Expr) == 16, "Expr must remain 16 bytes");
diff --git a/libc/src/__support/regex/regex_expr_pool.cpp b/libc/src/__support/regex/regex_expr_pool.cpp
index 5c2978a3707cf..3f8c4266782a1 100644
--- a/libc/src/__support/regex/regex_expr_pool.cpp
+++ b/libc/src/__support/regex/regex_expr_pool.cpp
@@ -14,6 +14,7 @@
#include "src/__support/regex/regex_expr_pool.h"
#include "hdr/regex_macros.h"
#include "src/__support/CPP/new.h"
+#include "src/__support/CPP/scope.h"
#include "src/__support/CPP/utility.h"
#include "src/__support/alloc-checker.h"
#include "src/__support/hash.h"
@@ -38,6 +39,9 @@ uint32_t hash_expr(const Expr &e) {
case ExprKind::Literal:
hasher.update(&e.ch, sizeof(e.ch));
break;
+ case ExprKind::ByteSet:
+ hasher.update(&e.byte_set_id, sizeof(e.byte_set_id));
+ break;
case ExprKind::Concat:
case ExprKind::Alt:
hasher.update(&e.bin.left, sizeof(e.bin.left));
@@ -53,45 +57,48 @@ uint32_t hash_expr(const Expr &e) {
// Bounded linear-probing range over a power-of-two open-addressing table.
class BucketProber {
cpp::span<ExprId> table;
- size_t mask;
size_t start_idx;
+ constexpr size_t mask() const { return table.size() - 1; }
+
public:
+ struct Sentinel {};
+
struct Iterator {
cpp::span<ExprId> table;
- size_t mask;
size_t idx;
size_t step;
+ constexpr size_t mask() const { return table.size() - 1; }
+
ExprId &operator*() const { return table[idx]; }
Iterator &operator++() {
- idx = (idx + 1) & mask;
+ idx = (idx + 1) & mask();
++step;
return *this;
}
- constexpr bool operator!=(const Iterator &other) const {
- return step != other.step;
- }
+ constexpr bool operator!=(Sentinel) const { return step != table.size(); }
};
BucketProber(cpp::span<ExprId> t, uint32_t hash)
- : table(t), mask(t.size() - 1), start_idx(hash & mask) {
- LIBC_ASSERT(!t.empty() && (t.size() & mask) == 0);
+ : table(t), start_idx(hash & (t.size() - 1)) {
+ LIBC_ASSERT(!t.empty() && (t.size() & mask()) == 0);
}
- Iterator begin() const { return {table, mask, start_idx, 0}; }
- Iterator end() const { return {table, mask, start_idx, table.size()}; }
+ Iterator begin() const { return {table, start_idx, 0}; }
+ Sentinel end() const { return {}; }
};
} // namespace
// TODO: When cpp::unique_ptr and cpp::make_unique are added to
// src/__support/CPP/, replace the manual AllocChecker + new[]/delete[] calls in
-// ensure_initialized(), ~ExprPool(), allocate_block(), and grow_hashtable()
-// with cpp::make_unique<ExprId[]>(...) and cpp::make_unique<Block>(),
-// eliminating explicit zeroing via inline_memset and the destructor cleanup
-// loop.
+// ensure_initialized(), ~ExprPool(), allocate_block(), make_byte_set(), and
+// grow_hashtable() with cpp::make_unique<ExprId[]>(...) and
+// cpp::make_unique<Block>(), eliminating explicit zeroing via inline_memset and
+// the destructor cleanup loop.
ExprPool::~ExprPool() {
+ delete[] byte_sets.data();
delete[] hashtable.data();
for (Block *blk : active_blocks())
delete blk;
@@ -99,25 +106,23 @@ ExprPool::~ExprPool() {
}
bool ExprPool::ensure_initialized() {
- if (node_count >= EMPTY_STR_ID)
+ if (block_count > 0)
return true;
- if (hashtable.empty()) {
- AllocChecker ac;
- ExprId *raw_table = new (ac) ExprId[INITIAL_HASH_CAPACITY];
- if (!ac)
- return false;
- hashtable = cpp::span<ExprId>(raw_table, INITIAL_HASH_CAPACITY);
- inline_memset(hashtable.data(), 0, hashtable.size_bytes());
- }
- if (block_count == 0 && !allocate_block())
+ AllocChecker ac;
+ ExprId *raw_table = new (ac) ExprId[INITIAL_HASH_CAPACITY];
+ if (!ac)
return false;
-
- auto empty_set_id = intern(Expr::make_empty_set());
- auto empty_str_id = intern(Expr::make_empty_str());
- if (!empty_set_id.has_value() || !empty_str_id.has_value())
+ cpp::scope_exit table_guard([raw_table] { delete[] raw_table; });
+ if (!allocate_block())
return false;
- LIBC_ASSERT(*empty_set_id == EMPTY_SET_ID);
- LIBC_ASSERT(*empty_str_id == EMPTY_STR_ID);
+ table_guard.release();
+ hashtable = cpp::span<ExprId>(raw_table, INITIAL_HASH_CAPACITY);
+ inline_memset(hashtable.data(), 0, hashtable.size_bytes());
+
+ [[maybe_unused]] auto empty_set_id = intern(Expr::make_empty_set());
+ [[maybe_unused]] auto empty_str_id = intern(Expr::make_empty_str());
+ LIBC_ASSERT(empty_set_id.has_value() && *empty_set_id == EMPTY_SET_ID);
+ LIBC_ASSERT(empty_str_id.has_value() && *empty_str_id == EMPTY_STR_ID);
return true;
}
@@ -135,8 +140,8 @@ ExprId &ExprPool::find_bucket(cpp::span<ExprId> table, const Expr &e) const {
bool ExprPool::allocate_block() {
AllocChecker ac;
if (block_count == blocks.size()) {
- size_t new_cap =
- blocks.empty() ? INITIAL_BLOCKS_CAPACITY : blocks.size() * 2;
+ size_t new_cap = blocks.empty() ? INITIAL_BLOCKS_CAPACITY
+ : blocks.size() * GROWTH_FACTOR;
Block **new_blocks = new (ac) Block *[new_cap];
if (!ac)
return false;
@@ -157,7 +162,7 @@ bool ExprPool::allocate_block() {
}
bool ExprPool::grow_hashtable() {
- size_t new_cap = hashtable.size() * 2;
+ size_t new_cap = hashtable.size() * GROWTH_FACTOR;
AllocChecker ac;
ExprId *raw_table = new (ac) ExprId[new_cap];
if (!ac)
@@ -175,7 +180,7 @@ bool ExprPool::grow_hashtable() {
}
ErrorOr<ExprId> ExprPool::intern(Expr e) {
- if ((hashtable.empty() || block_count == 0) && !ensure_initialized())
+ if (!ensure_initialized())
return Error(REG_ESPACE);
// Probe for an existing node with identical content.
@@ -222,6 +227,51 @@ ErrorOr<ExprId> ExprPool::make_lit(char c) {
return intern(Expr::make_literal(c));
}
+ErrorOr<ExprId> ExprPool::make_byte_set(const ByteSet &bs) {
+ size_t count = 0;
+ size_t last_bit = 0;
+ for (size_t b = 0; b < BYTE_SET_BITS; ++b) {
+ if (bs.test(b)) {
+ ++count;
+ last_bit = b;
+ }
+ }
+ if (count == 0)
+ return empty_set();
+ if (count == 1)
+ return make_lit(static_cast<char>(last_bit));
+
+ uint32_t bs_id = 0;
+ for (const ByteSet &existing : active_byte_sets()) {
+ if (existing == bs)
+ return intern(Expr::make_byte_set(bs_id));
+ ++bs_id;
+ }
+
+ if (byte_set_count == byte_sets.size()) {
+ size_t new_cap = byte_sets.empty() ? INITIAL_BYTE_SETS_CAPACITY
+ : byte_sets.size() * GROWTH_FACTOR;
+ AllocChecker ac;
+ ByteSet *new_sets = new (ac) ByteSet[new_cap];
+ if (!ac)
+ return Error(REG_ESPACE);
+ ByteSet *dst = new_sets;
+ for (const ByteSet &existing : active_byte_sets()) {
+ *dst = existing;
+ ++dst;
+ }
+ delete[] byte_sets.data();
+ byte_sets = cpp::span<ByteSet>(new_sets, new_cap);
+ }
+
+ ErrorOr<ExprId> id = intern(Expr::make_byte_set(bs_id));
+ if (!id.has_value())
+ return id;
+ byte_sets[byte_set_count] = bs;
+ ++byte_set_count;
+ return id;
+}
+
ErrorOr<ExprId> ExprPool::make_concat(ExprId l, ExprId r) {
if (l == INVALID_EXPR_ID || r == INVALID_EXPR_ID)
return Error(REG_BADPAT);
diff --git a/libc/src/__support/regex/regex_expr_pool.h b/libc/src/__support/regex/regex_expr_pool.h
index ec377863d6a8c..145f92493bc82 100644
--- a/libc/src/__support/regex/regex_expr_pool.h
+++ b/libc/src/__support/regex/regex_expr_pool.h
@@ -47,10 +47,15 @@ class ExprPool {
/// Initial directory capacity for 8 blocks (64B), holding up to 256 nodes
/// before directory growth.
static constexpr size_t INITIAL_BLOCKS_CAPACITY = 8;
+ /// Initial capacity for 4 unique ByteSets (4 * 32B = 128B), allocated lazily
+ /// on the first multi-bit ByteSet so pure literal patterns allocate 0 bytes.
+ static constexpr size_t INITIAL_BYTE_SETS_CAPACITY = 4;
/// Initial power-of-two hash table size: 64 slots (256B), holding up to 44
/// unique nodes at 70% max load factor while keeping initial pool footprint
/// at 512B + 64B + 256B = 832B (< 1 KiB).
static constexpr size_t INITIAL_HASH_CAPACITY = 64;
+ /// Scaling factor used when growing dynamic pool arrays and hash tables.
+ static constexpr size_t GROWTH_FACTOR = 2;
/// Grow the hash table when occupancy reaches 70% to keep linear-probing
/// chains short.
static constexpr double MAX_LOAD_FACTOR = 0.7;
@@ -63,8 +68,8 @@ class ExprPool {
// TODO: Once cpp::unique_ptr and AllocChecker-backed cpp::make_unique (and/or
// an owning unique_span) are available in src/__support/CPP/, replace raw
// Block* and span-backed manual new[]/delete[] ownership here so that Block
- // lifecycles, directory growth, and hash-table growth are managed via RAII
- // and ~ExprPool() can be defaulted.
+ // lifecycles, directory growth, ByteSet growth, and hash-table growth are
+ // managed via RAII and ~ExprPool() can be defaulted.
/// Span of allocated Block* slots (capacity = blocks.size()).
cpp::span<Block *> blocks;
@@ -72,6 +77,10 @@ class ExprPool {
/// Total number of nodes allocated across all blocks.
size_t node_count = 0;
+ /// Lazily allocated array of deduplicated 256-bit character sets.
+ cpp::span<ByteSet> byte_sets;
+ size_t byte_set_count = 0;
+
/// Open-addressing hash table storing ExprIds (0 = empty bucket).
cpp::span<ExprId> hashtable;
@@ -79,6 +88,10 @@ class ExprPool {
return blocks.first(block_count);
}
+ cpp::span<const ByteSet> active_byte_sets() const {
+ return byte_sets.first(byte_set_count);
+ }
+
/// Maximum number of nodes the current hash table can hold before rehashing.
size_t max_hashtable_nodes() const {
return static_cast<size_t>(static_cast<double>(hashtable.size()) *
@@ -115,16 +128,25 @@ class ExprPool {
/// Resolves an ExprId to a reference.
const Expr &get(ExprId id) const { return slot_at(id); }
+ /// Resolves a ByteSet index to its 256-bit character set.
+ const ByteSet &get_byte_set(uint32_t id) const {
+ LIBC_ASSERT(id < byte_set_count);
+ return byte_sets[id];
+ }
+
/// O(1) nullability query for a valid ExprId.
bool is_nullable(ExprId id) const { return get(id).nullable; }
/// Returns the number of unique AST nodes currently interned in this pool.
size_t size() const { return node_count; }
+ /// Returns the number of unique multi-bit ByteSets interned in this pool.
+ size_t num_byte_sets() const { return byte_set_count; }
+
/// Returns the current heap bytes allocated by this pool.
size_t allocated_bytes() const {
return (block_count * sizeof(Block)) + blocks.size_bytes() +
- hashtable.size_bytes();
+ hashtable.size_bytes() + byte_sets.size_bytes();
}
// TODO: Use fluent interface (and_then, transform) for these factories once
@@ -136,6 +158,8 @@ class ExprPool {
ErrorOr<ExprId> empty_str();
/// Creates or returns an existing Literal node for the given character.
ErrorOr<ExprId> make_lit(char c);
+ /// Creates or returns an existing node for the given 256-bit character set.
+ ErrorOr<ExprId> make_byte_set(const ByteSet &bs);
/// Normalizing factory for Concatenation (L · R).
///
/// Applies algebraic simplifications before interning:
diff --git a/libc/src/__support/regex/regex_matcher.cpp b/libc/src/__support/regex/regex_matcher.cpp
index b39386f1ae664..13d0346645f2f 100644
--- a/libc/src/__support/regex/regex_matcher.cpp
+++ b/libc/src/__support/regex/regex_matcher.cpp
@@ -32,6 +32,10 @@ ErrorOr<ExprId> derivative(ExprId id, char c, ExprPool &pool) {
return EMPTY_SET_ID;
case ExprKind::Literal:
return (c == e.ch) ? EMPTY_STR_ID : EMPTY_SET_ID;
+ case ExprKind::ByteSet:
+ return pool.get_byte_set(e.byte_set_id).test(static_cast<uint8_t>(c))
+ ? EMPTY_STR_ID
+ : EMPTY_SET_ID;
case ExprKind::Concat: {
// Product Rule:
// D_c(L . R) = (D_c(L) . R) | (nullable(L) ? D_c(R) : EmptySet)
diff --git a/libc/src/__support/regex/regex_parser.cpp b/libc/src/__support/regex/regex_parser.cpp
index 2a3f3d2c73dae..0406e598bb577 100644
--- a/libc/src/__support/regex/regex_parser.cpp
+++ b/libc/src/__support/regex/regex_parser.cpp
@@ -61,7 +61,7 @@ constexpr bool is_unimplemented_ere_escape(char c) {
} // namespace
-ErrorOr<ExprId> parse_ere(cpp::string_view pattern, [[maybe_unused]] int cflags,
+ErrorOr<ExprId> parse_ere(cpp::string_view pattern, int cflags,
ExprPool &pool) {
ErrorOr<ExprId> cur = pool.empty_str();
if (!cur.has_value())
@@ -71,6 +71,7 @@ ErrorOr<ExprId> parse_ere(cpp::string_view pattern, [[maybe_unused]] int cflags,
while (!rem.empty()) {
char c = rem.front();
rem.remove_prefix(1);
+ ErrorOr<ExprId> atom = INVALID_EXPR_ID;
switch (c) {
case '\\': {
if (rem.empty())
@@ -79,11 +80,22 @@ ErrorOr<ExprId> parse_ere(cpp::string_view pattern, [[maybe_unused]] int cflags,
rem.remove_prefix(1);
if (is_unimplemented_ere_escape(c))
return Error(REG_BADPAT);
+ atom = pool.make_lit(c);
+ break;
+ }
+ case '.': {
+ ByteSet bs;
+ if (cflags & REG_NEWLINE) {
+ bs.set_range(0, '\n' - 1);
+ bs.set_range('\n' + 1, BYTE_SET_BITS - 1);
+ } else {
+ bs.flip();
+ }
+ atom = pool.make_byte_set(bs);
break;
}
// TODO: Wire ERE metacharacters as each operator is implemented in
// subsequent patches.
- case '.':
case '^':
case '$':
case '*':
@@ -96,13 +108,13 @@ ErrorOr<ExprId> parse_ere(cpp::string_view pattern, [[maybe_unused]] int cflags,
case '|':
return Error(REG_BADPAT);
default:
+ atom = pool.make_lit(c);
break;
}
- ErrorOr<ExprId> lit = pool.make_lit(c);
- if (!lit.has_value())
- return lit;
- cur = pool.make_concat(*cur, *lit);
+ if (!atom.has_value())
+ return atom;
+ cur = pool.make_concat(*cur, *atom);
if (!cur.has_value())
return cur;
}
diff --git a/libc/test/src/__support/regex/expr_test.cpp b/libc/test/src/__support/regex/expr_test.cpp
index 0490c08dbd461..40f64767b8a63 100644
--- a/libc/test/src/__support/regex/expr_test.cpp
+++ b/libc/test/src/__support/regex/expr_test.cpp
@@ -15,6 +15,8 @@
#include "src/__support/regex/regex_expr_pool.h"
#include "test/UnitTest/Test.h"
+using LIBC_NAMESPACE::regex::ByteSet;
+using LIBC_NAMESPACE::regex::EMPTY_SET_ID;
using LIBC_NAMESPACE::regex::Expr;
using LIBC_NAMESPACE::regex::ExprId;
using LIBC_NAMESPACE::regex::ExprKind;
@@ -30,48 +32,83 @@ TEST(LlvmLibcRegexExprTest, Interning) {
auto lit_a_2 = pool.make_lit('a');
ASSERT_TRUE(lit_a_2.has_value());
// Hash-consing: same literal should return the same ExprId.
- EXPECT_EQ(lit_a.value(), lit_a_2.value());
+ EXPECT_EQ(*lit_a, *lit_a_2);
auto lit_b = pool.make_lit('b');
ASSERT_TRUE(lit_b.has_value());
- EXPECT_NE(lit_a.value(), lit_b.value());
+ EXPECT_NE(*lit_a, *lit_b);
auto empty_set_1 = pool.empty_set();
auto empty_set_2 = pool.empty_set();
- EXPECT_EQ(empty_set_1.value(), empty_set_2.value());
- EXPECT_FALSE(pool.is_nullable(empty_set_1.value()));
+ ASSERT_TRUE(empty_set_1.has_value());
+ ASSERT_TRUE(empty_set_2.has_value());
+ EXPECT_EQ(*empty_set_1, *empty_set_2);
+ EXPECT_FALSE(pool.is_nullable(*empty_set_1));
auto empty_str_1 = pool.empty_str();
auto empty_str_2 = pool.empty_str();
- EXPECT_EQ(empty_str_1.value(), empty_str_2.value());
- EXPECT_TRUE(pool.is_nullable(empty_str_1.value()));
+ ASSERT_TRUE(empty_str_1.has_value());
+ ASSERT_TRUE(empty_str_2.has_value());
+ EXPECT_EQ(*empty_str_1, *empty_str_2);
+ EXPECT_TRUE(pool.is_nullable(*empty_str_1));
EXPECT_LE(pool.allocated_bytes(), size_t(1024));
}
TEST(LlvmLibcRegexExprTest, RecursiveInterning) {
ExprPool pool;
- auto lit_a = pool.make_lit('a').value();
- auto lit_b = pool.make_lit('b').value();
- auto lit_c = pool.make_lit('c').value();
+ const ExprId lit_a = *pool.make_lit('a');
+ const ExprId lit_b = *pool.make_lit('b');
+ const ExprId lit_c = *pool.make_lit('c');
// Create (a · b) | c
- auto concat1 = pool.make_concat(lit_a, lit_b).value();
- auto alt1 = pool.make_alt(concat1, lit_c).value();
+ const ExprId concat1 = *pool.make_concat(lit_a, lit_b);
+ const ExprId alt1 = *pool.make_alt(concat1, lit_c);
// Create (a · b) | c again
- auto concat2 = pool.make_concat(lit_a, lit_b).value();
- auto alt2 = pool.make_alt(concat2, lit_c).value();
+ const ExprId concat2 = *pool.make_concat(lit_a, lit_b);
+ const ExprId alt2 = *pool.make_alt(concat2, lit_c);
EXPECT_EQ(alt1, alt2);
EXPECT_EQ(concat1, concat2);
}
+TEST(LlvmLibcRegexExprTest, ByteSetNormalizationAndDeduplication) {
+ ExprPool pool;
+ // 0-bit and 1-bit ByteSets simplify to EmptySet and Literal without
+ // allocating a ByteSet slot.
+ ByteSet bs;
+ EXPECT_EQ(*pool.make_byte_set(bs), EMPTY_SET_ID);
+ bs.set(static_cast<uint8_t>('x'));
+ EXPECT_EQ(*pool.make_byte_set(bs), *pool.make_lit('x'));
+ EXPECT_EQ(pool.num_byte_sets(), size_t(0));
+
+ // Multi-bit ByteSet allocates and deduplicates across identical sets.
+ bs.set(static_cast<uint8_t>('a'));
+ const ExprId bs1 = *pool.make_byte_set(bs);
+ const ExprId bs2 = *pool.make_byte_set(bs);
+ EXPECT_EQ(bs1, bs2);
+ EXPECT_EQ(pool.num_byte_sets(), size_t(1));
+ EXPECT_TRUE(pool.get_byte_set(0) == bs);
+ EXPECT_LE(pool.allocated_bytes(), size_t(1024));
+
+ // Grow past INITIAL_BYTE_SETS_CAPACITY (4).
+ for (uint8_t i = 0; i < 5; ++i) {
+ ByteSet multi;
+ multi.set(i);
+ multi.set(static_cast<uint8_t>(i + 10));
+ ASSERT_TRUE(pool.make_byte_set(multi).has_value());
+ }
+ EXPECT_EQ(pool.num_byte_sets(), size_t(6));
+ EXPECT_TRUE(Expr::make_byte_set(0) == Expr::make_byte_set(0));
+ EXPECT_FALSE(Expr::make_byte_set(0) == Expr::make_byte_set(1));
+}
+
TEST(LlvmLibcRegexExprTest, AlgebraicIdentitiesConcat) {
ExprPool pool;
- auto lit_a = *pool.make_lit('a');
- auto lit_b = *pool.make_lit('b');
- auto empty_set = *pool.empty_set();
- auto empty_str = *pool.empty_str();
+ const ExprId lit_a = *pool.make_lit('a');
+ const ExprId lit_b = *pool.make_lit('b');
+ const ExprId empty_set = *pool.empty_set();
+ const ExprId empty_str = *pool.empty_str();
// Invalid IDs return REG_BADPAT.
auto bad_left = pool.make_concat(INVALID_EXPR_ID, lit_a);
@@ -90,8 +127,8 @@ TEST(LlvmLibcRegexExprTest, AlgebraicIdentitiesConcat) {
EXPECT_EQ(*pool.make_concat(lit_a, empty_str), lit_a);
// Nullability of (L · R) requires both L and R to be nullable.
- auto opt_a = *pool.make_alt(lit_a, empty_str);
- auto opt_b = *pool.make_alt(lit_b, empty_str);
+ const ExprId opt_a = *pool.make_alt(lit_a, empty_str);
+ const ExprId opt_b = *pool.make_alt(lit_b, empty_str);
EXPECT_FALSE(pool.is_nullable(*pool.make_concat(lit_a, opt_b)));
EXPECT_FALSE(pool.is_nullable(*pool.make_concat(opt_a, lit_b)));
EXPECT_TRUE(pool.is_nullable(*pool.make_concat(opt_a, opt_b)));
@@ -108,10 +145,10 @@ TEST(LlvmLibcRegexExprTest, AlgebraicIdentitiesConcat) {
TEST(LlvmLibcRegexExprTest, AlgebraicIdentitiesAlt) {
ExprPool pool;
- auto lit_a = *pool.make_lit('a');
- auto lit_b = *pool.make_lit('b');
- auto empty_set = *pool.empty_set();
- auto empty_str = *pool.empty_str();
+ const ExprId lit_a = *pool.make_lit('a');
+ const ExprId lit_b = *pool.make_lit('b');
+ const ExprId empty_set = *pool.empty_set();
+ const ExprId empty_str = *pool.empty_str();
// Invalid IDs return REG_BADPAT.
auto bad_left = pool.make_alt(INVALID_EXPR_ID, lit_a);
@@ -133,17 +170,17 @@ TEST(LlvmLibcRegexExprTest, AlgebraicIdentitiesAlt) {
// Nullability of (L | R) when L is non-nullable and R (with higher ExprId) is
// nullable.
- auto opt_b = *pool.make_alt(lit_b, empty_str);
- auto opt_bb = *pool.make_concat(opt_b, opt_b);
+ const ExprId opt_b = *pool.make_alt(lit_b, empty_str);
+ const ExprId opt_bb = *pool.make_concat(opt_b, opt_b);
EXPECT_TRUE(pool.is_nullable(*pool.make_alt(lit_a, opt_bb)));
}
TEST(LlvmLibcRegexExprTest, HermeticHundredPools) {
for (size_t i = 0; i < 100; ++i) {
ExprPool pool;
- ExprId a = pool.make_lit('a').value();
- ExprId b = pool.make_lit('b').value();
- ExprId ab = pool.make_concat(a, b).value();
+ const ExprId a = *pool.make_lit('a');
+ const ExprId b = *pool.make_lit('b');
+ const ExprId ab = *pool.make_concat(a, b);
EXPECT_NE(ab, INVALID_EXPR_ID);
EXPECT_LE(pool.allocated_bytes(), size_t(1024));
}
@@ -153,16 +190,25 @@ TEST(LlvmLibcRegexExprTest, MemoryLimits) {
ExprPool pool;
// Verify that the pool correctly enforces MAX_NODE_LIMIT (10000 nodes).
// EmptySet (1), EmptyStr (2), and 'a' (3) occupy the first 3 slots.
- auto lit_a = pool.make_lit('a').value();
- auto current = lit_a;
+ const ExprId lit_a = *pool.make_lit('a');
+ ExprId current = lit_a;
for (size_t i = 0; i < 9997; ++i) {
auto next = pool.make_concat(lit_a, current);
ASSERT_TRUE(next.has_value());
- current = next.value();
+ current = *next;
}
// Next one should fail.
auto fail = pool.make_concat(lit_a, current);
ASSERT_FALSE(fail.has_value());
EXPECT_EQ(fail.error(), REG_ESPACE);
+
+ // A new multi-bit ByteSet at MAX_NODE_LIMIT fails without incrementing
+ // byte_set_count.
+ ByteSet bs;
+ bs.flip();
+ auto bs_fail = pool.make_byte_set(bs);
+ ASSERT_FALSE(bs_fail.has_value());
+ EXPECT_EQ(bs_fail.error(), REG_ESPACE);
+ EXPECT_EQ(pool.num_byte_sets(), size_t(0));
}
diff --git a/libc/test/src/__support/regex/matcher_test.cpp b/libc/test/src/__support/regex/matcher_test.cpp
index 53f178a0fc4c3..028d9cd8328b9 100644
--- a/libc/test/src/__support/regex/matcher_test.cpp
+++ b/libc/test/src/__support/regex/matcher_test.cpp
@@ -103,6 +103,42 @@ TEST(LlvmLibcRegexMatcherTest, OverlappingPrefixMatch) {
EXPECT_EQ(match(pool, *pat, "abababab"), MatchResult::NoMatch);
}
+TEST(LlvmLibcRegexMatcherTest, WildcardMatchingAndNewlineFlag) {
+ ExprPool pool;
+ ErrorOr<ExprId> pat = parse_ere("a.c", REG_EXTENDED, pool);
+ ASSERT_TRUE(pat.has_value());
+ EXPECT_EQ(pool.num_byte_sets(), size_t(1));
+ EXPECT_LE(pool.allocated_bytes(), size_t(1024));
+
+ EXPECT_EQ(match(pool, *pat, "xx abc yy"), MatchResult::Match);
+ EXPECT_EQ(match(pool, *pat, "a\nc"), MatchResult::Match);
+ EXPECT_EQ(match(pool, *pat,
+ "a\xff"
+ "c"),
+ MatchResult::Match);
+ EXPECT_EQ(match(pool, *pat, "ac"), MatchResult::NoMatch);
+ EXPECT_EQ(match(pool, *pat, "abbc"), MatchResult::NoMatch);
+
+ const size_t warm_nodes = pool.size();
+ const size_t warm_bytes = pool.allocated_bytes();
+ EXPECT_EQ(match(pool, *pat, "xx abc yy"), MatchResult::Match);
+ EXPECT_EQ(pool.size(), warm_nodes);
+ EXPECT_EQ(pool.allocated_bytes(), warm_bytes);
+
+ // A second '.' wildcard in the same mode shares the existing ByteSet slot.
+ ErrorOr<ExprId> pat2 = parse_ere("a.b.c", REG_EXTENDED, pool);
+ ASSERT_TRUE(pat2.has_value());
+ EXPECT_EQ(pool.num_byte_sets(), size_t(1));
+ EXPECT_EQ(match(pool, *pat2, "axbyc"), MatchResult::Match);
+
+ // Under REG_NEWLINE, '.' matches any byte except '\n'.
+ ErrorOr<ExprId> nl_pat = parse_ere("a.c", REG_EXTENDED | REG_NEWLINE, pool);
+ ASSERT_TRUE(nl_pat.has_value());
+ EXPECT_EQ(pool.num_byte_sets(), size_t(2));
+ EXPECT_EQ(match(pool, *nl_pat, "abc"), MatchResult::Match);
+ EXPECT_EQ(match(pool, *nl_pat, "a\nc"), MatchResult::NoMatch);
+}
+
TEST(LlvmLibcRegexMatcherTest, ParseEreEscapesAndErrors) {
ExprPool pool;
ErrorOr<ExprId> escaped_dot = parse_ere(R"(a\.b)", REG_EXTENDED, pool);
@@ -122,11 +158,11 @@ TEST(LlvmLibcRegexMatcherTest, ParseEreEscapesAndErrors) {
EXPECT_EQ(trailing_escape.error(), REG_EESCAPE);
const string_view bad_patterns[] = {
- ".", "^", "$", "*", "+", "?", "(",
- ")", "[", "{", "|", R"(\1)", R"(\2)", R"(\3)",
- R"(\4)", R"(\5)", R"(\6)", R"(\7)", R"(\8)", R"(\9)", R"(\b)",
- R"(\B)", R"(\<)", R"(\>)", R"(\A)", R"(\Z)", R"(\`)", R"(\')",
- R"(\w)", R"(\W)", R"(\s)", R"(\S)", R"(\d)", R"(\D)"};
+ "^", "$", "*", "+", "?", "(", ")",
+ "[", "{", "|", R"(\1)", R"(\2)", R"(\3)", R"(\4)",
+ R"(\5)", R"(\6)", R"(\7)", R"(\8)", R"(\9)", R"(\b)", R"(\B)",
+ R"(\<)", R"(\>)", R"(\A)", R"(\Z)", R"(\`)", R"(\')", R"(\w)",
+ R"(\W)", R"(\s)", R"(\S)", R"(\d)", R"(\D)"};
for (string_view bad : bad_patterns) {
ErrorOr<ExprId> err = parse_ere(bad, REG_EXTENDED, pool);
ASSERT_FALSE(err.has_value());
@@ -168,4 +204,12 @@ TEST(LlvmLibcRegexMatcherTest, PoolExhaustionPropagation) {
EXPECT_EQ(match(pool, need_bc, "a"), MatchResult::OutOfMemory);
EXPECT_EQ(match(pool, ab, "a"), MatchResult::OutOfMemory);
+
+ ErrorOr<ExprId> dot_exhausted = parse_ere(".", REG_EXTENDED, pool);
+ ASSERT_FALSE(dot_exhausted.has_value());
+ EXPECT_EQ(dot_exhausted.error(), REG_ESPACE);
+
+ ErrorOr<ExprId> concat_exhausted = parse_ere("ba", REG_EXTENDED, pool);
+ ASSERT_FALSE(concat_exhausted.has_value());
+ EXPECT_EQ(concat_exhausted.error(), REG_ESPACE);
}
diff --git a/libc/test/src/regex/regex_basic_test.cpp b/libc/test/src/regex/regex_basic_test.cpp
index 10aea4bb5d42b..4b0614afcbca6 100644
--- a/libc/test/src/regex/regex_basic_test.cpp
+++ b/libc/test/src/regex/regex_basic_test.cpp
@@ -103,18 +103,29 @@ TEST(LlvmLibcRegexTest, PoolExhaustionReturnsEspace) {
EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "a", 0, nullptr, 0), REG_ESPACE);
LIBC_NAMESPACE::regfree(&preg);
- // One additional repeated byte (9999 'a's) hits MAX_NODE_LIMIT in
- // make_concat; a novel byte ('b' at index 9998) hits MAX_NODE_LIMIT in
- // make_lit. Calling regfree afterwards must be a safe no-op in both cases.
+ // One additional byte (9999 'a's) hits MAX_NODE_LIMIT during regcomp;
+ // calling regfree afterwards must be a safe no-op.
pattern[9998] = 'a';
pattern[9999] = '\0';
EXPECT_EQ(LIBC_NAMESPACE::regcomp(&preg, pattern, REG_EXTENDED | REG_NOSUB),
REG_ESPACE);
LIBC_NAMESPACE::regfree(&preg);
+}
- pattern[9998] = 'b';
- EXPECT_EQ(LIBC_NAMESPACE::regcomp(&preg, pattern, REG_EXTENDED | REG_NOSUB),
- REG_ESPACE);
+TEST(LlvmLibcRegexTest, WildcardDot) {
+ regex_t preg;
+ ASSERT_EQ(LIBC_NAMESPACE::regcomp(&preg, "a.c", REG_EXTENDED | REG_NOSUB), 0);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "abc", 0, nullptr, 0), 0);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "axc", 0, nullptr, 0), 0);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "ac", 0, nullptr, 0), REG_NOMATCH);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "abbc", 0, nullptr, 0), REG_NOMATCH);
+ LIBC_NAMESPACE::regfree(&preg);
+
+ ASSERT_EQ(
+ LIBC_NAMESPACE::regcomp(&preg, R"(a\.b.c)", REG_EXTENDED | REG_NOSUB), 0);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "a.bXc", 0, nullptr, 0), 0);
+ EXPECT_EQ(LIBC_NAMESPACE::regexec(&preg, "aXbXc", 0, nullptr, 0),
+ REG_NOMATCH);
LIBC_NAMESPACE::regfree(&preg);
}
@@ -148,7 +159,7 @@ TEST(LlvmLibcRegexTest, UnimplementedSyntaxAndFlags) {
// Unimplemented ERE metacharacters and special escapes return REG_BADPAT.
const char *const bad_patterns[] = {
- "a.b", "^a", "a$", "a*", "a+", "a?", "(a)", "[a]",
+ "^a", "a$", "a*", "a+", "a?", "(a)", "[a]",
"a{1}", "a|b", R"(\1)", R"(\b)", R"(\w)", R"(\s)", R"(\d)"};
for (const char *pat : bad_patterns) {
EXPECT_EQ(LIBC_NAMESPACE::regcomp(&preg, pat, REG_EXTENDED), REG_BADPAT);
More information about the libc-commits
mailing list