[clang] [Clang][Lexer] Reopen "Detect SSE4.2 availability at runtime in fastParseASCIIIdentifier" (PR #175452)
Thibault Monnier via cfe-commits
cfe-commits at lists.llvm.org
Sun Jan 11 12:07:16 PST 2026
https://github.com/Thibault-Monnier updated https://github.com/llvm/llvm-project/pull/175452
>From 4fc9a07698e1a4627a050ba6fa9df3f1f8725451 Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Thu, 11 Dec 2025 22:02:35 +0100
Subject: [PATCH 1/7] Detect sse4.2 availability at runtime to use it on modern
processors
---
clang/lib/Lex/Lexer.cpp | 35 ++++++++++++++++++++++++++---------
1 file changed, 26 insertions(+), 9 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index b282a600c0e56..3b8fa0b9b7f36 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -46,9 +46,7 @@
#include <string>
#include <tuple>
-#ifdef __SSE4_2__
#include <nmmintrin.h>
-#endif
using namespace clang;
@@ -1921,9 +1919,17 @@ bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
}
static const char *
-fastParseASCIIIdentifier(const char *CurPtr,
- [[maybe_unused]] const char *BufferEnd) {
-#ifdef __SSE4_2__
+fastParseASCIIIdentifierScalar(const char *CurPtr,
+ [[maybe_unused]] const char *BufferEnd) {
+ unsigned char C = *CurPtr;
+ while (isAsciiIdentifierContinue(C))
+ C = *++CurPtr;
+ return CurPtr;
+}
+
+__attribute__((target("sse4.2"))) static const char *
+fastParseASCIIIdentifierSSE42(const char *CurPtr,
+ [[maybe_unused]] const char *BufferEnd) {
alignas(16) static constexpr char AsciiIdentifierRange[16] = {
'_', '_', 'A', 'Z', 'a', 'z', '0', '9',
};
@@ -1943,12 +1949,23 @@ fastParseASCIIIdentifier(const char *CurPtr,
continue;
return CurPtr;
}
+
+ return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
+}
+
+static bool supportsSSE42() {
+ static bool SupportsSSE42 = __builtin_cpu_supports("sse4.2");
+ return SupportsSSE42;
+}
+
+static const char *fastParseASCIIIdentifier(const char *CurPtr,
+ const char *BufferEnd) {
+#ifndef __SSE4_2__
+ if (LLVM_UNLIKELY(!supportsSSE42()))
+ return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
#endif
- unsigned char C = *CurPtr;
- while (isAsciiIdentifierContinue(C))
- C = *++CurPtr;
- return CurPtr;
+ return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
}
bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
>From ce3bf515e7a60bd58ff5871352979999f5864b4b Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Thu, 11 Dec 2025 23:15:40 +0100
Subject: [PATCH 2/7] Only on x86
---
clang/lib/Lex/Lexer.cpp | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 3b8fa0b9b7f36..c195237dae1f4 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -46,7 +46,9 @@
#include <string>
#include <tuple>
+#if defined(__i386__) || defined(__x86_64__)
#include <nmmintrin.h>
+#endif
using namespace clang;
@@ -1927,6 +1929,8 @@ fastParseASCIIIdentifierScalar(const char *CurPtr,
return CurPtr;
}
+#if defined(__i386__) || defined(__x86_64__)
+
__attribute__((target("sse4.2"))) static const char *
fastParseASCIIIdentifierSSE42(const char *CurPtr,
[[maybe_unused]] const char *BufferEnd) {
@@ -1958,14 +1962,22 @@ static bool supportsSSE42() {
return SupportsSSE42;
}
+#endif
+
static const char *fastParseASCIIIdentifier(const char *CurPtr,
const char *BufferEnd) {
+#if !defined(__i386__) && !defined(__x86_64__)
+ return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
+#else
+
#ifndef __SSE4_2__
if (LLVM_UNLIKELY(!supportsSSE42()))
return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
#endif
return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
+
+#endif
}
bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
>From 2109fdd371822ec77f870c5edbbdfccaaa7615be Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Sun, 14 Dec 2025 11:32:30 +0100
Subject: [PATCH 3/7] Not on windows
---
clang/lib/Lex/Lexer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index c195237dae1f4..86cfb47ca84d5 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -36,6 +36,7 @@
#include "llvm/Support/NativeFormatting.h"
#include "llvm/Support/Unicode.h"
#include "llvm/Support/UnicodeCharRanges.h"
+
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -1929,7 +1930,7 @@ fastParseASCIIIdentifierScalar(const char *CurPtr,
return CurPtr;
}
-#if defined(__i386__) || defined(__x86_64__)
+#if defined(__i386__) || defined(__x86_64__) && !defined(_WIN32)
__attribute__((target("sse4.2"))) static const char *
fastParseASCIIIdentifierSSE42(const char *CurPtr,
@@ -1966,7 +1967,7 @@ static bool supportsSSE42() {
static const char *fastParseASCIIIdentifier(const char *CurPtr,
const char *BufferEnd) {
-#if !defined(__i386__) && !defined(__x86_64__)
+#if !defined(__i386__) && !defined(__x86_64__) || defined(_WIN32)
return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
#else
>From d5485438edd460892bf210916827e0d92fc24065 Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Sun, 14 Dec 2025 14:37:43 +0100
Subject: [PATCH 4/7] Address comments
---
clang/lib/Lex/Lexer.cpp | 26 ++++++++++----------------
1 file changed, 10 insertions(+), 16 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 86cfb47ca84d5..470579df233d1 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1930,7 +1930,12 @@ fastParseASCIIIdentifierScalar(const char *CurPtr,
return CurPtr;
}
-#if defined(__i386__) || defined(__x86_64__) && !defined(_WIN32)
+// Fast path for lexing ASCII identifiers using SSE4.2 instructions.
+// Only enabled on x86/x86_64 when building with a compiler that supports
+// the 'target' attribute, which is used for runtime dispatch. Otherwise, we
+// fall back to the scalar implementation.
+#if (defined(__i386__) || defined(__x86_64__)) && defined(__has_attribute) && \
+ __has_attribute(target)
__attribute__((target("sse4.2"))) static const char *
fastParseASCIIIdentifierSSE42(const char *CurPtr,
@@ -1958,27 +1963,16 @@ fastParseASCIIIdentifierSSE42(const char *CurPtr,
return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
}
-static bool supportsSSE42() {
- static bool SupportsSSE42 = __builtin_cpu_supports("sse4.2");
- return SupportsSSE42;
+__attribute__((target("sse4.2"))) static const char *
+fastParseASCIIIdentifier(const char *CurPtr, const char *BufferEnd) {
+ return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
}
+__attribute__((target("default")))
#endif
-
static const char *fastParseASCIIIdentifier(const char *CurPtr,
const char *BufferEnd) {
-#if !defined(__i386__) && !defined(__x86_64__) || defined(_WIN32)
return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
-#else
-
-#ifndef __SSE4_2__
- if (LLVM_UNLIKELY(!supportsSSE42()))
- return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
-#endif
-
- return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
-
-#endif
}
bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
>From 82cf41e460d2fa1105e9abbf925837fa9e9c7b45 Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Sun, 14 Dec 2025 19:32:29 +0100
Subject: [PATCH 5/7] Not on MSVC
---
clang/lib/Lex/Lexer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 470579df233d1..58cd9348d3027 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1935,7 +1935,7 @@ fastParseASCIIIdentifierScalar(const char *CurPtr,
// the 'target' attribute, which is used for runtime dispatch. Otherwise, we
// fall back to the scalar implementation.
#if (defined(__i386__) || defined(__x86_64__)) && defined(__has_attribute) && \
- __has_attribute(target)
+ __has_attribute(target) && !defined(_MSC_VER)
__attribute__((target("sse4.2"))) static const char *
fastParseASCIIIdentifierSSE42(const char *CurPtr,
>From d7896bf4cd67116d8a3f59c74b51fce1c0f46951 Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Sun, 28 Dec 2025 17:48:49 +0100
Subject: [PATCH 6/7] Clean up
---
clang/lib/Lex/Lexer.cpp | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 58cd9348d3027..ba950dc350edb 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1921,9 +1921,7 @@ bool Lexer::LexUnicodeIdentifierStart(Token &Result, uint32_t C,
return true;
}
-static const char *
-fastParseASCIIIdentifierScalar(const char *CurPtr,
- [[maybe_unused]] const char *BufferEnd) {
+static const char *fastParseASCIIIdentifierScalar(const char *CurPtr) {
unsigned char C = *CurPtr;
while (isAsciiIdentifierContinue(C))
C = *++CurPtr;
@@ -1936,10 +1934,8 @@ fastParseASCIIIdentifierScalar(const char *CurPtr,
// fall back to the scalar implementation.
#if (defined(__i386__) || defined(__x86_64__)) && defined(__has_attribute) && \
__has_attribute(target) && !defined(_MSC_VER)
-
__attribute__((target("sse4.2"))) static const char *
-fastParseASCIIIdentifierSSE42(const char *CurPtr,
- [[maybe_unused]] const char *BufferEnd) {
+fastParseASCIIIdentifierSSE42(const char *CurPtr, const char *BufferEnd) {
alignas(16) static constexpr char AsciiIdentifierRange[16] = {
'_', '_', 'A', 'Z', 'a', 'z', '0', '9',
};
@@ -1960,7 +1956,7 @@ fastParseASCIIIdentifierSSE42(const char *CurPtr,
return CurPtr;
}
- return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
+ return fastParseASCIIIdentifierScalar(CurPtr);
}
__attribute__((target("sse4.2"))) static const char *
@@ -1972,7 +1968,7 @@ __attribute__((target("default")))
#endif
static const char *fastParseASCIIIdentifier(const char *CurPtr,
const char *BufferEnd) {
- return fastParseASCIIIdentifierScalar(CurPtr, BufferEnd);
+ return fastParseASCIIIdentifierScalar(CurPtr);
}
bool Lexer::LexIdentifierContinue(Token &Result, const char *CurPtr) {
>From 0ba35cb2d0e132f129c50196c1668a64592b5eaf Mon Sep 17 00:00:00 2001
From: Thibault-Monnier <thibaultmonni at gmail.com>
Date: Sun, 11 Jan 2026 21:05:05 +0100
Subject: [PATCH 7/7] Not on windows + fix unused parameter warning
---
clang/lib/Lex/Lexer.cpp | 16 +++-------------
1 file changed, 3 insertions(+), 13 deletions(-)
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index ba950dc350edb..d9eafc3e456dc 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -1933,7 +1933,7 @@ static const char *fastParseASCIIIdentifierScalar(const char *CurPtr) {
// the 'target' attribute, which is used for runtime dispatch. Otherwise, we
// fall back to the scalar implementation.
#if (defined(__i386__) || defined(__x86_64__)) && defined(__has_attribute) && \
- __has_attribute(target) && !defined(_MSC_VER)
+ __has_attribute(target) && !defined(_WIN32)
__attribute__((target("sse4.2"))) static const char *
fastParseASCIIIdentifierSSE42(const char *CurPtr, const char *BufferEnd) {
alignas(16) static constexpr char AsciiIdentifierRange[16] = {
@@ -1955,19 +1955,9 @@ fastParseASCIIIdentifierSSE42(const char *CurPtr, const char *BufferEnd) {
continue;
return CurPtr;
}
-
- return fastParseASCIIIdentifierScalar(CurPtr);
-}
-
-__attribute__((target("sse4.2"))) static const char *
-fastParseASCIIIdentifier(const char *CurPtr, const char *BufferEnd) {
- return fastParseASCIIIdentifierSSE42(CurPtr, BufferEnd);
-}
-
-__attribute__((target("default")))
#endif
-static const char *fastParseASCIIIdentifier(const char *CurPtr,
- const char *BufferEnd) {
+static const char *fastParseASCIIIdentifier(
+ const char *CurPtr, [[maybe_unused]] const char *BufferEnd) {
return fastParseASCIIIdentifierScalar(CurPtr);
}
More information about the cfe-commits
mailing list