[llvm-branch-commits] [Clang] Implement __builtin_speculative_load (PR #242576)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Nov 13 20:27:39 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang
Author: Aiden Grossman (boomanaiden154)
<details>
<summary>Changes</summary>
This patch implements a clang builtin exposign LLVM's
llvm.speculative.load intrinsic, which allows for performing potentially
OOB memory accesses, assuming the load will not trap on the underlying
hardware and that the program doesn't use the OOB parts of the load.
The intrinsic here is set to return a vector. We cannot use the overload
that returns a byte value given there is no type in clang currently that
lowers to the byte type, and the primary use case for
__builtin_speculative_load requires wide types whereas the byte type
will most likely only be used for char/std::byte. We can get the same
poison spreading semantics as the byte type by using clang-native
vectors with 8-bit wide lanes (e.g., char).
This is primarily intended to allow for a formally correct
implementation of LLVM libc's strlen, which needs speculative loads for
a fast implementation. The current implementation already does this, but
just does wide loads. Inlining such a strlen implementation (e.g., in
(Thin)LTO scenarios) causes problems as the compiler observes that the
load accesses memory that is potentially OOB and treats the entire load
as UB.
---
Full diff: https://github.com/llvm/llvm-project/pull/242576.diff
6 Files Affected:
- (modified) clang/docs/LanguageExtensions.md (+23)
- (modified) clang/include/clang/Basic/Builtins.td (+6)
- (modified) clang/lib/CodeGen/CGBuiltin.cpp (+23)
- (modified) clang/lib/Sema/SemaChecking.cpp (+81-5)
- (added) clang/test/CodeGen/builtin-speculative-load.c (+77)
- (added) clang/test/Sema/builtin-speculative-load.c (+29)
``````````diff
diff --git a/clang/docs/LanguageExtensions.md b/clang/docs/LanguageExtensions.md
index 9362de5c9e69a..a8e1c60ed8a2b 100644
--- a/clang/docs/LanguageExtensions.md
+++ b/clang/docs/LanguageExtensions.md
@@ -895,6 +895,29 @@ void scatter(v8b mask, v8i val, v8i idx, int *ptr) {
## Speculative Loads
+Clang provides an extension that allows developers to perform speculative loads, loads that might load from out-of-bounds memory provided the memory can be safely accessed on the underlying hardware.
+
+Query for this feature with `__has_builtin(__builtin_speculative_load)`.
+
+### Syntax
+
+The prototype for the `__builtin_speculative_load` intrinsic is as follows:
+
+```c++
+__builtin_speculative_load(vector_ty *load_addr, bool from_end, (*uint64_t)(...) oracle, ...);
+```
+
+All parameters after the oracle function are passed to the oracle function. Example:
+```c++
+typedef char v8 __attribute__((ext_vector_type(8)));
+
+v8 value = __builtin_speculative_load(input_ptr, false, oracle, input_size);
+```
+
+### Semantics
+
+`__builtin_speculative_load` loads a vector from `load_addr` that is potentially out of bounds of the memory object that `load_addr` points into. Any vector lanes containing any bytes that would have otherwise been out of bounds contain indeterminate values (corresponding to poison in LLVM). The oracle function must refer to a function with the `[[clang::oracle_function]]` attribute (described below) that will return the number of bytes that will be used from the load. These bytes are numbered starting from the beginning if `from_end` is false and vice versa.
+
### Oracle Functions
Oracle functions must be denoted with the `[[clang::oracle_function]]` attribute:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 15693d2fe128e..67d3f661babb3 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1562,6 +1562,12 @@ def MaskedScatter : Builtin {
let Prototype = "void(...)";
}
+def SpeculativeLoad : Builtin {
+ let Spellings = ["__builtin_speculative_load"];
+ let Attributes = [NoThrow, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
+
def AllocaUninitialized : Builtin {
let Spellings = ["__builtin_alloca_uninitialized"];
let Attributes = [FunctionWithBuiltinPrefix, NoThrow];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2ea329869284e..702827ef6d195 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -26,6 +26,7 @@
#include "TargetInfo.h"
#include "clang/AST/OSLog.h"
#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/Builtins.h"
#include "clang/Basic/DiagnosticFrontend.h"
#include "clang/Basic/TargetInfo.h"
#include "clang/CodeGenUtils/TargetUtils.h"
@@ -4745,6 +4746,28 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
Builder.CreateMaskedScatter(Val, PtrVec, Align.getAsAlign(), Mask);
return RValue();
}
+ case Builtin::BI__builtin_speculative_load: {
+ llvm::Value *Ptr = EmitScalarExpr(E->getArg(0));
+ llvm::Value *FromEnd = Builder.getInt1(
+ E->getArg(1)->EvaluateKnownConstInt(getContext()).getBoolValue());
+ llvm::Value *OracleFn = EmitScalarExpr(E->getArg(2));
+ llvm::SmallVector<llvm::Value *> OracleFnParams;
+ for (unsigned I = 3; I < E->getNumArgs(); ++I) {
+ OracleFnParams.push_back(EmitScalarExpr(E->getArg(I)));
+ }
+
+ llvm::Type *RetTy = CGM.getTypes().ConvertType(E->getType());
+
+ Function *F = CGM.getIntrinsic(Intrinsic::speculative_load,
+ {RetTy, Builder.getPtrTy()});
+
+ SmallVector<llvm::Value *> SpeculativeLoadCallArgs = {Ptr, FromEnd,
+ OracleFn};
+ SpeculativeLoadCallArgs.append(OracleFnParams);
+ llvm::Value *Result = Builder.CreateCall(F, SpeculativeLoadCallArgs);
+
+ return RValue::get(Result);
+ }
case Builtin::BI__builtin_isinf_sign: {
// isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index a273eef3526d3..5be4917502080 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -17,6 +17,7 @@
#include "clang/AST/ASTDiagnostic.h"
#include "clang/AST/Attr.h"
#include "clang/AST/AttrIterator.h"
+#include "clang/AST/Attrs.inc"
#include "clang/AST/CharUnits.h"
#include "clang/AST/Decl.h"
#include "clang/AST/DeclBase.h"
@@ -2847,7 +2848,7 @@ static bool CheckMaskedBuiltinArgs(Sema &S, Expr *MaskArg, Expr *PtrArg,
return false;
}
-static bool ConvertMaskedBuiltinArgs(Sema &S, CallExpr *TheCall) {
+static bool ConvertBuiltinArgs(Sema &S, CallExpr *TheCall) {
bool TypeDependent = false;
for (unsigned Arg = 0, E = TheCall->getNumArgs(); Arg != E; ++Arg) {
ExprResult Converted =
@@ -2867,7 +2868,7 @@ static ExprResult BuiltinMaskedLoad(Sema &S, CallExpr *TheCall) {
if (S.checkArgCountRange(TheCall, 2, 3))
return ExprError();
- if (ConvertMaskedBuiltinArgs(S, TheCall))
+ if (ConvertBuiltinArgs(S, TheCall))
return ExprError();
Expr *MaskArg = TheCall->getArg(0);
@@ -2903,7 +2904,7 @@ static ExprResult BuiltinMaskedStore(Sema &S, CallExpr *TheCall) {
if (S.checkArgCount(TheCall, 3))
return ExprError();
- if (ConvertMaskedBuiltinArgs(S, TheCall))
+ if (ConvertBuiltinArgs(S, TheCall))
return ExprError();
Expr *MaskArg = TheCall->getArg(0);
@@ -2952,7 +2953,7 @@ static ExprResult BuiltinMaskedGather(Sema &S, CallExpr *TheCall) {
if (S.checkArgCountRange(TheCall, 3, 4))
return ExprError();
- if (ConvertMaskedBuiltinArgs(S, TheCall))
+ if (ConvertBuiltinArgs(S, TheCall))
return ExprError();
Expr *MaskArg = TheCall->getArg(0);
@@ -3002,7 +3003,7 @@ static ExprResult BuiltinMaskedScatter(Sema &S, CallExpr *TheCall) {
if (S.checkArgCount(TheCall, 4))
return ExprError();
- if (ConvertMaskedBuiltinArgs(S, TheCall))
+ if (ConvertBuiltinArgs(S, TheCall))
return ExprError();
Expr *MaskArg = TheCall->getArg(0);
@@ -3054,6 +3055,79 @@ static ExprResult BuiltinMaskedScatter(Sema &S, CallExpr *TheCall) {
return TheCall;
}
+static ExprResult BuiltinSpeculativeLoad(Sema &S, CallExpr *TheCall) {
+ if (S.checkArgCountAtLeast(TheCall, 3))
+ return ExprError();
+
+ if (ConvertBuiltinArgs(S, TheCall))
+ return ExprError();
+
+ Expr *PtrArg = TheCall->getArg(0);
+ QualType PtrTy = PtrArg->getType();
+ if (!PtrTy->isPointerType())
+ return S.Diag(PtrArg->getExprLoc(),
+ diag::err_typecheck_indirection_requires_pointer)
+ << PtrTy << PtrArg->getSourceRange();
+
+ QualType PointeeTy = PtrTy->getPointeeType();
+ TheCall->setType(PointeeTy.getUnqualifiedType());
+
+ const auto *VT = PointeeTy->getAs<VectorType>();
+ if (!VT || !VT->getElementType()->isIntegerType())
+ return S.Diag(PtrArg->getExprLoc(),
+ diag::err_speculative_load_not_a_vector_of_integers);
+
+ InitializedEntity Entity = InitializedEntity::InitializeParameter(
+ S.Context, S.Context.BoolTy, /*consume=f*/ false);
+ ExprResult FromEnd =
+ S.PerformCopyInitialization(Entity, SourceLocation(), TheCall->getArg(1));
+ if (FromEnd.isInvalid())
+ return ExprError();
+ TheCall->setArg(1, FromEnd.get());
+ llvm::APSInt Val;
+ if (S.BuiltinConstantArg(TheCall, 1, Val))
+ return ExprError();
+
+ Expr *OracleFnArg = TheCall->getArg(2);
+ const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(
+ OracleFnArg->getAsBuiltinConstantDeclRef(S.getASTContext()));
+ if (!FD)
+ return S.Diag(OracleFnArg->getBeginLoc(),
+ diag::err_speculative_load_third_arg_must_be_function);
+
+ if (!FD->hasAttr<OracleFunctionAttr>())
+ return S.Diag(OracleFnArg->getBeginLoc(),
+ diag::err_speculative_load_function_oracle_attr);
+
+ if (TheCall->getNumArgs() - 3 != FD->getNumParams())
+ return S.Diag(TheCall->getBeginLoc(),
+ diag::err_speculative_load_arg_oracle_param_count_mismatch)
+ << FD->getNumParams() << TheCall->getNumArgs() - 3;
+
+ for (unsigned I = 0, N = FD->getNumParams(); I != N; ++I) {
+ ParmVarDecl *Param = const_cast<ParmVarDecl *>(FD->getParamDecl(I));
+ Expr *Arg = TheCall->getArg(3 + I);
+
+ if (S.RequireCompleteType(Arg->getBeginLoc(), Param->getType(),
+ diag::err_call_incomplete_argument, Arg))
+ return ExprError();
+
+ InitializedEntity Entity =
+ InitializedEntity::InitializeParameter(S.Context, Param);
+ ExprResult ArgRes =
+ S.PerformCopyInitialization(Entity, SourceLocation(), Arg);
+ if (ArgRes.isInvalid())
+ return ExprError();
+ TheCall->setArg(3 + I, ArgRes.get());
+ }
+
+ if (!S.checkAddressOfFunctionIsAvailable(FD, /*Complain=*/true,
+ TheCall->getBeginLoc()))
+ return ExprError();
+
+ return TheCall;
+}
+
static ExprResult BuiltinInvoke(Sema &S, CallExpr *TheCall) {
SourceLocation Loc = TheCall->getBeginLoc();
MutableArrayRef Args(TheCall->getArgs(), TheCall->getNumArgs());
@@ -3354,6 +3428,8 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
return BuiltinMaskedGather(*this, TheCall);
case Builtin::BI__builtin_masked_scatter:
return BuiltinMaskedScatter(*this, TheCall);
+ case Builtin::BI__builtin_speculative_load:
+ return BuiltinSpeculativeLoad(*this, TheCall);
case Builtin::BI__builtin_invoke:
return BuiltinInvoke(*this, TheCall);
case Builtin::BI__builtin_prefetch:
diff --git a/clang/test/CodeGen/builtin-speculative-load.c b/clang/test/CodeGen/builtin-speculative-load.c
new file mode 100644
index 0000000000000..0331e56c8ce6e
--- /dev/null
+++ b/clang/test/CodeGen/builtin-speculative-load.c
@@ -0,0 +1,77 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --include-generated-funcs --version 6
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+
+typedef char v8 __attribute__((ext_vector_type(8)));
+
+[[clang::oracle_function]] static long oracle(void) {
+ return 2;
+}
+
+v8 test_speculative_load(v8 *in) {
+ return __builtin_speculative_load(in, 0, oracle);
+}
+
+[[clang::oracle_function]] static long oracle_params(v8 *in) {
+ return 2;
+}
+
+v8 test_speculative_load_oracle_params(v8 *in) {
+ return __builtin_speculative_load(in, 0, oracle_params, in);
+}
+
+v8 test_speculative_load_from_end(v8 *in) {
+ return __builtin_speculative_load(in, 1, oracle);
+}
+// CHECK-LABEL: define dso_local double @test_speculative_load(
+// CHECK-SAME: ptr noundef [[IN:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[RETVAL:%.*]] = alloca <8 x i8>, align 8
+// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT: store ptr [[IN]], ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> (ptr, i1, ...) @llvm.speculative.load.v8i8.p0(ptr [[TMP0]], i1 false, ptr @oracle)
+// CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[RETVAL]], align 8
+// CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[RETVAL]], align 8
+// CHECK-NEXT: ret double [[TMP2]]
+//
+//
+// CHECK-LABEL: define internal i64 @oracle(
+// CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: ret i64 2
+//
+//
+// CHECK-LABEL: define dso_local double @test_speculative_load_oracle_params(
+// CHECK-SAME: ptr noundef [[IN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[RETVAL:%.*]] = alloca <8 x i8>, align 8
+// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT: store ptr [[IN]], ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP2:%.*]] = call <8 x i8> (ptr, i1, ...) @llvm.speculative.load.v8i8.p0(ptr [[TMP0]], i1 false, ptr @oracle_params, ptr [[TMP1]])
+// CHECK-NEXT: store <8 x i8> [[TMP2]], ptr [[RETVAL]], align 8
+// CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[RETVAL]], align 8
+// CHECK-NEXT: ret double [[TMP3]]
+//
+//
+// CHECK-LABEL: define internal i64 @oracle_params(
+// CHECK-SAME: ptr noundef [[IN:%.*]]) #[[ATTR1]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT: store ptr [[IN]], ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: ret i64 2
+//
+//
+// CHECK-LABEL: define dso_local double @test_speculative_load_from_end(
+// CHECK-SAME: ptr noundef [[IN:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[RETVAL:%.*]] = alloca <8 x i8>, align 8
+// CHECK-NEXT: [[IN_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT: store ptr [[IN]], ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[IN_ADDR]], align 8
+// CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> (ptr, i1, ...) @llvm.speculative.load.v8i8.p0(ptr [[TMP0]], i1 true, ptr @oracle)
+// CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[RETVAL]], align 8
+// CHECK-NEXT: [[TMP2:%.*]] = load double, ptr [[RETVAL]], align 8
+// CHECK-NEXT: ret double [[TMP2]]
+//
diff --git a/clang/test/Sema/builtin-speculative-load.c b/clang/test/Sema/builtin-speculative-load.c
new file mode 100644
index 0000000000000..87c40257958b9
--- /dev/null
+++ b/clang/test/Sema/builtin-speculative-load.c
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsyntax-only -verify %s
+
+typedef char v8 __attribute__((ext_vector_type(8)));
+typedef float v4 __attribute__((ext_vector_type(4)));
+
+static long fake_oracle(void) {
+ return 1;
+}
+
+[[clang::oracle_function]] static long oracle(void) {
+ return 1;
+}
+
+[[clang::oracle_function]] static long oracle_with_params(int a) {
+ return a;
+}
+
+void test_speculative_load(v8 *to_load, v4 *to_load2) {
+ int a = 5;
+ (void)__builtin_speculative_load(to_load); // expected-error {{too few arguments to function call}}
+ (void)__builtin_speculative_load(5, 1, oracle); // expected-error {{indirection requires pointer operand ('int' invalid)}}
+ (void)__builtin_speculative_load(to_load, a, oracle); // expected-error {{argument to '__builtin_speculative_load' must be a constant integer}}
+ (void)__builtin_speculative_load(to_load, 0, 5); // expected-error {{the 3rd argument to '__builtin_speculative_load' must be a function}}
+ (void)__builtin_speculative_load(to_load, 0, fake_oracle); // expected-error {{the oracle function passed to '__builtin_speculative_load' must have the oracle_function attribute}}
+ (void)__builtin_speculative_load(to_load, 0, oracle_with_params); // expected-error {{expected to have 1 arguments after the oracle function to match its parameter count, but there are 0}}
+ (void)__builtin_speculative_load(to_load, 0, oracle_with_params, "a"); // expected-error {{incompatible pointer to integer conversion passing 'char *' to parameter of type 'int'}} expected-note at 14 {{passing argument to parameter 'a' here}}
+ (void)__builtin_speculative_load(to_load2, 0, oracle_with_params, a); // expected-error {{the 1st argument to '__builtin_speculative_load' must be a pointer to a vector of integers}}
+ (void)__builtin_speculative_load(to_load, 0, oracle_with_params, a);
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/242576
More information about the llvm-branch-commits
mailing list