[clang] [llvm] [AArch64] Add MS atomic and exclusive-access intrinsics (PR #202416)
Adhemerval Zanella via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 1 09:04:21 PDT 2026
https://github.com/zatrazz updated https://github.com/llvm/llvm-project/pull/202416
>From 8c30cbf0b5540257a499dff4e695bbbc9e5cc560 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Mon, 13 Apr 2026 16:04:27 -0300
Subject: [PATCH 01/26] [aarch64] Add support for the __ldar{8|16|32|64} MS
intrinsics (#121689)
Adds support for the following MSVC intrinsics:
* `__ldar8` - maps to LDARB
* `__ldar16` - maps to LDARH
* `__ldar32` - maps to LDAR
* `__ldar64` - maps to LDAR
The emit is done using SeqCst atomic operations because wth RCPC enabled
the AArch64 backend lower acquire loads to LDARP* instead of LDAR*.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 7 ++++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 17 ++++++++++
clang/lib/Headers/arm64intr.h | 13 ++++++++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 32 +++++++++++++++++++
4 files changed, 69 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 4c49c874133fb..83ba7b39480ae 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -411,3 +411,10 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __hvc : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int, ...)">;
def __svc : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int, ...)">;
}
+
+let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES", Header = "intrin.h" in {
+ def __ldar8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char const volatile *)">;
+ def __ldar16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
+ def __ldar32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
+ def __ldar64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile*)">;
+}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 46f80979bcb97..2e624b10f34b9 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5280,6 +5280,23 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateTrunc(Call, Int32Ty);
}
+ if (BuiltinID == AArch64::BI__ldar8 || BuiltinID == AArch64::BI__ldar16 ||
+ BuiltinID == AArch64::BI__ldar32 || BuiltinID == AArch64::BI__ldar64) {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ QualType ElTy = E->getArg(0)->getType()->getPointeeType();
+ CharUnits LoadSize = CGM.getContext().getTypeSizeInChars(ElTy);
+ llvm::Type *ITy =
+ llvm::IntegerType::get(getLLVMContext(), LoadSize.getQuantity() * 8);
+ llvm::LoadInst *Load = Builder.CreateAlignedLoad(ITy, Ptr, LoadSize);
+ // We need SeqCst instead of Acquire because with RCPC enabled the AArch64
+ // lowers Acquire loads to LDARP* instead of LDAR*. The SeqCst has not RCPC
+ // override and always maps to LDAR*. This is the same apprach used by
+ // __iso_volatile_load (which uses Monotonic plus volatile for plain ldr).
+ Load->setAtomic(llvm::AtomicOrdering::SequentiallyConsistent);
+ Load->setVolatile(true);
+ return Load;
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 53a3d57a6e9d1..8eb36b5624ddb 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -41,5 +41,18 @@ typedef enum
_ARM64_BARRIER_OSHLD = 0x1
} _ARM64INTR_BARRIER_TYPE;
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+unsigned __int8 __ldar8(const volatile unsigned __int8 *);
+unsigned __int16 __ldar16(const volatile unsigned __int16 *);
+unsigned __int32 __ldar32(const volatile unsigned __int32 *);
+unsigned __int64 __ldar64(const volatile unsigned __int64 *);
+
+#ifdef __cplusplus
+}
+#endif
+
#endif /* __ARM64INTR_H */
#endif /* _MSC_VER */
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index bf2dcf5ce2afd..db83577da51b8 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -694,6 +694,38 @@ void check__prefetch2(void *arg1) {
// CHECK-LINUX: error: call to undeclared function '__prefetch2'
+unsigned char check__ldar8(unsigned char volatile *p) {
+ return __ldar8(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__ldar8(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = load atomic volatile i8, ptr %{{.*}} seq_cst, align 1
+// CHECK-MSCOMPAT: ret i8 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldar8'
+
+unsigned short check__ldar16(unsigned short volatile *p) {
+ return __ldar16(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @check__ldar16(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = load atomic volatile i16, ptr %{{.*}} seq_cst, align 2
+// CHECK-MSCOMPAT: ret i16 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldar16'
+
+unsigned int check__ldar32(unsigned int volatile *p) {
+ return __ldar32(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @check__ldar32(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = load atomic volatile i32, ptr %{{.*}} seq_cst, align 4
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldar32'
+
+unsigned long long int check__ldar64(unsigned long long int volatile *p) {
+ return __ldar64(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @check__ldar64(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = load atomic volatile i64, ptr %{{.*}} seq_cst, align 8
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldar64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From 472663aa7494c135437c0e5fa5d76c24074d15af Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Wed, 15 Apr 2026 09:43:49 -0300
Subject: [PATCH 02/26] [aarch64] Add support for the __stlr{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__stlr8` - maps to STLRB.
* `__stlr16` - maps to STLRH.
* `__stlr32` - maps to STLR.
* `__stlr64` - maps to STLR.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 12 +++++
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 44 +++++++++++++++++++
4 files changed, 65 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 83ba7b39480ae..8c1ea43934043 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -417,4 +417,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __ldar16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
def __ldar32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
def __ldar64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile*)">;
+ def __stlr8 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned char volatile *, unsigned char)">;
+ def __stlr16 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned short volatile *, unsigned short)">;
+ def __stlr32 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned int volatile *, unsigned int)">;
+ def __stlr64 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 2e624b10f34b9..c5c4787a5ee37 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5297,6 +5297,18 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Load;
}
+ if (BuiltinID == AArch64::BI__stlr8 || BuiltinID == AArch64::BI__stlr16 ||
+ BuiltinID == AArch64::BI__stlr32 || BuiltinID == AArch64::BI__stlr64) {
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ Value *Val = EmitScalarExpr(E->getArg(1));
+ QualType ElTy = E->getArg(0)->getType()->getPointeeType();
+ CharUnits StoreSize = CGM.getContext().getTypeSizeInChars(ElTy);
+ llvm::StoreInst *Store = Builder.CreateAlignedStore(Val, Ptr, StoreSize);
+ Store->setAtomic(llvm::AtomicOrdering::Release);
+ Store->setVolatile(true);
+ return Store;
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 8eb36b5624ddb..950ecd6aa38cc 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -50,6 +50,11 @@ unsigned __int16 __ldar16(const volatile unsigned __int16 *);
unsigned __int32 __ldar32(const volatile unsigned __int32 *);
unsigned __int64 __ldar64(const volatile unsigned __int64 *);
+void __stlr8(unsigned __int8 volatile *, unsigned __int8);
+void __stlr16(unsigned __int16 volatile *, unsigned __int16);
+void __stlr32(unsigned __int32 volatile *, unsigned __int32);
+void __stlr64(unsigned __int64 volatile *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index db83577da51b8..4d8538b9135f5 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -726,6 +726,50 @@ unsigned long long int check__ldar64(unsigned long long int volatile *p) {
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__ldar64'
+void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
+{
+ __stlr8 (p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}void @test__stlr8(ptr{{.*}}%p, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[DEST:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[VALUE:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: store atomic volatile i8 %[[VALUE]], ptr %[[DEST]] release, align 1
+// CHECK-MSCOMPAT: ret void
+// CHECK-LINUX: error: call to undeclared function '__stlr8'
+
+void test__stlr16(unsigned __int16 volatile *p, unsigned __int16 v)
+{
+ __stlr16 (p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}void @test__stlr16(ptr{{.*}}%p, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[DEST:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[VALUE:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: store atomic volatile i16 %[[VALUE]], ptr %[[DEST]] release, align 2
+// CHECK-MSCOMPAT: ret void
+// CHECK-LINUX: error: call to undeclared function '__stlr16'
+
+void test__stlr32(unsigned __int32 volatile *p, unsigned __int32 v)
+{
+ __stlr32(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}void @test__stlr32(ptr{{.*}}%p, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[DEST:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[VALUE:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: store atomic volatile i32 %[[VALUE]], ptr %[[DEST]] release, align 4
+// CHECK-MSCOMPAT: ret void
+// CHECK-LINUX: error: call to undeclared function '__stlr32'
+
+void test__stlr64(unsigned __int64 volatile *p, unsigned __int64 v)
+{
+ __stlr64(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}void @test__stlr64(ptr{{.*}}%p, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[DEST:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[VALUE:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: store atomic volatile i64 %[[VALUE]], ptr %[[DEST]] release, align 8
+// CHECK-MSCOMPAT: ret void
+// CHECK-LINUX: error: call to undeclared function '__stlr64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From 57561e8f5b069ae703cab84db15c23b02f572a40 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 07:53:47 -0300
Subject: [PATCH 03/26] [aarch64] Add support for the __ldxr{8|16|32|64} MS
intrinsics
These MSVC load-exclusive intrinsics lower to the existing llvm.aarch64.ldxr
intrinsic used by the ACLE __builtin_arm_ldrex builtin.
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 16 ++++++++++
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 31 +++++++++++++++++++
4 files changed, 57 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 8c1ea43934043..9c563c7886dc5 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -421,4 +421,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __stlr16 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned short volatile *, unsigned short)">;
def __stlr32 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned int volatile *, unsigned int)">;
def __stlr64 : AArch64NoPrefixTargetLibBuiltin<"void (unsigned long long int volatile *, unsigned long long int)">;
+
+ def __ldxr8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char const volatile *)">;
+ def __ldxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
+ def __ldxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
+ def __ldxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile *)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index c5c4787a5ee37..3d55fa2e46bb2 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5297,6 +5297,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Load;
}
+ if (BuiltinID == AArch64::BI__ldxr8 || BuiltinID == AArch64::BI__ldxr16 ||
+ BuiltinID == AArch64::BI__ldxr32 || BuiltinID == AArch64::BI__ldxr64) {
+ // Load-exclusive (LDXR*). Reuse the llvm.aarch64.ldxr lowering of the ACLE
+ // __builtin_arm_ldrex builtin.
+ Value *LoadAddr = EmitScalarExpr(E->getArg(0));
+ QualType Ty = E->getType();
+ llvm::Type *RealResTy = ConvertType(Ty);
+ llvm::Type *IntTy =
+ llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+ Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ldxr, DefaultPtrTy);
+ CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldxr");
+ Val->addParamAttr(
+ 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
+ return Builder.CreateTruncOrBitCast(Val, RealResTy);
+ }
+
if (BuiltinID == AArch64::BI__stlr8 || BuiltinID == AArch64::BI__stlr16 ||
BuiltinID == AArch64::BI__stlr32 || BuiltinID == AArch64::BI__stlr64) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 950ecd6aa38cc..ed411c213406b 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -55,6 +55,11 @@ void __stlr16(unsigned __int16 volatile *, unsigned __int16);
void __stlr32(unsigned __int32 volatile *, unsigned __int32);
void __stlr64(unsigned __int64 volatile *, unsigned __int64);
+unsigned __int8 __ldxr8(const volatile unsigned __int8 *);
+unsigned __int16 __ldxr16(const volatile unsigned __int16 *);
+unsigned __int32 __ldxr32(const volatile unsigned __int32 *);
+unsigned __int64 __ldxr64(const volatile unsigned __int64 *);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 4d8538b9135f5..e1beeb0cb17ad 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -726,6 +726,37 @@ unsigned long long int check__ldar64(unsigned long long int volatile *p) {
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__ldar64'
+unsigned char check__ldxr8(unsigned char volatile *p) {
+ return __ldxr8(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__ldxr8(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i8) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i8
+// CHECK-LINUX: error: call to undeclared function '__ldxr8'
+
+unsigned short check__ldxr16(unsigned short volatile *p) {
+ return __ldxr16(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @check__ldxr16(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i16) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i16
+// CHECK-LINUX: error: call to undeclared function '__ldxr16'
+
+unsigned int check__ldxr32(unsigned int volatile *p) {
+ return __ldxr32(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @check__ldxr32(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i32
+// CHECK-LINUX: error: call to undeclared function '__ldxr32'
+
+unsigned long long int check__ldxr64(unsigned long long int volatile *p) {
+ return __ldxr64(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @check__ldxr64(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i64) %{{.*}})
+// CHECK-LINUX: error: call to undeclared function '__ldxr64'
+
void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
{
__stlr8 (p, v);
>From daa8b72c4efede2d60c99b4f64596cb38b9795d3 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 08:11:24 -0300
Subject: [PATCH 04/26] [aarch64] Add support for the __ldaxr{8|16|32|64} MS
intrinsics
These MSVC load-acquire-exclusive intrinsics lower to the existing
llvm.aarch64.ldaxr intrinsic used by the ACLE __builtin_arm_ldaex builtin.
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 16 ++++++++++
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 31 +++++++++++++++++++
4 files changed, 57 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 9c563c7886dc5..69f3c3f81c3a6 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -426,4 +426,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __ldxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
def __ldxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
def __ldxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile *)">;
+
+ def __ldaxr8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char const volatile *)">;
+ def __ldaxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
+ def __ldaxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
+ def __ldaxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile *)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 3d55fa2e46bb2..a160136e6501f 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5313,6 +5313,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateTruncOrBitCast(Val, RealResTy);
}
+ if (BuiltinID == AArch64::BI__ldaxr8 || BuiltinID == AArch64::BI__ldaxr16 ||
+ BuiltinID == AArch64::BI__ldaxr32 || BuiltinID == AArch64::BI__ldaxr64) {
+ // Load-acquire-exclusive (LDAXR*). Reuse the llvm.aarch64.ldaxr lowering of
+ // the ACLE __builtin_arm_ldaex builtin.
+ Value *LoadAddr = EmitScalarExpr(E->getArg(0));
+ QualType Ty = E->getType();
+ llvm::Type *RealResTy = ConvertType(Ty);
+ llvm::Type *IntTy =
+ llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+ Function *F = CGM.getIntrinsic(Intrinsic::aarch64_ldaxr, DefaultPtrTy);
+ CallInst *Val = Builder.CreateCall(F, LoadAddr, "ldaxr");
+ Val->addParamAttr(
+ 0, Attribute::get(getLLVMContext(), Attribute::ElementType, IntTy));
+ return Builder.CreateTruncOrBitCast(Val, RealResTy);
+ }
+
if (BuiltinID == AArch64::BI__stlr8 || BuiltinID == AArch64::BI__stlr16 ||
BuiltinID == AArch64::BI__stlr32 || BuiltinID == AArch64::BI__stlr64) {
Value *Ptr = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index ed411c213406b..378f4b4c3d5e3 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -60,6 +60,11 @@ unsigned __int16 __ldxr16(const volatile unsigned __int16 *);
unsigned __int32 __ldxr32(const volatile unsigned __int32 *);
unsigned __int64 __ldxr64(const volatile unsigned __int64 *);
+unsigned __int8 __ldaxr8(const volatile unsigned __int8 *);
+unsigned __int16 __ldaxr16(const volatile unsigned __int16 *);
+unsigned __int32 __ldaxr32(const volatile unsigned __int32 *);
+unsigned __int64 __ldaxr64(const volatile unsigned __int64 *);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index e1beeb0cb17ad..3813978b90272 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -757,6 +757,37 @@ unsigned long long int check__ldxr64(unsigned long long int volatile *p) {
// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i64) %{{.*}})
// CHECK-LINUX: error: call to undeclared function '__ldxr64'
+unsigned char check__ldaxr8(unsigned char volatile *p) {
+ return __ldaxr8(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__ldaxr8(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldaxr.p0(ptr elementtype(i8) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i8
+// CHECK-LINUX: error: call to undeclared function '__ldaxr8'
+
+unsigned short check__ldaxr16(unsigned short volatile *p) {
+ return __ldaxr16(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @check__ldaxr16(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldaxr.p0(ptr elementtype(i16) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i16
+// CHECK-LINUX: error: call to undeclared function '__ldaxr16'
+
+unsigned int check__ldaxr32(unsigned int volatile *p) {
+ return __ldaxr32(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @check__ldaxr32(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldaxr.p0(ptr elementtype(i32) %{{.*}})
+// CHECK-MSCOMPAT: trunc i64 %[[RET]] to i32
+// CHECK-LINUX: error: call to undeclared function '__ldaxr32'
+
+unsigned long long int check__ldaxr64(unsigned long long int volatile *p) {
+ return __ldaxr64(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @check__ldaxr64(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldaxr.p0(ptr elementtype(i64) %{{.*}})
+// CHECK-LINUX: error: call to undeclared function '__ldaxr64'
+
void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
{
__stlr8 (p, v);
>From 33ca9997f64b61a64d60994c84ff4605ea1aa49a Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 08:13:59 -0300
Subject: [PATCH 05/26] [aarch64] Add support for the __stxr{8|16|32|64} MS
intrinsics
These MSVC store-exclusive intrinsics lower to the existing llvm.aarch64.stxr
intrinsic used by the ACLE __builtin_arm_strex builtin. MSVC takes the
arguments as (pointer, value) (the reverse of the ACLE builtin) and returns
the store status as unsigned char.
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 21 ++++++++++++
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 32 +++++++++++++++++++
4 files changed, 63 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 69f3c3f81c3a6..4e8a795343455 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -431,4 +431,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __ldaxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
def __ldaxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
def __ldaxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile *)">;
+
+ def __stxr8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __stxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned short volatile *, unsigned short)">;
+ def __stxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned int volatile *, unsigned int)">;
+ def __stxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index a160136e6501f..1851ce281ecc3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5341,6 +5341,27 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Store;
}
+ if (BuiltinID == AArch64::BI__stxr8 || BuiltinID == AArch64::BI__stxr16 ||
+ BuiltinID == AArch64::BI__stxr32 || BuiltinID == AArch64::BI__stxr64) {
+ // Store-exclusive (STXR*). Reuse the llvm.aarch64.stxr lowering of the ACLE
+ // __builtin_arm_strex builtin. MSVC takes (ptr, value) (the reverse of the
+ // ACLE builtin) and returns the store status as unsigned char.
+ Value *StoreAddr = EmitScalarExpr(E->getArg(0));
+ Value *StoreVal = EmitScalarExpr(E->getArg(1));
+
+ QualType Ty = E->getArg(1)->getType();
+ llvm::Type *StoreTy =
+ llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
+ StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
+
+ Function *F =
+ CGM.getIntrinsic(Intrinsic::aarch64_stxr, StoreAddr->getType());
+ CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
+ CI->addParamAttr(
+ 1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
+ return Builder.CreateTrunc(CI, ConvertType(E->getType()));
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 378f4b4c3d5e3..a0cdd669bca55 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -65,6 +65,11 @@ unsigned __int16 __ldaxr16(const volatile unsigned __int16 *);
unsigned __int32 __ldaxr32(const volatile unsigned __int32 *);
unsigned __int64 __ldaxr64(const volatile unsigned __int64 *);
+unsigned __int8 __stxr8(volatile unsigned __int8 *, unsigned __int8);
+unsigned __int8 __stxr16(volatile unsigned __int16 *, unsigned __int16);
+unsigned __int8 __stxr32(volatile unsigned __int32 *, unsigned __int32);
+unsigned __int8 __stxr64(volatile unsigned __int64 *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 3813978b90272..c912830282f0a 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -788,6 +788,38 @@ unsigned long long int check__ldaxr64(unsigned long long int volatile *p) {
// CHECK-MSCOMPAT: %[[RET:.*]] = call i64 @llvm.aarch64.ldaxr.p0(ptr elementtype(i64) %{{.*}})
// CHECK-LINUX: error: call to undeclared function '__ldaxr64'
+unsigned char check__stxr8(unsigned char volatile *p, unsigned char v) {
+ return __stxr8(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stxr8(ptr{{.*}}%p, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stxr.p0(i64 %{{.*}}, ptr elementtype(i8) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stxr8'
+
+unsigned char check__stxr16(unsigned short volatile *p, unsigned short v) {
+ return __stxr16(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stxr16(ptr{{.*}}%p, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stxr.p0(i64 %{{.*}}, ptr elementtype(i16) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stxr16'
+
+unsigned char check__stxr32(unsigned int volatile *p, unsigned int v) {
+ return __stxr32(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stxr32(ptr{{.*}}%p, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stxr.p0(i64 %{{.*}}, ptr elementtype(i32) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stxr32'
+
+unsigned char check__stxr64(unsigned long long int volatile *p, unsigned long long int v) {
+ return __stxr64(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stxr64(ptr{{.*}}%p, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stxr.p0(i64 %{{.*}}, ptr elementtype(i64) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stxr64'
+
void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
{
__stlr8 (p, v);
>From 0903296b84595440ec3637ad1e2aaeadfd0d36dc Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 08:17:34 -0300
Subject: [PATCH 06/26] [aarch64] Add support for the __stlxr{8|16|32|64} MS
intrinsics
These MSVC store-release-exclusive intrinsics lower to the existing
llvm.aarch64.stlxr intrinsic used by the ACLE __builtin_arm_stlex builtin.
As with __stxr, MSVC takes the arguments as (pointer, value) and returns the
store status as unsigned char.
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 19 +++++++----
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 32 +++++++++++++++++++
4 files changed, 55 insertions(+), 6 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 4e8a795343455..b0af6376a8b0e 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -436,4 +436,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __stxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned short volatile *, unsigned short)">;
def __stxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned int volatile *, unsigned int)">;
def __stxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned long long int volatile *, unsigned long long int)">;
+
+ def __stlxr8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __stlxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned short volatile *, unsigned short)">;
+ def __stlxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned int volatile *, unsigned int)">;
+ def __stlxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 1851ce281ecc3..854285ede516a 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5342,10 +5342,16 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
}
if (BuiltinID == AArch64::BI__stxr8 || BuiltinID == AArch64::BI__stxr16 ||
- BuiltinID == AArch64::BI__stxr32 || BuiltinID == AArch64::BI__stxr64) {
- // Store-exclusive (STXR*). Reuse the llvm.aarch64.stxr lowering of the ACLE
- // __builtin_arm_strex builtin. MSVC takes (ptr, value) (the reverse of the
- // ACLE builtin) and returns the store status as unsigned char.
+ BuiltinID == AArch64::BI__stxr32 || BuiltinID == AArch64::BI__stxr64 ||
+ BuiltinID == AArch64::BI__stlxr8 || BuiltinID == AArch64::BI__stlxr16 ||
+ BuiltinID == AArch64::BI__stlxr32 || BuiltinID == AArch64::BI__stlxr64) {
+ // Store-(release-)exclusive (STXR*/STLXR*). Reuse the llvm.aarch64.stxr /
+ // stlxr lowering of the ACLE __builtin_arm_strex / stlex builtins. MSVC
+ // takes (ptr, value) (the reverse of the ACLE builtins) and returns the
+ // store status as unsigned char.
+ bool IsRelease =
+ BuiltinID == AArch64::BI__stlxr8 || BuiltinID == AArch64::BI__stlxr16 ||
+ BuiltinID == AArch64::BI__stlxr32 || BuiltinID == AArch64::BI__stlxr64;
Value *StoreAddr = EmitScalarExpr(E->getArg(0));
Value *StoreVal = EmitScalarExpr(E->getArg(1));
@@ -5354,8 +5360,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
llvm::IntegerType::get(getLLVMContext(), getContext().getTypeSize(Ty));
StoreVal = Builder.CreateZExtOrBitCast(StoreVal, Int64Ty);
- Function *F =
- CGM.getIntrinsic(Intrinsic::aarch64_stxr, StoreAddr->getType());
+ Function *F = CGM.getIntrinsic(IsRelease ? Intrinsic::aarch64_stlxr
+ : Intrinsic::aarch64_stxr,
+ StoreAddr->getType());
CallInst *CI = Builder.CreateCall(F, {StoreVal, StoreAddr}, "stxr");
CI->addParamAttr(
1, Attribute::get(getLLVMContext(), Attribute::ElementType, StoreTy));
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index a0cdd669bca55..3bf6c2cb57091 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -70,6 +70,11 @@ unsigned __int8 __stxr16(volatile unsigned __int16 *, unsigned __int16);
unsigned __int8 __stxr32(volatile unsigned __int32 *, unsigned __int32);
unsigned __int8 __stxr64(volatile unsigned __int64 *, unsigned __int64);
+unsigned __int8 __stlxr8(volatile unsigned __int8 *, unsigned __int8);
+unsigned __int8 __stlxr16(volatile unsigned __int16 *, unsigned __int16);
+unsigned __int8 __stlxr32(volatile unsigned __int32 *, unsigned __int32);
+unsigned __int8 __stlxr64(volatile unsigned __int64 *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index c912830282f0a..cab9101058db3 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -820,6 +820,38 @@ unsigned char check__stxr64(unsigned long long int volatile *p, unsigned long lo
// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
// CHECK-LINUX: error: call to undeclared function '__stxr64'
+unsigned char check__stlxr8(unsigned char volatile *p, unsigned char v) {
+ return __stlxr8(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stlxr8(ptr{{.*}}%p, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stlxr.p0(i64 %{{.*}}, ptr elementtype(i8) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stlxr8'
+
+unsigned char check__stlxr16(unsigned short volatile *p, unsigned short v) {
+ return __stlxr16(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stlxr16(ptr{{.*}}%p, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stlxr.p0(i64 %{{.*}}, ptr elementtype(i16) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stlxr16'
+
+unsigned char check__stlxr32(unsigned int volatile *p, unsigned int v) {
+ return __stlxr32(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stlxr32(ptr{{.*}}%p, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stlxr.p0(i64 %{{.*}}, ptr elementtype(i32) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stlxr32'
+
+unsigned char check__stlxr64(unsigned long long int volatile *p, unsigned long long int v) {
+ return __stlxr64(p, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__stlxr64(ptr{{.*}}%p, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[ST:.*]] = call i32 @llvm.aarch64.stlxr.p0(i64 %{{.*}}, ptr elementtype(i64) %{{.*}})
+// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
+// CHECK-LINUX: error: call to undeclared function '__stlxr64'
+
void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
{
__stlr8 (p, v);
>From 608221bb378abf500773a21b41a6d6f1647779a2 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 09:54:55 -0300
Subject: [PATCH 07/26] [aarch64] Add a CRm operand to the llvm.aarch64.clrex
intrinsic
Parameterize llvm.aarch64.clrex with the CLREX CRm immediate (0-15) so it can
emit "clrex #CRm" instead of always "clrex" (CRm=15), mirroring the
llvm.aarch64.dmb family. The CLREX CRm operand is changed from imm0_15 to the
i32 imm32_0_15 to match the intrinsic argument; assembly is unaffected (same
parser match class). The ACLE __builtin_arm_clrex keeps CRm=15.
This is a prerequisite for implementing the MSVC __clrex(crm) intrinsic.
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 3 ++-
clang/test/CodeGen/builtins-arm-exclusive.c | 2 +-
llvm/include/llvm/IR/IntrinsicsAArch64.td | 2 +-
llvm/lib/IR/AutoUpgrade.cpp | 13 +++++++++++++
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 3 ++-
llvm/lib/Target/AArch64/AArch64InstrAtomics.td | 5 ++---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 3 ++-
llvm/test/Bitcode/aarch64-clrex-upgrade.ll | 14 ++++++++++++++
llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll | 4 ++--
9 files changed, 39 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/Bitcode/aarch64-clrex-upgrade.ll
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 854285ede516a..6760328f5d59e 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -4798,7 +4798,8 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
if (BuiltinID == clang::AArch64::BI__builtin_arm_clrex) {
Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
- return Builder.CreateCall(F);
+ // The ACLE __clrex clears the exclusive monitor with CRm == 15.
+ return Builder.CreateCall(F, {Builder.getInt32(15)});
}
if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
diff --git a/clang/test/CodeGen/builtins-arm-exclusive.c b/clang/test/CodeGen/builtins-arm-exclusive.c
index f27dcfc81f34b..6ca99f235201d 100644
--- a/clang/test/CodeGen/builtins-arm-exclusive.c
+++ b/clang/test/CodeGen/builtins-arm-exclusive.c
@@ -255,7 +255,7 @@ void test_clrex(void) {
__builtin_arm_clrex();
// CHECK: call void @llvm.arm.clrex()
-// CHECK-ARM64: call void @llvm.aarch64.clrex()
+// CHECK-ARM64: call void @llvm.aarch64.clrex(i32 15)
}
#ifdef __aarch64__
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 8356bbb8640a1..118f4a485ee61 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -32,7 +32,7 @@ def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
[llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
[IntrNoFree, IntrWillReturn]>;
-def int_aarch64_clrex : Intrinsic<[]>;
+def int_aarch64_clrex : Intrinsic<[], [llvm_i32_ty]>;
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index cd505273b5bfb..9843bc06aee05 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -976,6 +976,13 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
}
} else {
// 'aarch64.*'.
+ // Changed: 'aarch64.clrex' gained a CRm operand. Rename the old nullary
+ // form so the call is rewritten below to pass the previous implicit CRm
+ // value of 15.
+ if (Name == "clrex" && F->arg_size() == 0) {
+ rename(F);
+ return true;
+ }
if (Neon) {
// 'aarch64.neon.*'.
Intrinsic::ID ID = StringSwitch<Intrinsic::ID>(Name)
@@ -4934,6 +4941,12 @@ static Value *upgradeX86IntrinsicCall(StringRef Name, CallBase *CI, Function *F,
static Value *upgradeAArch64IntrinsicCall(StringRef Name, CallBase *CI,
Function *F, IRBuilder<> &Builder) {
+ if (Name == "clrex.old") {
+ // The old nullary 'aarch64.clrex' is upgraded to pass the previous implicit
+ // CRm value of 15.
+ return Builder.CreateIntrinsic(Intrinsic::aarch64_clrex, {},
+ {Builder.getInt32(15)});
+ }
if (Name.starts_with("neon.bfcvt")) {
if (Name.starts_with("neon.bfcvtn2")) {
SmallVector<int, 32> LoMask(4);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 70396749e3426..e9b2d679fd26a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33119,7 +33119,8 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilderBase &Builder,
void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
IRBuilderBase &Builder) const {
- Builder.CreateIntrinsic(Intrinsic::aarch64_clrex, {});
+ // Clear the exclusive monitor with CRm == 15.
+ Builder.CreateIntrinsic(Intrinsic::aarch64_clrex, {}, {Builder.getInt32(15)});
}
Value *AArch64TargetLowering::emitStoreConditional(IRBuilderBase &Builder,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2187f21abb70f..c324b38e56bc0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -477,9 +477,8 @@ def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
(STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
-// And clear exclusive.
-
-def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
+// And clear exclusive. The (int_aarch64_clrex imm) pattern is defined on the
+// CLREX instruction in AArch64InstrInfo.td.
//===----------------------------------
// Atomic cmpxchg for -O0
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e876ef7597454..c8a44ab4fbcce 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1636,7 +1636,8 @@ def : InstAlias<"tsb csync", (HINT 18), 1>, Requires<[HasTRACEV8_4]>;
// As far as LLVM is concerned this writes to the system's exclusive monitors.
let mayLoad = 1, mayStore = 1 in
-def CLREX : CRmSystemI<imm0_15, 0b010, "clrex">;
+def CLREX : CRmSystemI<imm32_0_15, 0b010, "clrex",
+ [(int_aarch64_clrex (i32 imm32_0_15:$CRm))]>;
// NOTE: ideally, this would have mayStore = 0, mayLoad = 0, but we cannot
// model patterns with sufficiently fine granularity.
diff --git a/llvm/test/Bitcode/aarch64-clrex-upgrade.ll b/llvm/test/Bitcode/aarch64-clrex-upgrade.ll
new file mode 100644
index 0000000000000..2c13bf98551b8
--- /dev/null
+++ b/llvm/test/Bitcode/aarch64-clrex-upgrade.ll
@@ -0,0 +1,14 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+
+; The nullary llvm.aarch64.clrex was given a CRm operand; the old form is
+; upgraded to pass the previous implicit CRm value of 15.
+
+define void @test_clrex() {
+; CHECK-LABEL: define void @test_clrex()
+; CHECK: call void @llvm.aarch64.clrex(i32 15)
+ call void @llvm.aarch64.clrex()
+ ret void
+}
+
+; CHECK: declare void @llvm.aarch64.clrex(i32)
+declare void @llvm.aarch64.clrex()
diff --git a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
index ccd191f163b01..11199a2423f8e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll
@@ -170,11 +170,11 @@ define dso_local void @test_clear() {
; CHECK: // %bb.0:
; CHECK-NEXT: clrex
; CHECK-NEXT: ret
- call void @llvm.aarch64.clrex()
+ call void @llvm.aarch64.clrex(i32 15)
ret void
}
-declare void @llvm.aarch64.clrex() nounwind
+declare void @llvm.aarch64.clrex(i32) nounwind
define dso_local i128 @test_load_acquire_i128(ptr %p) nounwind readonly {
; CHECK-LABEL: test_load_acquire_i128:
>From e10804ba3f0b9b47212c2521fd7358ab9452a847 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 5 Jun 2026 10:04:22 -0300
Subject: [PATCH 08/26] [aarch64] Add support for the __clrex MS intrinsic
The MSVC __clrex(crm) intrinsic clears the local exclusive monitor and emits
"clrex #crm". It lowers to the llvm.aarch64.clrex intrinsic, which now carries
the CRm immediate. Sema restricts crm to a constant in [0, 15].
---
clang/include/clang/Basic/BuiltinsAArch64.td | 2 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 8 ++++++++
clang/lib/Headers/arm64intr.h | 2 ++
clang/lib/Sema/SemaARM.cpp | 3 +++
clang/test/CodeGen/arm64-microsoft-intrinsics.c | 7 +++++++
clang/test/Sema/builtins-microsoft-arm64.c | 5 +++++
6 files changed, 27 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index b0af6376a8b0e..f4892f528c7e8 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -441,4 +441,6 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __stlxr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned short volatile *, unsigned short)">;
def __stlxr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned int volatile *, unsigned int)">;
def __stlxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned long long int volatile *, unsigned long long int)">;
+
+ def __clrex : AArch64NoPrefixTargetLibBuiltin<"void (unsigned char)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 6760328f5d59e..d68e727f21109 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -4802,6 +4802,14 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(F, {Builder.getInt32(15)});
}
+ if (BuiltinID == clang::AArch64::BI__clrex) {
+ // MSVC __clrex(crm) clears the exclusive monitor with the given CRm (a
+ // constant in [0, 15], enforced by Sema). Emit "clrex #crm".
+ Function *F = CGM.getIntrinsic(Intrinsic::aarch64_clrex);
+ Value *CRm = Builder.CreateZExt(EmitScalarExpr(E->getArg(0)), Int32Ty);
+ return Builder.CreateCall(F, {CRm});
+ }
+
if (BuiltinID == clang::AArch64::BI_ReadWriteBarrier)
return Builder.CreateFence(llvm::AtomicOrdering::SequentiallyConsistent,
llvm::SyncScope::SingleThread);
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 3bf6c2cb57091..2945eee4cb65a 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -75,6 +75,8 @@ unsigned __int8 __stlxr16(volatile unsigned __int16 *, unsigned __int16);
unsigned __int8 __stlxr32(volatile unsigned __int32 *, unsigned __int32);
unsigned __int8 __stlxr64(volatile unsigned __int64 *, unsigned __int64);
+void __clrex(unsigned __int8);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/lib/Sema/SemaARM.cpp b/clang/lib/Sema/SemaARM.cpp
index f38a35ec08fb0..e0ce4335cd74b 100644
--- a/clang/lib/Sema/SemaARM.cpp
+++ b/clang/lib/Sema/SemaARM.cpp
@@ -1212,6 +1212,9 @@ bool SemaARM::CheckAArch64BuiltinFunctionCall(const TargetInfo &TI,
return false;
}
+ if (BuiltinID == AArch64::BI__clrex)
+ return SemaRef.BuiltinConstantArgRange(TheCall, 0, 0, 15);
+
if (CheckNeonBuiltinFunctionCall(TI, BuiltinID, TheCall))
return true;
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index cab9101058db3..14f549443f304 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -852,6 +852,13 @@ unsigned char check__stlxr64(unsigned long long int volatile *p, unsigned long l
// CHECK-MSCOMPAT: trunc i32 %[[ST]] to i8
// CHECK-LINUX: error: call to undeclared function '__stlxr64'
+void check__clrex(void) {
+ __clrex(15);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}void @check__clrex(){{.*}}{
+// CHECK-MSCOMPAT: call void @llvm.aarch64.clrex(i32 15)
+// CHECK-LINUX: error: call to undeclared function '__clrex'
+
void test__stlr8(unsigned __int8 volatile *p, unsigned __int8 v)
{
__stlr8 (p, v);
diff --git a/clang/test/Sema/builtins-microsoft-arm64.c b/clang/test/Sema/builtins-microsoft-arm64.c
index 2093adbb349a4..a3da4dc5f8d4e 100644
--- a/clang/test/Sema/builtins-microsoft-arm64.c
+++ b/clang/test/Sema/builtins-microsoft-arm64.c
@@ -36,6 +36,11 @@ void check__svc(unsigned int x, double d, void *p, struct NonScalar s) {
__svc(1, s); // expected-error {{2nd argument to '__svc' must have integer, floating-point, or pointer type}}
}
+void check__clrex(unsigned char x) {
+ __clrex(16); // expected-error-re {{argument value {{.*}} is outside the valid range}}
+ __clrex(x); // expected-error {{argument to '__clrex' must be a constant integer}}
+}
+
void check__getReg(void) {
__getReg(-1); // expected-error-re {{argument value {{.*}} is outside the valid range}}
__getReg(32); // expected-error-re {{argument value {{.*}} is outside the valid range}}
>From dab068530f11292096fd287e421b8fd9aaddfe0e Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 10:52:29 -0300
Subject: [PATCH 09/26] [aarch64] Add support for cas builtin
The new intrinsic issues CAS* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __casX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 17 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 33 ++++++++++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 19 ++++++
.../lib/Target/AArch64/AArch64InstrFormats.td | 23 ++++---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 6 ++
.../test/CodeGen/AArch64/ms-intrinsics-cas.ll | 60 +++++++++++++++++++
6 files changed, 151 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 118f4a485ee61..86cb4e871eaf7 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -34,6 +34,23 @@ def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
def int_aarch64_clrex : Intrinsic<[], [llvm_i32_ty]>;
+// CAS intrinsics — emit CAS* (no acquire/release) directly, regardless of the
+// LSE target feature. CAS{H,b} comparand/value use i32; CAS{X}uses i64.
+// IntrWillReturn is intentionally omitted: on a target without LSE the encoding
+// is UNDEFINED and may trap, so these calls are not guaranteed to return.
+def int_aarch64_cas8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_cas16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_cas32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_cas64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 005f69b08a4d3..fed210ac1feee 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3262,6 +3262,25 @@ void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
#endif
}
+// The codegen-only CAS* "_cg" instructions are emitted by the MSVC __cas*
+// intrinsics regardless of the active -march, so they carry no LSE predicate.
+// When we emit textual assembly (-S/-save-temps) the assembler would otherwise
+// reject them, so bracket each with a .arch_extension directive enabling the
+// required feature. Returns the extension name ("lse") and the feature
+// controlling it, or an empty name for any other opcode.
+static std::pair<StringRef, unsigned>
+getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
+ switch (Opc) {
+ default:
+ return {StringRef(), 0};
+ case AArch64::CASB_cg:
+ case AArch64::CASH_cg:
+ case AArch64::CASW_cg:
+ case AArch64::CASX_cg:
+ return {"lse", AArch64::FeatureLSE};
+ }
+}
+
void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
@@ -3964,10 +3983,24 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
return;
+ // Codegen-only CAS*/SWP*/LDAPR* instructions are emitted without an LSE/RCPC
+ // predicate. When producing assembly, enable the needed extension around the
+ // instruction so the assembler accepts it. Skip this if the active subtarget
+ // already has the feature, otherwise the trailing "no" directive would
+ // wrongly disable it for the rest of the function.
+ auto [ArchExt, ArchExtFeature] =
+ getCodeGenOnlyAtomicArchExtension(MI->getOpcode());
+ bool ToggleArchExt = !ArchExt.empty() && !STI->hasFeature(ArchExtFeature);
+ if (ToggleArchExt)
+ TS->emitDirectiveArchExtension(ArchExt);
+
// Finally, do the automated lowerings for everything else.
MCInst TmpInst;
MCInstLowering.Lower(MI, TmpInst);
EmitToStreamer(*OutStreamer, TmpInst);
+
+ if (ToggleArchExt)
+ TS->emitDirectiveArchExtension((Twine("no") + ArchExt).str());
}
void AArch64AsmPrinter::recordIfImportCall(
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index c324b38e56bc0..9ec3c50ee8e32 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -567,6 +567,25 @@ let Predicates = [HasLSFE] in {
defm : LDBFPOPregister_patterns<"LDBFMIN", "atomic_load_fminimum">;
}
+// Unconditional patterns for the __cas* MSVC builtins to emit CAS* regardless
+// of the LSE subtarget feature. cas{b,h} use only the low bits of the
+// comparison register, so the zero extension masking emitted by CodeGen
+// (zext i8/i16 -> i32 -> 'and w,w,#mask') is redundant.
+def : Pat<(i32 (int_aarch64_cas8 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 255)), (and GPR32:$Rt, (i32 255)))),
+ (CASB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_cas8 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_cas16 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 65535)), (and GPR32:$Rt, (i32 65535)))),
+ (CASH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_cas16 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_cas32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASW_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_cas64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
+ (CASX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 68c80f5d112ae..bc22d7de4e1ce 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12359,7 +12359,7 @@ class CryptoRRRi2Tied<bits<1>op0, bits<2>op1, string asm>
// ST<OP>{<order>}[<size>] <Ws>, [<Xn|SP>]
// ST<OP>{<order>} <Xs>, [<Xn|SP>]
-let Predicates = [HasLSE], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
+let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
class BaseCASEncoding<dag oops, dag iops, string asm, string operands,
string cstr, list<dag> pattern>
: I<oops, iops, asm, operands, cstr, pattern> {
@@ -12380,7 +12380,6 @@ class BaseCASEncoding<dag oops, dag iops, string asm, string operands,
let Inst{14-10} = 0b11111;
let Inst{9-5} = Rn;
let Inst{4-0} = Rt;
- let Predicates = [HasLSE];
}
class BaseCAS<string order, string size, RegisterClass RC>
@@ -12391,13 +12390,23 @@ class BaseCAS<string order, string size, RegisterClass RC>
let NP = 1;
}
-multiclass CompareAndSwap<bits<1> Acq, bits<1> Rel, string order> {
- let Sz = 0b00, Acq = Acq, Rel = Rel in def B : BaseCAS<order, "b", GPR32>;
- let Sz = 0b01, Acq = Acq, Rel = Rel in def H : BaseCAS<order, "h", GPR32>;
- let Sz = 0b10, Acq = Acq, Rel = Rel in def W : BaseCAS<order, "", GPR32>;
- let Sz = 0b11, Acq = Acq, Rel = Rel in def X : BaseCAS<order, "", GPR64>;
+multiclass CompareAndSwap_impl<bits<1> Acq, bits<1> Rel, string order, list<Predicate> preds, string suffix> {
+ let Predicates = preds, Acq = Acq, Rel = Rel in {
+ let Sz = 0b00 in def "B" # suffix : BaseCAS<order, "b", GPR32>;
+ let Sz = 0b01 in def "H" # suffix : BaseCAS<order, "h", GPR32>;
+ let Sz = 0b10 in def "W" # suffix : BaseCAS<order, "", GPR32>;
+ let Sz = 0b11 in def "X" # suffix : BaseCAS<order, "", GPR64>;
+ }
}
+multiclass CompareAndSwap<bits<1> Acq, bits<1> Rel, string order>
+ : CompareAndSwap_impl<Acq, Rel, order, [HasLSE], "">;
+
+// For isCodeGenOnly use by MS intrinsics that must emit CAS* regardless
+// of the -march setting.
+multiclass CompareAndSwap_cg<bits<1> Acq, bits<1> Rel, string order>
+ : CompareAndSwap_impl<Acq, Rel, order, [], "_cg">;
+
class BaseCASP<string order, string size, RegisterOperand RC>
: BaseCASEncoding<(outs RC:$out),(ins RC:$Rs, RC:$Rt, GPR64sp:$Rn),
"casp" # order # size, "\t$Rs, $Rt, [$Rn]",
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index c8a44ab4fbcce..9a8e61881a638 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3114,6 +3114,12 @@ defm CASA : CompareAndSwap<1, 0, "a">;
defm CASL : CompareAndSwap<0, 1, "l">;
defm CASAL : CompareAndSwap<1, 1, "al">;
+// isCodeGenOnly=1: excluded from asm parser/disassembler, no HasLSE predicate.
+// Used by the __cas* MS intrinsics to force CAS* emission regardless of -march.
+let isCodeGenOnly = 1 in {
+ defm CAS : CompareAndSwap_cg<0, 0, "">;
+}
+
// v8.1 atomic CASP
defm CASP : CompareAndSwapPair<0, 0, "">;
defm CASPA : CompareAndSwapPair<1, 0, "a">;
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
new file mode 100644
index 0000000000000..cbaec2332b18d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-none-linux-gnu -fast-isel=0 -global-isel=false \
+; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+lse -fast-isel=0 \
+; RUN: -global-isel=false -verify-machineinstrs < %s | FileCheck %s
+
+; Tests for the __cas* MS builtins on AArch64. These lower to
+; llvm.aarch64.cas* intrinsics and must select the corresponding CAS instruction
+; regardless of whether +lse is present in the target features (the _cg ISel
+; variants have no feature predicate). When +lse is absent, the AsmPrinter
+; brackets the instruction with .arch_extension directives so that assembly
+; output re-assembles.
+
+define i32 @test_cas8(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_cas8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casb w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.cas8(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_cas16(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_cas16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: cash w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.cas16(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_cas32(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_cas32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: cas w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.cas32(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i64 @test_cas64(ptr %p, i64 %rs, i64 %rt) {
+; CHECK-LABEL: test_cas64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: cas x1, x2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.cas64(ptr %p, i64 %rs, i64 %rt)
+ ret i64 %r
+}
>From 7c0f36aaa594f28fbe35cb9de4a0846122f5af7e Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Wed, 15 Apr 2026 16:53:14 -0300
Subject: [PATCH 10/26] [aarch64] Add support for the __cas{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__cas8` - maps to CASB.
* `__cas16` - maps to CASH.
* `__cas32` - maps to CAS.
* `__cas64` - maps to CAS.
The emit is done using new intrisincs to issue CAS* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 41 ++++++++++++++
clang/lib/Headers/arm64intr.h | 9 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 56 +++++++++++++++++++
4 files changed, 111 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index f4892f528c7e8..aedc1c4fb3c02 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -443,4 +443,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __stlxr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned long long int volatile *, unsigned long long int)">;
def __clrex : AArch64NoPrefixTargetLibBuiltin<"void (unsigned char)">;
+
+ def __cas8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char, unsigned char)">;
+ def __cas16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
+ def __cas32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
+ def __cas64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index d68e727f21109..e9fb6ab01a181 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5378,6 +5378,47 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Builder.CreateTrunc(CI, ConvertType(E->getType()));
}
+ if (BuiltinID == AArch64::BI__cas8 || BuiltinID == AArch64::BI__cas16 ||
+ BuiltinID == AArch64::BI__cas32 || BuiltinID == AArch64::BI__cas64) {
+ unsigned IntrID;
+ llvm::Type *IntrArgTy;
+ switch (BuiltinID) {
+ case AArch64::BI__cas8:
+ IntrID = Intrinsic::aarch64_cas8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__cas16:
+ IntrID = Intrinsic::aarch64_cas16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__cas32:
+ IntrID = Intrinsic::aarch64_cas32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__cas64:
+ IntrID = Intrinsic::aarch64_cas64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
+ default:
+ llvm_unreachable("missing builtin ID in switch!");
+ }
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ Value *Comp = EmitScalarExpr(E->getArg(1));
+ Value *Val = EmitScalarExpr(E->getArg(2));
+ // For 8/16-bit we need to zext to GRP size
+ if (Comp->getType() != IntrArgTy)
+ Comp = Builder.CreateZExt(Comp, IntrArgTy);
+ if (Val->getType() != IntrArgTy)
+ Val = Builder.CreateZExt(Val, IntrArgTy);
+ Value *Result =
+ Builder.CreateCall(CGM.getIntrinsic(IntrID), {Ptr, Comp, Val});
+ // CAS{B/H} return i32 (zero-extended); truncate to declared type.
+ llvm::Type *RetTy = ConvertType(E->getType());
+ if (Result->getType() != RetTy)
+ Result = Builder.CreateTrunc(Result, RetTy);
+ return Result;
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 2945eee4cb65a..2540ac8bc5e4c 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -77,6 +77,15 @@ unsigned __int8 __stlxr64(volatile unsigned __int64 *, unsigned __int64);
void __clrex(unsigned __int8);
+unsigned __int8 __cas8(unsigned __int8 volatile *, unsigned __int8,
+ unsigned __int8);
+unsigned __int16 __cas16(unsigned __int16 volatile *, unsigned __int16,
+ unsigned __int16);
+unsigned __int32 __cas32(unsigned __int32 volatile *, unsigned __int32,
+ unsigned __int32);
+unsigned __int64 __cas64(unsigned __int64 volatile *, unsigned __int64,
+ unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 14f549443f304..06f523d993078 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -903,6 +903,62 @@ void test__stlr64(unsigned __int64 volatile *p, unsigned __int64 v)
// CHECK-MSCOMPAT: ret void
// CHECK-LINUX: error: call to undeclared function '__stlr64'
+unsigned char test__cas8(unsigned char volatile* t, unsigned char c, unsigned char v)
+{
+ return __cas8 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__cas8(ptr{{.*}}%t, i8{{.*}}%c, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i8, ptr %c.addr, align 1
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i8 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.cas8(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__cas8'
+
+unsigned short test__cas16(unsigned short volatile* t, unsigned short c, unsigned short v)
+{
+ return __cas16 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__cas16(ptr{{.*}}%t, i16{{.*}}%c, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i16, ptr %c.addr, align 2
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i16 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.cas16(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__cas16'
+
+unsigned int test__cas32(unsigned int volatile* t, unsigned int c, unsigned int v)
+{
+ return __cas32 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__cas32(ptr{{.*}}%t, i32{{.*}}%c, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i32, ptr %c.addr, align 4
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.cas32(ptr %[[TMPT]], i32 %[[TMPC]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__cas32'
+
+unsigned long long int test__cas64(unsigned long long int volatile* t,
+ unsigned long long int c,
+ unsigned long long int v)
+{
+ return __cas64 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__cas64(ptr{{.*}}%t, i64{{.*}}%c, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i64, ptr %c.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.cas64(ptr %[[TMPT]], i64 %[[TMPC]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__cas64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From fc0c4cf8d6b07d5b05cdce7538b8c6a44f8c7d2c Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 12:16:24 -0300
Subject: [PATCH 11/26] [aarch64] Add support for casa builtin
The new intrinsic issues CASA* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __casaX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 15 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 16 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-cas.ll | 48 +++++++++++++++++++
5 files changed, 84 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 86cb4e871eaf7..577dcc25fcf8d 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -51,6 +51,21 @@ def int_aarch64_cas64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
[IntrNoFree]>;
+// CASA intrinsics — emit CASA* (acquire, no release) directly, regardless of
+// the LSE target feature. Same type conventions as cas*.
+def int_aarch64_casa8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casa16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casa32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casa64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index fed210ac1feee..7693ae4bed6ec 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3277,6 +3277,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::CASH_cg:
case AArch64::CASW_cg:
case AArch64::CASX_cg:
+ case AArch64::CASAB_cg:
+ case AArch64::CASAH_cg:
+ case AArch64::CASAW_cg:
+ case AArch64::CASAX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 9ec3c50ee8e32..57b19d97b41a5 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -586,6 +586,22 @@ def : Pat<(i32 (int_aarch64_cas32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
def : Pat<(i64 (int_aarch64_cas64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
(CASX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+// Unconditional patterns for the __casa* MSVC builtins (acquire ordering).
+def : Pat<(i32 (int_aarch64_casa8 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 255)), (and GPR32:$Rt, (i32 255)))),
+ (CASAB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casa8 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASAB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casa16 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 65535)), (and GPR32:$Rt, (i32 65535)))),
+ (CASAH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casa16 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASAH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casa32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASAW_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_casa64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
+ (CASAX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 9a8e61881a638..1c21378a05789 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3118,6 +3118,7 @@ defm CASAL : CompareAndSwap<1, 1, "al">;
// Used by the __cas* MS intrinsics to force CAS* emission regardless of -march.
let isCodeGenOnly = 1 in {
defm CAS : CompareAndSwap_cg<0, 0, "">;
+ defm CASA : CompareAndSwap_cg<1, 0, "a">;
}
// v8.1 atomic CASP
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
index cbaec2332b18d..e2f54e72a55ff 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
@@ -58,3 +58,51 @@ define i64 @test_cas64(ptr %p, i64 %rs, i64 %rt) {
%r = call i64 @llvm.aarch64.cas64(ptr %p, i64 %rs, i64 %rt)
ret i64 %r
}
+
+define i32 @test_casa8(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casa8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casab w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casa8(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casa16(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casa16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casah w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casa16(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casa32(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casa32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casa w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casa32(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i64 @test_casa64(ptr %p, i64 %rs, i64 %rt) {
+; CHECK-LABEL: test_casa64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casa x1, x2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.casa64(ptr %p, i64 %rs, i64 %rt)
+ ret i64 %r
+}
>From 357204e3bcaedbe3c9cf775bc5cb150eb58f7b6a Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Wed, 15 Apr 2026 17:39:55 -0300
Subject: [PATCH 12/26] [aarch64] Add support for the __casa{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__casa8` - maps to CASAB.
* `__casa16` - maps to CASAH.
* `__casa32` - maps to CASA.
* `__casa64` - maps to CASA.
The emit is done using new intrisincs to issue CASA* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 ++++++-
clang/lib/Headers/arm64intr.h | 9 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 56 +++++++++++++++++++
4 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index aedc1c4fb3c02..5a209076ea52f 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -448,4 +448,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __cas16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
def __cas32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
def __cas64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
+ def __casa8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char, unsigned char)">;
+ def __casa16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
+ def __casa32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
+ def __casa64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index e9fb6ab01a181..ead7fa9700e2a 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5379,7 +5379,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
}
if (BuiltinID == AArch64::BI__cas8 || BuiltinID == AArch64::BI__cas16 ||
- BuiltinID == AArch64::BI__cas32 || BuiltinID == AArch64::BI__cas64) {
+ BuiltinID == AArch64::BI__cas32 || BuiltinID == AArch64::BI__cas64 ||
+ BuiltinID == AArch64::BI__casa8 || BuiltinID == AArch64::BI__casa16 ||
+ BuiltinID == AArch64::BI__casa32 || BuiltinID == AArch64::BI__casa64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5399,6 +5401,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_cas64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__casa8:
+ IntrID = Intrinsic::aarch64_casa8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casa16:
+ IntrID = Intrinsic::aarch64_casa16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casa32:
+ IntrID = Intrinsic::aarch64_casa32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casa64:
+ IntrID = Intrinsic::aarch64_casa64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 2540ac8bc5e4c..18e6cef556900 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -86,6 +86,15 @@ unsigned __int32 __cas32(unsigned __int32 volatile *, unsigned __int32,
unsigned __int64 __cas64(unsigned __int64 volatile *, unsigned __int64,
unsigned __int64);
+unsigned __int8 __casa8(unsigned __int8 volatile *, unsigned __int8,
+ unsigned __int8);
+unsigned __int16 __casa16(unsigned __int16 volatile *, unsigned __int16,
+ unsigned __int16);
+unsigned __int32 __casa32(unsigned __int32 volatile *, unsigned __int32,
+ unsigned __int32);
+unsigned __int64 __casa64(unsigned __int64 volatile *, unsigned __int64,
+ unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 06f523d993078..da8d2b136ae43 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -959,6 +959,62 @@ unsigned long long int test__cas64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__cas64'
+unsigned char test__casa8(unsigned char volatile* t, unsigned char c, unsigned char v)
+{
+ return __casa8 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__casa8(ptr{{.*}}%t, i8{{.*}}%c, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i8, ptr %c.addr, align 1
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i8 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casa8(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casa8'
+
+unsigned short test__casa16(unsigned short volatile* t, unsigned short c, unsigned short v)
+{
+ return __casa16 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__casa16(ptr{{.*}}%t, i16{{.*}}%c, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i16, ptr %c.addr, align 2
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i16 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casa16(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casa16'
+
+unsigned int test__casa32(unsigned int volatile* t, unsigned int c, unsigned int v)
+{
+ return __casa32 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__casa32(ptr{{.*}}%t, i32{{.*}}%c, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i32, ptr %c.addr, align 4
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casa32(ptr %[[TMPT]], i32 %[[TMPC]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casa32'
+
+unsigned long long int test__casa64(unsigned long long int volatile* t,
+ unsigned long long int c,
+ unsigned long long int v)
+{
+ return __casa64 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__casa64(ptr{{.*}}%t, i64{{.*}}%c, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i64, ptr %c.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.casa64(ptr %[[TMPT]], i64 %[[TMPC]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casa64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From 78d4ae9df82a01e309f57a6f4b6d73dac2c5d719 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 13:06:36 -0300
Subject: [PATCH 13/26] [aarch64] Add support for casl builtin
The new intrinsic issues CASL* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __caslX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 15 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 16 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-cas.ll | 48 +++++++++++++++++++
5 files changed, 84 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 577dcc25fcf8d..8e2fa394735c2 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -66,6 +66,21 @@ def int_aarch64_casa64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
[IntrNoFree]>;
+// CASL intrinsics — emit CASL* (release) directly, regardless of the LSE
+// target feature. Same type conventions as cas*.
+def int_aarch64_casl8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casl16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casl32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casl64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 7693ae4bed6ec..984a969c9fb05 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3281,6 +3281,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::CASAH_cg:
case AArch64::CASAW_cg:
case AArch64::CASAX_cg:
+ case AArch64::CASLB_cg:
+ case AArch64::CASLH_cg:
+ case AArch64::CASLW_cg:
+ case AArch64::CASLX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 57b19d97b41a5..12f108a77b518 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -602,6 +602,22 @@ def : Pat<(i32 (int_aarch64_casa32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
def : Pat<(i64 (int_aarch64_casa64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
(CASAX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+// Unconditional patterns for the __casl* MSVC builtins (release ordering).
+def : Pat<(i32 (int_aarch64_casl8 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 255)), (and GPR32:$Rt, (i32 255)))),
+ (CASLB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casl8 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASLB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casl16 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 65535)), (and GPR32:$Rt, (i32 65535)))),
+ (CASLH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casl16 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASLH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casl32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASLW_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_casl64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
+ (CASLX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 1c21378a05789..4c4ddebcf727c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3119,6 +3119,7 @@ defm CASAL : CompareAndSwap<1, 1, "al">;
let isCodeGenOnly = 1 in {
defm CAS : CompareAndSwap_cg<0, 0, "">;
defm CASA : CompareAndSwap_cg<1, 0, "a">;
+ defm CASL : CompareAndSwap_cg<0, 1, "l">;
}
// v8.1 atomic CASP
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
index e2f54e72a55ff..f17b136813c0e 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
@@ -106,3 +106,51 @@ define i64 @test_casa64(ptr %p, i64 %rs, i64 %rt) {
%r = call i64 @llvm.aarch64.casa64(ptr %p, i64 %rs, i64 %rt)
ret i64 %r
}
+
+define i32 @test_casl8(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casl8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: caslb w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casl8(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casl16(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casl16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: caslh w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casl16(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casl32(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casl32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casl w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casl32(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i64 @test_casl64(ptr %p, i64 %rs, i64 %rt) {
+; CHECK-LABEL: test_casl64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casl x1, x2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.casl64(ptr %p, i64 %rs, i64 %rt)
+ ret i64 %r
+}
>From b0b62dd01b97ece754dd07c7dead5cd5a44bbe07 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 16 Apr 2026 09:15:28 -0300
Subject: [PATCH 14/26] [aarch64] Add support for the __casl{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__casl8` - maps to CASLB.
* `__casl16` - maps to CASLH.
* `__casl32` - maps to CASL.
* `__casl64` - maps to CASL.
The emit is done using new intrisincs to issue CASL* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 ++++++-
clang/lib/Headers/arm64intr.h | 9 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 56 +++++++++++++++++++
4 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 5a209076ea52f..7ab646d36047d 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -452,4 +452,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __casa16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
def __casa32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
def __casa64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
+ def __casl8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char, unsigned char)">;
+ def __casl16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
+ def __casl32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
+ def __casl64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index ead7fa9700e2a..2ad363bf06afb 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5381,7 +5381,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
if (BuiltinID == AArch64::BI__cas8 || BuiltinID == AArch64::BI__cas16 ||
BuiltinID == AArch64::BI__cas32 || BuiltinID == AArch64::BI__cas64 ||
BuiltinID == AArch64::BI__casa8 || BuiltinID == AArch64::BI__casa16 ||
- BuiltinID == AArch64::BI__casa32 || BuiltinID == AArch64::BI__casa64) {
+ BuiltinID == AArch64::BI__casa32 || BuiltinID == AArch64::BI__casa64 ||
+ BuiltinID == AArch64::BI__casl8 || BuiltinID == AArch64::BI__casl16 ||
+ BuiltinID == AArch64::BI__casl32 || BuiltinID == AArch64::BI__casl64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5417,6 +5419,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_casa64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__casl8:
+ IntrID = Intrinsic::aarch64_casl8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casl16:
+ IntrID = Intrinsic::aarch64_casl16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casl32:
+ IntrID = Intrinsic::aarch64_casl32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casl64:
+ IntrID = Intrinsic::aarch64_casl64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 18e6cef556900..75c56c0ad8303 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -95,6 +95,15 @@ unsigned __int32 __casa32(unsigned __int32 volatile *, unsigned __int32,
unsigned __int64 __casa64(unsigned __int64 volatile *, unsigned __int64,
unsigned __int64);
+unsigned __int8 __casl8(unsigned __int8 volatile *, unsigned __int8,
+ unsigned __int8);
+unsigned __int16 __casl16(unsigned __int16 volatile *, unsigned __int16,
+ unsigned __int16);
+unsigned __int32 __casl32(unsigned __int32 volatile *, unsigned __int32,
+ unsigned __int32);
+unsigned __int64 __casl64(unsigned __int64 volatile *, unsigned __int64,
+ unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index da8d2b136ae43..658820c2f5d67 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1015,6 +1015,62 @@ unsigned long long int test__casa64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__casa64'
+unsigned char test__casl8(unsigned char volatile* t, unsigned char c, unsigned char v)
+{
+ return __casl8 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__casl8(ptr{{.*}}%t, i8{{.*}}%c, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i8, ptr %c.addr, align 1
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i8 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casl8(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casl8'
+
+unsigned short test__casl16(unsigned short volatile* t, unsigned short c, unsigned short v)
+{
+ return __casl16 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__casl16(ptr{{.*}}%t, i16{{.*}}%c, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i16, ptr %c.addr, align 2
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i16 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casl16(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casl16'
+
+unsigned int test__casl32(unsigned int volatile* t, unsigned int c, unsigned int v)
+{
+ return __casl32 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__casl32(ptr{{.*}}%t, i32{{.*}}%c, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i32, ptr %c.addr, align 4
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casl32(ptr %[[TMPT]], i32 %[[TMPC]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casl32'
+
+unsigned long long int test__casl64(unsigned long long int volatile* t,
+ unsigned long long int c,
+ unsigned long long int v)
+{
+ return __casl64 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__casl64(ptr{{.*}}%t, i64{{.*}}%c, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i64, ptr %c.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.casl64(ptr %[[TMPT]], i64 %[[TMPC]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casl64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From d1436d7a24e54a970fd2e4141a4a06904e6f252d Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 13:28:11 -0300
Subject: [PATCH 15/26] [aarch64] Add support for casal builtin
The new intrinsic issues CASAL* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __casalX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 15 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 16 +++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-cas.ll | 48 +++++++++++++++++++
5 files changed, 84 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 8e2fa394735c2..c29860dc85db6 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -81,6 +81,21 @@ def int_aarch64_casl64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
[IntrNoFree]>;
+// CASAL intrinsics — emit CASAL* (acquire/release) directly, regardless of the LSE
+// target feature. Same type conventions as cas*.
+def int_aarch64_casal8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casal16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casal32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_casal64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 984a969c9fb05..1ad285daa745c 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3285,6 +3285,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::CASLH_cg:
case AArch64::CASLW_cg:
case AArch64::CASLX_cg:
+ case AArch64::CASALB_cg:
+ case AArch64::CASALH_cg:
+ case AArch64::CASALW_cg:
+ case AArch64::CASALX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 12f108a77b518..f6fb0433cf7f5 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -618,6 +618,22 @@ def : Pat<(i32 (int_aarch64_casl32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
def : Pat<(i64 (int_aarch64_casl64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
(CASLX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+// Unconditional patterns for the __casal* MSVC builtins (acquire/release ordering).
+def : Pat<(i32 (int_aarch64_casal8 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 255)), (and GPR32:$Rt, (i32 255)))),
+ (CASALB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casal8 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASALB_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casal16 GPR64sp:$Rn,
+ (and GPR32:$Rs, (i32 65535)), (and GPR32:$Rt, (i32 65535)))),
+ (CASALH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casal16 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASALH_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_casal32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
+ (CASALW_cg GPR32:$Rs, GPR32:$Rt, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_casal64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
+ (CASALX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 4c4ddebcf727c..f0e38ede51be0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3120,6 +3120,7 @@ let isCodeGenOnly = 1 in {
defm CAS : CompareAndSwap_cg<0, 0, "">;
defm CASA : CompareAndSwap_cg<1, 0, "a">;
defm CASL : CompareAndSwap_cg<0, 1, "l">;
+ defm CASAL : CompareAndSwap_cg<1, 1, "al">;
}
// v8.1 atomic CASP
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
index f17b136813c0e..b113f4ea6d093 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-cas.ll
@@ -154,3 +154,51 @@ define i64 @test_casl64(ptr %p, i64 %rs, i64 %rt) {
%r = call i64 @llvm.aarch64.casl64(ptr %p, i64 %rs, i64 %rt)
ret i64 %r
}
+
+define i32 @test_casal8(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casal8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casalb w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casal8(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casal16(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casal16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casalh w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casal16(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @test_casal32(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: test_casal32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casal w1, w2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov w0, w1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.casal32(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i64 @test_casal64(ptr %p, i64 %rs, i64 %rt) {
+; CHECK-LABEL: test_casal64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: casal x1, x2, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: mov x0, x1
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.casal64(ptr %p, i64 %rs, i64 %rt)
+ ret i64 %r
+}
>From 8eaeb6d072b5db0fe0e08cf988d63f817ee41d1b Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 16 Apr 2026 10:47:20 -0300
Subject: [PATCH 16/26] [aarch64] Add support for the __casal{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__casal8` - maps to CASALB.
* `__casal16` - maps to CASALH.
* `__casal32` - maps to CASAL.
* `__casal64` - maps to CASAL.
The emit is done using new intrisincs to issue CASAL* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 ++++++-
clang/lib/Headers/arm64intr.h | 9 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 56 +++++++++++++++++++
4 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 7ab646d36047d..6a53f2379c1a5 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -456,4 +456,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __casl16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
def __casl32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
def __casl64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
+ def __casal8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char, unsigned char)">;
+ def __casal16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
+ def __casal32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
+ def __casal64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 2ad363bf06afb..55c5918bcaba1 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5383,7 +5383,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__casa8 || BuiltinID == AArch64::BI__casa16 ||
BuiltinID == AArch64::BI__casa32 || BuiltinID == AArch64::BI__casa64 ||
BuiltinID == AArch64::BI__casl8 || BuiltinID == AArch64::BI__casl16 ||
- BuiltinID == AArch64::BI__casl32 || BuiltinID == AArch64::BI__casl64) {
+ BuiltinID == AArch64::BI__casl32 || BuiltinID == AArch64::BI__casl64 ||
+ BuiltinID == AArch64::BI__casal8 || BuiltinID == AArch64::BI__casal16 ||
+ BuiltinID == AArch64::BI__casal32 || BuiltinID == AArch64::BI__casal64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5435,6 +5437,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_casl64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__casal8:
+ IntrID = Intrinsic::aarch64_casal8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casal16:
+ IntrID = Intrinsic::aarch64_casal16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casal32:
+ IntrID = Intrinsic::aarch64_casal32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__casal64:
+ IntrID = Intrinsic::aarch64_casal64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 75c56c0ad8303..ca0fc81742384 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -104,6 +104,15 @@ unsigned __int32 __casl32(unsigned __int32 volatile *, unsigned __int32,
unsigned __int64 __casl64(unsigned __int64 volatile *, unsigned __int64,
unsigned __int64);
+unsigned __int8 __casal8(unsigned __int8 volatile *, unsigned __int8,
+ unsigned __int8);
+unsigned __int16 __casal16(unsigned __int16 volatile *, unsigned __int16,
+ unsigned __int16);
+unsigned __int32 __casal32(unsigned __int32 volatile *, unsigned __int32,
+ unsigned __int32);
+unsigned __int64 __casal64(unsigned __int64 volatile *, unsigned __int64,
+ unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 658820c2f5d67..1cc3d0deb3023 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1071,6 +1071,62 @@ unsigned long long int test__casl64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__casl64'
+unsigned char test__casal8(unsigned char volatile* t, unsigned char c, unsigned char v)
+{
+ return __casal8 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__casal8(ptr{{.*}}%t, i8{{.*}}%c, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i8, ptr %c.addr, align 1
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i8 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casal8(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casal8'
+
+unsigned short test__casal16(unsigned short volatile* t, unsigned short c, unsigned short v)
+{
+ return __casal16 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__casal16(ptr{{.*}}%t, i16{{.*}}%c, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i16, ptr %c.addr, align 2
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTC:[0-9]+]] = zext i16 %[[TMPC]] to i32
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casal16(ptr %[[TMPT]], i32 %[[ZEXTC]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[RETT:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[RETT]]
+// CHECK-LINUX: error: call to undeclared function '__casal16'
+
+unsigned int test__casal32(unsigned int volatile* t, unsigned int c, unsigned int v)
+{
+ return __casal32 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__casal32(ptr{{.*}}%t, i32{{.*}}%c, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i32, ptr %c.addr, align 4
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.casal32(ptr %[[TMPT]], i32 %[[TMPC]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casal32'
+
+unsigned long long int test__casal64(unsigned long long int volatile* t,
+ unsigned long long int c,
+ unsigned long long int v)
+{
+ return __casal64 (t, c, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__casal64(ptr{{.*}}%t, i64{{.*}}%c, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPC:[0-9]+]] = load i64, ptr %c.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.casal64(ptr %[[TMPT]], i64 %[[TMPC]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__casal64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From a47d82ac58e135d65681a61d7eb90432ad7d6e7b Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 15:33:25 -0300
Subject: [PATCH 17/26] [aarch64] Add support for swp builtin
The new intrinsic issues SWP* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __swpX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 17 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 16 ++++++
.../lib/Target/AArch64/AArch64InstrFormats.td | 21 +++++--
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 6 ++
.../test/CodeGen/AArch64/ms-intrinsics-swp.ll | 56 +++++++++++++++++++
6 files changed, 114 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index c29860dc85db6..f7c1533518ed1 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -96,6 +96,23 @@ def int_aarch64_casal64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
[IntrNoFree]>;
+// SWP intrinsics — emit SWP* (no acquire/release) directly, regardless of the
+// LSE target feature. swp{b,h} value uses i32; swp64 uses i64.
+// IntrWillReturn is intentionally omitted: on a target without LSE the encoding
+// is UNDEFINED and may trap, so these calls are not guaranteed to return.
+def int_aarch64_swp8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swp16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swp32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swp64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 1ad285daa745c..225b503b74a0a 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3289,6 +3289,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::CASALH_cg:
case AArch64::CASALW_cg:
case AArch64::CASALX_cg:
+ case AArch64::SWPB_cg:
+ case AArch64::SWPH_cg:
+ case AArch64::SWPW_cg:
+ case AArch64::SWPX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index f6fb0433cf7f5..08d187363828c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -634,6 +634,22 @@ def : Pat<(i32 (int_aarch64_casal32 GPR64sp:$Rn, GPR32:$Rs, GPR32:$Rt)),
def : Pat<(i64 (int_aarch64_casal64 GPR64sp:$Rn, GPR64:$Rs, GPR64:$Rt)),
(CASALX_cg GPR64:$Rs, GPR64:$Rt, GPR64sp:$Rn)>;
+// Unconditional patterns for the __swp* MSVC builtins (no ordering).
+// As with swp{b,h}, the zero-extension mask may survive or be eliminated by the
+// DAG combiner, so both masked and plain forms are needed.
+def : Pat<(i32 (int_aarch64_swp8 GPR64sp:$Rn, (and GPR32:$Rs, (i32 255)))),
+ (SWPB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swp8 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swp16 GPR64sp:$Rn, (and GPR32:$Rs, (i32 65535)))),
+ (SWPH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swp16 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swp32 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPW_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_swp64 GPR64sp:$Rn, GPR64:$Rs)),
+ (SWPX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index bc22d7de4e1ce..f56202d7a25f1 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -12487,16 +12487,25 @@ class BaseSWP<string order, string size, RegisterClass RC>
let Inst{11-10} = 0b00;
let Inst{9-5} = Rn;
let Inst{4-0} = Rt;
- let Predicates = [HasLSE];
}
-multiclass Swap<bits<1> Acq, bits<1> Rel, string order> {
- let Sz = 0b00, Acq = Acq, Rel = Rel in def B : BaseSWP<order, "b", GPR32>;
- let Sz = 0b01, Acq = Acq, Rel = Rel in def H : BaseSWP<order, "h", GPR32>;
- let Sz = 0b10, Acq = Acq, Rel = Rel in def W : BaseSWP<order, "", GPR32>;
- let Sz = 0b11, Acq = Acq, Rel = Rel in def X : BaseSWP<order, "", GPR64>;
+multiclass Swap_impl<bits<1> Acq, bits<1> Rel, string order, list<Predicate> preds, string suffix> {
+ let Predicates = preds, Acq = Acq, Rel = Rel in {
+ let Sz = 0b00 in def "B" # suffix : BaseSWP<order, "b", GPR32>;
+ let Sz = 0b01 in def "H" # suffix : BaseSWP<order, "h", GPR32>;
+ let Sz = 0b10 in def "W" # suffix : BaseSWP<order, "", GPR32>;
+ let Sz = 0b11 in def "X" # suffix : BaseSWP<order, "", GPR64>;
+ }
}
+multiclass Swap<bits<1> Acq, bits<1> Rel, string order>
+ : Swap_impl<Acq, Rel, order, [HasLSE], "">;
+
+// Like Swap but without the HasLSE predicate — for isCodeGenOnly use by
+// MS intrinsics that must emit SWP* regardless of the -march setting.
+multiclass Swap_cg<bits<1> Acq, bits<1> Rel, string order>
+ : Swap_impl<Acq, Rel, order, [], "_cg">;
+
// v9.6a swap operations
class BaseSWPLSUI<string order, RegisterClass RC>
: I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "swpt" # order,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f0e38ede51be0..4eff622cd7435 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3149,6 +3149,12 @@ defm SWPA : Swap<1, 0, "a">;
defm SWPL : Swap<0, 1, "l">;
defm SWPAL : Swap<1, 1, "al">;
+// isCodeGenOnly=1: excluded from asm parser/disassembler, no HasLSE predicate.
+// Used by the __swp* MS intrinsics to force SWP* emission regardless of -march.
+let isCodeGenOnly = 1 in {
+ defm SWP : Swap_cg<0, 0, "">;
+}
+
// v9.6a atomic swap (FEAT_LSUI)
let Predicates = [HasLSUI] in {
defm SWPT : SwapLSUI<0, 0, "">;
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
new file mode 100644
index 0000000000000..207f81d54eac2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-windows-msvc -fast-isel=0 -global-isel=false \
+; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NOLSE
+; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+lse -fast-isel=0 \
+; RUN: -global-isel=false -verify-machineinstrs < %s | FileCheck %s
+
+; Tests for the __swp* MS builtins on AArch64. These lower to
+; llvm.aarch64.swp* intrinsics and must select SWP* regardless of whether
+; +lse is present in the target features (the _cg ISel variants have no
+; feature predicate). When +lse is absent, the AsmPrinter brackets the
+; instruction with .arch_extension directives so that assembly output
+; re-assembles.
+
+define i32 @test_swp8(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swp8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpb w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swp8(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swp16(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swp16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swph w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swp16(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swp32(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swp32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swp w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swp32(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i64 @test_swp64(ptr %p, i64 %v) {
+; CHECK-LABEL: test_swp64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swp x1, x0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.swp64(ptr %p, i64 %v)
+ ret i64 %r
+}
>From c85df29b0e10498b8d9ff18d5efed2f65d3c400e Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 17 Apr 2026 10:05:23 -0300
Subject: [PATCH 18/26] [aarch64] Add support for the __swp{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__swp8` - maps to SWPB.
* `__swp16` - maps to SWPH.
* `__swp32` - maps to SWP.
* `__swp64` - maps to SWP.
The emit is done using new intrisincs to issue SWP* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 36 ++++++++++++++
clang/lib/Headers/arm64intr.h | 5 ++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 49 +++++++++++++++++++
4 files changed, 95 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 6a53f2379c1a5..dd14615d161d4 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -460,4 +460,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __casal16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short, unsigned short)">;
def __casal32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int, unsigned int)">;
def __casal64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int, unsigned long long int)">;
+
+ def __swp8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __swp16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
+ def __swp32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
+ def __swp64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 55c5918bcaba1..9643f3f092868 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5473,6 +5473,42 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Result;
}
+ if (BuiltinID == AArch64::BI__swp8 || BuiltinID == AArch64::BI__swp16 ||
+ BuiltinID == AArch64::BI__swp32 || BuiltinID == AArch64::BI__swp64) {
+ unsigned IntrID;
+ llvm::Type *IntrArgTy;
+ switch (BuiltinID) {
+ case AArch64::BI__swp8:
+ IntrID = Intrinsic::aarch64_swp8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swp16:
+ IntrID = Intrinsic::aarch64_swp16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swp32:
+ IntrID = Intrinsic::aarch64_swp32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swp64:
+ IntrID = Intrinsic::aarch64_swp64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
+ default:
+ llvm_unreachable("missing builtin ID in switch!");
+ }
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ Value *Val = EmitScalarExpr(E->getArg(1));
+ if (Val->getType() != IntrArgTy)
+ Val = Builder.CreateZExt(Val, IntrArgTy);
+ Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntrID), {Ptr, Val});
+ // SWP{B/H} return i32 (zero-extended); truncate to declared type.
+ llvm::Type *RetTy = ConvertType(E->getType());
+ if (Result->getType() != RetTy)
+ Result = Builder.CreateTrunc(Result, RetTy);
+ return Result;
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index ca0fc81742384..49b6ca312b8e5 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -113,6 +113,11 @@ unsigned __int32 __casal32(unsigned __int32 volatile *, unsigned __int32,
unsigned __int64 __casal64(unsigned __int64 volatile *, unsigned __int64,
unsigned __int64);
+unsigned __int8 __swp8(unsigned __int8 volatile *, unsigned __int8);
+unsigned __int16 __swp16(unsigned __int16 volatile *, unsigned __int16);
+unsigned __int32 __swp32(unsigned __int32 volatile *, unsigned __int32);
+unsigned __int64 __swp64(unsigned __int64 volatile *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 1cc3d0deb3023..cd21646e031e5 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1127,6 +1127,55 @@ unsigned long long int test__casal64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__casal64'
+unsigned char test__swp8(unsigned char volatile* t, unsigned char v)
+{
+ return __swp8(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__swp8(ptr{{.*}}%t, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swp8(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swp8'
+
+unsigned short test__swp16(unsigned short volatile* t, unsigned short v)
+{
+ return __swp16(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__swp16(ptr{{.*}}%t, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swp16(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swp16'
+
+unsigned int test__swp32(unsigned int volatile* t, unsigned int v)
+{
+ return __swp32(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__swp32(ptr{{.*}}%t, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swp32(ptr %[[TMPT]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swp32'
+
+unsigned long long int test__swp64(unsigned long long int volatile* t,
+ unsigned long long int v)
+{
+ return __swp64(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__swp64(ptr{{.*}}%t, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.swp64(ptr %[[TMPT]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swp64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From cea0ffa1c2daac1a10738142a4b6af9120c02294 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 16:27:26 -0300
Subject: [PATCH 19/26] [aarch64] Add support for swpa builtin
The new intrinsic issues SWPA* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __swpaX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 14 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 13 ++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-swp.ll | 44 +++++++++++++++++++
5 files changed, 76 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index f7c1533518ed1..2bbe1205d89e9 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -113,6 +113,20 @@ def int_aarch64_swp64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty],
[IntrNoFree]>;
+// SWPA intrinsics — emit SWPA (acquire) directly, regardless of the LSE target
+def int_aarch64_swpa8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpa16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpa32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpa64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 225b503b74a0a..5eb5e7e4d8f3a 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3293,6 +3293,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::SWPH_cg:
case AArch64::SWPW_cg:
case AArch64::SWPX_cg:
+ case AArch64::SWPAB_cg:
+ case AArch64::SWPAH_cg:
+ case AArch64::SWPAW_cg:
+ case AArch64::SWPAX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 08d187363828c..ce12ff7ad8bee 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -650,6 +650,19 @@ def : Pat<(i32 (int_aarch64_swp32 GPR64sp:$Rn, GPR32:$Rs)),
def : Pat<(i64 (int_aarch64_swp64 GPR64sp:$Rn, GPR64:$Rs)),
(SWPX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpa8 GPR64sp:$Rn, (and GPR32:$Rs, (i32 255)))),
+ (SWPAB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpa8 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPAB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpa16 GPR64sp:$Rn, (and GPR32:$Rs, (i32 65535)))),
+ (SWPAH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpa16 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPAH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpa32 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPAW_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_swpa64 GPR64sp:$Rn, GPR64:$Rs)),
+ (SWPAX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 4eff622cd7435..41510b6e4edca 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3153,6 +3153,7 @@ defm SWPAL : Swap<1, 1, "al">;
// Used by the __swp* MS intrinsics to force SWP* emission regardless of -march.
let isCodeGenOnly = 1 in {
defm SWP : Swap_cg<0, 0, "">;
+ defm SWPA : Swap_cg<1, 0, "a">;
}
// v9.6a atomic swap (FEAT_LSUI)
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
index 207f81d54eac2..acc5ee76fc0c1 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
@@ -54,3 +54,47 @@ define i64 @test_swp64(ptr %p, i64 %v) {
%r = call i64 @llvm.aarch64.swp64(ptr %p, i64 %v)
ret i64 %r
}
+
+define i32 @test_swpa8(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpa8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpab w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpa8(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpa16(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpa16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpah w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpa16(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpa32(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpa32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpa w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpa32(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i64 @test_swpa64(ptr %p, i64 %v) {
+; CHECK-LABEL: test_swpa64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpa x1, x0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.swpa64(ptr %p, i64 %v)
+ ret i64 %r
+}
>From 5a8a9582b1a6c821093c2fdb00cd02de035f51b2 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 17 Apr 2026 12:41:03 -0300
Subject: [PATCH 20/26] [aarch64] Add support for the __swpa{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__swpa8` - maps to SWPAB.
* `__swpa16` - maps to SWPAH.
* `__swpa32` - maps to SWPA.
* `__swpa64` - maps to SWPA.
The emit is done using new intrisincs to issue SWPA* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 +++++++-
clang/lib/Headers/arm64intr.h | 5 ++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 49 +++++++++++++++++++
4 files changed, 77 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index dd14615d161d4..5ad36299a34e5 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -465,4 +465,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __swp16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
def __swp32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
def __swp64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
+ def __swpa8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __swpa16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
+ def __swpa32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
+ def __swpa64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 9643f3f092868..dacd3a7d4b2b8 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5474,7 +5474,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
}
if (BuiltinID == AArch64::BI__swp8 || BuiltinID == AArch64::BI__swp16 ||
- BuiltinID == AArch64::BI__swp32 || BuiltinID == AArch64::BI__swp64) {
+ BuiltinID == AArch64::BI__swp32 || BuiltinID == AArch64::BI__swp64 ||
+ BuiltinID == AArch64::BI__swpa8 || BuiltinID == AArch64::BI__swpa16 ||
+ BuiltinID == AArch64::BI__swpa32 || BuiltinID == AArch64::BI__swpa64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5494,6 +5496,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_swp64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__swpa8:
+ IntrID = Intrinsic::aarch64_swpa8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpa16:
+ IntrID = Intrinsic::aarch64_swpa16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpa32:
+ IntrID = Intrinsic::aarch64_swpa32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpa64:
+ IntrID = Intrinsic::aarch64_swpa64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 49b6ca312b8e5..21d7029d82533 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -118,6 +118,11 @@ unsigned __int16 __swp16(unsigned __int16 volatile *, unsigned __int16);
unsigned __int32 __swp32(unsigned __int32 volatile *, unsigned __int32);
unsigned __int64 __swp64(unsigned __int64 volatile *, unsigned __int64);
+unsigned __int8 __swpa8(unsigned __int8 volatile *, unsigned __int8);
+unsigned __int16 __swpa16(unsigned __int16 volatile *, unsigned __int16);
+unsigned __int32 __swpa32(unsigned __int32 volatile *, unsigned __int32);
+unsigned __int64 __swpa64(unsigned __int64 volatile *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index cd21646e031e5..282318f915258 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1176,6 +1176,55 @@ unsigned long long int test__swp64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__swp64'
+unsigned char test__swpa8(unsigned char volatile* t, unsigned char v)
+{
+ return __swpa8(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__swpa8(ptr{{.*}}%t, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpa8(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpa8'
+
+unsigned short test__swpa16(unsigned short volatile* t, unsigned short v)
+{
+ return __swpa16(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__swpa16(ptr{{.*}}%t, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpa16(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpa16'
+
+unsigned int test__swpa32(unsigned int volatile* t, unsigned int v)
+{
+ return __swpa32(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__swpa32(ptr{{.*}}%t, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpa32(ptr %[[TMPT]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpa32'
+
+unsigned long long int test__swpa64(unsigned long long int volatile* t,
+ unsigned long long int v)
+{
+ return __swpa64(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__swpa64(ptr{{.*}}%t, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.swpa64(ptr %[[TMPT]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpa64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From f8a69b4f291fd166aa79c0285fb89b4fea9dd010 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 18:06:06 -0300
Subject: [PATCH 21/26] [aarch64] Add support for swpl builtin
The new intrinsic issues SWPL* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __swplX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 14 ++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 13 ++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-swp.ll | 44 +++++++++++++++++++
5 files changed, 76 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 2bbe1205d89e9..1bb9c0c0b7e03 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -127,6 +127,20 @@ def int_aarch64_swpa64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty],
[IntrNoFree]>;
+// SWPL intrinsics — emit SWPL (release) directly, regardless of the LSE target
+def int_aarch64_swpl8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpl16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpl32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpl64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 5eb5e7e4d8f3a..d5091c8955e4a 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3297,6 +3297,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::SWPAH_cg:
case AArch64::SWPAW_cg:
case AArch64::SWPAX_cg:
+ case AArch64::SWPLB_cg:
+ case AArch64::SWPLH_cg:
+ case AArch64::SWPLW_cg:
+ case AArch64::SWPLX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index ce12ff7ad8bee..2496a8990da61 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -663,6 +663,19 @@ def : Pat<(i32 (int_aarch64_swpa32 GPR64sp:$Rn, GPR32:$Rs)),
def : Pat<(i64 (int_aarch64_swpa64 GPR64sp:$Rn, GPR64:$Rs)),
(SWPAX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpl8 GPR64sp:$Rn, (and GPR32:$Rs, (i32 255)))),
+ (SWPLB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpl8 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPLB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpl16 GPR64sp:$Rn, (and GPR32:$Rs, (i32 65535)))),
+ (SWPLH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpl16 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPLH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpl32 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPLW_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_swpl64 GPR64sp:$Rn, GPR64:$Rs)),
+ (SWPLX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 41510b6e4edca..a599433ab4df8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3154,6 +3154,7 @@ defm SWPAL : Swap<1, 1, "al">;
let isCodeGenOnly = 1 in {
defm SWP : Swap_cg<0, 0, "">;
defm SWPA : Swap_cg<1, 0, "a">;
+ defm SWPL : Swap_cg<0, 1, "l">;
}
// v9.6a atomic swap (FEAT_LSUI)
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
index acc5ee76fc0c1..095e035cf1c17 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
@@ -98,3 +98,47 @@ define i64 @test_swpa64(ptr %p, i64 %v) {
%r = call i64 @llvm.aarch64.swpa64(ptr %p, i64 %v)
ret i64 %r
}
+
+define i32 @test_swpl8(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpl8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swplb w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpl8(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpl16(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpl16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swplh w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpl16(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpl32(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpl32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpl w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpl32(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i64 @test_swpl64(ptr %p, i64 %v) {
+; CHECK-LABEL: test_swpl64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpl x1, x0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.swpl64(ptr %p, i64 %v)
+ ret i64 %r
+}
>From e8b5d819deecc2b84e29b99bb10d93058769760d Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 17 Apr 2026 13:02:47 -0300
Subject: [PATCH 22/26] [aarch64] Add support for the __swpl{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__swpl8` - maps to SWPLB.
* `__swpl16` - maps to SWPLH.
* `__swpl32` - maps to SWPL.
* `__swpl64` - maps to SWPL.
The emit is done using new intrisincs to issue SWPL* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 +++++++-
clang/lib/Headers/arm64intr.h | 5 ++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 49 +++++++++++++++++++
4 files changed, 77 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 5ad36299a34e5..8f4e5ccb81a1a 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -469,4 +469,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __swpa16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
def __swpa32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
def __swpa64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
+ def __swpl8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __swpl16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
+ def __swpl32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
+ def __swpl64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index dacd3a7d4b2b8..aae5f613c307a 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5476,7 +5476,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
if (BuiltinID == AArch64::BI__swp8 || BuiltinID == AArch64::BI__swp16 ||
BuiltinID == AArch64::BI__swp32 || BuiltinID == AArch64::BI__swp64 ||
BuiltinID == AArch64::BI__swpa8 || BuiltinID == AArch64::BI__swpa16 ||
- BuiltinID == AArch64::BI__swpa32 || BuiltinID == AArch64::BI__swpa64) {
+ BuiltinID == AArch64::BI__swpa32 || BuiltinID == AArch64::BI__swpa64 ||
+ BuiltinID == AArch64::BI__swpl8 || BuiltinID == AArch64::BI__swpl16 ||
+ BuiltinID == AArch64::BI__swpl32 || BuiltinID == AArch64::BI__swpl64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5512,6 +5514,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_swpa64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__swpl8:
+ IntrID = Intrinsic::aarch64_swpl8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpl16:
+ IntrID = Intrinsic::aarch64_swpl16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpl32:
+ IntrID = Intrinsic::aarch64_swpl32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpl64:
+ IntrID = Intrinsic::aarch64_swpl64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 21d7029d82533..707f43c51a422 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -123,6 +123,11 @@ unsigned __int16 __swpa16(unsigned __int16 volatile *, unsigned __int16);
unsigned __int32 __swpa32(unsigned __int32 volatile *, unsigned __int32);
unsigned __int64 __swpa64(unsigned __int64 volatile *, unsigned __int64);
+unsigned __int8 __swpl8(unsigned __int8 volatile *, unsigned __int8);
+unsigned __int16 __swpl16(unsigned __int16 volatile *, unsigned __int16);
+unsigned __int32 __swpl32(unsigned __int32 volatile *, unsigned __int32);
+unsigned __int64 __swpl64(unsigned __int64 volatile *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index 282318f915258..fbfaeb7cd941f 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1225,6 +1225,55 @@ unsigned long long int test__swpa64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__swpa64'
+unsigned char test__swpl8(unsigned char volatile* t, unsigned char v)
+{
+ return __swpl8(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__swpl8(ptr{{.*}}%t, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpl8(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpl8'
+
+unsigned short test__swpl16(unsigned short volatile* t, unsigned short v)
+{
+ return __swpl16(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__swpl16(ptr{{.*}}%t, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpl16(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpl16'
+
+unsigned int test__swpl32(unsigned int volatile* t, unsigned int v)
+{
+ return __swpl32(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__swpl32(ptr{{.*}}%t, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpl32(ptr %[[TMPT]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpl32'
+
+unsigned long long int test__swpl64(unsigned long long int volatile* t,
+ unsigned long long int v)
+{
+ return __swpl64(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__swpl64(ptr{{.*}}%t, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.swpl64(ptr %[[TMPT]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpl64'
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From 2dc5681c905c10183c8485b47c284f7f8f670a08 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Thu, 23 Apr 2026 18:23:55 -0300
Subject: [PATCH 23/26] [aarch64] Add support for swpal builtin
The new intrinsic issues SWPAL* instruction directly, regardless of LSE
target features. This will be used to implement MSVC __swpalX
builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 15 +++++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 4 ++
.../lib/Target/AArch64/AArch64InstrAtomics.td | 13 ++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 1 +
.../test/CodeGen/AArch64/ms-intrinsics-swp.ll | 44 +++++++++++++++++++
5 files changed, 77 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 1bb9c0c0b7e03..bd527c995c777 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -141,6 +141,21 @@ def int_aarch64_swpl64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty],
[IntrNoFree]>;
+// SWPAL intrinsics — emit SWPAL* (no acquire/release) directly, regardless of the
+// LSE target feature.
+def int_aarch64_swpal8 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpal16 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpal32 : Intrinsic<[llvm_i32_ty],
+ [llvm_ptr_ty, llvm_i32_ty],
+ [IntrNoFree]>;
+def int_aarch64_swpal64 : Intrinsic<[llvm_i64_ty],
+ [llvm_ptr_ty, llvm_i64_ty],
+ [IntrNoFree]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index d5091c8955e4a..379c4786ca58f 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3301,6 +3301,10 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::SWPLH_cg:
case AArch64::SWPLW_cg:
case AArch64::SWPLX_cg:
+ case AArch64::SWPALB_cg:
+ case AArch64::SWPALH_cg:
+ case AArch64::SWPALW_cg:
+ case AArch64::SWPALX_cg:
return {"lse", AArch64::FeatureLSE};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index 2496a8990da61..d7c6dc1290412 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -676,6 +676,19 @@ def : Pat<(i32 (int_aarch64_swpl32 GPR64sp:$Rn, GPR32:$Rs)),
def : Pat<(i64 (int_aarch64_swpl64 GPR64sp:$Rn, GPR64:$Rs)),
(SWPLX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpal8 GPR64sp:$Rn, (and GPR32:$Rs, (i32 255)))),
+ (SWPALB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpal8 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPALB_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpal16 GPR64sp:$Rn, (and GPR32:$Rs, (i32 65535)))),
+ (SWPALH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpal16 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPALH_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_swpal32 GPR64sp:$Rn, GPR32:$Rs)),
+ (SWPALW_cg GPR32:$Rs, GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_swpal64 GPR64sp:$Rn, GPR64:$Rs)),
+ (SWPALX_cg GPR64:$Rs, GPR64sp:$Rn)>;
+
// v8.9a/v9.4a FEAT_LRCPC patterns
let Predicates = [HasRCPC3, HasNEON] in {
// LDAP1 loads
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index a599433ab4df8..d2869c2a4e238 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -3155,6 +3155,7 @@ let isCodeGenOnly = 1 in {
defm SWP : Swap_cg<0, 0, "">;
defm SWPA : Swap_cg<1, 0, "a">;
defm SWPL : Swap_cg<0, 1, "l">;
+ defm SWPAL : Swap_cg<1, 1, "al">;
}
// v9.6a atomic swap (FEAT_LSUI)
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
index 095e035cf1c17..f10804403034d 100644
--- a/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-swp.ll
@@ -142,3 +142,47 @@ define i64 @test_swpl64(ptr %p, i64 %v) {
%r = call i64 @llvm.aarch64.swpl64(ptr %p, i64 %v)
ret i64 %r
}
+
+define i32 @test_swpal8(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpal8:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpalb w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpal8(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpal16(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpal16:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpalh w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpal16(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i32 @test_swpal32(ptr %p, i32 %v) {
+; CHECK-LABEL: test_swpal32:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpal w1, w0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.swpal32(ptr %p, i32 %v)
+ ret i32 %r
+}
+
+define i64 @test_swpal64(ptr %p, i64 %v) {
+; CHECK-LABEL: test_swpal64:
+; CHECK: // %bb.0:
+; NOLSE-NEXT: .arch_extension lse
+; CHECK-NEXT: swpal x1, x0, [x0]
+; NOLSE-NEXT: .arch_extension nolse
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.swpal64(ptr %p, i64 %v)
+ ret i64 %r
+}
>From 0ae11e1d9470f4f1987747e1332f27edff37f355 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Fri, 17 Apr 2026 13:20:15 -0300
Subject: [PATCH 24/26] [aarch64] Add support for the __swpal{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__swpal8` - maps to SWPALB.
* `__swpal16` - maps to SWPALH.
* `__swpal32` - maps to SWPAL.
* `__swpal64` - maps to SWPAL.
The emit is done using new intrisincs to issue SWPAL* instruction
directly, regardless of LSE target feature. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 4 ++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 20 +++++++-
clang/lib/Headers/arm64intr.h | 5 ++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 50 +++++++++++++++++++
4 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 8f4e5ccb81a1a..541e56514e0e1 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -473,4 +473,8 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __swpl16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
def __swpl32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
def __swpl64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
+ def __swpal8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char volatile *, unsigned char)">;
+ def __swpal16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
+ def __swpal32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
+ def __swpal64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index aae5f613c307a..558b7fbe658f6 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5478,7 +5478,9 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
BuiltinID == AArch64::BI__swpa8 || BuiltinID == AArch64::BI__swpa16 ||
BuiltinID == AArch64::BI__swpa32 || BuiltinID == AArch64::BI__swpa64 ||
BuiltinID == AArch64::BI__swpl8 || BuiltinID == AArch64::BI__swpl16 ||
- BuiltinID == AArch64::BI__swpl32 || BuiltinID == AArch64::BI__swpl64) {
+ BuiltinID == AArch64::BI__swpl32 || BuiltinID == AArch64::BI__swpl64 ||
+ BuiltinID == AArch64::BI__swpal8 || BuiltinID == AArch64::BI__swpal16 ||
+ BuiltinID == AArch64::BI__swpal32 || BuiltinID == AArch64::BI__swpal64) {
unsigned IntrID;
llvm::Type *IntrArgTy;
switch (BuiltinID) {
@@ -5530,6 +5532,22 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
IntrID = Intrinsic::aarch64_swpl64;
IntrArgTy = Builder.getInt64Ty();
break;
+ case AArch64::BI__swpal8:
+ IntrID = Intrinsic::aarch64_swpal8;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpal16:
+ IntrID = Intrinsic::aarch64_swpal16;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpal32:
+ IntrID = Intrinsic::aarch64_swpal32;
+ IntrArgTy = Builder.getInt32Ty();
+ break;
+ case AArch64::BI__swpal64:
+ IntrID = Intrinsic::aarch64_swpal64;
+ IntrArgTy = Builder.getInt64Ty();
+ break;
default:
llvm_unreachable("missing builtin ID in switch!");
}
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 707f43c51a422..3d6701cc78902 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -128,6 +128,11 @@ unsigned __int16 __swpl16(unsigned __int16 volatile *, unsigned __int16);
unsigned __int32 __swpl32(unsigned __int32 volatile *, unsigned __int32);
unsigned __int64 __swpl64(unsigned __int64 volatile *, unsigned __int64);
+unsigned __int8 __swpal8(unsigned __int8 volatile *, unsigned __int8);
+unsigned __int16 __swpal16(unsigned __int16 volatile *, unsigned __int16);
+unsigned __int32 __swpal32(unsigned __int32 volatile *, unsigned __int32);
+unsigned __int64 __swpal64(unsigned __int64 volatile *, unsigned __int64);
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index fbfaeb7cd941f..bb1f05e40991b 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1274,6 +1274,56 @@ unsigned long long int test__swpl64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__swpl64'
+unsigned char test__swpal8(unsigned char volatile* t, unsigned char v)
+{
+ return __swpal8(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @test__swpal8(ptr{{.*}}%t, i8{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i8, ptr %v.addr, align 1
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i8 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpal8(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpal8'
+
+unsigned short test__swpal16(unsigned short volatile* t, unsigned short v)
+{
+ return __swpal16(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @test__swpal16(ptr{{.*}}%t, i16{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i16, ptr %v.addr, align 2
+// CHECK-MSCOMPAT: %[[ZEXTV:[0-9]+]] = zext i16 %[[TMPV]] to i32
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpal16(ptr %[[TMPT]], i32 %[[ZEXTV]])
+// CHECK-MSCOMPAT: %[[TRUNC:[0-9]+]] = trunc i32 %[[RET]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[TRUNC]]
+// CHECK-LINUX: error: call to undeclared function '__swpal16'
+
+unsigned int test__swpal32(unsigned int volatile* t, unsigned int v)
+{
+ return __swpal32(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @test__swpal32(ptr{{.*}}%t, i32{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i32, ptr %v.addr, align 4
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.swpal32(ptr %[[TMPT]], i32 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpal32'
+
+unsigned long long int test__swpal64(unsigned long long int volatile* t,
+ unsigned long long int v)
+{
+ return __swpal64(t, v);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @test__swpal64(ptr{{.*}}%t, i64{{.*}}%v){{.*}}{
+// CHECK-MSCOMPAT: %[[TMPT:[0-9]+]] = load ptr, ptr %t.addr, align 8
+// CHECK-MSCOMPAT: %[[TMPV:[0-9]+]] = load i64, ptr %v.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.swpal64(ptr %[[TMPT]], i64 %[[TMPV]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__swpal64'
+
+
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
// CHECK-MSCOMPAT: ![[MD4]] = !{!"d5"}
>From 8ff48bfb994e03e3d847d13f7a233a88efc83b2c Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Wed, 22 Apr 2026 16:26:44 -0300
Subject: [PATCH 25/26] [aarch64] Add support for ldarp builtin
The new intrinsic issues LDAPR* instruction directly, regardless of RCPC
target features. This will be used to implement MSVC __ldaprX builtins.
---
llvm/include/llvm/IR/IntrinsicsAArch64.td | 13 +++++
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 18 ++++--
.../lib/Target/AArch64/AArch64InstrAtomics.td | 7 +++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 8 +++
.../ms-intrinsics-atomics-reassemble.ll | 36 ++++++++++++
.../CodeGen/AArch64/ms-intrinsics-ldapr.ll | 56 +++++++++++++++++++
6 files changed, 132 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/ms-intrinsics-atomics-reassemble.ll
create mode 100644 llvm/test/CodeGen/AArch64/ms-intrinsics-ldapr.ll
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index bd527c995c777..20f9763aef8cb 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -156,6 +156,19 @@ def int_aarch64_swpal64 : Intrinsic<[llvm_i64_ty],
[llvm_ptr_ty, llvm_i64_ty],
[IntrNoFree]>;
+// LDAPR intrinsics — emit LDAPR* directly, regardless of RCPC target feature.
+// LDAPRB/H/W return i32 (zero-extended into GPR32); LDAPRX returns i64.
+// IntrWillReturn is intentionally omitted: on a target without RCPC the encoding
+// is UNDEFINED and may trap, so these calls are not guaranteed to return.
+def int_aarch64_ldapr8 : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+ [IntrNoFree, IntrReadMem, IntrArgMemOnly]>;
+def int_aarch64_ldapr16 : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+ [IntrNoFree, IntrReadMem, IntrArgMemOnly]>;
+def int_aarch64_ldapr32 : Intrinsic<[llvm_i32_ty], [llvm_ptr_ty],
+ [IntrNoFree, IntrReadMem, IntrArgMemOnly]>;
+def int_aarch64_ldapr64 : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty],
+ [IntrNoFree, IntrReadMem, IntrArgMemOnly]>;
+
def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 379c4786ca58f..5d55a95248953 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -3262,12 +3262,13 @@ void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
#endif
}
-// The codegen-only CAS* "_cg" instructions are emitted by the MSVC __cas*
-// intrinsics regardless of the active -march, so they carry no LSE predicate.
-// When we emit textual assembly (-S/-save-temps) the assembler would otherwise
-// reject them, so bracket each with a .arch_extension directive enabling the
-// required feature. Returns the extension name ("lse") and the feature
-// controlling it, or an empty name for any other opcode.
+// The codegen-only CAS*/SWP*/LDAPR* "_cg" instructions are emitted by the MSVC
+// __cas*/__swp*/__ldapr* intrinsics regardless of the active -march, so they
+// carry no LSE/RCPC predicate. When we emit textual assembly (-S/-save-temps)
+// the assembler would otherwise reject them, so bracket each with a
+// .arch_extension directive enabling the required feature. Returns the
+// extension name ("lse"/"rcpc") and the feature controlling it, or an empty
+// name for any other opcode.
static std::pair<StringRef, unsigned>
getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
switch (Opc) {
@@ -3306,6 +3307,11 @@ getCodeGenOnlyAtomicArchExtension(unsigned Opc) {
case AArch64::SWPALW_cg:
case AArch64::SWPALX_cg:
return {"lse", AArch64::FeatureLSE};
+ case AArch64::LDAPRB_cg:
+ case AArch64::LDAPRH_cg:
+ case AArch64::LDAPRW_cg:
+ case AArch64::LDAPRX_cg:
+ return {"rcpc", AArch64::FeatureRCPC};
}
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
index d7c6dc1290412..77fd7134d9fb0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -60,6 +60,13 @@ let Predicates = [HasRCPC] in {
def : Pat<(acquiring_load<atomic_load_nonext_64> GPR64sp:$ptr), (LDAPRX GPR64sp:$ptr)>;
}
+// Unconditional patterns for the __ldapr* MSVC builtins. These force LDAPR*
+// emission regardless of the RCPC subtarget feature.
+def : Pat<(i32 (int_aarch64_ldapr8 GPR64sp:$Rn)), (LDAPRB_cg GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_ldapr16 GPR64sp:$Rn)), (LDAPRH_cg GPR64sp:$Rn)>;
+def : Pat<(i32 (int_aarch64_ldapr32 GPR64sp:$Rn)), (LDAPRW_cg GPR64sp:$Rn)>;
+def : Pat<(i64 (int_aarch64_ldapr64 GPR64sp:$Rn)), (LDAPRX_cg GPR64sp:$Rn)>;
+
// 8-bit loads
def : Pat<(seq_cst_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
def : Pat<(acquiring_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index d2869c2a4e238..2811ab62e0d95 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1970,6 +1970,14 @@ let Predicates = [HasRCPC] in {
def LDAPRX : RCPCLoad<0b11, "ldapr", GPR64>;
}
+// Code-gen-only LDAPR variants for the __ldapr* MSVC builtins.
+let isCodeGenOnly = 1 in {
+ def LDAPRB_cg : RCPCLoad<0b00, "ldaprb", GPR32>;
+ def LDAPRH_cg : RCPCLoad<0b01, "ldaprh", GPR32>;
+ def LDAPRW_cg : RCPCLoad<0b10, "ldapr", GPR32>;
+ def LDAPRX_cg : RCPCLoad<0b11, "ldapr", GPR64>;
+}
+
// v8.3a complex add and multiply-accumulate. No predicate here, that is done
// inside the multiclass as the FP16 versions need different predicates.
defm FCMLA : SIMDThreeSameVectorTiedComplexHSD<1, 0b110, complexrotateop,
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-atomics-reassemble.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-atomics-reassemble.ll
new file mode 100644
index 0000000000000..d4a1ec8972204
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-atomics-reassemble.ll
@@ -0,0 +1,36 @@
+; Verify that the codegen-only CAS*/SWP*/LDAPR* instructions emitted by the MS
+; __cas*/__swp*/__ldapr* intrinsics can be re-assembled when no -march feature
+; enables LSE/RCPC. The AsmPrinter brackets each such instruction with
+; .arch_extension directives so the integrated assembler (-S then assemble, or
+; -save-temps) accepts the otherwise feature-gated encoding.
+
+; RUN: llc -mtriple=aarch64-windows-msvc < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-windows-msvc < %s | \
+; RUN: llvm-mc -triple=aarch64-windows-msvc -filetype=obj -o /dev/null
+
+define i32 @cas(ptr %p, i32 %rs, i32 %rt) {
+; CHECK-LABEL: cas:
+; CHECK: .arch_extension lse
+; CHECK-NEXT: cas w1, w2, [x0]
+; CHECK-NEXT: .arch_extension nolse
+ %r = call i32 @llvm.aarch64.cas32(ptr %p, i32 %rs, i32 %rt)
+ ret i32 %r
+}
+
+define i32 @swp(ptr %p, i32 %rs) {
+; CHECK-LABEL: swp:
+; CHECK: .arch_extension lse
+; CHECK-NEXT: swp w1, w0, [x0]
+; CHECK-NEXT: .arch_extension nolse
+ %r = call i32 @llvm.aarch64.swp32(ptr %p, i32 %rs)
+ ret i32 %r
+}
+
+define i32 @ldapr(ptr %p) {
+; CHECK-LABEL: ldapr:
+; CHECK: .arch_extension rcpc
+; CHECK-NEXT: ldapr w0, [x0]
+; CHECK-NEXT: .arch_extension norcpc
+ %r = call i32 @llvm.aarch64.ldapr32(ptr %p)
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/ms-intrinsics-ldapr.ll b/llvm/test/CodeGen/AArch64/ms-intrinsics-ldapr.ll
new file mode 100644
index 0000000000000..1732819055e2f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ms-intrinsics-ldapr.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-windows-msvc -fast-isel=0 -global-isel=false \
+; RUN: -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,NORCPC
+; RUN: llc -mtriple=aarch64-windows-msvc -mattr=+rcpc -fast-isel=0 \
+; RUN: -global-isel=false -verify-machineinstrs < %s | FileCheck %s
+
+; Tests for the __ldapr* MS builtins on AArch64. These lower to the
+; llvm.aarch64.ldaprN intrinsics and must select ldaprb/h/w/x regardless
+; of whether +rcpc is present in the target features (the _cg ISel variants
+; have no feature predicate). When +rcpc is absent, the AsmPrinter brackets
+; the instruction with .arch_extension directives so that assembly output
+; re-assembles.
+
+define i32 @test_ldapr8(ptr %p) {
+; CHECK-LABEL: test_ldapr8:
+; CHECK: // %bb.0:
+; NORCPC-NEXT: .arch_extension rcpc
+; CHECK-NEXT: ldaprb w0, [x0]
+; NORCPC-NEXT: .arch_extension norcpc
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.ldapr8(ptr %p)
+ ret i32 %r
+}
+
+define i32 @test_ldapr16(ptr %p) {
+; CHECK-LABEL: test_ldapr16:
+; CHECK: // %bb.0:
+; NORCPC-NEXT: .arch_extension rcpc
+; CHECK-NEXT: ldaprh w0, [x0]
+; NORCPC-NEXT: .arch_extension norcpc
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.ldapr16(ptr %p)
+ ret i32 %r
+}
+
+define i32 @test_ldapr32(ptr %p) {
+; CHECK-LABEL: test_ldapr32:
+; CHECK: // %bb.0:
+; NORCPC-NEXT: .arch_extension rcpc
+; CHECK-NEXT: ldapr w0, [x0]
+; NORCPC-NEXT: .arch_extension norcpc
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.aarch64.ldapr32(ptr %p)
+ ret i32 %r
+}
+
+define i64 @test_ldapr64(ptr %p) {
+; CHECK-LABEL: test_ldapr64:
+; CHECK: // %bb.0:
+; NORCPC-NEXT: .arch_extension rcpc
+; CHECK-NEXT: ldapr x0, [x0]
+; NORCPC-NEXT: .arch_extension norcpc
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.aarch64.ldapr64(ptr %p)
+ ret i64 %r
+}
>From 2a6899e123353de5d442c05d36c044529c0069c5 Mon Sep 17 00:00:00 2001
From: Adhemerval Zanella <zatrazz at gmail.com>
Date: Tue, 14 Apr 2026 11:27:39 -0300
Subject: [PATCH 26/26] [aarch64] Add support for the __ldapr{8|16|32|64} MS
intrinsics
Adds support for the following MSVC intrinsics:
* `__ldapr8` - LDAPRB
* `__ldapr16` - LDAPRH
* `__ldapr32` - LDAPR
* `__ldapr64` - LDAPR
The emit is done using new intrisincs to issue LDAPR* instruction
directly, regardless of RCPC target features. This mimics MSVC compiler
and the idea of the builtin to mimic an inline asm.
These are documented at:
<https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-180>
---
clang/include/clang/Basic/BuiltinsAArch64.td | 5 +++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 26 +++++++++++++
clang/lib/Headers/arm64intr.h | 5 +++
.../test/CodeGen/arm64-microsoft-intrinsics.c | 37 +++++++++++++++++++
4 files changed, 73 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.td b/clang/include/clang/Basic/BuiltinsAArch64.td
index 541e56514e0e1..67b8babe0d646 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.td
+++ b/clang/include/clang/Basic/BuiltinsAArch64.td
@@ -477,4 +477,9 @@ let Attributes = [NoThrow, RequireDeclaration], Languages = "ALL_MS_LANGUAGES",
def __swpal16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short volatile *, unsigned short)">;
def __swpal32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int volatile *, unsigned int)">;
def __swpal64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int volatile *, unsigned long long int)">;
+
+ def __ldapr8 : AArch64NoPrefixTargetLibBuiltin<"unsigned char (unsigned char const volatile *)">;
+ def __ldapr16 : AArch64NoPrefixTargetLibBuiltin<"unsigned short (unsigned short const volatile *)">;
+ def __ldapr32 : AArch64NoPrefixTargetLibBuiltin<"unsigned int (unsigned int const volatile *)">;
+ def __ldapr64 : AArch64NoPrefixTargetLibBuiltin<"unsigned long long int (unsigned long long int const volatile *)">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 558b7fbe658f6..22f0f45567a7a 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -5563,6 +5563,32 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
return Result;
}
+ if (BuiltinID == AArch64::BI__ldapr8 || BuiltinID == AArch64::BI__ldapr16 ||
+ BuiltinID == AArch64::BI__ldapr32 || BuiltinID == AArch64::BI__ldapr64) {
+ unsigned IntrID;
+ switch (BuiltinID) {
+ case AArch64::BI__ldapr8:
+ IntrID = Intrinsic::aarch64_ldapr8;
+ break;
+ case AArch64::BI__ldapr16:
+ IntrID = Intrinsic::aarch64_ldapr16;
+ break;
+ case AArch64::BI__ldapr32:
+ IntrID = Intrinsic::aarch64_ldapr32;
+ break;
+ default:
+ IntrID = Intrinsic::aarch64_ldapr64;
+ break;
+ }
+ Value *Ptr = EmitScalarExpr(E->getArg(0));
+ Value *Result = Builder.CreateCall(CGM.getIntrinsic(IntrID), Ptr);
+ // LDAPRB/H return i32 (zero-extended); truncate to match the declared type.
+ llvm::Type *RetTy = ConvertType(E->getType());
+ if (Result->getType() != RetTy)
+ Result = Builder.CreateTrunc(Result, RetTy);
+ return Result;
+ }
+
if (BuiltinID == NEON::BI__builtin_neon_vcvth_bf16_f32)
return Builder.CreateFPTrunc(
Builder.CreateBitCast(EmitScalarExpr(E->getArg(0)),
diff --git a/clang/lib/Headers/arm64intr.h b/clang/lib/Headers/arm64intr.h
index 3d6701cc78902..ea7d2109278bf 100644
--- a/clang/lib/Headers/arm64intr.h
+++ b/clang/lib/Headers/arm64intr.h
@@ -50,6 +50,11 @@ unsigned __int16 __ldar16(const volatile unsigned __int16 *);
unsigned __int32 __ldar32(const volatile unsigned __int32 *);
unsigned __int64 __ldar64(const volatile unsigned __int64 *);
+unsigned __int8 __ldapr8(const volatile unsigned __int8 *);
+unsigned __int16 __ldapr16(const volatile unsigned __int16 *);
+unsigned __int32 __ldapr32(const volatile unsigned __int32 *);
+unsigned __int64 __ldapr64(const volatile unsigned __int64 *);
+
void __stlr8(unsigned __int8 volatile *, unsigned __int8);
void __stlr16(unsigned __int16 volatile *, unsigned __int16);
void __stlr32(unsigned __int32 volatile *, unsigned __int32);
diff --git a/clang/test/CodeGen/arm64-microsoft-intrinsics.c b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
index bb1f05e40991b..74187180d62b1 100644
--- a/clang/test/CodeGen/arm64-microsoft-intrinsics.c
+++ b/clang/test/CodeGen/arm64-microsoft-intrinsics.c
@@ -1323,6 +1323,43 @@ unsigned long long int test__swpal64(unsigned long long int volatile* t,
// CHECK-MSCOMPAT: ret i64 %[[RET]]
// CHECK-LINUX: error: call to undeclared function '__swpal64'
+unsigned char check__ldapr8(unsigned char volatile *p) {
+ return __ldapr8(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i8 @check__ldapr8(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[LOAD:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[TMP:[0-9]+]] = call i32 @llvm.aarch64.ldapr8(ptr %[[LOAD]])
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = trunc i32 %[[TMP]] to i8
+// CHECK-MSCOMPAT: ret i8 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldapr8'
+
+unsigned short check__ldapr16(unsigned short volatile *p) {
+ return __ldapr16(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i16 @check__ldapr16(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[LOAD:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[TMP:[0-9]+]] = call i32 @llvm.aarch64.ldapr16(ptr %[[LOAD]])
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = trunc i32 %[[TMP]] to i16
+// CHECK-MSCOMPAT: ret i16 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldapr16'
+
+unsigned int check__ldapr32(unsigned int volatile *p) {
+ return __ldapr32(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i32 @check__ldapr32(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[LOAD:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i32 @llvm.aarch64.ldapr32(ptr %[[LOAD]])
+// CHECK-MSCOMPAT: ret i32 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldapr32'
+
+unsigned long long int check__ldapr64(unsigned long long int volatile *p) {
+ return __ldapr64(p);
+}
+// CHECK-MSCOMPAT-LABEL: define{{.*}}i64 @check__ldapr64(ptr{{.*}}%p){{.*}}{
+// CHECK-MSCOMPAT: %[[LOAD:[0-9]+]] = load ptr, ptr %p.addr, align 8
+// CHECK-MSCOMPAT: %[[RET:[0-9]+]] = call i64 @llvm.aarch64.ldapr64(ptr %[[LOAD]])
+// CHECK-MSCOMPAT: ret i64 %[[RET]]
+// CHECK-LINUX: error: call to undeclared function '__ldapr64'
// CHECK-MSCOMPAT: ![[MD2]] = !{!"x18"}
// CHECK-MSCOMPAT: ![[MD3]] = !{!"sp"}
More information about the llvm-commits
mailing list