[llvm-branch-commits] [clang] [llvm] [PowerPC] Add support for AMO store builtins (PR #170933)
Maryam Moghadas via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Jan 15 13:32:53 PST 2026
https://github.com/maryammo updated https://github.com/llvm/llvm-project/pull/170933
>From eec7926a03e86c6670fadd9d342cf073b64fd81a Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo at ca.ibm.com>
Date: Wed, 3 Dec 2025 22:59:57 +0000
Subject: [PATCH 1/3] [PowerPC] Add support for AMO store builtins
---
clang/include/clang/Basic/BuiltinsPPC.def | 5 +
clang/lib/CodeGen/TargetBuiltins/PPC.cpp | 14 +++
clang/lib/Headers/amo.h | 89 +++++++++++++++
clang/lib/Sema/SemaPPC.cpp | 28 +++++
clang/test/CodeGen/PowerPC/builtins-amo-err.c | 24 ++++
clang/test/CodeGen/PowerPC/builtins-ppc-amo.c | 63 ++++++++++
clang/test/CodeGen/PowerPC/ppc-amo-header.c | 108 ++++++++++++++++++
llvm/include/llvm/IR/IntrinsicsPowerPC.td | 9 ++
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 24 ++++
llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 2 +-
llvm/lib/Target/PowerPC/PPCInstrInfo.td | 2 +-
llvm/test/CodeGen/PowerPC/amo-enable.ll | 32 ++++++
12 files changed, 398 insertions(+), 2 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index 7689daf00e6a7..f518429136e3c 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -1010,6 +1010,11 @@ TARGET_BUILTIN(__builtin_amo_lwat_cond, "UiUi*Ii", "", "isa-v30-instructions")
TARGET_BUILTIN(__builtin_amo_ldat_cond, "ULiULi*Ii", "", "isa-v30-instructions")
TARGET_BUILTIN(__builtin_amo_lwat_cond_s, "SiSi*Ii", "", "isa-v30-instructions")
TARGET_BUILTIN(__builtin_amo_ldat_cond_s, "SLiSLi*Ii", "", "isa-v30-instructions")
+TARGET_BUILTIN(__builtin_amo_stwat, "vUi*UiIi", "", "isa-v30-instructions")
+TARGET_BUILTIN(__builtin_amo_stdat, "vULi*ULiIi", "", "isa-v30-instructions")
+TARGET_BUILTIN(__builtin_amo_stwat_s, "vSi*SiIi", "", "isa-v30-instructions")
+TARGET_BUILTIN(__builtin_amo_stdat_s, "vSLi*SLiIi", "", "isa-v30-instructions")
+
// Set the floating point rounding mode
BUILTIN(__builtin_setrnd, "di", "")
diff --git a/clang/lib/CodeGen/TargetBuiltins/PPC.cpp b/clang/lib/CodeGen/TargetBuiltins/PPC.cpp
index bccb6acdb4e06..6568959351a5d 100644
--- a/clang/lib/CodeGen/TargetBuiltins/PPC.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/PPC.cpp
@@ -1386,5 +1386,19 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID,
return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_ldat_cond),
{Op0, Op1});
}
+ case PPC::BI__builtin_amo_stwat_s: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_stwat),
+ {Op0, Op1, Op2});
+ }
+ case PPC::BI__builtin_amo_stdat_s: {
+ Value *Op0 = EmitScalarExpr(E->getArg(0));
+ Value *Op1 = EmitScalarExpr(E->getArg(1));
+ Value *Op2 = EmitScalarExpr(E->getArg(2));
+ return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_stdat),
+ {Op0, Op1, Op2});
+ }
}
}
diff --git a/clang/lib/Headers/amo.h b/clang/lib/Headers/amo.h
index 97eff35e9c5dc..c7563f203d1f8 100644
--- a/clang/lib/Headers/amo.h
+++ b/clang/lib/Headers/amo.h
@@ -124,6 +124,95 @@ static inline int64_t amo_ldat_sswap(int64_t *ptr, int64_t val) {
return __builtin_amo_ldat_s(ptr, val, _AMO_LD_SWAP);
}
+/* AMO Store Operation Codes (FC values) */
+enum _AMO_ST {
+ _AMO_ST_ADD = 0x00, /* Store Add */
+ _AMO_ST_XOR = 0x01, /* Store Xor */
+ _AMO_ST_IOR = 0x02, /* Store Ior */
+ _AMO_ST_AND = 0x03, /* Store And */
+ _AMO_ST_UMAX = 0x04, /* Store Unsigned Maximum */
+ _AMO_ST_SMAX = 0x05, /* Store Signed Maximum */
+ _AMO_ST_UMIN = 0x06, /* Store Unsigned Minimum */
+ _AMO_ST_SMIN = 0x07, /* Store Signed Minimum */
+ _AMO_ST_TWIN = 0x18 /* Store Twin */
+};
+
+/* 32-bit unsigned AMO store operations */
+static inline void amo_stwat_add(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stwat_xor(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_XOR);
+}
+
+static inline void amo_stwat_ior(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_IOR);
+}
+
+static inline void amo_stwat_and(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_AND);
+}
+
+static inline void amo_stwat_umax(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_UMAX);
+}
+
+static inline void amo_stwat_umin(uint32_t *ptr, uint32_t val) {
+ __builtin_amo_stwat(ptr, val, _AMO_ST_UMIN);
+}
+
+/* 32-bit signed AMO store operations */
+static inline void amo_stwat_sadd(int32_t *ptr, int32_t val) {
+ __builtin_amo_stwat_s(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stwat_smax(int32_t *ptr, int32_t val) {
+ __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMAX);
+}
+
+static inline void amo_stwat_smin(int32_t *ptr, int32_t val) {
+ __builtin_amo_stwat_s(ptr, val, _AMO_ST_SMIN);
+}
+
+/* 64-bit unsigned AMO store operations */
+static inline void amo_stdat_add(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stdat_xor(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_XOR);
+}
+
+static inline void amo_stdat_ior(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_IOR);
+}
+
+static inline void amo_stdat_and(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_AND);
+}
+
+static inline void amo_stdat_umax(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_UMAX);
+}
+
+static inline void amo_stdat_umin(uint64_t *ptr, uint64_t val) {
+ __builtin_amo_stdat(ptr, val, _AMO_ST_UMIN);
+}
+
+/* 64-bit signed AMO store operations */
+static inline void amo_stdat_sadd(int64_t *ptr, int64_t val) {
+ __builtin_amo_stdat_s(ptr, val, _AMO_ST_ADD);
+}
+
+static inline void amo_stdat_smax(int64_t *ptr, int64_t val) {
+ __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMAX);
+}
+
+static inline void amo_stdat_smin(int64_t *ptr, int64_t val) {
+ __builtin_amo_stdat_s(ptr, val, _AMO_ST_SMIN);
+}
+
#ifdef __cplusplus
}
#endif
diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp
index a7e76a9917372..1bf806d996887 100644
--- a/clang/lib/Sema/SemaPPC.cpp
+++ b/clang/lib/Sema/SemaPPC.cpp
@@ -95,6 +95,10 @@ static bool isPPC_64Builtin(unsigned BuiltinID) {
case PPC::BI__builtin_amo_ldat_cond:
case PPC::BI__builtin_amo_lwat_cond_s:
case PPC::BI__builtin_amo_ldat_cond_s:
+ case PPC::BI__builtin_amo_stwat:
+ case PPC::BI__builtin_amo_stdat:
+ case PPC::BI__builtin_amo_stwat_s:
+ case PPC::BI__builtin_amo_stdat_s:
return true;
}
return false;
@@ -300,6 +304,30 @@ bool SemaPPC::CheckPPCBuiltinFunctionCall(const TargetInfo &TI,
return SemaRef.Diag(Arg->getBeginLoc(), diag::err_argument_invalid_range)
<< toString(Result, 10) << "24, 25" << "28" << Arg->getSourceRange();
}
+ case PPC::BI__builtin_amo_stwat:
+ case PPC::BI__builtin_amo_stdat:
+ case PPC::BI__builtin_amo_stwat_s:
+ case PPC::BI__builtin_amo_stdat_s: {
+ llvm::APSInt Result;
+ if (SemaRef.BuiltinConstantArg(TheCall, 2, Result))
+ return true;
+ unsigned Val = Result.getZExtValue();
+
+ bool IsUnsigned = (BuiltinID == PPC::BI__builtin_amo_stwat ||
+ BuiltinID == PPC::BI__builtin_amo_stdat);
+
+ bool IsValid = IsUnsigned
+ ? llvm::is_contained({0u, 1u, 2u, 3u, 4u, 6u, 24u}, Val)
+ : llvm::is_contained({0u, 5u, 7u, 24u}, Val);
+
+ if (IsValid)
+ return false;
+
+ Expr *Arg = TheCall->getArg(2);
+ return SemaRef.Diag(Arg->getBeginLoc(), diag::err_argument_invalid_range)
+ << toString(Result, 10) << (IsUnsigned ? "0-4, 6" : "0, 5, 7")
+ << "24" << Arg->getSourceRange();
+ }
}
llvm_unreachable("must return from switch");
}
diff --git a/clang/test/CodeGen/PowerPC/builtins-amo-err.c b/clang/test/CodeGen/PowerPC/builtins-amo-err.c
index ad6be9e867856..f99efd0505818 100644
--- a/clang/test/CodeGen/PowerPC/builtins-amo-err.c
+++ b/clang/test/CodeGen/PowerPC/builtins-amo-err.c
@@ -51,4 +51,28 @@ void test_amo() {
__builtin_amo_ldat_cond_s(ptr6, 28);
// FC-ERROR: argument value 0 is outside the valid range [24, 25, 28]
__builtin_amo_ldat_cond_s(ptr6, 0);
+
+ unsigned int *ptr9, value9;
+ // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets
+ __builtin_amo_stwat(ptr9, value9, 0);
+ // FC-ERROR: error: argument value 5 is outside the valid range [0-4, 6, 24]
+ __builtin_amo_stwat(ptr9, value9, 5);
+
+ unsigned long int *ptr10, value10;
+ // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets
+ __builtin_amo_stdat(ptr10, value10, 24);
+ // FC-ERROR: error: argument value 10 is outside the valid range [0-4, 6, 24]
+ __builtin_amo_stdat(ptr10, value10, 10);
+
+ signed int *ptr11, value11;
+ // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets
+ __builtin_amo_stwat_s(ptr11, value11, 0);
+ // FC-ERROR: error: argument value 1 is outside the valid range [0, 5, 7, 24]
+ __builtin_amo_stwat_s(ptr11, value11, 1);
+
+ signed long int *ptr12, value12;
+ // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets
+ __builtin_amo_stdat_s(ptr12, value12, 24);
+ // FC-ERROR: error: argument value 6 is outside the valid range [0, 5, 7, 24]
+ __builtin_amo_stdat_s(ptr12, value12, 6);
}
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c b/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c
index 0bf28e85730e1..ffbedd8c50202 100644
--- a/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c
@@ -156,6 +156,69 @@ void test_signed_ldat_cond(long int *ptr, long int * resp) {
*resp = res;
}
+// CHECK-LABEL: define dso_local void @test_unsigned_stwat(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef zeroext [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.ppc.amo.stwat(ptr [[PTR]], i32 [[VALUE]], i32 24)
+// CHECK-NEXT: ret void
+//
+// AIX-LABEL: define void @test_unsigned_stwat(
+// AIX-SAME: ptr noundef [[PTR:%.*]], i32 noundef zeroext [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AIX-NEXT: [[ENTRY:.*:]]
+// AIX-NEXT: tail call void @llvm.ppc.amo.stwat(ptr [[PTR]], i32 [[VALUE]], i32 24)
+// AIX-NEXT: ret void
+//
+void test_unsigned_stwat(unsigned int *ptr, unsigned int value, unsigned int * resp) {
+ __builtin_amo_stwat(ptr, value, 24);
+}
+
+// CHECK-LABEL: define dso_local void @test_unsigned_stdat(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i64 noundef [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.ppc.amo.stdat(ptr [[PTR]], i64 [[VALUE]], i32 3)
+// CHECK-NEXT: ret void
+//
+// AIX-LABEL: define void @test_unsigned_stdat(
+// AIX-SAME: ptr noundef [[PTR:%.*]], i64 noundef [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AIX-NEXT: [[ENTRY:.*:]]
+// AIX-NEXT: tail call void @llvm.ppc.amo.stdat(ptr [[PTR]], i64 [[VALUE]], i32 3)
+// AIX-NEXT: ret void
+//
+void test_unsigned_stdat(unsigned long int *ptr, unsigned long int value, unsigned long int * resp) {
+ __builtin_amo_stdat(ptr, value, 3);
+}
+
+// CHECK-LABEL: define dso_local void @test_signed_stwat(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i32 noundef signext [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.ppc.amo.stwat(ptr [[PTR]], i32 [[VALUE]], i32 24)
+// CHECK-NEXT: ret void
+//
+// AIX-LABEL: define void @test_signed_stwat(
+// AIX-SAME: ptr noundef [[PTR:%.*]], i32 noundef signext [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AIX-NEXT: [[ENTRY:.*:]]
+// AIX-NEXT: tail call void @llvm.ppc.amo.stwat(ptr [[PTR]], i32 [[VALUE]], i32 24)
+// AIX-NEXT: ret void
+//
+void test_signed_stwat(int *ptr, int value, int * resp) {
+ __builtin_amo_stwat_s(ptr, value, 24);
+}
+
+// CHECK-LABEL: define dso_local void @test_signed_stdat(
+// CHECK-SAME: ptr noundef [[PTR:%.*]], i64 noundef [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: tail call void @llvm.ppc.amo.stdat(ptr [[PTR]], i64 [[VALUE]], i32 5)
+// CHECK-NEXT: ret void
+//
+// AIX-LABEL: define void @test_signed_stdat(
+// AIX-SAME: ptr noundef [[PTR:%.*]], i64 noundef [[VALUE:%.*]], ptr noundef readnone captures(none) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// AIX-NEXT: [[ENTRY:.*:]]
+// AIX-NEXT: tail call void @llvm.ppc.amo.stdat(ptr [[PTR]], i64 [[VALUE]], i32 5)
+// AIX-NEXT: ret void
+//
+void test_signed_stdat(long int *ptr, long int value, long int * resp) {
+ __builtin_amo_stdat_s(ptr, value, 5);
+}
//.
// CHECK: [[INT_TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
// CHECK: [[META3]] = !{!"int", [[META4:![0-9]+]], i64 0}
diff --git a/clang/test/CodeGen/PowerPC/ppc-amo-header.c b/clang/test/CodeGen/PowerPC/ppc-amo-header.c
index ffc87561fa3c4..e0acbf2dd8fa2 100644
--- a/clang/test/CodeGen/PowerPC/ppc-amo-header.c
+++ b/clang/test/CodeGen/PowerPC/ppc-amo-header.c
@@ -137,3 +137,111 @@ int64_t test_ldat_sswap(int64_t *ptr, int64_t val) {
// CHECK: call i64 @llvm.ppc.amo.ldat(ptr %{{.*}}, i64 %{{.*}}, i32 8)
return amo_ldat_sswap(ptr, val);
}
+
+void test_stwat_add(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_add
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 0)
+ return amo_stwat_add(ptr, val);
+}
+
+void test_stwat_xor(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_xor
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 1)
+ return amo_stwat_xor(ptr, val);
+}
+
+void test_stwat_ior(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_ior
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 2)
+ return amo_stwat_ior(ptr, val);
+}
+
+void test_stwat_and(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_and
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 3)
+ return amo_stwat_and(ptr, val);
+}
+
+void test_stwat_umax(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_umax
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 4)
+ return amo_stwat_umax(ptr, val);
+}
+
+void test_stwat_umin(uint32_t *ptr, uint32_t val) {
+ // CHECK-LABEL: @test_stwat_umin
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 6)
+ return amo_stwat_umin(ptr, val);
+}
+
+void test_stwat_sadd(int32_t *ptr, int32_t val) {
+ // CHECK-LABEL: @test_stwat_sadd
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 0)
+ return amo_stwat_sadd(ptr, val);
+}
+
+void test_stwat_smax(int32_t *ptr, int32_t val) {
+ // CHECK-LABEL: @test_stwat_smax
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 5)
+ return amo_stwat_smax(ptr, val);
+}
+
+void test_stwat_smin(int32_t *ptr, int32_t val) {
+ // CHECK-LABEL: @test_stwat_smin
+ // CHECK: call void @llvm.ppc.amo.stwat(ptr %{{.*}}, i32 %{{.*}}, i32 7)
+ return amo_stwat_smin(ptr, val);
+}
+
+void test_stdat_add(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_add
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 0)
+ return amo_stdat_add(ptr, val);
+}
+
+void test_stdat_xor(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_xor
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 1)
+ return amo_stdat_xor(ptr, val);
+}
+
+void test_stdat_ior(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_ior
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 2)
+ return amo_stdat_ior(ptr, val);
+}
+
+void test_stdat_and(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_and
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 3)
+ return amo_stdat_and(ptr, val);
+}
+
+void test_stdat_umax(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_umax
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 4)
+ return amo_stdat_umax(ptr, val);
+}
+
+void test_stdat_umin(uint64_t *ptr, uint64_t val) {
+ // CHECK-LABEL: @test_stdat_umin
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 6)
+ return amo_stdat_umin(ptr, val);
+}
+
+void test_stdat_sadd(int64_t *ptr, int64_t val) {
+ // CHECK-LABEL: @test_stdat_sadd
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 0)
+ return amo_stdat_sadd(ptr, val);
+}
+
+void test_stdat_smax(int64_t *ptr, int64_t val) {
+ // CHECK-LABEL: @test_stdat_smax
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 5)
+ return amo_stdat_smax(ptr, val);
+}
+
+void test_stdat_smin(int64_t *ptr, int64_t val) {
+ // CHECK-LABEL: @test_stdat_smin
+ // CHECK: call void @llvm.ppc.amo.stdat(ptr %{{.*}}, i64 %{{.*}}, i32 7)
+ return amo_stdat_smin(ptr, val);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index e13f40454fba2..ce27e9163560b 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -2158,4 +2158,13 @@ let TargetPrefix = "ppc" in {
DefaultAttrsIntrinsic<[llvm_i64_ty],[llvm_ptr_ty,
llvm_i32_ty],
[IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
+
+ def int_ppc_amo_stwat : ClangBuiltin<"__builtin_amo_stwat">,
+ DefaultAttrsIntrinsic<[],[llvm_ptr_ty,
+ llvm_i32_ty, llvm_i32_ty],
+ [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
+ def int_ppc_amo_stdat : ClangBuiltin<"__builtin_amo_stdat">,
+ DefaultAttrsIntrinsic<[],[llvm_ptr_ty,
+ llvm_i64_ty, llvm_i32_ty],
+ [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
}
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 77cdee443c0bf..773125d9af3b6 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11509,6 +11509,30 @@ SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
return DAG.getStore(DAG.getEntryNode(), DL, Op.getOperand(ArgStart + 2),
Op.getOperand(ArgStart + 1), MachinePointerInfo());
}
+ case Intrinsic::ppc_amo_stwat:
+ case Intrinsic::ppc_amo_stdat: {
+ SDLoc dl(Op);
+ SDValue Chain = Op.getOperand(0);
+ SDValue Ptr = Op.getOperand(ArgStart + 1);
+ SDValue Val = Op.getOperand(ArgStart + 2);
+ SDValue FC = Op.getOperand(ArgStart + 3);
+
+ bool IsStwat =
+ Op.getConstantOperandVal(ArgStart) == Intrinsic::ppc_amo_stwat;
+ if (isa<ConstantSDNode>(Val)) {
+ MVT VT = IsStwat ? MVT::i32 : MVT::i64;
+ MachineFunction &MF = DAG.getMachineFunction();
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+ Register ValReg = MRI.createVirtualRegister(getRegClassFor(VT));
+
+ // Materialize constant Val using CopyToReg/CopyFromReg.
+ SDValue CopyChain = DAG.getCopyToReg(Chain, dl, ValReg, Val);
+ Val = DAG.getCopyFromReg(CopyChain, dl, ValReg, VT);
+ }
+ MachineSDNode *MNode = DAG.getMachineNode(IsStwat ? PPC::STWAT : PPC::STDAT,
+ dl, MVT::Other, {Val, Ptr, FC});
+ return SDValue(MNode, 0);
+ }
default:
break;
}
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 9fa284716dc71..45614923d9477 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -412,7 +412,7 @@ def : Pat<(int_ppc_cmpxchg_i128 ForceXForm:$ptr,
g8rc:$new_hi))>;
let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
-def STDAT : X_RD5_RS5_IM5<31, 742, (outs), (ins g8rc:$RST, g8rc:$RA, u5imm:$RB),
+def STDAT : X_RD5_RS5_IM5<31, 742, (outs), (ins g8rc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
"stdat $RST, $RA, $RB", IIC_LdStStore>, isPPC64,
Requires<[IsISA3_0]>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 67bbed63bfc13..b7b687fe197a1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -2157,7 +2157,7 @@ def STWCX : XForm_1_memOp<31, 150, (outs), (ins gprc:$RST, (memrr $RA, $RB):$add
}
let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
-def STWAT : X_RD5_RS5_IM5<31, 710, (outs), (ins gprc:$RST, gprc:$RA, u5imm:$RB),
+def STWAT : X_RD5_RS5_IM5<31, 710, (outs), (ins gprc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
"stwat $RST, $RA, $RB", IIC_LdStStore>,
Requires<[IsISA3_0]>;
diff --git a/llvm/test/CodeGen/PowerPC/amo-enable.ll b/llvm/test/CodeGen/PowerPC/amo-enable.ll
index c5889ba9624b7..549f262022ab1 100644
--- a/llvm/test/CodeGen/PowerPC/amo-enable.ll
+++ b/llvm/test/CodeGen/PowerPC/amo-enable.ll
@@ -82,7 +82,39 @@ entry:
ret void
}
+define void @test_stwat(ptr noundef %ptr, i32 noundef %value) {
+; CHECK-LABEL: test_stwat:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: stwat r4, r3, 0
+; CHECK-NEXT: blr
+;
+; CHECK-BE-LABEL: test_stwat:
+; CHECK-BE: # %bb.0: # %entry
+; CHECK-BE-NEXT: stwat r4, r3, 0
+; CHECK-BE-NEXT: blr
+entry:
+ tail call void @llvm.ppc.amo.stwat(ptr %ptr, i32 %value, i32 0)
+ ret void
+}
+
+define void @test_stdat(ptr noundef %ptr, i64 noundef %value) {
+; CHECK-LABEL: test_stdat:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: stdat r4, r3, 0
+; CHECK-NEXT: blr
+;
+; CHECK-BE-LABEL: test_stdat:
+; CHECK-BE: # %bb.0: # %entry
+; CHECK-BE-NEXT: stdat r4, r3, 0
+; CHECK-BE-NEXT: blr
+entry:
+ tail call void @llvm.ppc.amo.stdat(ptr %ptr, i64 %value, i32 0)
+ ret void
+}
+
declare i64 @llvm.ppc.amo.ldat(ptr, i64, i32 immarg)
declare i32 @llvm.ppc.amo.lwat(ptr, i32, i32 immarg)
declare i64 @llvm.ppc.amo.ldat.cond(ptr, i32 immarg)
declare i32 @llvm.ppc.amo.lwat.cond(ptr, i32 immarg)
+declare void @llvm.ppc.amo.stwat(ptr, i32, i32 immarg)
+declare void @llvm.ppc.amo.stdat(ptr, i64, i32 immarg)
>From 981c37fa0b94312de7855d5ea49b51954eec44e0 Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo at ca.ibm.com>
Date: Mon, 12 Jan 2026 21:45:10 +0000
Subject: [PATCH 2/3] Add PPCISD node for AMO stores
---
llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 16 +---------------
llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 2 ++
llvm/lib/Target/PowerPC/PPCInstrInfo.td | 10 ++++++++++
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 773125d9af3b6..b9c9d45914c41 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -11517,21 +11517,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
SDValue Val = Op.getOperand(ArgStart + 2);
SDValue FC = Op.getOperand(ArgStart + 3);
- bool IsStwat =
- Op.getConstantOperandVal(ArgStart) == Intrinsic::ppc_amo_stwat;
- if (isa<ConstantSDNode>(Val)) {
- MVT VT = IsStwat ? MVT::i32 : MVT::i64;
- MachineFunction &MF = DAG.getMachineFunction();
- MachineRegisterInfo &MRI = MF.getRegInfo();
- Register ValReg = MRI.createVirtualRegister(getRegClassFor(VT));
-
- // Materialize constant Val using CopyToReg/CopyFromReg.
- SDValue CopyChain = DAG.getCopyToReg(Chain, dl, ValReg, Val);
- Val = DAG.getCopyFromReg(CopyChain, dl, ValReg, VT);
- }
- MachineSDNode *MNode = DAG.getMachineNode(IsStwat ? PPC::STWAT : PPC::STDAT,
- dl, MVT::Other, {Val, Ptr, FC});
- return SDValue(MNode, 0);
+ return DAG.getNode(PPCISD::STAT, dl, MVT::Other, Chain, Val, Ptr, FC);
}
default:
break;
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 45614923d9477..59abbf36ee14d 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -2007,3 +2007,5 @@ def : Pat<(i64 (int_ppc_mfspr timm:$SPR)),
(MFSPR8 $SPR)>;
def : Pat<(int_ppc_mtspr timm:$SPR, g8rc:$RT),
(MTSPR8 $SPR, $RT)>;
+def : Pat<(PPCstat i64:$val, ptr_rc_nor0:$ptr, timm:$fc),
+ (STDAT $val, $ptr, $fc)>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index b7b687fe197a1..f8761ae138f7c 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -143,6 +143,10 @@ def SDT_PPCBinaryArithWithFlagsInOut : SDTypeProfile<2, 3, [
SDTCisVT<1, i32>,
]>;
+def SDT_PPCstat : SDTypeProfile<0, 3, [
+ SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
+]>;
+
//===----------------------------------------------------------------------===//
// PowerPC specific DAG Nodes.
//
@@ -679,6 +683,9 @@ def PPCsubc : SDNode<"PPCISD::SUBC", SDT_PPCBinaryArithWithFlagsOut,
def PPCsube : SDNode<"PPCISD::SUBE", SDT_PPCBinaryArithWithFlagsInOut,
[]>;
+def PPCstat : SDNode<"PPCISD::STAT", SDT_PPCstat,
+ [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+
//===----------------------------------------------------------------------===//
// PowerPC specific transformation functions and pattern fragments.
//
@@ -5604,3 +5611,6 @@ def : Pat<(int_ppc_dcbtt ForceXForm:$dst),
def : Pat<(int_ppc_stfiw ForceXForm:$dst, f64:$XT),
(STFIWX f64:$XT, ForceXForm:$dst)>;
+
+def : Pat<(PPCstat i32:$val, ptr_rc_nor0:$ptr, timm:$fc),
+ (STWAT $val, $ptr, $fc)>;
>From c3863f0ba0c83b603beafc62a303e9f822954adc Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo at ca.ibm.com>
Date: Thu, 15 Jan 2026 21:30:38 +0000
Subject: [PATCH 3/3] Use XForm_base_r3xo_memOp for STWAT/STDAT to enable
direct patterns
---
llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 7 +++----
llvm/lib/Target/PowerPC/PPCInstrInfo.td | 7 +++----
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 59abbf36ee14d..5d862f830eac4 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -412,8 +412,9 @@ def : Pat<(int_ppc_cmpxchg_i128 ForceXForm:$ptr,
g8rc:$new_hi))>;
let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
-def STDAT : X_RD5_RS5_IM5<31, 742, (outs), (ins g8rc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
- "stdat $RST, $RA, $RB", IIC_LdStStore>, isPPC64,
+def STDAT : XForm_base_r3xo_memOp<31, 742, (outs), (ins g8rc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
+ "stdat $RST, $RA, $RB", IIC_LdStStore,
+ [(PPCstat i64:$RST, ptr_rc_nor0:$RA, timm:$RB)]>, isPPC64,
Requires<[IsISA3_0]>;
let Interpretation64Bit = 1, isCodeGenOnly = 1 in {
@@ -2007,5 +2008,3 @@ def : Pat<(i64 (int_ppc_mfspr timm:$SPR)),
(MFSPR8 $SPR)>;
def : Pat<(int_ppc_mtspr timm:$SPR, g8rc:$RT),
(MTSPR8 $SPR, $RT)>;
-def : Pat<(PPCstat i64:$val, ptr_rc_nor0:$ptr, timm:$fc),
- (STDAT $val, $ptr, $fc)>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index f8761ae138f7c..3f9c482979e61 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -2164,8 +2164,9 @@ def STWCX : XForm_1_memOp<31, 150, (outs), (ins gprc:$RST, (memrr $RA, $RB):$add
}
let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in
-def STWAT : X_RD5_RS5_IM5<31, 710, (outs), (ins gprc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
- "stwat $RST, $RA, $RB", IIC_LdStStore>,
+def STWAT : XForm_base_r3xo_memOp<31, 710, (outs), (ins gprc:$RST, ptr_rc_nor0:$RA, u5imm:$RB),
+ "stwat $RST, $RA, $RB", IIC_LdStStore,
+ [(PPCstat i32:$RST, ptr_rc_nor0:$RA, timm:$RB)]>,
Requires<[IsISA3_0]>;
let isTrap = 1, hasCtrlDep = 1 in
@@ -5612,5 +5613,3 @@ def : Pat<(int_ppc_dcbtt ForceXForm:$dst),
def : Pat<(int_ppc_stfiw ForceXForm:$dst, f64:$XT),
(STFIWX f64:$XT, ForceXForm:$dst)>;
-def : Pat<(PPCstat i32:$val, ptr_rc_nor0:$ptr, timm:$fc),
- (STWAT $val, $ptr, $fc)>;
More information about the llvm-branch-commits
mailing list