[llvm-branch-commits] [clang] [llvm] [InstCombine] Canonicalize atomicrmw sub of a constant to add (PR #232626)
Harrison Hao via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Oct 16 20:35:30 PDT 2026
https://github.com/harrisonGPU updated https://github.com/llvm/llvm-project/pull/232626
>From 88487fb605a7600cca177263ba96a268802b864e Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 16 Oct 2026 14:35:13 +0800
Subject: [PATCH 1/3] [InstCombine] Canonicalize atomicrmw sub of a constant to
add
Rewrite `atomicrmw sub ptr, C` as `atomicrmw add ptr, -C`. This
matches the canonical form InstCombine already uses for plain `sub`.
In wrapping arithmetic $old - C = old + (-C)$, so both the stored
value and the returned old value stay the same, including when C is
`INT_MIN`. Ordering, syncscope and metadata are kept.
```llvm
; before
%old = atomicrmw sub ptr %p, i32 1 syncscope("agent") monotonic
; after
%old = atomicrmw add ptr %p, i32 -1 syncscope("agent") monotonic
```
This benefits AMDGPU, for example: https://godbolt.org/z/39xsYro3o
---
llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp | 7 +++++++
llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll | 4 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index 4d6b0684abe4da..f9eab910fc56ae 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -134,6 +134,13 @@ Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
: AtomicRMWInst::Add);
return replaceOperand(RMWI, 1, X);
}
+
+ // atomicrmw sub(ptr, C) -> atomicrmw add(ptr, -C)
+ const APInt *C;
+ if (Op == AtomicRMWInst::Sub && match(Val, m_APInt(C)) && !C->isZero()) {
+ RMWI.setOperation(AtomicRMWInst::Add);
+ return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), -*C));
+ }
}
if (!isIdempotentRMW(RMWI))
diff --git a/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll b/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
index 9d304429335d57..147ae828fdac1d 100644
--- a/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
+++ b/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
@@ -97,7 +97,7 @@ define i64 @fn_add_of_two_vars(ptr %p, i64 %a, i64 %b) {
define i32 @fn_sub_const(ptr %a) {
; CHECK-LABEL: define i32 @fn_sub_const(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[A]], i32 5 syncscope("agent") acq_rel, align 4
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[A]], i32 -5 syncscope("agent") acq_rel, align 4
; CHECK-NEXT: ret i32 [[OLD]]
;
%old = atomicrmw sub ptr %a, i32 5 syncscope("agent") acq_rel
@@ -107,7 +107,7 @@ define i32 @fn_sub_const(ptr %a) {
define i64 @fn_sub_const_int_min(ptr %a) {
; CHECK-LABEL: define i64 @fn_sub_const_int_min(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[A]], i64 -9223372036854775808 monotonic, align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[A]], i64 -9223372036854775808 monotonic, align 8
; CHECK-NEXT: ret i64 [[OLD]]
;
%old = atomicrmw sub ptr %a, i64 -9223372036854775808 monotonic
>From 075886b9f43e46fe4032e754280d90d980de9a29 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Fri, 16 Oct 2026 16:00:35 +0800
Subject: [PATCH 2/3] Fix clang lit test
---
clang/test/CodeGen/ms-intrinsics-other.c | 8 ++++----
clang/test/CodeGen/ms-intrinsics.c | 24 ++++++++++++------------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/clang/test/CodeGen/ms-intrinsics-other.c b/clang/test/CodeGen/ms-intrinsics-other.c
index 9f183aee85824b..6f55bcb86fdd6b 100644
--- a/clang/test/CodeGen/ms-intrinsics-other.c
+++ b/clang/test/CodeGen/ms-intrinsics-other.c
@@ -178,7 +178,7 @@ LONG test_InterlockedDecrement(LONG volatile *Addend) {
return _InterlockedDecrement(Addend);
}
// CHECK: define{{.*}}i32 @test_InterlockedDecrement(ptr{{.*}}%Addend){{.*}}{
-// CHECK: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 seq_cst, align 4
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 seq_cst, align 4
// CHECK: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK: ret i32 [[RESULT]]
// CHECK: }
@@ -427,7 +427,7 @@ LONG test_InterlockedDecrement_acq(LONG volatile *Addend) {
return _InterlockedDecrement_acq(Addend);
}
// CHECK-ARM: define{{.*}}i32 @test_InterlockedDecrement_acq(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 acquire, align 4
+// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 acquire, align 4
// CHECK-ARM: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM: ret i32 [[RESULT]]
// CHECK-ARM: }
@@ -436,7 +436,7 @@ LONG test_InterlockedDecrement_rel(LONG volatile *Addend) {
return _InterlockedDecrement_rel(Addend);
}
// CHECK-ARM: define{{.*}}i32 @test_InterlockedDecrement_rel(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 release, align 4
+// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 release, align 4
// CHECK-ARM: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM: ret i32 [[RESULT]]
// CHECK-ARM: }
@@ -445,7 +445,7 @@ LONG test_InterlockedDecrement_nf(LONG volatile *Addend) {
return _InterlockedDecrement_nf(Addend);
}
// CHECK-ARM: define{{.*}}i32 @test_InterlockedDecrement_nf(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 monotonic, align 4
+// CHECK-ARM: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 monotonic, align 4
// CHECK-ARM: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM: ret i32 [[RESULT]]
// CHECK-ARM: }
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 8b719ef8cc1fd4..d15310e41a2676 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -572,7 +572,7 @@ short test_InterlockedDecrement16(short volatile *Addend) {
return _InterlockedDecrement16(Addend);
}
// CHECK: define{{.*}}i16 @test_InterlockedDecrement16(ptr{{.*}}%Addend){{.*}}{
-// CHECK: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i16 1 seq_cst, align 2
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i16 -1 seq_cst, align 2
// CHECK: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1
// CHECK: ret i16 [[RESULT]]
// CHECK: }
@@ -581,7 +581,7 @@ long test_InterlockedDecrement(long volatile *Addend) {
return _InterlockedDecrement(Addend);
}
// CHECK: define{{.*}}i32 @test_InterlockedDecrement(ptr{{.*}}%Addend){{.*}}{
-// CHECK: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 seq_cst, align 4
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 seq_cst, align 4
// CHECK: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK: ret i32 [[RESULT]]
// CHECK: }
@@ -677,7 +677,7 @@ __int64 test_InterlockedDecrement64(__int64 volatile *Addend) {
return _InterlockedDecrement64(Addend);
}
// CHECK: define{{.*}}i64 @test_InterlockedDecrement64(ptr{{.*}}%Addend){{.*}}{
-// CHECK: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i64 1 seq_cst, align 8
+// CHECK: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i64 -1 seq_cst, align 8
// CHECK: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
// CHECK: ret i64 [[RESULT]]
// CHECK: }
@@ -1385,7 +1385,7 @@ short test_InterlockedDecrement16_acq(short volatile *Addend) {
return _InterlockedDecrement16_acq(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_acq(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i16 1 acquire, align 2
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i16 -1 acquire, align 2
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1
// CHECK-ARM-ARM64: ret i16 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1394,7 +1394,7 @@ short test_InterlockedDecrement16_rel(short volatile *Addend) {
return _InterlockedDecrement16_rel(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_rel(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i16 1 release, align 2
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i16 -1 release, align 2
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1
// CHECK-ARM-ARM64: ret i16 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1403,7 +1403,7 @@ short test_InterlockedDecrement16_nf(short volatile *Addend) {
return _InterlockedDecrement16_nf(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i16 @test_InterlockedDecrement16_nf(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i16 1 monotonic, align 2
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i16 -1 monotonic, align 2
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i16 [[TMP]], -1
// CHECK-ARM-ARM64: ret i16 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1412,7 +1412,7 @@ long test_InterlockedDecrement_acq(long volatile *Addend) {
return _InterlockedDecrement_acq(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_acq(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 acquire, align 4
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 acquire, align 4
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM-ARM64: ret i32 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1421,7 +1421,7 @@ long test_InterlockedDecrement_rel(long volatile *Addend) {
return _InterlockedDecrement_rel(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_rel(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 release, align 4
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 release, align 4
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM-ARM64: ret i32 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1430,7 +1430,7 @@ long test_InterlockedDecrement_nf(long volatile *Addend) {
return _InterlockedDecrement_nf(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i32 @test_InterlockedDecrement_nf(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i32 1 monotonic, align 4
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i32 -1 monotonic, align 4
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i32 [[TMP]], -1
// CHECK-ARM-ARM64: ret i32 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1439,7 +1439,7 @@ __int64 test_InterlockedDecrement64_acq(__int64 volatile *Addend) {
return _InterlockedDecrement64_acq(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_acq(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i64 1 acquire, align 8
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i64 -1 acquire, align 8
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
// CHECK-ARM-ARM64: ret i64 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1448,7 +1448,7 @@ __int64 test_InterlockedDecrement64_rel(__int64 volatile *Addend) {
return _InterlockedDecrement64_rel(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_rel(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i64 1 release, align 8
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i64 -1 release, align 8
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
// CHECK-ARM-ARM64: ret i64 [[RESULT]]
// CHECK-ARM-ARM64: }
@@ -1457,7 +1457,7 @@ __int64 test_InterlockedDecrement64_nf(__int64 volatile *Addend) {
return _InterlockedDecrement64_nf(Addend);
}
// CHECK-ARM-ARM64: define{{.*}}i64 @test_InterlockedDecrement64_nf(ptr{{.*}}%Addend){{.*}}{
-// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw sub ptr %Addend, i64 1 monotonic, align 8
+// CHECK-ARM-ARM64: [[TMP:%[0-9]+]] = atomicrmw add ptr %Addend, i64 -1 monotonic, align 8
// CHECK-ARM-ARM64: [[RESULT:%[0-9]+]] = add i64 [[TMP]], -1
// CHECK-ARM-ARM64: ret i64 [[RESULT]]
// CHECK-ARM-ARM64: }
>From 7f658fb2a2ae664d570639402fc50f8212698f71 Mon Sep 17 00:00:00 2001
From: Harrison Hao <tsworld1314 at gmail.com>
Date: Sat, 17 Oct 2026 11:07:27 +0800
Subject: [PATCH 3/3] Support vector
---
llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp | 7 ++++---
llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll | 6 +++---
2 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index f9eab910fc56ae..0dcf1585aef963 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -136,10 +136,11 @@ Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
}
// atomicrmw sub(ptr, C) -> atomicrmw add(ptr, -C)
- const APInt *C;
- if (Op == AtomicRMWInst::Sub && match(Val, m_APInt(C)) && !C->isZero()) {
+ Constant *C;
+ if (Op == AtomicRMWInst::Sub && match(Val, m_ImmConstant(C)) &&
+ !C->isNullValue()) {
RMWI.setOperation(AtomicRMWInst::Add);
- return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), -*C));
+ return replaceOperand(RMWI, 1, ConstantExpr::getNeg(C));
}
}
diff --git a/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll b/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
index 147ae828fdac1d..0fbee0d6ce8780 100644
--- a/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
+++ b/llvm/test/Transforms/InstCombine/atomicrmw-add-neg.ll
@@ -139,7 +139,7 @@ define i64 @neg_fn_sub_constexpr(ptr %a) {
define <2 x i32> @fn_sub_const_vec(ptr %a) {
; CHECK-LABEL: define <2 x i32> @fn_sub_const_vec(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[A]], <2 x i32> splat (i32 5) monotonic, align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[A]], <2 x i32> splat (i32 -5) monotonic, align 8
; CHECK-NEXT: ret <2 x i32> [[OLD]]
;
%old = atomicrmw sub ptr %a, <2 x i32> splat (i32 5) monotonic
@@ -149,7 +149,7 @@ define <2 x i32> @fn_sub_const_vec(ptr %a) {
define <2 x i64> @fn_sub_const_vec_elementwise(ptr %a) {
; CHECK-LABEL: define <2 x i64> @fn_sub_const_vec_elementwise(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[OLD:%.*]] = atomicrmw elementwise sub ptr [[A]], <2 x i64> <i64 1, i64 -9223372036854775808> monotonic, align 16
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw elementwise add ptr [[A]], <2 x i64> <i64 -1, i64 -9223372036854775808> monotonic, align 16
; CHECK-NEXT: ret <2 x i64> [[OLD]]
;
%old = atomicrmw elementwise sub ptr %a, <2 x i64> <i64 1, i64 -9223372036854775808> monotonic
@@ -159,7 +159,7 @@ define <2 x i64> @fn_sub_const_vec_elementwise(ptr %a) {
define <2 x i32> @fn_sub_const_vec_poison(ptr %a) {
; CHECK-LABEL: define <2 x i32> @fn_sub_const_vec_poison(
; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: [[OLD:%.*]] = atomicrmw sub ptr [[A]], <2 x i32> <i32 1, i32 poison> monotonic, align 8
+; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[A]], <2 x i32> <i32 -1, i32 poison> monotonic, align 8
; CHECK-NEXT: ret <2 x i32> [[OLD]]
;
%old = atomicrmw sub ptr %a, <2 x i32> <i32 1, i32 poison> monotonic
More information about the llvm-branch-commits
mailing list