[llvm] [AMDGPU] Constant folding for wave-reduce intrinsics (PR #212755)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 02:00:26 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/212755
>From a2a2bf8d64e9cb76c8d3001bc5374f74cf893a90 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 29 Jul 2026 17:47:23 +0530
Subject: [PATCH 1/2] [AMDGPU] Constant folding for wave-reduce intrinsics
---
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 68 ++-
.../atomic_optimizations_mul_one.ll | 396 +++++++++++++++++-
2 files changed, 443 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f2b670558de1c..badc5e95b7eb7 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5974,11 +5974,34 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// parity the result will be the same as the input value.
Register ParityRegister =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister)
.addReg(NewAccumulator->getOperand(0).getReg())
.addImm(1)
.setOperandDead(3); // Dead scc
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ unsigned XorMovOp = Opc == AMDGPU::S_XOR_B32
+ ? AMDGPU::S_MOV_B32
+ : AMDGPU::S_MOV_B64_IMM_PSEUDO;
+ if (Imm == 0) { // 0 * parity(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(XorMovOp), DstReg).addImm(0);
+ break;
+ }
+ if (Imm == 1) { // 1 * parity(exec) = parity(exec)
+ if (Opc == AMDGPU::S_XOR_B32) {
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg)
+ .addReg(ParityRegister);
+ } else {
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
+ }
+ break;
+ }
+ }
if (Opc == AMDGPU::S_XOR_B32) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
@@ -6002,7 +6025,15 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
}
case AMDGPU::S_SUB_I32: {
Register NegatedVal = MRI.createVirtualRegister(DstRegClass);
-
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addImm(0);
+ break;
+ }
+ }
// Take the negation of the source operand.
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal)
.addImm(0)
@@ -6013,6 +6044,20 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
break;
}
case AMDGPU::S_ADD_I32: {
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addImm(0);
+ break;
+ }
+ if (Imm == 1) { // 1 * bitcount(exec) = bitcount(exec)
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
+ .addReg(NewAccumulator->getOperand(0).getReg());
+ break;
+ }
+ }
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
.addReg(NewAccumulator->getOperand(0).getReg());
@@ -6035,6 +6080,25 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
MRI.getRegClass(SrcReg), ST, MRI);
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 0) { // 0 * bitcount(exec) = 0
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO), DstReg)
+ .addImm(0);
+ break;
+ }
+ if (Imm == 1 && Opc == AMDGPU::S_ADD_U64_PSEUDO) {
+ // 1 * bitcount(exec) = bitcount(exec)
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg,
+ NewAccumulator->getOperand(0).getReg(), DstHi);
+ break;
+ }
+ }
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
index 4d08b2c582d2b..e8ab55c208970 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgpu-- -passes=amdgpu-atomic-optimizer %s | FileCheck -check-prefix=IR %s
; RUN: llc -global-isel -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefix=GCN %s
@@ -149,6 +149,242 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
ret void
}
+define amdgpu_cs void @atomic_add_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_add v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB4_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_sub v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB5_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB5_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_sub_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -181,13 +417,13 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: s_cbranch_execz .LBB9_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: .LBB9_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -233,20 +469,142 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB4_2
+; GCN-NEXT: s_cbranch_execz .LBB10_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: .LBB10_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
+define amdgpu_cs void @atomic_xor_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP9:%.*]] = mul i32 0, [[TMP8]]
+; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
+; IR: [[BB11]]:
+; IR-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 [[TMP9]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB13]]
+; IR: [[BB13]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_xor v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_constant_0_i64(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0_i64(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; IR-NEXT: [[TMP8:%.*]] = mul i64 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_0_i64:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB12_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB12_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_constant_1_i64(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_1_i64(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_constant_1_i64:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB13_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: s_and_b64 s[4:5], s[4:5], 1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB13_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_xor_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -281,14 +639,14 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB5_2
+; GCN-NEXT: s_cbranch_execz .LBB14_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB5_2:
+; GCN-NEXT: .LBB14_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -334,13 +692,13 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: s_cbranch_execz .LBB15_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: .LBB15_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -381,13 +739,13 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: s_cbranch_execz .LBB16_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: .LBB16_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -434,13 +792,13 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: s_cbranch_execz .LBB17_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: .LBB17_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -481,13 +839,13 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB9_2
+; GCN-NEXT: s_cbranch_execz .LBB18_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB9_2:
+; GCN-NEXT: .LBB18_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -535,14 +893,14 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB10_2
+; GCN-NEXT: s_cbranch_execz .LBB19_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB10_2:
+; GCN-NEXT: .LBB19_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -585,14 +943,14 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: s_cbranch_execz .LBB20_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: .LBB20_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
>From 58c12ff41b8bcee09552cf73da9336cfb6fcafff Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Thu, 30 Jul 2026 14:29:32 +0530
Subject: [PATCH 2/2] 1. Move test out of globalIsel directory. 2. Rename
functions with type.
---
.../atomic_optimizations_mul_one.ll | 254 +++++++++---------
1 file changed, 127 insertions(+), 127 deletions(-)
rename llvm/test/CodeGen/AMDGPU/{GlobalISel => }/atomic_optimizations_mul_one.ll (96%)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
similarity index 96%
rename from llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
rename to llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
index e8ab55c208970..c869e47a19987 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
@@ -13,8 +13,8 @@ declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32, ptr addrspace(8),
declare void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32>, ptr addrspace(8), i32, i32, i32, i32 immarg)
-define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add(
+define amdgpu_cs void @atomic_add_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -33,7 +33,7 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add:
+; GCN-LABEL: atomic_add_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
@@ -53,8 +53,8 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add_and_format(
+define amdgpu_cs void @atomic_add_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i32_constant_0(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -65,52 +65,34 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
-; IR: [[TMP9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB11]]
-; IR: [[BB11]]:
-; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
-; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
-; IR-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP5]]
-; IR-NEXT: call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> [[ARG]], <4 x i32> [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add_and_format:
+; GCN-LABEL: atomic_add_i32_constant_0:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB1_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_add v0, v0, s[0:3], 0 idxen
; GCN-NEXT: .LBB1_2:
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_readfirstlane_b32 s4, v1
-; GCN-NEXT: v_add_i32_e32 v4, vcc, s4, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NEXT: v_mov_b32_e32 v3, s3
-; GCN-NEXT: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
; GCN-NEXT: s_endpgm
.entry:
- %a = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
- call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> %arg, <4 x i32> %arg, i32 %a, i32 0, i32 0, i32 0)
+ call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
-define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_sub(
+define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -120,37 +102,38 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
-; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB11]]
-; IR: [[BB11]]:
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_sub:
+; GCN-LABEL: atomic_add_i64_constant_1:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[4:5], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GCN-NEXT: s_cbranch_execz .LBB2_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
; GCN-NEXT: .LBB2_2:
; GCN-NEXT: s_endpgm
.entry:
- call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
-define amdgpu_cs void @atomic_add_constant_0(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add_constant_0(
+define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_0(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -160,17 +143,16 @@ define amdgpu_cs void @atomic_add_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
-; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB12]]
-; IR: [[BB12]]:
+; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add_constant_0:
+; GCN-LABEL: atomic_add_i64_constant_0:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
@@ -179,16 +161,18 @@ define amdgpu_cs void @atomic_add_constant_0(<4 x i32> inreg %arg) {
; GCN-NEXT: s_cbranch_execz .LBB3_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: buffer_atomic_add v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
; GCN-NEXT: .LBB3_2:
; GCN-NEXT: s_endpgm
.entry:
- call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
-define amdgpu_cs void @atomic_sub_constant_0(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_sub_constant_0(
+define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -199,34 +183,52 @@ define amdgpu_cs void @atomic_sub_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
-; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
-; IR: [[BB10]]:
-; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB12]]
-; IR: [[BB12]]:
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[TMP9:.*]], label %[[BB11:.*]]
+; IR: [[TMP9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: [[TMP12:%.*]] = phi i32 [ poison, [[DOTENTRY:%.*]] ], [ [[TMP10]], %[[TMP9]] ]
+; IR-NEXT: [[TMP13:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[TMP12]])
+; IR-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[TMP5]]
+; IR-NEXT: call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> [[ARG]], <4 x i32> [[ARG]], i32 [[TMP14]], i32 0, i32 0, i32 0)
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_sub_constant_0:
+; GCN-LABEL: atomic_add_and_format:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT: s_cbranch_execz .LBB4_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: buffer_atomic_sub v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v1, s6
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_readfirstlane_b32 s4, v1
+; GCN-NEXT: v_add_i32_e32 v4, vcc, s4, v0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_waitcnt expcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: buffer_store_format_xyzw v[0:3], v4, s[0:3], 0 idxen
; GCN-NEXT: s_endpgm
.entry:
- call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ %a = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ call void @llvm.amdgcn.struct.buffer.store.format.v4i32(<4 x i32> %arg, <4 x i32> %arg, i32 %a, i32 0, i32 0, i32 0)
ret void
}
-define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_1(
+define amdgpu_cs void @atomic_sub_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -236,38 +238,37 @@ define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
-; IR: [[BB8]]:
-; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB10]]
-; IR: [[BB10]]:
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add_i64_constant_1:
+; GCN-LABEL: atomic_sub_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: s_mov_b64 s[6:7], exec
-; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
-; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: s_mov_b64 s[4:5], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GCN-NEXT: s_cbranch_execz .LBB5_2
; GCN-NEXT: ; %bb.1:
-; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
; GCN-NEXT: .LBB5_2:
; GCN-NEXT: s_endpgm
.entry:
- call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
-define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_0(
+define amdgpu_cs void @atomic_sub_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i32_constant_0(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -277,16 +278,17 @@ define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
-; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
-; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
-; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
-; IR: [[BB9]]:
-; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
-; IR-NEXT: br label %[[BB11]]
-; IR: [[BB11]]:
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add_i64_constant_0:
+; GCN-LABEL: atomic_sub_i32_constant_0:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
@@ -295,13 +297,11 @@ define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
; GCN-NEXT: s_cbranch_execz .LBB6_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: buffer_atomic_sub v0, v0, s[0:3], 0 idxen
; GCN-NEXT: .LBB6_2:
; GCN-NEXT: s_endpgm
.entry:
- call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
@@ -441,8 +441,8 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_xor(
+define amdgpu_cs void @atomic_xor_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -462,7 +462,7 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; IR: [[BB12]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_xor:
+; GCN-LABEL: atomic_xor_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
@@ -483,8 +483,8 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_xor_constant_0(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0(
+define amdgpu_cs void @atomic_xor_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i32_constant_0(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -505,7 +505,7 @@ define amdgpu_cs void @atomic_xor_constant_0(<4 x i32> inreg %arg) {
; IR: [[BB13]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_xor_constant_0:
+; GCN-LABEL: atomic_xor_i32_constant_0:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
@@ -522,8 +522,8 @@ define amdgpu_cs void @atomic_xor_constant_0(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_xor_constant_0_i64(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_0_i64(
+define amdgpu_cs void @atomic_xor_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i64_constant_0(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -543,7 +543,7 @@ define amdgpu_cs void @atomic_xor_constant_0_i64(<4 x i32> inreg %arg) {
; IR: [[BB12]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_xor_constant_0_i64:
+; GCN-LABEL: atomic_xor_i64_constant_0:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
@@ -562,8 +562,8 @@ define amdgpu_cs void @atomic_xor_constant_0_i64(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_xor_constant_1_i64(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_xor_constant_1_i64(
+define amdgpu_cs void @atomic_xor_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i64_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -582,7 +582,7 @@ define amdgpu_cs void @atomic_xor_constant_1_i64(<4 x i32> inreg %arg) {
; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_xor_constant_1_i64:
+; GCN-LABEL: atomic_xor_i64_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[6:7], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
More information about the llvm-commits
mailing list