[llvm] [AMDGPU] Constant folding for wave-reduce intrinsics (PR #212755)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 22:33:46 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/212755
>From a2a07c755ecf214c6e490363e3c740b20be5b4d5 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 29 Jul 2026 17:47:23 +0530
Subject: [PATCH] [AMDGPU] Constant folding for wave-reduce intrinsics
---
llvm/lib/Analysis/ConstantFolding.cpp | 10 +
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 43 +-
.../atomic_optimizations_mul_one.ll | 422 ++++++++++++++++--
3 files changed, 441 insertions(+), 34 deletions(-)
rename llvm/test/CodeGen/AMDGPU/{GlobalISel => }/atomic_optimizations_mul_one.ll (63%)
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index a822ab9c5a748..b83c9bb13ecb4 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1804,6 +1804,9 @@ static bool canConstantFoldIntrinsic(Intrinsic::ID ID, bool IsStrictFP) {
case Intrinsic::amdgcn_wave_reduce_min:
case Intrinsic::amdgcn_wave_reduce_and:
case Intrinsic::amdgcn_wave_reduce_or:
+ case Intrinsic::amdgcn_wave_reduce_xor:
+ case Intrinsic::amdgcn_wave_reduce_add:
+ case Intrinsic::amdgcn_wave_reduce_sub:
case Intrinsic::amdgcn_s_wqm:
case Intrinsic::amdgcn_s_quadmask:
case Intrinsic::amdgcn_s_bitreplicate:
@@ -3951,6 +3954,13 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
if (!C0 || !C1)
return Constant::getNullValue(Ty);
return ConstantInt::get(Ty, APIntOps::mulhu(*C0, *C1));
+ case Intrinsic::amdgcn_wave_reduce_add:
+ case Intrinsic::amdgcn_wave_reduce_sub:
+ case Intrinsic::amdgcn_wave_reduce_xor: {
+ if (C0 && C0->isZero())
+ return Constant::getNullValue(Ty);
+ return nullptr;
+ }
case Intrinsic::amdgcn_wave_reduce_umin:
case Intrinsic::amdgcn_wave_reduce_umax:
case Intrinsic::amdgcn_wave_reduce_max:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index f718d9e403209..f9a4ba4c1bd1d 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -6066,11 +6066,27 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
// parity the result will be the same as the input value.
Register ParityRegister =
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
-
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister)
.addReg(NewAccumulator->getOperand(0).getReg())
.addImm(1)
.setOperandDead(3); // Dead scc
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 1) { // 1 * parity(exec) = parity(exec)
+ if (Opc == AMDGPU::S_XOR_B32) {
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg)
+ .addReg(ParityRegister);
+ } else {
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg, ParityRegister, DstHi);
+ }
+ break;
+ }
+ }
if (Opc == AMDGPU::S_XOR_B32) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
@@ -6094,7 +6110,6 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
}
case AMDGPU::S_SUB_I32: {
Register NegatedVal = MRI.createVirtualRegister(DstRegClass);
-
// Take the negation of the source operand.
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedVal)
.addImm(0)
@@ -6105,6 +6120,16 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
break;
}
case AMDGPU::S_ADD_I32: {
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 1) { // 1 * bitcount(exec) = bitcount(exec)
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::COPY), DstReg)
+ .addReg(NewAccumulator->getOperand(0).getReg());
+ break;
+ }
+ }
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg)
.addReg(SrcReg)
.addReg(NewAccumulator->getOperand(0).getReg());
@@ -6127,6 +6152,20 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI,
MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
auto [Op1L, Op1H] = ExtractSubRegs(MI, MI.getOperand(1),
MRI.getRegClass(SrcReg), ST, MRI);
+ // Check if Src is a known identity constant.
+ MachineInstr *SrcDef = MRI.getVRegDef(SrcReg);
+ if (SrcDef && SrcDef->isMoveImmediate()) {
+ int64_t Imm = SrcDef->getOperand(1).getImm();
+ if (Imm == 1 && Opc == AMDGPU::S_ADD_U64_PSEUDO) {
+ // 1 * bitcount(exec) = bitcount(exec)
+ Register DstHi =
+ MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+ BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstHi).addImm(0);
+ BuildRegSequence(BB, MI, DstReg,
+ NewAccumulator->getOperand(0).getReg(), DstHi);
+ break;
+ }
+ }
if (Opc == AMDGPU::S_SUB_U64_PSEUDO) {
BuildMI(BB, MI, DL, TII->get(AMDGPU::S_SUB_I32), NegatedValLo)
.addImm(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
similarity index 63%
rename from llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
rename to llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
index 4d08b2c582d2b..c869e47a19987 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_mul_one.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgpu-- -passes=amdgpu-atomic-optimizer %s | FileCheck -check-prefix=IR %s
; RUN: llc -global-isel -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefix=GCN %s
@@ -13,8 +13,8 @@ declare i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32, ptr addrspace(8),
declare void @llvm.amdgcn.struct.ptr.buffer.store.format.v4i32(<4 x i32>, ptr addrspace(8), i32, i32, i32, i32 immarg)
-define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_add(
+define amdgpu_cs void @atomic_add_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -33,7 +33,7 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_add:
+; GCN-LABEL: atomic_add_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
@@ -53,6 +53,124 @@ define amdgpu_cs void @atomic_add(<4 x i32> inreg %arg) {
ret void
}
+define amdgpu_cs void @atomic_add_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i32_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i32_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB1_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_add v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB1_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_add_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB2_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB2_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_add_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_add_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_add_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_add_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_add_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -85,13 +203,13 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB1_2
+; GCN-NEXT: s_cbranch_execz .LBB4_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB1_2:
+; GCN-NEXT: .LBB4_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -109,8 +227,8 @@ define amdgpu_cs void @atomic_add_and_format(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_sub(
+define amdgpu_cs void @atomic_sub_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -129,26 +247,144 @@ define amdgpu_cs void @atomic_sub(<4 x i32> inreg %arg) {
; IR: [[BB11]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_sub:
+; GCN-LABEL: atomic_sub_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB2_2
+; GCN-NEXT: s_cbranch_execz .LBB5_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB2_2:
+; GCN-NEXT: .LBB5_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
+define amdgpu_cs void @atomic_sub_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i32_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = mul i32 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i32_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_sub v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.sub.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP7]], label %[[BB8:.*]], label %[[BB10:.*]]
+; IR: [[BB8]]:
+; IR-NEXT: [[TMP9:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP6]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB10]]
+; IR: [[BB10]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_sub_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_sub_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = mul i64 0, [[TMP6]]
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_sub_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_sub_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.sub.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_sub_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -181,13 +417,13 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB3_2
+; GCN-NEXT: s_cbranch_execz .LBB9_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB3_2:
+; GCN-NEXT: .LBB9_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -205,8 +441,8 @@ define amdgpu_cs void @atomic_sub_and_format(<4 x i32> inreg %arg) {
ret void
}
-define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
-; IR-LABEL: define amdgpu_cs void @atomic_xor(
+define amdgpu_cs void @atomic_xor_i32_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i32_constant_1(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
; IR-NEXT: [[_ENTRY:.*:]]
; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
@@ -226,27 +462,149 @@ define amdgpu_cs void @atomic_xor(<4 x i32> inreg %arg) {
; IR: [[BB12]]:
; IR-NEXT: ret void
;
-; GCN-LABEL: atomic_xor:
+; GCN-LABEL: atomic_xor_i32_constant_1:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: s_mov_b64 s[4:5], exec
; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s4, 0
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB4_2
+; GCN-NEXT: s_cbranch_execz .LBB10_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB4_2:
+; GCN-NEXT: .LBB10_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
ret void
}
+define amdgpu_cs void @atomic_xor_i32_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i32_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
+; IR-NEXT: [[TMP8:%.*]] = and i32 [[TMP7]], 1
+; IR-NEXT: [[TMP9:%.*]] = mul i32 0, [[TMP8]]
+; IR-NEXT: [[TMP10:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB13:.*]]
+; IR: [[BB11]]:
+; IR-NEXT: [[TMP12:%.*]] = call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 [[TMP9]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB13]]
+; IR: [[BB13]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_i32_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: buffer_atomic_xor v0, v0, s[0:3], 0 idxen
+; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i32 @llvm.amdgcn.struct.buffer.atomic.xor.i32(i32 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_i64_constant_0(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i64_constant_0(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; IR-NEXT: [[TMP8:%.*]] = mul i64 0, [[TMP7]]
+; IR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP9]], label %[[BB10:.*]], label %[[BB12:.*]]
+; IR: [[BB10]]:
+; IR-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP8]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB12]]
+; IR: [[BB12]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_i64_constant_0:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GCN-NEXT: s_cbranch_execz .LBB12_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB12_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 0, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define amdgpu_cs void @atomic_xor_i64_constant_1(<4 x i32> inreg %arg) {
+; IR-LABEL: define amdgpu_cs void @atomic_xor_i64_constant_1(
+; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
+; IR-NEXT: [[_ENTRY:.*:]]
+; IR-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true)
+; IR-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; IR-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP0]], 32
+; IR-NEXT: [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; IR-NEXT: [[TMP4:%.*]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 [[TMP1]], i32 0)
+; IR-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mbcnt.hi(i32 [[TMP3]], i32 [[TMP4]])
+; IR-NEXT: [[TMP6:%.*]] = call i64 @llvm.ctpop.i64(i64 [[TMP0]])
+; IR-NEXT: [[TMP7:%.*]] = and i64 [[TMP6]], 1
+; IR-NEXT: [[TMP8:%.*]] = icmp eq i32 [[TMP5]], 0
+; IR-NEXT: br i1 [[TMP8]], label %[[BB9:.*]], label %[[BB11:.*]]
+; IR: [[BB9]]:
+; IR-NEXT: [[TMP10:%.*]] = call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 [[TMP7]], <4 x i32> [[ARG]], i32 0, i32 0, i32 0, i32 0)
+; IR-NEXT: br label %[[BB11]]
+; IR: [[BB11]]:
+; IR-NEXT: ret void
+;
+; GCN-LABEL: atomic_xor_i64_constant_1:
+; GCN: ; %bb.0: ; %.entry
+; GCN-NEXT: s_mov_b64 s[6:7], exec
+; GCN-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
+; GCN-NEXT: s_mov_b32 s5, 0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GCN-NEXT: s_cbranch_execz .LBB13_2
+; GCN-NEXT: ; %bb.1:
+; GCN-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
+; GCN-NEXT: s_and_b64 s[4:5], s[4:5], 1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: buffer_atomic_xor_x2 v[0:1], v2, s[0:3], 0 idxen
+; GCN-NEXT: .LBB13_2:
+; GCN-NEXT: s_endpgm
+.entry:
+ call i64 @llvm.amdgcn.struct.buffer.atomic.xor.i64(i64 1, <4 x i32> %arg, i32 0, i32 0, i32 0, i32 0)
+ ret void
+}
+
define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; IR-LABEL: define amdgpu_cs void @atomic_xor_and_format(
; IR-SAME: <4 x i32> inreg [[ARG:%.*]]) {
@@ -281,14 +639,14 @@ define amdgpu_cs void @atomic_xor_and_format(<4 x i32> inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB5_2
+; GCN-NEXT: s_cbranch_execz .LBB14_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB5_2:
+; GCN-NEXT: .LBB14_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -334,13 +692,13 @@ define amdgpu_cs void @atomic_ptr_add(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB6_2
+; GCN-NEXT: s_cbranch_execz .LBB15_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_add v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB6_2:
+; GCN-NEXT: .LBB15_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.add.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -381,13 +739,13 @@ define amdgpu_cs void @atomic_ptr_add_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB7_2
+; GCN-NEXT: s_cbranch_execz .LBB16_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_add v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB7_2:
+; GCN-NEXT: .LBB16_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -434,13 +792,13 @@ define amdgpu_cs void @atomic_ptr_sub(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB8_2
+; GCN-NEXT: s_cbranch_execz .LBB17_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_sub v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB8_2:
+; GCN-NEXT: .LBB17_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.sub.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -481,13 +839,13 @@ define amdgpu_cs void @atomic_ptr_sub_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB9_2
+; GCN-NEXT: s_cbranch_execz .LBB18_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_sub v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB9_2:
+; GCN-NEXT: .LBB18_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
@@ -535,14 +893,14 @@ define amdgpu_cs void @atomic_ptr_xor(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s5, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB10_2
+; GCN-NEXT: s_cbranch_execz .LBB19_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GCN-NEXT: s_and_b32 s4, s4, 1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: buffer_atomic_xor v0, v1, s[0:3], 0 idxen
-; GCN-NEXT: .LBB10_2:
+; GCN-NEXT: .LBB19_2:
; GCN-NEXT: s_endpgm
.entry:
call i32 @llvm.amdgcn.struct.ptr.buffer.atomic.xor.i32(i32 1, ptr addrspace(8) %arg, i32 0, i32 0, i32 0, i32 0)
@@ -585,14 +943,14 @@ define amdgpu_cs void @atomic_ptr_xor_and_format(ptr addrspace(8) inreg %arg) {
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: ; implicit-def: $vgpr1
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_cbranch_execz .LBB11_2
+; GCN-NEXT: s_cbranch_execz .LBB20_2
; GCN-NEXT: ; %bb.1:
; GCN-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GCN-NEXT: s_and_b32 s6, s6, 1
; GCN-NEXT: v_mov_b32_e32 v1, s6
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: buffer_atomic_xor v1, v2, s[0:3], 0 idxen glc
-; GCN-NEXT: .LBB11_2:
+; GCN-NEXT: .LBB20_2:
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_readfirstlane_b32 s4, v1
More information about the llvm-commits
mailing list