[llvm] 5df6669 - [AMDGPU] Enforce alignment of image vaddr on gfx90a
Stanislav Mekhanoshin via llvm-commits
llvm-commits at lists.llvm.org
Tue May 24 10:05:48 PDT 2022
Author: Stanislav Mekhanoshin
Date: 2022-05-24T10:05:39-07:00
New Revision: 5df6669d45bc72df8a9ac7fb0f4f4cfc00444e0d
URL: https://github.com/llvm/llvm-project/commit/5df6669d45bc72df8a9ac7fb0f4f4cfc00444e0d
DIFF: https://github.com/llvm/llvm-project/commit/5df6669d45bc72df8a9ac7fb0f4f4cfc00444e0d.diff
LOG: [AMDGPU] Enforce alignment of image vaddr on gfx90a
Even though single address image instructions only use a single VGPR
HW accesses 4 or 5 which creates alignment requirement.
Fixes: SWDEV-316648
Differential Revision: https://reviews.llvm.org/D126009
Added:
llvm/test/CodeGen/AMDGPU/verify-image-vaddr-align.mir
Modified:
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
llvm/lib/Target/AMDGPU/MIMGInstructions.td
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/lib/Target/AMDGPU/SIInstrInfo.h
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 25e947dfb022d..2e507b31ec813 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1434,23 +1434,7 @@ bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
if (HasVSrc) {
Register VSrc = MI.getOperand(1).getReg();
-
- if (STI.needsAlignedVGPRs()) {
- // Add implicit aligned super-reg to force alignment on the data operand.
- Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
- BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
- Register NewVR =
- MRI->createVirtualRegister(&AMDGPU::VReg_64_Align2RegClass);
- BuildMI(*MBB, &*MIB, DL, TII.get(AMDGPU::REG_SEQUENCE), NewVR)
- .addReg(VSrc, 0, MI.getOperand(1).getSubReg())
- .addImm(AMDGPU::sub0)
- .addReg(Undef)
- .addImm(AMDGPU::sub1);
- MIB.addReg(NewVR, 0, AMDGPU::sub0);
- MIB.addReg(NewVR, RegState::Implicit);
- } else {
- MIB.addReg(VSrc);
- }
+ MIB.addReg(VSrc);
if (!RBI.constrainGenericRegister(VSrc, AMDGPU::VGPR_32RegClass, *MRI))
return false;
@@ -1459,6 +1443,8 @@ bool AMDGPUInstructionSelector::selectDSGWSIntrinsic(MachineInstr &MI,
MIB.addImm(ImmOffset)
.cloneMemRefs(MI);
+ TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::data0);
+
MI.eraseFromParent();
return true;
}
@@ -1753,7 +1739,9 @@ bool AMDGPUInstructionSelector::selectImageIntrinsic(
}
MI.eraseFromParent();
- return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+ constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+ TII.enforceOperandRCAlignment(*MIB, AMDGPU::OpName::vaddr);
+ return true;
}
bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 9427355a1fbc8..6358547a983e3 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -478,6 +478,7 @@ multiclass MIMG_Store_Addr_Helper <mimgopc op, string asm,
if op.HAS_BASE then {
def _V1 : MIMG_Store_Helper <op, asm, data_rc, VGPR_32,
!if(enableDisasm, "AMDGPU", "")>;
+ let hasPostISelHook = 1 in
def _V1_gfx90a : MIMG_Store_Helper_gfx90a <op, asm, data_rc, VGPR_32,
!if(enableDisasm, "GFX90A", "")>;
def _V1_gfx10 : MIMG_Store_gfx10 <op, asm, data_rc, VGPR_32,
@@ -622,6 +623,7 @@ multiclass MIMG_Atomic_Addr_Helper_m <mimgopc op, string asm,
}
if op.HAS_VI then {
def _V1_vi : MIMG_Atomic_vi <op, asm, data_rc, VGPR_32, enableDasm>;
+ let hasPostISelHook = 1 in
def _V1_gfx90a : MIMG_Atomic_gfx90a <op, asm, data_rc, VGPR_32, enableDasm>;
}
if op.HAS_BASE then {
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 3b65454967e1e..933f7b34954d1 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -4257,29 +4257,7 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
case AMDGPU::DS_GWS_INIT:
case AMDGPU::DS_GWS_SEMA_BR:
case AMDGPU::DS_GWS_BARRIER:
- if (Subtarget->needsAlignedVGPRs()) {
- // Add implicit aligned super-reg to force alignment on the data operand.
- const DebugLoc &DL = MI.getDebugLoc();
- MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
- const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
- MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
- Register DataReg = Op->getReg();
- bool IsAGPR = TRI->isAGPR(MRI, DataReg);
- Register Undef = MRI.createVirtualRegister(
- IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), Undef);
- Register NewVR =
- MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
- : &AMDGPU::VReg_64_Align2RegClass);
- BuildMI(*BB, MI, DL, TII->get(AMDGPU::REG_SEQUENCE), NewVR)
- .addReg(DataReg, 0, Op->getSubReg())
- .addImm(AMDGPU::sub0)
- .addReg(Undef)
- .addImm(AMDGPU::sub1);
- Op->setReg(NewVR);
- Op->setSubReg(AMDGPU::sub0);
- MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
- }
+ TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::data0);
LLVM_FALLTHROUGH;
case AMDGPU::DS_GWS_SEMA_V:
case AMDGPU::DS_GWS_SEMA_P:
@@ -11832,8 +11810,11 @@ void SITargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
return;
}
- if (TII->isMIMG(MI) && !MI.mayStore())
- AddIMGInit(MI);
+ if (TII->isMIMG(MI)) {
+ if (!MI.mayStore())
+ AddIMGInit(MI);
+ TII->enforceOperandRCAlignment(MI, AMDGPU::OpName::vaddr);
+ }
}
static SDValue buildSMovImm32(SelectionDAG &DAG, const SDLoc &DL,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index b23e884a5d0e1..a0b2a6d7439bd 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4619,25 +4619,36 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
}
- if (ST.needsAlignedVGPRs() &&
- (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
- MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
- MI.getOpcode() == AMDGPU::DS_GWS_BARRIER)) {
- const MachineOperand *Op = getNamedOperand(MI, AMDGPU::OpName::data0);
- Register Reg = Op->getReg();
- bool Aligned = true;
- if (Reg.isPhysical()) {
- Aligned = !(RI.getHWRegIndex(Reg) & 1);
- } else {
+ if (ST.needsAlignedVGPRs()) {
+ const auto isAlignedReg = [&MI, &MRI, this](unsigned OpName) -> bool {
+ const MachineOperand *Op = getNamedOperand(MI, OpName);
+ if (!Op)
+ return true;
+ Register Reg = Op->getReg();
+ if (Reg.isPhysical())
+ return !(RI.getHWRegIndex(Reg) & 1);
const TargetRegisterClass &RC = *MRI.getRegClass(Reg);
- Aligned = RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
- !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
+ return RI.getRegSizeInBits(RC) > 32 && RI.isProperlyAlignedRC(RC) &&
+ !(RI.getChannelFromSubReg(Op->getSubReg()) & 1);
+ };
+
+ if (MI.getOpcode() == AMDGPU::DS_GWS_INIT ||
+ MI.getOpcode() == AMDGPU::DS_GWS_SEMA_BR ||
+ MI.getOpcode() == AMDGPU::DS_GWS_BARRIER) {
+
+ if (!isAlignedReg(AMDGPU::OpName::data0)) {
+ ErrInfo = "Subtarget requires even aligned vector registers "
+ "for DS_GWS instructions";
+ return false;
+ }
}
- if (!Aligned) {
- ErrInfo = "Subtarget requires even aligned vector registers "
- "for DS_GWS instructions";
- return false;
+ if (isMIMG(MI)) {
+ if (!isAlignedReg(AMDGPU::OpName::vaddr)) {
+ ErrInfo = "Subtarget requires even aligned vector registers "
+ "for vaddr operand of image instructions";
+ return false;
+ }
}
}
@@ -8427,3 +8438,37 @@ bool SIInstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
return false;
}
+
+void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI,
+ unsigned OpName) const {
+ if (!ST.needsAlignedVGPRs())
+ return;
+
+ int OpNo = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
+ if (OpNo < 0)
+ return;
+ MachineOperand &Op = MI.getOperand(OpNo);
+ if (getOpSize(MI, OpNo) > 4)
+ return;
+
+ // Add implicit aligned super-reg to force alignment on the data operand.
+ const DebugLoc &DL = MI.getDebugLoc();
+ MachineBasicBlock *BB = MI.getParent();
+ MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
+ Register DataReg = Op.getReg();
+ bool IsAGPR = RI.isAGPR(MRI, DataReg);
+ Register Undef = MRI.createVirtualRegister(
+ IsAGPR ? &AMDGPU::AGPR_32RegClass : &AMDGPU::VGPR_32RegClass);
+ BuildMI(*BB, MI, DL, get(AMDGPU::IMPLICIT_DEF), Undef);
+ Register NewVR =
+ MRI.createVirtualRegister(IsAGPR ? &AMDGPU::AReg_64_Align2RegClass
+ : &AMDGPU::VReg_64_Align2RegClass);
+ BuildMI(*BB, MI, DL, get(AMDGPU::REG_SEQUENCE), NewVR)
+ .addReg(DataReg, 0, Op.getSubReg())
+ .addImm(AMDGPU::sub0)
+ .addReg(Undef)
+ .addImm(AMDGPU::sub1);
+ Op.setReg(NewVR);
+ Op.setSubReg(AMDGPU::sub0);
+ MI.addOperand(MachineOperand::CreateReg(NewVR, false, true));
+}
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index a3357be4ff512..6c6da68736937 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1150,6 +1150,11 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
static unsigned getDSShaderTypeValue(const MachineFunction &MF);
const TargetSchedModel &getSchedModel() const { return SchedModel; }
+
+ // Enforce operand's \p OpName even alignment if required by target.
+ // This is used if an operand is a 32 bit register but needs to be aligned
+ // regardless.
+ void enforceOperandRCAlignment(MachineInstr &MI, unsigned OpName) const;
};
/// \brief Returns true if a reg:subreg pair P has a TRC class
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
index a617ed96e79be..4315f34724e51 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
@@ -1,9 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -o - %s | FileCheck -check-prefix=GFX6 %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX900 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -o - %s | FileCheck -check-prefix=GFX90A %s
; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10 %s
-
define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
; GFX6-LABEL: atomic_swap_i32_1d:
; GFX6: ; %bb.0: ; %main_body
@@ -33,6 +34,35 @@ define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_swap_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_swap_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_swap v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_swap_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -81,6 +111,35 @@ define amdgpu_ps float @atomic_add_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_add v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -129,6 +188,35 @@ define amdgpu_ps float @atomic_sub_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_sub_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_sub_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_sub v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_sub_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -177,6 +265,35 @@ define amdgpu_ps float @atomic_smin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_smin_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smin_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_smin v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_smin_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -225,6 +342,35 @@ define amdgpu_ps float @atomic_umin_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_umin_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umin_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_umin v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_umin_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -273,6 +419,35 @@ define amdgpu_ps float @atomic_smax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_smax_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smax_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_smax v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_smax_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -321,6 +496,35 @@ define amdgpu_ps float @atomic_umax_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_umax_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umax_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_umax v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_umax_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -369,6 +573,35 @@ define amdgpu_ps float @atomic_and_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_and_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_and_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_and v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_and_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -417,6 +650,35 @@ define amdgpu_ps float @atomic_or_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_or_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_or_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_or v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_or_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -465,6 +727,35 @@ define amdgpu_ps float @atomic_xor_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_xor_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_xor_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_xor v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_xor_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -513,6 +804,35 @@ define amdgpu_ps float @atomic_inc_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_inc_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_inc_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_inc v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_inc_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -561,6 +881,35 @@ define amdgpu_ps float @atomic_dec_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_dec_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_dec_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_dec v0, v2, s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_dec_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -609,6 +958,34 @@ define amdgpu_ps float @atomic_cmpswap_i32_1d(<8 x i32> inreg %rsrc, i32 %cmp, i
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_cmpswap_i32_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_cmpswap_i32_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_cmpswap_i32_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -655,6 +1032,32 @@ define amdgpu_ps void @atomic_cmpswap_i32_1d_no_return(<8 x i32> inreg %rsrc, i3
; GFX8-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
; GFX8-NEXT: s_endpgm
;
+; GFX900-LABEL: atomic_cmpswap_i32_1d_no_return:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_endpgm
+;
+; GFX90A-LABEL: atomic_cmpswap_i32_1d_no_return:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_endpgm
+;
; GFX10-LABEL: atomic_cmpswap_i32_1d_no_return:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -701,6 +1104,36 @@ define amdgpu_ps float @atomic_add_i32_2d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_2d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: image_atomic_add v0, v[4:5], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -749,6 +1182,37 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i32
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_3d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_3d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_3d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -797,6 +1261,37 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_cube:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_cube:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_cube:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -845,6 +1340,36 @@ define amdgpu_ps float @atomic_add_i32_1darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_1darray:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1darray:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: image_atomic_add v0, v[4:5], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_1darray:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -893,6 +1418,37 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_2darray:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2darray:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_2darray:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -941,6 +1497,37 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_2dmsaa:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2dmsaa:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v5, v2
+; GFX90A-NEXT: v_mov_b32_e32 v6, v3
+; GFX90A-NEXT: image_atomic_add v0, v[4:6], s[0:7] dmask:0x1 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_2dmsaa:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -989,6 +1576,38 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_2darraymsaa:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_2darraymsaa:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v6, v1
+; GFX90A-NEXT: v_mov_b32_e32 v7, v2
+; GFX90A-NEXT: v_mov_b32_e32 v8, v3
+; GFX90A-NEXT: v_mov_b32_e32 v9, v4
+; GFX90A-NEXT: image_atomic_add v0, v[6:9], s[0:7] dmask:0x1 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_2darraymsaa:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1037,6 +1656,35 @@ define amdgpu_ps float @atomic_add_i32_1d_slc(<8 x i32> inreg %rsrc, i32 %data,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i32_1d_slc:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i32_1d_slc:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: v_mov_b32_e32 v2, v1
+; GFX90A-NEXT: image_atomic_add v0, v2, s[0:7] dmask:0x1 unorm glc slc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i32_1d_slc:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1085,6 +1733,34 @@ define amdgpu_ps <2 x float> @atomic_swap_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_swap_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_swap_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_swap_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1133,6 +1809,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1181,6 +1885,34 @@ define amdgpu_ps <2 x float> @atomic_sub_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_sub_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_sub_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_sub v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_sub_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1229,6 +1961,34 @@ define amdgpu_ps <2 x float> @atomic_smin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_smin_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smin_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_smin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_smin_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1277,6 +2037,34 @@ define amdgpu_ps <2 x float> @atomic_umin_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_umin_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umin_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_umin v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_umin_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1325,6 +2113,34 @@ define amdgpu_ps <2 x float> @atomic_smax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_smax_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_smax_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_smax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_smax_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1373,6 +2189,34 @@ define amdgpu_ps <2 x float> @atomic_umax_i64_1d(<8 x i32> inreg %rsrc, i64 %dat
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_umax_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_umax_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_umax v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_umax_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1421,6 +2265,34 @@ define amdgpu_ps <2 x float> @atomic_and_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_and_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_and_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_and v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_and_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1469,6 +2341,34 @@ define amdgpu_ps <2 x float> @atomic_or_i64_1d(<8 x i32> inreg %rsrc, i64 %data,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_or_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_or_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_or v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_or_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1517,6 +2417,34 @@ define amdgpu_ps <2 x float> @atomic_xor_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_xor_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_xor_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_xor v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_xor_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1565,6 +2493,34 @@ define amdgpu_ps <2 x float> @atomic_inc_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_inc_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_inc_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_inc v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_inc_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1613,6 +2569,34 @@ define amdgpu_ps <2 x float> @atomic_dec_i64_1d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_dec_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_dec_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_dec v[0:1], v2, s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_dec_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1661,6 +2645,34 @@ define amdgpu_ps <2 x float> @atomic_cmpswap_i64_1d(<8 x i32> inreg %rsrc, i64 %
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_cmpswap_i64_1d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_cmpswap_i64_1d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_cmpswap_i64_1d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1707,6 +2719,32 @@ define amdgpu_ps void @atomic_cmpswap_i64_1d_no_return(<8 x i32> inreg %rsrc, i6
; GFX8-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
; GFX8-NEXT: s_endpgm
;
+; GFX900-LABEL: atomic_cmpswap_i64_1d_no_return:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX900-NEXT: s_endpgm
+;
+; GFX90A-LABEL: atomic_cmpswap_i64_1d_no_return:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
+; GFX90A-NEXT: s_endpgm
+;
; GFX10-LABEL: atomic_cmpswap_i64_1d_no_return:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1753,6 +2791,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_2d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_2d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1801,6 +2867,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_3d:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_3d:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_3d:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1849,6 +2943,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_cube:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_cube:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_cube:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1897,6 +3019,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1darray(<8 x i32> inreg %rsrc, i64
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_1darray:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1darray:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_1darray:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1945,6 +3095,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_2darray:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2darray:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_2darray:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -1993,6 +3171,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_2dmsaa:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2dmsaa:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:4], s[0:7] dmask:0x3 unorm glc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_2dmsaa:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -2041,6 +3247,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_2darraymsaa:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_2darraymsaa:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v[2:5], s[0:7] dmask:0x3 unorm glc da
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_2darraymsaa:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
@@ -2089,6 +3323,34 @@ define amdgpu_ps <2 x float> @atomic_add_i64_1d_slc(<8 x i32> inreg %rsrc, i64 %
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: ; return to shader part epilog
;
+; GFX900-LABEL: atomic_add_i64_1d_slc:
+; GFX900: ; %bb.0: ; %main_body
+; GFX900-NEXT: s_mov_b32 s0, s2
+; GFX900-NEXT: s_mov_b32 s1, s3
+; GFX900-NEXT: s_mov_b32 s2, s4
+; GFX900-NEXT: s_mov_b32 s3, s5
+; GFX900-NEXT: s_mov_b32 s4, s6
+; GFX900-NEXT: s_mov_b32 s5, s7
+; GFX900-NEXT: s_mov_b32 s6, s8
+; GFX900-NEXT: s_mov_b32 s7, s9
+; GFX900-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: ; return to shader part epilog
+;
+; GFX90A-LABEL: atomic_add_i64_1d_slc:
+; GFX90A: ; %bb.0: ; %main_body
+; GFX90A-NEXT: s_mov_b32 s0, s2
+; GFX90A-NEXT: s_mov_b32 s1, s3
+; GFX90A-NEXT: s_mov_b32 s2, s4
+; GFX90A-NEXT: s_mov_b32 s3, s5
+; GFX90A-NEXT: s_mov_b32 s4, s6
+; GFX90A-NEXT: s_mov_b32 s5, s7
+; GFX90A-NEXT: s_mov_b32 s6, s8
+; GFX90A-NEXT: s_mov_b32 s7, s9
+; GFX90A-NEXT: image_atomic_add v[0:1], v2, s[0:7] dmask:0x3 unorm glc slc
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: ; return to shader part epilog
+;
; GFX10-LABEL: atomic_add_i64_1d_slc:
; GFX10: ; %bb.0: ; %main_body
; GFX10-NEXT: s_mov_b32 s0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
index 5aea86567b498..4ba5fa09f5878 100644
--- a/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
+++ b/llvm/test/CodeGen/AMDGPU/alloc-aligned-tuples-gfx90a.mir
@@ -73,7 +73,7 @@ body: |
%0:vreg_64_align2 = COPY $vgpr0_vgpr1
%1:vgpr_32 = COPY $vgpr2
- %2:vreg_160_align2 = IMAGE_LOAD_V5_V1 %1, undef %3:sgpr_256, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
+ %2:vreg_160_align2 = IMAGE_LOAD_V5_V1 %0.sub0, undef %3:sgpr_256, 0, 0, 0, 0, 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s32), addrspace 4)
GLOBAL_STORE_DWORDX4 %0, %2.sub0_sub1_sub2_sub3, 0, 0, implicit $exec
GLOBAL_STORE_DWORD %0, %1, 0, 0, implicit $exec
...
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll
index 40bd98fae3e65..d05a98336fb23 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.dim.ll
@@ -1,10 +1,11 @@
; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX6789 %s
; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX6789 %s
-; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX90A %s
; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs -show-mc-encoding < %s | FileCheck -check-prefix=GCN -check-prefix=GFX10 %s
; GCN-LABEL: {{^}}atomic_swap_1d:
; GFX6789: image_atomic_swap v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_swap v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_swap v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -15,6 +16,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_swap_1d_i64:
; GFX6789: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
+; GFX90A: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
; GFX10: image_atomic_swap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps <2 x float> @atomic_swap_1d_i64(<8 x i32> inreg %rsrc, i64 %data, i32 %s) {
main_body:
@@ -25,6 +27,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_1d:
; GFX6789: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -35,6 +38,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_sub_1d:
; GFX6789: image_atomic_sub v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_sub v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_sub v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -45,6 +49,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_smin_1d:
; GFX6789: image_atomic_smin v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_smin v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_smin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -55,6 +60,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_umin_1d:
; GFX6789: image_atomic_umin v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_umin v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_umin v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -65,6 +71,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_smax_1d:
; GFX6789: image_atomic_smax v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_smax v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_smax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -75,6 +82,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_umax_1d:
; GFX6789: image_atomic_umax v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_umax v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_umax v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -85,6 +93,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_and_1d:
; GFX6789: image_atomic_and v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_and v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_and v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -95,6 +104,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_or_1d:
; GFX6789: image_atomic_or v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_or v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_or v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -105,6 +115,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_xor_1d:
; GFX6789: image_atomic_xor v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_xor v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_xor v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -115,6 +126,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_inc_1d:
; GFX6789: image_atomic_inc v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_inc v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_inc v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -125,6 +137,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_dec_1d:
; GFX6789: image_atomic_dec v0, v1, s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_dec v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_dec v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
@@ -135,6 +148,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_cmpswap_1d:
; GFX6789: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
+; GFX90A: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc{{$}}
; GFX10: image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %swap, i32 %s) {
main_body:
@@ -145,6 +159,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_cmpswap_1d_64:
; GFX6789: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
+; GFX90A: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc{{$}}
; GFX10: image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc ;
define amdgpu_ps <2 x float> @atomic_cmpswap_1d_64(<8 x i32> inreg %rsrc, i64 %cmp, i64 %swap, i32 %s) {
main_body:
@@ -155,6 +170,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_2d:
; GFX6789: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D unorm glc ;
define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t) {
main_body:
@@ -165,6 +181,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_3d:
; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc ;
define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %r) {
main_body:
@@ -175,6 +192,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_cube:
; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc ;
define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %face) {
main_body:
@@ -185,6 +203,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_1darray:
; GFX6789: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
; GFX10: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D_ARRAY unorm glc ;
define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %slice) {
main_body:
@@ -195,6 +214,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_2darray:
; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc ;
define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice) {
main_body:
@@ -205,6 +225,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_2dmsaa:
; GFX6789: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 unorm glc{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[02468]}}], s[0:7] dmask:0x1 unorm glc{{$}}
; GFX10: image_atomic_add v0, v[1:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc ;
define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %fragid) {
main_body:
@@ -215,6 +236,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_2darraymsaa:
; GFX6789: image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 unorm glc da{{$}}
+; GFX90A: image_atomic_add v0, v[{{[02468]}}:{{[13579]}}], s[0:7] dmask:0x1 unorm glc da{{$}}
; GFX10: image_atomic_add v0, v[1:4], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc ;
define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
main_body:
@@ -225,6 +247,7 @@ main_body:
; GCN-LABEL: {{^}}atomic_add_1d_slc:
; GFX6789: image_atomic_add v0, v1, s[0:7] dmask:0x1 unorm glc slc{{$}}
+; GFX90A: image_atomic_add v0, v{{[02468]}}, s[0:7] dmask:0x1 unorm glc slc{{$}}
; GFX10: image_atomic_add v0, v1, s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_1D unorm glc slc ;
define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
index d5e5bc1d052c6..15269a88e61c7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.gfx90a.ll
@@ -1,4 +1,5 @@
; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
; GCN-LABEL: {{^}}load_1d:
; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}}
@@ -97,6 +98,16 @@ main_body:
ret <4 x float> %v
}
+; GCN-LABEL: {{^}}load_1d_addr_align:
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_load v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm{{$}}
+define amdgpu_ps <4 x float> @load_1d_addr_align(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+main_body:
+ %s1 = extractelement <2 x i32> %s, i32 1
+ %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x float> %v
+}
+
; GCN-LABEL: {{^}}store_1d:
; GCN: image_store v[0:3], v4, s[0:7] dmask:0xf unorm{{$}}
define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, i32 %s) {
@@ -178,7 +189,8 @@ main_body:
}
; GCN-LABEL: {{^}}store_1d_V1:
-; GCN: image_store v0, v1, s[0:7] dmask:0x2 unorm{{$}}
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_store v0, [[VADDR]], s[0:7] dmask:0x2 unorm{{$}}
define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, i32 %s) {
main_body:
call void @llvm.amdgcn.image.store.1d.f32.i32(float %vdata, i32 2, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
@@ -256,7 +268,7 @@ main_body:
}
; GCN-LABEL: image_load_mmo
-; GCN: image_load v1, v[2:3], s[0:7] dmask:0x1 unorm
+; GCN: image_load v1, v[{{[0-9:]+}}], s[0:7] dmask:0x1 unorm
define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, float addrspace(3)* %lds, <2 x i32> %c) #0 {
store float 0.000000e+00, float addrspace(3)* %lds
%c0 = extractelement <2 x i32> %c, i32 0
@@ -267,6 +279,15 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, float addrspace(3)
ret float %tex
}
+; GCN: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; GCN: image_get_resinfo v[0:3], [[VADDR]], s[0:7] dmask:0xf unorm
+define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i32> %s) {
+main_body:
+ %s1 = extractelement <2 x i32> %s, i32 1
+ %v = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %s1, <8 x i32> %rsrc, i32 0, i32 0)
+ ret <4 x float> %v
+}
+
declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
declare {float,i32} @llvm.amdgcn.image.load.1d.f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
declare {<2 x float>,i32} @llvm.amdgcn.image.load.1d.v2f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
@@ -301,6 +322,8 @@ declare <2 x float> @llvm.amdgcn.image.load.1d.v2f32.i32(i32, i32, <8 x i32>, i3
declare void @llvm.amdgcn.image.store.1d.f32.i32(float, i32, i32, <8 x i32>, i32, i32) #0
declare void @llvm.amdgcn.image.store.1d.v2f32.i32(<2 x float>, i32, i32, <8 x i32>, i32, i32) #0
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #2
+
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll
index dead4ecbaae7d..e712a18b74df3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.gfx90a.ll
@@ -1,4 +1,5 @@
-; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A %s
+; RUN: llc -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A,SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A,GISEL %s
; GFX90A-LABEL: {{^}}sample_1d:
; GFX90A-NOT: s_wqm_b64
@@ -66,6 +67,20 @@ main_body:
ret <4 x float> %v
}
+; Address register must be even aligned.
+
+; GFX90A-LABEL: {{^}}sample_1d_addr_align:
+; GFX90A: v_mov_b32_e32 [[VADDR:v[0-9]?[02468]]], v1
+; SDAG: image_sample v{{[0-9]+}}, [[VADDR]], s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0x1
+; GISEL: image_sample v[{{[0-9:]+}}], [[VADDR]], s[{{[0-9:]+}}], s[{{[0-9:]+}}] dmask:0xf
+define amdgpu_ps float @sample_1d_addr_align(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <2 x float> %s) {
+main_body:
+ %s1 = extractelement <2 x float> %s, i32 1
+ %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s1, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 0, i32 0)
+ %v1 = extractelement <4 x float> %v, i32 0
+ ret float %v1
+}
+
declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32)
declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/verify-image-vaddr-align.mir b/llvm/test/CodeGen/AMDGPU/verify-image-vaddr-align.mir
new file mode 100644
index 0000000000000..07f4067ad0b19
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/verify-image-vaddr-align.mir
@@ -0,0 +1,27 @@
+# RUN: not --crash llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -run-pass=machineverifier -o /dev/null %s 2>&1 | FileCheck -check-prefix=GFX90A-ERR %s
+
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %4:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx90a %0.sub1:vreg_128_align2
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: $vgpr0 = IMAGE_SAMPLE_V1_V1_gfx90a $vgpr1,
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %5:vgpr_32 = IMAGE_LOAD_V1_V1_gfx90a %0.sub1:vreg_128_align2
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: IMAGE_STORE_V1_V1_gfx90a $vgpr1,
+# GFX90A-ERR: *** Bad machine code: Subtarget requires even aligned vector registers for vaddr operand of image instructions ***
+# GFX90A-ERR: %6:vgpr_32 = IMAGE_ATOMIC_SWAP_V1_V1_gfx90a %6:vgpr_32(tied-def 0), $vgpr1,
+---
+name: image_sample_odd_vgpr
+body: |
+ bb.0:
+ %0:vreg_128_align2 = IMPLICIT_DEF
+ %1:areg_128_align2 = IMPLICIT_DEF
+ %2:sgpr_256 = IMPLICIT_DEF
+ %3:sgpr_128 = IMPLICIT_DEF
+
+ %4:vgpr_32 = IMAGE_SAMPLE_V1_V1_gfx90a %0.sub1, %2, %3, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from custom "ImageResource")
+ $vgpr0 = IMAGE_SAMPLE_V1_V1_gfx90a $vgpr1, %2, %3, 1, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from custom "ImageResource")
+ %5:vgpr_32 = IMAGE_LOAD_V1_V1_gfx90a %0.sub1, %2, 8, -1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32) from custom "ImageResource")
+ IMAGE_STORE_V1_V1_gfx90a $vgpr1, %5, %2, 2, -1, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store (s32) into custom "ImageResource")
+ %6:vgpr_32 = IMAGE_ATOMIC_SWAP_V1_V1_gfx90a %6:vgpr_32, $vgpr1, %2, 1, -1, 1, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on custom "ImageResource")
+...
More information about the llvm-commits
mailing list