[llvm] [AMDGPU] using divergent/uniform information in ISel of zext (PR #174539)
Zeng Wu via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 00:08:50 PDT 2026
https://github.com/zwu-2025 updated https://github.com/llvm/llvm-project/pull/174539
>From b20ed163ed103c6d02036bd64af37ab5c610d7c2 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Mon, 23 Mar 2026 17:49:18 +0000
Subject: [PATCH 01/31] changes decision order of UseRC and use
common-sub-regclass
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 21 +++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 322766dee5aa9..20c02317019c6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -24,9 +24,11 @@
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/PseudoProbe.h"
+#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
@@ -103,12 +105,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
bool MatchReg = true;
MVT VT = Op.getSimpleValueType();
-
- // FIXME: The Untyped check is a workaround for SystemZ i128 inline assembly
- // using i128, when it should probably be using v2i64.
- const TargetRegisterClass *UseRC =
- VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
-
+ const TargetRegisterClass *UseRC = nullptr;
for (SDNode *User : Op->users()) {
bool Match = true;
if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2) == Op) {
@@ -132,6 +129,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
RC = TRI->getAllocatableClass(
TII->getRegClass(II, i + II.getNumDefs()));
}
+
if (!UseRC)
UseRC = RC;
else if (RC) {
@@ -150,6 +148,17 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
break;
}
+ // FIXME: The Untyped check is a workaround for SystemZ i128 inline assembly
+ // using i128, when it should probably be using v2i64.
+ const TargetRegisterClass *RegClassForVT =
+ VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
+
+ if (UseRC == nullptr || !UseRC->isAllocatable()) {
+ UseRC = RegClassForVT;
+ } else if (auto CommonSubClass = TRI->getCommonSubClass(UseRC, RegClassForVT)) {
+ UseRC = CommonSubClass;
+ }
+
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
>From ce4a770ba8f8b677a497bae30bb97d4b48abf087 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Mon, 23 Mar 2026 21:58:21 +0000
Subject: [PATCH 02/31] Select S_AND for ZEXT Only if directly from ICMP and
UNIFORM
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 61 +++++++++++++++++--
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 +
.../llvm.amdgcn.uniform_ext_i1_to_i32.ll | 47 ++++++++++++++
3 files changed, 105 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index cc2058a5a1d4a..d630cf46b7dbe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -325,10 +325,57 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
return false;
}
-void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
- if (!Subtarget->d16PreservesUnusedBits())
- return;
+bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
+ // For the DAG:
+ // t134: i1 = setcc t52, Constant:i32<0>, setne:ch
+ // t133: i32 = zero_extend t134
+ // After DAG selections, it is:
+ // t134: i1 = S_CMP_LG_U32 t52, t19
+ // t133: i32,i1 = S_AND_B32 t134, TargetConstant:i32<1>
+ //
+ // And the listed MIR are:
+ // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
+ // %27:sreg_32 = COPY $scc
+ // %28:sreg_32 = S_AND_B32 killed %27:sreg_32, 1, implicit-def dead $scc
+ //
+ // After SI-Fix-SGPR-Copies, if wave size is 64, the fixSCCCopy will change it to
+ // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
+ // %79:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit $scc
+ // %27:sreg_32 = COPY %79:sreg_64_xexec; illegal vgpr to sgpr copy
+ //
+ // This preprocess changes DAG
+ // t133: i32 = zero_extend t134 into --> S_CSELEECT_B32 -1 0
+ // if it is valid.
+ if (N->isDivergent()) return false;
+
+ // If to i64, it is probably used for lane mask, then we don't do the changes here.
+ if (N->getValueType(0) != MVT::i32) return false;
+
+ auto CondNode = N->getOperand(0);
+ if (CondNode->getOpcode() == ISD::FREEZE) {
+ CondNode = CondNode->getOperand(0);
+ }
+
+ if (CondNode->getOpcode() != ISD::SETCC)
+ return false;
+
+ if (CondNode->getOperand(0).getValueType() != MVT::i32 || CondNode->getOperand(1).getValueType() != MVT::i32)
+ return false;
+
+ SDLoc DL(N);
+ SDValue TrueValue = CurDAG->getAllOnesConstant(DL, MVT::i32, true);
+ SDValue FalseValue = CurDAG->getTargetConstant(0, DL, MVT::i32);
+ SDValue Chain = CurDAG->getEntryNode(); // Basic chain to anchor the copy
+ SDValue CopyToSCC = CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
+ SmallVector<SDValue, 3> Ops = {TrueValue, FalseValue, CopyToSCC.getValue(1)};
+ MachineSDNode *CSelectNode = CurDAG->getMachineNode(AMDGPU::S_CSELECT_B32, DL, CurDAG->getVTList(MVT::i32), Ops);
+ CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
+
+ return true;
+}
+
+void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
bool MadeChange = false;
@@ -340,8 +387,14 @@ void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
switch (N->getOpcode()) {
case ISD::BUILD_VECTOR:
// TODO: Match load d16 from shl (extload:i16), 16
- MadeChange |= matchLoadD16FromBuildVector(N);
+ if (Subtarget->d16PreservesUnusedBits())
+ MadeChange |= matchLoadD16FromBuildVector(N);
+
break;
+ case ISD::ZERO_EXTEND:
+ MadeChange |= preprocessZeroExtend(N);
+ break;
+
default:
break;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index ffeb6dfdb3f90..9cc34cc769cf0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -67,6 +67,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
bool runOnMachineFunction(MachineFunction &MF) override;
bool matchLoadD16FromBuildVector(SDNode *N) const;
+ bool preprocessZeroExtend(SDNode *N) const;
void PreprocessISelDAG() override;
void Select(SDNode *N) override;
void PostprocessISelDAG() override;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
new file mode 100644
index 0000000000000..f5cc63694be9a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+entry:
+ ; GFX950-LABEL: zext_i1_to_i64_uniform
+ ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
+ ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+ ; GFX950-NEXT: s_mov_b32 s4, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v2, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v1, s4
+ ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+ ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+ ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+ ; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+ ; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+ ; GFX950-NEXT: s_endpgm
+
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+entry:
+ ; GFX950-LABEL: zext_i1_to_i32_uniform
+ ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
+ ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+ ; GFX950-NEXT: v_mov_b32_e32 v0, 0
+ ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+ ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+ ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+ ; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+ ; GFX950-NEXT: s_cselect_b32 s0, -1, 0
+ ; GFX950-NEXT: v_mov_b32_e32 v1, s0
+ ; GFX950-NEXT: global_store_dword v0, v1, s[2:3]
+ ; GFX950-NEXT: s_endpgm
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+attributes #0 = { nounwind }
>From 5cf7ac1a475840ed11253ef222af48b3eadf7702 Mon Sep 17 00:00:00 2001
From: zwu-2025 <Zeng.Wu2 at amd.com>
Date: Sun, 19 Apr 2026 05:25:02 +0000
Subject: [PATCH 03/31] validate scc is not cross bb before and after
si-fix-sgpr-copies
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 80 ++++++++++++++++++++++
1 file changed, 80 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 5994eeb54095b..066f11c932fe5 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -159,6 +159,8 @@ class SIFixSGPRCopies {
MachineBasicBlock *BlockToInsertTo,
MachineBasicBlock::iterator PointToInsertTo,
const DebugLoc &DL);
+
+ void validateNoCrossBlockSCC(MachineFunction &MF);
};
class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
@@ -623,6 +625,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getProperties().hasSelected())
return false;
+ validateNoCrossBlockSCC(MF);
+
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
@@ -781,8 +785,12 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
}
lowerVGPR2SGPRCopies(MF);
+
+ validateNoCrossBlockSCC(MF);
// Postprocessing
fixSCCCopies(MF);
+ validateNoCrossBlockSCC(MF);
+
for (auto *MI : S2VCopies) {
// Check if it is still valid
if (MI->isCopy()) {
@@ -806,6 +814,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
+ validateNoCrossBlockSCC(MF);
+
SiblingPenalty.clear();
V2SCopies.clear();
SCCCopies.clear();
@@ -1012,6 +1022,10 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
AnalysisWorklist.push_back(U);
}
}
+
+ for (auto I : Info.SChain) {
+ LLVM_DEBUG(dbgs() << "Chain 12345:"; I->dump(););
+ }
V2SCopies[Info.ID] = std::move(Info);
}
@@ -1055,6 +1069,72 @@ bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
return Info->NeedToBeConvertedToVALU;
}
+void SIFixSGPRCopies::validateNoCrossBlockSCC(MachineFunction &MF) {
+
+ for (const MachineBasicBlock &MBB : MF) {
+ // assert(!MBB.isLiveIn(AMDGPU::SCC) && "AMDGPU::SCC is live-in to a basic
+ // block! Cross-BB SCC liveness detected.");
+
+ bool SCCDefinedInCurrentBlock = false;
+
+ for (const MachineInstr &MI : MBB) {
+ bool ReadsSCC = false;
+ bool ModifiesSCC = false;
+
+ for (const MachineOperand &MO : MI.operands()) {
+ if (MO.isRegMask() && MO.clobbersPhysReg(AMDGPU::SCC)) {
+ ModifiesSCC = true;
+ continue;
+ }
+
+ if (!MO.isReg())
+ continue;
+
+ Register Reg = MO.getReg();
+ if (!Reg || !Reg.isPhysical())
+ continue;
+
+ unsigned BitSize = 0;
+ if (Reg.isPhysical()) {
+ const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+ BitSize = TRI->getRegSizeInBits(*RC);
+ } else {
+ if (MRI->getType(Reg).isValid()) {
+ BitSize = MRI->getType(Reg).getSizeInBits();
+ } else {
+ const TargetRegisterClass *RC = MRI->getRegClass(Reg);
+ if (RC)
+ BitSize = TRI->getRegSizeInBits(*RC);
+ }
+ }
+
+ if (BitSize != 32)
+ continue;
+
+ if (TRI->regsOverlap(Reg, AMDGPU::SCC)) {
+ if (MO.readsReg())
+ ReadsSCC = true;
+ if (MO.isDef())
+ ModifiesSCC = true;
+ }
+ }
+
+ if (ReadsSCC) {
+ if (!SCCDefinedInCurrentBlock) {
+ errs() << "Violation in MBB: " << MBB.getName() << "\n";
+ errs() << "Instruction: " << MI << "\n";
+ llvm_unreachable(
+ "SCC is read before being defined in the same basic block!");
+ }
+ }
+
+ if (ModifiesSCC) {
+ SCCDefinedInCurrentBlock = true;
+ }
+ }
+ }
+}
+
void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
SmallVector<unsigned, 8> LoweringWorklist;
>From 1b0fc97a167201c1ecc9610c7c1667e2f97e0f0b Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 18:18:46 +0000
Subject: [PATCH 04/31] comments
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 9 +--
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 72 ++++++++-----------
2 files changed, 35 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 20c02317019c6..17c1b048bd4d2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -153,10 +153,11 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
const TargetRegisterClass *RegClassForVT =
VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
- if (UseRC == nullptr || !UseRC->isAllocatable()) {
- UseRC = RegClassForVT;
- } else if (auto CommonSubClass = TRI->getCommonSubClass(UseRC, RegClassForVT)) {
- UseRC = CommonSubClass;
+ if (!UseRC || !UseRC->isAllocatable()) {
+ UseRC = RegClassForVT;
+ } else if (const TargetRegisterClass *CommonSubClass =
+ TRI->getCommonSubClass(UseRC, RegClassForVT)) {
+ UseRC = CommonSubClass;
}
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index d630cf46b7dbe..472dfae9405e3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -326,53 +326,41 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
}
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
- // For the DAG:
- // t134: i1 = setcc t52, Constant:i32<0>, setne:ch
- // t133: i32 = zero_extend t134
- // After DAG selections, it is:
- // t134: i1 = S_CMP_LG_U32 t52, t19
- // t133: i32,i1 = S_AND_B32 t134, TargetConstant:i32<1>
- //
- // And the listed MIR are:
- // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
- // %27:sreg_32 = COPY $scc
- // %28:sreg_32 = S_AND_B32 killed %27:sreg_32, 1, implicit-def dead $scc
- //
- // After SI-Fix-SGPR-Copies, if wave size is 64, the fixSCCCopy will change it to
- // S_CMP_LG_U32 killed %25:sreg_32, %26:sreg_32, implicit-def $scc
- // %79:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit $scc
- // %27:sreg_32 = COPY %79:sreg_64_xexec; illegal vgpr to sgpr copy
- //
- // This preprocess changes DAG
- // t133: i32 = zero_extend t134 into --> S_CSELEECT_B32 -1 0
- // if it is valid.
- if (N->isDivergent()) return false;
-
- // If to i64, it is probably used for lane mask, then we don't do the changes here.
- if (N->getValueType(0) != MVT::i32) return false;
-
- auto CondNode = N->getOperand(0);
- if (CondNode->getOpcode() == ISD::FREEZE) {
- CondNode = CondNode->getOperand(0);
- }
+ // If the extension of uniform i1 to i32 is not used as lanemask, we want to
+ // the result type is int32. This method mutates node `zero_extend i1 to i32`
+ // directly to S_CSELECT_B32. Instead of change the following
+ // SI-Fix-SGPR-Copies to support it, we make changes earlier bofore inst
+ // selection.
+ if (N->isDivergent())
+ return false;
- if (CondNode->getOpcode() != ISD::SETCC)
- return false;
+ // If to i64, it is probably used for lane mask, then we don't do the changes
+ // here.
+ if (N->getValueType(0) != MVT::i32)
+ return false;
- if (CondNode->getOperand(0).getValueType() != MVT::i32 || CondNode->getOperand(1).getValueType() != MVT::i32)
- return false;
+ SDValue CondNode = N->getOperand(0);
+ if (CondNode->getOpcode() != ISD::SETCC)
+ return false;
- SDLoc DL(N);
- SDValue TrueValue = CurDAG->getAllOnesConstant(DL, MVT::i32, true);
- SDValue FalseValue = CurDAG->getTargetConstant(0, DL, MVT::i32);
+ if (CondNode->getOperand(0).getValueType() != MVT::i32 ||
+ CondNode->getOperand(1).getValueType() != MVT::i32)
+ return false;
- SDValue Chain = CurDAG->getEntryNode(); // Basic chain to anchor the copy
- SDValue CopyToSCC = CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
- SmallVector<SDValue, 3> Ops = {TrueValue, FalseValue, CopyToSCC.getValue(1)};
- MachineSDNode *CSelectNode = CurDAG->getMachineNode(AMDGPU::S_CSELECT_B32, DL, CurDAG->getVTList(MVT::i32), Ops);
- CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
+ SDLoc DL(N);
+ SDValue TrueValue = CurDAG->getTargetConstant(1, DL, MVT::i32);
+ SDValue FalseValue = CurDAG->getTargetConstant(0, DL, MVT::i32);
+
+ SDValue Chain = CurDAG->getEntryNode(); // Basic chain to anchor the copy
+ SDValue CopyToSCC =
+ CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
+ SDValue Ops[4] = {TrueValue, FalseValue, CopyToSCC.getValue(1),
+ CopyToSCC.getValue(1)};
+ MachineSDNode *CSelectNode = CurDAG->getMachineNode(
+ AMDGPU::S_CSELECT_B32, DL, CurDAG->getVTList(MVT::i32), Ops);
+ CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
- return true;
+ return true;
}
void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
>From def6ea135cf851d40586cf4d58c4847b95ea3f41 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 19:08:37 +0000
Subject: [PATCH 05/31] new basic lit-test
---
...amdgcn.uniform_ext_i1_to_i32.ll => uniform_ext_i1_to_i32.ll} | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
rename llvm/test/CodeGen/AMDGPU/{llvm.amdgcn.uniform_ext_i1_to_i32.ll => uniform_ext_i1_to_i32.ll} (94%)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
similarity index 94%
rename from llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
rename to llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
index f5cc63694be9a..97aaf9eb4b681 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.uniform_ext_i1_to_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
declare i32 @llvm.amdgcn.workitem.id.x() #0
>From 646aebb184adb3c638b425951e6411dd3b2f9802 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 19:42:53 +0000
Subject: [PATCH 06/31] cleanup
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 74 -------------------
.../CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll | 37 ++++++++--
2 files changed, 32 insertions(+), 79 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 066f11c932fe5..603665611e2bf 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -159,8 +159,6 @@ class SIFixSGPRCopies {
MachineBasicBlock *BlockToInsertTo,
MachineBasicBlock::iterator PointToInsertTo,
const DebugLoc &DL);
-
- void validateNoCrossBlockSCC(MachineFunction &MF);
};
class SIFixSGPRCopiesLegacy : public MachineFunctionPass {
@@ -625,8 +623,6 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getProperties().hasSelected())
return false;
- validateNoCrossBlockSCC(MF);
-
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
MRI = &MF.getRegInfo();
TRI = ST.getRegisterInfo();
@@ -786,10 +782,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
lowerVGPR2SGPRCopies(MF);
- validateNoCrossBlockSCC(MF);
// Postprocessing
fixSCCCopies(MF);
- validateNoCrossBlockSCC(MF);
for (auto *MI : S2VCopies) {
// Check if it is still valid
@@ -814,8 +808,6 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
if (MF.getTarget().getOptLevel() > CodeGenOptLevel::None && EnableM0Merge)
hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
- validateNoCrossBlockSCC(MF);
-
SiblingPenalty.clear();
V2SCopies.clear();
SCCCopies.clear();
@@ -1069,72 +1061,6 @@ bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
return Info->NeedToBeConvertedToVALU;
}
-void SIFixSGPRCopies::validateNoCrossBlockSCC(MachineFunction &MF) {
-
- for (const MachineBasicBlock &MBB : MF) {
- // assert(!MBB.isLiveIn(AMDGPU::SCC) && "AMDGPU::SCC is live-in to a basic
- // block! Cross-BB SCC liveness detected.");
-
- bool SCCDefinedInCurrentBlock = false;
-
- for (const MachineInstr &MI : MBB) {
- bool ReadsSCC = false;
- bool ModifiesSCC = false;
-
- for (const MachineOperand &MO : MI.operands()) {
- if (MO.isRegMask() && MO.clobbersPhysReg(AMDGPU::SCC)) {
- ModifiesSCC = true;
- continue;
- }
-
- if (!MO.isReg())
- continue;
-
- Register Reg = MO.getReg();
- if (!Reg || !Reg.isPhysical())
- continue;
-
- unsigned BitSize = 0;
- if (Reg.isPhysical()) {
- const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
- BitSize = TRI->getRegSizeInBits(*RC);
- } else {
- if (MRI->getType(Reg).isValid()) {
- BitSize = MRI->getType(Reg).getSizeInBits();
- } else {
- const TargetRegisterClass *RC = MRI->getRegClass(Reg);
- if (RC)
- BitSize = TRI->getRegSizeInBits(*RC);
- }
- }
-
- if (BitSize != 32)
- continue;
-
- if (TRI->regsOverlap(Reg, AMDGPU::SCC)) {
- if (MO.readsReg())
- ReadsSCC = true;
- if (MO.isDef())
- ModifiesSCC = true;
- }
- }
-
- if (ReadsSCC) {
- if (!SCCDefinedInCurrentBlock) {
- errs() << "Violation in MBB: " << MBB.getName() << "\n";
- errs() << "Instruction: " << MI << "\n";
- llvm_unreachable(
- "SCC is read before being defined in the same basic block!");
- }
- }
-
- if (ModifiesSCC) {
- SCCDefinedInCurrentBlock = true;
- }
- }
- }
-}
-
void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
SmallVector<unsigned, 8> LoweringWorklist;
diff --git a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
index 97aaf9eb4b681..183c8487a9698 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
@@ -3,9 +3,36 @@
declare i32 @llvm.amdgcn.workitem.id.x() #0
-define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+; divergent i64
+define i64 @zext_i1_to_i64(i32 %a, i32 %b) #0 {
entry:
- ; GFX950-LABEL: zext_i1_to_i64_uniform
+ ; GFX950-LABEL: zext_i1_to_i64
+ ; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+ ; GFX950-NEXT: v_mov_b32_e32 v1, 0
+ ; GFX950-NEXT: s_nop 0
+ ; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+ ; GFX950-NEXT: s_setpc_b64 s[30:31]
+
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i64
+ ret i64 %tmp2
+}
+
+; divergent i32
+define i32 @zext_i1_to_i32(i1 %pred) #0 {
+entry:
+ ; GFX950-LABEL: zext_i1_to_i32
+ ; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+ ; GFX950-NEXT: v_and_b32_e32 v0, 1, v0
+ ; GFX950-NEXT: s_setpc_b64 s[30:31]
+ %tmp2 = zext i1 %pred to i32
+ ret i32 %tmp2
+}
+
+define amdgpu_kernel void @kernel_zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+entry:
+ ; GFX950-LABEL: kernel_zext_i1_to_i64_uniform
; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX950-NEXT: s_mov_b32 s4, 0
@@ -24,9 +51,9 @@ entry:
ret void
}
-define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+define amdgpu_kernel void @kernel_zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
entry:
- ; GFX950-LABEL: zext_i1_to_i32_uniform
+ ; GFX950-LABEL: kernel_zext_i1_to_i32_uniform
; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX950-NEXT: v_mov_b32_e32 v0, 0
@@ -34,7 +61,7 @@ entry:
; GFX950-NEXT: s_cmp_eq_u32 s0, s1
; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
- ; GFX950-NEXT: s_cselect_b32 s0, -1, 0
+ ; GFX950-NEXT: s_cselect_b32 s0, 1, 0
; GFX950-NEXT: v_mov_b32_e32 v1, s0
; GFX950-NEXT: global_store_dword v0, v1, s[2:3]
; GFX950-NEXT: s_endpgm
>From 21211ec274ee2d045b01d52b7de9b2d9ff31d17e Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 19:45:14 +0000
Subject: [PATCH 07/31] cleanup
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 5 -----
1 file changed, 5 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 603665611e2bf..ef0224ec723a0 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -781,10 +781,8 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
}
lowerVGPR2SGPRCopies(MF);
-
// Postprocessing
fixSCCCopies(MF);
-
for (auto *MI : S2VCopies) {
// Check if it is still valid
if (MI->isCopy()) {
@@ -1015,9 +1013,6 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
}
}
- for (auto I : Info.SChain) {
- LLVM_DEBUG(dbgs() << "Chain 12345:"; I->dump(););
- }
V2SCopies[Info.ID] = std::move(Info);
}
>From 4770cf73336db313e7254d8ecbb1be924f2075cf Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 19:45:49 +0000
Subject: [PATCH 08/31] cleanup
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index ef0224ec723a0..5994eeb54095b 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1012,7 +1012,6 @@ void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
AnalysisWorklist.push_back(U);
}
}
-
V2SCopies[Info.ID] = std::move(Info);
}
>From 60d960b8654c74da71e29fcad25d386fda57cf83 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 20:05:21 +0000
Subject: [PATCH 09/31] comments
---
llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 1 -
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 13 ++++++-------
2 files changed, 6 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 17c1b048bd4d2..2ec959f392738 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -129,7 +129,6 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
RC = TRI->getAllocatableClass(
TII->getRegClass(II, i + II.getNumDefs()));
}
-
if (!UseRC)
UseRC = RC;
else if (RC) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 472dfae9405e3..19f68260c2a1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -325,17 +325,14 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
return false;
}
+// TODO: To support sext and any_ext.
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
- // If the extension of uniform i1 to i32 is not used as lanemask, we want to
- // the result type is int32. This method mutates node `zero_extend i1 to i32`
- // directly to S_CSELECT_B32. Instead of change the following
- // SI-Fix-SGPR-Copies to support it, we make changes earlier bofore inst
- // selection.
+ // This is special case for pattern of common compare + select if it can be
+ // selected to SALU. The pattern will be directly selected to `s_cselect`, to
+ // avoid an intermediate i1 virtual register to be interpreted as a lane mask.
if (N->isDivergent())
return false;
- // If to i64, it is probably used for lane mask, then we don't do the changes
- // here.
if (N->getValueType(0) != MVT::i32)
return false;
@@ -343,6 +340,8 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
if (CondNode->getOpcode() != ISD::SETCC)
return false;
+ // TODO: To support the operand type is int64 if s_cmp_i64 is supported on
+ // some targets
if (CondNode->getOperand(0).getValueType() != MVT::i32 ||
CondNode->getOperand(1).getValueType() != MVT::i32)
return false;
>From 59a38790c0da80c3a8ac817a5d7ca191252c0ca4 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Fri, 15 May 2026 20:10:48 +0000
Subject: [PATCH 10/31] change TODO
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 19f68260c2a1b..780d334096c6e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -325,7 +325,6 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
return false;
}
-// TODO: To support sext and any_ext.
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// This is special case for pattern of common compare + select if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
@@ -379,8 +378,9 @@ void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
break;
case ISD::ZERO_EXTEND:
- MadeChange |= preprocessZeroExtend(N);
- break;
+ // TODO: To support sext and any_ext.
+ MadeChange |= preprocessZeroExtend(N);
+ break;
default:
break;
>From 57178ea2ea2f74faccf77045f88434283f3ce7fe Mon Sep 17 00:00:00 2001
From: Zeng Wu <zengwu13 at amd.com>
Date: Fri, 15 May 2026 16:53:58 -0700
Subject: [PATCH 11/31] Apply suggestion from @arsenm
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 780d334096c6e..89d0b8493a90c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -328,7 +328,9 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// This is special case for pattern of common compare + select if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
- // avoid an intermediate i1 virtual register to be interpreted as a lane mask.
+ // This is special case for the common pattern of compare + select if it can be
+ // selected to SALU. The pattern will be directly selected to `s_cselect`, to
+ // avoid an intermediate i1 virtual register which will be interpreted as a lane mask.
if (N->isDivergent())
return false;
>From 32f09ca3ed971c2cb7bf21672618064a380ef0ea Mon Sep 17 00:00:00 2001
From: Zeng Wu <zengwu13 at amd.com>
Date: Fri, 15 May 2026 16:56:00 -0700
Subject: [PATCH 12/31] Apply suggestion from @arsenm
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 89d0b8493a90c..62a2cf58a8de5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -344,7 +344,7 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// TODO: To support the operand type is int64 if s_cmp_i64 is supported on
// some targets
if (CondNode->getOperand(0).getValueType() != MVT::i32 ||
- CondNode->getOperand(1).getValueType() != MVT::i32)
+ if (CondNode->getOperand(0).getValueType() != MVT::i32)
return false;
SDLoc DL(N);
>From cda7d165698df2f8ae98625bc7b7b879fc44baf3 Mon Sep 17 00:00:00 2001
From: Zeng Wu <zengwu13 at amd.com>
Date: Fri, 15 May 2026 16:56:24 -0700
Subject: [PATCH 13/31] Apply suggestion from @arsenm
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 62a2cf58a8de5..6774944c592ba 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -344,6 +344,7 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// TODO: To support the operand type is int64 if s_cmp_i64 is supported on
// some targets
if (CondNode->getOperand(0).getValueType() != MVT::i32 ||
+ if (CondNode->getOperand(0).getValueType() != MVT::i32)
if (CondNode->getOperand(0).getValueType() != MVT::i32)
return false;
>From 262117fcc5a313ad3eed2192ac50fbe4faeeae9f Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Sat, 16 May 2026 01:04:22 +0000
Subject: [PATCH 14/31] support i64
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 10 ++--
.../CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll | 48 +++++++++++++++++--
2 files changed, 49 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 6774944c592ba..6635c4ee4b532 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -326,7 +326,7 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
}
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
- // This is special case for pattern of common compare + select if it can be
+ // This is special case for common pattern of compare + select if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
// This is special case for the common pattern of compare + select if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
@@ -334,7 +334,8 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
if (N->isDivergent())
return false;
- if (N->getValueType(0) != MVT::i32)
+ EVT ResType = N->getOperand(0).getValueType();
+ if (ResType != MVT::i32 && ResType != MVT::i64)
return false;
SDValue CondNode = N->getOperand(0);
@@ -343,8 +344,6 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// TODO: To support the operand type is int64 if s_cmp_i64 is supported on
// some targets
- if (CondNode->getOperand(0).getValueType() != MVT::i32 ||
- if (CondNode->getOperand(0).getValueType() != MVT::i32)
if (CondNode->getOperand(0).getValueType() != MVT::i32)
return false;
@@ -357,8 +356,9 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
SDValue Ops[4] = {TrueValue, FalseValue, CopyToSCC.getValue(1),
CopyToSCC.getValue(1)};
+ auto SelectCode = ResType == MVT::i32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
MachineSDNode *CSelectNode = CurDAG->getMachineNode(
- AMDGPU::S_CSELECT_B32, DL, CurDAG->getVTList(MVT::i32), Ops);
+ SelectCode, DL, CurDAG->getVTList(MVT::i32), Ops);
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
index 183c8487a9698..06e6fe7b09751 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
@@ -4,9 +4,9 @@
declare i32 @llvm.amdgcn.workitem.id.x() #0
; divergent i64
-define i64 @zext_i1_to_i64(i32 %a, i32 %b) #0 {
+define i64 @test_zext_i1_to_i64(i32 %a, i32 %b) #0 {
entry:
- ; GFX950-LABEL: zext_i1_to_i64
+ ; GFX950-LABEL: test_zext_i1_to_i64
; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: v_mov_b32_e32 v1, 0
@@ -20,9 +20,9 @@ entry:
}
; divergent i32
-define i32 @zext_i1_to_i32(i1 %pred) #0 {
+define i32 @test_zext_i1_to_i32(i1 %pred) #0 {
entry:
- ; GFX950-LABEL: zext_i1_to_i32
+ ; GFX950-LABEL: test_zext_i1_to_i32
; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_and_b32_e32 v0, 1, v0
; GFX950-NEXT: s_setpc_b64 s[30:31]
@@ -30,6 +30,46 @@ entry:
ret i32 %tmp2
}
+; uniform i32
+define i32 @test_uniform_zext(i32 inreg %val) {
+
+; GFX950-LABEL: test_uniform_zext:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lg_u32 s0, 0
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+
+entry:
+ %cmp = icmp ne i32 %val, 0
+ %zext = zext i1 %cmp to i32
+
+ ret i32 %zext
+}
+
+; uniform i64
+define i64 @test_uniform_zext_i64(i32 inreg %val) {
+; GFX950-LABEL: test_uniform_zext_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lg_u32 s0, 0
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+
+entry:
+ %cmp = icmp ne i32 %val, 0
+ %zext = zext i1 %cmp to i64
+
+ ret i64 %zext
+}
+
define amdgpu_kernel void @kernel_zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
entry:
; GFX950-LABEL: kernel_zext_i1_to_i64_uniform
>From fcabf71751c26d2405be90b2227cf03b1008b9ee Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Sat, 16 May 2026 23:52:13 +0000
Subject: [PATCH 15/31] lit test
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 6635c4ee4b532..211bf6e9fdc81 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -356,7 +356,8 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
SDValue Ops[4] = {TrueValue, FalseValue, CopyToSCC.getValue(1),
CopyToSCC.getValue(1)};
- auto SelectCode = ResType == MVT::i32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
+ unsigned SelectCode =
+ ResType == MVT::i32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
MachineSDNode *CSelectNode = CurDAG->getMachineNode(
SelectCode, DL, CurDAG->getVTList(MVT::i32), Ops);
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
>From 43cf80e83fccd3f9a092c0e05e10476739ad85e2 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Sat, 16 May 2026 23:52:13 +0000
Subject: [PATCH 16/31] extend lit tests to cover all ops in setcc and operand
types i16, i32 and i64
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 8 +-
.../CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll | 114 -
.../AMDGPU/zero_extend_i1_to_i32_i64.ll | 2028 +++++++++++++++++
3 files changed, 2032 insertions(+), 118 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 211bf6e9fdc81..5d27b02250d8b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -334,7 +334,7 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
if (N->isDivergent())
return false;
- EVT ResType = N->getOperand(0).getValueType();
+ EVT ResType = N->getValueType(0);
if (ResType != MVT::i32 && ResType != MVT::i64)
return false;
@@ -356,10 +356,10 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
SDValue Ops[4] = {TrueValue, FalseValue, CopyToSCC.getValue(1),
CopyToSCC.getValue(1)};
- unsigned SelectCode =
- ResType == MVT::i32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
+ bool IsInt32 = ResType == MVT::i32;
+ unsigned SelectCode = IsInt32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
MachineSDNode *CSelectNode = CurDAG->getMachineNode(
- SelectCode, DL, CurDAG->getVTList(MVT::i32), Ops);
+ SelectCode, DL, CurDAG->getVTList(IsInt32 ? MVT::i32 : MVT::i64), Ops);
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
return true;
diff --git a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll b/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
deleted file mode 100644
index 06e6fe7b09751..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/uniform_ext_i1_to_i32.ll
+++ /dev/null
@@ -1,114 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
-
-declare i32 @llvm.amdgcn.workitem.id.x() #0
-
-; divergent i64
-define i64 @test_zext_i1_to_i64(i32 %a, i32 %b) #0 {
-entry:
- ; GFX950-LABEL: test_zext_i1_to_i64
- ; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
- ; GFX950-NEXT: v_mov_b32_e32 v1, 0
- ; GFX950-NEXT: s_nop 0
- ; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
- ; GFX950-NEXT: s_setpc_b64 s[30:31]
-
- %cmp = icmp eq i32 %a, %b
- %tmp2 = zext i1 %cmp to i64
- ret i64 %tmp2
-}
-
-; divergent i32
-define i32 @test_zext_i1_to_i32(i1 %pred) #0 {
-entry:
- ; GFX950-LABEL: test_zext_i1_to_i32
- ; GFX950: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
- ; GFX950-NEXT: v_and_b32_e32 v0, 1, v0
- ; GFX950-NEXT: s_setpc_b64 s[30:31]
- %tmp2 = zext i1 %pred to i32
- ret i32 %tmp2
-}
-
-; uniform i32
-define i32 @test_uniform_zext(i32 inreg %val) {
-
-; GFX950-LABEL: test_uniform_zext:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lg_u32 s0, 0
-; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX950-NEXT: s_cselect_b32 s0, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v0, s0
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-
-entry:
- %cmp = icmp ne i32 %val, 0
- %zext = zext i1 %cmp to i32
-
- ret i32 %zext
-}
-
-; uniform i64
-define i64 @test_uniform_zext_i64(i32 inreg %val) {
-; GFX950-LABEL: test_uniform_zext_i64:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lg_u32 s0, 0
-; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX950-NEXT: s_cselect_b32 s0, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v0, s0
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-
-entry:
- %cmp = icmp ne i32 %val, 0
- %zext = zext i1 %cmp to i64
-
- ret i64 %zext
-}
-
-define amdgpu_kernel void @kernel_zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
-entry:
- ; GFX950-LABEL: kernel_zext_i1_to_i64_uniform
- ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
- ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
- ; GFX950-NEXT: s_mov_b32 s4, 0
- ; GFX950-NEXT: v_mov_b32_e32 v2, 0
- ; GFX950-NEXT: v_mov_b32_e32 v1, s4
- ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
- ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
- ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
- ; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
- ; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
- ; GFX950-NEXT: s_endpgm
-
- %cmp = icmp eq i32 %a, %b
- %tmp2 = zext i1 %cmp to i64
- store i64 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_kernel void @kernel_zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
-entry:
- ; GFX950-LABEL: kernel_zext_i1_to_i32_uniform
- ; GFX950: s_load_dwordx2 s[0:1], s[4:5], 0x30
- ; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
- ; GFX950-NEXT: v_mov_b32_e32 v0, 0
- ; GFX950-NEXT: s_waitcnt lgkmcnt(0)
- ; GFX950-NEXT: s_cmp_eq_u32 s0, s1
- ; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
- ; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
- ; GFX950-NEXT: s_cselect_b32 s0, 1, 0
- ; GFX950-NEXT: v_mov_b32_e32 v1, s0
- ; GFX950-NEXT: global_store_dword v0, v1, s[2:3]
- ; GFX950-NEXT: s_endpgm
- %cmp = icmp eq i32 %a, %b
- %tmp2 = zext i1 %cmp to i32
- store i32 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
new file mode 100644
index 0000000000000..5566df56b098c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
@@ -0,0 +1,2028 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck --check-prefix=GFX950 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GFX1250 %s
+
+declare i32 @llvm.amdgcn.workitem.id.x() #0
+
+; ============================================================================
+; Divergent i32 compares
+; ============================================================================
+
+define i32 @divergent_i32_eq_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i64 @divergent_i32_eq_i64(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i64
+ ret i64 %zext
+}
+
+define i32 @divergent_i32_ne_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ne_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ne i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ugt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ugt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_uge_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp uge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ult_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ult i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ule_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ule i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sgt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sgt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sge_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_slt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp slt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sle_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: divergent_i32_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sle i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+; ============================================================================
+; Uniform i32 compares
+; ============================================================================
+
+define i32 @uniform_i32_eq_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i64 @uniform_i32_eq_i64(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i64
+ ret i64 %zext
+}
+
+define i32 @uniform_i32_ne_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lg_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lg_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ne i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_ugt_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_gt_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_gt_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ugt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_uge_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_ge_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_ge_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp uge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_ult_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lt_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ult i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_ule_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_le_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_le_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ule i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_sgt_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_gt_i32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_gt_i32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sgt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_sge_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_ge_i32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_ge_i32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_slt_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lt_i32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lt_i32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp slt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @uniform_i32_sle_i32(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: uniform_i32_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: s_cmp_le_i32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v0, s0
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: uniform_i32_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_le_i32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sle i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+
+; ============================================================================
+; AMDGPU_KERNEL ABI tests
+; ============================================================================
+
+; ----------------------------------------------------------------------------
+; Uniform i32 compares
+; ----------------------------------------------------------------------------
+
+define amdgpu_kernel void @kernel_uniform_i32_eq_i32(
+; GFX950-LABEL: kernel_uniform_i32_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_eq_i64(
+; GFX950-LABEL: kernel_uniform_i32_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i64
+ store i64 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_ne_i32(
+; GFX950-LABEL: kernel_uniform_i32_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lg_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp ne i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_ugt_i32(
+; GFX950-LABEL: kernel_uniform_i32_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_gt_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_gt_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp ugt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_uge_i32(
+; GFX950-LABEL: kernel_uniform_i32_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_ge_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_ge_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp uge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_ult_i32(
+; GFX950-LABEL: kernel_uniform_i32_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lt_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lt_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp ult i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_ule_i32(
+; GFX950-LABEL: kernel_uniform_i32_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_le_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_le_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp ule i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_sgt_i32(
+; GFX950-LABEL: kernel_uniform_i32_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_gt_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_gt_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp sgt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_sge_i32(
+; GFX950-LABEL: kernel_uniform_i32_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_ge_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_ge_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp sge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_slt_i32(
+; GFX950-LABEL: kernel_uniform_i32_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lt_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lt_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp slt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i32_sle_i32(
+; GFX950-LABEL: kernel_uniform_i32_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_le_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i32_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_le_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp sle i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+; ----------------------------------------------------------------------------
+; Uniform i16 compares
+; ----------------------------------------------------------------------------
+
+define amdgpu_kernel void @kernel_uniform_i16_eq_i32(
+; GFX950-LABEL: kernel_uniform_i16_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_eq_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp eq i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_eq_i64(
+; GFX950-LABEL: kernel_uniform_i16_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_eq_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp eq i16 %a, %b
+ %zext = zext i1 %cmp to i64
+ store i64 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_ne_i32(
+; GFX950-LABEL: kernel_uniform_i16_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_lg_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp ne i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_ugt_i32(
+; GFX950-LABEL: kernel_uniform_i16_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_gt_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_gt_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp ugt i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_uge_i32(
+; GFX950-LABEL: kernel_uniform_i16_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_ge_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_ge_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp uge i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_ult_i32(
+; GFX950-LABEL: kernel_uniform_i16_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_lt_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_lt_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp ult i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_ule_i32(
+; GFX950-LABEL: kernel_uniform_i16_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_lshr_b32 s3, s2, 16
+; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX950-NEXT: s_cmp_le_u32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_le_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp ule i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_sgt_i32(
+; GFX950-LABEL: kernel_uniform_i16_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_ashr_i32 s3, s2, 16
+; GFX950-NEXT: s_sext_i32_i16 s2, s2
+; GFX950-NEXT: s_cmp_gt_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
+; GFX1250-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_gt_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp sgt i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_sge_i32(
+; GFX950-LABEL: kernel_uniform_i16_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_ashr_i32 s3, s2, 16
+; GFX950-NEXT: s_sext_i32_i16 s2, s2
+; GFX950-NEXT: s_cmp_ge_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
+; GFX1250-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_ge_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp sge i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_slt_i32(
+; GFX950-LABEL: kernel_uniform_i16_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_ashr_i32 s3, s2, 16
+; GFX950-NEXT: s_sext_i32_i16 s2, s2
+; GFX950-NEXT: s_cmp_lt_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
+; GFX1250-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_lt_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp slt i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i16_sle_i32(
+; GFX950-LABEL: kernel_uniform_i16_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_ashr_i32 s3, s2, 16
+; GFX950-NEXT: s_sext_i32_i16 s2, s2
+; GFX950-NEXT: s_cmp_le_i32 s2, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i16_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
+; GFX1250-NEXT: s_sext_i32_i16 s2, s2
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_le_i32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i16 %a,
+ i16 %b) #0 {
+entry:
+ %cmp = icmp sle i16 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+; ----------------------------------------------------------------------------
+; Divergent i64 compares
+; ----------------------------------------------------------------------------
+
+define amdgpu_kernel void @kernel_i64_eq_i32(
+; GFX950-LABEL: kernel_i64_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp eq i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_i64_eq_i64(
+; GFX950-LABEL: kernel_i64_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_mov_b32 s4, 0
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s4
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
+; GFX1250-NEXT: s_mov_b32 s2, 0
+; GFX1250-NEXT: s_cselect_b32 s3, -1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s3
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp eq i64 %a, %b
+ %zext = zext i1 %cmp to i64
+ store i64 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_i64_ne_i32(
+; GFX950-LABEL: kernel_i64_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_lg_u64 s[2:3], s[6:7]
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_lg_u64 s[2:3], s[6:7]
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp ne i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_i64_ugt_i32(
+; GFX950-LABEL: kernel_i64_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_u64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp ugt i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_i64_uge_i32(
+; GFX950-LABEL: kernel_i64_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_u64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp uge i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_i64_ult_i32(
+; GFX950-LABEL: kernel_i64_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_i64_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_u64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp ult i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i64_ule_i32(
+; GFX950-LABEL: kernel_uniform_i64_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_le_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i64_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_u64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp ule i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i64_sgt_i32(
+; GFX950-LABEL: kernel_uniform_i64_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i64_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_i64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp sgt i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i64_sge_i32(
+; GFX950-LABEL: kernel_uniform_i64_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i64_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_i64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp sge i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i64_slt_i32(
+; GFX950-LABEL: kernel_uniform_i64_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i64_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_i64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp slt i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @kernel_uniform_i64_sle_i32(
+; GFX950-LABEL: kernel_uniform_i64_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
+; GFX950-NEXT: v_cmp_le_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_uniform_i64_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_i64_e64 s2, s[2:3], s[6:7]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i64 %a,
+ i64 %b) #0 {
+entry:
+ %cmp = icmp sle i64 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+
+; To test divergent with amdgpu kernel ABI.
+define amdgpu_kernel void @kernel_divergent_i32_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: kernel_divergent_i32_cmp_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_divergent_i32_cmp_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+; To test divergent zero i1 to i64 with amdgpu kernel ABI
+define amdgpu_kernel void @kernel_divergent_i64_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: kernel_divergent_i64_cmp_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_mov_b32 s2, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: kernel_divergent_i64_cmp_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_mov_b32 s3, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+; To test divergent with amdgpu kernel ABI.
+define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: zext_i1_to_i32_uniform:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: zext_i1_to_i32_uniform:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+; TO test zero_extend from i1 to 64 uniform
+define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+; GFX950-LABEL: zext_i1_to_i64_uniform:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b64 s[0:1], 1, 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[2:3]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: zext_i1_to_i64_uniform:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b64 s[0:1], 1, 0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
>From 3d0256116891c124c9518e14a1fb1654987d8cbc Mon Sep 17 00:00:00 2001
From: Zeng Wu <zengwu13 at amd.com>
Date: Sat, 16 May 2026 22:57:54 -0700
Subject: [PATCH 17/31] Update llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
Co-authored-by: Shilei Tian <i at tianshilei.me>
---
llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 2ec959f392738..d7d9cb4132c5d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -157,7 +157,11 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
} else if (const TargetRegisterClass *CommonSubClass =
TRI->getCommonSubClass(UseRC, RegClassForVT)) {
UseRC = CommonSubClass;
- }
+ if (!UseRC || !UseRC->isAllocatable())
+ UseRC = RegClassForVT;
+ else if (const TargetRegisterClass *CommonSubClass =
+ TRI->getCommonSubClass(UseRC, RegClassForVT))
+ UseRC = CommonSubClass;
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
>From c3531d6fcfea039443a515bec2c133ee4822d4dc Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Sun, 17 May 2026 06:56:22 +0000
Subject: [PATCH 18/31] update lit test
---
.../AMDGPU/zero_extend_i1_to_i32_i64.ll | 24 +++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
index 5566df56b098c..7fa407866dac9 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
@@ -2026,3 +2026,27 @@ entry:
store i64 %tmp2, ptr addrspace(1) %out
ret void
}
+
+define amdgpu_ps i32 @zext_i1_to_i32_uniform_amdgpu_ps_abi(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: ; return to shader part epilog
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i32
+ ret i32 %tmp2
+}
>From 523573a37dae4f88fb7fe6202e95ad9bbf9c4f77 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Mon, 18 May 2026 04:35:13 +0000
Subject: [PATCH 19/31] format
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 5d27b02250d8b..3cf6086d1dac0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -378,7 +378,7 @@ void AMDGPUDAGToDAGISel::PreprocessISelDAG() {
case ISD::BUILD_VECTOR:
// TODO: Match load d16 from shl (extload:i16), 16
if (Subtarget->d16PreservesUnusedBits())
- MadeChange |= matchLoadD16FromBuildVector(N);
+ MadeChange |= matchLoadD16FromBuildVector(N);
break;
case ISD::ZERO_EXTEND:
>From 29589b4433535ca01bda62515060e1ad20bc57c9 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Mon, 18 May 2026 04:45:11 +0000
Subject: [PATCH 20/31] apply clang-format for all relevant changes
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 59 ++++++++++---------
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 7 ++-
2 files changed, 34 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index d7d9cb4132c5d..e83037f264e7f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -157,36 +157,37 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
} else if (const TargetRegisterClass *CommonSubClass =
TRI->getCommonSubClass(UseRC, RegClassForVT)) {
UseRC = CommonSubClass;
- if (!UseRC || !UseRC->isAllocatable())
- UseRC = RegClassForVT;
- else if (const TargetRegisterClass *CommonSubClass =
+ if (!UseRC || !UseRC->isAllocatable())
+ UseRC = RegClassForVT;
+ else if (const TargetRegisterClass *CommonSubClass =
TRI->getCommonSubClass(UseRC, RegClassForVT))
- UseRC = CommonSubClass;
-
- const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
- SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
-
- // Figure out the register class to create for the destreg.
- if (VRBase) {
- DstRC = MRI->getRegClass(VRBase);
- } else if (UseRC) {
- assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
- "Incompatible phys register def and uses!");
- DstRC = UseRC;
- } else
- DstRC = SrcRC;
-
- // If all uses are reading from the src physical register and copying the
- // register is either impossible or very expensive, then don't create a copy.
- if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
- VRBase = SrcReg;
- } else {
- // Create the reg, emit the copy.
- VRBase = MRI->createVirtualRegister(DstRC);
- BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::COPY),
- VRBase)
- .addReg(SrcReg);
- }
+ UseRC = CommonSubClass;
+
+ const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
+ SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
+
+ // Figure out the register class to create for the destreg.
+ if (VRBase) {
+ DstRC = MRI->getRegClass(VRBase);
+ } else if (UseRC) {
+ assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
+ "Incompatible phys register def and uses!");
+ DstRC = UseRC;
+ } else
+ DstRC = SrcRC;
+
+ // If all uses are reading from the src physical register and copying the
+ // register is either impossible or very expensive, then don't create a
+ // copy.
+ if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
+ VRBase = SrcReg;
+ } else {
+ // Create the reg, emit the copy.
+ VRBase = MRI->createVirtualRegister(DstRC);
+ BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::COPY),
+ VRBase)
+ .addReg(SrcReg);
+ }
if (IsClone)
VRBaseMap.erase(Op);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 3cf6086d1dac0..838a46878d609 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -328,9 +328,10 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// This is special case for common pattern of compare + select if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
- // This is special case for the common pattern of compare + select if it can be
- // selected to SALU. The pattern will be directly selected to `s_cselect`, to
- // avoid an intermediate i1 virtual register which will be interpreted as a lane mask.
+ // This is special case for the common pattern of compare + select if it can
+ // be selected to SALU. The pattern will be directly selected to `s_cselect`,
+ // to avoid an intermediate i1 virtual register which will be interpreted as a
+ // lane mask.
if (N->isDivergent())
return false;
>From 89c333fb68ea04ff0fc4674851101d30d5ecd8e8 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Mon, 18 May 2026 04:51:34 +0000
Subject: [PATCH 21/31] format
---
llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index e83037f264e7f..5b60315d3f214 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -188,6 +188,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
VRBase)
.addReg(SrcReg);
}
+ }
if (IsClone)
VRBaseMap.erase(Op);
>From ae1e0dc3e6987444883d5020c334da82d4cd90bb Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Mon, 18 May 2026 18:12:03 +0000
Subject: [PATCH 22/31] fix rebase error
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 63 ++++++++++---------
1 file changed, 32 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 5b60315d3f214..ce0e7b73cdfb9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -157,37 +157,38 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
} else if (const TargetRegisterClass *CommonSubClass =
TRI->getCommonSubClass(UseRC, RegClassForVT)) {
UseRC = CommonSubClass;
- if (!UseRC || !UseRC->isAllocatable())
- UseRC = RegClassForVT;
- else if (const TargetRegisterClass *CommonSubClass =
- TRI->getCommonSubClass(UseRC, RegClassForVT))
- UseRC = CommonSubClass;
-
- const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
- SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
-
- // Figure out the register class to create for the destreg.
- if (VRBase) {
- DstRC = MRI->getRegClass(VRBase);
- } else if (UseRC) {
- assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
- "Incompatible phys register def and uses!");
- DstRC = UseRC;
- } else
- DstRC = SrcRC;
-
- // If all uses are reading from the src physical register and copying the
- // register is either impossible or very expensive, then don't create a
- // copy.
- if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
- VRBase = SrcReg;
- } else {
- // Create the reg, emit the copy.
- VRBase = MRI->createVirtualRegister(DstRC);
- BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::COPY),
- VRBase)
- .addReg(SrcReg);
- }
+ }
+
+ if (!UseRC || !UseRC->isAllocatable())
+ UseRC = RegClassForVT;
+ else if (const TargetRegisterClass *CommonSubClass =
+ TRI->getCommonSubClass(UseRC, RegClassForVT))
+ UseRC = CommonSubClass;
+
+ const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
+ SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
+
+ // Figure out the register class to create for the destreg.
+ if (VRBase) {
+ DstRC = MRI->getRegClass(VRBase);
+ } else if (UseRC) {
+ assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
+ "Incompatible phys register def and uses!");
+ DstRC = UseRC;
+ } else
+ DstRC = SrcRC;
+
+ // If all uses are reading from the src physical register and copying the
+ // register is either impossible or very expensive, then don't create a
+ // copy.
+ if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
+ VRBase = SrcReg;
+ } else {
+ // Create the reg, emit the copy.
+ VRBase = MRI->createVirtualRegister(DstRC);
+ BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::COPY),
+ VRBase)
+ .addReg(SrcReg);
}
if (IsClone)
>From 7c5513d9d0fb39b6e8dbe1d80d2bef7c4a4e46fa Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Tue, 19 May 2026 06:50:17 +0000
Subject: [PATCH 23/31] refactor lit-test
---
llvm/test/CodeGen/AMDGPU/zero_extend.ll | 686 +++++++
.../AMDGPU/zero_extend_i1_to_i32_i64.ll | 1703 +++++------------
2 files changed, 1203 insertions(+), 1186 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend.ll b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
index f0f8eaa7ab0de..0199aa6c00427 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
@@ -1,6 +1,8 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SI %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,VI %s
; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GFX1250 %s
; R600: {{^}}s_mad_zext_i32_to_i64:
; R600: MEM_RAT_CACHELESS STORE_RAW
@@ -10,6 +12,60 @@
; GCN: v_mov_b32_e32 v[[V_ZERO:[0-9]]], 0{{$}}
; GCN: buffer_store_dwordx2 v[0:[[V_ZERO]]]
define amdgpu_kernel void @s_mad_zext_i32_to_i64(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) #0 {
+; SI-LABEL: s_mad_zext_i32_to_i64:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mul_i32 s0, s0, s1
+; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_mad_zext_i32_to_i64:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mul_i32 s0, s0, s1
+; VI-NEXT: s_add_i32 s0, s0, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT: s_endpgm
+;
+; R600-LABEL: s_mad_zext_i32_to_i64:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: MULLO_INT * T0.X, KC0[2].Z, KC0[2].W,
+; R600-NEXT: ADD_INT T0.X, PS, KC0[3].X,
+; R600-NEXT: MOV T0.Y, 0.0,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX1250-LABEL: s_mad_zext_i32_to_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mul_i32 s0, s0, s1
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_add_co_i32 s0, s0, s2
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s0
+; GFX1250-NEXT: global_store_b64 v1, v[0:1], s[6:7]
+; GFX1250-NEXT: s_endpgm
entry:
%tmp0 = mul i32 %a, %b
%tmp1 = add i32 %tmp0, %c
@@ -21,6 +77,61 @@ entry:
; GCN-LABEL: {{^}}s_cmp_zext_i1_to_i32
; GCN: v_cndmask_b32
define amdgpu_kernel void @s_cmp_zext_i1_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
+; SI-LABEL: s_cmp_zext_i1_to_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_cmp_zext_i1_to_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; R600-LABEL: s_cmp_zext_i1_to_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: ALU 3, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: SETE_INT * T0.W, KC0[2].Z, KC0[2].W,
+; R600-NEXT: AND_INT T0.X, PV.W, 1,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX1250-LABEL: s_cmp_zext_i1_to_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
entry:
%tmp0 = icmp eq i32 %a, %b
%tmp1 = zext i1 %tmp0 to i32
@@ -30,6 +141,60 @@ entry:
; GCN-LABEL: {{^}}s_arg_zext_i1_to_i64:
define amdgpu_kernel void @s_arg_zext_i1_to_i64(ptr addrspace(1) %out, i1 zeroext %arg) #0 {
+; SI-LABEL: s_arg_zext_i1_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s6, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s4, s6, 1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_arg_zext_i1_to_i64:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s4, s6, 1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; R600-LABEL: s_arg_zext_i1_to_i64:
+; R600: ; %bb.0:
+; R600-NEXT: ALU 0, @8, KC0[], KC1[]
+; R600-NEXT: TEX 0 @6
+; R600-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: Fetch clause starting at 6:
+; R600-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; R600-NEXT: ALU clause starting at 8:
+; R600-NEXT: MOV * T0.X, 0.0,
+; R600-NEXT: ALU clause starting at 9:
+; R600-NEXT: BFE_INT T0.X, T0.X, 0.0, 1,
+; R600-NEXT: MOV T0.Y, 0.0,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX1250-LABEL: s_arg_zext_i1_to_i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
%ext = zext i1 %arg to i64
store i64 %ext, ptr addrspace(1) %out, align 8
ret void
@@ -40,6 +205,65 @@ define amdgpu_kernel void @s_arg_zext_i1_to_i64(ptr addrspace(1) %out, i1 zeroex
; GCN-DAG: s_cmp_eq_u32
; GCN: v_cndmask_b32
define amdgpu_kernel void @s_cmp_zext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
+; SI-LABEL: s_cmp_zext_i1_to_i64:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b64 s[4:5], 1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_cmp_zext_i1_to_i64:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b64 s[4:5], 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; R600-LABEL: s_cmp_zext_i1_to_i64:
+; R600: ; %bb.0:
+; R600-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: ALU clause starting at 4:
+; R600-NEXT: SETE_INT * T0.W, KC0[2].Z, KC0[2].W,
+; R600-NEXT: AND_INT T0.X, PV.W, 1,
+; R600-NEXT: MOV T0.Y, 0.0,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX1250-LABEL: s_cmp_zext_i1_to_i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: s_endpgm
%cmp = icmp eq i32 %a, %b
%ext = zext i1 %cmp to i64
store i64 %ext, ptr addrspace(1) %out, align 8
@@ -59,10 +283,472 @@ define amdgpu_kernel void @s_cmp_zext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i
; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
; GCN: buffer_store_short [[RESULT]]
define amdgpu_kernel void @s_cmp_zext_i1_to_i16(ptr addrspace(1) %out, [8 x i32], i16 zeroext %a, [8 x i32], i16 zeroext %b) #0 {
+; SI-LABEL: s_cmp_zext_i1_to_i16:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s6, s[4:5], 0x13
+; SI-NEXT: s_load_dword s7, s[4:5], 0x1c
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s4, s6, 0xffff
+; SI-NEXT: s_and_b32 s5, s7, 0xffff
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_cmp_zext_i1_to_i16:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s6, s[4:5], 0x70
+; VI-NEXT: s_load_dword s7, s[4:5], 0x4c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s4, s6, 0xffff
+; VI-NEXT: s_and_b32 s5, s7, 0xffff
+; VI-NEXT: s_cmp_eq_u32 s5, s4
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; R600-LABEL: s_cmp_zext_i1_to_i16:
+; R600: ; %bb.0:
+; R600-NEXT: ALU 0, @10, KC0[], KC1[]
+; R600-NEXT: TEX 1 @6
+; R600-NEXT: ALU 14, @11, KC0[CB0:0-32], KC1[]
+; R600-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+; R600-NEXT: Fetch clause starting at 6:
+; R600-NEXT: VTX_READ_16 T1.X, T0.X, 72, #3
+; R600-NEXT: VTX_READ_16 T0.X, T0.X, 108, #3
+; R600-NEXT: ALU clause starting at 10:
+; R600-NEXT: MOV * T0.X, 0.0,
+; R600-NEXT: ALU clause starting at 11:
+; R600-NEXT: BFE_INT T0.Z, T1.X, 0.0, literal.x,
+; R600-NEXT: BFE_INT T0.W, T0.X, 0.0, literal.x, BS:VEC_120/SCL_212
+; R600-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; R600-NEXT: 16(2.242078e-44), 3(4.203895e-45)
+; R600-NEXT: SETE_INT * T0.W, PV.Z, PV.W,
+; R600-NEXT: AND_INT T0.W, PV.W, 1,
+; R600-NEXT: LSHL * T1.W, T1.W, literal.x,
+; R600-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; R600-NEXT: LSHL T0.X, PV.W, PS,
+; R600-NEXT: LSHL * T0.W, literal.x, PS,
+; R600-NEXT: 65535(9.183409e-41), 0(0.000000e+00)
+; R600-NEXT: MOV T0.Y, 0.0,
+; R600-NEXT: MOV * T0.Z, 0.0,
+; R600-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; R600-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+;
+; GFX1250-LABEL: s_cmp_zext_i1_to_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x70 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_eq_u32 s3, s2
+; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%tmp0 = icmp eq i16 %a, %b
%tmp1 = zext i1 %tmp0 to i16
store i16 %tmp1, ptr addrspace(1) %out
ret void
}
+; ============================================================================
+; Divergent i32 compares
+; ============================================================================
+
+define i32 @divergent_i32_eq_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_eq_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_eq_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i64 @divergent_i32_eq_i64(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_eq_i64:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_eq_i64:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i64
+ ret i64 %zext
+}
+
+define i32 @divergent_i32_ne_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_ne_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_ne_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ne_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ne i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ugt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ugt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_ugt_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_ugt_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_ugt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ugt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_uge_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_uge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_uge_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_uge_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_uge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp uge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ult_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ult_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_ult_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_ult_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_ult_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ult i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_ule_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_ule_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_ule_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_ule_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_ule_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp ule i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sgt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sgt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_sgt_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_sgt_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_sgt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sgt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sge_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sge_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_sge_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_sge_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_sge_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_ge_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sge i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_slt_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_slt_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_slt_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_slt_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_slt_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp slt i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
+define i32 @divergent_i32_sle_i32(i32 %a, i32 %b) #0 {
+; GFX950-LABEL: divergent_i32_sle_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-LABEL: divergent_i32_sle_i32:
+; GCN: ; %bb.0: ; %entry
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; R600-LABEL: divergent_i32_sle_i32:
+; R600: ; %bb.0: ; %entry
+; R600-NEXT: CF_END
+; R600-NEXT: PAD
+;
+; GFX1250-LABEL: divergent_i32_sle_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_cmp_le_i32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+entry:
+ %cmp = icmp sle i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ ret i32 %zext
+}
+
attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
index 7fa407866dac9..7aef9c35cd745 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
@@ -4,254 +4,6 @@
declare i32 @llvm.amdgcn.workitem.id.x() #0
-; ============================================================================
-; Divergent i32 compares
-; ============================================================================
-
-define i32 @divergent_i32_eq_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_eq_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_eq_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp eq i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i64 @divergent_i32_eq_i64(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_eq_i64:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_eq_i64:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp eq i32 %a, %b
- %zext = zext i1 %cmp to i64
- ret i64 %zext
-}
-
-define i32 @divergent_i32_ne_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_ne_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_ne_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_ne_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp ne i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_ugt_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_ugt_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_ugt_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp ugt i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_uge_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_uge_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_uge_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_ge_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp uge i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_ult_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_ult_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_ult_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp ult i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_ule_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_ule_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_ule_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_le_u32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp ule i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_sgt_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_sgt_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_sgt_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp sgt i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_sge_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_sge_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_sge_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_ge_i32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp sge i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_slt_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_slt_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_slt_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp slt i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
-define i32 @divergent_i32_sle_i32(i32 %a, i32 %b) #0 {
-; GFX950-LABEL: divergent_i32_sle_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250-LABEL: divergent_i32_sle_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_le_i32_e32 vcc_lo, v0, v1
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
-entry:
- %cmp = icmp sle i32 %a, %b
- %zext = zext i1 %cmp to i32
- ret i32 %zext
-}
-
; ============================================================================
; Uniform i32 compares
; ============================================================================
@@ -568,114 +320,261 @@ entry:
; ============================================================================
-; AMDGPU_KERNEL ABI tests
+; AMDGPU_PS ABI tests
; ============================================================================
-; ----------------------------------------------------------------------------
-; Uniform i32 compares
-; ----------------------------------------------------------------------------
-define amdgpu_kernel void @kernel_uniform_i32_eq_i32(
-; GFX950-LABEL: kernel_uniform_i32_eq_i32:
+; To test divergent with amdgpu kernel ABI.
+define amdgpu_kernel void @kernel_divergent_i32_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: kernel_divergent_i32_cmp_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_eq_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_eq_i32:
+; GFX1250-LABEL: kernel_divergent_i32_cmp_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- ptr addrspace(1) %out,
- i32 %a,
- i32 %b) #0 {
entry:
- %cmp = icmp eq i32 %a, %b
- %zext = zext i1 %cmp to i32
- store i32 %zext, ptr addrspace(1) %out
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_eq_i64(
-; GFX950-LABEL: kernel_uniform_i32_eq_i64:
+; To test divergent zero i1 to i64 with amdgpu kernel ABI
+define amdgpu_kernel void @kernel_divergent_i64_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: kernel_divergent_i64_cmp_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_eq_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b64 s[2:3], 1, 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_mov_b32 s2, 0
+; GFX950-NEXT: v_mov_b32_e32 v1, s2
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_eq_i64:
+; GFX1250-LABEL: kernel_divergent_i64_cmp_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: s_mov_b32 s3, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b64 s[2:3], 1, 0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
- ptr addrspace(1) %out,
- i32 %a,
- i32 %b) #0 {
entry:
- %cmp = icmp eq i32 %a, %b
- %zext = zext i1 %cmp to i64
- store i64 %zext, ptr addrspace(1) %out
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+; To test divergent with amdgpu kernel ABI.
+define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
+; GFX950-LABEL: zext_i1_to_i32_uniform:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX950-NEXT: v_mov_b32_e32 v1, 0
+; GFX950-NEXT: s_waitcnt lgkmcnt(0)
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: zext_i1_to_i32_uniform:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
+ %cmp = icmp eq i32 %a, %workitem.id
+ %tmp2 = zext i1 %cmp to i32
+ store i32 %tmp2, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_ne_i32(
-; GFX950-LABEL: kernel_uniform_i32_ne_i32:
+; TO test zero_extend from i1 to 64 uniform
+define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
+; GFX950-LABEL: zext_i1_to_i64_uniform:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
+; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lg_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b64 s[0:1], 1, 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[2:3]
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_ne_i32:
+; GFX1250-LABEL: zext_i1_to_i64_uniform:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b64 s[0:1], 1, 0
+; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT: s_endpgm
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i64
+ store i64 %tmp2, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps i32 @zext_i1_to_i32_uniform_amdgpu_ps_abi(i32 inreg %a, i32 inreg %b) #0 {
+; GFX950-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: s_cmp_eq_u32 s0, s1
+; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX950-NEXT: s_cselect_b32 s0, 1, 0
+; GFX950-NEXT: ; return to shader part epilog
+;
+; GFX1250-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-NEXT: ; return to shader part epilog
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %tmp2 = zext i1 %cmp to i32
+ ret i32 %tmp2
+}
+
+; ----------------------------------------------------------------------------
+; Uniform i32 compares
+; ----------------------------------------------------------------------------
+
+define amdgpu_ps void @ps_uniform_i32_eq_i32(
+; GFX950-LABEL: ps_uniform_i32_eq_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: ps_uniform_i32_eq_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i32
+ store i32 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @ps_uniform_i32_eq_i64(
+; GFX950-LABEL: ps_uniform_i32_eq_i64:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_mov_b32 s0, 0
+; GFX950-NEXT: v_mov_b32_e32 v3, s0
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: ps_uniform_i32_eq_i64:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-NEXT: s_endpgm
+ ptr addrspace(1) %out,
+ i32 %a,
+ i32 %b) #0 {
+entry:
+ %cmp = icmp eq i32 %a, %b
+ %zext = zext i1 %cmp to i64
+ store i64 %zext, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_ps void @ps_uniform_i32_ne_i32(
+; GFX950-LABEL: ps_uniform_i32_ne_i32:
+; GFX950: ; %bb.0: ; %entry
+; GFX950-NEXT: v_cmp_ne_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
+; GFX950-NEXT: s_endpgm
+;
+; GFX1250-LABEL: ps_uniform_i32_ne_i32:
+; GFX1250: ; %bb.0: ; %entry
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -687,32 +586,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_ugt_i32(
-; GFX950-LABEL: kernel_uniform_i32_ugt_i32:
+define amdgpu_ps void @ps_uniform_i32_ugt_i32(
+; GFX950-LABEL: ps_uniform_i32_ugt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_gt_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_gt_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_ugt_i32:
+; GFX1250-LABEL: ps_uniform_i32_ugt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_gt_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -724,32 +612,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_uge_i32(
-; GFX950-LABEL: kernel_uniform_i32_uge_i32:
+define amdgpu_ps void @ps_uniform_i32_uge_i32(
+; GFX950-LABEL: ps_uniform_i32_uge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_ge_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_uge_i32:
+; GFX1250-LABEL: ps_uniform_i32_uge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_ge_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -761,32 +638,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_ult_i32(
-; GFX950-LABEL: kernel_uniform_i32_ult_i32:
+define amdgpu_ps void @ps_uniform_i32_ult_i32(
+; GFX950-LABEL: ps_uniform_i32_ult_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lt_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_lt_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_ult_i32:
+; GFX1250-LABEL: ps_uniform_i32_ult_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_lt_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -798,32 +664,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_ule_i32(
-; GFX950-LABEL: kernel_uniform_i32_ule_i32:
+define amdgpu_ps void @ps_uniform_i32_ule_i32(
+; GFX950-LABEL: ps_uniform_i32_ule_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_le_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_le_u32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_ule_i32:
+; GFX1250-LABEL: ps_uniform_i32_ule_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_le_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_u32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -835,32 +690,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_sgt_i32(
-; GFX950-LABEL: kernel_uniform_i32_sgt_i32:
+define amdgpu_ps void @ps_uniform_i32_sgt_i32(
+; GFX950-LABEL: ps_uniform_i32_sgt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_gt_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_gt_i32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_sgt_i32:
+; GFX1250-LABEL: ps_uniform_i32_sgt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_gt_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_i32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -872,32 +716,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_sge_i32(
-; GFX950-LABEL: kernel_uniform_i32_sge_i32:
+define amdgpu_ps void @ps_uniform_i32_sge_i32(
+; GFX950-LABEL: ps_uniform_i32_sge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_ge_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ge_i32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_sge_i32:
+; GFX1250-LABEL: ps_uniform_i32_sge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_ge_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_i32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -909,32 +742,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_slt_i32(
-; GFX950-LABEL: kernel_uniform_i32_slt_i32:
+define amdgpu_ps void @ps_uniform_i32_slt_i32(
+; GFX950-LABEL: ps_uniform_i32_slt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lt_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_lt_i32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_slt_i32:
+; GFX1250-LABEL: ps_uniform_i32_slt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_lt_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_i32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -946,32 +768,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i32_sle_i32(
-; GFX950-LABEL: kernel_uniform_i32_sle_i32:
+define amdgpu_ps void @ps_uniform_i32_sle_i32(
+; GFX950-LABEL: ps_uniform_i32_sle_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_le_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_le_i32_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i32_sle_i32:
+; GFX1250-LABEL: ps_uniform_i32_sle_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_le_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_i32_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i32 %a,
@@ -987,38 +798,21 @@ entry:
; Uniform i16 compares
; ----------------------------------------------------------------------------
-define amdgpu_kernel void @kernel_uniform_i16_eq_i32(
-; GFX950-LABEL: kernel_uniform_i16_eq_i32:
+define amdgpu_ps void @ps_uniform_i16_eq_i32(
+; GFX950-LABEL: ps_uniform_i16_eq_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_eq_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_eq_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_eq_i32:
+; GFX1250-LABEL: ps_uniform_i16_eq_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1030,39 +824,25 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_eq_i64(
-; GFX950-LABEL: kernel_uniform_i16_eq_i64:
+define amdgpu_ps void @ps_uniform_i16_eq_i64(
+; GFX950-LABEL: ps_uniform_i16_eq_i64:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_eq_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b64 s[2:3], 1, 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
+; GFX950-NEXT: v_cmp_eq_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_mov_b32 s0, 0
+; GFX950-NEXT: v_mov_b32_e32 v3, s0
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_eq_i64:
+; GFX1250-LABEL: ps_uniform_i16_eq_i64:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_eq_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b64 s[2:3], 1, 0
+; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: s_mov_b32 s0, 0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1074,38 +854,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_ne_i32(
-; GFX950-LABEL: kernel_uniform_i16_ne_i32:
+define amdgpu_ps void @ps_uniform_i16_ne_i32(
+; GFX950-LABEL: ps_uniform_i16_ne_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_lg_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ne_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_ne_i32:
+; GFX1250-LABEL: ps_uniform_i16_ne_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_lg_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ne_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1117,38 +880,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_ugt_i32(
-; GFX950-LABEL: kernel_uniform_i16_ugt_i32:
+define amdgpu_ps void @ps_uniform_i16_ugt_i32(
+; GFX950-LABEL: ps_uniform_i16_ugt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_gt_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_gt_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_ugt_i32:
+; GFX1250-LABEL: ps_uniform_i16_ugt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_gt_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1160,38 +906,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_uge_i32(
-; GFX950-LABEL: kernel_uniform_i16_uge_i32:
+define amdgpu_ps void @ps_uniform_i16_uge_i32(
+; GFX950-LABEL: ps_uniform_i16_uge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_ge_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ge_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_uge_i32:
+; GFX1250-LABEL: ps_uniform_i16_uge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_ge_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1203,38 +932,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_ult_i32(
-; GFX950-LABEL: kernel_uniform_i16_ult_i32:
+define amdgpu_ps void @ps_uniform_i16_ult_i32(
+; GFX950-LABEL: ps_uniform_i16_ult_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_lt_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_lt_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_ult_i32:
+; GFX1250-LABEL: ps_uniform_i16_ult_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_lt_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1246,38 +958,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_ule_i32(
-; GFX950-LABEL: kernel_uniform_i16_ule_i32:
+define amdgpu_ps void @ps_uniform_i16_ule_i32(
+; GFX950-LABEL: ps_uniform_i16_ule_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_lshr_b32 s3, s2, 16
-; GFX950-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX950-NEXT: s_cmp_le_u32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_le_u16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_ule_i32:
+; GFX1250-LABEL: ps_uniform_i16_ule_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 16
-; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_le_u32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1289,38 +984,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_sgt_i32(
-; GFX950-LABEL: kernel_uniform_i16_sgt_i32:
+define amdgpu_ps void @ps_uniform_i16_sgt_i32(
+; GFX950-LABEL: ps_uniform_i16_sgt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_ashr_i32 s3, s2, 16
-; GFX950-NEXT: s_sext_i32_i16 s2, s2
-; GFX950-NEXT: s_cmp_gt_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_gt_i16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_sgt_i32:
+; GFX1250-LABEL: ps_uniform_i16_sgt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
-; GFX1250-NEXT: s_sext_i32_i16 s2, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_gt_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1332,38 +1010,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_sge_i32(
-; GFX950-LABEL: kernel_uniform_i16_sge_i32:
+define amdgpu_ps void @ps_uniform_i16_sge_i32(
+; GFX950-LABEL: ps_uniform_i16_sge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_ashr_i32 s3, s2, 16
-; GFX950-NEXT: s_sext_i32_i16 s2, s2
-; GFX950-NEXT: s_cmp_ge_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ge_i16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_sge_i32:
+; GFX1250-LABEL: ps_uniform_i16_sge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
-; GFX1250-NEXT: s_sext_i32_i16 s2, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_ge_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1375,38 +1036,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_slt_i32(
-; GFX950-LABEL: kernel_uniform_i16_slt_i32:
+define amdgpu_ps void @ps_uniform_i16_slt_i32(
+; GFX950-LABEL: ps_uniform_i16_slt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_ashr_i32 s3, s2, 16
-; GFX950-NEXT: s_sext_i32_i16 s2, s2
-; GFX950-NEXT: s_cmp_lt_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_lt_i16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_slt_i32:
+; GFX1250-LABEL: ps_uniform_i16_slt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
-; GFX1250-NEXT: s_sext_i32_i16 s2, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_lt_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1418,38 +1062,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i16_sle_i32(
-; GFX950-LABEL: kernel_uniform_i16_sle_i32:
+define amdgpu_ps void @ps_uniform_i16_sle_i32(
+; GFX950-LABEL: ps_uniform_i16_sle_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_ashr_i32 s3, s2, 16
-; GFX950-NEXT: s_sext_i32_i16 s2, s2
-; GFX950-NEXT: s_cmp_le_i32 s2, s3
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX950-NEXT: s_cselect_b32 s2, 1, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_le_i16_e32 vcc, v2, v3
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i16_sle_i32:
+; GFX1250-LABEL: ps_uniform_i16_sle_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
-; GFX1250-NEXT: s_sext_i32_i16 s2, s2
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_le_i32 s2, s3
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i16 %a,
@@ -1465,32 +1092,21 @@ entry:
; Divergent i64 compares
; ----------------------------------------------------------------------------
-define amdgpu_kernel void @kernel_i64_eq_i32(
-; GFX950-LABEL: kernel_i64_eq_i32:
+define amdgpu_ps void @ps_i64_eq_i32(
+; GFX950-LABEL: ps_i64_eq_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_eq_i32:
+; GFX1250-LABEL: ps_i64_eq_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1502,34 +1118,25 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_i64_eq_i64(
-; GFX950-LABEL: kernel_i64_eq_i64:
+define amdgpu_ps void @ps_i64_eq_i64(
+; GFX950-LABEL: ps_i64_eq_i64:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_mov_b32 s4, 0
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s4
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX950-NEXT: s_mov_b32 s0, 0
+; GFX950-NEXT: v_mov_b32_e32 v3, s0
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_eq_i64:
+; GFX1250-LABEL: ps_i64_eq_i64:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_eq_u64 s[2:3], s[6:7]
-; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s3, -1, 0
-; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s3
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mov_b32_e32 v3, s0
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1541,32 +1148,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_i64_ne_i32(
-; GFX950-LABEL: kernel_i64_ne_i32:
+define amdgpu_ps void @ps_i64_ne_i32(
+; GFX950-LABEL: ps_i64_ne_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_lg_u64 s[2:3], s[6:7]
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GFX950-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX950-NEXT: v_cmp_ne_u64_e32 vcc, v[2:3], v[4:5]
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_ne_i32:
+; GFX1250-LABEL: ps_i64_ne_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_lg_u64 s[2:3], s[6:7]
-; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1578,32 +1174,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_i64_ugt_i32(
-; GFX950-LABEL: kernel_i64_ugt_i32:
+define amdgpu_ps void @ps_i64_ugt_i32(
+; GFX950-LABEL: ps_i64_ugt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_ugt_i32:
+; GFX1250-LABEL: ps_i64_ugt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_gt_u64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1615,32 +1200,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_i64_uge_i32(
-; GFX950-LABEL: kernel_i64_uge_i32:
+define amdgpu_ps void @ps_i64_uge_i32(
+; GFX950-LABEL: ps_i64_uge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_uge_i32:
+; GFX1250-LABEL: ps_i64_uge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_ge_u64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1652,32 +1226,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_i64_ult_i32(
-; GFX950-LABEL: kernel_i64_ult_i32:
+define amdgpu_ps void @ps_i64_ult_i32(
+; GFX950-LABEL: ps_i64_ult_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_lt_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_i64_ult_i32:
+; GFX1250-LABEL: ps_i64_ult_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_lt_u64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1689,32 +1252,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i64_ule_i32(
-; GFX950-LABEL: kernel_uniform_i64_ule_i32:
+define amdgpu_ps void @ps_uniform_i64_ule_i32(
+; GFX950-LABEL: ps_uniform_i64_ule_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_le_u64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_le_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i64_ule_i32:
+; GFX1250-LABEL: ps_uniform_i64_ule_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_le_u64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1726,32 +1278,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i64_sgt_i32(
-; GFX950-LABEL: kernel_uniform_i64_sgt_i32:
+define amdgpu_ps void @ps_uniform_i64_sgt_i32(
+; GFX950-LABEL: ps_uniform_i64_sgt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_gt_i64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i64_sgt_i32:
+; GFX1250-LABEL: ps_uniform_i64_sgt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_gt_i64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_gt_i64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1763,32 +1304,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i64_sge_i32(
-; GFX950-LABEL: kernel_uniform_i64_sge_i32:
+define amdgpu_ps void @ps_uniform_i64_sge_i32(
+; GFX950-LABEL: ps_uniform_i64_sge_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_ge_i64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i64_sge_i32:
+; GFX1250-LABEL: ps_uniform_i64_sge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_ge_i64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_ge_i64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1800,32 +1330,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i64_slt_i32(
-; GFX950-LABEL: kernel_uniform_i64_slt_i32:
+define amdgpu_ps void @ps_uniform_i64_slt_i32(
+; GFX950-LABEL: ps_uniform_i64_slt_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_lt_i64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i64_slt_i32:
+; GFX1250-LABEL: ps_uniform_i64_slt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_lt_i64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_lt_i64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1837,32 +1356,21 @@ entry:
ret void
}
-define amdgpu_kernel void @kernel_uniform_i64_sle_i32(
-; GFX950-LABEL: kernel_uniform_i64_sle_i32:
+define amdgpu_ps void @ps_uniform_i64_sle_i32(
+; GFX950-LABEL: ps_uniform_i64_sle_i32:
; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
-; GFX950-NEXT: v_cmp_le_i64_e32 vcc, s[2:3], v[0:1]
+; GFX950-NEXT: v_cmp_le_i64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX950-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX950-NEXT: global_store_dword v[0:1], v2, off
; GFX950-NEXT: s_endpgm
;
-; GFX1250-LABEL: kernel_uniform_i64_sle_i32:
+; GFX1250-LABEL: ps_uniform_i64_sle_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_le_i64_e64 s2, s[2:3], s[6:7]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cmp_le_i64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) %out,
i64 %a,
@@ -1873,180 +1381,3 @@ entry:
store i32 %zext, ptr addrspace(1) %out
ret void
}
-
-
-; To test divergent with amdgpu kernel ABI.
-define amdgpu_kernel void @kernel_divergent_i32_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
-; GFX950-LABEL: kernel_divergent_i32_cmp_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX950-NEXT: s_endpgm
-;
-; GFX1250-LABEL: kernel_divergent_i32_cmp_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
-; GFX1250-NEXT: s_endpgm
-entry:
- %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
- %cmp = icmp eq i32 %a, %workitem.id
- %tmp2 = zext i1 %cmp to i32
- store i32 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-; To test divergent zero i1 to i64 with amdgpu kernel ABI
-define amdgpu_kernel void @kernel_divergent_i64_cmp_i32(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
-; GFX950-LABEL: kernel_divergent_i64_cmp_i32:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX950-NEXT: v_mov_b32_e32 v2, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
-; GFX950-NEXT: s_mov_b32 s2, 0
-; GFX950-NEXT: v_mov_b32_e32 v1, s2
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX950-NEXT: s_endpgm
-;
-; GFX1250-LABEL: kernel_divergent_i64_cmp_i32:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_mov_b32 s3, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX1250-NEXT: s_endpgm
-entry:
- %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
- %cmp = icmp eq i32 %a, %workitem.id
- %tmp2 = zext i1 %cmp to i64
- store i64 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-; To test divergent with amdgpu kernel ABI.
-define amdgpu_kernel void @zext_i1_to_i32_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a) #0 {
-; GFX950-LABEL: zext_i1_to_i32_uniform:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dword s2, s[4:5], 0x30
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX950-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
-; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX950-NEXT: s_endpgm
-;
-; GFX1250-LABEL: zext_i1_to_i32_uniform:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x30 nv
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, s2, v0
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
-; GFX1250-NEXT: s_endpgm
-entry:
- %workitem.id = call i32 @llvm.amdgcn.workitem.id.x()
- %cmp = icmp eq i32 %a, %workitem.id
- %tmp2 = zext i1 %cmp to i32
- store i32 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-; TO test zero_extend from i1 to 64 uniform
-define amdgpu_kernel void @zext_i1_to_i64_uniform(ptr addrspace(1) %out, i1 %pred, i32 %a, i32 %b) #0 {
-; GFX950-LABEL: zext_i1_to_i64_uniform:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x30
-; GFX950-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_cmp_eq_u32 s0, s1
-; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX950-NEXT: s_cselect_b64 s[0:1], 1, 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: global_store_dwordx2 v0, v[2:3], s[2:3]
-; GFX950-NEXT: s_endpgm
-;
-; GFX1250-LABEL: zext_i1_to_i64_uniform:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x30 nv
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
-; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_cselect_b64 s[0:1], 1, 0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[2:3]
-; GFX1250-NEXT: s_endpgm
-entry:
- %cmp = icmp eq i32 %a, %b
- %tmp2 = zext i1 %cmp to i64
- store i64 %tmp2, ptr addrspace(1) %out
- ret void
-}
-
-define amdgpu_ps i32 @zext_i1_to_i32_uniform_amdgpu_ps_abi(i32 inreg %a, i32 inreg %b) #0 {
-; GFX950-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
-; GFX950: ; %bb.0: ; %entry
-; GFX950-NEXT: s_cmp_eq_u32 s0, s1
-; GFX950-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX950-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX950-NEXT: s_cselect_b32 s0, 1, 0
-; GFX950-NEXT: ; return to shader part epilog
-;
-; GFX1250-LABEL: zext_i1_to_i32_uniform_amdgpu_ps_abi:
-; GFX1250: ; %bb.0: ; %entry
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_cmp_eq_u32 s0, s1
-; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
-; GFX1250-NEXT: ; return to shader part epilog
-entry:
- %cmp = icmp eq i32 %a, %b
- %tmp2 = zext i1 %cmp to i32
- ret i32 %tmp2
-}
>From 040375168a312b51606d09a6cd3af7ebf047f3ae Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Thu, 21 May 2026 06:16:05 +0000
Subject: [PATCH 24/31] comments
---
llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 3 +--
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 7 +++++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index ce0e7b73cdfb9..a867ddfc600a2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -179,8 +179,7 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
DstRC = SrcRC;
// If all uses are reading from the src physical register and copying the
- // register is either impossible or very expensive, then don't create a
- // copy.
+ // register is either impossible or very expensive, then don't create a copy.
if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
VRBase = SrcReg;
} else {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 838a46878d609..a74656888a5e8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -355,8 +355,11 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
SDValue Chain = CurDAG->getEntryNode(); // Basic chain to anchor the copy
SDValue CopyToSCC =
CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
- SDValue Ops[4] = {TrueValue, FalseValue, CopyToSCC.getValue(1),
- CopyToSCC.getValue(1)};
+ SDValue Ops[4] = {
+ TrueValue, FalseValue,
+ CopyToSCC.getValue(1), // Explicitly state SCC is read
+ CopyToSCC.getValue(2) // Glue
+ };
bool IsInt32 = ResType == MVT::i32;
unsigned SelectCode = IsInt32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
MachineSDNode *CSelectNode = CurDAG->getMachineNode(
>From 8039fc571f2850e99f885825f48e63983f16e7b5 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Thu, 21 May 2026 06:29:33 +0000
Subject: [PATCH 25/31] comments
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a74656888a5e8..efe9eb0dd940f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -357,8 +357,8 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
CurDAG->getCopyToReg(Chain, DL, AMDGPU::SCC, CondNode, SDValue());
SDValue Ops[4] = {
TrueValue, FalseValue,
- CopyToSCC.getValue(1), // Explicitly state SCC is read
- CopyToSCC.getValue(2) // Glue
+ CurDAG->getRegister(AMDGPU::SCC, MVT::i1), // Explicitly state SCC is read
+ CopyToSCC.getValue(1) // Glue
};
bool IsInt32 = ResType == MVT::i32;
unsigned SelectCode = IsInt32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
>From f9ac002a7cf27222faad60a2a29a73530efad996 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Thu, 21 May 2026 18:16:29 +0000
Subject: [PATCH 26/31] redundant cleanup
---
llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 6 ------
1 file changed, 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index a867ddfc600a2..2ec959f392738 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -159,12 +159,6 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
UseRC = CommonSubClass;
}
- if (!UseRC || !UseRC->isAllocatable())
- UseRC = RegClassForVT;
- else if (const TargetRegisterClass *CommonSubClass =
- TRI->getCommonSubClass(UseRC, RegClassForVT))
- UseRC = CommonSubClass;
-
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
>From d0a2697520a29c7357288db7f96ab965923d619e Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Thu, 28 May 2026 04:36:13 +0000
Subject: [PATCH 27/31] comments
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index efe9eb0dd940f..1720d994d49a2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -326,10 +326,8 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
}
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
- // This is special case for common pattern of compare + select if it can be
+ // This is special case for zext from setcc if it can be
// selected to SALU. The pattern will be directly selected to `s_cselect`, to
- // This is special case for the common pattern of compare + select if it can
- // be selected to SALU. The pattern will be directly selected to `s_cselect`,
// to avoid an intermediate i1 virtual register which will be interpreted as a
// lane mask.
if (N->isDivergent())
@@ -360,10 +358,11 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
CurDAG->getRegister(AMDGPU::SCC, MVT::i1), // Explicitly state SCC is read
CopyToSCC.getValue(1) // Glue
};
- bool IsInt32 = ResType == MVT::i32;
- unsigned SelectCode = IsInt32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
- MachineSDNode *CSelectNode = CurDAG->getMachineNode(
- SelectCode, DL, CurDAG->getVTList(IsInt32 ? MVT::i32 : MVT::i64), Ops);
+
+ unsigned SelectCode =
+ ResType == MVT::i32 ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64;
+ MachineSDNode *CSelectNode =
+ CurDAG->getMachineNode(SelectCode, DL, CurDAG->getVTList(ResType), Ops);
CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), SDValue(CSelectNode, 0));
return true;
>From 9295530ce379e9938843fe2ffd3284a0e71afb81 Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Thu, 28 May 2026 04:40:41 +0000
Subject: [PATCH 28/31] comments
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 1720d994d49a2..bc1387c479273 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -327,7 +327,7 @@ bool AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(SDNode *N) const {
bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
// This is special case for zext from setcc if it can be
- // selected to SALU. The pattern will be directly selected to `s_cselect`, to
+ // selected to SALU. The pattern will be directly selected to `s_cselect`,
// to avoid an intermediate i1 virtual register which will be interpreted as a
// lane mask.
if (N->isDivergent())
>From 2ea3bb4998bfed497ecc370fd357781efbe7e834 Mon Sep 17 00:00:00 2001
From: Zeng Wu <zengwu13 at amd.com>
Date: Fri, 29 May 2026 08:52:11 -0700
Subject: [PATCH 29/31] Update llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index bc1387c479273..cabb9f7c37163 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -342,7 +342,7 @@ bool AMDGPUDAGToDAGISel::preprocessZeroExtend(SDNode *N) const {
return false;
// TODO: To support the operand type is int64 if s_cmp_i64 is supported on
- // some targets
+ // TODO: Support i64 if s_cmp_i64 is available
if (CondNode->getOperand(0).getValueType() != MVT::i32)
return false;
>From 3bcc551a054021fb12fb573b0d9c31897273aaaf Mon Sep 17 00:00:00 2001
From: root <Zeng.Wu2 at amd.com>
Date: Sat, 30 May 2026 05:26:05 +0000
Subject: [PATCH 30/31] update lit-test
---
.../test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll | 632 +++--
.../AMDGPU/amdgpu-cs-chain-fp-nosave.ll | 52 +-
llvm/test/CodeGen/AMDGPU/anyext.ll | 14 +-
.../AMDGPU/atomic-optimizer-strict-wqm.ll | 8 +-
.../atomic_optimizations_global_pointer.ll | 197 +-
.../atomic_optimizations_local_pointer.ll | 263 +-
.../atomic_optimizations_pixelshader.ll | 10 +-
llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 100 +-
.../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 40 +-
llvm/test/CodeGen/AMDGPU/commute-compares.ll | 4 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 492 ++--
.../CodeGen/AMDGPU/extract_vector_dynelt.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 349 ++-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 442 ++--
llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll | 2271 +++++++++++------
.../CodeGen/AMDGPU/flat_atomics_i64_system.ll | 1253 ++++++---
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 1762 ++++++-------
llvm/test/CodeGen/AMDGPU/fptrunc.ll | 152 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 22 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 202 +-
.../CodeGen/AMDGPU/gep-const-address-space.ll | 14 +-
.../AMDGPU/global-atomic-fadd.f32-no-rtn.ll | 4 +-
.../AMDGPU/global-atomic-fadd.f32-rtn.ll | 4 +-
.../AMDGPU/global_atomics_scan_fadd.ll | 377 +--
.../AMDGPU/global_atomics_scan_fmax.ll | 175 +-
.../AMDGPU/global_atomics_scan_fmin.ll | 175 +-
.../AMDGPU/global_atomics_scan_fsub.ll | 377 +--
.../AMDGPU/implicit-kernarg-backend-usage.ll | 34 +-
.../AMDGPU/infer-addrspace-flat-atomic.ll | 14 +-
.../CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll | 5 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.private.ll | 37 +-
.../CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll | 38 +-
.../CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll | 26 +-
.../AMDGPU/llvm.amdgcn.readfirstlane.ll | 10 +-
llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll | 1204 ++++-----
llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll | 72 +-
llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll | 24 +-
llvm/test/CodeGen/AMDGPU/min.ll | 298 ++-
llvm/test/CodeGen/AMDGPU/s_cmp_0.ll | 84 +-
llvm/test/CodeGen/AMDGPU/saddo.ll | 88 +-
.../test/CodeGen/AMDGPU/setcc-multiple-use.ll | 7 +-
llvm/test/CodeGen/AMDGPU/setcc-opt.ll | 1063 +++++++-
.../test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll | 7 +-
.../si-unify-exit-multiple-unreachables.ll | 9 +-
llvm/test/CodeGen/AMDGPU/sminmax.ll | 1077 +++++++-
llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll | 21 +-
llvm/test/CodeGen/AMDGPU/ssubo.ll | 86 +-
.../test/CodeGen/AMDGPU/trunc-cmp-constant.ll | 24 +-
llvm/test/CodeGen/AMDGPU/uaddo.ll | 23 +-
llvm/test/CodeGen/AMDGPU/usubo.ll | 23 +-
llvm/test/CodeGen/AMDGPU/zero_extend.ll | 229 +-
51 files changed, 8734 insertions(+), 5164 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index 3dac24ed89fa0..d039093aed3a1 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -14483,11 +14483,13 @@ define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB193_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB193_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
@@ -14526,11 +14528,13 @@ define void @flat_atomic_xchg_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB193_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB193_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
@@ -14573,11 +14577,13 @@ define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB194_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB194_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_swap_x2 v[2:3], v[2:3], v[0:1] glc
@@ -14610,11 +14616,13 @@ define void @flat_atomic_xchg_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB194_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB194_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_swap_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -14650,14 +14658,16 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB195_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB195_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc
@@ -14697,14 +14707,16 @@ define void @flat_atomic_add_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB195_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB195_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -14747,11 +14759,13 @@ define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB196_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB196_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] glc
@@ -14786,11 +14800,13 @@ define void @flat_atomic_add_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB196_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB196_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_add_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -14827,14 +14843,16 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB197_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB197_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] glc
@@ -14874,14 +14892,16 @@ define void @flat_atomic_sub_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB197_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB197_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -14926,11 +14946,13 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB198_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB198_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_sub_x2 v[2:3], v[2:3], v[0:1] glc
@@ -14965,11 +14987,13 @@ define void @flat_atomic_sub_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB198_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB198_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -15008,14 +15032,16 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB199_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB199_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15055,14 +15081,16 @@ define void @flat_atomic_and_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB199_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB199_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -15106,11 +15134,13 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB200_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB200_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_and_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15145,11 +15175,13 @@ define void @flat_atomic_and_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB200_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB200_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -15187,14 +15219,16 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB201_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB201_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15251,14 +15285,16 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB201_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB201_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15319,11 +15355,13 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB202_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB202_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15377,11 +15415,13 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB202_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB202_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -15437,14 +15477,16 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB203_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB203_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15484,14 +15526,16 @@ define void @flat_atomic_or_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB203_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB203_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -15535,11 +15579,13 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB204_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB204_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_or_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15574,11 +15620,13 @@ define void @flat_atomic_or_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB204_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB204_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -15616,14 +15664,16 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB205_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB205_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15663,14 +15713,16 @@ define void @flat_atomic_xor_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB205_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB205_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -15714,11 +15766,13 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB206_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB206_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_xor_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15753,11 +15807,13 @@ define void @flat_atomic_xor_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB206_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB206_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -15795,14 +15851,16 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB207_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB207_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15843,14 +15901,16 @@ define void @flat_atomic_max_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB207_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB207_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -15895,11 +15955,13 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB208_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB208_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smax_x2 v[2:3], v[2:3], v[0:1] glc
@@ -15935,11 +15997,13 @@ define void @flat_atomic_max_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB208_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB208_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -15979,14 +16043,16 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB209_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB209_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16027,14 +16093,16 @@ define void @flat_atomic_min_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB209_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB209_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -16079,11 +16147,13 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB210_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB210_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_smin_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16119,11 +16189,13 @@ define void @flat_atomic_min_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB210_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB210_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -16163,14 +16235,16 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB211_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB211_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16211,14 +16285,16 @@ define void @flat_atomic_umax_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB211_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB211_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -16263,11 +16339,13 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB212_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB212_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umax_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16303,11 +16381,13 @@ define void @flat_atomic_umax_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB212_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB212_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -16347,14 +16427,16 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB213_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB213_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16395,14 +16477,16 @@ define void @flat_atomic_umin_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB213_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB213_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -16447,11 +16531,13 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB214_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB214_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_umin_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16487,11 +16573,13 @@ define void @flat_atomic_umin_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB214_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB214_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -16531,14 +16619,16 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB215_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB215_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16581,14 +16671,16 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB215_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB215_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_inc_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -16634,11 +16726,13 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB216_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB216_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
@@ -16676,11 +16770,13 @@ define void @flat_atomic_uinc_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB216_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB216_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -16721,14 +16817,16 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB217_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB217_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16773,14 +16871,16 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB217_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB217_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc0
@@ -16828,11 +16928,13 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB218_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB218_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] glc
@@ -16872,11 +16974,13 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB218_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB218_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc0
@@ -16918,14 +17022,16 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB219_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB219_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -16984,14 +17090,16 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB219_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB219_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17058,11 +17166,13 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB220_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB220_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17118,11 +17228,13 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB220_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB220_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17184,14 +17296,16 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB221_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB221_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17248,14 +17362,16 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB221_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB221_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17320,11 +17436,13 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB222_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB222_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17378,11 +17496,13 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB222_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB222_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -17447,18 +17567,22 @@ define void @flat_atomic_fadd_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB223_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB223_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB223_4
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB223_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc
@@ -17529,17 +17653,21 @@ define void @flat_atomic_fadd_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB224_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB224_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB224_4
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB224_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v0, s[4:5] glc
@@ -18289,20 +18417,24 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_shared_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB235_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB235_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB235_4
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB235_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc
@@ -18352,20 +18484,24 @@ define void @flat_atomic_fadd_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_shared_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB235_3
+; GFX950-NEXT: s_cbranch_scc0 .LBB235_3
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_vccz .LBB235_4
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc0 .LBB235_4
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc0
@@ -18420,17 +18556,21 @@ define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB236_3
+; GFX90A-NEXT: s_cbranch_scc0 .LBB236_3
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: s_cbranch_vccz .LBB236_4
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
+; GFX90A-NEXT: s_cbranch_scc0 .LBB236_4
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[4:5] glc
@@ -18475,17 +18615,21 @@ define void @flat_atomic_fadd_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB236_3
+; GFX950-NEXT: s_cbranch_scc0 .LBB236_3
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-NEXT: s_cbranch_vccz .LBB236_4
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-NEXT: s_cbranch_scc0 .LBB236_4
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc0
@@ -18533,14 +18677,16 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB237_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB237_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18590,14 +18736,16 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB237_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB237_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18653,11 +18801,13 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB238_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB238_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18704,11 +18854,13 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB238_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB238_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -18759,14 +18911,16 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB239_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB239_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] glc
@@ -18806,14 +18960,16 @@ define void @flat_atomic_fmax_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB239_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB239_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
@@ -18858,11 +19014,13 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB240_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB240_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] glc
@@ -18898,11 +19056,13 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB240_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB240_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
@@ -18941,14 +19101,16 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX90A-NEXT: s_cbranch_vccz .LBB241_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB241_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] glc
@@ -18988,14 +19150,16 @@ define void @flat_atomic_fmin_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v1, a1
-; GFX950-NEXT: s_cbranch_vccz .LBB241_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB241_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
@@ -19040,11 +19204,13 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[2:3]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB242_2
+; GFX90A-NEXT: s_cbranch_scc0 .LBB242_2
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] glc
@@ -19080,11 +19246,13 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB242_2
+; GFX950-NEXT: s_cbranch_scc0 .LBB242_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
@@ -19123,14 +19291,16 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB243_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB243_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19188,14 +19358,16 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB243_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB243_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19260,11 +19432,13 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB244_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB244_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19319,11 +19493,13 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB244_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB244_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19384,14 +19560,16 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_mov_b64 s[6:7], src_private_base
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a[0:1]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_cbranch_vccz .LBB245_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB245_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19449,14 +19627,16 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
+; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def a[0:1]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: v_accvgpr_read_b32 v5, a1
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX950-NEXT: s_cbranch_vccz .LBB245_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB245_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19521,11 +19701,13 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_addc_u32 s5, s17, 0
; GFX90A-NEXT: s_cmp_eq_u32 s5, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GFX90A-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
+; GFX90A-NEXT: s_cmp_lg_u32 s6, 1
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:5]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: s_cbranch_vccz .LBB246_4
+; GFX90A-NEXT: s_cbranch_scc0 .LBB246_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
@@ -19580,11 +19762,13 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_addc_u32 s1, s1, 0
; GFX950-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v[4:5]
; GFX950-NEXT: ;;#ASMEND
-; GFX950-NEXT: s_cbranch_vccz .LBB246_4
+; GFX950-NEXT: s_cbranch_scc0 .LBB246_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
index 236fcc95f223f..bbb6b8d68c557 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
@@ -170,11 +170,11 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) {
; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX12-NEXT: s_cbranch_scc1 .LBB3_1
; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 s0, s32
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_lshl_add_u32 v1, s2, 6, s0
-; GFX12-NEXT: scratch_store_b32 off, v0, s0
-; GFX12-NEXT: v_readfirstlane_b32 s32, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, s32
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_lshl_add_u32 v2, s2, 6, v1
+; GFX12-NEXT: scratch_store_b32 v1, v0, off
+; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_endpgm
;
; GFX942-LABEL: test_alloca_var:
@@ -195,11 +195,10 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) {
; GFX942-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX942-NEXT: s_cbranch_scc1 .LBB3_1
; GFX942-NEXT: ; %bb.2:
-; GFX942-NEXT: s_mov_b32 s0, s32
-; GFX942-NEXT: v_mov_b32_e32 v1, s0
-; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
-; GFX942-NEXT: scratch_store_dword off, v0, s0
-; GFX942-NEXT: v_readfirstlane_b32 s32, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s32
+; GFX942-NEXT: v_lshl_add_u32 v2, s2, 6, v1
+; GFX942-NEXT: scratch_store_dword v1, v0, off
+; GFX942-NEXT: v_readfirstlane_b32 s32, v2
; GFX942-NEXT: s_endpgm
%v = alloca i32, i32 %count, align 4, addrspace(5)
store i32 0, ptr addrspace(5) %v, align 4
@@ -347,11 +346,12 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) {
; GFX12-NEXT: s_add_co_u32 s0, s0, foo at gotpcrel32@lo+12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
-; GFX12-NEXT: s_mov_b32 s3, s32
+; GFX12-NEXT: v_mov_b32_e32 v1, s32
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX12-NEXT: v_lshl_add_u32 v1, s2, 6, s3
-; GFX12-NEXT: scratch_store_b32 off, v0, s3
-; GFX12-NEXT: v_readfirstlane_b32 s32, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_lshl_add_u32 v2, s2, 6, v1
+; GFX12-NEXT: scratch_store_b32 v1, v0, off
+; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_swappc_b64 s[30:31], s[0:1]
@@ -379,11 +379,10 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) {
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX942-NEXT: s_mov_b32 s3, s32
-; GFX942-NEXT: v_mov_b32_e32 v1, s3
-; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
-; GFX942-NEXT: scratch_store_dword off, v0, s3
-; GFX942-NEXT: v_readfirstlane_b32 s32, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, s32
+; GFX942-NEXT: v_lshl_add_u32 v2, s2, 6, v1
+; GFX942-NEXT: scratch_store_dword v1, v0, off
+; GFX942-NEXT: v_readfirstlane_b32 s32, v2
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX942-NEXT: s_endpgm
@@ -533,15 +532,15 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) {
; GFX12-NEXT: s_add_co_u32 s0, s0, foo at gotpcrel32@lo+12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
-; GFX12-NEXT: s_mov_b32 s4, s32
+; GFX12-NEXT: v_mov_b32_e32 v41, s32
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX12-NEXT: v_lshl_add_u32 v0, s2, 6, s4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshl_add_u32 v0, s2, 6, v41
; GFX12-NEXT: v_readfirstlane_b32 s32, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX12-NEXT: scratch_store_b32 off, v40, s4
+; GFX12-NEXT: scratch_store_b32 v41, v40, off
; GFX12-NEXT: s_endpgm
;
; GFX942-LABEL: test_call_and_alloca_var:
@@ -566,14 +565,13 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) {
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX942-NEXT: s_mov_b32 s4, s32
-; GFX942-NEXT: v_mov_b32_e32 v0, s4
-; GFX942-NEXT: v_lshl_add_u32 v0, s2, 6, v0
+; GFX942-NEXT: v_mov_b32_e32 v41, s32
+; GFX942-NEXT: v_lshl_add_u32 v0, s2, 6, v41
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_readfirstlane_b32 s32, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX942-NEXT: scratch_store_dword off, v40, s4
+; GFX942-NEXT: scratch_store_dword v41, v40, off
; GFX942-NEXT: s_endpgm
%v = alloca i32, i32 %count, align 4, addrspace(5)
call amdgpu_gfx void @foo()
diff --git a/llvm/test/CodeGen/AMDGPU/anyext.ll b/llvm/test/CodeGen/AMDGPU/anyext.ll
index 1492119a6022d..6dda5e92cb19a 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=verde < %s | FileCheck --check-prefix=GCN %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX8 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GFX9 %s
@@ -16,7 +16,9 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u32 s6, 0
; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN-NEXT: s_cselect_b32 s4, 1, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
@@ -29,7 +31,9 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_cmp_lg_u32 s6, 0
; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
@@ -42,7 +46,9 @@ define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s6, 0
; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
index 8170e57b9e75c..c80a0f91bf7cb 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10
declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)
@@ -26,9 +26,11 @@ define amdgpu_ps void @main(i32 %arg) {
; GFX10-NEXT: s_and_saveexec_b32 s3, s1
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.3: ; in Loop: Header=BB0_2 Depth=1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, exec_lo
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v1
-; GFX10-NEXT: s_and_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT: s_mov_b32 exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.4: ; in Loop: Header=BB0_2 Depth=1
; GFX10-NEXT: s_mov_b32 s5, s4
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 01de2f36ec74d..6d94badcf4671 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS,GFX7LESS_ITERATIVE %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8_ITERATIVE %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_ITERATIVE %s
@@ -1257,6 +1257,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: add_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1273,7 +1274,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -1377,8 +1378,9 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-LABEL: add_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1398,11 +1400,11 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132_DPP-NEXT: s_mov_b32 s4, s6
; GFX1132_DPP-NEXT: s_mov_b32 s6, -1
@@ -1505,8 +1507,9 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-LABEL: add_i32_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1526,11 +1529,11 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232_DPP-NEXT: s_mov_b32 s4, s6
; GFX1232_DPP-NEXT: s_mov_b32 s6, -1
@@ -1719,21 +1722,20 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s5, s5, 5
; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
+; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1032-NEXT: s_mov_b32 s9, s3
; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
@@ -1791,21 +1793,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1132-NEXT: v_mov_b32_e32 v1, 0
; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s5, s5, 5
; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
+; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1132-NEXT: s_mov_b32 s9, s3
; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -1864,21 +1866,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB3_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1232-NEXT: v_mov_b32_e32 v1, 0
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -3057,6 +3059,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: add_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -3104,7 +3107,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s7, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v9, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 16
; GFX1032_DPP-NEXT: v_writelane_b32 v7, s7, 16
@@ -3253,6 +3256,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: add_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -3300,8 +3304,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -3452,6 +3455,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_DPP-LABEL: add_i64_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
+; GFX1232_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -3501,8 +3505,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -5176,6 +5179,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: sub_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -5192,7 +5196,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -5323,6 +5327,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: sub_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -5345,8 +5350,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -5468,8 +5472,9 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-LABEL: sub_i32_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -5489,11 +5494,11 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232_DPP-NEXT: s_mov_b32 s4, s6
; GFX1232_DPP-NEXT: s_mov_b32 s6, -1
@@ -5762,20 +5767,20 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, v2, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v2, 0
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s10, s6, 5
; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: v_mul_lo_u32 v5, v0, 5
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
@@ -5783,19 +5788,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_mov_b32_e32 v8, v1
-; GFX1032-NEXT: v_mov_b32_e32 v7, v0
-; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
-; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo
-; GFX1032-NEXT: v_mov_b32_e32 v0, v5
-; GFX1032-NEXT: v_mov_b32_e32 v2, v7
-; GFX1032-NEXT: v_mov_b32_e32 v1, v6
-; GFX1032-NEXT: v_mov_b32_e32 v3, v8
+; GFX1032-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032-NEXT: v_mov_b32_e32 v8, v0
+; GFX1032-NEXT: v_sub_co_u32 v6, vcc_lo, v8, v5
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v9, vcc_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, v6
+; GFX1032-NEXT: v_mov_b32_e32 v2, v8
+; GFX1032-NEXT: v_mov_b32_e32 v1, v7
+; GFX1032-NEXT: v_mov_b32_e32 v3, v9
; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
@@ -5883,21 +5888,22 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, v2, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB9_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-NEXT: v_bcnt_u32_b32 v0, v2, 0
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s10, s6, 5
; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mul_lo_u32 v5, v0, 5
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX1132-NEXT: s_mov_b32 s4, s2
@@ -5905,19 +5911,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
-; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
+; GFX1132-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX1132-NEXT: v_sub_co_u32 v6, vcc_lo, v8, v5
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, v5
-; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, v6
+; GFX1132-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-NEXT: v_mov_b32_e32 v1, v6
+; GFX1132-NEXT: v_mov_b32_e32 v1, v7
; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
@@ -5984,21 +5990,21 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_mov_b32 s6, exec_lo
+; GFX1232-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB9_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1232-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1232-NEXT: v_mov_b32_e32 v1, 0
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
-; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
+; GFX1232-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -7551,6 +7557,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: sub_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -7598,7 +7605,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v9, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 16
@@ -7786,6 +7793,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: sub_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -7833,8 +7841,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 16
; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 16
@@ -8006,6 +8013,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_DPP-LABEL: sub_i64_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
+; GFX1232_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -8055,8 +8063,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -8314,8 +8321,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8479,9 +8487,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8534,9 +8543,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8705,9 +8715,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8761,9 +8772,10 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10219,8 +10231,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s3, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10383,9 +10396,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10437,9 +10451,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10607,9 +10622,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10662,9 +10678,10 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 95908b2b666cf..18a17a8286134 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn - -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX7LESS,GFX7LESS_ITERATIVE %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global - -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8_ITERATIVE %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -mattr=-flat-for-global - -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9_ITERATIVE %s
@@ -887,6 +887,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: add_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -900,7 +901,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -992,6 +993,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: add_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1006,12 +1008,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -1351,6 +1353,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i32_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1360,7 +1363,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
@@ -1413,7 +1416,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1132_DPP-LABEL: add_i32_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
@@ -1427,11 +1430,10 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -1576,17 +1578,16 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
@@ -1637,19 +1638,18 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
+; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -2549,6 +2549,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: add_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -2593,7 +2594,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v9, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
@@ -2735,6 +2736,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: add_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -2775,12 +2777,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, v8, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -3235,6 +3237,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i64_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -3267,7 +3270,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2
@@ -3347,7 +3350,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1132_DPP-LABEL: add_i64_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
@@ -3381,11 +3384,11 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v5, 0
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v3
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -4279,6 +4282,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: sub_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4292,7 +4296,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -4384,6 +4388,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: sub_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -4398,12 +4403,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -4743,6 +4748,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1032_DPP-LABEL: sub_i32_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -4752,7 +4758,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
@@ -4805,7 +4811,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1132_DPP-LABEL: sub_i32_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
@@ -4819,11 +4825,10 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -4972,17 +4977,16 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_mov_b32 s1, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s1, s1, 5
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
@@ -5039,19 +5043,18 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
+; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_mul_i32 s1, s1, 5
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -5962,6 +5965,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: sub_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -6006,7 +6010,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v9, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
@@ -6148,6 +6152,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: sub_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -6188,12 +6193,12 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, v8, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -6671,6 +6676,7 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: and_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1
@@ -6684,7 +6690,7 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -6777,8 +6783,9 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: and_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6790,12 +6797,12 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_and_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -7360,6 +7367,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: and_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s2
@@ -7384,7 +7392,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -7509,7 +7517,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: and_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s2
@@ -7541,8 +7549,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -8020,6 +8027,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: or_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8033,7 +8041,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -8125,6 +8133,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: or_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -8139,12 +8148,12 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_or_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -8708,6 +8717,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: or_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -8732,7 +8742,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -8857,7 +8867,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: or_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
@@ -8889,8 +8899,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -9368,6 +9377,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: xor_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -9381,7 +9391,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -9473,6 +9483,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: xor_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -9487,12 +9498,12 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_xor_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -10056,6 +10067,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: xor_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -10080,7 +10092,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -10205,7 +10217,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: xor_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
@@ -10237,8 +10249,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -10716,6 +10727,7 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: max_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v3, 1
@@ -10729,7 +10741,7 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -10822,8 +10834,9 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: max_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10835,12 +10848,12 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_max_i32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -11011,7 +11024,8 @@ define amdgpu_kernel void @max_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: max_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11076,8 +11090,9 @@ define amdgpu_kernel void @max_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: max_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11789,6 +11804,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: max_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s2
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, 1
@@ -11838,7 +11854,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -12005,7 +12021,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: max_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s2
@@ -12055,13 +12071,13 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -12540,6 +12556,7 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: min_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fffffff, v0, s0
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v3, -2
@@ -12553,7 +12570,7 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -12646,8 +12663,9 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: min_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fffffff, v0, s0
; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v3, -2
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12659,12 +12677,12 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_min_i32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -12835,7 +12853,8 @@ define amdgpu_kernel void @min_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: min_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -12900,8 +12919,9 @@ define amdgpu_kernel void @min_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: min_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -13613,6 +13633,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: min_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s2
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, -2
@@ -13662,7 +13683,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -13829,7 +13850,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: min_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s2
@@ -13879,13 +13900,13 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, -1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -14364,6 +14385,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umax_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -14377,7 +14399,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -14469,6 +14491,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umax_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -14483,12 +14506,12 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_max_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -14655,7 +14678,8 @@ define amdgpu_kernel void @umax_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: umax_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -14720,8 +14744,9 @@ define amdgpu_kernel void @umax_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: umax_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -15428,6 +15453,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umax_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
@@ -15477,7 +15503,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -15644,7 +15670,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umax_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
@@ -15688,13 +15714,13 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_cndmask_b32 v1, v3, v1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -16173,6 +16199,7 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umin_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1
@@ -16186,7 +16213,7 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -16279,8 +16306,9 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: umin_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -16292,12 +16320,12 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_min_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -16465,7 +16493,8 @@ define amdgpu_kernel void @umin_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: umin_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -16530,8 +16559,9 @@ define amdgpu_kernel void @umin_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: umin_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -17236,6 +17266,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umin_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s2
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1
@@ -17285,7 +17316,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -17452,7 +17483,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umin_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s2
@@ -17496,13 +17527,13 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_cndmask_b32 v1, v3, v1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index e7ade6614795c..031420831dbaa 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn-- - -amdgpu-atomic-optimizer-strategy=DPP -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX7 %s
; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global - -amdgpu-atomic-optimizer-strategy=DPP -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX8 %s
; RUN: llc -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global - -amdgpu-atomic-optimizer-strategy=DPP -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX89,GFX9 %s
@@ -438,6 +438,7 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1032-NEXT: s_and_saveexec_b32 s8, s9
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.1:
+; GFX1032-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032-NEXT: s_or_saveexec_b32 s9, -1
; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, v0, s9
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
@@ -451,7 +452,7 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1032-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-NEXT: v_readlane_b32 s10, v1, 15
; GFX1032-NEXT: s_mov_b32 exec_lo, s9
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1032-NEXT: s_or_saveexec_b32 s9, -1
; GFX1032-NEXT: v_writelane_b32 v3, s10, 16
; GFX1032-NEXT: s_mov_b32 exec_lo, s9
@@ -560,6 +561,7 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1132-NEXT: s_and_saveexec_b32 s8, s9
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.1:
+; GFX1132-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-NEXT: s_or_saveexec_b32 s9, -1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-NEXT: v_cndmask_b32_e64 v1, 0, v0, s9
@@ -577,11 +579,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1132-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-NEXT: v_readlane_b32 s10, v1, 15
; GFX1132-NEXT: s_mov_b32 exec_lo, s9
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
; GFX1132-NEXT: s_or_saveexec_b32 s9, -1
; GFX1132-NEXT: v_writelane_b32 v3, s10, 16
; GFX1132-NEXT: s_mov_b32 exec_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0
; GFX1132-NEXT: s_and_saveexec_b32 s9, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index ae42404fd3818..23bd4e5a6bbf1 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
@@ -459,10 +459,12 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX908-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX908-NEXT: s_cmp_eq_u32 s7, s5
; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX908-NEXT: s_cselect_b32 s6, 1, 0
; GFX908-NEXT: s_mov_b64 s[4:5], -1
-; GFX908-NEXT: s_mov_b32 s6, 1
-; GFX908-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX908-NEXT: s_mov_b32 s7, 1
+; GFX908-NEXT: s_cmp_lg_u32 s6, s7
+; GFX908-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_cbranch_vccnz .LBB4_3
@@ -508,10 +510,12 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s6, 1
-; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX90A-NEXT: s_mov_b32 s7, 1
+; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB4_3
@@ -557,10 +561,12 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT: s_cselect_b32 s2, 1, 0
; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s2, 1
-; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2
+; GFX942-NEXT: s_mov_b32 s3, 1
+; GFX942-NEXT: s_cmp_lg_u32 s2, s3
+; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB4_3
@@ -604,10 +610,12 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX1100-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX1100-NEXT: s_cmp_eq_u32 s3, s1
; GFX1100-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1100-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1100-NEXT: s_cselect_b32 s1, 1, 0
; GFX1100-NEXT: s_mov_b32 s0, -1
-; GFX1100-NEXT: s_mov_b32 s1, 1
-; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1
+; GFX1100-NEXT: s_mov_b32 s2, 1
+; GFX1100-NEXT: s_cmp_lg_u32 s1, s2
+; GFX1100-NEXT: s_cselect_b32 s1, -1, 0
; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1100-NEXT: s_cbranch_vccnz .LBB4_3
@@ -660,11 +668,14 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX1200-NEXT: s_cmp_eq_u32 s3, s1
; GFX1200-NEXT: s_cselect_b32 s0, -1, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1200-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GFX1200-NEXT: s_mov_b32 s0, -1
-; GFX1200-NEXT: s_mov_b32 s1, 1
+; GFX1200-NEXT: s_mov_b32 s2, 1
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_cmp_lg_u32 s1, s2
+; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s1
; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -723,10 +734,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX908-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX908-NEXT: s_cmp_eq_u32 s7, s5
; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX908-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX908-NEXT: s_cselect_b32 s6, 1, 0
; GFX908-NEXT: s_mov_b64 s[4:5], -1
-; GFX908-NEXT: s_mov_b32 s6, 1
-; GFX908-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX908-NEXT: s_mov_b32 s7, 1
+; GFX908-NEXT: s_cmp_lg_u32 s6, s7
+; GFX908-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_cbranch_vccnz .LBB5_2
@@ -790,10 +803,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s6, 1
-; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX90A-NEXT: s_mov_b32 s7, 1
+; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3
@@ -813,10 +828,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s6, 1
-; GFX90A-NEXT: v_cmp_ne_u32_e64 s[6:7], v2, s6
+; GFX90A-NEXT: s_mov_b32 s7, 1
+; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
+; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5
@@ -882,10 +899,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT: s_cselect_b32 s2, 1, 0
; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s2, 1
-; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2
+; GFX942-NEXT: s_mov_b32 s3, 1
+; GFX942-NEXT: s_cmp_lg_u32 s2, s3
+; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_3
@@ -905,10 +924,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX942-NEXT: s_cselect_b32 s2, 1, 0
; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s2, 1
-; GFX942-NEXT: v_cmp_ne_u32_e64 s[2:3], v2, s2
+; GFX942-NEXT: s_mov_b32 s3, 1
+; GFX942-NEXT: s_cmp_lg_u32 s2, s3
+; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_5
@@ -972,10 +993,12 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX1100-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX1100-NEXT: s_cmp_eq_u32 s3, s1
; GFX1100-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1100-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1100-NEXT: s_cselect_b32 s1, 1, 0
; GFX1100-NEXT: s_mov_b32 s0, -1
-; GFX1100-NEXT: s_mov_b32 s1, 1
-; GFX1100-NEXT: v_cmp_ne_u32_e64 s1, v2, s1
+; GFX1100-NEXT: s_mov_b32 s2, 1
+; GFX1100-NEXT: s_cmp_lg_u32 s1, s2
+; GFX1100-NEXT: s_cselect_b32 s1, -1, 0
; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1100-NEXT: s_cbranch_vccnz .LBB5_2
@@ -1045,11 +1068,14 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX1200-NEXT: s_cmp_eq_u32 s3, s1
; GFX1200-NEXT: s_cselect_b32 s0, -1, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1200-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GFX1200-NEXT: s_mov_b32 s0, -1
-; GFX1200-NEXT: s_mov_b32 s1, 1
+; GFX1200-NEXT: s_mov_b32 s2, 1
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_cmp_lg_u32 s1, s2
+; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cmp_ne_u32_e64 s1, v2, s1
; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
index ecafe94d4cd55..18a8e89d33534 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -475,9 +475,10 @@ define amdgpu_gfx void @test34(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_min_i32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmpk_lt_i32 s0, 0x3e9
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -495,9 +496,10 @@ define amdgpu_gfx void @test35(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_max_i32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e8
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -515,9 +517,10 @@ define amdgpu_gfx void @test36(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_min_u32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmp_lt_u32 s0, s6
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -535,9 +538,10 @@ define amdgpu_gfx void @test37(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_max_i32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmp_ge_i32 s0, s6
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -555,9 +559,10 @@ define amdgpu_gfx void @test38(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_max_u32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmpk_lt_u32 s0, 0x3e9
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -575,9 +580,10 @@ define amdgpu_gfx void @test39(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_min_i32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmpk_gt_i32 s0, 0x3e7
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -595,9 +601,10 @@ define amdgpu_gfx void @test40(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_max_i32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmp_le_i32 s0, s6
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -615,9 +622,10 @@ define amdgpu_gfx void @test41(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_min_u32 s0, s4, s5
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_cmp_ge_u32 s0, s6
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GCN-NEXT: s_and_b32 s0, s0, exec_lo
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/commute-compares.ll b/llvm/test/CodeGen/AMDGPU/commute-compares.ll
index e4fb014af46ad..9233982d1e873 100644
--- a/llvm/test/CodeGen/AMDGPU/commute-compares.ll
+++ b/llvm/test/CodeGen/AMDGPU/commute-compares.ll
@@ -1450,7 +1450,9 @@ define amdgpu_kernel void @commute_frameindex(ptr addrspace(1) nocapture %out) #
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_mov_b32 s4, 0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GCN-NEXT: s_and_b64 s[4:5], vcc, exec
+; GCN-NEXT: s_cselect_b32 s4, 1, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index d19a260db3550..08a3af88907ae 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -262,13 +262,23 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB3_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_3
+; GFX9-SDAG-NEXT: ; %bb.4:
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent:
@@ -317,12 +327,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB3_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent:
@@ -496,13 +507,23 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB5_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB5_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB5_3
+; GFX9-SDAG-NEXT: ; %bb.4:
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent_under_aligned:
@@ -549,12 +570,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB5_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent_under_aligned:
@@ -601,7 +623,7 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x2000
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_4
+; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_6
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.0
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s5, s5, 15
; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
@@ -620,18 +642,29 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_2
; GFX9-SDAG-NEXT: ; %bb.3:
-; GFX9-SDAG-NEXT: s_mov_b32 s5, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 3
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; GFX9-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s9
+; GFX9-SDAG-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 4
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s5
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 4
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-SDAG-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s5, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s5
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: .LBB6_4: ; %bb.1
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB6_4
+; GFX9-SDAG-NEXT: ; %bb.5:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: .LBB6_6: ; %bb.1
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-SDAG-NEXT: s_and_b32 s4, s4, -16
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
@@ -730,14 +763,15 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB6_2
; GFX11-SDAG-NEXT: ; %bb.3:
-; GFX11-SDAG-NEXT: s_mov_b32 s3, s32
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, 3 :: v_dual_mov_b32 v2, 4
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s2, 5, s3
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s32 :: v_dual_mov_b32 v3, 4
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 3
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s1 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s3 dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v3, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s2, 5, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: .LBB6_4: ; %bb.1
; GFX11-SDAG-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v1, 2
@@ -831,7 +865,7 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-SDAG-NEXT: s_mov_b32 s4, 0
-; GFX9-SDAG-NEXT: s_cbranch_scc0 .LBB7_6
+; GFX9-SDAG-NEXT: s_cbranch_scc0 .LBB7_8
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.1
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
@@ -844,15 +878,26 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB7_2
; GFX9-SDAG-NEXT: ; %bb.3:
-; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s4, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s4, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-SDAG-NEXT: .LBB7_4: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_5
-; GFX9-SDAG-NEXT: .LBB7_4: ; %bb.0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_4
+; GFX9-SDAG-NEXT: ; %bb.5:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_7
+; GFX9-SDAG-NEXT: .LBB7_6: ; %bb.0
; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0xfff
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s5, s5, 15
; GFX9-SDAG-NEXT: s_and_b32 s4, s4, 0xfffff000
@@ -863,10 +908,10 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-SDAG-NEXT: s_add_i32 s32, s4, s5
; GFX9-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: .LBB7_5: ; %bb.2
+; GFX9-SDAG-NEXT: .LBB7_7: ; %bb.2
; GFX9-SDAG-NEXT: s_endpgm
-; GFX9-SDAG-NEXT: .LBB7_6:
-; GFX9-SDAG-NEXT: s_branch .LBB7_4
+; GFX9-SDAG-NEXT: .LBB7_8:
+; GFX9-SDAG-NEXT: s_branch .LBB7_6
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_control_flow:
; GFX9-GISEL: ; %bb.0: ; %entry
@@ -942,12 +987,13 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB7_2
; GFX11-SDAG-NEXT: ; %bb.3:
-; GFX11-SDAG-NEXT: s_mov_b32 s2, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 1
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s2
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s2 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB7_5
; GFX11-SDAG-NEXT: .LBB7_4: ; %bb.0
; GFX11-SDAG-NEXT: s_lshl2_add_u32 s1, s1, 15
@@ -1049,13 +1095,24 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB8_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB8_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB8_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1108,12 +1165,13 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB8_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1313,13 +1371,24 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB10_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 22
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 22
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB10_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB10_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1372,12 +1441,13 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB10_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 22
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 22
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1436,13 +1506,24 @@ define void @test_dynamic_stackalloc_device_divergent() {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB11_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7b
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB11_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB11_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1498,12 +1579,13 @@ define void @test_dynamic_stackalloc_device_divergent() {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB11_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1712,13 +1794,24 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB13_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB13_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB13_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1774,12 +1867,13 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB13_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1834,7 +1928,7 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: s_mov_b32 s34, s32
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x3000
; GFX9-SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-SDAG-NEXT: s_cbranch_execz .LBB14_6
+; GFX9-SDAG-NEXT: s_cbranch_execz .LBB14_8
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.0
; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
@@ -1866,40 +1960,62 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
; GFX9-SDAG-NEXT: ; %bb.5:
-; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s10, 6, v1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 3
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-SDAG-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s10, 6, v1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s9
+; GFX9-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 4
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 4
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-SDAG-NEXT: .LBB14_6: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s9, v1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s9, v1
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v2, off, s[0:3], s9
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: .LBB14_6: ; %bb.1
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr2
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB14_6
+; GFX9-SDAG-NEXT: ; %bb.7:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: .LBB14_8: ; %bb.1
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB14_7: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: .LBB14_9: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: s_ff1_i32_b64 s6, s[4:5]
; GFX9-SDAG-NEXT: v_readlane_b32 s7, v0, s6
; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s6
; GFX9-SDAG-NEXT: s_max_u32 s8, s8, s7
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_7
-; GFX9-SDAG-NEXT: ; %bb.8:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s33
+; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_9
+; GFX9-SDAG-NEXT: ; %bb.10:
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s33
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 2
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 2
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB14_11: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB14_11
+; GFX9-SDAG-NEXT: ; %bb.12:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
; GFX9-SDAG-NEXT: s_mov_b32 s34, s14
; GFX9-SDAG-NEXT: s_mov_b32 s33, s13
@@ -2034,14 +2150,15 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
; GFX11-SDAG-NEXT: ; %bb.5:
-; GFX11-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 4
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s3, 5, s4
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s2 dlc
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s32 :: v_dual_mov_b32 v4, 4
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 3
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s2 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s4 dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 v1, v4, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, s3, 5, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v2
; GFX11-SDAG-NEXT: .LBB14_6: ; %bb.1
; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
@@ -2057,15 +2174,16 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB14_7
; GFX11-SDAG-NEXT: ; %bb.8:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 2
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s33 dlc
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s32 :: v_dual_mov_b32 v3, 2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s33 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s1 dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v3, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
; GFX11-SDAG-NEXT: s_mov_b32 s33, s7
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s34
; GFX11-SDAG-NEXT: s_mov_b32 s34, s8
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -2207,7 +2325,7 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: ; implicit-def: $vgpr31
; GFX9-SDAG-NEXT: .LBB15_4: ; %Flow
; GFX9-SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-SDAG-NEXT: s_cbranch_execz .LBB15_8
+; GFX9-SDAG-NEXT: s_cbranch_execz .LBB15_10
; GFX9-SDAG-NEXT: ; %bb.5: ; %bb.0
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
@@ -2222,14 +2340,25 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
; GFX9-SDAG-NEXT: ; %bb.7:
-; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-SDAG-NEXT: .LBB15_8: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s8, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s8
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: .LBB15_8: ; %bb.2
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB15_8
+; GFX9-SDAG-NEXT: ; %bb.9:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: .LBB15_10: ; %bb.2
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
; GFX9-SDAG-NEXT: s_mov_b32 s34, s13
@@ -2358,12 +2487,13 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
; GFX11-SDAG-NEXT: ; %bb.7:
-; GFX11-SDAG-NEXT: s_mov_b32 s2, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 1
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s2
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s2 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s1, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: .LBB15_8: ; %bb.2
; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s34
@@ -2476,13 +2606,24 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB16_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB16_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB16_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -2539,12 +2680,13 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB16_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -2604,13 +2746,24 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB17_1
; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
+; GFX9-SDAG-NEXT: .LBB17_3: ; =>This Inner Loop Header: Depth=1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
+; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
+; GFX9-SDAG-NEXT: s_nop 2
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
+; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
+; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB17_3
+; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -2663,12 +2816,13 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB17_1
; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s0, 5, s1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index 975695b03c114..69978d6c9a83c 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -1948,9 +1948,9 @@ define amdgpu_kernel void @bit128_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u32 s2, 1
; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
; GCN-NEXT: s_cselect_b32 s3, s3, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 3
; GCN-NEXT: s_cselect_b32 s3, s3, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 2c02d7d9b3b7d..145bd639d8aa5 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope --check-prefixes=SI %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope --check-prefixes=VI %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope --check-prefixes=GFX9 %s
@@ -1089,13 +1089,13 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; SI-NEXT: s_and_b32 s3, s1, 0x1ff
; SI-NEXT: s_or_b32 s0, s3, s0
; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: s_lshr_b32 s0, s1, 8
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s3, s1, 8
; SI-NEXT: s_bfe_u32 s1, s1, 0xb0014
-; SI-NEXT: s_and_b32 s0, s0, 0xffe
-; SI-NEXT: v_readfirstlane_b32 s3, v0
+; SI-NEXT: s_and_b32 s3, s3, 0xffe
; SI-NEXT: s_sub_i32 s4, 0x3f1, s1
-; SI-NEXT: s_or_b32 s0, s0, s3
+; SI-NEXT: s_or_b32 s0, s3, s0
; SI-NEXT: v_med3_i32 v0, s4, 0, 13
; SI-NEXT: s_or_b32 s3, s0, 0x1000
; SI-NEXT: v_readfirstlane_b32 s4, v0
@@ -1134,13 +1134,13 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; VI-NEXT: s_and_b32 s3, s1, 0x1ff
; VI-NEXT: s_or_b32 s0, s3, s0
; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: s_lshr_b32 s0, s1, 8
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: s_lshr_b32 s3, s1, 8
; VI-NEXT: s_bfe_u32 s1, s1, 0xb0014
-; VI-NEXT: s_and_b32 s0, s0, 0xffe
-; VI-NEXT: v_readfirstlane_b32 s3, v0
+; VI-NEXT: s_and_b32 s3, s3, 0xffe
; VI-NEXT: s_sub_i32 s4, 0x3f1, s1
-; VI-NEXT: s_or_b32 s0, s0, s3
+; VI-NEXT: s_or_b32 s0, s3, s0
; VI-NEXT: v_med3_i32 v0, s4, 0, 13
; VI-NEXT: s_or_b32 s3, s0, 0x1000
; VI-NEXT: v_readfirstlane_b32 s4, v0
@@ -1181,13 +1181,13 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; GFX9-NEXT: s_and_b32 s3, s1, 0x1ff
; GFX9-NEXT: s_or_b32 s0, s3, s0
; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX9-NEXT: s_lshr_b32 s0, s1, 8
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: s_lshr_b32 s3, s1, 8
; GFX9-NEXT: s_bfe_u32 s1, s1, 0xb0014
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffe
-; GFX9-NEXT: v_readfirstlane_b32 s3, v0
+; GFX9-NEXT: s_and_b32 s3, s3, 0xffe
; GFX9-NEXT: s_sub_i32 s4, 0x3f1, s1
-; GFX9-NEXT: s_or_b32 s0, s0, s3
+; GFX9-NEXT: s_or_b32 s0, s3, s0
; GFX9-NEXT: v_med3_i32 v0, s4, 0, 13
; GFX9-NEXT: s_or_b32 s3, s0, 0x1000
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -1226,50 +1226,50 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f64_sign_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0x1ff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
; GFX11-TRUE16-NEXT: s_or_b32 s0, s3, s0
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-TRUE16-NEXT: s_bfe_u32 s0, s1, 0xb0014
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_bfe_u32 s3, s1, 0xb0014
; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_sub_i32 s3, 0x3f1, s0
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, 0x3f1, s3
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xffe
-; GFX11-TRUE16-NEXT: v_med3_i32 v1, s3, 0, 13
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-TRUE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
-; GFX11-TRUE16-NEXT: s_or_b32 s3, s1, 0x1000
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, s4
+; GFX11-TRUE16-NEXT: v_med3_i32 v0, s4, 0, 13
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s0, 0x1000
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s5, s4
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, s3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-TRUE16-NEXT: s_addk_i32 s0, 0xfc10
-; GFX11-TRUE16-NEXT: s_or_b32 s3, s5, s3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s0, 12
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, s1
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-TRUE16-NEXT: s_or_b32 s1, s5, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s3, 12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_or_b32 s4, s1, s4
-; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s0, 1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, s3, s4
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s3, 7
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s0, s4
+; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 2
; GFX11-TRUE16-NEXT: s_or_b32 s4, s4, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_add_i32 s3, s3, s4
-; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s0, 31
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
; GFX11-TRUE16-NEXT: s_movk_i32 s4, 0x7e00
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, s4, 0x7c00
-; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s0, 0x40f
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s1, s3
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s4, 0x7c00
+; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
@@ -1283,47 +1283,47 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_or_b32 s0, s3, s0
; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-FAKE16-NEXT: s_bfe_u32 s0, s1, 0xb0014
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_bfe_u32 s3, s1, 0xb0014
; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_sub_i32 s3, 0x3f1, s0
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, 0x3f1, s3
; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffe
-; GFX11-FAKE16-NEXT: v_med3_i32 v1, s3, 0, 13
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-FAKE16-NEXT: v_med3_i32 v0, s4, 0, 13
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s0, 0x1000
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s1, s3
-; GFX11-FAKE16-NEXT: s_or_b32 s3, s1, 0x1000
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, s4
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s5, s4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, s3
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-FAKE16-NEXT: s_addk_i32 s0, 0xfc10
-; GFX11-FAKE16-NEXT: s_or_b32 s3, s5, s3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s0, 12
-; GFX11-FAKE16-NEXT: s_or_b32 s4, s1, s4
-; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s0, 1
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, s3, s4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s3, 7
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, s1
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-FAKE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-FAKE16-NEXT: s_or_b32 s1, s5, s1
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s3, 12
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s0, s4
+; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, s1, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 2
; GFX11-FAKE16-NEXT: s_or_b32 s4, s4, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_add_i32 s3, s3, s4
-; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s0, 31
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s4
+; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
; GFX11-FAKE16-NEXT: s_movk_i32 s4, 0x7e00
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, s4, 0x7c00
-; GFX11-FAKE16-NEXT: s_cmpk_eq_i32 s0, 0x40f
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s1, s3
+; GFX11-FAKE16-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s4, 0x7c00
+; GFX11-FAKE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3776,9 +3776,9 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; SI-NEXT: s_and_b32 s6, s3, 0x1ff
; SI-NEXT: s_or_b32 s2, s6, s2
; SI-NEXT: s_cselect_b64 s[6:7], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[6:7]
+; SI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; SI-NEXT: s_cselect_b32 s2, 1, 0
; SI-NEXT: s_bfe_u32 s3, s3, 0xb0014
-; SI-NEXT: v_readfirstlane_b32 s2, v0
; SI-NEXT: s_sub_i32 s7, 0x3f1, s3
; SI-NEXT: s_or_b32 s2, s8, s2
; SI-NEXT: v_med3_i32 v0, s7, 0, 13
@@ -3814,9 +3814,9 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; SI-NEXT: s_and_b32 s2, s1, 0x1ff
; SI-NEXT: s_or_b32 s0, s2, s0
; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
; SI-NEXT: s_bfe_u32 s1, s1, 0xb0014
-; SI-NEXT: v_readfirstlane_b32 s0, v0
; SI-NEXT: s_sub_i32 s3, 0x3f1, s1
; SI-NEXT: s_or_b32 s0, s8, s0
; SI-NEXT: v_med3_i32 v0, s3, 0, 13
@@ -3863,9 +3863,9 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; VI-NEXT: s_and_b32 s6, s3, 0x1ff
; VI-NEXT: s_or_b32 s2, s6, s2
; VI-NEXT: s_cselect_b64 s[6:7], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[6:7]
+; VI-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; VI-NEXT: s_cselect_b32 s2, 1, 0
; VI-NEXT: s_bfe_u32 s3, s3, 0xb0014
-; VI-NEXT: v_readfirstlane_b32 s2, v0
; VI-NEXT: s_sub_i32 s6, 0x3f1, s3
; VI-NEXT: s_or_b32 s2, s5, s2
; VI-NEXT: v_med3_i32 v0, s6, 0, 13
@@ -3902,9 +3902,9 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; VI-NEXT: s_and_b32 s2, s1, 0x1ff
; VI-NEXT: s_or_b32 s0, s2, s0
; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: s_bfe_u32 s1, s1, 0xb0014
-; VI-NEXT: v_readfirstlane_b32 s0, v0
; VI-NEXT: s_sub_i32 s3, 0x3f1, s1
; VI-NEXT: s_or_b32 s0, s7, s0
; VI-NEXT: v_med3_i32 v0, s3, 0, 13
@@ -3950,9 +3950,9 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; GFX9-NEXT: s_and_b32 s6, s3, 0x1ff
; GFX9-NEXT: s_or_b32 s2, s6, s2
; GFX9-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[6:7]
+; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
; GFX9-NEXT: s_bfe_u32 s6, s3, 0xb0014
-; GFX9-NEXT: v_readfirstlane_b32 s2, v0
; GFX9-NEXT: s_sub_i32 s7, 0x3f1, s6
; GFX9-NEXT: s_or_b32 s2, s5, s2
; GFX9-NEXT: v_med3_i32 v0, s7, 0, 13
@@ -3991,8 +3991,8 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; GFX9-NEXT: s_and_b32 s2, s1, 0x1ff
; GFX9-NEXT: s_or_b32 s0, s2, s0
; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: s_bfe_u32 s3, s1, 0xb0014
; GFX9-NEXT: s_or_b32 s0, s6, s0
; GFX9-NEXT: s_sub_i32 s6, 0x3f1, s3
@@ -4041,46 +4041,44 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; GFX11-NEXT: s_and_b32 s5, s5, 0xffe
; GFX11-NEXT: s_or_b32 s2, s6, s2
; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
-; GFX11-NEXT: s_bfe_u32 s2, s3, 0xb0014
-; GFX11-NEXT: s_sub_i32 s6, 0x3f1, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_med3_i32 v1, s6, 0, 13
-; GFX11-NEXT: v_readfirstlane_b32 s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s7, v1
-; GFX11-NEXT: s_or_b32 s5, s5, s6
-; GFX11-NEXT: s_or_b32 s6, s5, 0x1000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s8, s6, s7
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_bfe_u32 s6, s3, 0xb0014
+; GFX11-NEXT: s_or_b32 s2, s5, s2
+; GFX11-NEXT: s_sub_i32 s7, 0x3f1, s6
+; GFX11-NEXT: s_or_b32 s5, s2, 0x1000
+; GFX11-NEXT: v_med3_i32 v0, s7, 0, 13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s7, v0
+; GFX11-NEXT: s_lshr_b32 s8, s5, s7
; GFX11-NEXT: s_lshl_b32 s7, s8, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmp_lg_u32 s7, s6
-; GFX11-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-NEXT: s_addk_i32 s2, 0xfc10
-; GFX11-NEXT: s_or_b32 s6, s8, s6
-; GFX11-NEXT: s_lshl_b32 s7, s2, 12
-; GFX11-NEXT: s_or_b32 s7, s5, s7
-; GFX11-NEXT: s_cmp_lt_i32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s6, s6, s7
+; GFX11-NEXT: s_cmp_lg_u32 s7, s5
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_addk_i32 s6, 0xfc10
+; GFX11-NEXT: s_or_b32 s5, s8, s5
+; GFX11-NEXT: s_lshl_b32 s7, s6, 12
+; GFX11-NEXT: s_or_b32 s7, s2, s7
+; GFX11-NEXT: s_cmp_lt_i32 s6, 1
+; GFX11-NEXT: s_cselect_b32 s5, s5, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s7, s6, 7
+; GFX11-NEXT: s_and_b32 s7, s5, 7
; GFX11-NEXT: s_cmp_gt_i32 s7, 5
; GFX11-NEXT: s_cselect_b32 s8, 1, 0
; GFX11-NEXT: s_cmp_eq_u32 s7, 3
; GFX11-NEXT: s_cselect_b32 s7, 1, 0
-; GFX11-NEXT: s_lshr_b32 s6, s6, 2
+; GFX11-NEXT: s_lshr_b32 s5, s5, 2
; GFX11-NEXT: s_or_b32 s7, s7, s8
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s6, s6, s7
-; GFX11-NEXT: s_cmp_lt_i32 s2, 31
+; GFX11-NEXT: s_add_i32 s5, s5, s7
+; GFX11-NEXT: s_cmp_lt_i32 s6, 31
; GFX11-NEXT: s_movk_i32 s7, 0x7e00
-; GFX11-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; GFX11-NEXT: s_cmp_lg_u32 s5, 0
-; GFX11-NEXT: s_cselect_b32 s5, s7, 0x7c00
-; GFX11-NEXT: s_cmpk_eq_i32 s2, 0x40f
-; GFX11-NEXT: s_cselect_b32 s2, s5, s6
+; GFX11-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-NEXT: s_cselect_b32 s2, s7, 0x7c00
+; GFX11-NEXT: s_cmpk_eq_i32 s6, 0x40f
+; GFX11-NEXT: s_cselect_b32 s2, s2, s5
; GFX11-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-NEXT: s_lshr_b32 s5, s1, 8
; GFX11-NEXT: s_and_b32 s3, s3, 0x8000
@@ -4089,29 +4087,27 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; GFX11-NEXT: s_or_b32 s2, s3, s2
; GFX11-NEXT: s_or_b32 s0, s6, s0
; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-NEXT: s_bfe_u32 s0, s1, 0xb0014
-; GFX11-NEXT: s_sub_i32 s3, 0x3f1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_med3_i32 v1, s3, 0, 13
-; GFX11-NEXT: v_readfirstlane_b32 s3, v0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_bfe_u32 s3, s1, 0xb0014
+; GFX11-NEXT: s_or_b32 s0, s5, s0
+; GFX11-NEXT: s_sub_i32 s6, 0x3f1, s3
+; GFX11-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX11-NEXT: v_med3_i32 v0, s6, 0, 13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_readfirstlane_b32 s6, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-NEXT: s_or_b32 s3, s5, s3
-; GFX11-NEXT: s_or_b32 s5, s3, 0x1000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshr_b32 s8, s5, s6
; GFX11-NEXT: s_lshl_b32 s6, s8, s6
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_lg_u32 s6, s5
; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_addk_i32 s0, 0xfc10
+; GFX11-NEXT: s_addk_i32 s3, 0xfc10
; GFX11-NEXT: s_or_b32 s5, s8, s5
-; GFX11-NEXT: s_lshl_b32 s6, s0, 12
-; GFX11-NEXT: s_or_b32 s6, s3, s6
-; GFX11-NEXT: s_cmp_lt_i32 s0, 1
+; GFX11-NEXT: s_lshl_b32 s6, s3, 12
+; GFX11-NEXT: s_or_b32 s6, s0, s6
+; GFX11-NEXT: s_cmp_lt_i32 s3, 1
; GFX11-NEXT: s_cselect_b32 s5, s5, s6
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s6, s5, 7
@@ -4123,12 +4119,12 @@ define amdgpu_ps i32 @s_copysign_out_v2f16_mag_v2f64_sign_v2f16(<2 x double> inr
; GFX11-NEXT: s_or_b32 s6, s6, s8
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_add_i32 s5, s5, s6
-; GFX11-NEXT: s_cmp_lt_i32 s0, 31
+; GFX11-NEXT: s_cmp_lt_i32 s3, 31
; GFX11-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s3, s7, 0x7c00
-; GFX11-NEXT: s_cmpk_eq_i32 s0, 0x40f
-; GFX11-NEXT: s_cselect_b32 s0, s3, s5
+; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cselect_b32 s0, s7, 0x7c00
+; GFX11-NEXT: s_cmpk_eq_i32 s3, 0x40f
+; GFX11-NEXT: s_cselect_b32 s0, s0, s5
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s1, s1, 0x8000
@@ -6772,13 +6768,13 @@ define amdgpu_ps i32 @s_copysign_f16_0_f64(double inreg %sign) {
; SI-NEXT: s_and_b32 s2, s1, 0x1ff
; SI-NEXT: s_or_b32 s0, s2, s0
; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; SI-NEXT: s_lshr_b32 s0, s1, 8
+; SI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s2, s1, 8
; SI-NEXT: s_bfe_u32 s3, s1, 0xb0014
-; SI-NEXT: s_and_b32 s0, s0, 0xffe
-; SI-NEXT: v_readfirstlane_b32 s2, v0
+; SI-NEXT: s_and_b32 s2, s2, 0xffe
; SI-NEXT: s_sub_i32 s4, 0x3f1, s3
-; SI-NEXT: s_or_b32 s0, s0, s2
+; SI-NEXT: s_or_b32 s0, s2, s0
; SI-NEXT: v_med3_i32 v0, s4, 0, 13
; SI-NEXT: s_or_b32 s2, s0, 0x1000
; SI-NEXT: v_readfirstlane_b32 s4, v0
@@ -6817,13 +6813,13 @@ define amdgpu_ps i32 @s_copysign_f16_0_f64(double inreg %sign) {
; VI-NEXT: s_and_b32 s2, s1, 0x1ff
; VI-NEXT: s_or_b32 s0, s2, s0
; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; VI-NEXT: s_lshr_b32 s0, s1, 8
+; VI-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: s_lshr_b32 s2, s1, 8
; VI-NEXT: s_bfe_u32 s3, s1, 0xb0014
-; VI-NEXT: s_and_b32 s0, s0, 0xffe
-; VI-NEXT: v_readfirstlane_b32 s2, v0
+; VI-NEXT: s_and_b32 s2, s2, 0xffe
; VI-NEXT: s_sub_i32 s4, 0x3f1, s3
-; VI-NEXT: s_or_b32 s0, s0, s2
+; VI-NEXT: s_or_b32 s0, s2, s0
; VI-NEXT: v_med3_i32 v0, s4, 0, 13
; VI-NEXT: s_or_b32 s2, s0, 0x1000
; VI-NEXT: v_readfirstlane_b32 s4, v0
@@ -6862,13 +6858,13 @@ define amdgpu_ps i32 @s_copysign_f16_0_f64(double inreg %sign) {
; GFX9-NEXT: s_and_b32 s2, s1, 0x1ff
; GFX9-NEXT: s_or_b32 s0, s2, s0
; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; GFX9-NEXT: s_lshr_b32 s0, s1, 8
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: s_lshr_b32 s2, s1, 8
; GFX9-NEXT: s_bfe_u32 s3, s1, 0xb0014
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffe
-; GFX9-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffe
; GFX9-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_or_b32 s0, s2, s0
; GFX9-NEXT: v_med3_i32 v0, s4, 0, 13
; GFX9-NEXT: s_or_b32 s2, s0, 0x1000
; GFX9-NEXT: v_readfirstlane_b32 s4, v0
@@ -6905,51 +6901,52 @@ define amdgpu_ps i32 @s_copysign_f16_0_f64(double inreg %sign) {
; GFX11-LABEL: s_copysign_f16_0_f64:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_and_b32 s2, s1, 0x1ff
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_or_b32 s0, s2, s0
; GFX11-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_sub_i32 s3, 0x3f1, s2
+; GFX11-NEXT: v_med3_i32 v0, s3, 0, 13
; GFX11-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-NEXT: s_bfe_u32 s0, s1, 0xb0014
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX11-NEXT: s_sub_i32 s2, 0x3f1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_med3_i32 v1, s2, 0, 13
-; GFX11-NEXT: v_readfirstlane_b32 s2, v0
-; GFX11-NEXT: v_readfirstlane_b32 s4, v1
-; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-NEXT: s_or_b32 s0, s3, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s3, s2, 0x1000
+; GFX11-NEXT: s_or_b32 s3, s0, 0x1000
; GFX11-NEXT: s_lshr_b32 s5, s3, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_lshl_b32 s4, s5, s4
; GFX11-NEXT: s_cmp_lg_u32 s4, s3
; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_addk_i32 s0, 0xfc10
+; GFX11-NEXT: s_addk_i32 s2, 0xfc10
; GFX11-NEXT: s_or_b32 s3, s5, s3
-; GFX11-NEXT: s_lshl_b32 s4, s0, 12
+; GFX11-NEXT: s_lshl_b32 s4, s2, 12
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s2, s2, s4
-; GFX11-NEXT: s_cmp_lt_i32 s0, 1
-; GFX11-NEXT: s_cselect_b32 s2, s3, s2
-; GFX11-NEXT: s_and_b32 s3, s2, 7
+; GFX11-NEXT: s_or_b32 s0, s0, s4
+; GFX11-NEXT: s_cmp_lt_i32 s2, 1
+; GFX11-NEXT: s_cselect_b32 s0, s3, s0
+; GFX11-NEXT: s_and_b32 s3, s0, 7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_cmp_gt_i32 s3, 5
; GFX11-NEXT: s_cselect_b32 s4, 1, 0
; GFX11-NEXT: s_cmp_eq_u32 s3, 3
; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_lshr_b32 s2, s2, 2
+; GFX11-NEXT: s_lshr_b32 s0, s0, 2
; GFX11-NEXT: s_or_b32 s3, s3, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s2, s2, s3
-; GFX11-NEXT: s_cmp_lt_i32 s0, 31
+; GFX11-NEXT: s_add_i32 s0, s0, s3
+; GFX11-NEXT: s_cmp_lt_i32 s2, 31
; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_cmpk_lg_i32 s0, 0x40f
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_and_b32 s0, s0, s3
+; GFX11-NEXT: s_cmpk_lg_i32 s2, 0x40f
+; GFX11-NEXT: s_cselect_b32 s2, -1, 0
+; GFX11-NEXT: s_and_b32 s2, s2, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; GFX11-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, s0, 0x7c00
; GFX11-NEXT: s_lshr_b32 s1, s1, 16
; GFX11-NEXT: s_or_b32 s0, s1, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index a6fd0b0b0b2d2..869e41a7c7ea9 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG %s
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx950 < %s | FileCheck -check-prefixes=GFX950,GFX950-SDAG %s
@@ -10,7 +10,7 @@
define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -53,7 +53,7 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_2047:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:2047 scope:SCOPE_DEV
@@ -97,7 +97,7 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_2047(ptr inreg %sbase, i
define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_nortn_offset_neg2048:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v1, s[2:3] offset:-2048 scope:SCOPE_DEV
@@ -146,7 +146,7 @@ define amdgpu_ps void @flat_xchg_saddr_i32_nortn_offset_neg2048(ptr inreg %sbase
define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -189,7 +189,7 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_2048:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -233,7 +233,7 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_2048(ptr inreg %sbase, i32 %voff
define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xchg_saddr_i32_rtn_neg2048:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_swap_b32 v0, v0, v1, s[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -290,7 +290,7 @@ define amdgpu_ps float @flat_xchg_saddr_i32_rtn_neg2048(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -306,7 +306,7 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
@@ -361,7 +361,7 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn(i32 %voffset, i
define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -377,7 +377,7 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
@@ -433,7 +433,7 @@ define amdgpu_ps float @flat_xchg_saddr_uniform_ptr_in_vgprs_rtn_immoffset(i32 %
define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -449,7 +449,7 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
@@ -503,7 +503,7 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn(i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32 %voffset, i32 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -519,7 +519,7 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-GISEL-NEXT: ds_load_b64 v[2:3], v2
; GFX1250-GISEL-NEXT: s_wait_dscnt 0x0
@@ -581,7 +581,7 @@ define amdgpu_ps void @flat_xchg_saddr_uniform_ptr_in_vgprs_nortn_immoffset(i32
define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -626,7 +626,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -762,7 +762,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -810,7 +810,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -956,7 +956,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -993,7 +993,7 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -1104,7 +1104,7 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xchg_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -1143,7 +1143,7 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_xchg_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -1269,7 +1269,7 @@ define amdgpu_ps void @flat_xchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1312,7 +1312,7 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_add_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1362,7 +1362,7 @@ define amdgpu_ps float @flat_add_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -1404,7 +1404,7 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_add_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_add_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
@@ -1453,7 +1453,7 @@ define amdgpu_ps void @flat_add_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1498,7 +1498,7 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1638,7 +1638,7 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -1686,7 +1686,7 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1836,7 +1836,7 @@ define amdgpu_ps <2 x float> @flat_add_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -1876,7 +1876,7 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -1998,7 +1998,7 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_add_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -2040,7 +2040,7 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_add_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -2177,7 +2177,7 @@ define amdgpu_ps void @flat_add_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2220,7 +2220,7 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_sub_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2270,7 +2270,7 @@ define amdgpu_ps float @flat_sub_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -2312,7 +2312,7 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_sub_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_sub_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
@@ -2361,7 +2361,7 @@ define amdgpu_ps void @flat_sub_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2406,7 +2406,7 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2548,7 +2548,7 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -2596,7 +2596,7 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -2748,7 +2748,7 @@ define amdgpu_ps <2 x float> @flat_sub_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -2788,7 +2788,7 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -2912,7 +2912,7 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_sub_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -2954,7 +2954,7 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_sub_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -3093,7 +3093,7 @@ define amdgpu_ps void @flat_sub_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3136,7 +3136,7 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_and_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3186,7 +3186,7 @@ define amdgpu_ps float @flat_and_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -3228,7 +3228,7 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_and_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_and_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
@@ -3277,7 +3277,7 @@ define amdgpu_ps void @flat_and_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3323,7 +3323,7 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3464,7 +3464,7 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -3513,7 +3513,7 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3664,7 +3664,7 @@ define amdgpu_ps <2 x float> @flat_and_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -3705,7 +3705,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -3828,7 +3828,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_and_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -3871,7 +3871,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_and_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -4009,7 +4009,7 @@ define amdgpu_ps void @flat_and_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4052,7 +4052,7 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i3
define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_or_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4102,7 +4102,7 @@ define amdgpu_ps float @flat_or_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voff
define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -4144,7 +4144,7 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_or_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_or_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
@@ -4193,7 +4193,7 @@ define amdgpu_ps void @flat_or_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4239,7 +4239,7 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4380,7 +4380,7 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -4429,7 +4429,7 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -4580,7 +4580,7 @@ define amdgpu_ps <2 x float> @flat_or_saddr_i64_rtn_neg128(ptr inreg %sbase, i32
define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -4621,7 +4621,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -4744,7 +4744,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_or_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -4787,7 +4787,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_or_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -4925,7 +4925,7 @@ define amdgpu_ps void @flat_or_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4968,7 +4968,7 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_xor_b32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5018,7 +5018,7 @@ define amdgpu_ps float @flat_xor_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] scope:SCOPE_DEV
@@ -5060,7 +5060,7 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_xor_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: flat_atomic_xor_b32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
@@ -5109,7 +5109,7 @@ define amdgpu_ps void @flat_xor_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5155,7 +5155,7 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5296,7 +5296,7 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -5345,7 +5345,7 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5496,7 +5496,7 @@ define amdgpu_ps <2 x float> @flat_xor_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -5537,7 +5537,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -5660,7 +5660,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -5703,7 +5703,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_xor_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -5841,7 +5841,7 @@ define amdgpu_ps void @flat_xor_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -5874,7 +5874,7 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -5914,7 +5914,7 @@ define amdgpu_ps float @flat_max_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -5946,7 +5946,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_max_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_i32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -5985,7 +5985,7 @@ define amdgpu_ps void @flat_max_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6026,7 +6026,7 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6162,7 +6162,7 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -6206,7 +6206,7 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6352,7 +6352,7 @@ define amdgpu_ps <2 x float> @flat_max_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -6389,7 +6389,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -6506,7 +6506,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_max_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -6545,7 +6545,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_max_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -6677,7 +6677,7 @@ define amdgpu_ps void @flat_max_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -6710,7 +6710,7 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -6750,7 +6750,7 @@ define amdgpu_ps float @flat_min_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -6782,7 +6782,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_min_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_i32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -6821,7 +6821,7 @@ define amdgpu_ps void @flat_min_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6862,7 +6862,7 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6998,7 +6998,7 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -7042,7 +7042,7 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7188,7 +7188,7 @@ define amdgpu_ps <2 x float> @flat_min_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -7225,7 +7225,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -7342,7 +7342,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_min_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -7381,7 +7381,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_min_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -7513,7 +7513,7 @@ define amdgpu_ps void @flat_min_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -7546,7 +7546,7 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -7586,7 +7586,7 @@ define amdgpu_ps float @flat_umax_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -7618,7 +7618,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umax_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_max_u32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -7657,7 +7657,7 @@ define amdgpu_ps void @flat_umax_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7698,7 +7698,7 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7834,7 +7834,7 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -7878,7 +7878,7 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8024,7 +8024,7 @@ define amdgpu_ps <2 x float> @flat_umax_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -8061,7 +8061,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -8178,7 +8178,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umax_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -8217,7 +8217,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_umax_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -8349,7 +8349,7 @@ define amdgpu_ps void @flat_umax_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -8382,7 +8382,7 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -8422,7 +8422,7 @@ define amdgpu_ps float @flat_umin_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3]
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -8454,7 +8454,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_umin_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_min_u32 v0, v1, s[2:3] offset:-128
; GFX1250-NEXT: s_wait_storecnt_dscnt 0x0
; GFX1250-NEXT: s_endpgm
@@ -8493,7 +8493,7 @@ define amdgpu_ps void @flat_umin_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8534,7 +8534,7 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8670,7 +8670,7 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -8714,7 +8714,7 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8860,7 +8860,7 @@ define amdgpu_ps <2 x float> @flat_umin_saddr_i64_rtn_neg128(ptr inreg %sbase, i
define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -8897,7 +8897,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -9014,7 +9014,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_umin_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -9053,7 +9053,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
;
; GFX1250-GISEL-LABEL: flat_umin_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -9185,7 +9185,7 @@ define amdgpu_ps void @flat_umin_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %v
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -9231,7 +9231,7 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn(ptr inreg %sbase, i32 %voffse
define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -9284,7 +9284,7 @@ define amdgpu_ps float @flat_cmpxchg_saddr_i32_rtn_neg128(ptr inreg %sbase, i32
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -9328,7 +9328,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %cmp, i32 %data) {
; GFX1250-LABEL: flat_cmpxchg_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-NEXT: global_wb scope:SCOPE_SYS
; GFX1250-NEXT: s_wait_storecnt 0x0
@@ -9379,7 +9379,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i32_nortn_neg128(ptr inreg %sbase, i32
define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v3
@@ -9426,7 +9426,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
@@ -9577,7 +9577,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn(ptr inreg %sbase, i32 %
define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -9627,7 +9627,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v8, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v6, v3
@@ -9788,7 +9788,7 @@ define amdgpu_ps <2 x float> @flat_cmpxchg_saddr_i64_rtn_neg128(ptr inreg %sbase
define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v5, v4
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v4, v3
@@ -9830,7 +9830,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -9962,7 +9962,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn(ptr inreg %sbase, i32 %voffs
define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %cmp, i64 %data) {
; GFX1250-SDAG-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v7, v2 :: v_dual_mov_b32 v6, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -10006,7 +10006,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
;
; GFX1250-GISEL-LABEL: flat_cmpxchg_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v9, v2
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v7, v4
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
@@ -10153,7 +10153,7 @@ define amdgpu_ps void @flat_cmpxchg_saddr_i64_nortn_neg128(ptr inreg %sbase, i32
define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -10186,7 +10186,7 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -10226,7 +10226,7 @@ define amdgpu_ps float @flat_inc_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
;
@@ -10255,7 +10255,7 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_inc_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_inc_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
;
@@ -10291,7 +10291,7 @@ define amdgpu_ps void @flat_inc_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10336,7 +10336,7 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10482,7 +10482,7 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -10530,7 +10530,7 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10686,7 +10686,7 @@ define amdgpu_ps <2 x float> @flat_inc_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -10724,7 +10724,7 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -10846,7 +10846,7 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_inc_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -10886,7 +10886,7 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_inc_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -11024,7 +11024,7 @@ define amdgpu_ps void @flat_inc_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_rtn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -11057,7 +11057,7 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn(ptr inreg %sbase, i32 %voffset, i
define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_rtn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v0, v1, s[2:3] offset:-128 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: ; return to shader part epilog
@@ -11097,7 +11097,7 @@ define amdgpu_ps float @flat_dec_saddr_i32_rtn_neg128(ptr inreg %sbase, i32 %vof
define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_nortn:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
;
@@ -11126,7 +11126,7 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %voffset, i32 %data) {
; GFX1250-LABEL: flat_dec_saddr_i32_nortn_neg128:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: flat_atomic_dec_u32 v0, v1, s[2:3] offset:-128 scope:SCOPE_DEV
; GFX1250-NEXT: s_endpgm
;
@@ -11162,7 +11162,7 @@ define amdgpu_ps void @flat_dec_saddr_i32_nortn_neg128(ptr inreg %sbase, i32 %vo
define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -11210,7 +11210,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -11360,7 +11360,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff
define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_rtn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -11411,7 +11411,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_rtn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v1
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -11571,7 +11571,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3
define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s0, exec_lo
@@ -11612,7 +11612,7 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -11738,7 +11738,7 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset,
define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %voffset, i64 %data) {
; GFX1250-SDAG-LABEL: flat_dec_saddr_i64_nortn_neg128:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], 0xffffffffffffff80
@@ -11781,7 +11781,7 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo
;
; GFX1250-GISEL-LABEL: flat_dec_saddr_i64_nortn_neg128:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v2
; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-GISEL-NEXT: s_mov_b32 s0, exec_lo
@@ -11925,15 +11925,21 @@ define double @flat_atomic_fadd_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB110_3
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB110_3
; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB110_4
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB110_4
; GFX1250-SDAG-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
@@ -12034,14 +12040,18 @@ define double @flat_atomic_fadd_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB110_3
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-SDAG-NEXT: s_cbranch_scc0 .LBB110_3
; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.check.private
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB110_4
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-SDAG-NEXT: s_cbranch_scc0 .LBB110_4
; GFX950-SDAG-NEXT: ; %bb.2: ; %atomicrmw.global
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX950-SDAG-NEXT: global_atomic_add_f64 v[2:3], v2, v[0:1], s[0:1] sc0
@@ -12138,9 +12148,12 @@ define void @flat_atomic_fadd_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB111_3
+; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB111_3
; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow2
; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB111_8
@@ -12151,9 +12164,12 @@ define void @flat_atomic_fadd_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB111_5
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB111_5
; GFX1250-SDAG-NEXT: ; %bb.4: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: s_mov_b32 s2, 0
@@ -12246,9 +12262,11 @@ define void @flat_atomic_fadd_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB111_3
+; GFX950-SDAG-NEXT: s_cbranch_scc1 .LBB111_3
; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow2
; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB111_8
@@ -12259,9 +12277,11 @@ define void @flat_atomic_fadd_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], src_private_base
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB111_5
+; GFX950-SDAG-NEXT: s_cbranch_scc0 .LBB111_5
; GFX950-SDAG-NEXT: ; %bb.4: ; %atomicrmw.global
; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX950-SDAG-NEXT: global_atomic_add_f64 v2, v[0:1], s[0:1]
@@ -12345,9 +12365,11 @@ define double @flat_atomic_fmax_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB112_2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB112_2
; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
@@ -12420,8 +12442,10 @@ define double @flat_atomic_fmax_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB112_2
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-SDAG-NEXT: s_cbranch_scc0 .LBB112_2
; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-SDAG-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
@@ -12493,10 +12517,12 @@ define void @flat_atomic_fmax_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB113_3
+; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB113_3
; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB113_4
@@ -12567,9 +12593,11 @@ define void @flat_atomic_fmax_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB113_3
+; GFX950-SDAG-NEXT: s_cbranch_scc1 .LBB113_3
; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB113_4
@@ -12638,9 +12666,11 @@ define double @flat_atomic_fmin_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
-; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB114_2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc0 .LBB114_2
; GFX1250-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-SDAG-NEXT: s_wait_storecnt 0x0
@@ -12713,8 +12743,10 @@ define double @flat_atomic_fmin_f64_saddr_rtn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB114_2
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
+; GFX950-SDAG-NEXT: s_cbranch_scc0 .LBB114_2
; GFX950-SDAG-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-SDAG-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
@@ -12786,10 +12818,12 @@ define void @flat_atomic_fmin_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX1250-SDAG-NEXT: s_xor_b32 s2, s1, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s2, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX1250-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB115_3
+; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB115_3
; GFX1250-SDAG-NEXT: ; %bb.1: ; %Flow
; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX1250-SDAG-NEXT: s_cbranch_vccz .LBB115_4
@@ -12860,9 +12894,11 @@ define void @flat_atomic_fmin_f64_saddr_nortn(ptr inreg %ptr, double %data) {
; GFX950-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s1, s3
; GFX950-SDAG-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GFX950-SDAG-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-SDAG-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-SDAG-NEXT: s_cbranch_vccnz .LBB115_3
+; GFX950-SDAG-NEXT: s_cbranch_scc1 .LBB115_3
; GFX950-SDAG-NEXT: ; %bb.1: ; %Flow
; GFX950-SDAG-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GFX950-SDAG-NEXT: s_cbranch_vccz .LBB115_4
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index 25fbdbc83b2b9..ca03986123945 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
@@ -19,9 +19,11 @@ define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB0_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB0_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB0_4
@@ -69,9 +71,11 @@ define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB0_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB0_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB0_4
@@ -112,9 +116,12 @@ define amdgpu_kernel void @atomic_add_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB0_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB0_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB0_4
@@ -160,8 +167,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB1_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB1_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -212,8 +221,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB1_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB1_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -258,8 +269,11 @@ define amdgpu_kernel void @atomic_add_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB1_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB1_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -310,9 +324,11 @@ define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB2_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB2_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB2_4
@@ -349,24 +365,27 @@ define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB2_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB2_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB2_4
@@ -412,9 +431,12 @@ define amdgpu_kernel void @atomic_add_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB2_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB2_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB2_4
@@ -463,8 +485,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB3_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB3_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -507,8 +531,8 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -517,8 +541,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB3_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB3_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -564,8 +590,11 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB3_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB3_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -611,9 +640,11 @@ define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB4_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB4_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB4_4
@@ -659,9 +690,11 @@ define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB4_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB4_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB4_4
@@ -700,10 +733,12 @@ define amdgpu_kernel void @atomic_add_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB4_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB4_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB4_4
@@ -746,8 +781,10 @@ define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB5_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB5_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -796,8 +833,10 @@ define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB5_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB5_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -840,9 +879,11 @@ define amdgpu_kernel void @atomic_add_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB5_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB5_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -890,9 +931,11 @@ define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB6_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB6_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB6_4
@@ -942,9 +985,11 @@ define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB6_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB6_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB6_4
@@ -988,10 +1033,12 @@ define amdgpu_kernel void @atomic_add_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB6_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB6_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB6_4
@@ -1037,8 +1084,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB7_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB7_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -1089,8 +1138,10 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB7_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB7_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -1134,9 +1185,11 @@ define amdgpu_kernel void @atomic_add_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB7_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB7_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -1183,9 +1236,11 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB8_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB8_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB8_4
@@ -1232,9 +1287,11 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB8_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB8_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB8_4
@@ -1274,9 +1331,12 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB8_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB8_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB8_4
@@ -1321,8 +1381,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB9_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB9_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -1372,8 +1434,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB9_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB9_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -1417,8 +1481,11 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB9_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB9_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -1468,9 +1535,11 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB10_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB10_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB10_4
@@ -1506,24 +1575,27 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB10_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB10_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB10_4
@@ -1568,9 +1640,12 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB10_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB10_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB10_4
@@ -1618,8 +1693,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB11_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB11_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -1661,8 +1738,8 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -1671,8 +1748,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB11_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB11_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -1717,8 +1796,11 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB11_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB11_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -1763,9 +1845,11 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB12_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB12_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB12_4
@@ -1810,9 +1894,11 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB12_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB12_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB12_4
@@ -1850,10 +1936,12 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB12_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB12_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB12_4
@@ -1895,8 +1983,10 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB13_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB13_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -1944,8 +2034,10 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB13_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB13_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -1987,9 +2079,11 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB13_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB13_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -2036,9 +2130,11 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB14_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB14_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB14_4
@@ -2087,9 +2183,11 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB14_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB14_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB14_4
@@ -2132,10 +2230,12 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB14_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB14_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB14_4
@@ -2180,8 +2280,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB15_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB15_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -2231,8 +2333,10 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB15_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB15_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -2275,9 +2379,11 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB15_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB15_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -2323,9 +2429,11 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB16_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB16_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB16_4
@@ -2373,9 +2481,11 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB16_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB16_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB16_4
@@ -2416,9 +2526,12 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB16_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB16_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB16_4
@@ -2464,8 +2577,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB17_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB17_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -2516,8 +2631,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB17_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB17_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -2562,8 +2679,11 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB17_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB17_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -2614,9 +2734,11 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB18_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB18_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB18_4
@@ -2653,24 +2775,27 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB18_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB18_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB18_4
@@ -2716,9 +2841,12 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB18_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB18_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB18_4
@@ -2767,8 +2895,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB19_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB19_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -2811,8 +2941,8 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -2821,8 +2951,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB19_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB19_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -2868,8 +3000,11 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB19_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB19_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -2915,9 +3050,11 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB20_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB20_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB20_4
@@ -2963,9 +3100,11 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB20_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB20_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB20_4
@@ -3004,10 +3143,12 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB20_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB20_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB20_4
@@ -3050,8 +3191,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB21_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB21_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -3100,8 +3243,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB21_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB21_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -3144,9 +3289,11 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB21_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB21_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -3194,9 +3341,11 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB22_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB22_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB22_4
@@ -3246,9 +3395,11 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB22_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB22_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB22_4
@@ -3292,10 +3443,12 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB22_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB22_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB22_4
@@ -3341,8 +3494,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB23_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB23_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -3393,8 +3548,10 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB23_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB23_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -3438,9 +3595,11 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB23_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB23_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -3487,9 +3646,11 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB24_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB24_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB24_4
@@ -3537,9 +3698,11 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB24_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB24_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB24_4
@@ -3580,9 +3743,12 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB24_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB24_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB24_4
@@ -3628,8 +3794,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB25_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB25_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -3681,8 +3849,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB25_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB25_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -3728,8 +3898,11 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB25_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB25_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -3780,9 +3953,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB26_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB26_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB26_4
@@ -3819,24 +3994,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB26_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB26_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB26_4
@@ -3882,9 +4060,12 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB26_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB26_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB26_4
@@ -3933,8 +4114,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB27_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB27_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -3978,8 +4161,8 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -3988,8 +4171,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB27_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB27_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -4036,8 +4221,11 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB27_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB27_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -4083,9 +4271,11 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB28_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB28_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB28_4
@@ -4131,9 +4321,11 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB28_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB28_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB28_4
@@ -4172,10 +4364,12 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB28_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB28_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB28_4
@@ -4218,8 +4412,10 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB29_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB29_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -4269,8 +4465,10 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB29_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB29_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -4314,9 +4512,11 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB29_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB29_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -4364,9 +4564,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB30_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB30_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB30_4
@@ -4416,9 +4618,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB30_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB30_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB30_4
@@ -4462,10 +4666,12 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB30_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB30_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB30_4
@@ -4511,8 +4717,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB31_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB31_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -4564,8 +4772,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB31_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB31_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -4610,9 +4820,11 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB31_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB31_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -4659,9 +4871,11 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB32_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB32_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB32_4
@@ -4709,9 +4923,11 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB32_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB32_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB32_4
@@ -4752,9 +4968,12 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB32_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB32_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB32_4
@@ -4800,8 +5019,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB33_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB33_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -4853,8 +5074,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB33_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB33_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -4900,8 +5123,11 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB33_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB33_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -4952,9 +5178,11 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB34_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB34_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB34_4
@@ -4991,24 +5219,27 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB34_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB34_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB34_4
@@ -5054,9 +5285,12 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB34_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB34_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB34_4
@@ -5105,8 +5339,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB35_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB35_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -5150,8 +5386,8 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -5160,8 +5396,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB35_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB35_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -5208,8 +5446,11 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB35_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB35_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -5255,9 +5496,11 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB36_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB36_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB36_4
@@ -5303,9 +5546,11 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB36_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB36_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB36_4
@@ -5344,10 +5589,12 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB36_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB36_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB36_4
@@ -5390,8 +5637,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB37_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB37_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -5441,8 +5690,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB37_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB37_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -5486,9 +5737,11 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB37_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB37_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -5536,9 +5789,11 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB38_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB38_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB38_4
@@ -5588,9 +5843,11 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB38_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB38_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB38_4
@@ -5634,10 +5891,12 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB38_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB38_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB38_4
@@ -5683,8 +5942,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB39_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB39_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -5736,8 +5997,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB39_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB39_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -5782,9 +6045,11 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB39_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB39_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -5831,9 +6096,11 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB40_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB40_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB40_4
@@ -5881,9 +6148,11 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB40_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB40_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB40_4
@@ -5924,9 +6193,12 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB40_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB40_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB40_4
@@ -5972,8 +6244,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB41_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB41_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -6025,8 +6299,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB41_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB41_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -6072,8 +6348,11 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB41_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB41_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -6124,9 +6403,11 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB42_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB42_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB42_4
@@ -6163,24 +6444,27 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB42_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB42_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB42_4
@@ -6226,9 +6510,12 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB42_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB42_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB42_4
@@ -6277,8 +6564,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB43_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB43_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -6322,8 +6611,8 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -6332,8 +6621,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB43_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB43_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -6380,8 +6671,11 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB43_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB43_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -6427,9 +6721,11 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB44_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB44_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB44_4
@@ -6475,9 +6771,11 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB44_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB44_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB44_4
@@ -6516,10 +6814,12 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB44_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB44_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB44_4
@@ -6562,8 +6862,10 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB45_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB45_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -6613,8 +6915,10 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB45_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB45_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -6658,9 +6962,11 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB45_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB45_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -6708,9 +7014,11 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB46_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB46_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB46_4
@@ -6760,9 +7068,11 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB46_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB46_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB46_4
@@ -6806,10 +7116,12 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB46_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB46_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB46_4
@@ -6855,8 +7167,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB47_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB47_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -6908,8 +7222,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB47_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB47_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -6954,9 +7270,11 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB47_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB47_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -7003,9 +7321,11 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB48_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB48_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB48_4
@@ -7053,9 +7373,11 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB48_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB48_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB48_4
@@ -7096,9 +7418,12 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB48_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB48_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB48_4
@@ -7144,8 +7469,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB49_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB49_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -7197,8 +7524,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB49_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB49_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -7244,8 +7573,11 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB49_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB49_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -7296,9 +7628,11 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB50_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB50_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB50_4
@@ -7335,24 +7669,27 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB50_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB50_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB50_4
@@ -7398,9 +7735,12 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB50_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB50_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB50_4
@@ -7449,8 +7789,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB51_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB51_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -7494,8 +7836,8 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -7504,8 +7846,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB51_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB51_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -7552,8 +7896,11 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB51_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB51_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -7599,9 +7946,11 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB52_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB52_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB52_4
@@ -7647,9 +7996,11 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB52_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB52_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB52_4
@@ -7688,10 +8039,12 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB52_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB52_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB52_4
@@ -7734,8 +8087,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB53_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB53_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -7785,8 +8140,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB53_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB53_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -7830,9 +8187,11 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB53_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB53_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -7880,9 +8239,11 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB54_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB54_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB54_4
@@ -7932,9 +8293,11 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB54_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB54_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB54_4
@@ -7978,10 +8341,12 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB54_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB54_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB54_4
@@ -8027,8 +8392,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB55_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB55_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -8080,8 +8447,10 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB55_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB55_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -8126,9 +8495,11 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB55_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB55_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -8175,9 +8546,11 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB56_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB56_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB56_4
@@ -8224,9 +8597,11 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB56_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB56_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB56_4
@@ -8266,9 +8641,12 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB56_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB56_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB56_4
@@ -8313,8 +8691,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB57_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB57_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -8364,8 +8744,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB57_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB57_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -8409,8 +8791,11 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB57_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB57_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -8460,9 +8845,11 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB58_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB58_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB58_4
@@ -8498,24 +8885,27 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB58_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB58_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB58_4
@@ -8560,9 +8950,12 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB58_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB58_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB58_4
@@ -8610,8 +9003,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB59_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB59_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -8653,8 +9048,8 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -8663,8 +9058,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB59_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB59_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -8709,8 +9106,11 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB59_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB59_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -8755,9 +9155,11 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB60_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB60_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB60_4
@@ -8802,9 +9204,11 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB60_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB60_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB60_4
@@ -8842,10 +9246,12 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB60_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB60_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB60_4
@@ -8887,8 +9293,10 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB61_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB61_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -8936,8 +9344,10 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB61_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB61_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -8979,9 +9389,11 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB61_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB61_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -9028,9 +9440,11 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB62_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB62_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB62_4
@@ -9079,9 +9493,11 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB62_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB62_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB62_4
@@ -9124,10 +9540,12 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB62_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB62_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB62_4
@@ -9172,8 +9590,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB63_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB63_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -9223,8 +9643,10 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB63_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB63_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -9267,9 +9689,11 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB63_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB63_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -9315,9 +9739,11 @@ define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB64_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB64_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB64_4
@@ -9360,9 +9786,11 @@ define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB64_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB64_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB64_4
@@ -9398,9 +9826,12 @@ define amdgpu_kernel void @atomic_xchg_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB64_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB64_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB64_4
@@ -9441,9 +9872,11 @@ define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB65_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB65_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB65_4
@@ -9486,9 +9919,11 @@ define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB65_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB65_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB65_4
@@ -9524,9 +9959,12 @@ define amdgpu_kernel void @atomic_xchg_f64_offset(ptr %out, double %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB65_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB65_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB65_4
@@ -9567,9 +10005,11 @@ define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB66_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB66_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB66_4
@@ -9612,9 +10052,11 @@ define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB66_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB66_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB66_4
@@ -9650,9 +10092,12 @@ define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr %out, ptr %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB66_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB66_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB66_4
@@ -9694,8 +10139,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB67_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB67_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -9744,8 +10191,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB67_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB67_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -9788,8 +10237,11 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB67_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB67_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -9838,9 +10290,11 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB68_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB68_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB68_4
@@ -9872,24 +10326,27 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB68_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB68_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB68_4
@@ -9930,9 +10387,12 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB68_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB68_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB68_4
@@ -9977,8 +10437,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB69_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB69_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -10019,8 +10481,8 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -10029,8 +10491,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB69_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB69_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -10074,8 +10538,11 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB69_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB69_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -10119,9 +10586,11 @@ define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB70_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB70_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB70_4
@@ -10162,9 +10631,11 @@ define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB70_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB70_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB70_4
@@ -10198,10 +10669,12 @@ define amdgpu_kernel void @atomic_xchg_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB70_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB70_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB70_4
@@ -10240,8 +10713,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB71_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB71_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -10288,8 +10763,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB71_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB71_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -10330,9 +10807,11 @@ define amdgpu_kernel void @atomic_xchg_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB71_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB71_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -10378,9 +10857,11 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB72_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB72_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB72_4
@@ -10425,9 +10906,11 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB72_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB72_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB72_4
@@ -10466,10 +10949,12 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB72_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB72_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB72_4
@@ -10511,8 +10996,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB73_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB73_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -10561,8 +11048,10 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB73_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB73_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -10604,9 +11093,11 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB73_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB73_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -10651,9 +11142,11 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB74_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB74_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB74_4
@@ -10700,9 +11193,11 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB74_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB74_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB74_4
@@ -10742,9 +11237,12 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB74_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB74_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB74_4
@@ -10789,8 +11287,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB75_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB75_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -10840,8 +11340,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB75_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB75_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -10885,8 +11387,11 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB75_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB75_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -10936,9 +11441,11 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB76_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB76_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB76_4
@@ -10974,24 +11481,27 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB76_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB76_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB76_4
@@ -11036,9 +11546,12 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB76_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB76_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB76_4
@@ -11086,8 +11599,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB77_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB77_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -11129,8 +11644,8 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -11139,8 +11654,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB77_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB77_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -11185,8 +11702,11 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB77_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB77_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -11231,9 +11751,11 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB78_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB78_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB78_4
@@ -11278,9 +11800,11 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB78_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB78_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB78_4
@@ -11318,10 +11842,12 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB78_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB78_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB78_4
@@ -11363,8 +11889,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB79_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB79_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -11412,8 +11940,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB79_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB79_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -11455,9 +11985,11 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB79_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB79_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -11504,9 +12036,11 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB80_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB80_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB80_4
@@ -11555,9 +12089,11 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB80_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB80_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB80_4
@@ -11600,10 +12136,12 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB80_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB80_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB80_4
@@ -11648,8 +12186,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB81_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB81_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -11699,8 +12239,10 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB81_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB81_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -11743,9 +12285,11 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB81_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB81_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -12204,9 +12748,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
; GCN1-NEXT: s_mov_b64 s[6:7], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB90_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB90_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT: s_cbranch_vccz .LBB90_4
@@ -12258,9 +12804,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
; GCN2-NEXT: s_mov_b64 s[6:7], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB90_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB90_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT: s_cbranch_vccz .LBB90_4
@@ -12306,9 +12854,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_offset(ptr %out, i64 %in, i64 %old
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
; GFX12-NEXT: s_mov_b32 s6, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB90_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB90_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT: s_cbranch_vccz .LBB90_4
@@ -12355,9 +12906,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %ol
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
; GCN1-NEXT: s_mov_b64 s[6:7], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB91_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB91_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT: s_cbranch_vccz .LBB91_4
@@ -12409,9 +12962,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %ol
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
; GCN2-NEXT: s_mov_b64 s[6:7], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB91_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB91_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT: s_cbranch_vccz .LBB91_4
@@ -12457,9 +13012,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_soffset(ptr %out, i64 %in, i64 %ol
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
; GFX12-NEXT: s_mov_b32 s6, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB91_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB91_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT: s_cbranch_vccz .LBB91_4
@@ -12505,8 +13063,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i6
; GCN1-NEXT: s_addc_u32 s1, s9, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB92_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB92_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v5, s1
; GCN1-NEXT: v_mov_b32_e32 v0, s12
@@ -12559,8 +13119,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i6
; GCN2-NEXT: s_addc_u32 s1, s9, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB92_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB92_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v5, s1
; GCN2-NEXT: v_mov_b32_e32 v0, s12
@@ -12606,8 +13168,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_offset(ptr %out, ptr %out2, i6
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s8, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX12-NEXT: s_cbranch_vccz .LBB92_2
+; GFX12-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX12-NEXT: s_cselect_b32 s8, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s8, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB92_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
@@ -12659,9 +13224,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_mov_b64 s[2:3], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB93_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB93_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT: s_cbranch_vccz .LBB93_4
@@ -12705,8 +13272,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[12:13], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -12715,9 +13282,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_mov_b64 s[2:3], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB93_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB93_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT: s_cbranch_vccz .LBB93_4
@@ -12764,9 +13333,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64_offset(ptr %out, i64 %in, i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB93_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB93_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB93_4
@@ -12817,8 +13389,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %o
; GCN1-NEXT: s_addc_u32 s3, s3, 0
; GCN1-NEXT: s_cmp_eq_u32 s3, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN1-NEXT: s_cbranch_vccz .LBB94_2
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB94_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v5, s3
; GCN1-NEXT: v_mov_b32_e32 v0, s12
@@ -12864,9 +13438,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %o
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GCN2-NEXT: s_load_dword s4, s[4:5], 0x10c
-; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[2:3], s[14:15], 3
; GCN2-NEXT: s_add_u32 s2, s8, s2
@@ -12875,8 +13449,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %o
; GCN2-NEXT: s_addc_u32 s3, s3, 0
; GCN2-NEXT: s_cmp_eq_u32 s3, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN2-NEXT: s_cbranch_vccz .LBB94_2
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB94_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v5, s3
; GCN2-NEXT: v_mov_b32_e32 v0, s12
@@ -12927,8 +13503,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64_offset(ptr %out, ptr %o
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s3, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_vccz .LBB94_2
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB94_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
@@ -12977,9 +13556,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
; GCN1-NEXT: s_mov_b64 s[6:7], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB95_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB95_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN1-NEXT: s_cbranch_vccz .LBB95_4
@@ -13029,9 +13610,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
; GCN2-NEXT: s_mov_b64 s[6:7], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB95_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB95_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GCN2-NEXT: s_cbranch_vccz .LBB95_4
@@ -13075,10 +13658,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i64(ptr %out, i64 %in, i64 %old) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
; GFX12-NEXT: s_mov_b32 s6, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB95_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB95_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
; GFX12-NEXT: s_cbranch_vccz .LBB95_4
@@ -13121,8 +13706,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s9, s0
; GCN1-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; GCN1-NEXT: s_cbranch_vccz .LBB96_2
+; GCN1-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GCN1-NEXT: s_cselect_b32 s0, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s0, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB96_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v4, s8
; GCN1-NEXT: v_mov_b32_e32 v0, s12
@@ -13173,8 +13760,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s9, s0
; GCN2-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; GCN2-NEXT: s_cbranch_vccz .LBB96_2
+; GCN2-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GCN2-NEXT: s_cselect_b32 s0, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s0, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB96_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v4, s8
; GCN2-NEXT: v_mov_b32_e32 v0, s12
@@ -13218,9 +13807,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret(ptr %out, ptr %out2, i64 %in,
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s8, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s8
-; GFX12-NEXT: s_cbranch_vccz .LBB96_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX12-NEXT: s_cselect_b32 s8, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s8, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB96_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
@@ -13269,9 +13860,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %ind
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_mov_b64 s[2:3], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB97_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB97_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN1-NEXT: s_cbranch_vccz .LBB97_4
@@ -13323,9 +13916,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %ind
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_mov_b64 s[2:3], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB97_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB97_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN2-NEXT: s_cbranch_vccz .LBB97_4
@@ -13370,10 +13965,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_addr64(ptr %out, i64 %in, i64 %ind
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB97_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB97_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB97_4
@@ -13421,8 +14018,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i6
; GCN1-NEXT: s_addc_u32 s3, s9, s3
; GCN1-NEXT: s_cmp_eq_u32 s3, s6
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN1-NEXT: s_cbranch_vccz .LBB98_2
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB98_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v5, s3
; GCN1-NEXT: v_mov_b32_e32 v0, s12
@@ -13477,8 +14076,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i6
; GCN2-NEXT: s_addc_u32 s3, s9, s3
; GCN2-NEXT: s_cmp_eq_u32 s3, s6
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN2-NEXT: s_cbranch_vccz .LBB98_2
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB98_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v5, s3
; GCN2-NEXT: v_mov_b32_e32 v0, s12
@@ -13527,9 +14128,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i64_ret_addr64(ptr %out, ptr %out2, i6
; GFX12-NEXT: s_add_nc_u64 s[2:3], s[8:9], s[2:3]
; GFX12-NEXT: s_cmp_eq_u32 s3, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_vccz .LBB98_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB98_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s12 :: v_dual_mov_b32 v1, s13
; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
@@ -13990,9 +14593,11 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB107_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB107_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB107_4
@@ -14042,9 +14647,11 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB107_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB107_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB107_4
@@ -14087,9 +14694,12 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB107_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB107_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB107_4
@@ -14138,8 +14748,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB108_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB108_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -14192,8 +14804,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB108_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB108_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -14240,8 +14854,11 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB108_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB108_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -14295,9 +14912,11 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB109_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB109_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB109_4
@@ -14336,24 +14955,27 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB109_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB109_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB109_4
@@ -14401,9 +15023,12 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB109_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB109_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB109_4
@@ -14455,8 +15080,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB110_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB110_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -14501,8 +15128,8 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -14511,8 +15138,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB110_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB110_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -14560,8 +15189,11 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB110_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB110_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -14610,9 +15242,11 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB111_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB111_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB111_4
@@ -14660,9 +15294,11 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB111_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB111_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB111_4
@@ -14703,10 +15339,12 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB111_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB111_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB111_4
@@ -14752,8 +15390,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s6
; GCN1-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN1-NEXT: s_cbranch_vccz .LBB112_2
+; GCN1-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN1-NEXT: s_cselect_b32 s6, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s6, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB112_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s4
@@ -14804,8 +15444,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s6
; GCN2-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GCN2-NEXT: s_cbranch_vccz .LBB112_2
+; GCN2-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GCN2-NEXT: s_cselect_b32 s6, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s6, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB112_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s4
@@ -14850,9 +15492,11 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB112_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB112_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -14903,9 +15547,11 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB113_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB113_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB113_4
@@ -14957,9 +15603,11 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB113_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB113_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB113_4
@@ -15005,10 +15653,12 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB113_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB113_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB113_4
@@ -15057,8 +15707,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB114_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB114_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -15111,8 +15763,10 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB114_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB114_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -15158,9 +15812,11 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB114_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB114_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -15210,9 +15866,11 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB115_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB115_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB115_4
@@ -15265,9 +15923,11 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB115_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB115_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB115_4
@@ -15313,9 +15973,12 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB115_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB115_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB115_4
@@ -15367,8 +16030,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s3, s9, 0
; GCN1-NEXT: s_cmp_eq_u32 s3, s6
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN1-NEXT: s_cbranch_vccz .LBB116_2
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB116_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s2
; GCN1-NEXT: v_mov_b32_e32 v3, s1
@@ -15425,8 +16090,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s3, s9, 0
; GCN2-NEXT: s_cmp_eq_u32 s3, s6
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
-; GCN2-NEXT: s_cbranch_vccz .LBB116_2
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB116_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s2
; GCN2-NEXT: v_mov_b32_e32 v3, s1
@@ -15477,8 +16144,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB116_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB116_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -15535,9 +16205,11 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB117_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB117_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB117_4
@@ -15579,24 +16251,27 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB117_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB117_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB117_4
@@ -15647,9 +16322,12 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB117_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB117_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB117_4
@@ -15704,8 +16382,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB118_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB118_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -15754,8 +16434,8 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -15764,8 +16444,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB118_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB118_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -15817,8 +16499,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB118_2
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB118_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -15870,9 +16555,11 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB119_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB119_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB119_4
@@ -15923,9 +16610,11 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB119_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB119_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB119_4
@@ -15969,10 +16658,12 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB119_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB119_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB119_4
@@ -16021,8 +16712,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s9, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB120_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB120_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s8
; GCN1-NEXT: v_mov_b32_e32 v3, s1
@@ -16077,8 +16770,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s9, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB120_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB120_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s8
; GCN2-NEXT: v_mov_b32_e32 v3, s1
@@ -16127,9 +16822,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB120_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB120_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
@@ -16183,9 +16880,11 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB121_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB121_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccz .LBB121_4
@@ -16240,9 +16939,11 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB121_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB121_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccz .LBB121_4
@@ -16291,10 +16992,12 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[4:5]
; GFX12-NEXT: s_cmp_eq_u32 s1, s7
; GFX12-NEXT: s_cselect_b32 s4, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX12-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 1
; GFX12-NEXT: s_mov_b32 s4, -1
-; GFX12-NEXT: s_cbranch_vccnz .LBB121_3
+; GFX12-NEXT: s_cbranch_scc1 .LBB121_3
; GFX12-NEXT: ; %bb.1: ; %Flow
; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
; GFX12-NEXT: s_cbranch_vccz .LBB121_4
@@ -16346,8 +17049,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB122_2
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB122_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v2, s12
@@ -16404,8 +17109,10 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB122_2
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB122_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v2, s12
@@ -16455,9 +17162,11 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[6:7]
; GFX12-NEXT: s_cmp_eq_u32 s1, s9
; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccz .LBB122_2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX12-NEXT: s_cselect_b32 s6, 1, 0
+; GFX12-NEXT: s_cmp_lg_u32 s6, 1
+; GFX12-NEXT: s_cbranch_scc0 .LBB122_2
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
index 9e27f6badfdac..9e5102fd2d047 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefix=GCN1 %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefix=GCN2 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN3 %s
@@ -468,9 +468,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB4_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB4_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB4_4
@@ -507,9 +509,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB4_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB4_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB4_4
@@ -543,9 +547,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB4_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB4_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB4_4
@@ -585,9 +591,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB5_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB5_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB5_4
@@ -626,9 +634,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB5_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB5_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB5_4
@@ -664,9 +674,11 @@ define amdgpu_gfx void @flat_atomic_xchg_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB5_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB5_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB5_4
@@ -705,8 +717,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB6_2
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB6_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -744,8 +758,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB6_2
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB6_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -780,8 +796,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB6_2
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB6_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -822,8 +840,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB7_2
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB7_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -863,8 +883,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB7_2
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB7_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -901,8 +923,10 @@ define amdgpu_gfx i64 @flat_atomic_xchg_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB7_2
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB7_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -1628,9 +1652,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB14_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB14_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB14_4
@@ -1667,9 +1693,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB14_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB14_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB14_4
@@ -1703,9 +1731,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_scalar(ptr inreg %ptr, double
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB14_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB14_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB14_4
@@ -1745,9 +1775,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB15_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB15_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB15_4
@@ -1786,9 +1818,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB15_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB15_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB15_4
@@ -1824,9 +1858,11 @@ define amdgpu_gfx void @flat_atomic_xchg_f64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB15_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB15_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB15_4
@@ -1865,8 +1901,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB16_2
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB16_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -1904,8 +1942,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB16_2
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB16_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -1940,8 +1980,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_scalar(ptr inreg %ptr, double
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB16_2
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB16_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -1982,8 +2024,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB17_2
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB17_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -2023,8 +2067,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB17_2
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB17_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -2061,8 +2107,10 @@ define amdgpu_gfx double @flat_atomic_xchg_f64_ret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB17_2
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB17_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -2846,9 +2894,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB24_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB24_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccz .LBB24_4
@@ -2890,9 +2940,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB24_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB24_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccz .LBB24_4
@@ -2931,9 +2983,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB24_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB24_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccz .LBB24_4
@@ -2978,9 +3032,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB25_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB25_3
; GCN1-NEXT: ; %bb.1: ; %Flow
; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccz .LBB25_4
@@ -3024,9 +3080,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB25_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB25_3
; GCN2-NEXT: ; %bb.1: ; %Flow
; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccz .LBB25_4
@@ -3067,9 +3125,11 @@ define amdgpu_gfx void @flat_atomic_add_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB25_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB25_3
; GCN3-NEXT: ; %bb.1: ; %Flow
; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccz .LBB25_4
@@ -3113,8 +3173,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB26_2
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB26_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -3155,8 +3217,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB26_2
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB26_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -3194,8 +3258,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB26_2
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB26_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -3239,8 +3305,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB27_2
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB27_2
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v2, s6
@@ -3283,8 +3351,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB27_2
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB27_2
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v2, s6
@@ -3324,8 +3394,10 @@ define amdgpu_gfx i64 @flat_atomic_add_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB27_2
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB27_2
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v0, s34
; GCN3-NEXT: v_mov_b32_e32 v2, s6
@@ -4349,9 +4421,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB34_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB34_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB34_6
@@ -4412,9 +4486,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB34_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB34_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB34_6
@@ -4472,9 +4548,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB34_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB34_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB34_6
@@ -4533,9 +4611,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB35_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB35_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB35_6
@@ -4598,9 +4678,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB35_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB35_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB35_6
@@ -4660,9 +4742,11 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB35_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB35_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB35_6
@@ -4720,8 +4804,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB36_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB36_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -4781,8 +4867,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB36_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB36_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -4839,8 +4927,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB36_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB36_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -4898,8 +4988,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB37_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB37_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -4961,8 +5053,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB37_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB37_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -5021,8 +5115,10 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB37_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB37_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -6060,9 +6156,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB44_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB44_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB44_6
@@ -6121,9 +6219,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB44_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB44_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB44_6
@@ -6179,9 +6279,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB44_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB44_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB44_6
@@ -6238,9 +6340,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB45_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB45_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB45_6
@@ -6301,9 +6405,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB45_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB45_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB45_6
@@ -6361,9 +6467,11 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB45_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB45_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB45_6
@@ -6419,8 +6527,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB46_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB46_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -6478,8 +6588,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB46_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB46_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -6534,8 +6646,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB46_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB46_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -6591,8 +6705,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB47_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB47_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -6652,8 +6768,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB47_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB47_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -6710,8 +6828,10 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB47_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB47_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -7795,9 +7915,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB54_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB54_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB54_6
@@ -7860,9 +7982,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB54_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB54_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB54_6
@@ -7922,9 +8046,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB54_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB54_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB54_6
@@ -7985,9 +8111,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB55_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB55_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB55_6
@@ -8052,9 +8180,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB55_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB55_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB55_6
@@ -8116,9 +8246,11 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB55_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB55_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB55_6
@@ -8178,8 +8310,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB56_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB56_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -8241,8 +8375,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB56_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB56_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -8301,8 +8437,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB56_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB56_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -8362,8 +8500,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB57_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB57_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -8427,8 +8567,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB57_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB57_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -8489,8 +8631,10 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB57_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB57_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -9662,9 +9806,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB64_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB64_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB64_6
@@ -9723,9 +9869,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB64_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB64_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB64_6
@@ -9781,9 +9929,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB64_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB64_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB64_6
@@ -9840,9 +9990,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB65_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB65_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB65_6
@@ -9903,9 +10055,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB65_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB65_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB65_6
@@ -9963,9 +10117,11 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB65_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB65_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB65_6
@@ -10021,8 +10177,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB66_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB66_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -10080,8 +10238,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB66_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB66_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -10136,8 +10296,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB66_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB66_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -10193,8 +10355,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB67_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB67_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -10254,8 +10418,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB67_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB67_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -10312,8 +10478,10 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB67_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB67_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -11349,9 +11517,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB74_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB74_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB74_6
@@ -11410,9 +11580,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB74_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB74_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB74_6
@@ -11468,9 +11640,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB74_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB74_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB74_6
@@ -11527,9 +11701,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB75_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB75_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB75_6
@@ -11590,9 +11766,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB75_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB75_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB75_6
@@ -11650,9 +11828,11 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB75_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB75_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB75_6
@@ -11708,8 +11888,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB76_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB76_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -11767,8 +11949,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB76_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB76_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -11823,8 +12007,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB76_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB76_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -11880,8 +12066,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB77_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB77_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -11941,8 +12129,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB77_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB77_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -11999,8 +12189,10 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB77_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB77_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -13036,9 +13228,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB84_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB84_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB84_6
@@ -13102,9 +13296,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB84_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB84_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB84_6
@@ -13165,9 +13361,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB84_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB84_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB84_6
@@ -13229,9 +13427,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB85_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB85_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB85_6
@@ -13297,9 +13497,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB85_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB85_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB85_6
@@ -13362,9 +13564,11 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB85_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB85_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB85_6
@@ -13425,8 +13629,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB86_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB86_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -13489,8 +13695,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB86_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB86_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -13550,8 +13758,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB86_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB86_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -13612,8 +13822,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB87_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB87_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -13678,8 +13890,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB87_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB87_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -13741,8 +13955,10 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB87_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB87_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -13813,9 +14029,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB88_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB88_3
; GCN1-NEXT: ; %bb.1: ; %Flow6
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccnz .LBB88_6
@@ -13869,24 +14087,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB88_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB88_3
; GCN2-NEXT: ; %bb.1: ; %Flow6
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccnz .LBB88_6
@@ -13954,9 +14175,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s7
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN3-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN3-NEXT: s_cselect_b32 s4, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s4, 1
; GCN3-NEXT: s_mov_b64 s[4:5], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB88_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB88_3
; GCN3-NEXT: ; %bb.1: ; %Flow6
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN3-NEXT: s_cbranch_vccnz .LBB88_6
@@ -14028,8 +14251,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB89_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB89_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -14089,8 +14314,8 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -14099,8 +14324,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB89_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB89_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -14169,8 +14396,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB89_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB89_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -14244,9 +14473,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB90_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB90_3
; GCN1-NEXT: ; %bb.1: ; %Flow6
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccnz .LBB90_6
@@ -14313,9 +14544,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_cmp_eq_u32 s1, s8
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB90_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB90_3
; GCN2-NEXT: ; %bb.1: ; %Flow6
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccnz .LBB90_6
@@ -14381,9 +14614,11 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: s_cmp_eq_u32 s1, s7
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN3-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN3-NEXT: s_cselect_b32 s4, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s4, 1
; GCN3-NEXT: s_mov_b64 s[4:5], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB90_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB90_3
; GCN3-NEXT: ; %bb.1: ; %Flow6
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN3-NEXT: s_cbranch_vccnz .LBB90_6
@@ -14452,8 +14687,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB91_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB91_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -14521,8 +14758,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB91_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB91_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -14589,8 +14828,10 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB91_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB91_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -15635,9 +15876,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB98_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB98_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB98_6
@@ -15701,9 +15944,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB98_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB98_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB98_6
@@ -15764,9 +16009,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB98_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB98_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB98_6
@@ -15828,9 +16075,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB99_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB99_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB99_6
@@ -15896,9 +16145,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB99_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB99_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB99_6
@@ -15961,9 +16212,11 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB99_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB99_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB99_6
@@ -16024,8 +16277,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB100_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB100_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -16088,8 +16343,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB100_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB100_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -16149,8 +16406,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB100_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB100_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -16211,8 +16470,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB101_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB101_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -16277,8 +16538,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB101_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB101_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -16340,8 +16603,10 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB101_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB101_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -16412,9 +16677,11 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB102_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB102_3
; GCN1-NEXT: ; %bb.1: ; %Flow6
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccnz .LBB102_6
@@ -16468,24 +16735,27 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB102_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB102_3
; GCN2-NEXT: ; %bb.1: ; %Flow6
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccnz .LBB102_6
@@ -16553,9 +16823,11 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s7
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN3-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN3-NEXT: s_cselect_b32 s4, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s4, 1
; GCN3-NEXT: s_mov_b64 s[4:5], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB102_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB102_3
; GCN3-NEXT: ; %bb.1: ; %Flow6
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN3-NEXT: s_cbranch_vccnz .LBB102_6
@@ -16627,8 +16899,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB103_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB103_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -16688,8 +16962,8 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -16698,8 +16972,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB103_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB103_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -16768,8 +17044,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB103_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB103_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -16842,8 +17120,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB104_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB104_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -16911,8 +17191,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB104_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB104_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -16979,8 +17261,10 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB104_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB104_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -18025,9 +18309,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB111_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB111_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB111_6
@@ -18091,9 +18377,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB111_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB111_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB111_6
@@ -18154,9 +18442,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB111_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB111_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB111_6
@@ -18218,9 +18508,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB112_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB112_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB112_6
@@ -18286,9 +18578,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB112_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB112_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB112_6
@@ -18351,9 +18645,11 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB112_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB112_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB112_6
@@ -18414,8 +18710,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB113_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB113_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -18478,8 +18776,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB113_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB113_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -18539,8 +18839,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB113_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB113_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -18601,8 +18903,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB114_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB114_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -18667,8 +18971,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB114_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB114_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -18730,8 +19036,10 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB114_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB114_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -19772,9 +20080,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB121_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB121_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB121_6
@@ -19838,9 +20148,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB121_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB121_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB121_6
@@ -19901,9 +20213,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB121_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB121_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB121_6
@@ -19965,9 +20279,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB122_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB122_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB122_6
@@ -20033,9 +20349,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB122_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB122_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB122_6
@@ -20098,9 +20416,11 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB122_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB122_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB122_6
@@ -20161,8 +20481,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB123_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB123_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -20225,8 +20547,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB123_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB123_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -20286,8 +20610,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB123_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB123_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -20348,8 +20674,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB124_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB124_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -20414,8 +20742,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB124_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB124_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -20477,8 +20807,10 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB124_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB124_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -20549,9 +20881,11 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s8
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB125_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB125_3
; GCN1-NEXT: ; %bb.1: ; %Flow6
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccnz .LBB125_6
@@ -20605,24 +20939,27 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2: ; %bb.0: ; %entry
; GCN2-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; GCN2-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
-; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0xfc
+; GCN2-NEXT: s_mov_b32 s90, -1
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_addc_u32 s89, s89, 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; GCN2-NEXT: s_load_dword s4, s[4:5], 0xfc
+; GCN2-NEXT: s_add_u32 s0, s0, s6
+; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: s_cmp_eq_u32 s1, s8
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB125_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB125_3
; GCN2-NEXT: ; %bb.1: ; %Flow6
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccnz .LBB125_6
@@ -20690,9 +21027,11 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s7
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN3-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN3-NEXT: s_cselect_b32 s4, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s4, 1
; GCN3-NEXT: s_mov_b64 s[4:5], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB125_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB125_3
; GCN3-NEXT: ; %bb.1: ; %Flow6
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN3-NEXT: s_cbranch_vccnz .LBB125_6
@@ -20764,8 +21103,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB126_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB126_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -20825,8 +21166,8 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_mov_b32 s91, 0xe80000
; GCN2-NEXT: s_add_u32 s88, s88, s11
; GCN2-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_addc_u32 s89, s89, 0
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x104
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GCN2-NEXT: s_add_u32 s0, s8, s0
@@ -20835,8 +21176,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB126_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB126_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -20905,8 +21248,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, 0
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB126_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB126_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -20976,9 +21321,11 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s1, s4
; GCN1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN1-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN1-NEXT: s_cselect_b32 s4, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s4, 1
; GCN1-NEXT: s_mov_b64 s[4:5], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB127_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB127_3
; GCN1-NEXT: ; %bb.1: ; %Flow5
; GCN1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN1-NEXT: s_cbranch_vccnz .LBB127_6
@@ -21041,9 +21388,11 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s1, s4
; GCN2-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN2-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN2-NEXT: s_cselect_b32 s4, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s4, 1
; GCN2-NEXT: s_mov_b64 s[4:5], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB127_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB127_3
; GCN2-NEXT: ; %bb.1: ; %Flow5
; GCN2-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN2-NEXT: s_cbranch_vccnz .LBB127_6
@@ -21105,9 +21454,11 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_cmp_eq_u32 s1, s7
; GCN3-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; GCN3-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GCN3-NEXT: s_cselect_b32 s4, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s4, 1
; GCN3-NEXT: s_mov_b64 s[4:5], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB127_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB127_3
; GCN3-NEXT: ; %bb.1: ; %Flow5
; GCN3-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN3-NEXT: s_cbranch_vccnz .LBB127_6
@@ -21175,8 +21526,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s9, s1
; GCN1-NEXT: s_cmp_eq_u32 s1, s2
; GCN1-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN1-NEXT: s_cbranch_vccz .LBB128_4
+; GCN1-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN1-NEXT: s_cselect_b32 s2, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s2, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB128_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: v_mov_b32_e32 v2, s0
@@ -21244,8 +21597,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s9, s1
; GCN2-NEXT: s_cmp_eq_u32 s1, s2
; GCN2-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN2-NEXT: s_cbranch_vccz .LBB128_4
+; GCN2-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN2-NEXT: s_cselect_b32 s2, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s2, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB128_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: v_mov_b32_e32 v2, s0
@@ -21312,8 +21667,10 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: s_cmp_eq_u32 s1, s3
; GCN3-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[2:3]
-; GCN3-NEXT: s_cbranch_vccz .LBB128_4
+; GCN3-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GCN3-NEXT: s_cselect_b32 s2, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s2, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB128_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: v_mov_b32_e32 v2, s0
@@ -22418,9 +22775,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB135_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB135_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB135_6
@@ -22485,9 +22844,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB135_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB135_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB135_6
@@ -22549,9 +22910,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB135_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB135_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB135_6
@@ -22614,9 +22977,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
; GCN1-NEXT: s_mov_b64 s[36:37], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB136_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB136_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN1-NEXT: s_cbranch_vccnz .LBB136_6
@@ -22683,9 +23048,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
; GCN2-NEXT: s_mov_b64 s[36:37], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB136_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB136_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN2-NEXT: s_cbranch_vccnz .LBB136_6
@@ -22749,9 +23116,11 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_mov_b64 s[36:37], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB136_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB136_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[36:37]
; GCN3-NEXT: s_cbranch_vccnz .LBB136_6
@@ -22813,8 +23182,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB137_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB137_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -22878,8 +23249,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB137_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB137_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -22940,8 +23313,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB137_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB137_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -23003,8 +23378,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s35, s36
; GCN1-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT: s_cbranch_vccz .LBB138_4
+; GCN1-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN1-NEXT: s_cselect_b32 s36, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s36, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB138_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s36, s34, 4
; GCN1-NEXT: s_addc_u32 s37, s35, 0
@@ -23070,8 +23447,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s35, s36
; GCN2-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT: s_cbranch_vccz .LBB138_4
+; GCN2-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN2-NEXT: s_cselect_b32 s36, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s36, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB138_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s36, s34, 4
; GCN2-NEXT: s_addc_u32 s37, s35, 0
@@ -23134,8 +23513,10 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: s_addc_u32 s35, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s35, s37
; GCN3-NEXT: s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT: s_cbranch_vccz .LBB138_4
+; GCN3-NEXT: s_and_b64 s[36:37], s[36:37], exec
+; GCN3-NEXT: s_cselect_b32 s36, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s36, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB138_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s34
; GCN3-NEXT: v_mov_b32_e32 v3, s35
@@ -24297,9 +24678,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB145_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB145_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB145_6
@@ -24371,9 +24754,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB145_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB145_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB145_6
@@ -24442,9 +24827,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB145_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB145_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB145_6
@@ -24514,9 +24901,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s39, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
; GCN1-NEXT: s_mov_b64 s[34:35], -1
-; GCN1-NEXT: s_cbranch_vccnz .LBB146_3
+; GCN1-NEXT: s_cbranch_scc1 .LBB146_3
; GCN1-NEXT: ; %bb.1: ; %Flow3
; GCN1-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN1-NEXT: s_cbranch_vccnz .LBB146_6
@@ -24590,9 +24979,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s39, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
; GCN2-NEXT: s_mov_b64 s[34:35], -1
-; GCN2-NEXT: s_cbranch_vccnz .LBB146_3
+; GCN2-NEXT: s_cbranch_scc1 .LBB146_3
; GCN2-NEXT: ; %bb.1: ; %Flow3
; GCN2-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN2-NEXT: s_cbranch_vccnz .LBB146_6
@@ -24663,9 +25054,11 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN3-NEXT: s_addc_u32 s39, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s39, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_mov_b64 s[34:35], -1
-; GCN3-NEXT: s_cbranch_vccnz .LBB146_3
+; GCN3-NEXT: s_cbranch_scc1 .LBB146_3
; GCN3-NEXT: ; %bb.1: ; %Flow3
; GCN3-NEXT: s_and_b64 vcc, exec, s[34:35]
; GCN3-NEXT: s_cbranch_vccnz .LBB146_6
@@ -24734,8 +25127,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s5, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB147_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB147_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s4, 4
; GCN1-NEXT: s_addc_u32 s35, s5, 0
@@ -24807,8 +25202,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s5, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB147_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB147_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s4, 4
; GCN2-NEXT: s_addc_u32 s35, s5, 0
@@ -24877,8 +25274,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN3-NEXT: s_mov_b64 s[34:35], src_private_base
; GCN3-NEXT: s_cmp_eq_u32 s5, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB147_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB147_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: v_mov_b32_e32 v3, s5
@@ -24948,8 +25347,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_cmp_eq_u32 s39, s34
; GCN1-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN1-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT: s_cbranch_vccz .LBB148_4
+; GCN1-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT: s_cselect_b32 s34, 1, 0
+; GCN1-NEXT: s_cmp_lg_u32 s34, 1
+; GCN1-NEXT: s_cbranch_scc0 .LBB148_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN1-NEXT: s_add_u32 s34, s38, 4
; GCN1-NEXT: s_addc_u32 s35, s39, 0
@@ -25023,8 +25424,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_cmp_eq_u32 s39, s34
; GCN2-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN2-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT: s_cbranch_vccz .LBB148_4
+; GCN2-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN2-NEXT: s_cselect_b32 s34, 1, 0
+; GCN2-NEXT: s_cmp_lg_u32 s34, 1
+; GCN2-NEXT: s_cbranch_scc0 .LBB148_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN2-NEXT: s_add_u32 s34, s38, 4
; GCN2-NEXT: s_addc_u32 s35, s39, 0
@@ -25095,8 +25498,10 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: s_addc_u32 s39, s5, 0
; GCN3-NEXT: s_cmp_eq_u32 s39, s35
; GCN3-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT: s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT: s_cbranch_vccz .LBB148_4
+; GCN3-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GCN3-NEXT: s_cselect_b32 s34, 1, 0
+; GCN3-NEXT: s_cmp_lg_u32 s34, 1
+; GCN3-NEXT: s_cbranch_scc0 .LBB148_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
; GCN3-NEXT: v_mov_b32_e32 v2, s38
; GCN3-NEXT: v_mov_b32_e32 v3, s39
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 9dd8d8f230217..e2d25e7a8c3ee 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -199,8 +199,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -212,15 +212,15 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-TRUE16-NEXT: s_endpgm
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -232,21 +232,21 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -256,14 +256,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -462,8 +462,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -475,15 +475,15 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX1250-SDAG-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-TRUE16-NEXT: s_endpgm
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -495,21 +495,21 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
; GFX1250-SDAG-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -519,14 +519,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_afn(ptr addrspace(1) %r,
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -553,47 +553,47 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; SI-SDAG-NEXT: s_mov_b32 s8, s6
; SI-SDAG-NEXT: s_mov_b32 s9, s7
; SI-SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; SI-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; SI-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; SI-SDAG-NEXT: v_readfirstlane_b32 s1, v1
-; SI-SDAG-NEXT: s_and_b32 s6, s1, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s7, s1, 8
-; SI-SDAG-NEXT: s_bfe_u32 s8, s1, 0xb0014
-; SI-SDAG-NEXT: v_or_b32_e32 v0, s6, v0
-; SI-SDAG-NEXT: s_and_b32 s6, s7, 0xffe
-; SI-SDAG-NEXT: s_sub_i32 s7, 0x3f1, s8
+; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v1
+; SI-SDAG-NEXT: s_and_b32 s0, s7, 0x1ff
+; SI-SDAG-NEXT: v_or_b32_e32 v0, s0, v0
; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-SDAG-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; SI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; SI-SDAG-NEXT: s_or_b32 s6, s6, s7
-; SI-SDAG-NEXT: s_or_b32 s7, s6, 0x1000
-; SI-SDAG-NEXT: s_lshr_b32 s10, s7, s9
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-SDAG-NEXT: s_lshr_b32 s1, s7, 8
+; SI-SDAG-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; SI-SDAG-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s8
+; SI-SDAG-NEXT: s_or_b32 s0, s1, s0
+; SI-SDAG-NEXT: v_med3_i32 v0, s9, 0, 13
+; SI-SDAG-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-SDAG-NEXT: v_readfirstlane_b32 s9, v0
+; SI-SDAG-NEXT: s_lshr_b32 s10, s1, s9
; SI-SDAG-NEXT: s_lshl_b32 s9, s10, s9
-; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s7
-; SI-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s1
+; SI-SDAG-NEXT: s_cselect_b32 s1, 1, 0
; SI-SDAG-NEXT: s_addk_i32 s8, 0xfc10
-; SI-SDAG-NEXT: s_or_b32 s7, s10, s7
+; SI-SDAG-NEXT: s_or_b32 s1, s10, s1
; SI-SDAG-NEXT: s_lshl_b32 s9, s8, 12
-; SI-SDAG-NEXT: s_or_b32 s9, s6, s9
+; SI-SDAG-NEXT: s_or_b32 s9, s0, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s8, 1
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, s9
-; SI-SDAG-NEXT: s_and_b32 s9, s7, 7
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, s9
+; SI-SDAG-NEXT: s_and_b32 s9, s1, 7
; SI-SDAG-NEXT: s_cmp_gt_i32 s9, 5
; SI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
; SI-SDAG-NEXT: s_cmp_eq_u32 s9, 3
; SI-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; SI-SDAG-NEXT: s_lshr_b32 s7, s7, 2
+; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 2
; SI-SDAG-NEXT: s_or_b32 s9, s9, s10
-; SI-SDAG-NEXT: s_add_i32 s7, s7, s9
+; SI-SDAG-NEXT: s_add_i32 s1, s1, s9
; SI-SDAG-NEXT: s_cmp_lt_i32 s8, 31
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s6, 0
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, 0x7c00
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s0, 0
+; SI-SDAG-NEXT: s_cselect_b32 s0, s6, 0x7c00
; SI-SDAG-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s7
-; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 16
+; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s1
+; SI-SDAG-NEXT: s_lshr_b32 s1, s7, 16
; SI-SDAG-NEXT: s_and_b32 s1, s1, 0x8000
; SI-SDAG-NEXT: s_or_b32 s6, s1, s0
; SI-SDAG-NEXT: s_mov_b32 s0, s4
@@ -668,47 +668,47 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; VI-SDAG-NEXT: s_mov_b32 s1, s5
; VI-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; VI-SDAG-NEXT: v_readfirstlane_b32 s4, v1
-; VI-SDAG-NEXT: s_and_b32 s5, s4, 0x1ff
-; VI-SDAG-NEXT: v_or_b32_e32 v0, s5, v0
-; VI-SDAG-NEXT: s_lshr_b32 s7, s4, 8
-; VI-SDAG-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; VI-SDAG-NEXT: v_readfirstlane_b32 s7, v1
+; VI-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
+; VI-SDAG-NEXT: v_or_b32_e32 v0, s4, v0
; VI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; VI-SDAG-NEXT: s_and_b32 s5, s7, 0xffe
-; VI-SDAG-NEXT: s_sub_i32 s7, 0x3f1, s8
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; VI-SDAG-NEXT: v_med3_i32 v1, s7, 0, 13
-; VI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; VI-SDAG-NEXT: s_or_b32 s5, s5, s7
-; VI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; VI-SDAG-NEXT: s_or_b32 s7, s5, 0x1000
-; VI-SDAG-NEXT: s_lshr_b32 s10, s7, s9
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; VI-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; VI-SDAG-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; VI-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; VI-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s8
+; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
+; VI-SDAG-NEXT: v_med3_i32 v0, s9, 0, 13
+; VI-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; VI-SDAG-NEXT: v_readfirstlane_b32 s9, v0
+; VI-SDAG-NEXT: s_lshr_b32 s10, s5, s9
; VI-SDAG-NEXT: s_lshl_b32 s9, s10, s9
-; VI-SDAG-NEXT: s_cmp_lg_u32 s9, s7
-; VI-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; VI-SDAG-NEXT: s_cmp_lg_u32 s9, s5
+; VI-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; VI-SDAG-NEXT: s_addk_i32 s8, 0xfc10
; VI-SDAG-NEXT: s_lshl_b32 s9, s8, 12
-; VI-SDAG-NEXT: s_or_b32 s7, s10, s7
-; VI-SDAG-NEXT: s_or_b32 s9, s5, s9
+; VI-SDAG-NEXT: s_or_b32 s5, s10, s5
+; VI-SDAG-NEXT: s_or_b32 s9, s4, s9
; VI-SDAG-NEXT: s_cmp_lt_i32 s8, 1
-; VI-SDAG-NEXT: s_cselect_b32 s7, s7, s9
-; VI-SDAG-NEXT: s_and_b32 s9, s7, 7
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, s9
+; VI-SDAG-NEXT: s_and_b32 s9, s5, 7
; VI-SDAG-NEXT: s_cmp_gt_i32 s9, 5
; VI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
; VI-SDAG-NEXT: s_cmp_eq_u32 s9, 3
; VI-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; VI-SDAG-NEXT: s_lshr_b32 s7, s7, 2
+; VI-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; VI-SDAG-NEXT: s_or_b32 s9, s9, s10
-; VI-SDAG-NEXT: s_add_i32 s7, s7, s9
+; VI-SDAG-NEXT: s_add_i32 s5, s5, s9
; VI-SDAG-NEXT: s_cmp_lt_i32 s8, 31
-; VI-SDAG-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; VI-SDAG-NEXT: s_cmp_lg_u32 s5, 0
-; VI-SDAG-NEXT: s_cselect_b32 s5, s6, 0x7c00
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; VI-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; VI-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; VI-SDAG-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; VI-SDAG-NEXT: s_cselect_b32 s5, s5, s7
-; VI-SDAG-NEXT: s_lshr_b32 s4, s4, 16
-; VI-SDAG-NEXT: s_and_b32 s4, s4, 0x8000
-; VI-SDAG-NEXT: s_or_b32 s4, s4, s5
+; VI-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; VI-SDAG-NEXT: s_lshr_b32 s5, s7, 16
+; VI-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
+; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
; VI-SDAG-NEXT: s_endpgm
@@ -777,47 +777,47 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX9-SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; GFX9-SDAG-NEXT: s_mov_b32 s0, s8
; GFX9-SDAG-NEXT: s_mov_b32 s1, s9
-; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX9-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX9-SDAG-NEXT: s_and_b32 s6, s5, 0x1ff
-; GFX9-SDAG-NEXT: v_or_b32_e32 v0, s6, v0
-; GFX9-SDAG-NEXT: s_lshr_b32 s7, s5, 8
-; GFX9-SDAG-NEXT: s_bfe_u32 s8, s5, 0xb0014
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s7, v1
+; GFX9-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, s4, v0
; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT: s_and_b32 s6, s7, 0xffe
-; GFX9-SDAG-NEXT: s_sub_i32 s7, 0x3f1, s8
-; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-SDAG-NEXT: v_med3_i32 v1, s7, 0, 13
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-SDAG-NEXT: s_or_b32 s6, s6, s7
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX9-SDAG-NEXT: s_or_b32 s7, s6, 0x1000
-; GFX9-SDAG-NEXT: s_lshr_b32 s10, s7, s9
+; GFX9-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; GFX9-SDAG-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX9-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s8
+; GFX9-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX9-SDAG-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX9-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s9, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s10, s5, s9
; GFX9-SDAG-NEXT: s_lshl_b32 s9, s10, s9
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s9, s7
-; GFX9-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s9, s5
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; GFX9-SDAG-NEXT: s_addk_i32 s8, 0xfc10
; GFX9-SDAG-NEXT: s_lshl_b32 s9, s8, 12
-; GFX9-SDAG-NEXT: s_or_b32 s7, s10, s7
-; GFX9-SDAG-NEXT: s_or_b32 s9, s6, s9
+; GFX9-SDAG-NEXT: s_or_b32 s5, s10, s5
+; GFX9-SDAG-NEXT: s_or_b32 s9, s4, s9
; GFX9-SDAG-NEXT: s_cmp_lt_i32 s8, 1
-; GFX9-SDAG-NEXT: s_cselect_b32 s7, s7, s9
-; GFX9-SDAG-NEXT: s_and_b32 s9, s7, 7
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, s9
+; GFX9-SDAG-NEXT: s_and_b32 s9, s5, 7
; GFX9-SDAG-NEXT: s_cmp_gt_i32 s9, 5
; GFX9-SDAG-NEXT: s_cselect_b32 s10, 1, 0
; GFX9-SDAG-NEXT: s_cmp_eq_u32 s9, 3
; GFX9-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-SDAG-NEXT: s_lshr_b32 s7, s7, 2
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; GFX9-SDAG-NEXT: s_or_b32 s9, s9, s10
-; GFX9-SDAG-NEXT: s_add_i32 s7, s7, s9
+; GFX9-SDAG-NEXT: s_add_i32 s5, s5, s9
; GFX9-SDAG-NEXT: s_cmp_lt_i32 s8, 31
-; GFX9-SDAG-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s6, 0
-; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; GFX9-SDAG-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s7
-; GFX9-SDAG-NEXT: s_lshr_b32 s5, s5, 16
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s7, 16
; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
; GFX9-SDAG-NEXT: s_or_b32 s4, s5, s4
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
@@ -888,47 +888,47 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX950-SDAG-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; GFX950-SDAG-NEXT: s_mov_b32 s0, s8
; GFX950-SDAG-NEXT: s_mov_b32 s1, s9
-; GFX950-SDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX950-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; GFX950-SDAG-NEXT: s_and_b32 s6, s5, 0x1ff
-; GFX950-SDAG-NEXT: v_or_b32_e32 v0, s6, v0
-; GFX950-SDAG-NEXT: s_lshr_b32 s7, s5, 8
-; GFX950-SDAG-NEXT: s_bfe_u32 s8, s5, 0xb0014
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v1
+; GFX950-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
+; GFX950-SDAG-NEXT: v_or_b32_e32 v0, s4, v0
; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SDAG-NEXT: s_and_b32 s6, s7, 0xffe
-; GFX950-SDAG-NEXT: s_sub_i32 s7, 0x3f1, s8
-; GFX950-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-SDAG-NEXT: v_med3_i32 v1, s7, 0, 13
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; GFX950-SDAG-NEXT: s_or_b32 s6, s6, s7
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX950-SDAG-NEXT: s_or_b32 s7, s6, 0x1000
-; GFX950-SDAG-NEXT: s_lshr_b32 s10, s7, s9
+; GFX950-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; GFX950-SDAG-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX950-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s8
+; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX950-SDAG-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX950-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s9, v0
+; GFX950-SDAG-NEXT: s_lshr_b32 s10, s5, s9
; GFX950-SDAG-NEXT: s_lshl_b32 s9, s10, s9
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s9, s7
-; GFX950-SDAG-NEXT: s_cselect_b32 s7, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s9, s5
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; GFX950-SDAG-NEXT: s_addk_i32 s8, 0xfc10
; GFX950-SDAG-NEXT: s_lshl_b32 s9, s8, 12
-; GFX950-SDAG-NEXT: s_or_b32 s7, s10, s7
-; GFX950-SDAG-NEXT: s_or_b32 s9, s6, s9
+; GFX950-SDAG-NEXT: s_or_b32 s5, s10, s5
+; GFX950-SDAG-NEXT: s_or_b32 s9, s4, s9
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s8, 1
-; GFX950-SDAG-NEXT: s_cselect_b32 s7, s7, s9
-; GFX950-SDAG-NEXT: s_and_b32 s9, s7, 7
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, s9
+; GFX950-SDAG-NEXT: s_and_b32 s9, s5, 7
; GFX950-SDAG-NEXT: s_cmp_gt_i32 s9, 5
; GFX950-SDAG-NEXT: s_cselect_b32 s10, 1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s9, 3
; GFX950-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; GFX950-SDAG-NEXT: s_lshr_b32 s7, s7, 2
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; GFX950-SDAG-NEXT: s_or_b32 s9, s9, s10
-; GFX950-SDAG-NEXT: s_add_i32 s7, s7, s9
+; GFX950-SDAG-NEXT: s_add_i32 s5, s5, s9
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s8, 31
-; GFX950-SDAG-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s6, 0
-; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s7
-; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 16
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s7, 16
; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, s4
@@ -1000,34 +1000,36 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s3, v0
-; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v1, s4, 0, 13
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, s5, s4
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s5, 0x3f1, s4
+; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v0, s5, 0, 13
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s3, s5, s3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SDAG-TRUE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-SDAG-TRUE16-NEXT: s_addk_i32 s4, 0xfc10
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s9, s5
-; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s4, s8
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s3, s8
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s8, s5, 7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -1036,19 +1038,19 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s8, s9
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_add_i32 s5, s5, s8
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX11-SDAG-TRUE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
-; GFX11-SDAG-TRUE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s4, s5
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s8, 0x7c00
+; GFX11-SDAG-TRUE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX11-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
@@ -1067,34 +1069,36 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s3, v0
-; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v1, s4, 0, 13
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s5, 0x3f1, s4
+; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v0, s5, 0, 13
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s3, s5, s3
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SDAG-FAKE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-SDAG-FAKE16-NEXT: s_addk_i32 s4, 0xfc10
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s9, s5
-; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s4, s8
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s3, s8
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s8, s5, 7
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -1103,19 +1107,19 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s8, s9
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_add_i32 s5, s5, s8
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
-; GFX11-SDAG-FAKE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s4, s5
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s8, 0x7c00
+; GFX11-SDAG-FAKE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX11-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
@@ -1232,8 +1236,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -1245,34 +1249,36 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s2, v1
; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s3, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s4, 0x3f1, s3
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1250-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v1, s4, 0, 13
-; GFX1250-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v1
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s5, 0x3f1, s4
+; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v0, s5, 0, 13
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s3, s5, s3
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-SDAG-TRUE16-NEXT: s_addk_co_i32 s3, 0xfc10
+; GFX1250-SDAG-TRUE16-NEXT: s_addk_co_i32 s4, 0xfc10
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s9, s5
-; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s4, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s3, s8
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s8, s5, 7
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -1281,27 +1287,27 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s8, s9
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_add_co_i32 s5, s5, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX1250-SDAG-TRUE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s3, 0x40f
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s4, s5
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s8, 0x7c00
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s4, 0x40f
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, s5
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s3
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-TRUE16-NEXT: s_endpgm
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -1313,34 +1319,36 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s2, v1
; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s3, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s4, 0x3f1, s3
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1250-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v1, s4, 0, 13
-; GFX1250-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v1
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s5, 0x3f1, s4
+; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v0, s5, 0, 13
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s3, s5, s3
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-SDAG-FAKE16-NEXT: s_addk_co_i32 s3, 0xfc10
+; GFX1250-SDAG-FAKE16-NEXT: s_addk_co_i32 s4, 0xfc10
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s9, s5
-; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s4, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s3, s8
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s8, s5, 7
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -1349,27 +1357,27 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s8, s9
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_add_co_i32 s5, s5, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX1250-SDAG-FAKE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s3, 0x40f
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s4, s5
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s8, 0x7c00
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s4, 0x40f
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, s5
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s3
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -1424,8 +1432,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f64_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -1685,8 +1693,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -1699,7 +1707,7 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -1707,8 +1715,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -1721,7 +1729,7 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -1729,15 +1737,15 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f32_f64_e32 v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -1745,15 +1753,15 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f64_to_f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
@@ -1974,8 +1982,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -1993,8 +2001,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -2012,8 +2020,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -2027,8 +2035,8 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f32_to_v2f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b64 s[2:3], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -2060,93 +2068,93 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; SI-SDAG-NEXT: s_mov_b32 s8, s6
; SI-SDAG-NEXT: s_mov_b32 s9, s7
; SI-SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; SI-SDAG-NEXT: s_movk_i32 s0, 0x7e00
+; SI-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; SI-SDAG-NEXT: v_readfirstlane_b32 s1, v3
-; SI-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; SI-SDAG-NEXT: s_and_b32 s7, s1, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s8, s1, 8
-; SI-SDAG-NEXT: s_bfe_u32 s9, s1, 0xb0014
-; SI-SDAG-NEXT: v_or_b32_e32 v1, s7, v2
-; SI-SDAG-NEXT: s_and_b32 s7, s8, 0xffe
-; SI-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s9
-; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; SI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; SI-SDAG-NEXT: v_med3_i32 v2, s8, 0, 13
+; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v3
; SI-SDAG-NEXT: v_readfirstlane_b32 s8, v1
-; SI-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; SI-SDAG-NEXT: s_or_b32 s7, s7, s8
-; SI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; SI-SDAG-NEXT: s_lshr_b32 s11, s8, s10
+; SI-SDAG-NEXT: s_and_b32 s0, s7, 0x1ff
+; SI-SDAG-NEXT: v_or_b32_e32 v1, s0, v2
+; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-SDAG-NEXT: s_lshr_b32 s1, s7, 8
+; SI-SDAG-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; SI-SDAG-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-SDAG-NEXT: s_or_b32 s0, s1, s0
+; SI-SDAG-NEXT: v_med3_i32 v1, s10, 0, 13
+; SI-SDAG-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-SDAG-NEXT: v_readfirstlane_b32 s10, v1
+; SI-SDAG-NEXT: s_lshr_b32 s11, s1, s10
; SI-SDAG-NEXT: s_lshl_b32 s10, s11, s10
-; SI-SDAG-NEXT: s_cmp_lg_u32 s10, s8
-; SI-SDAG-NEXT: s_cselect_b32 s8, 1, 0
+; SI-SDAG-NEXT: s_cmp_lg_u32 s10, s1
+; SI-SDAG-NEXT: s_cselect_b32 s1, 1, 0
; SI-SDAG-NEXT: s_addk_i32 s9, 0xfc10
-; SI-SDAG-NEXT: s_or_b32 s8, s11, s8
+; SI-SDAG-NEXT: s_or_b32 s1, s11, s1
; SI-SDAG-NEXT: s_lshl_b32 s10, s9, 12
-; SI-SDAG-NEXT: s_or_b32 s10, s7, s10
+; SI-SDAG-NEXT: s_or_b32 s10, s0, s10
; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 1
-; SI-SDAG-NEXT: s_cselect_b32 s8, s8, s10
-; SI-SDAG-NEXT: s_and_b32 s10, s8, 7
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, s10
+; SI-SDAG-NEXT: s_and_b32 s10, s1, 7
; SI-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; SI-SDAG-NEXT: s_cselect_b32 s11, 1, 0
; SI-SDAG-NEXT: s_cmp_eq_u32 s10, 3
; SI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
-; SI-SDAG-NEXT: s_lshr_b32 s8, s8, 2
+; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 2
; SI-SDAG-NEXT: s_or_b32 s10, s10, s11
-; SI-SDAG-NEXT: s_add_i32 s8, s8, s10
+; SI-SDAG-NEXT: s_add_i32 s1, s1, s10
; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 31
-; SI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; SI-SDAG-NEXT: s_cselect_b32 s7, s0, 0x7c00
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s0, 0
+; SI-SDAG-NEXT: s_cselect_b32 s0, s6, 0x7c00
; SI-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s7, s7, s8
-; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 16
-; SI-SDAG-NEXT: s_and_b32 s8, s6, 0x1ff
-; SI-SDAG-NEXT: s_lshr_b32 s9, s6, 8
-; SI-SDAG-NEXT: s_bfe_u32 s10, s6, 0xb0014
+; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s1
+; SI-SDAG-NEXT: s_lshr_b32 s1, s7, 16
+; SI-SDAG-NEXT: s_and_b32 s7, s8, 0x1ff
; SI-SDAG-NEXT: s_and_b32 s1, s1, 0x8000
-; SI-SDAG-NEXT: v_or_b32_e32 v0, s8, v0
-; SI-SDAG-NEXT: s_and_b32 s8, s9, 0xffe
-; SI-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s10
-; SI-SDAG-NEXT: s_or_b32 s1, s1, s7
+; SI-SDAG-NEXT: v_or_b32_e32 v0, s7, v0
+; SI-SDAG-NEXT: s_or_b32 s0, s1, s0
; SI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-SDAG-NEXT: v_med3_i32 v1, s9, 0, 13
-; SI-SDAG-NEXT: s_lshl_b32 s1, s1, 16
-; SI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; SI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; SI-SDAG-NEXT: s_or_b32 s7, s8, s7
-; SI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; SI-SDAG-NEXT: s_lshr_b32 s11, s8, s9
-; SI-SDAG-NEXT: s_lshl_b32 s9, s11, s9
-; SI-SDAG-NEXT: s_cmp_lg_u32 s9, s8
-; SI-SDAG-NEXT: s_cselect_b32 s8, 1, 0
-; SI-SDAG-NEXT: s_addk_i32 s10, 0xfc10
-; SI-SDAG-NEXT: s_or_b32 s8, s11, s8
-; SI-SDAG-NEXT: s_lshl_b32 s9, s10, 12
-; SI-SDAG-NEXT: s_or_b32 s9, s7, s9
-; SI-SDAG-NEXT: s_cmp_lt_i32 s10, 1
-; SI-SDAG-NEXT: s_cselect_b32 s8, s8, s9
-; SI-SDAG-NEXT: s_and_b32 s9, s8, 7
-; SI-SDAG-NEXT: s_cmp_gt_i32 s9, 5
+; SI-SDAG-NEXT: s_lshl_b32 s7, s0, 16
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; SI-SDAG-NEXT: s_lshr_b32 s1, s8, 8
+; SI-SDAG-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; SI-SDAG-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-SDAG-NEXT: s_or_b32 s0, s1, s0
+; SI-SDAG-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-SDAG-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-SDAG-NEXT: v_readfirstlane_b32 s10, v0
+; SI-SDAG-NEXT: s_lshr_b32 s11, s1, s10
+; SI-SDAG-NEXT: s_lshl_b32 s10, s11, s10
+; SI-SDAG-NEXT: s_cmp_lg_u32 s10, s1
+; SI-SDAG-NEXT: s_cselect_b32 s1, 1, 0
+; SI-SDAG-NEXT: s_addk_i32 s9, 0xfc10
+; SI-SDAG-NEXT: s_or_b32 s1, s11, s1
+; SI-SDAG-NEXT: s_lshl_b32 s10, s9, 12
+; SI-SDAG-NEXT: s_or_b32 s10, s0, s10
+; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 1
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, s10
+; SI-SDAG-NEXT: s_and_b32 s10, s1, 7
+; SI-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; SI-SDAG-NEXT: s_cselect_b32 s11, 1, 0
-; SI-SDAG-NEXT: s_cmp_eq_u32 s9, 3
-; SI-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; SI-SDAG-NEXT: s_lshr_b32 s8, s8, 2
-; SI-SDAG-NEXT: s_or_b32 s9, s9, s11
-; SI-SDAG-NEXT: s_add_i32 s8, s8, s9
-; SI-SDAG-NEXT: s_cmp_lt_i32 s10, 31
-; SI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; SI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, 0x7c00
-; SI-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s8
-; SI-SDAG-NEXT: s_lshr_b32 s6, s6, 16
-; SI-SDAG-NEXT: s_and_b32 s6, s6, 0x8000
-; SI-SDAG-NEXT: s_or_b32 s0, s6, s0
+; SI-SDAG-NEXT: s_cmp_eq_u32 s10, 3
+; SI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
+; SI-SDAG-NEXT: s_lshr_b32 s1, s1, 2
+; SI-SDAG-NEXT: s_or_b32 s10, s10, s11
+; SI-SDAG-NEXT: s_add_i32 s1, s1, s10
+; SI-SDAG-NEXT: s_cmp_lt_i32 s9, 31
+; SI-SDAG-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-SDAG-NEXT: s_cmp_lg_u32 s0, 0
+; SI-SDAG-NEXT: s_cselect_b32 s0, s6, 0x7c00
+; SI-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-SDAG-NEXT: s_cselect_b32 s0, s0, s1
+; SI-SDAG-NEXT: s_lshr_b32 s1, s8, 16
+; SI-SDAG-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-SDAG-NEXT: s_or_b32 s0, s1, s0
; SI-SDAG-NEXT: s_and_b32 s0, s0, 0xffff
-; SI-SDAG-NEXT: s_or_b32 s6, s0, s1
+; SI-SDAG-NEXT: s_or_b32 s6, s0, s7
; SI-SDAG-NEXT: s_mov_b32 s0, s4
; SI-SDAG-NEXT: s_mov_b32 s1, s5
; SI-SDAG-NEXT: v_mov_b32_e32 v0, s6
@@ -2262,91 +2270,91 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; VI-SDAG-NEXT: s_mov_b32 s1, s5
; VI-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
-; VI-SDAG-NEXT: v_readfirstlane_b32 s4, v3
-; VI-SDAG-NEXT: s_and_b32 s7, s4, 0x1ff
-; VI-SDAG-NEXT: v_readfirstlane_b32 s5, v1
-; VI-SDAG-NEXT: v_or_b32_e32 v1, s7, v2
-; VI-SDAG-NEXT: s_lshr_b32 s8, s4, 8
-; VI-SDAG-NEXT: s_bfe_u32 s9, s4, 0xb0014
-; VI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; VI-SDAG-NEXT: s_and_b32 s7, s8, 0xffe
-; VI-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s9
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; VI-SDAG-NEXT: v_med3_i32 v2, s8, 0, 13
+; VI-SDAG-NEXT: v_readfirstlane_b32 s7, v3
+; VI-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
; VI-SDAG-NEXT: v_readfirstlane_b32 s8, v1
-; VI-SDAG-NEXT: s_or_b32 s7, s7, s8
-; VI-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; VI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; VI-SDAG-NEXT: s_lshr_b32 s11, s8, s10
+; VI-SDAG-NEXT: v_or_b32_e32 v1, s4, v2
+; VI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; VI-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; VI-SDAG-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; VI-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; VI-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
+; VI-SDAG-NEXT: v_med3_i32 v1, s10, 0, 13
+; VI-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; VI-SDAG-NEXT: v_readfirstlane_b32 s10, v1
+; VI-SDAG-NEXT: s_lshr_b32 s11, s5, s10
; VI-SDAG-NEXT: s_lshl_b32 s10, s11, s10
-; VI-SDAG-NEXT: s_cmp_lg_u32 s10, s8
-; VI-SDAG-NEXT: s_cselect_b32 s8, 1, 0
+; VI-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; VI-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; VI-SDAG-NEXT: s_addk_i32 s9, 0xfc10
; VI-SDAG-NEXT: s_lshl_b32 s10, s9, 12
-; VI-SDAG-NEXT: s_or_b32 s8, s11, s8
-; VI-SDAG-NEXT: s_or_b32 s10, s7, s10
+; VI-SDAG-NEXT: s_or_b32 s5, s11, s5
+; VI-SDAG-NEXT: s_or_b32 s10, s4, s10
; VI-SDAG-NEXT: s_cmp_lt_i32 s9, 1
-; VI-SDAG-NEXT: s_cselect_b32 s8, s8, s10
-; VI-SDAG-NEXT: s_and_b32 s10, s8, 7
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; VI-SDAG-NEXT: s_and_b32 s10, s5, 7
; VI-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; VI-SDAG-NEXT: s_cselect_b32 s11, 1, 0
; VI-SDAG-NEXT: s_cmp_eq_u32 s10, 3
; VI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
-; VI-SDAG-NEXT: s_lshr_b32 s8, s8, 2
+; VI-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; VI-SDAG-NEXT: s_or_b32 s10, s10, s11
-; VI-SDAG-NEXT: s_add_i32 s8, s8, s10
+; VI-SDAG-NEXT: s_add_i32 s5, s5, s10
; VI-SDAG-NEXT: s_cmp_lt_i32 s9, 31
-; VI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; VI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; VI-SDAG-NEXT: s_cselect_b32 s7, s6, 0x7c00
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; VI-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; VI-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; VI-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
-; VI-SDAG-NEXT: s_cselect_b32 s7, s7, s8
-; VI-SDAG-NEXT: s_and_b32 s8, s5, 0x1ff
-; VI-SDAG-NEXT: v_or_b32_e32 v0, s8, v0
-; VI-SDAG-NEXT: s_lshr_b32 s4, s4, 16
+; VI-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; VI-SDAG-NEXT: s_lshr_b32 s5, s7, 16
+; VI-SDAG-NEXT: s_and_b32 s7, s8, 0x1ff
+; VI-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
+; VI-SDAG-NEXT: v_or_b32_e32 v0, s7, v0
+; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
; VI-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; VI-SDAG-NEXT: s_lshr_b32 s9, s5, 8
-; VI-SDAG-NEXT: s_bfe_u32 s10, s5, 0xb0014
-; VI-SDAG-NEXT: s_and_b32 s4, s4, 0x8000
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; VI-SDAG-NEXT: s_and_b32 s8, s9, 0xffe
-; VI-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s10
-; VI-SDAG-NEXT: s_or_b32 s4, s4, s7
-; VI-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; VI-SDAG-NEXT: v_med3_i32 v1, s9, 0, 13
-; VI-SDAG-NEXT: s_or_b32 s7, s8, s7
-; VI-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; VI-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; VI-SDAG-NEXT: s_lshr_b32 s11, s8, s9
-; VI-SDAG-NEXT: s_lshl_b32 s4, s4, 16
-; VI-SDAG-NEXT: s_lshl_b32 s9, s11, s9
-; VI-SDAG-NEXT: s_cmp_lg_u32 s9, s8
-; VI-SDAG-NEXT: s_cselect_b32 s8, 1, 0
-; VI-SDAG-NEXT: s_addk_i32 s10, 0xfc10
-; VI-SDAG-NEXT: s_lshl_b32 s9, s10, 12
-; VI-SDAG-NEXT: s_or_b32 s8, s11, s8
-; VI-SDAG-NEXT: s_or_b32 s9, s7, s9
-; VI-SDAG-NEXT: s_cmp_lt_i32 s10, 1
-; VI-SDAG-NEXT: s_cselect_b32 s8, s8, s9
-; VI-SDAG-NEXT: s_and_b32 s9, s8, 7
-; VI-SDAG-NEXT: s_cmp_gt_i32 s9, 5
+; VI-SDAG-NEXT: s_lshl_b32 s7, s4, 16
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; VI-SDAG-NEXT: s_lshr_b32 s5, s8, 8
+; VI-SDAG-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; VI-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; VI-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
+; VI-SDAG-NEXT: v_med3_i32 v0, s10, 0, 13
+; VI-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; VI-SDAG-NEXT: v_readfirstlane_b32 s10, v0
+; VI-SDAG-NEXT: s_lshr_b32 s11, s5, s10
+; VI-SDAG-NEXT: s_lshl_b32 s10, s11, s10
+; VI-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; VI-SDAG-NEXT: s_cselect_b32 s5, 1, 0
+; VI-SDAG-NEXT: s_addk_i32 s9, 0xfc10
+; VI-SDAG-NEXT: s_lshl_b32 s10, s9, 12
+; VI-SDAG-NEXT: s_or_b32 s5, s11, s5
+; VI-SDAG-NEXT: s_or_b32 s10, s4, s10
+; VI-SDAG-NEXT: s_cmp_lt_i32 s9, 1
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; VI-SDAG-NEXT: s_and_b32 s10, s5, 7
+; VI-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; VI-SDAG-NEXT: s_cselect_b32 s11, 1, 0
-; VI-SDAG-NEXT: s_cmp_eq_u32 s9, 3
-; VI-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; VI-SDAG-NEXT: s_lshr_b32 s8, s8, 2
-; VI-SDAG-NEXT: s_or_b32 s9, s9, s11
-; VI-SDAG-NEXT: s_add_i32 s8, s8, s9
-; VI-SDAG-NEXT: s_cmp_lt_i32 s10, 31
-; VI-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; VI-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; VI-SDAG-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; VI-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; VI-SDAG-NEXT: s_cselect_b32 s6, s6, s8
-; VI-SDAG-NEXT: s_lshr_b32 s5, s5, 16
+; VI-SDAG-NEXT: s_cmp_eq_u32 s10, 3
+; VI-SDAG-NEXT: s_cselect_b32 s10, 1, 0
+; VI-SDAG-NEXT: s_lshr_b32 s5, s5, 2
+; VI-SDAG-NEXT: s_or_b32 s10, s10, s11
+; VI-SDAG-NEXT: s_add_i32 s5, s5, s10
+; VI-SDAG-NEXT: s_cmp_lt_i32 s9, 31
+; VI-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; VI-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; VI-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; VI-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; VI-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; VI-SDAG-NEXT: s_lshr_b32 s5, s8, 16
; VI-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
-; VI-SDAG-NEXT: s_or_b32 s5, s5, s6
-; VI-SDAG-NEXT: s_and_b32 s5, s5, 0xffff
; VI-SDAG-NEXT: s_or_b32 s4, s5, s4
+; VI-SDAG-NEXT: s_and_b32 s4, s4, 0xffff
+; VI-SDAG-NEXT: s_or_b32 s4, s4, s7
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-SDAG-NEXT: s_endpgm
@@ -2458,91 +2466,91 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX9-SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; GFX9-SDAG-NEXT: s_mov_b32 s0, s8
; GFX9-SDAG-NEXT: s_mov_b32 s1, s9
-; GFX9-SDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX9-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s5, v3
-; GFX9-SDAG-NEXT: s_and_b32 s7, s5, 0x1ff
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; GFX9-SDAG-NEXT: v_or_b32_e32 v1, s7, v2
-; GFX9-SDAG-NEXT: s_lshr_b32 s8, s5, 8
-; GFX9-SDAG-NEXT: s_bfe_u32 s9, s5, 0xb0014
-; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX9-SDAG-NEXT: s_and_b32 s7, s8, 0xffe
-; GFX9-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s9
-; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX9-SDAG-NEXT: v_med3_i32 v2, s8, 0, 13
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s7, v3
+; GFX9-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s8, v1
-; GFX9-SDAG-NEXT: s_or_b32 s7, s7, s8
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX9-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; GFX9-SDAG-NEXT: s_lshr_b32 s11, s8, s10
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, s4, v2
+; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX9-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; GFX9-SDAG-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX9-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; GFX9-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX9-SDAG-NEXT: v_med3_i32 v1, s10, 0, 13
+; GFX9-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s10, v1
+; GFX9-SDAG-NEXT: s_lshr_b32 s11, s5, s10
; GFX9-SDAG-NEXT: s_lshl_b32 s10, s11, s10
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s10, s8
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, 1, 0
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; GFX9-SDAG-NEXT: s_addk_i32 s9, 0xfc10
; GFX9-SDAG-NEXT: s_lshl_b32 s10, s9, 12
-; GFX9-SDAG-NEXT: s_or_b32 s8, s11, s8
-; GFX9-SDAG-NEXT: s_or_b32 s10, s7, s10
+; GFX9-SDAG-NEXT: s_or_b32 s5, s11, s5
+; GFX9-SDAG-NEXT: s_or_b32 s10, s4, s10
; GFX9-SDAG-NEXT: s_cmp_lt_i32 s9, 1
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, s8, s10
-; GFX9-SDAG-NEXT: s_and_b32 s10, s8, 7
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; GFX9-SDAG-NEXT: s_and_b32 s10, s5, 7
; GFX9-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; GFX9-SDAG-NEXT: s_cselect_b32 s11, 1, 0
; GFX9-SDAG-NEXT: s_cmp_eq_u32 s10, 3
; GFX9-SDAG-NEXT: s_cselect_b32 s10, 1, 0
-; GFX9-SDAG-NEXT: s_lshr_b32 s8, s8, 2
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; GFX9-SDAG-NEXT: s_or_b32 s10, s10, s11
-; GFX9-SDAG-NEXT: s_add_i32 s8, s8, s10
+; GFX9-SDAG-NEXT: s_add_i32 s5, s5, s10
; GFX9-SDAG-NEXT: s_cmp_lt_i32 s9, 31
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; GFX9-SDAG-NEXT: s_cselect_b32 s7, s4, 0x7c00
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; GFX9-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
-; GFX9-SDAG-NEXT: s_cselect_b32 s7, s7, s8
-; GFX9-SDAG-NEXT: s_and_b32 s8, s6, 0x1ff
-; GFX9-SDAG-NEXT: v_or_b32_e32 v0, s8, v0
-; GFX9-SDAG-NEXT: s_lshr_b32 s5, s5, 16
-; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX9-SDAG-NEXT: s_lshr_b32 s9, s6, 8
-; GFX9-SDAG-NEXT: s_bfe_u32 s10, s6, 0xb0014
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s7, 16
+; GFX9-SDAG-NEXT: s_and_b32 s7, s8, 0x1ff
; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
-; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX9-SDAG-NEXT: s_and_b32 s8, s9, 0xffe
-; GFX9-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s10
-; GFX9-SDAG-NEXT: s_or_b32 s5, s5, s7
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; GFX9-SDAG-NEXT: v_med3_i32 v1, s9, 0, 13
-; GFX9-SDAG-NEXT: s_or_b32 s7, s8, s7
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX9-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; GFX9-SDAG-NEXT: s_lshr_b32 s11, s8, s9
-; GFX9-SDAG-NEXT: s_lshl_b32 s9, s11, s9
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s9, s8
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, 1, 0
-; GFX9-SDAG-NEXT: s_addk_i32 s10, 0xfc10
-; GFX9-SDAG-NEXT: s_lshl_b32 s9, s10, 12
-; GFX9-SDAG-NEXT: s_or_b32 s8, s11, s8
-; GFX9-SDAG-NEXT: s_or_b32 s9, s7, s9
-; GFX9-SDAG-NEXT: s_cmp_lt_i32 s10, 1
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, s8, s9
-; GFX9-SDAG-NEXT: s_and_b32 s9, s8, 7
-; GFX9-SDAG-NEXT: s_cmp_gt_i32 s9, 5
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, s7, v0
+; GFX9-SDAG-NEXT: s_or_b32 s7, s5, s4
+; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s8, 8
+; GFX9-SDAG-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX9-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; GFX9-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX9-SDAG-NEXT: v_med3_i32 v0, s10, 0, 13
+; GFX9-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s10, v0
+; GFX9-SDAG-NEXT: s_lshr_b32 s11, s5, s10
+; GFX9-SDAG-NEXT: s_lshl_b32 s10, s11, s10
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-SDAG-NEXT: s_addk_i32 s9, 0xfc10
+; GFX9-SDAG-NEXT: s_lshl_b32 s10, s9, 12
+; GFX9-SDAG-NEXT: s_or_b32 s5, s11, s5
+; GFX9-SDAG-NEXT: s_or_b32 s10, s4, s10
+; GFX9-SDAG-NEXT: s_cmp_lt_i32 s9, 1
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; GFX9-SDAG-NEXT: s_and_b32 s10, s5, 7
+; GFX9-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; GFX9-SDAG-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-SDAG-NEXT: s_cmp_eq_u32 s9, 3
-; GFX9-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; GFX9-SDAG-NEXT: s_lshr_b32 s8, s8, 2
-; GFX9-SDAG-NEXT: s_or_b32 s9, s9, s11
-; GFX9-SDAG-NEXT: s_add_i32 s8, s8, s9
-; GFX9-SDAG-NEXT: s_cmp_lt_i32 s10, 31
-; GFX9-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; GFX9-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
-; GFX9-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s8
-; GFX9-SDAG-NEXT: s_lshr_b32 s6, s6, 16
-; GFX9-SDAG-NEXT: s_and_b32 s6, s6, 0x8000
-; GFX9-SDAG-NEXT: s_or_b32 s4, s6, s4
-; GFX9-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s5
+; GFX9-SDAG-NEXT: s_cmp_eq_u32 s10, 3
+; GFX9-SDAG-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s5, 2
+; GFX9-SDAG-NEXT: s_or_b32 s10, s10, s11
+; GFX9-SDAG-NEXT: s_add_i32 s5, s5, s10
+; GFX9-SDAG-NEXT: s_cmp_lt_i32 s9, 31
+; GFX9-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; GFX9-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; GFX9-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX9-SDAG-NEXT: s_lshr_b32 s5, s8, 16
+; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
+; GFX9-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX9-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s7
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-SDAG-NEXT: s_endpgm
@@ -2651,91 +2659,91 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX950-SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; GFX950-SDAG-NEXT: s_mov_b32 s0, s8
; GFX950-SDAG-NEXT: s_mov_b32 s1, s9
-; GFX950-SDAG-NEXT: s_movk_i32 s4, 0x7e00
+; GFX950-SDAG-NEXT: s_movk_i32 s6, 0x7e00
; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s5, v3
-; GFX950-SDAG-NEXT: s_and_b32 s7, s5, 0x1ff
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; GFX950-SDAG-NEXT: v_or_b32_e32 v1, s7, v2
-; GFX950-SDAG-NEXT: s_lshr_b32 s8, s5, 8
-; GFX950-SDAG-NEXT: s_bfe_u32 s9, s5, 0xb0014
-; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX950-SDAG-NEXT: s_and_b32 s7, s8, 0xffe
-; GFX950-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s9
-; GFX950-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX950-SDAG-NEXT: v_med3_i32 v2, s8, 0, 13
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v3
+; GFX950-SDAG-NEXT: s_and_b32 s4, s7, 0x1ff
; GFX950-SDAG-NEXT: v_readfirstlane_b32 s8, v1
-; GFX950-SDAG-NEXT: s_or_b32 s7, s7, s8
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s10, v2
-; GFX950-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; GFX950-SDAG-NEXT: s_lshr_b32 s11, s8, s10
+; GFX950-SDAG-NEXT: v_or_b32_e32 v1, s4, v2
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
+; GFX950-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s7, 8
+; GFX950-SDAG-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX950-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX950-SDAG-NEXT: v_med3_i32 v1, s10, 0, 13
+; GFX950-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s10, v1
+; GFX950-SDAG-NEXT: s_lshr_b32 s11, s5, s10
; GFX950-SDAG-NEXT: s_lshl_b32 s10, s11, s10
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s10, s8
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, 1, 0
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, 1, 0
; GFX950-SDAG-NEXT: s_addk_i32 s9, 0xfc10
; GFX950-SDAG-NEXT: s_lshl_b32 s10, s9, 12
-; GFX950-SDAG-NEXT: s_or_b32 s8, s11, s8
-; GFX950-SDAG-NEXT: s_or_b32 s10, s7, s10
+; GFX950-SDAG-NEXT: s_or_b32 s5, s11, s5
+; GFX950-SDAG-NEXT: s_or_b32 s10, s4, s10
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 1
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, s10
-; GFX950-SDAG-NEXT: s_and_b32 s10, s8, 7
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; GFX950-SDAG-NEXT: s_and_b32 s10, s5, 7
; GFX950-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; GFX950-SDAG-NEXT: s_cselect_b32 s11, 1, 0
; GFX950-SDAG-NEXT: s_cmp_eq_u32 s10, 3
; GFX950-SDAG-NEXT: s_cselect_b32 s10, 1, 0
-; GFX950-SDAG-NEXT: s_lshr_b32 s8, s8, 2
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 2
; GFX950-SDAG-NEXT: s_or_b32 s10, s10, s11
-; GFX950-SDAG-NEXT: s_add_i32 s8, s8, s10
+; GFX950-SDAG-NEXT: s_add_i32 s5, s5, s10
; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 31
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; GFX950-SDAG-NEXT: s_cselect_b32 s7, s4, 0x7c00
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
-; GFX950-SDAG-NEXT: s_cselect_b32 s7, s7, s8
-; GFX950-SDAG-NEXT: s_and_b32 s8, s6, 0x1ff
-; GFX950-SDAG-NEXT: v_or_b32_e32 v0, s8, v0
-; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 16
-; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX950-SDAG-NEXT: s_lshr_b32 s9, s6, 8
-; GFX950-SDAG-NEXT: s_bfe_u32 s10, s6, 0xb0014
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s7, 16
+; GFX950-SDAG-NEXT: s_and_b32 s7, s8, 0x1ff
; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
-; GFX950-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX950-SDAG-NEXT: s_and_b32 s8, s9, 0xffe
-; GFX950-SDAG-NEXT: s_sub_i32 s9, 0x3f1, s10
-; GFX950-SDAG-NEXT: s_or_b32 s5, s5, s7
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s7, v0
-; GFX950-SDAG-NEXT: v_med3_i32 v1, s9, 0, 13
-; GFX950-SDAG-NEXT: s_or_b32 s7, s8, s7
-; GFX950-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX950-SDAG-NEXT: s_or_b32 s8, s7, 0x1000
-; GFX950-SDAG-NEXT: s_lshr_b32 s11, s8, s9
-; GFX950-SDAG-NEXT: s_lshl_b32 s9, s11, s9
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s9, s8
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, 1, 0
-; GFX950-SDAG-NEXT: s_addk_i32 s10, 0xfc10
-; GFX950-SDAG-NEXT: s_lshl_b32 s9, s10, 12
-; GFX950-SDAG-NEXT: s_or_b32 s8, s11, s8
-; GFX950-SDAG-NEXT: s_or_b32 s9, s7, s9
-; GFX950-SDAG-NEXT: s_cmp_lt_i32 s10, 1
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, s9
-; GFX950-SDAG-NEXT: s_and_b32 s9, s8, 7
-; GFX950-SDAG-NEXT: s_cmp_gt_i32 s9, 5
+; GFX950-SDAG-NEXT: v_or_b32_e32 v0, s7, v0
+; GFX950-SDAG-NEXT: s_or_b32 s7, s5, s4
+; GFX950-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX950-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s8, 8
+; GFX950-SDAG-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX950-SDAG-NEXT: s_sub_i32 s10, 0x3f1, s9
+; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX950-SDAG-NEXT: v_med3_i32 v0, s10, 0, 13
+; GFX950-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX950-SDAG-NEXT: v_readfirstlane_b32 s10, v0
+; GFX950-SDAG-NEXT: s_lshr_b32 s11, s5, s10
+; GFX950-SDAG-NEXT: s_lshl_b32 s10, s11, s10
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s10, s5
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, 1, 0
+; GFX950-SDAG-NEXT: s_addk_i32 s9, 0xfc10
+; GFX950-SDAG-NEXT: s_lshl_b32 s10, s9, 12
+; GFX950-SDAG-NEXT: s_or_b32 s5, s11, s5
+; GFX950-SDAG-NEXT: s_or_b32 s10, s4, s10
+; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 1
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, s10
+; GFX950-SDAG-NEXT: s_and_b32 s10, s5, 7
+; GFX950-SDAG-NEXT: s_cmp_gt_i32 s10, 5
; GFX950-SDAG-NEXT: s_cselect_b32 s11, 1, 0
-; GFX950-SDAG-NEXT: s_cmp_eq_u32 s9, 3
-; GFX950-SDAG-NEXT: s_cselect_b32 s9, 1, 0
-; GFX950-SDAG-NEXT: s_lshr_b32 s8, s8, 2
-; GFX950-SDAG-NEXT: s_or_b32 s9, s9, s11
-; GFX950-SDAG-NEXT: s_add_i32 s8, s8, s9
-; GFX950-SDAG-NEXT: s_cmp_lt_i32 s10, 31
-; GFX950-SDAG-NEXT: s_cselect_b32 s8, s8, 0x7c00
-; GFX950-SDAG-NEXT: s_cmp_lg_u32 s7, 0
-; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
-; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s8
-; GFX950-SDAG-NEXT: s_lshr_b32 s6, s6, 16
-; GFX950-SDAG-NEXT: s_and_b32 s6, s6, 0x8000
-; GFX950-SDAG-NEXT: s_or_b32 s4, s6, s4
-; GFX950-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s5
+; GFX950-SDAG-NEXT: s_cmp_eq_u32 s10, 3
+; GFX950-SDAG-NEXT: s_cselect_b32 s10, 1, 0
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s5, 2
+; GFX950-SDAG-NEXT: s_or_b32 s10, s10, s11
+; GFX950-SDAG-NEXT: s_add_i32 s5, s5, s10
+; GFX950-SDAG-NEXT: s_cmp_lt_i32 s9, 31
+; GFX950-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; GFX950-SDAG-NEXT: s_cmp_lg_u32 s4, 0
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; GFX950-SDAG-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; GFX950-SDAG-NEXT: s_cselect_b32 s4, s4, s5
+; GFX950-SDAG-NEXT: s_lshr_b32 s5, s8, 16
+; GFX950-SDAG-NEXT: s_and_b32 s5, s5, 0x8000
+; GFX950-SDAG-NEXT: s_or_b32 s4, s5, s4
+; GFX950-SDAG-NEXT: s_pack_ll_b32_b16 s4, s4, s7
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, s4
; GFX950-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX950-SDAG-NEXT: s_endpgm
@@ -2845,34 +2853,36 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s2, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, s3, v2
-; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v3, s4, 0, 13
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, s5, s4
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s5, 0x3f1, s4
+; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v2, s5, 0, 13
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v2
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s3, s5, s3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SDAG-TRUE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-SDAG-TRUE16-NEXT: s_addk_i32 s4, 0xfc10
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, s9, s5
-; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s4, s8
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s3, s8
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s8, s5, 7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -2881,65 +2891,67 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s8, s8, s9
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_add_i32 s5, s5, s8
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX11-SDAG-TRUE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s8, 0x7c00
-; GFX11-SDAG-TRUE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s9, s5
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s4, 0x1ff
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s10, s4, 8
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s5, v0
-; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s5, s4, 0xb0014
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s10, s10, 0xffe
-; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s9, 0x3f1, s5
+; GFX11-SDAG-TRUE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s9, s5
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s5, s3, 0x1ff
; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v1, s9, 0, 13
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s5, v0
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s9, s10, s9
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s10, s9, 0x1000
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-SDAG-TRUE16-NEXT: s_bfe_u32 s5, s3, 0xb0014
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s12, s10, s11
-; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s11, s12, s11
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s11, s10
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-TRUE16-NEXT: s_sub_i32 s9, 0x3f1, s5
+; GFX11-SDAG-TRUE16-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s3, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s9, s9, 0xffe
+; GFX11-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, s9, s4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s9, s4, 0x1000
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s11, s9, s10
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s10, s11, s10
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s10, s9
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-TRUE16-NEXT: s_addk_i32 s5, 0xfc10
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s3, s12, s3
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s9, s11, s9
; GFX11-SDAG-TRUE16-NEXT: s_lshl_b32 s10, s5, 12
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s10, s9, s10
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s10, s4, s10
; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s5, 1
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, s10
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s10, s3, 7
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s9, s10
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s10, s9, 7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s10, 5
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s11, 1, 0
; GFX11-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s10, 3
; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s9, 2
; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s10, s10, s11
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: s_add_i32 s3, s3, s10
+; GFX11-SDAG-TRUE16-NEXT: s_add_i32 s9, s9, s10
; GFX11-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s5, 31
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s9, 0
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s8, s8, 0x7c00
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s9, 0x7c00
+; GFX11-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
; GFX11-SDAG-TRUE16-NEXT: s_cmpk_eq_i32 s5, 0x40f
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s8, s3
-; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s4, s4, 16
+; GFX11-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s4, s9
+; GFX11-SDAG-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s4, s4, 0x8000
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s3, s4, s3
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s3, s3, 0x8000
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s3, s3, s4
; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, s0
; GFX11-SDAG-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2961,34 +2973,36 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s2, v3
; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, s3, v2
-; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s4, 0x3f1, s3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v3, s4, 0, 13
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s5, 0x3f1, s4
+; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v2, s5, 0, 13
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v2
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s3, s5, s3
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SDAG-FAKE16-NEXT: s_addk_i32 s3, 0xfc10
+; GFX11-SDAG-FAKE16-NEXT: s_addk_i32 s4, 0xfc10
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, s9, s5
-; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s4, s8
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s3, s8
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s8, s5, 7
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -2997,65 +3011,67 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s8, s8, s9
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_add_i32 s5, s5, s8
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX11-SDAG-FAKE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s8, 0x7c00
-; GFX11-SDAG-FAKE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s9, s5
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s4, 0x1ff
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s10, s4, 8
-; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s5, v0
-; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s5, s4, 0xb0014
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s10, s10, 0xffe
-; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s9, 0x3f1, s5
+; GFX11-SDAG-FAKE16-NEXT: s_cmpk_eq_i32 s4, 0x40f
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s9, s5
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s5, s3, 0x1ff
; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v1, s9, 0, 13
+; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s5, v0
; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s3
-; GFX11-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s11, v1
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s9, s10, s9
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s10, s9, 0x1000
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s4
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-SDAG-FAKE16-NEXT: s_bfe_u32 s5, s3, 0xb0014
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s12, s10, s11
-; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s11, s12, s11
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s11, s10
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-SDAG-FAKE16-NEXT: s_sub_i32 s9, 0x3f1, s5
+; GFX11-SDAG-FAKE16-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s3, 8
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s9, s9, 0xffe
+; GFX11-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, s9, s4
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s9, s4, 0x1000
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s11, s9, s10
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s10, s11, s10
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s10, s9
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX11-SDAG-FAKE16-NEXT: s_addk_i32 s5, 0xfc10
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s3, s12, s3
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s9, s11, s9
; GFX11-SDAG-FAKE16-NEXT: s_lshl_b32 s10, s5, 12
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s10, s9, s10
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s10, s4, s10
; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s5, 1
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, s10
-; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s10, s3, 7
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s9, s10
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s10, s9, 7
+; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s10, 5
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s11, 1, 0
; GFX11-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s10, 3
; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s9, 2
; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s10, s10, s11
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_add_i32 s3, s3, s10
+; GFX11-SDAG-FAKE16-NEXT: s_add_i32 s9, s9, s10
; GFX11-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s5, 31
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s9, 0
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s8, s8, 0x7c00
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s9, 0x7c00
+; GFX11-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
; GFX11-SDAG-FAKE16-NEXT: s_cmpk_eq_i32 s5, 0x40f
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s8, s3
-; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s4, s4, 16
+; GFX11-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s4, s9
+; GFX11-SDAG-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s4, s4, 0x8000
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s3, s4, s3
+; GFX11-SDAG-FAKE16-NEXT: s_and_b32 s3, s3, 0x8000
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s3, s3, s4
; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, s0
; GFX11-SDAG-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -3263,8 +3279,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -3276,34 +3292,36 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s2, v3
; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, s3, v2
-; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s4, 0x3f1, s3
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1250-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v3, s4, 0, 13
-; GFX1250-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s5, 0x3f1, s4
+; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v2, s5, 0, 13
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s8, v2
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s3, s5, s3
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-SDAG-TRUE16-NEXT: s_addk_co_i32 s3, 0xfc10
+; GFX1250-SDAG-TRUE16-NEXT: s_addk_co_i32 s4, 0xfc10
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s5, s9, s5
-; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s4, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s3, s8
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s8, s5, 7
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -3312,65 +3330,67 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s8, s8, s9
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_add_co_i32 s5, s5, s8
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX1250-SDAG-TRUE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s8, 0x7c00
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s3, 0x40f
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s9, s5
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s4, 0x1ff
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s10, s4, 8
-; GFX1250-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s5, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s5, s4, 0xb0014
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s10, s10, 0xffe
-; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s9, 0x3f1, s5
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s4, 0x40f
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s9, s5
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s5, s3, 0x1ff
; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX1250-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v1, s9, 0, 13
+; GFX1250-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, s5, v0
; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s2, s2, 0x8000
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s3
-; GFX1250-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s11, v1
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s9, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s9, s10, s9
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s10, s9, 0x1000
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1250-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_bfe_u32 s5, s3, 0xb0014
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s12, s10, s11
-; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s11, s12, s11
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s11, s10
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_sub_co_i32 s9, 0x3f1, s5
+; GFX1250-SDAG-TRUE16-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s3, 8
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s9, s9, 0xffe
+; GFX1250-SDAG-TRUE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s4, s9, s4
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s9, s4, 0x1000
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s11, s9, s10
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s10, s11, s10
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s10, s9
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_addk_co_i32 s5, 0xfc10
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s3, s12, s3
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s9, s11, s9
; GFX1250-SDAG-TRUE16-NEXT: s_lshl_b32 s10, s5, 12
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s10, s9, s10
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s10, s4, s10
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s5, 1
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, s10
-; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s10, s3, 7
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s9, s10
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s10, s9, 7
+; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_gt_i32 s10, 5
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s11, 1, 0
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s10, 3
; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s10, 1, 0
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s9, s9, 2
; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s10, s10, s11
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: s_add_co_i32 s3, s3, s10
+; GFX1250-SDAG-TRUE16-NEXT: s_add_co_i32 s9, s9, s10
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lt_i32 s5, 31
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s9, 0
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s8, s8, 0x7c00
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s9, s9, 0x7c00
+; GFX1250-SDAG-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
; GFX1250-SDAG-TRUE16-NEXT: s_cmp_eq_u32 s5, 0x40f
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s3, s8, s3
-; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s4, s4, 16
+; GFX1250-SDAG-TRUE16-NEXT: s_cselect_b32 s4, s4, s9
+; GFX1250-SDAG-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s4, s4, 0x8000
-; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s3, s4, s3
+; GFX1250-SDAG-TRUE16-NEXT: s_and_b32 s3, s3, 0x8000
+; GFX1250-SDAG-TRUE16-NEXT: s_or_b32 s3, s3, s4
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s4, s0
; GFX1250-SDAG-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -3380,8 +3400,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -3393,34 +3413,36 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s2, v3
; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s3, s2, 0x1ff
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_or_b32_e32 v2, s3, v2
-; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s3, s2, 0xb0014
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s4, 0x3f1, s3
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1250-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v3, s4, 0, 13
-; GFX1250-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v3
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v2
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s4, 0x1000
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s3, vcc_lo, exec_lo
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s4, s2, 0xb0014
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s5, 0x3f1, s4
+; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v2, s5, 0, 13
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s5, 0xffe
+; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s8, v2
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s3, s5, s3
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s3, 0x1000
; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s5, s8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s9, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s8, s5
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-SDAG-FAKE16-NEXT: s_addk_co_i32 s3, 0xfc10
+; GFX1250-SDAG-FAKE16-NEXT: s_addk_co_i32 s4, 0xfc10
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s5, s9, s5
-; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s3, 12
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s4, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 1
+; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s8, s4, 12
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s3, s8
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 1
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s8, s5, 7
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s8, 5
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s8, 3
@@ -3429,65 +3451,67 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s8, s8, s9
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_add_co_i32 s5, s5, s8
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s3, 31
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s4, 31
; GFX1250-SDAG-FAKE16-NEXT: s_movk_i32 s8, 0x7e00
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s3, 0
+; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s3, v1
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s8, 0x7c00
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s3, 0x40f
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s9, s5
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s4, 0x1ff
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s10, s4, 8
-; GFX1250-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s5, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s5, s4, 0xb0014
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s10, s10, 0xffe
-; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s9, 0x3f1, s5
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s4, 0x40f
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s9, s5
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s5, s3, 0x1ff
; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s2, s2, 16
-; GFX1250-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v1, s9, 0, 13
+; GFX1250-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, s5, v0
; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s2, s2, 0x8000
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s3
-; GFX1250-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s11, v1
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s9, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s9, s10, s9
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s10, s9, 0x1000
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1250-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_bfe_u32 s5, s3, 0xb0014
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s12, s10, s11
-; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s11, s12, s11
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s11, s10
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_sub_co_i32 s9, 0x3f1, s5
+; GFX1250-SDAG-FAKE16-NEXT: v_med3_i32 v0, s9, 0, 13
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s3, 8
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s9, s9, 0xffe
+; GFX1250-SDAG-FAKE16-NEXT: v_readfirstlane_b32 s10, v0
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s4, s9, s4
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s9, s4, 0x1000
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s11, s9, s10
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s10, s11, s10
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s10, s9
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, 1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_addk_co_i32 s5, 0xfc10
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s3, s12, s3
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s9, s11, s9
; GFX1250-SDAG-FAKE16-NEXT: s_lshl_b32 s10, s5, 12
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s10, s9, s10
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s10, s4, s10
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s5, 1
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, s10
-; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s10, s3, 7
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s9, s10
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s10, s9, 7
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_gt_i32 s10, 5
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s11, 1, 0
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s10, 3
; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s10, 1, 0
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s3, s3, 2
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s9, s9, 2
; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s10, s10, s11
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: s_add_co_i32 s3, s3, s10
+; GFX1250-SDAG-FAKE16-NEXT: s_add_co_i32 s9, s9, s10
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lt_i32 s5, 31
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s9, 0
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s8, s8, 0x7c00
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s9, s9, 0x7c00
+; GFX1250-SDAG-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s8, 0x7c00
; GFX1250-SDAG-FAKE16-NEXT: s_cmp_eq_u32 s5, 0x40f
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s3, s8, s3
-; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s4, s4, 16
+; GFX1250-SDAG-FAKE16-NEXT: s_cselect_b32 s4, s4, s9
+; GFX1250-SDAG-FAKE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s4, s4, 0x8000
-; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s3, s4, s3
+; GFX1250-SDAG-FAKE16-NEXT: s_and_b32 s3, s3, 0x8000
+; GFX1250-SDAG-FAKE16-NEXT: s_or_b32 s3, s3, s4
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s4, s0
; GFX1250-SDAG-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s3, s2
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -3497,8 +3521,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
@@ -3597,8 +3621,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f64_to_v2f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
@@ -3938,8 +3962,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -3960,8 +3984,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -3982,8 +4006,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
@@ -3999,8 +4023,8 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_v2f64_to_v2f16_afn:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[4:7], s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
@@ -4009,7 +4033,7 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v0, s[4:5]
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v1, s[6:7]
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v1, v1
@@ -4209,8 +4233,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -4223,7 +4247,7 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4231,8 +4255,8 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -4245,7 +4269,7 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_xor_b32_e32 v0, 0x80000000, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4253,15 +4277,15 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_xor_b32 s2, s2, 0x80000000
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -4271,15 +4295,15 @@ define amdgpu_kernel void @fneg_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fneg_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_xor_b32 s2, s2, 0x80000000
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -4479,8 +4503,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -4493,7 +4517,7 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4501,8 +4525,8 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -4515,7 +4539,7 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4523,15 +4547,15 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_bitset0_b32 s2, 31
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -4541,15 +4565,15 @@ define amdgpu_kernel void @fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_bitset0_b32 s2, 31
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -4749,8 +4773,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -4763,7 +4787,7 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_or_b32_e32 v0, 0x80000000, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4771,8 +4795,8 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-SDAG-FAKE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -4785,7 +4809,7 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, 0x80000000, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
@@ -4793,15 +4817,15 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-TRUE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_bitset1_b32 s2, 31
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
@@ -4811,15 +4835,15 @@ define amdgpu_kernel void @fneg_fabs_fptrunc_f32_to_f16(
;
; GFX1250-GISEL-FAKE16-LABEL: fneg_fabs_fptrunc_f32_to_f16:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_bitset1_b32 s2, 31
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
@@ -5027,8 +5051,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -5040,7 +5064,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX1250-SDAG-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -5049,8 +5073,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -5062,7 +5086,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
; GFX1250-SDAG-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -5071,14 +5095,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
@@ -5089,14 +5113,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_zext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-FAKE16-NEXT: s_and_b32 s2, 0xffff, s2
@@ -5305,8 +5329,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -5319,7 +5343,7 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -5328,8 +5352,8 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -5342,7 +5366,7 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -5351,15 +5375,15 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_bitset0_b32 s2, 31
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-TRUE16-NEXT: s_and_b32 s2, 0xffff, s2
@@ -5370,15 +5394,15 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_bitset0_b32 s2, 31
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-FAKE16-NEXT: s_and_b32 s2, 0xffff, s2
@@ -5597,8 +5621,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-SDAG-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-SDAG-TRUE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s10, s6
@@ -5610,7 +5634,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX1250-SDAG-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
@@ -5619,8 +5643,8 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-SDAG-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-SDAG-FAKE16: ; %bb.0: ; %entry
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s6, -1
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s10, s6
@@ -5632,7 +5656,7 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
; GFX1250-SDAG-FAKE16-NEXT: buffer_load_b32 v0, off, s[8:11], null
; GFX1250-SDAG-FAKE16-NEXT: s_mov_b32 s5, s1
; GFX1250-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
@@ -5641,14 +5665,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-GISEL-TRUE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-TRUE16-NEXT: s_sext_i32_i16 s2, s2
@@ -5659,14 +5683,14 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_sext_i32(
;
; GFX1250-GISEL-FAKE16-LABEL: fptrunc_f32_to_f16_sext_i32:
; GFX1250-GISEL-FAKE16: ; %bb.0: ; %entry
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_load_b32 s2, s[2:3], 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1250-GISEL-FAKE16-NEXT: s_sext_i32_i16 s2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index 0a2c59668714d..54a4b4bb1b6c3 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -183,23 +183,23 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; SI-NEXT: s_and_b32 s8, s0, 0xffe
; SI-NEXT: s_or_b32 s0, s1, s6
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: s_bfe_u32 s0, s7, 0xb0014
-; SI-NEXT: v_readfirstlane_b32 s1, v0
-; SI-NEXT: s_sub_i32 s6, 0x3f1, s0
-; SI-NEXT: s_or_b32 s1, s8, s1
+; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_bfe_u32 s1, s7, 0xb0014
+; SI-NEXT: s_or_b32 s0, s8, s0
+; SI-NEXT: s_sub_i32 s6, 0x3f1, s1
+; SI-NEXT: s_or_b32 s8, s0, 0x1000
; SI-NEXT: v_med3_i32 v0, s6, 0, 13
-; SI-NEXT: s_or_b32 s6, s1, 0x1000
-; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_lshr_b32 s9, s6, s8
-; SI-NEXT: s_lshl_b32 s8, s9, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s6
+; SI-NEXT: v_readfirstlane_b32 s6, v0
+; SI-NEXT: s_lshr_b32 s9, s8, s6
+; SI-NEXT: s_lshl_b32 s6, s9, s6
+; SI-NEXT: s_cmp_lg_u32 s6, s8
; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_addk_i32 s0, 0xfc10
+; SI-NEXT: s_addk_i32 s1, 0xfc10
; SI-NEXT: s_or_b32 s6, s9, s6
-; SI-NEXT: s_lshl_b32 s8, s0, 12
-; SI-NEXT: s_or_b32 s8, s1, s8
-; SI-NEXT: s_cmp_lt_i32 s0, 1
+; SI-NEXT: s_lshl_b32 s8, s1, 12
+; SI-NEXT: s_or_b32 s8, s0, s8
+; SI-NEXT: s_cmp_lt_i32 s1, 1
; SI-NEXT: s_cselect_b32 s6, s6, s8
; SI-NEXT: s_and_b32 s8, s6, 7
; SI-NEXT: s_cmp_gt_i32 s8, 5
@@ -209,12 +209,12 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; SI-NEXT: s_lshr_b32 s6, s6, 2
; SI-NEXT: s_or_b32 s8, s8, s9
; SI-NEXT: s_add_i32 s6, s6, s8
-; SI-NEXT: s_cmp_lt_i32 s0, 31
+; SI-NEXT: s_cmp_lt_i32 s1, 31
; SI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s1, 0
-; SI-NEXT: s_cselect_b32 s1, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s0, 0x40f
-; SI-NEXT: s_cselect_b32 s0, s1, s6
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s1, 0x40f
+; SI-NEXT: s_cselect_b32 s0, s0, s6
; SI-NEXT: s_lshr_b32 s1, s7, 16
; SI-NEXT: s_and_b32 s1, s1, 0x8000
; SI-NEXT: s_or_b32 s6, s1, s0
@@ -238,8 +238,8 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; VI-SDAG-NEXT: s_or_b32 s4, s4, s6
; VI-SDAG-NEXT: s_mov_b32 s1, s5
; VI-SDAG-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-SDAG-NEXT: v_readfirstlane_b32 s4, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-SDAG-NEXT: s_cselect_b32 s4, 1, 0
; VI-SDAG-NEXT: s_bfe_u32 s6, s7, 0xb0014
; VI-SDAG-NEXT: s_or_b32 s4, s8, s4
; VI-SDAG-NEXT: s_sub_i32 s8, 0x3f1, s6
@@ -338,39 +338,39 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; GFX10-SDAG-NEXT: s_and_b32 s4, s4, 0xffe
; GFX10-SDAG-NEXT: s_or_b32 s2, s5, s2
; GFX10-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
-; GFX10-SDAG-NEXT: s_bfe_u32 s2, s3, 0xb0014
-; GFX10-SDAG-NEXT: s_sub_i32 s5, 0x3f1, s2
-; GFX10-SDAG-NEXT: v_med3_i32 v1, s5, 0, 13
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; GFX10-SDAG-NEXT: s_or_b32 s4, s4, s5
-; GFX10-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
-; GFX10-SDAG-NEXT: s_lshr_b32 s7, s5, s6
+; GFX10-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX10-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX10-SDAG-NEXT: s_bfe_u32 s5, s3, 0xb0014
+; GFX10-SDAG-NEXT: s_or_b32 s2, s4, s2
+; GFX10-SDAG-NEXT: s_sub_i32 s6, 0x3f1, s5
+; GFX10-SDAG-NEXT: s_or_b32 s4, s2, 0x1000
+; GFX10-SDAG-NEXT: v_med3_i32 v0, s6, 0, 13
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX10-SDAG-NEXT: s_lshr_b32 s7, s4, s6
; GFX10-SDAG-NEXT: s_lshl_b32 s6, s7, s6
-; GFX10-SDAG-NEXT: s_cmp_lg_u32 s6, s5
-; GFX10-SDAG-NEXT: s_cselect_b32 s5, 1, 0
-; GFX10-SDAG-NEXT: s_addk_i32 s2, 0xfc10
-; GFX10-SDAG-NEXT: s_or_b32 s5, s7, s5
-; GFX10-SDAG-NEXT: s_lshl_b32 s6, s2, 12
-; GFX10-SDAG-NEXT: s_or_b32 s6, s4, s6
-; GFX10-SDAG-NEXT: s_cmp_lt_i32 s2, 1
-; GFX10-SDAG-NEXT: s_cselect_b32 s5, s5, s6
-; GFX10-SDAG-NEXT: s_and_b32 s6, s5, 7
+; GFX10-SDAG-NEXT: s_cmp_lg_u32 s6, s4
+; GFX10-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-SDAG-NEXT: s_addk_i32 s5, 0xfc10
+; GFX10-SDAG-NEXT: s_or_b32 s4, s7, s4
+; GFX10-SDAG-NEXT: s_lshl_b32 s6, s5, 12
+; GFX10-SDAG-NEXT: s_or_b32 s6, s2, s6
+; GFX10-SDAG-NEXT: s_cmp_lt_i32 s5, 1
+; GFX10-SDAG-NEXT: s_cselect_b32 s4, s4, s6
+; GFX10-SDAG-NEXT: s_and_b32 s6, s4, 7
; GFX10-SDAG-NEXT: s_cmp_gt_i32 s6, 5
; GFX10-SDAG-NEXT: s_cselect_b32 s7, 1, 0
; GFX10-SDAG-NEXT: s_cmp_eq_u32 s6, 3
; GFX10-SDAG-NEXT: s_cselect_b32 s6, 1, 0
-; GFX10-SDAG-NEXT: s_lshr_b32 s5, s5, 2
+; GFX10-SDAG-NEXT: s_lshr_b32 s4, s4, 2
; GFX10-SDAG-NEXT: s_or_b32 s6, s6, s7
-; GFX10-SDAG-NEXT: s_add_i32 s5, s5, s6
-; GFX10-SDAG-NEXT: s_cmp_lt_i32 s2, 31
+; GFX10-SDAG-NEXT: s_add_i32 s4, s4, s6
+; GFX10-SDAG-NEXT: s_cmp_lt_i32 s5, 31
; GFX10-SDAG-NEXT: s_movk_i32 s6, 0x7e00
-; GFX10-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX10-SDAG-NEXT: s_cmp_lg_u32 s4, 0
-; GFX10-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
-; GFX10-SDAG-NEXT: s_cmpk_eq_i32 s2, 0x40f
-; GFX10-SDAG-NEXT: s_cselect_b32 s2, s4, s5
+; GFX10-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
+; GFX10-SDAG-NEXT: s_cmp_lg_u32 s2, 0
+; GFX10-SDAG-NEXT: s_cselect_b32 s2, s6, 0x7c00
+; GFX10-SDAG-NEXT: s_cmpk_eq_i32 s5, 0x40f
+; GFX10-SDAG-NEXT: s_cselect_b32 s2, s2, s4
; GFX10-SDAG-NEXT: s_lshr_b32 s3, s3, 16
; GFX10-SDAG-NEXT: s_and_b32 s3, s3, 0x8000
; GFX10-SDAG-NEXT: s_or_b32 s2, s3, s2
@@ -440,46 +440,44 @@ define amdgpu_kernel void @fptrunc_f64_to_f16(ptr addrspace(1) %out, double %in)
; GFX11-SDAG-NEXT: s_and_b32 s4, s4, 0xffe
; GFX11-SDAG-NEXT: s_or_b32 s2, s5, s2
; GFX11-SDAG-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
-; GFX11-SDAG-NEXT: s_bfe_u32 s2, s3, 0xb0014
-; GFX11-SDAG-NEXT: s_sub_i32 s5, 0x3f1, s2
-; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_med3_i32 v1, s5, 0, 13
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-SDAG-NEXT: s_or_b32 s4, s4, s5
-; GFX11-SDAG-NEXT: s_or_b32 s5, s4, 0x1000
-; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_lshr_b32 s7, s5, s6
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-SDAG-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-SDAG-NEXT: s_bfe_u32 s5, s3, 0xb0014
+; GFX11-SDAG-NEXT: s_or_b32 s2, s4, s2
+; GFX11-SDAG-NEXT: s_sub_i32 s6, 0x3f1, s5
+; GFX11-SDAG-NEXT: s_or_b32 s4, s2, 0x1000
+; GFX11-SDAG-NEXT: v_med3_i32 v0, s6, 0, 13
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s6, v0
+; GFX11-SDAG-NEXT: s_lshr_b32 s7, s4, s6
; GFX11-SDAG-NEXT: s_lshl_b32 s6, s7, s6
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s6, s5
-; GFX11-SDAG-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-SDAG-NEXT: s_addk_i32 s2, 0xfc10
-; GFX11-SDAG-NEXT: s_or_b32 s5, s7, s5
-; GFX11-SDAG-NEXT: s_lshl_b32 s6, s2, 12
-; GFX11-SDAG-NEXT: s_or_b32 s6, s4, s6
-; GFX11-SDAG-NEXT: s_cmp_lt_i32 s2, 1
-; GFX11-SDAG-NEXT: s_cselect_b32 s5, s5, s6
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s6, s4
+; GFX11-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-SDAG-NEXT: s_addk_i32 s5, 0xfc10
+; GFX11-SDAG-NEXT: s_or_b32 s4, s7, s4
+; GFX11-SDAG-NEXT: s_lshl_b32 s6, s5, 12
+; GFX11-SDAG-NEXT: s_or_b32 s6, s2, s6
+; GFX11-SDAG-NEXT: s_cmp_lt_i32 s5, 1
+; GFX11-SDAG-NEXT: s_cselect_b32 s4, s4, s6
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_and_b32 s6, s5, 7
+; GFX11-SDAG-NEXT: s_and_b32 s6, s4, 7
; GFX11-SDAG-NEXT: s_cmp_gt_i32 s6, 5
; GFX11-SDAG-NEXT: s_cselect_b32 s7, 1, 0
; GFX11-SDAG-NEXT: s_cmp_eq_u32 s6, 3
; GFX11-SDAG-NEXT: s_cselect_b32 s6, 1, 0
-; GFX11-SDAG-NEXT: s_lshr_b32 s5, s5, 2
+; GFX11-SDAG-NEXT: s_lshr_b32 s4, s4, 2
; GFX11-SDAG-NEXT: s_or_b32 s6, s6, s7
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_add_i32 s5, s5, s6
-; GFX11-SDAG-NEXT: s_cmp_lt_i32 s2, 31
+; GFX11-SDAG-NEXT: s_add_i32 s4, s4, s6
+; GFX11-SDAG-NEXT: s_cmp_lt_i32 s5, 31
; GFX11-SDAG-NEXT: s_movk_i32 s6, 0x7e00
-; GFX11-SDAG-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-NEXT: s_cselect_b32 s4, s6, 0x7c00
-; GFX11-SDAG-NEXT: s_cmpk_eq_i32 s2, 0x40f
-; GFX11-SDAG-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-SDAG-NEXT: s_cselect_b32 s4, s4, 0x7c00
+; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-SDAG-NEXT: s_cselect_b32 s2, s6, 0x7c00
+; GFX11-SDAG-NEXT: s_cmpk_eq_i32 s5, 0x40f
+; GFX11-SDAG-NEXT: s_cselect_b32 s2, s2, s4
; GFX11-SDAG-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-NEXT: s_and_b32 s3, s3, 0x8000
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index cab9bc15fdf63..68351942428b8 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx600 < %s | FileCheck -check-prefixes=GFX6,GFX6-SDAG %s
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx600 < %s | FileCheck -check-prefixes=GFX6,GFX6-GISEL %s
@@ -13695,12 +13695,14 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-SDAG-NEXT: s_cmp_eq_u32 s16, s17
-; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
; GFX6-SDAG-NEXT: s_cselect_b64 s[8:9], -1, 0
+; GFX6-SDAG-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX6-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT: s_cselect_b32 s8, 1, 0
; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX6-SDAG-NEXT: s_mov_b32 s4, s6
; GFX6-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX6-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[8:9]
+; GFX6-SDAG-NEXT: v_mov_b32_e32 v2, s8
; GFX6-SDAG-NEXT: buffer_store_byte v2, v[0:1], s[4:7], 0 addr64
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -13725,12 +13727,14 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX7-SDAG: ; %bb.0:
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: s_cmp_eq_u32 s16, s17
-; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
; GFX7-SDAG-NEXT: s_cselect_b64 s[8:9], -1, 0
+; GFX7-SDAG-NEXT: s_and_b64 s[8:9], s[8:9], exec
+; GFX7-SDAG-NEXT: s_mov_b32 s6, 0
+; GFX7-SDAG-NEXT: s_cselect_b32 s8, 1, 0
; GFX7-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX7-SDAG-NEXT: s_mov_b32 s4, s6
; GFX7-SDAG-NEXT: s_mov_b32 s5, s6
-; GFX7-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[8:9]
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v2, s8
; GFX7-SDAG-NEXT: buffer_store_byte v2, v[0:1], s[4:7], 0 addr64
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -13782,7 +13786,9 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: s_cmp_eq_u32 s16, s17
; GFX10-SDAG-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10-SDAG-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX10-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX10-SDAG-NEXT: global_store_byte v[0:1], v2, off
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -13803,7 +13809,9 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; GFX11-SDAG-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX11-SDAG-NEXT: global_store_b8 v[0:1], v2, off
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index bf153a88982e0..ae6a8dad78225 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mattr=+mad-mac-f32-insts < %s | FileCheck --check-prefix=SI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck --check-prefix=CI %s
; RUN: llc -amdgpu-scalarize-global-loads=false -enable-misched=0 -mtriple=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=VI %s
@@ -1269,48 +1269,48 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; SI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
+; SI-NEXT: v_readfirstlane_b32 s6, v1
+; SI-NEXT: s_and_b32 s4, s6, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_lshr_b32 s5, s4, 8
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s6, 8
; SI-NEXT: s_and_b32 s5, s5, 0xffe
-; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_or_b32 s6, s5, 0x1000
-; SI-NEXT: s_bfe_u32 s7, s4, 0xb0014
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: s_bfe_u32 s7, s6, 0xb0014
; SI-NEXT: s_sub_i32 s8, 0x3f1, s7
; SI-NEXT: v_med3_i32 v0, s8, 0, 13
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_lshr_b32 s9, s6, s8
+; SI-NEXT: s_lshr_b32 s9, s5, s8
; SI-NEXT: s_lshl_b32 s8, s9, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s6
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_or_b32 s6, s9, s6
+; SI-NEXT: s_cmp_lg_u32 s8, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_or_b32 s5, s9, s5
; SI-NEXT: s_addk_i32 s7, 0xfc10
; SI-NEXT: s_lshl_b32 s8, s7, 12
-; SI-NEXT: s_or_b32 s8, s5, s8
+; SI-NEXT: s_or_b32 s8, s4, s8
; SI-NEXT: s_cmp_lt_i32 s7, 1
-; SI-NEXT: s_cselect_b32 s6, s6, s8
-; SI-NEXT: s_and_b32 s8, s6, 7
+; SI-NEXT: s_cselect_b32 s5, s5, s8
+; SI-NEXT: s_and_b32 s8, s5, 7
; SI-NEXT: s_cmp_gt_i32 s8, 5
; SI-NEXT: s_cselect_b32 s9, 1, 0
; SI-NEXT: s_cmp_eq_u32 s8, 3
; SI-NEXT: s_cselect_b32 s8, 1, 0
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_lshr_b32 s6, s6, 2
-; SI-NEXT: s_add_i32 s6, s6, s8
+; SI-NEXT: s_lshr_b32 s5, s5, 2
+; SI-NEXT: s_add_i32 s5, s5, s8
; SI-NEXT: s_cmp_lt_i32 s7, 31
-; SI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_movk_i32 s5, 0x7e00
; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_movk_i32 s4, 0x7e00
+; SI-NEXT: s_cselect_b32 s4, s4, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s6
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s6, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1359,48 +1359,48 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; CI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; CI-NEXT: v_readfirstlane_b32 s4, v1
-; CI-NEXT: s_and_b32 s5, s4, 0x1ff
-; CI-NEXT: v_or_b32_e32 v0, s5, v0
+; CI-NEXT: v_readfirstlane_b32 s6, v1
+; CI-NEXT: s_and_b32 s4, s6, 0x1ff
+; CI-NEXT: v_or_b32_e32 v0, s4, v0
; CI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-NEXT: s_lshr_b32 s5, s4, 8
-; CI-NEXT: s_bfe_u32 s7, s4, 0xb0014
+; CI-NEXT: s_and_b64 s[4:5], vcc, exec
+; CI-NEXT: s_cselect_b32 s4, 1, 0
+; CI-NEXT: s_lshr_b32 s5, s6, 8
+; CI-NEXT: s_bfe_u32 s7, s6, 0xb0014
; CI-NEXT: s_and_b32 s5, s5, 0xffe
-; CI-NEXT: v_readfirstlane_b32 s6, v0
; CI-NEXT: s_sub_i32 s8, 0x3f1, s7
-; CI-NEXT: s_or_b32 s5, s5, s6
+; CI-NEXT: s_or_b32 s4, s5, s4
; CI-NEXT: v_med3_i32 v0, s8, 0, 13
-; CI-NEXT: s_or_b32 s6, s5, 0x1000
+; CI-NEXT: s_or_b32 s5, s4, 0x1000
; CI-NEXT: v_readfirstlane_b32 s8, v0
-; CI-NEXT: s_lshr_b32 s9, s6, s8
+; CI-NEXT: s_lshr_b32 s9, s5, s8
; CI-NEXT: s_lshl_b32 s8, s9, s8
-; CI-NEXT: s_cmp_lg_u32 s8, s6
-; CI-NEXT: s_cselect_b32 s6, 1, 0
+; CI-NEXT: s_cmp_lg_u32 s8, s5
+; CI-NEXT: s_cselect_b32 s5, 1, 0
; CI-NEXT: s_addk_i32 s7, 0xfc10
; CI-NEXT: s_lshl_b32 s8, s7, 12
-; CI-NEXT: s_or_b32 s6, s9, s6
-; CI-NEXT: s_or_b32 s8, s5, s8
+; CI-NEXT: s_or_b32 s5, s9, s5
+; CI-NEXT: s_or_b32 s8, s4, s8
; CI-NEXT: s_cmp_lt_i32 s7, 1
-; CI-NEXT: s_cselect_b32 s6, s6, s8
-; CI-NEXT: s_and_b32 s8, s6, 7
+; CI-NEXT: s_cselect_b32 s5, s5, s8
+; CI-NEXT: s_and_b32 s8, s5, 7
; CI-NEXT: s_cmp_gt_i32 s8, 5
; CI-NEXT: s_cselect_b32 s9, 1, 0
; CI-NEXT: s_cmp_eq_u32 s8, 3
; CI-NEXT: s_cselect_b32 s8, 1, 0
; CI-NEXT: s_or_b32 s8, s8, s9
-; CI-NEXT: s_lshr_b32 s6, s6, 2
-; CI-NEXT: s_add_i32 s6, s6, s8
+; CI-NEXT: s_lshr_b32 s5, s5, 2
+; CI-NEXT: s_add_i32 s5, s5, s8
; CI-NEXT: s_cmp_lt_i32 s7, 31
-; CI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; CI-NEXT: s_cmp_lg_u32 s5, 0
-; CI-NEXT: s_movk_i32 s5, 0x7e00
; CI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; CI-NEXT: s_cmp_lg_u32 s4, 0
+; CI-NEXT: s_movk_i32 s4, 0x7e00
+; CI-NEXT: s_cselect_b32 s4, s4, 0x7c00
; CI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; CI-NEXT: s_cselect_b32 s5, s5, s6
-; CI-NEXT: s_lshr_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s4, s4, 0x8000
-; CI-NEXT: s_or_b32 s4, s4, s5
+; CI-NEXT: s_cselect_b32 s4, s4, s5
+; CI-NEXT: s_lshr_b32 s5, s6, 16
+; CI-NEXT: s_and_b32 s5, s5, 0x8000
+; CI-NEXT: s_or_b32 s4, s5, s4
; CI-NEXT: v_mov_b32_e32 v0, s4
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
@@ -1753,48 +1753,48 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; SI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
+; SI-NEXT: v_readfirstlane_b32 s6, v1
+; SI-NEXT: s_and_b32 s4, s6, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_lshr_b32 s5, s4, 8
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s6, 8
; SI-NEXT: s_and_b32 s5, s5, 0xffe
-; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_or_b32 s6, s5, 0x1000
-; SI-NEXT: s_bfe_u32 s7, s4, 0xb0014
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: s_bfe_u32 s7, s6, 0xb0014
; SI-NEXT: s_sub_i32 s8, 0x3f1, s7
; SI-NEXT: v_med3_i32 v0, s8, 0, 13
; SI-NEXT: v_readfirstlane_b32 s8, v0
-; SI-NEXT: s_lshr_b32 s9, s6, s8
+; SI-NEXT: s_lshr_b32 s9, s5, s8
; SI-NEXT: s_lshl_b32 s8, s9, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s6
-; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_or_b32 s6, s9, s6
+; SI-NEXT: s_cmp_lg_u32 s8, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_or_b32 s5, s9, s5
; SI-NEXT: s_addk_i32 s7, 0xfc10
; SI-NEXT: s_lshl_b32 s8, s7, 12
-; SI-NEXT: s_or_b32 s8, s5, s8
+; SI-NEXT: s_or_b32 s8, s4, s8
; SI-NEXT: s_cmp_lt_i32 s7, 1
-; SI-NEXT: s_cselect_b32 s6, s6, s8
-; SI-NEXT: s_and_b32 s8, s6, 7
+; SI-NEXT: s_cselect_b32 s5, s5, s8
+; SI-NEXT: s_and_b32 s8, s5, 7
; SI-NEXT: s_cmp_gt_i32 s8, 5
; SI-NEXT: s_cselect_b32 s9, 1, 0
; SI-NEXT: s_cmp_eq_u32 s8, 3
; SI-NEXT: s_cselect_b32 s8, 1, 0
; SI-NEXT: s_or_b32 s8, s8, s9
-; SI-NEXT: s_lshr_b32 s6, s6, 2
-; SI-NEXT: s_add_i32 s6, s6, s8
+; SI-NEXT: s_lshr_b32 s5, s5, 2
+; SI-NEXT: s_add_i32 s5, s5, s8
; SI-NEXT: s_cmp_lt_i32 s7, 31
-; SI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_movk_i32 s5, 0x7e00
; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_movk_i32 s4, 0x7e00
+; SI-NEXT: s_cselect_b32 s4, s4, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s6
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s6, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1843,48 +1843,48 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; CI-NEXT: v_fma_f64 v[0:1], v[0:1], v[4:5], v[2:3]
-; CI-NEXT: v_readfirstlane_b32 s4, v1
-; CI-NEXT: s_and_b32 s5, s4, 0x1ff
-; CI-NEXT: v_or_b32_e32 v0, s5, v0
+; CI-NEXT: v_readfirstlane_b32 s6, v1
+; CI-NEXT: s_and_b32 s4, s6, 0x1ff
+; CI-NEXT: v_or_b32_e32 v0, s4, v0
; CI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; CI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; CI-NEXT: s_lshr_b32 s5, s4, 8
-; CI-NEXT: s_bfe_u32 s7, s4, 0xb0014
+; CI-NEXT: s_and_b64 s[4:5], vcc, exec
+; CI-NEXT: s_cselect_b32 s4, 1, 0
+; CI-NEXT: s_lshr_b32 s5, s6, 8
+; CI-NEXT: s_bfe_u32 s7, s6, 0xb0014
; CI-NEXT: s_and_b32 s5, s5, 0xffe
-; CI-NEXT: v_readfirstlane_b32 s6, v0
; CI-NEXT: s_sub_i32 s8, 0x3f1, s7
-; CI-NEXT: s_or_b32 s5, s5, s6
+; CI-NEXT: s_or_b32 s4, s5, s4
; CI-NEXT: v_med3_i32 v0, s8, 0, 13
-; CI-NEXT: s_or_b32 s6, s5, 0x1000
+; CI-NEXT: s_or_b32 s5, s4, 0x1000
; CI-NEXT: v_readfirstlane_b32 s8, v0
-; CI-NEXT: s_lshr_b32 s9, s6, s8
+; CI-NEXT: s_lshr_b32 s9, s5, s8
; CI-NEXT: s_lshl_b32 s8, s9, s8
-; CI-NEXT: s_cmp_lg_u32 s8, s6
-; CI-NEXT: s_cselect_b32 s6, 1, 0
+; CI-NEXT: s_cmp_lg_u32 s8, s5
+; CI-NEXT: s_cselect_b32 s5, 1, 0
; CI-NEXT: s_addk_i32 s7, 0xfc10
; CI-NEXT: s_lshl_b32 s8, s7, 12
-; CI-NEXT: s_or_b32 s6, s9, s6
-; CI-NEXT: s_or_b32 s8, s5, s8
+; CI-NEXT: s_or_b32 s5, s9, s5
+; CI-NEXT: s_or_b32 s8, s4, s8
; CI-NEXT: s_cmp_lt_i32 s7, 1
-; CI-NEXT: s_cselect_b32 s6, s6, s8
-; CI-NEXT: s_and_b32 s8, s6, 7
+; CI-NEXT: s_cselect_b32 s5, s5, s8
+; CI-NEXT: s_and_b32 s8, s5, 7
; CI-NEXT: s_cmp_gt_i32 s8, 5
; CI-NEXT: s_cselect_b32 s9, 1, 0
; CI-NEXT: s_cmp_eq_u32 s8, 3
; CI-NEXT: s_cselect_b32 s8, 1, 0
; CI-NEXT: s_or_b32 s8, s8, s9
-; CI-NEXT: s_lshr_b32 s6, s6, 2
-; CI-NEXT: s_add_i32 s6, s6, s8
+; CI-NEXT: s_lshr_b32 s5, s5, 2
+; CI-NEXT: s_add_i32 s5, s5, s8
; CI-NEXT: s_cmp_lt_i32 s7, 31
-; CI-NEXT: s_cselect_b32 s6, s6, 0x7c00
-; CI-NEXT: s_cmp_lg_u32 s5, 0
-; CI-NEXT: s_movk_i32 s5, 0x7e00
; CI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; CI-NEXT: s_cmp_lg_u32 s4, 0
+; CI-NEXT: s_movk_i32 s4, 0x7e00
+; CI-NEXT: s_cselect_b32 s4, s4, 0x7c00
; CI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; CI-NEXT: s_cselect_b32 s5, s5, s6
-; CI-NEXT: s_lshr_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s4, s4, 0x8000
-; CI-NEXT: s_or_b32 s4, s4, s5
+; CI-NEXT: s_cselect_b32 s4, s4, s5
+; CI-NEXT: s_lshr_b32 s5, s6, 16
+; CI-NEXT: s_and_b32 s5, s5, 0x8000
+; CI-NEXT: s_or_b32 s4, s5, s4
; CI-NEXT: v_mov_b32_e32 v0, s4
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll b/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
index 872c2cf569dcc..301613420d607 100644
--- a/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/gep-const-address-space.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -amdgpu-atomic-optimizer-strategy=None < %s | FileCheck %s
define protected amdgpu_kernel void @IllegalGEPConst(i32 %a, ptr addrspace(1) %b, double %c) {
@@ -22,9 +22,11 @@ define protected amdgpu_kernel void @IllegalGEPConst(i32 %a, ptr addrspace(1) %b
; CHECK-NEXT: s_addc_u32 s1, s1, -1
; CHECK-NEXT: s_cmp_eq_u32 s1, s7
; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s4, 1
; CHECK-NEXT: s_mov_b64 s[4:5], -1
-; CHECK-NEXT: s_cbranch_vccnz .LBB0_3
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_3
; CHECK-NEXT: ; %bb.1: ; %Flow6
; CHECK-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; CHECK-NEXT: s_cbranch_vccz .LBB0_8
@@ -34,9 +36,11 @@ define protected amdgpu_kernel void @IllegalGEPConst(i32 %a, ptr addrspace(1) %b
; CHECK-NEXT: s_mov_b64 s[4:5], src_private_base
; CHECK-NEXT: s_cmp_eq_u32 s1, s5
; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; CHECK-NEXT: s_andn2_b64 vcc, exec, s[4:5]
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s4, 1
; CHECK-NEXT: s_mov_b64 s[4:5], -1
-; CHECK-NEXT: s_cbranch_vccz .LBB0_5
+; CHECK-NEXT: s_cbranch_scc0 .LBB0_5
; CHECK-NEXT: ; %bb.4: ; %atomicrmw.global
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
index 85549b86a9a6c..1edc47fe9b5e9 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefixes=GFX908_GFX11_GFX12,GFX908 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefixes=GFX90A_GFX942 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefixes=GFX90A_GFX942 %s
@@ -175,7 +175,7 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11_GFX12-NEXT: bb.1 (%ir-block.5):
; GFX11_GFX12-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
+ ; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $exec_lo
; GFX11_GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11_GFX12-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
index aede3928d7b0f..38c41dde0bf9c 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefixes=GFX90A_GFX942,GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx942 -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefixes=GFX90A_GFX942,GFX942 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -stop-after=amdgpu-isel -amdgpu-atomic-optimizer-strategy=DPP < %s | FileCheck -check-prefix=GFX11 %s
@@ -223,7 +223,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX11-NEXT: bb.1 (%ir-block.5):
; GFX11-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $exec_lo
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index 103ca48a7dc5f..db365de33aad0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefix=GFX7LESS %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
@@ -964,6 +964,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -981,7 +982,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -1088,6 +1089,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -1110,11 +1112,10 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1260,27 +1261,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1327,11 +1328,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -1480,27 +1482,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1547,11 +1549,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -2177,6 +2180,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -2194,7 +2198,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -2301,6 +2305,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -2323,11 +2328,10 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2473,27 +2477,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2555,12 +2559,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -2723,27 +2728,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2805,12 +2810,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -3450,6 +3456,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -3467,7 +3474,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -3574,6 +3581,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -3596,11 +3604,10 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4219,6 +4226,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -4236,7 +4244,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -4343,6 +4351,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -4365,11 +4374,10 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4515,27 +4523,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4597,12 +4605,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -4765,27 +4774,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4847,12 +4856,13 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -5517,6 +5527,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -5534,7 +5545,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -5654,6 +5665,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -5676,12 +5688,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6820,6 +6831,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6847,7 +6859,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -6983,6 +6995,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -7019,7 +7032,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -7185,24 +7198,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7269,12 +7282,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -7442,24 +7456,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7526,12 +7540,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -8247,6 +8262,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8274,7 +8290,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -8410,6 +8426,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -8446,7 +8463,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -8612,24 +8629,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -8696,12 +8713,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -8869,24 +8887,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -8953,12 +8971,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -9674,6 +9693,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -9701,7 +9721,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -9837,6 +9857,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -9873,7 +9894,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -10583,6 +10604,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -10610,7 +10632,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -10746,6 +10768,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -10782,7 +10805,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -10948,24 +10971,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11032,12 +11055,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -11205,24 +11229,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11289,12 +11313,13 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -12010,6 +12035,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -12037,7 +12063,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -12173,6 +12199,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -12209,7 +12236,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 2160976599dd7..cbde0a9a08b36 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefix=GFX7LESS %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
@@ -96,7 +96,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
@@ -128,9 +129,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
@@ -221,7 +223,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_2
@@ -253,9 +256,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -872,6 +876,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -895,7 +900,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1000,6 +1005,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -1028,11 +1034,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1128,7 +1133,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_2
@@ -1160,9 +1166,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
@@ -1253,7 +1260,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_2
@@ -1285,9 +1293,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1905,6 +1914,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -1928,7 +1938,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2033,6 +2043,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -2061,11 +2072,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2162,7 +2172,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
@@ -2194,9 +2205,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
@@ -2287,7 +2299,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_2
@@ -2319,9 +2332,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2938,6 +2952,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -2961,7 +2976,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3066,6 +3081,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -3094,11 +3110,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -3201,7 +3216,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_2
@@ -3251,10 +3267,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
@@ -3367,7 +3384,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_2
@@ -3417,10 +3435,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4133,6 +4152,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4166,7 +4186,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -4303,6 +4323,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -4344,7 +4365,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -4465,7 +4486,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
@@ -4515,10 +4537,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
@@ -4631,7 +4654,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_2
@@ -4681,10 +4705,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -5397,6 +5422,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -5430,7 +5456,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -5567,6 +5593,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -5608,7 +5635,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -5729,7 +5756,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_2
@@ -5779,10 +5807,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
@@ -5895,7 +5924,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_2
@@ -5945,10 +5975,11 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6661,6 +6692,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6694,7 +6726,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -6831,6 +6863,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -6872,7 +6905,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -6986,7 +7019,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
@@ -7018,9 +7052,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
@@ -7111,7 +7146,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_2
@@ -7143,9 +7179,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7240,7 +7277,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
@@ -7272,9 +7310,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
@@ -7365,7 +7404,8 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_2
@@ -7397,9 +7437,10 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 029fb9c118344..070967d658105 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefix=GFX7LESS %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
@@ -96,7 +96,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
@@ -128,9 +129,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
@@ -221,7 +223,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_2
@@ -253,9 +256,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -872,6 +876,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -895,7 +900,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1000,6 +1005,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -1028,11 +1034,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1128,7 +1133,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_2
@@ -1160,9 +1166,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
@@ -1253,7 +1260,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_2
@@ -1285,9 +1293,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1905,6 +1914,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -1928,7 +1938,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2033,6 +2043,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -2061,11 +2072,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2162,7 +2172,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
@@ -2194,9 +2205,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
@@ -2287,7 +2299,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_2
@@ -2319,9 +2332,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2938,6 +2952,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -2961,7 +2976,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3066,6 +3081,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -3094,11 +3110,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -3201,7 +3216,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_2
@@ -3251,10 +3267,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
@@ -3367,7 +3384,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_2
@@ -3417,10 +3435,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4133,6 +4152,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4166,7 +4186,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -4303,6 +4323,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -4344,7 +4365,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -4465,7 +4486,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
@@ -4515,10 +4537,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
@@ -4631,7 +4654,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_2
@@ -4681,10 +4705,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -5397,6 +5422,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -5430,7 +5456,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -5567,6 +5593,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -5608,7 +5635,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -5729,7 +5756,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_2
@@ -5779,10 +5807,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
@@ -5895,7 +5924,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_2
@@ -5945,10 +5975,11 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6661,6 +6692,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6694,7 +6726,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -6831,6 +6863,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -6872,7 +6905,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
@@ -6986,7 +7019,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
@@ -7018,9 +7052,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
@@ -7111,7 +7146,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_2
@@ -7143,9 +7179,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7240,7 +7277,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
@@ -7272,9 +7310,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
@@ -7365,7 +7404,8 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_2
@@ -7397,9 +7437,10 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 3250d95bb0b7d..a9519048e3ddb 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefix=GFX7LESS %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 -amdgpu-atomic-optimizer-strategy=Iterative < %s | FileCheck -enable-var-scope -check-prefixes=GFX1064 %s
@@ -1050,6 +1050,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -1067,7 +1068,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -1187,6 +1188,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -1209,12 +1211,11 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1372,27 +1373,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1454,12 +1455,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -1622,27 +1624,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1704,12 +1706,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -2375,6 +2378,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -2392,7 +2396,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -2512,6 +2516,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -2534,12 +2539,11 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2697,27 +2701,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2779,12 +2783,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -2947,27 +2952,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -3029,12 +3034,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -3700,6 +3706,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -3717,7 +3724,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -3837,6 +3844,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -3859,12 +3867,11 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4521,6 +4528,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -4538,7 +4546,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -4658,6 +4666,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -4680,12 +4689,11 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4843,27 +4851,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4925,12 +4933,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -5093,27 +5102,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -5175,12 +5184,13 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -5845,6 +5855,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -5862,7 +5873,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -5982,6 +5993,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -6004,12 +6016,11 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7148,6 +7159,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -7175,7 +7187,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -7311,6 +7323,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -7347,7 +7360,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -7513,24 +7526,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7597,12 +7610,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -7770,24 +7784,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7854,12 +7868,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -8574,6 +8589,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8601,7 +8617,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -8737,6 +8753,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -8773,7 +8790,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -8939,24 +8956,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -9023,12 +9040,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -9196,24 +9214,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -9280,12 +9298,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -10001,6 +10020,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -10028,7 +10048,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -10164,6 +10184,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -10200,7 +10221,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -10910,6 +10931,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -10937,7 +10959,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -11073,6 +11095,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -11109,7 +11132,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -11274,24 +11297,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11358,12 +11381,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -11531,24 +11555,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
-; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
+; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11615,12 +11639,13 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -12336,6 +12361,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: s_mov_b32 s32, 0
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -12363,7 +12389,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -12499,6 +12525,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -12535,7 +12562,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
index 2daed9b69384f..017c9360ce7dd 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-kernarg-backend-usage.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 | FileCheck --check-prefix=GFX8V4 %s
; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 | FileCheck --check-prefix=GFX8V5 %s
; RUN: sed 's/CODE_OBJECT_VERSION/600/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 | FileCheck --check-prefix=GFX8V5 %s
@@ -117,7 +117,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V4-NEXT: s_cmp_eq_u32 s1, s0
; GFX8V4-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8V4-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V4-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX8V4-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8V4-NEXT: v_mov_b32_e32 v0, s0
; GFX8V4-NEXT: flat_store_dword v[0:1], v0
; GFX8V4-NEXT: s_waitcnt vmcnt(0)
; GFX8V4-NEXT: s_endpgm
@@ -129,7 +131,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V5-NEXT: s_cmp_eq_u32 s1, s0
; GFX8V5-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8V5-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V5-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX8V5-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8V5-NEXT: v_mov_b32_e32 v0, s0
; GFX8V5-NEXT: flat_store_dword v[0:1], v0
; GFX8V5-NEXT: s_waitcnt vmcnt(0)
; GFX8V5-NEXT: s_endpgm
@@ -141,7 +145,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
; GFX9V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX9V4-NEXT: s_cmp_eq_u32 s0, s1
; GFX9V4-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9V4-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V4-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9V4-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9V4-NEXT: v_mov_b32_e32 v0, s0
; GFX9V4-NEXT: global_store_dword v[0:1], v0, off
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: s_endpgm
@@ -153,7 +159,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_shared(ptr %ptr) #0 {
; GFX9V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX9V5-NEXT: s_cmp_eq_u32 s0, s1
; GFX9V5-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9V5-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V5-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9V5-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9V5-NEXT: v_mov_b32_e32 v0, s0
; GFX9V5-NEXT: global_store_dword v[0:1], v0, off
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: s_endpgm
@@ -171,7 +179,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
; GFX8V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V4-NEXT: s_cmp_eq_u32 s1, s0
; GFX8V4-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8V4-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V4-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX8V4-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8V4-NEXT: v_mov_b32_e32 v0, s0
; GFX8V4-NEXT: flat_store_dword v[0:1], v0
; GFX8V4-NEXT: s_waitcnt vmcnt(0)
; GFX8V4-NEXT: s_endpgm
@@ -183,7 +193,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
; GFX8V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX8V5-NEXT: s_cmp_eq_u32 s1, s0
; GFX8V5-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8V5-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8V5-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX8V5-NEXT: s_cselect_b32 s0, 1, 0
+; GFX8V5-NEXT: v_mov_b32_e32 v0, s0
; GFX8V5-NEXT: flat_store_dword v[0:1], v0
; GFX8V5-NEXT: s_waitcnt vmcnt(0)
; GFX8V5-NEXT: s_endpgm
@@ -195,7 +207,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
; GFX9V4-NEXT: s_waitcnt lgkmcnt(0)
; GFX9V4-NEXT: s_cmp_eq_u32 s0, s1
; GFX9V4-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9V4-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V4-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9V4-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9V4-NEXT: v_mov_b32_e32 v0, s0
; GFX9V4-NEXT: global_store_dword v[0:1], v0, off
; GFX9V4-NEXT: s_waitcnt vmcnt(0)
; GFX9V4-NEXT: s_endpgm
@@ -207,7 +221,9 @@ define amdgpu_kernel void @llvm_amdgcn_is_private(ptr %ptr) #0 {
; GFX9V5-NEXT: s_waitcnt lgkmcnt(0)
; GFX9V5-NEXT: s_cmp_eq_u32 s0, s1
; GFX9V5-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9V5-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9V5-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9V5-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9V5-NEXT: v_mov_b32_e32 v0, s0
; GFX9V5-NEXT: global_store_dword v[0:1], v0, off
; GFX9V5-NEXT: s_waitcnt vmcnt(0)
; GFX9V5-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
index 36df710529599..3fe6c638bfded 100644
--- a/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/infer-addrspace-flat-atomic.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s
define protected amdgpu_kernel void @InferNothing(i32 %a, ptr %b, double %c) {
@@ -127,9 +127,11 @@ define protected amdgpu_kernel void @InferPHI(i32 %a, ptr addrspace(1) %b, doubl
; CHECK-NEXT: s_mov_b64 s[0:1], src_shared_base
; CHECK-NEXT: s_cmp_eq_u32 s5, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s0, 1
; CHECK-NEXT: s_mov_b64 s[0:1], -1
-; CHECK-NEXT: s_cbranch_vccnz .LBB3_5
+; CHECK-NEXT: s_cbranch_scc1 .LBB3_5
; CHECK-NEXT: ; %bb.3: ; %Flow6
; CHECK-NEXT: s_andn2_b64 vcc, exec, s[0:1]
; CHECK-NEXT: s_cbranch_vccz .LBB3_10
@@ -139,9 +141,11 @@ define protected amdgpu_kernel void @InferPHI(i32 %a, ptr addrspace(1) %b, doubl
; CHECK-NEXT: s_mov_b64 s[0:1], src_private_base
; CHECK-NEXT: s_cmp_eq_u32 s5, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: s_andn2_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s0, 1
; CHECK-NEXT: s_mov_b64 s[0:1], -1
-; CHECK-NEXT: s_cbranch_vccz .LBB3_7
+; CHECK-NEXT: s_cbranch_scc0 .LBB3_7
; CHECK-NEXT: ; %bb.6: ; %atomicrmw.global
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index c1f3a12dba578..686afb16332b1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32 < %s | FileCheck -check-prefixes=CHECK,GFX10 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32 < %s | FileCheck -check-prefixes=CHECK,GFX11 %s
@@ -21,7 +21,8 @@ define amdgpu_cs i32 @constant_false() {
define amdgpu_cs i32 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_mov_b32 s0, exec_lo
+; CHECK-NEXT: v_mov_b32_e32 v0, exec_lo
+; CHECK-NEXT: v_readfirstlane_b32 s0, v0
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
ret i32 %ballot
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
index 7de4aa95b3cc2..835365961f2dd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.private.ll
@@ -61,8 +61,8 @@ define amdgpu_kernel void @is_private_vgpr(ptr addrspace(1) %ptr.ptr) {
;
; GFX1250-LABEL: is_private_vgpr:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v0, s[0:1] scale_offset scope:SCOPE_SYS
@@ -140,8 +140,10 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_eq_u32 s0, s1
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; SI-NEXT: s_cbranch_vccnz .LBB1_2
+; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s0, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_2
; SI-NEXT: ; %bb.1: ; %bb0
; SI-NEXT: s_mov_b32 s3, 0x100f000
; SI-NEXT: s_mov_b32 s2, -1
@@ -161,8 +163,10 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
; CI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; CI-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; CI-SDAG-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CI-SDAG-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; CI-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; CI-SDAG-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CI-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; CI-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; CI-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; CI-SDAG-NEXT: ; %bb.1: ; %bb0
; CI-SDAG-NEXT: v_mov_b32_e32 v0, 0
; CI-SDAG-NEXT: flat_store_dword v[0:1], v0
@@ -177,8 +181,10 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; GFX9-SDAG-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-SDAG-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; GFX9-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; GFX9-SDAG-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: global_store_dword v[0:1], v0, off
@@ -188,15 +194,18 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
;
; GFX1250-SDAG-LABEL: is_private_sgpr:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_xor_b32 s0, s0, src_flat_scratch_base_hi
; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-SDAG-NEXT: s_cmp_lt_u32 s0, 0x4000000
; GFX1250-SDAG-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; GFX1250-SDAG-NEXT: ; %bb.1: ; %bb0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v0, off scope:SCOPE_SYS
@@ -266,8 +275,8 @@ define amdgpu_kernel void @is_private_sgpr(ptr %ptr) {
;
; GFX1250-GISEL-LABEL: is_private_sgpr:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_xor_b32 s0, s1, src_flat_scratch_base_hi
; GFX1250-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
index 3aa1a10ffc918..2810aa883e6b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.is.shared.ll
@@ -94,8 +94,8 @@ define amdgpu_kernel void @is_local_vgpr(ptr addrspace(1) %ptr.ptr) {
;
; GFX1250-LABEL: is_local_vgpr:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_mov_b64 s[0:1], src_shared_base
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -206,8 +206,10 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_eq_u32 s0, s1
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; SI-NEXT: s_cbranch_vccnz .LBB1_2
+; SI-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s0, 1
+; SI-NEXT: s_cbranch_scc1 .LBB1_2
; SI-NEXT: ; %bb.1: ; %bb0
; SI-NEXT: s_mov_b32 s3, 0x100f000
; SI-NEXT: s_mov_b32 s2, -1
@@ -227,8 +229,10 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
; CI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; CI-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; CI-SDAG-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CI-SDAG-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; CI-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; CI-SDAG-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CI-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; CI-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; CI-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; CI-SDAG-NEXT: ; %bb.1: ; %bb0
; CI-SDAG-NEXT: v_mov_b32_e32 v0, 0
; CI-SDAG-NEXT: flat_store_dword v[0:1], v0
@@ -243,8 +247,10 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; GFX9-SDAG-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-SDAG-NEXT: s_andn2_b64 vcc, exec, s[0:1]
-; GFX9-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; GFX9-SDAG-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: global_store_dword v[0:1], v0, off
@@ -254,15 +260,17 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
;
; GFX1250-SDAG-LABEL: is_local_sgpr:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_mov_b64 s[0:1], src_shared_base
-; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4
+; GFX1250-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x4 nv
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_cmp_eq_u32 s0, s1
; GFX1250-SDAG-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-SDAG-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
-; GFX1250-SDAG-NEXT: s_cbranch_vccnz .LBB1_2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1250-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-SDAG-NEXT: s_cmp_lg_u32 s0, 1
+; GFX1250-SDAG-NEXT: s_cbranch_scc1 .LBB1_2
; GFX1250-SDAG-NEXT: ; %bb.1: ; %bb0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: global_store_b32 v[0:1], v0, off scope:SCOPE_SYS
@@ -332,8 +340,8 @@ define amdgpu_kernel void @is_local_sgpr(ptr %ptr) {
;
; GFX1250-GISEL-LABEL: is_local_sgpr:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
; GFX1250-GISEL-NEXT: s_mov_b64 s[0:1], src_shared_base
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s3, s1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
index b9bf76c1423b6..32044e23cc9a9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
@@ -193,15 +193,16 @@ define amdgpu_kernel void @test_scc_quadmask_32(i32 %val0, i32 %val1, ptr addrsp
; GFX11-SDAG-LABEL: test_scc_quadmask_32:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_and_b32 s0, s0, 1
; GFX11-SDAG-NEXT: s_quadmask_b32 s1, s1
; GFX11-SDAG-NEXT: s_cmp_eq_u32 s0, 0
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, s1
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, s1
; GFX11-SDAG-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, s0
; GFX11-SDAG-NEXT: global_store_b32 v2, v3, s[2:3]
; GFX11-SDAG-NEXT: global_store_b32 v[0:1], v4, off
; GFX11-SDAG-NEXT: s_endpgm
@@ -239,18 +240,19 @@ define amdgpu_kernel void @test_scc_quadmask_64(i32 %val0, i64 %val1, ptr addrsp
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_and_b32 s4, s6, 1
; GFX11-SDAG-NEXT: s_quadmask_b64 s[0:1], s[0:1]
; GFX11-SDAG-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, s1
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-SDAG-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11-SDAG-NEXT: global_store_b64 v4, v[2:3], s[2:3]
-; GFX11-SDAG-NEXT: global_store_b32 v[0:1], v5, off
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-SDAG-NEXT: s_cselect_b32 s4, -1, 0
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s1
+; GFX11-SDAG-NEXT: s_and_b32 s0, s4, exec_lo
+; GFX11-SDAG-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v5, s0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX11-SDAG-NEXT: global_store_b64 v4, v[0:1], s[2:3]
+; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v5, off
; GFX11-SDAG-NEXT: s_endpgm
%and = and i32 %val0, 1
%result = call i64 @llvm.amdgcn.s.quadmask.i64(i64 %val1) nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index 14f2873b64647..a69c95e3f4bc0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -511,16 +511,18 @@ define amdgpu_kernel void @test_readfirstlane_m0(ptr addrspace(1) %out) {
; CHECK-SDAG-LABEL: test_readfirstlane_m0:
; CHECK-SDAG: ; %bb.0:
; CHECK-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-SDAG-NEXT: s_add_i32 s12, s12, s17
-; CHECK-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CHECK-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CHECK-SDAG-NEXT: ;;#ASMSTART
; CHECK-SDAG-NEXT: s_mov_b32 m0, -1
; CHECK-SDAG-NEXT: ;;#ASMEND
+; CHECK-SDAG-NEXT: v_mov_b32_e32 v0, m0
+; CHECK-SDAG-NEXT: s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; CHECK-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-SDAG-NEXT: v_mov_b32_e32 v2, m0
+; CHECK-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CHECK-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; CHECK-SDAG-NEXT: v_mov_b32_e32 v2, s2
; CHECK-SDAG-NEXT: flat_store_dword v[0:1], v2
; CHECK-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
index e8bf198f89855..bcdc3670d53cc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fma.f16.ll
@@ -42,47 +42,47 @@ define amdgpu_kernel void @fma_f16(
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
; SI-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s0, v1
-; SI-NEXT: s_and_b32 s1, s0, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s1, v0
-; SI-NEXT: s_lshr_b32 s3, s0, 8
-; SI-NEXT: s_bfe_u32 s4, s0, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s3, v1
+; SI-NEXT: s_and_b32 s0, s3, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s0, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s1, s3, 0xffe
-; SI-NEXT: s_sub_i32 s3, 0x3f1, s4
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s3, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s3, v0
-; SI-NEXT: s_or_b32 s1, s1, s3
-; SI-NEXT: v_readfirstlane_b32 s5, v1
-; SI-NEXT: s_or_b32 s3, s1, 0x1000
-; SI-NEXT: s_lshr_b32 s6, s3, s5
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s3, 8
+; SI-NEXT: s_bfe_u32 s4, s3, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s5, 0x3f1, s4
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v0, s5, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s5, v0
+; SI-NEXT: s_lshr_b32 s6, s1, s5
; SI-NEXT: s_lshl_b32 s5, s6, s5
-; SI-NEXT: s_cmp_lg_u32 s5, s3
-; SI-NEXT: s_cselect_b32 s3, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s5, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
; SI-NEXT: s_addk_i32 s4, 0xfc10
; SI-NEXT: s_lshl_b32 s5, s4, 12
-; SI-NEXT: s_or_b32 s3, s6, s3
-; SI-NEXT: s_or_b32 s5, s1, s5
+; SI-NEXT: s_or_b32 s1, s6, s1
+; SI-NEXT: s_or_b32 s5, s0, s5
; SI-NEXT: s_cmp_lt_i32 s4, 1
-; SI-NEXT: s_cselect_b32 s3, s3, s5
-; SI-NEXT: s_and_b32 s5, s3, 7
+; SI-NEXT: s_cselect_b32 s1, s1, s5
+; SI-NEXT: s_and_b32 s5, s1, 7
; SI-NEXT: s_cmp_gt_i32 s5, 5
; SI-NEXT: s_cselect_b32 s6, 1, 0
; SI-NEXT: s_cmp_eq_u32 s5, 3
; SI-NEXT: s_cselect_b32 s5, 1, 0
-; SI-NEXT: s_lshr_b32 s3, s3, 2
+; SI-NEXT: s_lshr_b32 s1, s1, 2
; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_add_i32 s3, s3, s5
+; SI-NEXT: s_add_i32 s1, s1, s5
; SI-NEXT: s_cmp_lt_i32 s4, 31
-; SI-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s1, 0
-; SI-NEXT: s_cselect_b32 s1, s2, 0x7c00
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s4, 0x40f
-; SI-NEXT: s_cselect_b32 s1, s1, s3
-; SI-NEXT: s_lshr_b32 s0, s0, 16
-; SI-NEXT: s_and_b32 s0, s0, 0x8000
-; SI-NEXT: s_or_b32 s0, s0, s1
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s3, 16
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-NEXT: s_or_b32 s0, s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
@@ -182,47 +182,47 @@ define amdgpu_kernel void @fma_f16_imm_a(
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s10, s7, s9
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s9, 0x3f1, s8
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s9, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s9, v0
+; SI-NEXT: s_lshr_b32 s10, s5, s9
; SI-NEXT: s_lshl_b32 s9, s10, s9
-; SI-NEXT: s_cmp_lg_u32 s9, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
; SI-NEXT: s_addk_i32 s8, 0xfc10
; SI-NEXT: s_lshl_b32 s9, s8, 12
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_or_b32 s9, s5, s9
+; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_or_b32 s9, s4, s9
; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s7, 7
+; SI-NEXT: s_cselect_b32 s5, s5, s9
+; SI-NEXT: s_and_b32 s9, s5, 7
; SI-NEXT: s_cmp_gt_i32 s9, 5
; SI-NEXT: s_cselect_b32 s10, 1, 0
; SI-NEXT: s_cmp_eq_u32 s9, 3
; SI-NEXT: s_cselect_b32 s9, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s7, s7, s9
+; SI-NEXT: s_add_i32 s5, s5, s9
; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -310,47 +310,47 @@ define amdgpu_kernel void @fma_f16_imm_b(
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s10, s7, s9
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s9, 0x3f1, s8
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s9, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s9, v0
+; SI-NEXT: s_lshr_b32 s10, s5, s9
; SI-NEXT: s_lshl_b32 s9, s10, s9
-; SI-NEXT: s_cmp_lg_u32 s9, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
; SI-NEXT: s_addk_i32 s8, 0xfc10
; SI-NEXT: s_lshl_b32 s9, s8, 12
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_or_b32 s9, s5, s9
+; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_or_b32 s9, s4, s9
; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s7, 7
+; SI-NEXT: s_cselect_b32 s5, s5, s9
+; SI-NEXT: s_and_b32 s9, s5, 7
; SI-NEXT: s_cmp_gt_i32 s9, 5
; SI-NEXT: s_cselect_b32 s10, 1, 0
; SI-NEXT: s_cmp_eq_u32 s9, 3
; SI-NEXT: s_cselect_b32 s9, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s7, s7, s9
+; SI-NEXT: s_add_i32 s5, s5, s9
; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -438,47 +438,47 @@ define amdgpu_kernel void @fma_f16_imm_c(
; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[4:5]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s7, v1
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s9, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s10, s7, s9
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s8, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s9, 0x3f1, s8
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s9, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s9, v0
+; SI-NEXT: s_lshr_b32 s10, s5, s9
; SI-NEXT: s_lshl_b32 s9, s10, s9
-; SI-NEXT: s_cmp_lg_u32 s9, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s9, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
; SI-NEXT: s_addk_i32 s8, 0xfc10
; SI-NEXT: s_lshl_b32 s9, s8, 12
-; SI-NEXT: s_or_b32 s7, s10, s7
-; SI-NEXT: s_or_b32 s9, s5, s9
+; SI-NEXT: s_or_b32 s5, s10, s5
+; SI-NEXT: s_or_b32 s9, s4, s9
; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s9
-; SI-NEXT: s_and_b32 s9, s7, 7
+; SI-NEXT: s_cselect_b32 s5, s5, s9
+; SI-NEXT: s_and_b32 s9, s5, 7
; SI-NEXT: s_cmp_gt_i32 s9, 5
; SI-NEXT: s_cselect_b32 s10, 1, 0
; SI-NEXT: s_cmp_eq_u32 s9, 3
; SI-NEXT: s_cselect_b32 s9, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s9, s9, s10
-; SI-NEXT: s_add_i32 s7, s7, s9
+; SI-NEXT: s_add_i32 s5, s5, s9
; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -582,92 +582,92 @@ define amdgpu_kernel void @fma_v2f16(
; SI-NEXT: v_cvt_f64_f32_e32 v[8:9], v9
; SI-NEXT: v_cvt_f64_f32_e32 v[10:11], v10
; SI-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s0, v1
-; SI-NEXT: s_and_b32 s1, s0, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s1, v0
-; SI-NEXT: s_lshr_b32 s3, s0, 8
-; SI-NEXT: s_bfe_u32 s4, s0, 0xb0014
+; SI-NEXT: v_readfirstlane_b32 s3, v1
+; SI-NEXT: s_and_b32 s0, s3, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s0, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s1, s3, 0xffe
-; SI-NEXT: s_sub_i32 s3, 0x3f1, s4
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s3, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s3, v0
-; SI-NEXT: s_or_b32 s1, s1, s3
-; SI-NEXT: v_readfirstlane_b32 s6, v1
-; SI-NEXT: s_or_b32 s3, s1, 0x1000
-; SI-NEXT: s_lshr_b32 s7, s3, s6
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s3, 8
+; SI-NEXT: s_bfe_u32 s5, s3, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s6, 0x3f1, s5
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v0, s6, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s6, v0
+; SI-NEXT: s_lshr_b32 s7, s1, s6
; SI-NEXT: s_lshl_b32 s6, s7, s6
-; SI-NEXT: s_cmp_lg_u32 s6, s3
-; SI-NEXT: s_cselect_b32 s3, 1, 0
-; SI-NEXT: s_addk_i32 s4, 0xfc10
-; SI-NEXT: s_lshl_b32 s6, s4, 12
-; SI-NEXT: s_or_b32 s3, s7, s3
-; SI-NEXT: s_or_b32 s6, s1, s6
-; SI-NEXT: s_cmp_lt_i32 s4, 1
-; SI-NEXT: s_cselect_b32 s3, s3, s6
-; SI-NEXT: s_and_b32 s6, s3, 7
+; SI-NEXT: s_cmp_lg_u32 s6, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
+; SI-NEXT: s_addk_i32 s5, 0xfc10
+; SI-NEXT: s_lshl_b32 s6, s5, 12
+; SI-NEXT: s_or_b32 s1, s7, s1
+; SI-NEXT: s_or_b32 s6, s0, s6
+; SI-NEXT: s_cmp_lt_i32 s5, 1
+; SI-NEXT: s_cselect_b32 s1, s1, s6
+; SI-NEXT: s_and_b32 s6, s1, 7
; SI-NEXT: s_cmp_gt_i32 s6, 5
; SI-NEXT: s_cselect_b32 s7, 1, 0
; SI-NEXT: s_cmp_eq_u32 s6, 3
; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_lshr_b32 s3, s3, 2
+; SI-NEXT: s_lshr_b32 s1, s1, 2
; SI-NEXT: s_or_b32 s6, s6, s7
-; SI-NEXT: s_add_i32 s3, s3, s6
+; SI-NEXT: s_add_i32 s1, s1, s6
+; SI-NEXT: s_cmp_lt_i32 s5, 31
; SI-NEXT: v_fma_f64 v[2:3], v[10:11], v[8:9], v[6:7]
-; SI-NEXT: s_cmp_lt_i32 s4, 31
-; SI-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s1, 0
-; SI-NEXT: v_readfirstlane_b32 s5, v3
-; SI-NEXT: s_cselect_b32 s1, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s4, 0x40f
-; SI-NEXT: s_cselect_b32 s1, s1, s3
-; SI-NEXT: s_and_b32 s3, s5, 0x1ff
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s5, 0x40f
+; SI-NEXT: v_readfirstlane_b32 s4, v3
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s3, 16
+; SI-NEXT: s_and_b32 s3, s4, 0x1ff
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
; SI-NEXT: v_or_b32_e32 v0, s3, v2
-; SI-NEXT: s_lshr_b32 s0, s0, 16
+; SI-NEXT: s_or_b32 s0, s1, s0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_lshr_b32 s4, s5, 8
-; SI-NEXT: s_bfe_u32 s6, s5, 0xb0014
-; SI-NEXT: s_and_b32 s0, s0, 0x8000
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_and_b32 s3, s4, 0xffe
-; SI-NEXT: s_sub_i32 s4, 0x3f1, s6
-; SI-NEXT: s_or_b32 s0, s0, s1
-; SI-NEXT: v_readfirstlane_b32 s1, v0
-; SI-NEXT: v_med3_i32 v1, s4, 0, 13
-; SI-NEXT: s_or_b32 s1, s3, s1
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_or_b32 s3, s1, 0x1000
-; SI-NEXT: s_lshr_b32 s7, s3, s4
-; SI-NEXT: s_and_b32 s0, s0, 0xffff
-; SI-NEXT: s_lshl_b32 s4, s7, s4
-; SI-NEXT: s_cmp_lg_u32 s4, s3
-; SI-NEXT: s_cselect_b32 s3, 1, 0
-; SI-NEXT: s_addk_i32 s6, 0xfc10
-; SI-NEXT: s_lshl_b32 s4, s6, 12
-; SI-NEXT: s_or_b32 s3, s7, s3
-; SI-NEXT: s_or_b32 s4, s1, s4
-; SI-NEXT: s_cmp_lt_i32 s6, 1
-; SI-NEXT: s_cselect_b32 s3, s3, s4
-; SI-NEXT: s_and_b32 s4, s3, 7
-; SI-NEXT: s_cmp_gt_i32 s4, 5
+; SI-NEXT: s_and_b32 s3, s0, 0xffff
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s4, 8
+; SI-NEXT: s_bfe_u32 s5, s4, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s6, 0x3f1, s5
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v0, s6, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s6, v0
+; SI-NEXT: s_lshr_b32 s7, s1, s6
+; SI-NEXT: s_lshl_b32 s6, s7, s6
+; SI-NEXT: s_cmp_lg_u32 s6, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
+; SI-NEXT: s_addk_i32 s5, 0xfc10
+; SI-NEXT: s_lshl_b32 s6, s5, 12
+; SI-NEXT: s_or_b32 s1, s7, s1
+; SI-NEXT: s_or_b32 s6, s0, s6
+; SI-NEXT: s_cmp_lt_i32 s5, 1
+; SI-NEXT: s_cselect_b32 s1, s1, s6
+; SI-NEXT: s_and_b32 s6, s1, 7
+; SI-NEXT: s_cmp_gt_i32 s6, 5
; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s4, 3
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_lshr_b32 s3, s3, 2
-; SI-NEXT: s_or_b32 s4, s4, s7
-; SI-NEXT: s_add_i32 s3, s3, s4
-; SI-NEXT: s_cmp_lt_i32 s6, 31
-; SI-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s1, 0
-; SI-NEXT: s_cselect_b32 s1, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s6, 0x40f
-; SI-NEXT: s_cselect_b32 s1, s1, s3
-; SI-NEXT: s_lshr_b32 s2, s5, 16
-; SI-NEXT: s_and_b32 s2, s2, 0x8000
-; SI-NEXT: s_or_b32 s1, s2, s1
-; SI-NEXT: s_lshl_b32 s1, s1, 16
-; SI-NEXT: s_or_b32 s0, s0, s1
+; SI-NEXT: s_cmp_eq_u32 s6, 3
+; SI-NEXT: s_cselect_b32 s6, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s1, 2
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: s_add_i32 s1, s1, s6
+; SI-NEXT: s_cmp_lt_i32 s5, 31
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s5, 0x40f
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s4, 16
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: s_lshl_b32 s0, s0, 16
+; SI-NEXT: s_or_b32 s0, s3, s0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
@@ -765,9 +765,9 @@ define amdgpu_kernel void @fma_v2f16_imm_a(
; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s5, 0x40080000
-; SI-NEXT: s_mov_b32 s0, s8
-; SI-NEXT: s_movk_i32 s6, 0x7e00
; SI-NEXT: s_mov_b32 s1, s9
+; SI-NEXT: s_movk_i32 s6, 0x7e00
+; SI-NEXT: s_mov_b32 s0, s8
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
@@ -781,92 +781,92 @@ define amdgpu_kernel void @fma_v2f16_imm_a(
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v5
; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: v_fma_f64 v[2:3], v[6:7], s[4:5], v[4:5]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s10, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s10
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
; SI-NEXT: s_lshl_b32 s10, s11, s10
-; SI-NEXT: s_cmp_lg_u32 s10, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s8, 0xfc10
-; SI-NEXT: s_lshl_b32 s10, s8, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s10, s5, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s10
-; SI-NEXT: s_and_b32 s10, s7, 7
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
; SI-NEXT: s_cmp_eq_u32 s10, 3
; SI-NEXT: s_cselect_b32 s10, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_add_i32 s7, s7, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_readfirstlane_b32 s9, v3
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_and_b32 s7, s9, 0x1ff
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: v_readfirstlane_b32 s8, v3
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s7, s8, 0x1ff
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
; SI-NEXT: v_or_b32_e32 v0, s7, v2
-; SI-NEXT: s_lshr_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_lshr_b32 s8, s9, 8
-; SI-NEXT: s_bfe_u32 s10, s9, 0xb0014
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_and_b32 s7, s8, 0xffe
-; SI-NEXT: s_sub_i32 s8, 0x3f1, s10
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readfirstlane_b32 s5, v0
-; SI-NEXT: v_med3_i32 v1, s8, 0, 13
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: v_readfirstlane_b32 s8, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s8
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s11, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s10, 0xfc10
-; SI-NEXT: s_lshl_b32 s8, s10, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s8, s5, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s8
-; SI-NEXT: s_and_b32 s8, s7, 7
-; SI-NEXT: s_cmp_gt_i32 s8, 5
+; SI-NEXT: s_and_b32 s7, s4, 0xffff
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s8, 8
+; SI-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
+; SI-NEXT: s_lshl_b32 s10, s11, s10
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
+; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s8, 3
-; SI-NEXT: s_cselect_b32 s8, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
-; SI-NEXT: s_or_b32 s8, s8, s11
-; SI-NEXT: s_add_i32 s7, s7, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s6, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0x8000
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cmp_eq_u32 s10, 3
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s5, 2
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s8, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -950,9 +950,9 @@ define amdgpu_kernel void @fma_v2f16_imm_b(
; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s5, 0x40080000
-; SI-NEXT: s_mov_b32 s0, s8
-; SI-NEXT: s_movk_i32 s6, 0x7e00
; SI-NEXT: s_mov_b32 s1, s9
+; SI-NEXT: s_movk_i32 s6, 0x7e00
+; SI-NEXT: s_mov_b32 s0, s8
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
@@ -966,92 +966,92 @@ define amdgpu_kernel void @fma_v2f16_imm_b(
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v5
; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], s[4:5], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: v_fma_f64 v[2:3], v[6:7], s[4:5], v[4:5]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s10, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s10
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
; SI-NEXT: s_lshl_b32 s10, s11, s10
-; SI-NEXT: s_cmp_lg_u32 s10, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s8, 0xfc10
-; SI-NEXT: s_lshl_b32 s10, s8, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s10, s5, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s10
-; SI-NEXT: s_and_b32 s10, s7, 7
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
; SI-NEXT: s_cmp_eq_u32 s10, 3
; SI-NEXT: s_cselect_b32 s10, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_add_i32 s7, s7, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_readfirstlane_b32 s9, v3
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_and_b32 s7, s9, 0x1ff
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: v_readfirstlane_b32 s8, v3
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s7, s8, 0x1ff
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
; SI-NEXT: v_or_b32_e32 v0, s7, v2
-; SI-NEXT: s_lshr_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_lshr_b32 s8, s9, 8
-; SI-NEXT: s_bfe_u32 s10, s9, 0xb0014
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_and_b32 s7, s8, 0xffe
-; SI-NEXT: s_sub_i32 s8, 0x3f1, s10
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readfirstlane_b32 s5, v0
-; SI-NEXT: v_med3_i32 v1, s8, 0, 13
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: v_readfirstlane_b32 s8, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s8
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s11, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s10, 0xfc10
-; SI-NEXT: s_lshl_b32 s8, s10, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s8, s5, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s8
-; SI-NEXT: s_and_b32 s8, s7, 7
-; SI-NEXT: s_cmp_gt_i32 s8, 5
+; SI-NEXT: s_and_b32 s7, s4, 0xffff
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s8, 8
+; SI-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
+; SI-NEXT: s_lshl_b32 s10, s11, s10
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
+; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s8, 3
-; SI-NEXT: s_cselect_b32 s8, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
-; SI-NEXT: s_or_b32 s8, s8, s11
-; SI-NEXT: s_add_i32 s7, s7, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s6, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0x8000
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cmp_eq_u32 s10, 3
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s5, 2
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s8, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1135,9 +1135,9 @@ define amdgpu_kernel void @fma_v2f16_imm_c(
; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0
; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s5, 0x40080000
-; SI-NEXT: s_mov_b32 s0, s8
-; SI-NEXT: s_movk_i32 s6, 0x7e00
; SI-NEXT: s_mov_b32 s1, s9
+; SI-NEXT: s_movk_i32 s6, 0x7e00
+; SI-NEXT: s_mov_b32 s0, s8
; SI-NEXT: s_waitcnt vmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
@@ -1151,92 +1151,92 @@ define amdgpu_kernel void @fma_v2f16_imm_c(
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v5
; SI-NEXT: v_cvt_f64_f32_e32 v[6:7], v6
; SI-NEXT: v_fma_f64 v[0:1], v[2:3], v[0:1], s[4:5]
+; SI-NEXT: v_readfirstlane_b32 s7, v1
; SI-NEXT: v_fma_f64 v[2:3], v[6:7], v[4:5], s[4:5]
-; SI-NEXT: v_readfirstlane_b32 s4, v1
-; SI-NEXT: s_and_b32 s5, s4, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s5, v0
-; SI-NEXT: s_lshr_b32 s7, s4, 8
-; SI-NEXT: s_bfe_u32 s8, s4, 0xb0014
+; SI-NEXT: s_and_b32 s4, s7, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s4, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_b32 s5, s7, 0xffe
-; SI-NEXT: s_sub_i32 s7, 0x3f1, s8
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v1, s7, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_or_b32 s5, s5, s7
-; SI-NEXT: v_readfirstlane_b32 s10, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s10
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s7, 8
+; SI-NEXT: s_bfe_u32 s9, s7, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
; SI-NEXT: s_lshl_b32 s10, s11, s10
-; SI-NEXT: s_cmp_lg_u32 s10, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s8, 0xfc10
-; SI-NEXT: s_lshl_b32 s10, s8, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s10, s5, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s10
-; SI-NEXT: s_and_b32 s10, s7, 7
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
; SI-NEXT: s_cmp_eq_u32 s10, 3
; SI-NEXT: s_cselect_b32 s10, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
+; SI-NEXT: s_lshr_b32 s5, s5, 2
; SI-NEXT: s_or_b32 s10, s10, s11
-; SI-NEXT: s_add_i32 s7, s7, s10
-; SI-NEXT: s_cmp_lt_i32 s8, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: v_readfirstlane_b32 s9, v3
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s8, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_and_b32 s7, s9, 0x1ff
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: v_readfirstlane_b32 s8, v3
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s7, 16
+; SI-NEXT: s_and_b32 s7, s8, 0x1ff
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
; SI-NEXT: v_or_b32_e32 v0, s7, v2
-; SI-NEXT: s_lshr_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s5, s4
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: s_lshr_b32 s8, s9, 8
-; SI-NEXT: s_bfe_u32 s10, s9, 0xb0014
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_and_b32 s7, s8, 0xffe
-; SI-NEXT: s_sub_i32 s8, 0x3f1, s10
-; SI-NEXT: s_or_b32 s4, s4, s5
-; SI-NEXT: v_readfirstlane_b32 s5, v0
-; SI-NEXT: v_med3_i32 v1, s8, 0, 13
-; SI-NEXT: s_or_b32 s5, s7, s5
-; SI-NEXT: v_readfirstlane_b32 s8, v1
-; SI-NEXT: s_or_b32 s7, s5, 0x1000
-; SI-NEXT: s_lshr_b32 s11, s7, s8
-; SI-NEXT: s_and_b32 s4, s4, 0xffff
-; SI-NEXT: s_lshl_b32 s8, s11, s8
-; SI-NEXT: s_cmp_lg_u32 s8, s7
-; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_addk_i32 s10, 0xfc10
-; SI-NEXT: s_lshl_b32 s8, s10, 12
-; SI-NEXT: s_or_b32 s7, s11, s7
-; SI-NEXT: s_or_b32 s8, s5, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 1
-; SI-NEXT: s_cselect_b32 s7, s7, s8
-; SI-NEXT: s_and_b32 s8, s7, 7
-; SI-NEXT: s_cmp_gt_i32 s8, 5
+; SI-NEXT: s_and_b32 s7, s4, 0xffff
+; SI-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s8, 8
+; SI-NEXT: s_bfe_u32 s9, s8, 0xb0014
+; SI-NEXT: s_and_b32 s5, s5, 0xffe
+; SI-NEXT: s_sub_i32 s10, 0x3f1, s9
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: v_med3_i32 v0, s10, 0, 13
+; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s10, v0
+; SI-NEXT: s_lshr_b32 s11, s5, s10
+; SI-NEXT: s_lshl_b32 s10, s11, s10
+; SI-NEXT: s_cmp_lg_u32 s10, s5
+; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_addk_i32 s9, 0xfc10
+; SI-NEXT: s_lshl_b32 s10, s9, 12
+; SI-NEXT: s_or_b32 s5, s11, s5
+; SI-NEXT: s_or_b32 s10, s4, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 1
+; SI-NEXT: s_cselect_b32 s5, s5, s10
+; SI-NEXT: s_and_b32 s10, s5, 7
+; SI-NEXT: s_cmp_gt_i32 s10, 5
; SI-NEXT: s_cselect_b32 s11, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s8, 3
-; SI-NEXT: s_cselect_b32 s8, 1, 0
-; SI-NEXT: s_lshr_b32 s7, s7, 2
-; SI-NEXT: s_or_b32 s8, s8, s11
-; SI-NEXT: s_add_i32 s7, s7, s8
-; SI-NEXT: s_cmp_lt_i32 s10, 31
-; SI-NEXT: s_cselect_b32 s7, s7, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s5, 0
-; SI-NEXT: s_cselect_b32 s5, s6, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s10, 0x40f
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_lshr_b32 s6, s9, 16
-; SI-NEXT: s_and_b32 s6, s6, 0x8000
-; SI-NEXT: s_or_b32 s5, s6, s5
-; SI-NEXT: s_lshl_b32 s5, s5, 16
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_cmp_eq_u32 s10, 3
+; SI-NEXT: s_cselect_b32 s10, 1, 0
+; SI-NEXT: s_lshr_b32 s5, s5, 2
+; SI-NEXT: s_or_b32 s10, s10, s11
+; SI-NEXT: s_add_i32 s5, s5, s10
+; SI-NEXT: s_cmp_lt_i32 s9, 31
+; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s4, 0
+; SI-NEXT: s_cselect_b32 s4, s6, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s9, 0x40f
+; SI-NEXT: s_cselect_b32 s4, s4, s5
+; SI-NEXT: s_lshr_b32 s5, s8, 16
+; SI-NEXT: s_and_b32 s5, s5, 0x8000
+; SI-NEXT: s_or_b32 s4, s5, s4
+; SI-NEXT: s_lshl_b32 s4, s4, 16
+; SI-NEXT: s_or_b32 s4, s7, s4
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
@@ -1320,222 +1320,222 @@ define amdgpu_kernel void @fma_v4f16(
; SI-NEXT: s_mov_b32 s4, s6
; SI-NEXT: s_mov_b32 s5, s7
; SI-NEXT: s_mov_b32 s6, s10
-; SI-NEXT: buffer_load_dwordx2 v[5:6], off, s[12:15], 0
; SI-NEXT: s_mov_b32 s7, s11
-; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[16:19], 0
-; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT: buffer_load_dwordx2 v[0:1], off, s[12:15], 0
+; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[16:19], 0
+; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b32 s8, s0
; SI-NEXT: s_mov_b32 s9, s1
; SI-NEXT: s_movk_i32 s2, 0x7e00
; SI-NEXT: s_waitcnt vmcnt(2)
-; SI-NEXT: v_cvt_f32_f16_e32 v10, v6
+; SI-NEXT: v_cvt_f32_f16_e32 v12, v1
; SI-NEXT: s_waitcnt vmcnt(1)
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v11, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v13, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v15, v5
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v13, v4
+; SI-NEXT: v_cvt_f32_f16_e32 v14, v2
+; SI-NEXT: v_cvt_f32_f16_e32 v15, v0
+; SI-NEXT: v_cvt_f32_f16_e32 v4, v6
+; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v5
+; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v3
+; SI-NEXT: v_cvt_f64_f32_e32 v[5:6], v12
+; SI-NEXT: v_cvt_f32_f16_e32 v16, v11
; SI-NEXT: v_cvt_f32_f16_e32 v17, v9
-; SI-NEXT: v_cvt_f32_f16_e32 v18, v8
-; SI-NEXT: v_cvt_f32_f16_e32 v19, v7
-; SI-NEXT: v_cvt_f64_f32_e32 v[5:6], v3
-; SI-NEXT: v_cvt_f64_f32_e32 v[7:8], v1
-; SI-NEXT: v_cvt_f64_f32_e32 v[9:10], v10
-; SI-NEXT: v_cvt_f64_f32_e32 v[11:12], v11
-; SI-NEXT: v_cvt_f64_f32_e32 v[13:14], v13
-; SI-NEXT: v_cvt_f64_f32_e32 v[15:16], v15
-; SI-NEXT: v_fma_f64 v[5:6], v[9:10], v[7:8], v[5:6]
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; SI-NEXT: v_readfirstlane_b32 s0, v6
-; SI-NEXT: v_fma_f64 v[2:3], v[15:16], v[13:14], v[11:12]
-; SI-NEXT: s_and_b32 s3, s0, 0x1ff
-; SI-NEXT: v_readfirstlane_b32 s1, v3
-; SI-NEXT: v_or_b32_e32 v3, s3, v5
-; SI-NEXT: s_lshr_b32 s4, s0, 8
-; SI-NEXT: s_bfe_u32 s5, s0, 0xb0014
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: s_and_b32 s3, s4, 0xffe
-; SI-NEXT: s_sub_i32 s4, 0x3f1, s5
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; SI-NEXT: v_med3_i32 v5, s4, 0, 13
-; SI-NEXT: v_readfirstlane_b32 s4, v3
-; SI-NEXT: s_or_b32 s3, s3, s4
+; SI-NEXT: v_cvt_f32_f16_e32 v18, v7
+; SI-NEXT: v_fma_f64 v[5:6], v[5:6], v[2:3], v[0:1]
+; SI-NEXT: v_cvt_f32_f16_e32 v19, v10
+; SI-NEXT: v_readfirstlane_b32 s3, v6
+; SI-NEXT: s_and_b32 s0, s3, 0x1ff
+; SI-NEXT: v_or_b32_e32 v5, s0, v5
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s3, 8
+; SI-NEXT: s_bfe_u32 s6, s3, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s7, 0x3f1, s6
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v5, s7, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
; SI-NEXT: v_readfirstlane_b32 s7, v5
-; SI-NEXT: s_or_b32 s4, s3, 0x1000
-; SI-NEXT: s_lshr_b32 s12, s4, s7
+; SI-NEXT: s_lshr_b32 s12, s1, s7
; SI-NEXT: s_lshl_b32 s7, s12, s7
-; SI-NEXT: s_cmp_lg_u32 s7, s4
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_addk_i32 s5, 0xfc10
-; SI-NEXT: s_lshl_b32 s7, s5, 12
-; SI-NEXT: s_or_b32 s4, s12, s4
-; SI-NEXT: s_or_b32 s7, s3, s7
-; SI-NEXT: s_cmp_lt_i32 s5, 1
-; SI-NEXT: s_cselect_b32 s4, s4, s7
-; SI-NEXT: s_and_b32 s7, s4, 7
+; SI-NEXT: s_cmp_lg_u32 s7, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
+; SI-NEXT: s_addk_i32 s6, 0xfc10
+; SI-NEXT: s_lshl_b32 s7, s6, 12
+; SI-NEXT: s_or_b32 s1, s12, s1
+; SI-NEXT: s_or_b32 s7, s0, s7
+; SI-NEXT: s_cmp_lt_i32 s6, 1
+; SI-NEXT: s_cselect_b32 s1, s1, s7
+; SI-NEXT: s_and_b32 s7, s1, 7
; SI-NEXT: s_cmp_gt_i32 s7, 5
-; SI-NEXT: v_cvt_f64_f32_e32 v[7:8], v17
-; SI-NEXT: v_cvt_f64_f32_e32 v[9:10], v18
-; SI-NEXT: v_cvt_f64_f32_e32 v[17:18], v19
; SI-NEXT: s_cselect_b32 s12, 1, 0
; SI-NEXT: s_cmp_eq_u32 s7, 3
+; SI-NEXT: v_cvt_f32_f16_e32 v20, v8
+; SI-NEXT: v_cvt_f64_f32_e32 v[7:8], v13
+; SI-NEXT: v_cvt_f64_f32_e32 v[9:10], v14
+; SI-NEXT: v_cvt_f64_f32_e32 v[11:12], v15
+; SI-NEXT: v_cvt_f64_f32_e32 v[13:14], v16
+; SI-NEXT: v_cvt_f64_f32_e32 v[15:16], v17
+; SI-NEXT: v_cvt_f64_f32_e32 v[17:18], v18
; SI-NEXT: s_cselect_b32 s7, 1, 0
-; SI-NEXT: s_lshr_b32 s4, s4, 2
+; SI-NEXT: s_lshr_b32 s1, s1, 2
; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_add_i32 s4, s4, s7
-; SI-NEXT: v_fma_f64 v[7:8], v[17:18], v[9:10], v[7:8]
-; SI-NEXT: s_cmp_lt_i32 s5, 31
-; SI-NEXT: s_cselect_b32 s4, s4, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s3, 0
-; SI-NEXT: v_readfirstlane_b32 s6, v8
-; SI-NEXT: s_cselect_b32 s3, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s5, 0x40f
-; SI-NEXT: s_cselect_b32 s3, s3, s4
-; SI-NEXT: s_and_b32 s4, s6, 0x1ff
-; SI-NEXT: v_or_b32_e32 v3, s4, v7
-; SI-NEXT: s_lshr_b32 s0, s0, 16
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; SI-NEXT: s_lshr_b32 s5, s6, 8
-; SI-NEXT: s_bfe_u32 s7, s6, 0xb0014
-; SI-NEXT: s_and_b32 s0, s0, 0x8000
-; SI-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; SI-NEXT: s_and_b32 s4, s5, 0xffe
-; SI-NEXT: s_sub_i32 s5, 0x3f1, s7
-; SI-NEXT: s_or_b32 s0, s0, s3
-; SI-NEXT: v_readfirstlane_b32 s3, v3
-; SI-NEXT: v_med3_i32 v5, s5, 0, 13
-; SI-NEXT: s_or_b32 s3, s4, s3
-; SI-NEXT: v_readfirstlane_b32 s5, v5
-; SI-NEXT: s_or_b32 s4, s3, 0x1000
-; SI-NEXT: s_lshr_b32 s12, s4, s5
-; SI-NEXT: s_and_b32 s0, s0, 0xffff
-; SI-NEXT: s_lshl_b32 s5, s12, s5
-; SI-NEXT: s_cmp_lg_u32 s5, s4
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_addk_i32 s7, 0xfc10
-; SI-NEXT: s_lshl_b32 s5, s7, 12
-; SI-NEXT: s_or_b32 s4, s12, s4
-; SI-NEXT: s_or_b32 s5, s3, s5
-; SI-NEXT: s_cmp_lt_i32 s7, 1
-; SI-NEXT: s_cselect_b32 s4, s4, s5
-; SI-NEXT: s_and_b32 s5, s4, 7
-; SI-NEXT: s_cmp_gt_i32 s5, 5
+; SI-NEXT: s_add_i32 s1, s1, s7
+; SI-NEXT: s_cmp_lt_i32 s6, 31
+; SI-NEXT: v_fma_f64 v[7:8], v[11:12], v[9:10], v[7:8]
+; SI-NEXT: v_fma_f64 v[9:10], v[17:18], v[15:16], v[13:14]
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s6, 0x40f
+; SI-NEXT: v_readfirstlane_b32 s5, v10
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s3, 16
+; SI-NEXT: s_and_b32 s3, s5, 0x1ff
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-NEXT: v_or_b32_e32 v5, s3, v9
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; SI-NEXT: s_and_b32 s3, s0, 0xffff
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s5, 8
+; SI-NEXT: s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s7, 0x3f1, s6
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v5, s7, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s7, v5
+; SI-NEXT: s_lshr_b32 s12, s1, s7
+; SI-NEXT: s_lshl_b32 s7, s12, s7
+; SI-NEXT: s_cmp_lg_u32 s7, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
+; SI-NEXT: s_addk_i32 s6, 0xfc10
+; SI-NEXT: s_lshl_b32 s7, s6, 12
+; SI-NEXT: s_or_b32 s1, s12, s1
+; SI-NEXT: s_or_b32 s7, s0, s7
+; SI-NEXT: s_cmp_lt_i32 s6, 1
+; SI-NEXT: s_cselect_b32 s1, s1, s7
+; SI-NEXT: s_and_b32 s7, s1, 7
+; SI-NEXT: s_cmp_gt_i32 s7, 5
; SI-NEXT: s_cselect_b32 s12, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s5, 3
-; SI-NEXT: s_cselect_b32 s5, 1, 0
-; SI-NEXT: s_lshr_b32 s4, s4, 2
-; SI-NEXT: s_or_b32 s5, s5, s12
-; SI-NEXT: s_add_i32 s4, s4, s5
-; SI-NEXT: s_cmp_lt_i32 s7, 31
-; SI-NEXT: s_cselect_b32 s4, s4, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s3, 0
-; SI-NEXT: s_cselect_b32 s3, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; SI-NEXT: s_cselect_b32 s3, s3, s4
-; SI-NEXT: s_and_b32 s5, s1, 0x1ff
-; SI-NEXT: s_lshr_b32 s4, s6, 16
-; SI-NEXT: v_or_b32_e32 v2, s5, v2
-; SI-NEXT: s_and_b32 s4, s4, 0x8000
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2
-; SI-NEXT: s_lshr_b32 s6, s1, 8
-; SI-NEXT: s_bfe_u32 s7, s1, 0xb0014
-; SI-NEXT: s_or_b32 s3, s4, s3
-; SI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; SI-NEXT: s_and_b32 s5, s6, 0xffe
-; SI-NEXT: s_sub_i32 s6, 0x3f1, s7
-; SI-NEXT: s_lshl_b32 s3, s3, 16
-; SI-NEXT: v_readfirstlane_b32 s4, v2
-; SI-NEXT: v_med3_i32 v3, s6, 0, 13
-; SI-NEXT: s_or_b32 s0, s0, s3
-; SI-NEXT: s_or_b32 s3, s5, s4
-; SI-NEXT: v_readfirstlane_b32 s6, v3
-; SI-NEXT: s_or_b32 s4, s3, 0x1000
-; SI-NEXT: s_lshr_b32 s5, s4, s6
-; SI-NEXT: s_lshl_b32 s6, s5, s6
-; SI-NEXT: s_cmp_lg_u32 s6, s4
-; SI-NEXT: s_cselect_b32 s4, 1, 0
-; SI-NEXT: s_addk_i32 s7, 0xfc10
-; SI-NEXT: s_or_b32 s4, s5, s4
-; SI-NEXT: s_lshl_b32 s5, s7, 12
-; SI-NEXT: s_or_b32 s5, s3, s5
-; SI-NEXT: s_cmp_lt_i32 s7, 1
-; SI-NEXT: s_cselect_b32 s4, s4, s5
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: s_and_b32 s5, s4, 7
-; SI-NEXT: s_cmp_gt_i32 s5, 5
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-NEXT: s_cmp_eq_u32 s7, 3
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s1, 2
+; SI-NEXT: s_or_b32 s7, s7, s12
+; SI-NEXT: s_add_i32 s1, s1, s7
+; SI-NEXT: s_cmp_lt_i32 s6, 31
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s6, 0x40f
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s5, 16
+; SI-NEXT: v_readfirstlane_b32 s4, v8
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-NEXT: s_and_b32 s5, s4, 0x1ff
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_or_b32_e32 v5, s5, v7
+; SI-NEXT: s_lshl_b32 s0, s0, 16
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
+; SI-NEXT: s_or_b32 s3, s3, s0
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s4, 8
+; SI-NEXT: s_bfe_u32 s5, s4, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
+; SI-NEXT: s_sub_i32 s6, 0x3f1, s5
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_med3_i32 v5, s6, 0, 13
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
+; SI-NEXT: v_readfirstlane_b32 s6, v5
+; SI-NEXT: s_lshr_b32 s7, s1, s6
+; SI-NEXT: s_lshl_b32 s6, s7, s6
+; SI-NEXT: s_cmp_lg_u32 s6, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
+; SI-NEXT: s_addk_i32 s5, 0xfc10
+; SI-NEXT: s_lshl_b32 s6, s5, 12
+; SI-NEXT: s_or_b32 s1, s7, s1
+; SI-NEXT: s_or_b32 s6, s0, s6
+; SI-NEXT: s_cmp_lt_i32 s5, 1
+; SI-NEXT: s_cselect_b32 s1, s1, s6
+; SI-NEXT: s_and_b32 s6, s1, 7
+; SI-NEXT: s_cmp_gt_i32 s6, 5
+; SI-NEXT: s_cselect_b32 s7, 1, 0
+; SI-NEXT: s_cmp_eq_u32 s6, 3
; SI-NEXT: s_cselect_b32 s6, 1, 0
-; SI-NEXT: s_cmp_eq_u32 s5, 3
-; SI-NEXT: s_cselect_b32 s5, 1, 0
-; SI-NEXT: s_or_b32 s5, s5, s6
-; SI-NEXT: s_lshr_b32 s4, s4, 2
-; SI-NEXT: s_add_i32 s4, s4, s5
-; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
+; SI-NEXT: s_lshr_b32 s1, s1, 2
+; SI-NEXT: s_or_b32 s6, s6, s7
+; SI-NEXT: v_cvt_f64_f32_e32 v[0:1], v19
+; SI-NEXT: v_cvt_f64_f32_e32 v[2:3], v20
+; SI-NEXT: s_add_i32 s1, s1, s6
; SI-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; SI-NEXT: s_cmp_lt_i32 s7, 31
-; SI-NEXT: s_cselect_b32 s4, s4, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s3, 0
-; SI-NEXT: s_cselect_b32 s3, s2, 0x7c00
-; SI-NEXT: s_cmpk_eq_i32 s7, 0x40f
-; SI-NEXT: s_cselect_b32 s3, s3, s4
-; SI-NEXT: s_lshr_b32 s1, s1, 16
+; SI-NEXT: s_cmp_lt_i32 s5, 31
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
+; SI-NEXT: s_cmpk_eq_i32 s5, 0x40f
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s4, 16
; SI-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
; SI-NEXT: s_and_b32 s1, s1, 0x8000
-; SI-NEXT: s_or_b32 s1, s1, s3
-; SI-NEXT: v_readfirstlane_b32 s3, v1
-; SI-NEXT: s_and_b32 s4, s3, 0x1ff
-; SI-NEXT: v_or_b32_e32 v0, s4, v0
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: v_readfirstlane_b32 s5, v1
+; SI-NEXT: s_and_b32 s4, s0, 0xffff
+; SI-NEXT: s_and_b32 s0, s5, 0x1ff
+; SI-NEXT: v_or_b32_e32 v0, s0, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_lshr_b32 s4, s3, 8
-; SI-NEXT: s_bfe_u32 s6, s3, 0xb0014
-; SI-NEXT: s_and_b32 s4, s4, 0xffe
-; SI-NEXT: v_readfirstlane_b32 s5, v0
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: s_lshr_b32 s1, s5, 8
+; SI-NEXT: s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT: s_and_b32 s1, s1, 0xffe
; SI-NEXT: s_sub_i32 s7, 0x3f1, s6
-; SI-NEXT: s_or_b32 s4, s4, s5
+; SI-NEXT: s_or_b32 s0, s1, s0
; SI-NEXT: v_med3_i32 v0, s7, 0, 13
-; SI-NEXT: s_or_b32 s5, s4, 0x1000
+; SI-NEXT: s_or_b32 s1, s0, 0x1000
; SI-NEXT: v_readfirstlane_b32 s7, v0
-; SI-NEXT: s_lshr_b32 s12, s5, s7
-; SI-NEXT: s_and_b32 s1, s1, 0xffff
+; SI-NEXT: s_lshr_b32 s12, s1, s7
; SI-NEXT: s_lshl_b32 s7, s12, s7
-; SI-NEXT: s_cmp_lg_u32 s7, s5
-; SI-NEXT: s_cselect_b32 s5, 1, 0
+; SI-NEXT: s_cmp_lg_u32 s7, s1
+; SI-NEXT: s_cselect_b32 s1, 1, 0
; SI-NEXT: s_addk_i32 s6, 0xfc10
; SI-NEXT: s_lshl_b32 s7, s6, 12
-; SI-NEXT: s_or_b32 s5, s12, s5
-; SI-NEXT: s_or_b32 s7, s4, s7
+; SI-NEXT: s_or_b32 s1, s12, s1
+; SI-NEXT: s_or_b32 s7, s0, s7
; SI-NEXT: s_cmp_lt_i32 s6, 1
-; SI-NEXT: s_cselect_b32 s5, s5, s7
-; SI-NEXT: s_and_b32 s7, s5, 7
+; SI-NEXT: s_cselect_b32 s1, s1, s7
+; SI-NEXT: s_and_b32 s7, s1, 7
; SI-NEXT: s_cmp_gt_i32 s7, 5
; SI-NEXT: s_cselect_b32 s12, 1, 0
; SI-NEXT: s_cmp_eq_u32 s7, 3
; SI-NEXT: s_cselect_b32 s7, 1, 0
; SI-NEXT: s_or_b32 s7, s7, s12
-; SI-NEXT: s_lshr_b32 s5, s5, 2
-; SI-NEXT: s_add_i32 s5, s5, s7
+; SI-NEXT: s_lshr_b32 s1, s1, 2
+; SI-NEXT: s_add_i32 s1, s1, s7
; SI-NEXT: s_cmp_lt_i32 s6, 31
-; SI-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; SI-NEXT: s_cmp_lg_u32 s4, 0
-; SI-NEXT: s_cselect_b32 s2, s2, 0x7c00
+; SI-NEXT: s_cselect_b32 s1, s1, 0x7c00
+; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cselect_b32 s0, s2, 0x7c00
; SI-NEXT: s_cmpk_eq_i32 s6, 0x40f
-; SI-NEXT: s_cselect_b32 s2, s2, s5
-; SI-NEXT: s_lshr_b32 s3, s3, 16
-; SI-NEXT: s_and_b32 s3, s3, 0x8000
-; SI-NEXT: s_or_b32 s2, s3, s2
-; SI-NEXT: s_lshl_b32 s2, s2, 16
-; SI-NEXT: s_or_b32 s1, s1, s2
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: s_cselect_b32 s0, s0, s1
+; SI-NEXT: s_lshr_b32 s1, s5, 16
+; SI-NEXT: s_and_b32 s1, s1, 0x8000
+; SI-NEXT: s_or_b32 s0, s1, s0
+; SI-NEXT: s_lshl_b32 s0, s0, 16
+; SI-NEXT: s_or_b32 s0, s4, s0
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s3
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
index 0c8dbe865a872..4d545880523a1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.set.rounding.ll
@@ -1162,53 +1162,27 @@ define amdgpu_gfx void @s_set_rounding_i3_zeroext(i3 zeroext inreg %rounding) {
}
define amdgpu_gfx void @s_set_rounding_select_0_1(i32 inreg %cond) {
-; GFX6-LABEL: s_set_rounding_select_0_1:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: s_cmp_lg_u32 s4, 0
-; GFX6-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[34:35]
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX6-NEXT: v_lshr_b32_e32 v0, 0xa50f, v0
-; GFX6-NEXT: v_readfirstlane_b32 s34, v0
-; GFX6-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX7-LABEL: s_set_rounding_select_0_1:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_cmp_lg_u32 s4, 0
-; GFX7-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GFX7-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[34:35]
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX7-NEXT: v_lshr_b32_e32 v0, 0xa50f, v0
-; GFX7-NEXT: v_readfirstlane_b32 s34, v0
-; GFX7-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX7-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: s_set_rounding_select_0_1:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_cmp_lg_u32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[34:35]
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX8-NEXT: s_mov_b32 s34, 0xa50f
-; GFX8-NEXT: v_lshrrev_b32_e64 v0, v0, s34
-; GFX8-NEXT: v_readfirstlane_b32 s34, v0
-; GFX8-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX678-LABEL: s_set_rounding_select_0_1:
+; GFX678: ; %bb.0:
+; GFX678-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX678-NEXT: s_cmp_lg_u32 s4, 0
+; GFX678-NEXT: s_cselect_b64 s[34:35], -1, 0
+; GFX678-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GFX678-NEXT: s_cselect_b32 s34, 1, 0
+; GFX678-NEXT: s_lshl_b32 s34, s34, 2
+; GFX678-NEXT: s_lshr_b32 s34, 0xa50f, s34
+; GFX678-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
+; GFX678-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: s_set_rounding_select_0_1:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s4, 0
; GFX9-NEXT: s_cselect_b64 s[34:35], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[34:35]
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: s_mov_b32 s34, 0xa50f
-; GFX9-NEXT: v_lshrrev_b32_e64 v0, v0, s34
-; GFX9-NEXT: v_readfirstlane_b32 s34, v0
+; GFX9-NEXT: s_and_b64 s[34:35], s[34:35], exec
+; GFX9-NEXT: s_cselect_b32 s34, 1, 0
+; GFX9-NEXT: s_lshl_b32 s34, s34, 2
+; GFX9-NEXT: s_lshr_b32 s34, 0xa50f, s34
; GFX9-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -1217,10 +1191,10 @@ define amdgpu_gfx void @s_set_rounding_select_0_1(i32 inreg %cond) {
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u32 s4, 0
; GFX10-NEXT: s_cselect_b32 s34, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s34
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-NEXT: v_lshrrev_b32_e64 v0, v0, 0xa50f
-; GFX10-NEXT: v_readfirstlane_b32 s34, v0
+; GFX10-NEXT: s_and_b32 s34, s34, exec_lo
+; GFX10-NEXT: s_cselect_b32 s34, 1, 0
+; GFX10-NEXT: s_lshl_b32 s34, s34, 2
+; GFX10-NEXT: s_lshr_b32 s34, 0xa50f, s34
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s34
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
@@ -1229,10 +1203,10 @@ define amdgpu_gfx void @s_set_rounding_select_0_1(i32 inreg %cond) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s4, 0
; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: v_lshrrev_b32_e64 v0, v0, 0xa50f
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_lshl_b32 s0, s0, 2
+; GFX11-NEXT: s_lshr_b32 s0, 0xa50f, s0
; GFX11-NEXT: s_setreg_b32 hwreg(HW_REG_MODE, 0, 4), s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %cond, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll b/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
index caedcc5e44ba1..853ac67dc0bba 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
@@ -335,14 +335,14 @@ define amdgpu_gfx ptr addrspace(5) @sponentry_gfx_dyn_alloc(i32 %val) #0 {
; DAGISEL-NEXT: s_cmp_lg_u32 s1, 0
; DAGISEL-NEXT: s_cbranch_scc1 .LBB9_1
; DAGISEL-NEXT: ; %bb.2:
-; DAGISEL-NEXT: s_mov_b32 s1, s32
-; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; DAGISEL-NEXT: v_lshl_add_u32 v1, s0, 5, s1
+; DAGISEL-NEXT: v_mov_b32_e32 v1, s32
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 off, v0, s1 scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 v1, v0, off scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
; DAGISEL-NEXT: v_mov_b32_e32 v0, s33
-; DAGISEL-NEXT: v_readfirstlane_b32 s32, v1
+; DAGISEL-NEXT: v_lshl_add_u32 v2, s0, 5, v1
+; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; DAGISEL-NEXT: v_readfirstlane_b32 s32, v2
; DAGISEL-NEXT: s_mov_b32 s32, s33
; DAGISEL-NEXT: s_mov_b32 s33, s34
; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -517,16 +517,16 @@ define amdgpu_cs_chain void @sponentry_cs_chain_dyn_alloc(i32 %val) #0 {
; DAGISEL-NEXT: s_cmp_lg_u32 s1, 0
; DAGISEL-NEXT: s_cbranch_scc1 .LBB12_1
; DAGISEL-NEXT: ; %bb.2:
-; DAGISEL-NEXT: s_mov_b32 s1, s32
-; DAGISEL-NEXT: v_mov_b32_e32 v1, s33
-; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; DAGISEL-NEXT: v_lshl_add_u32 v0, s0, 5, s1
+; DAGISEL-NEXT: v_mov_b32_e32 v0, s32
+; DAGISEL-NEXT: v_mov_b32_e32 v2, s33
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 off, v8, s1 scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 v0, v8, off scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 off, v1, s1 scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 v0, v2, off scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: v_readfirstlane_b32 s32, v0
+; DAGISEL-NEXT: v_lshl_add_u32 v1, s0, 5, v0
+; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; DAGISEL-NEXT: v_readfirstlane_b32 s32, v1
; DAGISEL-NEXT: s_alloc_vgpr 0
; DAGISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 5283233a0b461..80fd4c43ab5e6 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -129,10 +129,10 @@ define amdgpu_kernel void @v_test_imin_sle_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imin_sle_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -227,8 +227,8 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imin_sle_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -315,8 +315,8 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
;
; GFX1250-LABEL: s_test_imin_sle_v1i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -442,10 +442,10 @@ define amdgpu_kernel void @s_test_imin_sle_v4i32(ptr addrspace(1) %out, <4 x i32
;
; GFX1250-LABEL: s_test_imin_sle_v4i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s11, s15
@@ -578,11 +578,11 @@ define amdgpu_kernel void @s_test_imin_sle_i8(ptr addrspace(1) %out, [8 x i32],
;
; GFX1250-LABEL: s_test_imin_sle_i8:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i8 s2, s2
@@ -834,11 +834,11 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
;
; GFX1250-LABEL: s_test_imin_sle_v4i8:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i16 s5, s2
@@ -980,8 +980,8 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
;
; GFX1250-LABEL: s_test_imin_sle_v2i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_i16 v1, s2, s3
@@ -1177,10 +1177,10 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
;
; GFX1250-LABEL: s_test_imin_sle_v4i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_i16 v1, s1, s3
@@ -1313,10 +1313,10 @@ define amdgpu_kernel void @v_test_imin_slt_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_imin_slt_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -1491,10 +1491,10 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-TRUE16-LABEL: v_test_imin_slt_i16:
; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_clause 0x1
@@ -1507,10 +1507,10 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-FAKE16-LABEL: v_test_imin_slt_i16:
; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: s_clause 0x1
@@ -1606,8 +1606,8 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_imin_slt_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -1711,10 +1711,10 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
;
; GFX1250-LABEL: s_test_imin_slt_v2i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s0, s0, s2
@@ -1809,8 +1809,8 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imin_slt_imm_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, 8
@@ -1904,8 +1904,8 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX1250-LABEL: s_test_imin_sle_imm_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, 8
@@ -2039,10 +2039,10 @@ define amdgpu_kernel void @v_test_umin_ule_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umin_ule_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2200,10 +2200,10 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_umin_ule_v3i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 4, v0
@@ -2404,10 +2404,10 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_umin_ule_v3i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2508,8 +2508,8 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umin_ule_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -2643,10 +2643,10 @@ define amdgpu_kernel void @v_test_umin_ult_i32(ptr addrspace(1) %out, ptr addrsp
;
; GFX1250-LABEL: v_test_umin_ult_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2804,10 +2804,10 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-TRUE16-LABEL: v_test_umin_ult_i8:
; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_clause 0x1
@@ -2820,10 +2820,10 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
;
; GFX1250-FAKE16-LABEL: v_test_umin_ult_i8:
; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: s_clause 0x1
@@ -2919,8 +2919,8 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
;
; GFX1250-LABEL: s_test_umin_ult_i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -2984,12 +2984,13 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_cmp_lt_u32 s4, s5
; CI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CI-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
; CI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; CI-NEXT: s_cselect_b32 s0, s4, s5
-; CI-NEXT: v_mov_b32_e32 v5, s0
-; CI-NEXT: flat_store_dword v[0:1], v5
-; CI-NEXT: flat_store_byte v[2:3], v4
+; CI-NEXT: s_cselect_b32 s1, 1, 0
+; CI-NEXT: v_mov_b32_e32 v4, s0
+; CI-NEXT: v_mov_b32_e32 v5, s1
+; CI-NEXT: flat_store_dword v[0:1], v4
+; CI-NEXT: flat_store_byte v[2:3], v5
; CI-NEXT: s_endpgm
;
; VI-LABEL: v_test_umin_ult_i32_multi_use:
@@ -3008,12 +3009,13 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lt_u32 s4, s5
; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v4, 0, 1, s[0:1]
; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: s_cselect_b32 s0, s4, s5
-; VI-NEXT: v_mov_b32_e32 v5, s0
-; VI-NEXT: flat_store_dword v[0:1], v5
-; VI-NEXT: flat_store_byte v[2:3], v4
+; VI-NEXT: s_cselect_b32 s1, 1, 0
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: flat_store_dword v[0:1], v4
+; VI-NEXT: flat_store_byte v[2:3], v5
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_umin_ult_i32_multi_use:
@@ -3026,71 +3028,75 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lt_u32 s8, s9
; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, s8, s9
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
-; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
+; GFX9-NEXT: s_cselect_b32 s5, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: global_store_byte v0, v2, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: v_test_umin_ult_i32_multi_use:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s8, s[4:5], 0x0
; GFX10-NEXT: s_load_dword s9, s[6:7], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lt_u32 s8, s9
; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s4
; GFX10-NEXT: s_and_b32 s4, s4, exec_lo
; GFX10-NEXT: s_cselect_b32 s4, s8, s9
-; GFX10-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-NEXT: global_store_dword v1, v2, s[0:1]
-; GFX10-NEXT: global_store_byte v1, v0, s[2:3]
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: global_store_byte v0, v2, s[2:3]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_test_umin_ult_i32_multi_use:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x0
; GFX11-NEXT: s_load_b32 s5, s[6:7], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_lt_u32 s4, s5
; GFX11-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s6
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
; GFX11-NEXT: s_cselect_b32 s4, s4, s5
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v2, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]
-; GFX11-NEXT: global_store_b8 v1, v0, s[2:3]
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]
; GFX11-NEXT: s_endpgm
;
; GFX1250-LABEL: v_test_umin_ult_i32_multi_use:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
+; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s0, s[12:13], 0x0
; GFX1250-NEXT: s_load_b32 s1, s[14:15], 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s2, s2, exec_lo
; GFX1250-NEXT: s_cselect_b32 s0, s0, s1
-; GFX1250-NEXT: v_mov_b32_e32 v2, s0
+; GFX1250-NEXT: s_cselect_b32 s1, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_store_b32 v1, v2, s[8:9]
-; GFX1250-NEXT: global_store_b8 v1, v0, s[10:11]
+; GFX1250-NEXT: global_store_b32 v0, v1, s[8:9]
+; GFX1250-NEXT: global_store_b8 v0, v2, s[10:11]
; GFX1250-NEXT: s_endpgm
%a = load i32, ptr addrspace(1) %aptr, align 4
%b = load i32, ptr addrspace(1) %bptr, align 4
@@ -3162,9 +3168,11 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; CI-NEXT: v_mov_b32_e32 v3, s3
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_cmp_lt_u32_e32 vcc, v4, v5
+; CI-NEXT: s_and_b64 s[0:1], vcc, exec
; CI-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
+; CI-NEXT: s_cselect_b32 s0, 1, 0
; CI-NEXT: flat_store_short v[0:1], v4
-; CI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: flat_store_byte v[2:3], v0
; CI-NEXT: s_endpgm
;
@@ -3190,9 +3198,11 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v7, 0xffff, v5
; VI-NEXT: v_cmp_lt_u32_e32 vcc, v7, v6
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
; VI-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
+; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_short v[0:1], v4
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_byte v[2:3], v0
; VI-NEXT: s_endpgm
;
@@ -3207,7 +3217,9 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX9-NEXT: v_cmp_lt_u32_sdwa vcc, v1, v2 src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-NEXT: global_store_short v0, v1, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
@@ -3221,8 +3233,10 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX10-NEXT: global_load_ushort v2, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cmp_lt_u32_sdwa vcc_lo, v1, v2 src0_sel:WORD_0 src1_sel:WORD_0
+; GFX10-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s4, 1, 0
; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
; GFX10-NEXT: global_store_short v0, v1, s[0:1]
; GFX10-NEXT: global_store_byte v0, v2, s[2:3]
; GFX10-NEXT: s_endpgm
@@ -3241,8 +3255,10 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT: global_store_b8 v0, v2, s[2:3]
@@ -3262,8 +3278,10 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cmp_lt_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: global_store_b8 v0, v2, s[2:3]
@@ -3271,8 +3289,8 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
;
; GFX1250-LABEL: v_test_umin_ult_i16_multi_use:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -3284,8 +3302,10 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v4, v3
+; GFX1250-NEXT: s_and_b32 s0, vcc_lo, exec_lo
+; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: v_mov_b32_e32 v2, s0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b16 v0, v1, s[8:9]
; GFX1250-NEXT: global_store_b8 v0, v2, s[10:11]
@@ -3372,8 +3392,8 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
;
; GFX1250-LABEL: s_test_umin_ult_v1i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -3562,10 +3582,10 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
;
; GFX1250-LABEL: s_test_umin_ult_v8i32:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b512 s[8:23], s[4:5], 0x20
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b512 s[8:23], s[4:5], 0x20 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s4, s9, s17
@@ -3879,10 +3899,10 @@ define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16
;
; GFX1250-LABEL: s_test_umin_ult_v8i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_u16 v3, s11, s15
@@ -4004,11 +4024,11 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umin_ult_i16(ptr addrspac
;
; GFX1250-LABEL: simplify_demanded_bits_test_umin_ult_i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
@@ -4134,11 +4154,11 @@ define amdgpu_kernel void @simplify_demanded_bits_test_min_slt_i16(ptr addrspace
;
; GFX1250-LABEL: simplify_demanded_bits_test_min_slt_i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i16 s2, s2
@@ -4267,8 +4287,8 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
;
; GFX1250-LABEL: s_test_imin_sle_i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
@@ -4396,10 +4416,10 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umin_ult_i64:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
@@ -4521,10 +4541,10 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_umin_ule_i64:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
@@ -4646,10 +4666,10 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imin_slt_i64:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
@@ -4771,10 +4791,10 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
;
; GFX1250-LABEL: test_imin_sle_i64:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
@@ -4934,10 +4954,10 @@ define amdgpu_kernel void @v_test_imin_sle_v2i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_imin_sle_v2i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -5105,10 +5125,10 @@ define amdgpu_kernel void @v_test_imin_ule_v2i16(ptr addrspace(1) %out, ptr addr
;
; GFX1250-LABEL: v_test_imin_ule_v2i16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
diff --git a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
index d66b9029dbf99..2bbb31eed423c 100644
--- a/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_cmp_0.ll
@@ -15,8 +15,8 @@ define amdgpu_ps i32 @add32(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], 0, -1
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = add i32 %val0, 1
call void asm "; use $0", "s"(i32 %result)
@@ -30,8 +30,8 @@ define amdgpu_ps i32 @shl32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_lshl_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = shl i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -88,8 +88,8 @@ define amdgpu_ps i32 @lshr32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_lshr_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = lshr i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -116,8 +116,8 @@ define amdgpu_ps i32 @ashr32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_ashr_i32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = ashr i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -147,8 +147,8 @@ define amdgpu_ps i32 @abs32(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%neg = sub i32 0, %val0
%cond = icmp sgt i32 %val0, %neg
@@ -164,8 +164,8 @@ define amdgpu_ps i32 @absdiff32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_absdiff_i32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%diff = sub i32 %val0, %val1
%result = call i32 @llvm.abs.i32(i32 %diff, i1 false)
@@ -179,8 +179,8 @@ define amdgpu_ps i32 @and32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_and_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = and i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -210,8 +210,8 @@ define amdgpu_ps i32 @and32_clear_one_bit(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = and i32 %val0, 2147483647
call void asm "; use $0", "s"(i32 %result)
@@ -244,8 +244,8 @@ define amdgpu_ps i32 @or32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_or_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = or i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -272,8 +272,8 @@ define amdgpu_ps i32 @xor32(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_xor_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = xor i32 %val0, %val1
%cmp = icmp ne i32 %result, 0
@@ -303,8 +303,8 @@ define amdgpu_ps i32 @nand32(i32 inreg %val0, i32 inreg %val1) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = and i32 %val0, %val1
%result2 = xor i32 %result, -1
@@ -341,8 +341,8 @@ define amdgpu_ps i32 @nor32(i32 inreg %val0, i32 inreg %val1) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = or i32 %val0, %val1
%result2 = xor i32 %result, -1
@@ -379,8 +379,8 @@ define amdgpu_ps i32 @xnor32(i32 inreg %val0, i32 inreg %val1) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = xor i32 %val0, %val1
%result2 = xor i32 %result, -1
@@ -414,8 +414,8 @@ define amdgpu_ps i32 @andn232(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_andn2_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%nval1 = xor i32 %val1, -1
%result = and i32 %val0, %nval1
@@ -444,8 +444,8 @@ define amdgpu_ps i32 @orn232(i32 inreg %val0, i32 inreg %val1) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_orn2_b32 s0, s0, s1
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%nval1 = xor i32 %val1, -1
%result = or i32 %val0, %nval1
@@ -474,8 +474,8 @@ define amdgpu_ps i32 @bfe_i32(i32 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_bfe_i32 s0, s0, 0x80010
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%shl = shl i32 %val0, 8
%result = ashr i32 %shl, 24
@@ -511,8 +511,8 @@ define amdgpu_ps i32 @bfe_u32(i32 inreg %val0) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_bfe_u32 s0, s0, 0x80010
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%shl = shl i32 %val0, 8
%result = lshr i32 %shl, 24
@@ -552,8 +552,8 @@ define amdgpu_ps i32 @bcnt032(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = call i32 @llvm.ctpop.i32(i32 %val0) nounwind readnone
%result2 = sub i32 32, %result
@@ -593,8 +593,8 @@ define amdgpu_ps i32 @bcnt132(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = call i32 @llvm.ctpop.i32(i32 %val0) nounwind readnone
call void asm "; use $0", "s"(i32 %result)
@@ -631,8 +631,8 @@ define amdgpu_ps i32 @quadmask32(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = call i32 @llvm.amdgcn.s.quadmask.i32(i32 %val0) nounwind readnone
call void asm "; use $0", "s"(i32 %result)
@@ -667,8 +667,8 @@ define amdgpu_ps i32 @not32(i32 inreg %val0) {
; CHECK-NEXT: ; use s0
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_cselect_b32 s0, 1, 0
; CHECK-NEXT: ; return to shader part epilog
%result = xor i32 %val0, -1
call void asm "; use $0", "s"(i32 %result)
diff --git a/llvm/test/CodeGen/AMDGPU/saddo.ll b/llvm/test/CodeGen/AMDGPU/saddo.ll
index 0e9ea5430d0ea..88b1c4511fc40 100644
--- a/llvm/test/CodeGen/AMDGPU/saddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddo.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefix=SI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefix=VI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 | FileCheck %s --check-prefix=GFX9
@@ -185,10 +185,12 @@ define amdgpu_kernel void @s_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_add_i32 s4, s6, s7
; GFX9-NEXT: v_add_i32 v1, s6, v1 clamp
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
@@ -197,15 +199,17 @@ define amdgpu_kernel void @s_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_add_nc_i32 v0, s6, s7 clamp
; GFX10-NEXT: s_add_i32 s4, s6, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v2, s[0:1]
-; GFX10-NEXT: global_store_byte v1, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: global_store_byte v0, v2, s[2:3]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_saddo_i32:
@@ -217,12 +221,16 @@ define amdgpu_kernel void @s_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: v_add_nc_i32 v0, s6, s7 clamp
; GFX11-NEXT: s_add_i32 s4, s6, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_and_b32 s5, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]
-; GFX11-NEXT: global_store_b8 v1, v0, s[2:3]
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]
; GFX11-NEXT: s_endpgm
%sadd = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 %a, i32 %b) nounwind
%val = extractvalue { i32, i1 } %sadd, 0
@@ -298,8 +306,10 @@ define amdgpu_kernel void @v_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_add_i32 v3, v1, v2 clamp
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v1, v3
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: global_store_dword v0, v1, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_byte v0, v1, s[10:11]
; GFX9-NEXT: s_endpgm
;
@@ -315,7 +325,9 @@ define amdgpu_kernel void @v_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: v_add_nc_i32 v3, v1, v2 clamp
; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT: s_and_b32 s0, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: global_store_dword v0, v1, s[8:9]
; GFX10-NEXT: global_store_byte v0, v2, s[10:11]
; GFX10-NEXT: s_endpgm
@@ -331,9 +343,11 @@ define amdgpu_kernel void @v_saddo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_nc_i32 v3, v1, v2 clamp
; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]
@@ -666,10 +680,14 @@ define amdgpu_kernel void @v_saddo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; GFX9-NEXT: v_add_i32 v1, v1, v3 clamp
; GFX9-NEXT: v_add_u32_e32 v4, v0, v2
; GFX9-NEXT: v_add_i32 v0, v0, v2 clamp
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v5, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], v5, v1
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v4, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]
; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[10:11]
; GFX9-NEXT: s_endpgm
@@ -688,34 +706,42 @@ define amdgpu_kernel void @v_saddo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; GFX10-NEXT: v_add_nc_u32_e32 v3, v0, v2
; GFX10-NEXT: v_add_nc_i32 v0, v0, v2 clamp
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u32_e64 s0, v3, v0
+; GFX10-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s1, 1, 0
+; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v5, v[3:4], s[8:9]
; GFX10-NEXT: global_store_dwordx2 v5, v[0:1], s[10:11]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_saddo_v2i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-NEXT: s_load_b256 s[4:11], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b64 v[0:1], v5, s[4:5]
-; GFX11-NEXT: global_load_b64 v[2:3], v5, s[6:7]
+; GFX11-NEXT: global_load_b64 v[0:1], v5, s[8:9]
+; GFX11-NEXT: global_load_b64 v[2:3], v5, s[10:11]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_nc_u32_e32 v4, v1, v3
; GFX11-NEXT: v_add_nc_i32 v1, v1, v3 clamp
; GFX11-NEXT: v_add_nc_u32_e32 v3, v0, v2
; GFX11-NEXT: v_add_nc_i32 v0, v0, v2 clamp
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_ne_u32_e64 s0, v3, v0
+; GFX11-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b64 v5, v[3:4], s[0:1]
-; GFX11-NEXT: global_store_b64 v5, v[0:1], s[2:3]
+; GFX11-NEXT: global_store_b64 v5, v[3:4], s[4:5]
+; GFX11-NEXT: global_store_b64 v5, v[0:1], s[6:7]
; GFX11-NEXT: s_endpgm
%a = load <2 x i32>, ptr addrspace(1) %aptr, align 4
%b = load <2 x i32>, ptr addrspace(1) %bptr, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-multiple-use.ll b/llvm/test/CodeGen/AMDGPU/setcc-multiple-use.ll
index 15aceeb8250f9..6414259d3f96d 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-multiple-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-multiple-use.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 < %s | FileCheck %s
; SelectionDAG generates a setcc node with multiple uses:
@@ -15,9 +15,10 @@ define i32 @f() {
; CHECK-NEXT: ds_read_b32 v0, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; CHECK-NEXT: s_and_b32 s4, vcc_lo, exec_lo
; CHECK-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; CHECK-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc_lo
+; CHECK-NEXT: s_cselect_b32 s4, 1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, s4, vcc_lo
; CHECK-NEXT: s_setpc_b64 s[30:31]
bb:
%i = load i32, ptr addrspace(3) zeroinitializer, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
index 031a55a2dc758..5816ef289d764 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
@@ -1,18 +1,59 @@
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
-; FUNC-LABEL: {{^}}sext_bool_icmp_eq_0:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT:buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
-
-; EG: SETNE_INT * [[CMP:T[0-9]+]].[[CMPCHAN:[XYZW]]], KC0[2].Z, KC0[2].W
-; EG: AND_INT T{{[0-9]+.[XYZW]}}, PS, 1
define amdgpu_kernel void @sext_bool_icmp_eq_0(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_eq_0:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_lg_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_eq_0:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_lg_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_eq_0:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETNE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, 0
@@ -20,17 +61,57 @@ define amdgpu_kernel void @sext_bool_icmp_eq_0(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}sext_bool_icmp_ne_0:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
-
-; EG: SETNE_INT * [[CMP:T[0-9]+]].[[CMPCHAN:[XYZW]]], KC0[2].Z, KC0[2].W
-; EG: AND_INT T{{[0-9]+.[XYZW]}}, PS, 1
define amdgpu_kernel void @sext_bool_icmp_ne_0(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_ne_0:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_lg_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_ne_0:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_lg_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_ne_0:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETNE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 0
@@ -38,14 +119,57 @@ define amdgpu_kernel void @sext_bool_icmp_ne_0(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}sext_bool_icmp_eq_neg1:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_eq_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @sext_bool_icmp_eq_neg1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_eq_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_eq_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_eq_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, -1
@@ -53,14 +177,57 @@ define amdgpu_kernel void @sext_bool_icmp_eq_neg1(ptr addrspace(1) %out, i32 %a,
ret void
}
-; FUNC-LABEL: {{^}}sext_bool_icmp_ne_neg1:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_eq_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @sext_bool_icmp_ne_neg1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_ne_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_ne_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_ne_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, -1
@@ -68,14 +235,57 @@ define amdgpu_kernel void @sext_bool_icmp_ne_neg1(ptr addrspace(1) %out, i32 %a,
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_eq_0:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_eq_0(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_eq_0:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_lg_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_eq_0:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_lg_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_eq_0:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETNE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, 0
@@ -83,14 +293,57 @@ define amdgpu_kernel void @zext_bool_icmp_eq_0(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_ne_0:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_lg_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_ne_0(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_ne_0:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_lg_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_ne_0:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_lg_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_ne_0:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETNE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 0
@@ -98,14 +351,57 @@ define amdgpu_kernel void @zext_bool_icmp_ne_0(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_eq_1:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_eq_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_eq_1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_eq_1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_eq_1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_eq_1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, 1
@@ -113,13 +409,57 @@ define amdgpu_kernel void @zext_bool_icmp_eq_1(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_ne_1:
-; GCN-NOT: v_cmp
-; GCN: s_cmp_eq_u32
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN-NEXT: buffer_store_byte [[RESULT]]
define amdgpu_kernel void @zext_bool_icmp_ne_1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_ne_1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_cmp_eq_u32 s4, s5
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_ne_1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; VI-NEXT: s_cmp_eq_u32 s4, s5
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_ne_1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 12, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETE_INT * T1.W, KC0[2].Z, KC0[2].W,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 1
@@ -128,11 +468,45 @@ define amdgpu_kernel void @zext_bool_icmp_ne_1(ptr addrspace(1) %out, i32 %a, i3
}
; Reduces to false:
-; FUNC-LABEL: {{^}}zext_bool_icmp_eq_neg1:
-; GCN: v_mov_b32_e32 [[TMP:v[0-9]+]], 0{{$}}
-; GCN: buffer_store_byte [[TMP]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_eq_neg1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_eq_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_eq_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_eq_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.X, 0.0,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, -1
@@ -141,11 +515,45 @@ define amdgpu_kernel void @zext_bool_icmp_eq_neg1(ptr addrspace(1) %out, i32 %a,
}
; Reduces to true:
-; FUNC-LABEL: {{^}}zext_bool_icmp_ne_neg1:
-; GCN: v_mov_b32_e32 [[TMP:v[0-9]+]], 1{{$}}
-; GCN: buffer_store_byte [[TMP]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_ne_neg1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_ne_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_ne_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_ne_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.X, 1, T0.W,
+; EG-NEXT: MOV T1.Y, 0.0,
+; EG-NEXT: MOV T1.Z, 0.0,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, -1
@@ -153,29 +561,142 @@ define amdgpu_kernel void @zext_bool_icmp_ne_neg1(ptr addrspace(1) %out, i32 %a,
ret void
}
-; FUNC-LABEL: {{^}}cmp_zext_k_i8max:
-; GCN: s_load_dword [[VALUE:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}
-; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VALUE]], 0xff
-; GCN: s_cmpk_lg_i32 [[B]], 0xff
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN: buffer_store_byte [[RESULT]]
-; GCN: s_endpgm
define amdgpu_kernel void @cmp_zext_k_i8max(ptr addrspace(1) %out, i8 %b) nounwind {
+; SI-LABEL: cmp_zext_k_i8max:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_cmpk_lg_i32 s2, 0xff
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: cmp_zext_k_i8max:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s4, s6, 0xff
+; VI-NEXT: s_cmpk_lg_i32 s4, 0xff
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: cmp_zext_k_i8max:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 12, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: AND_INT T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: SETNE_INT * T1.W, T0.X, literal.y,
+; EG-NEXT: 3(4.203895e-45), 255(3.573311e-43)
+; EG-NEXT: AND_INT T1.W, PS, 1,
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%b.ext = zext i8 %b to i32
%icmp0 = icmp ne i32 %b.ext, 255
store i1 %icmp0, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}cmp_sext_k_neg1:
-; GCN: buffer_load_sbyte [[B:v[0-9]+]]
-; GCN: v_cmp_ne_u32_e32 vcc, -1, [[B]]{{$}}
-; GCN-NEXT: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, vcc
-; GCN: buffer_store_byte [[RESULT]]
-; GCN: s_endpgm
define amdgpu_kernel void @cmp_sext_k_neg1(ptr addrspace(1) %out, ptr addrspace(1) %b.ptr) nounwind {
+; SI-LABEL: cmp_sext_k_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s10, s6
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s2
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: buffer_load_sbyte v0, off, s[8:11], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v0
+; SI-NEXT: s_and_b64 s[2:3], vcc, exec
+; SI-NEXT: s_cselect_b32 s2, 1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: cmp_sext_k_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_mov_b32 s10, s6
+; VI-NEXT: s_mov_b32 s11, s7
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s8, s2
+; VI-NEXT: s_mov_b32 s9, s3
+; VI-NEXT: buffer_load_sbyte v0, off, s[8:11], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v0
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: cmp_sext_k_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, KC0[2].Z,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: AND_INT T0.W, T0.X, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: SETNE_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, 1,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%b = load i8, ptr addrspace(1) %b.ptr
%b.ext = sext i8 %b to i32
%icmp0 = icmp ne i32 %b.ext, -1
@@ -183,11 +704,53 @@ define amdgpu_kernel void @cmp_sext_k_neg1(ptr addrspace(1) %out, ptr addrspace(
ret void
}
-; FUNC-LABEL: {{^}}v_cmp_sext_k_neg1_i8_sext_arg:
-; GCN: v_cmp_ne_u32_e32 vcc, -1, v0
-; GCN: v_cndmask_b32_e64 [[SELECT:v[0-9]+]], 0, 1, vcc
-; GCN: buffer_store_byte [[SELECT]]
define void @v_cmp_sext_k_neg1_i8_sext_arg(i8 signext %b) nounwind {
+; SI-LABEL: v_cmp_sext_k_neg1_i8_sext_arg:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-LABEL: v_cmp_sext_k_neg1_i8_sext_arg:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, -1, v0
+; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; EG-LABEL: v_cmp_sext_k_neg1_i8_sext_arg:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 10, @9, KC0[], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 36, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: BFE_INT * T0.W, T0.X, 0.0, literal.x,
+; EG-NEXT: 8(1.121039e-44), 0(0.000000e+00)
+; EG-NEXT: SETNE_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: -1(nan), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.X, PV.W, 1,
+; EG-NEXT: MOV * T0.W, literal.x,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: MOV * T1.X, literal.x,
+; EG-NEXT: 0(0.000000e+00), 0(0.000000e+00)
%b.ext = sext i8 %b to i32
%icmp0 = icmp ne i32 %b.ext, -1
store i1 %icmp0, ptr addrspace(1) poison
@@ -198,38 +761,157 @@ define void @v_cmp_sext_k_neg1_i8_sext_arg(i8 signext %b) nounwind {
; 255. Seems to be because of ordering problems when not allowing load widths to be reduced.
; Should do a buffer_load_sbyte and compare with -1
-; FUNC-LABEL: {{^}}cmp_sext_k_neg1_i8_arg:
-; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
-; VI: s_load_dword [[VAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VAL]], 0xff
-; GCN: s_cmpk_lg_i32 [[B]], 0xff{{$}}
-; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN: buffer_store_byte [[RESULT]]
-; GCN: s_endpgm
define amdgpu_kernel void @cmp_sext_k_neg1_i8_arg(ptr addrspace(1) %out, i8 %b) nounwind {
+; SI-LABEL: cmp_sext_k_neg1_i8_arg:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_and_b32 s2, s2, 0xff
+; SI-NEXT: s_cmpk_lg_i32 s2, 0xff
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; SI-NEXT: s_cselect_b32 s4, 1, 0
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: cmp_sext_k_neg1_i8_arg:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_and_b32 s4, s6, 0xff
+; VI-NEXT: s_cmpk_lg_i32 s4, 0xff
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; VI-NEXT: s_cselect_b32 s4, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: cmp_sext_k_neg1_i8_arg:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @8, KC0[], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_8 T0.X, T0.X, 40, #3
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: MOV * T0.X, 0.0,
+; EG-NEXT: ALU clause starting at 9:
+; EG-NEXT: AND_INT T0.W, T0.X, literal.x,
+; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y,
+; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45)
+; EG-NEXT: SETNE_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: AND_INT T0.W, PV.W, 1,
+; EG-NEXT: LSHL * T1.W, T1.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL T0.X, PV.W, PS,
+; EG-NEXT: LSHL * T0.W, literal.x, PS,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV * T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%b.ext = sext i8 %b to i32
%icmp0 = icmp ne i32 %b.ext, -1
store i1 %icmp0, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}cmp_zext_k_neg1:
-; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], 1{{$}}
-; GCN: buffer_store_byte [[RESULT]]
-; GCN: s_endpgm
define amdgpu_kernel void @cmp_zext_k_neg1(ptr addrspace(1) %out, i8 %b) nounwind {
+; SI-LABEL: cmp_zext_k_neg1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: cmp_zext_k_neg1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: cmp_zext_k_neg1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.X, 1, T0.W,
+; EG-NEXT: MOV T1.Y, 0.0,
+; EG-NEXT: MOV T1.Z, 0.0,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%b.ext = zext i8 %b to i32
%icmp0 = icmp ne i32 %b.ext, -1
store i1 %icmp0, ptr addrspace(1) %out
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_ne_k:
-; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], 1{{$}}
-; GCN: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_ne_k(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_ne_k:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_ne_k:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_ne_k:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.X, 1, T0.W,
+; EG-NEXT: MOV T1.Y, 0.0,
+; EG-NEXT: MOV T1.Z, 0.0,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 2
@@ -237,11 +919,45 @@ define amdgpu_kernel void @zext_bool_icmp_ne_k(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}zext_bool_icmp_eq_k:
-; GCN: v_mov_b32_e32 [[RESULT:v[0-9]+]], 0{{$}}
-; GCN: buffer_store_byte [[RESULT]]
-; GCN-NEXT: s_endpgm
define amdgpu_kernel void @zext_bool_icmp_eq_k(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: zext_bool_icmp_eq_k:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: zext_bool_icmp_eq_k:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: zext_bool_icmp_eq_k:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.X, 0.0,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = zext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, 2
@@ -253,10 +969,45 @@ define amdgpu_kernel void @zext_bool_icmp_eq_k(ptr addrspace(1) %out, i32 %a, i3
; DAGCombiner
; This really folds away to false
-; FUNC-LABEL: {{^}}sext_bool_icmp_eq_1:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 0{{$}}
-; GCN: buffer_store_byte [[K]]
define amdgpu_kernel void @sext_bool_icmp_eq_1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_eq_1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_eq_1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_eq_1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T0.X, 0.0,
+; EG-NEXT: MOV T0.Y, 0.0,
+; EG-NEXT: MOV T0.Z, 0.0,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp eq i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp eq i32 %ext, 1
@@ -264,10 +1015,45 @@ define amdgpu_kernel void @sext_bool_icmp_eq_1(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}sext_bool_icmp_ne_1:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 1{{$}}
-; GCN: buffer_store_byte [[K]]
define amdgpu_kernel void @sext_bool_icmp_ne_1(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_ne_1:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_ne_1:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_ne_1:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.X, 1, T0.W,
+; EG-NEXT: MOV T1.Y, 0.0,
+; EG-NEXT: MOV T1.Z, 0.0,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 1
@@ -275,10 +1061,45 @@ define amdgpu_kernel void @sext_bool_icmp_ne_1(ptr addrspace(1) %out, i32 %a, i3
ret void
}
-; FUNC-LABEL: {{^}}sext_bool_icmp_ne_k:
-; GCN: v_mov_b32_e32 [[K:v[0-9]+]], 1{{$}}
-; GCN: buffer_store_byte [[K]]
define amdgpu_kernel void @sext_bool_icmp_ne_k(ptr addrspace(1) %out, i32 %a, i32 %b) nounwind {
+; SI-LABEL: sext_bool_icmp_ne_k:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: sext_bool_icmp_ne_k:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_endpgm
+;
+; EG-LABEL: sext_bool_icmp_ne_k:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 10, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT MSKOR T1.XW, T0.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: AND_INT * T0.W, KC0[2].Y, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T0.W, PV.W, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: LSHL * T1.W, literal.x, PV.W,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: LSHL T1.X, 1, T0.W,
+; EG-NEXT: MOV T1.Y, 0.0,
+; EG-NEXT: MOV T1.Z, 0.0,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%icmp0 = icmp ne i32 %a, %b
%ext = sext i1 %icmp0 to i32
%icmp1 = icmp ne i32 %ext, 2
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
index ef535b99c6549..21eae1beac801 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-to-vreg1-copy.ll
@@ -8,20 +8,19 @@ define amdgpu_kernel void @copy_to_vreg_1(i32 %0) {
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT: v_mov_b64_e32 v[2:3], 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_sub_i32 s5, 1, s4
; GCN-NEXT: s_cmp_lt_u32 s4, 2
; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
; GCN-NEXT: s_and_b64 s[2:3], s[0:1], exec
; GCN-NEXT: s_cselect_b32 s2, s5, 1
+; GCN-NEXT: s_cselect_b32 s3, 1, 0
; GCN-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; GCN-NEXT: s_addc_u32 s0, 1, 0
; GCN-NEXT: s_cmp_ge_u32 s2, s4
-; GCN-NEXT: v_readfirstlane_b32 s1, v2
-; GCN-NEXT: s_cselect_b32 s4, s0, s1
+; GCN-NEXT: s_cselect_b32 s4, s0, s3
; GCN-NEXT: s_mov_b64 s[0:1], 0
-; GCN-NEXT: v_mov_b64_e32 v[2:3], 0
; GCN-NEXT: s_mov_b64 s[2:3], 0
; GCN-NEXT: s_branch .LBB0_3
; GCN-NEXT: .LBB0_1: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
index 004c27971131d..dc493e3281b65 100644
--- a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
@@ -139,13 +139,15 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_cmpk_eq_i32 s1, 0x100
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_cselect_b32 s1, 1, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ; %bb.1: ; %if.then
; CHECK-NEXT: s_cmp_eq_u32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_cselect_b32 s1, 1, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: .LBB1_2: ; %if.end6.sink.split
@@ -170,7 +172,8 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; CHECK-NEXT: ; %bb.5: ; %if.then3
; CHECK-NEXT: s_cmp_eq_u32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_cselect_b32 s1, 1, 0
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_branch .LBB1_2
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.ll b/llvm/test/CodeGen/AMDGPU/sminmax.ll
index dbcb4b75e7818..d7786d2431173 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.ll
@@ -1,14 +1,60 @@
-; RUN: llc -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -check-prefixes=EG,FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefix=VI %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG %s
-; FUNC-LABEL: {{^}}s_abs_i32:
-; GCN: s_abs_i32
-; GCN: s_add_i32
-
-; EG: MAX_INT
define amdgpu_kernel void @s_abs_i32(ptr addrspace(1) %out, i32 %val) nounwind {
+; SI-LABEL: s_abs_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s6, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_abs_i32 s4, s6
+; SI-NEXT: s_add_i32 s4, s4, 2
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_abs_i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_abs_i32 s2, s2
+; VI-NEXT: s_add_i32 s2, s2, 2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_abs_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_abs_i32 s2, s2
+; GFX9-NEXT: s_add_i32 s2, s2, 2
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: s_abs_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 4, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: SUB_INT * T0.W, 0.0, KC0[2].Z,
+; EG-NEXT: MAX_INT * T0.W, KC0[2].Z, PV.W,
+; EG-NEXT: ADD_INT T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%neg = sub i32 0, %val
%cond = icmp sgt i32 %val, %neg
%res = select i1 %cond, i32 %val, i32 %neg
@@ -17,17 +63,80 @@ define amdgpu_kernel void @s_abs_i32(ptr addrspace(1) %out, i32 %val) nounwind {
ret void
}
-; FUNC-LABEL: {{^}}v_abs_i32:
-; SIVI: v_sub_{{i|u}}32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
-; GFX9: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SRC:v[0-9]+]]
-
-; GCN: v_max_i32_e32 {{v[0-9]+}}, [[SRC]], [[NEG]]
-
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2
-
-; EG: MAX_INT
define amdgpu_kernel void @v_abs_i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
+; SI-LABEL: v_abs_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_sub_i32_e32 v1, vcc, 0, v0
+; SI-NEXT: v_max_i32_e32 v0, v0, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 2, v0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_abs_i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT: flat_load_dword v2, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_sub_u32_e32 v3, vcc, 0, v2
+; VI-NEXT: v_max_i32_e32 v2, v2, v3
+; VI-NEXT: v_add_u32_e32 v2, vcc, 2, v2
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_abs_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_sub_u32_e32 v2, 0, v0
+; GFX9-NEXT: v_max_i32_e32 v0, v0, v2
+; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_abs_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 4, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: SUB_INT * T0.W, 0.0, T0.X,
+; EG-NEXT: MAX_INT * T0.W, T0.X, PV.W,
+; EG-NEXT: ADD_INT T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep.in = getelementptr inbounds i32, ptr addrspace(1) %src, i32 %tid
%val = load i32, ptr addrspace(1) %gep.in, align 4
@@ -39,12 +148,80 @@ define amdgpu_kernel void @v_abs_i32(ptr addrspace(1) %out, ptr addrspace(1) %sr
ret void
}
-; GCN-LABEL: {{^}}v_abs_i32_repeat_user:
-; SIVI: v_sub_{{i|u}}32_e32 [[NEG:v[0-9]+]], vcc, 0, [[SRC:v[0-9]+]]
-; GFX9: v_sub_u32_e32 [[NEG:v[0-9]+]], 0, [[SRC:v[0-9]+]]
-; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[SRC]], [[NEG]]
-; GCN: v_mul_lo_u32 v{{[0-9]+}}, [[MAX]], [[MAX]]
define amdgpu_kernel void @v_abs_i32_repeat_user(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
+; SI-LABEL: v_abs_i32_repeat_user:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_sub_i32_e32 v1, vcc, 0, v0
+; SI-NEXT: v_max_i32_e32 v0, v0, v1
+; SI-NEXT: v_mul_lo_u32 v0, v0, v0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_abs_i32_repeat_user:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_sub_u32_e32 v1, vcc, 0, v0
+; VI-NEXT: v_max_i32_e32 v0, v0, v1
+; VI-NEXT: v_mul_lo_u32 v2, v0, v0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_abs_i32_repeat_user:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_sub_u32_e32 v1, 0, v0
+; GFX9-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_abs_i32_repeat_user:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 4, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: SUB_INT * T0.W, 0.0, T0.X,
+; EG-NEXT: MAX_INT * T0.W, T0.X, PV.W,
+; EG-NEXT: LSHR T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: MULLO_INT * T1.X, PV.W, PV.W,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep.in = getelementptr inbounds i32, ptr addrspace(1) %src, i32 %tid
%val = load i32, ptr addrspace(1) %gep.in, align 4
@@ -56,15 +233,68 @@ define amdgpu_kernel void @v_abs_i32_repeat_user(ptr addrspace(1) %out, ptr addr
ret void
}
-; FUNC-LABEL: {{^}}s_abs_v2i32:
-; GCN: s_abs_i32
-; GCN: s_abs_i32
-; GCN: s_add_i32
-; GCN: s_add_i32
-
-; EG: MAX_INT
-; EG: MAX_INT
define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) nounwind {
+; SI-LABEL: s_abs_v2i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_abs_i32 s4, s4
+; SI-NEXT: s_abs_i32 s5, s5
+; SI-NEXT: s_add_i32 s5, s5, 2
+; SI-NEXT: s_add_i32 s4, s4, 2
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_abs_v2i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_abs_i32 s2, s2
+; VI-NEXT: s_abs_i32 s3, s3
+; VI-NEXT: s_add_i32 s3, s3, 2
+; VI-NEXT: s_add_i32 s2, s2, 2
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_abs_v2i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_abs_i32 s2, s2
+; GFX9-NEXT: s_abs_i32 s3, s3
+; GFX9-NEXT: s_add_i32 s3, s3, 2
+; GFX9-NEXT: s_add_i32 s2, s2, 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: s_abs_v2i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 8, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: SUB_INT T0.W, 0.0, KC0[3].X,
+; EG-NEXT: SUB_INT * T1.W, 0.0, KC0[2].W,
+; EG-NEXT: MAX_INT * T0.W, KC0[3].X, PV.W,
+; EG-NEXT: ADD_INT T0.Y, PV.W, literal.x,
+; EG-NEXT: MAX_INT * T0.W, KC0[2].W, T1.W,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%z0 = insertelement <2 x i32> poison, i32 0, i32 0
%z1 = insertelement <2 x i32> %z0, i32 0, i32 1
%t0 = insertelement <2 x i32> poison, i32 2, i32 0
@@ -77,25 +307,93 @@ define amdgpu_kernel void @s_abs_v2i32(ptr addrspace(1) %out, <2 x i32> %val) no
ret void
}
-; FUNC-LABEL: {{^}}v_abs_v2i32:
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
-
-; GFX9-DAG: v_sub_u32_e32 [[NEG0:v[0-9]+]], 0, [[SRC0:v[0-9]+]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG1:v[0-9]+]], 0, [[SRC1:v[0-9]+]]
-
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC0]], [[NEG0]]
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC1]], [[NEG1]]
-
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc
-
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-
-; EG: MAX_INT
-; EG: MAX_INT
define amdgpu_kernel void @v_abs_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
+; SI-LABEL: v_abs_v2i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_sub_i32_e32 v2, vcc, 0, v1
+; SI-NEXT: v_sub_i32_e32 v3, vcc, 0, v0
+; SI-NEXT: v_max_i32_e32 v0, v0, v3
+; SI-NEXT: v_max_i32_e32 v1, v1, v2
+; SI-NEXT: v_add_i32_e32 v1, vcc, 2, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 2, v0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_abs_v2i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_sub_u32_e32 v4, vcc, 0, v1
+; VI-NEXT: v_sub_u32_e32 v5, vcc, 0, v0
+; VI-NEXT: v_max_i32_e32 v0, v0, v5
+; VI-NEXT: v_max_i32_e32 v1, v1, v4
+; VI-NEXT: v_add_u32_e32 v1, vcc, 2, v1
+; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0
+; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_abs_v2i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_sub_u32_e32 v3, 0, v1
+; GFX9-NEXT: v_sub_u32_e32 v4, 0, v0
+; GFX9-NEXT: v_max_i32_e32 v0, v0, v4
+; GFX9-NEXT: v_max_i32_e32 v1, v1, v3
+; GFX9-NEXT: v_add_u32_e32 v1, 2, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_abs_v2i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 8, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: SUB_INT T0.W, 0.0, T0.Y,
+; EG-NEXT: SUB_INT * T1.W, 0.0, T0.X,
+; EG-NEXT: MAX_INT * T0.W, T0.Y, PV.W,
+; EG-NEXT: ADD_INT T0.Y, PV.W, literal.x,
+; EG-NEXT: MAX_INT * T0.W, T0.X, T1.W,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%z0 = insertelement <2 x i32> poison, i32 0, i32 0
%z1 = insertelement <2 x i32> %z0, i32 0, i32 1
%t0 = insertelement <2 x i32> poison, i32 2, i32 0
@@ -111,23 +409,98 @@ define amdgpu_kernel void @v_abs_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %
ret void
}
-; FUNC-LABEL: {{^}}s_abs_v4i32:
; TODO: this should use s_abs_i32
-; GCN: s_abs_i32
-; GCN: s_abs_i32
-; GCN: s_abs_i32
-; GCN: s_abs_i32
-; GCN: s_add_i32
-; GCN: s_add_i32
-; GCN: s_add_i32
-; GCN: s_add_i32
-
-; EG: MAX_INT
-; EG: MAX_INT
-; EG: MAX_INT
-; EG: MAX_INT
define amdgpu_kernel void @s_abs_v4i32(ptr addrspace(1) %out, <4 x i32> %val) nounwind {
+; SI-LABEL: s_abs_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_abs_i32 s0, s0
+; SI-NEXT: s_abs_i32 s1, s1
+; SI-NEXT: s_abs_i32 s2, s2
+; SI-NEXT: s_abs_i32 s3, s3
+; SI-NEXT: s_add_i32 s3, s3, 2
+; SI-NEXT: s_add_i32 s2, s2, 2
+; SI-NEXT: s_add_i32 s1, s1, 2
+; SI-NEXT: s_add_i32 s0, s0, 2
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_abs_v4i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_abs_i32 s0, s0
+; VI-NEXT: s_abs_i32 s1, s1
+; VI-NEXT: s_abs_i32 s2, s2
+; VI-NEXT: s_abs_i32 s3, s3
+; VI-NEXT: s_add_i32 s3, s3, 2
+; VI-NEXT: s_add_i32 s2, s2, 2
+; VI-NEXT: s_add_i32 s1, s1, 2
+; VI-NEXT: s_add_i32 s0, s0, 2
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_abs_v4i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_abs_i32 s0, s0
+; GFX9-NEXT: s_abs_i32 s1, s1
+; GFX9-NEXT: s_abs_i32 s2, s2
+; GFX9-NEXT: s_abs_i32 s3, s3
+; GFX9-NEXT: s_add_i32 s3, s3, 2
+; GFX9-NEXT: s_add_i32 s2, s2, 2
+; GFX9-NEXT: s_add_i32 s1, s1, 2
+; GFX9-NEXT: s_add_i32 s0, s0, 2
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: s_abs_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 16, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: SUB_INT * T0.W, 0.0, KC0[4].X,
+; EG-NEXT: MAX_INT T0.W, KC0[4].X, PV.W,
+; EG-NEXT: SUB_INT * T1.W, 0.0, KC0[3].W,
+; EG-NEXT: SUB_INT T0.Z, 0.0, KC0[3].Z,
+; EG-NEXT: MAX_INT T1.W, KC0[3].W, PS,
+; EG-NEXT: ADD_INT * T2.W, PV.W, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T2.Z, PV.W, literal.x,
+; EG-NEXT: SUB_INT T0.W, 0.0, KC0[3].Y,
+; EG-NEXT: MAX_INT * T1.W, KC0[3].Z, PV.Z,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T2.Y, PS, literal.x,
+; EG-NEXT: MAX_INT * T0.W, KC0[3].Y, PV.W,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T2.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%z0 = insertelement <4 x i32> poison, i32 0, i32 0
%z1 = insertelement <4 x i32> %z0, i32 0, i32 1
%z2 = insertelement <4 x i32> %z1, i32 0, i32 2
@@ -144,38 +517,119 @@ define amdgpu_kernel void @s_abs_v4i32(ptr addrspace(1) %out, <4 x i32> %val) no
ret void
}
-; FUNC-LABEL: {{^}}v_abs_v4i32:
-
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG0:v[0-9]+]], vcc, 0, [[SRC0:v[0-9]+]]
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG1:v[0-9]+]], vcc, 0, [[SRC1:v[0-9]+]]
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG2:v[0-9]+]], vcc, 0, [[SRC2:v[0-9]+]]
-; SIVI-DAG: v_sub_{{i|u}}32_e32 [[NEG3:v[0-9]+]], vcc, 0, [[SRC3:v[0-9]+]]
-
-; GFX9-DAG: v_sub_u32_e32 [[NEG0:v[0-9]+]], 0, [[SRC0:v[0-9]+]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG1:v[0-9]+]], 0, [[SRC1:v[0-9]+]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG2:v[0-9]+]], 0, [[SRC2:v[0-9]+]]
-; GFX9-DAG: v_sub_u32_e32 [[NEG3:v[0-9]+]], 0, [[SRC3:v[0-9]+]]
-
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC0]], [[NEG0]]
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC1]], [[NEG1]]
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC2]], [[NEG2]]
-; GCN-DAG: v_max_i32_e32 {{v[0-9]+}}, [[SRC3]], [[NEG3]]
-
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
-; SIVI: v_add_{{i|u}}32_e32 v{{[0-9]+}}, vcc,
-
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-; GFX9: v_add_u32_e32 v{{[0-9]+}}, 2,
-
-; EG: MAX_INT
-; EG: MAX_INT
-; EG: MAX_INT
-; EG: MAX_INT
define amdgpu_kernel void @v_abs_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %src) nounwind {
+; SI-LABEL: v_abs_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b64 s[8:9], s[2:3]
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_sub_i32_e32 v4, vcc, 0, v3
+; SI-NEXT: v_sub_i32_e32 v5, vcc, 0, v2
+; SI-NEXT: v_sub_i32_e32 v6, vcc, 0, v1
+; SI-NEXT: v_sub_i32_e32 v7, vcc, 0, v0
+; SI-NEXT: v_max_i32_e32 v0, v0, v7
+; SI-NEXT: v_max_i32_e32 v1, v1, v6
+; SI-NEXT: v_max_i32_e32 v2, v2, v5
+; SI-NEXT: v_max_i32_e32 v3, v3, v4
+; SI-NEXT: v_add_i32_e32 v3, vcc, 2, v3
+; SI-NEXT: v_add_i32_e32 v2, vcc, 2, v2
+; SI-NEXT: v_add_i32_e32 v1, vcc, 2, v1
+; SI-NEXT: v_add_i32_e32 v0, vcc, 2, v0
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_abs_v4i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_sub_u32_e32 v6, vcc, 0, v3
+; VI-NEXT: v_sub_u32_e32 v7, vcc, 0, v2
+; VI-NEXT: v_sub_u32_e32 v8, vcc, 0, v1
+; VI-NEXT: v_sub_u32_e32 v9, vcc, 0, v0
+; VI-NEXT: v_max_i32_e32 v0, v0, v9
+; VI-NEXT: v_max_i32_e32 v1, v1, v8
+; VI-NEXT: v_max_i32_e32 v2, v2, v7
+; VI-NEXT: v_max_i32_e32 v3, v3, v6
+; VI-NEXT: v_add_u32_e32 v3, vcc, 2, v3
+; VI-NEXT: v_add_u32_e32 v2, vcc, 2, v2
+; VI-NEXT: v_add_u32_e32 v1, vcc, 2, v1
+; VI-NEXT: v_add_u32_e32 v0, vcc, 2, v0
+; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_abs_v4i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_sub_u32_e32 v5, 0, v3
+; GFX9-NEXT: v_sub_u32_e32 v6, 0, v2
+; GFX9-NEXT: v_sub_u32_e32 v7, 0, v1
+; GFX9-NEXT: v_sub_u32_e32 v8, 0, v0
+; GFX9-NEXT: v_max_i32_e32 v0, v0, v8
+; GFX9-NEXT: v_max_i32_e32 v1, v1, v7
+; GFX9-NEXT: v_max_i32_e32 v2, v2, v6
+; GFX9-NEXT: v_max_i32_e32 v3, v3, v5
+; GFX9-NEXT: v_add_u32_e32 v3, 2, v3
+; GFX9-NEXT: v_add_u32_e32 v2, 2, v2
+; GFX9-NEXT: v_add_u32_e32 v1, 2, v1
+; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_abs_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @6
+; EG-NEXT: ALU 16, @11, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 6:
+; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1
+; EG-NEXT: ALU clause starting at 8:
+; EG-NEXT: LSHL * T0.W, T0.X, literal.x,
+; EG-NEXT: 4(5.605194e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W,
+; EG-NEXT: ALU clause starting at 11:
+; EG-NEXT: SUB_INT * T1.W, 0.0, T0.W,
+; EG-NEXT: MAX_INT T0.W, T0.W, PV.W,
+; EG-NEXT: SUB_INT * T1.W, 0.0, T0.Z,
+; EG-NEXT: SUB_INT T1.Z, 0.0, T0.Y,
+; EG-NEXT: MAX_INT T1.W, T0.Z, PS,
+; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T0.Z, PV.W, literal.x,
+; EG-NEXT: SUB_INT T1.W, 0.0, T0.X,
+; EG-NEXT: MAX_INT * T2.W, T0.Y, PV.Z,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T0.Y, PS, literal.x,
+; EG-NEXT: MAX_INT * T1.W, T0.X, PV.W,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: ADD_INT T0.X, PV.W, literal.x,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%z0 = insertelement <4 x i32> poison, i32 0, i32 0
%z1 = insertelement <4 x i32> %z0, i32 0, i32 1
%z2 = insertelement <4 x i32> %z1, i32 0, i32 2
@@ -195,13 +649,81 @@ define amdgpu_kernel void @v_abs_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %
ret void
}
-; FUNC-LABEL: {{^}}s_min_max_i32:
-; GCN: s_load_dword [[VAL0:s[0-9]+]]
-; GCN: s_load_dword [[VAL1:s[0-9]+]]
-
-; GCN-DAG: s_min_i32 s{{[0-9]+}}, [[VAL0]], [[VAL1]]
-; GCN-DAG: s_max_i32 s{{[0-9]+}}, [[VAL0]], [[VAL1]]
define amdgpu_kernel void @s_min_max_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, [8 x i32], i32 %val0, [8 x i32], i32 %val1) nounwind {
+; SI-LABEL: s_min_max_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dword s8, s[4:5], 0x15
+; SI-NEXT: s_load_dword s9, s[4:5], 0x1e
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_max_i32 s10, s8, s9
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_min_i32 s8, s8, s9
+; SI-NEXT: v_mov_b32_e32 v0, s10
+; SI-NEXT: s_mov_b32 s0, s2
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s2, s6
+; SI-NEXT: s_mov_b32 s3, s7
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_min_max_i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s6, s[4:5], 0x54
+; VI-NEXT: s_load_dword s4, s[4:5], 0x78
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_max_i32 s0, s6, s4
+; VI-NEXT: s_min_i32 s1, s6, s4
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: flat_store_dword v[0:1], v4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s1
+; VI-NEXT: flat_store_dword v[2:3], v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_min_max_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x54
+; GFX9-NEXT: s_load_dword s7, s[4:5], 0x78
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_max_i32 s4, s6, s7
+; GFX9-NEXT: s_min_i32 s5, s6, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: s_min_max_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 5, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T3.X, T2.X, 0
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.X, T0.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: LSHR * T0.X, KC0[2].Z, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: MIN_INT * T1.X, KC0[4].W, KC0[7].X,
+; EG-NEXT: LSHR * T2.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: MAX_INT * T3.X, KC0[4].W, KC0[7].X,
%cond0 = icmp sgt i32 %val0, %val1
%sel0 = select i1 %cond0, i32 %val0, i32 %val1
%sel1 = select i1 %cond0, i32 %val1, i32 %val0
@@ -211,13 +733,103 @@ define amdgpu_kernel void @s_min_max_i32(ptr addrspace(1) %out0, ptr addrspace(1
ret void
}
-; FUNC-LABEL: {{^}}v_min_max_i32:
-; GCN: {{buffer|flat|global}}_load_dword [[VAL0:v[0-9]+]]
-; GCN: {{buffer|flat|global}}_load_dword [[VAL1:v[0-9]+]]
-
-; GCN-DAG: v_min_i32_e32 v{{[0-9]+}}, [[VAL0]], [[VAL1]]
-; GCN-DAG: v_max_i32_e32 v{{[0-9]+}}, [[VAL0]], [[VAL1]]
define amdgpu_kernel void @v_min_max_i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %ptr0, ptr addrspace(1) %ptr1) nounwind {
+; SI-LABEL: v_min_max_i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s4, s2
+; SI-NEXT: s_mov_b32 s5, s3
+; SI-NEXT: v_max_i32_e32 v2, v0, v1
+; SI-NEXT: v_min_i32_e32 v0, v0, v1
+; SI-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_min_max_i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: flat_load_dword v4, v[0:1] glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_load_dword v5, v[2:3] glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_max_i32_e32 v6, v4, v5
+; VI-NEXT: v_min_i32_e32 v4, v4, v5
+; VI-NEXT: flat_store_dword v[0:1], v6
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_dword v[2:3], v4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_min_max_i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[12:13] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v0, s[14:15] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_max_i32_e32 v3, v1, v2
+; GFX9-NEXT: v_min_i32_e32 v1, v1, v2
+; GFX9-NEXT: global_store_dword v0, v3, s[8:9]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[10:11]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_min_max_i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @12, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @8
+; EG-NEXT: ALU 0, @13, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @10
+; EG-NEXT: ALU 5, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 0
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T2.X, T3.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: Fetch clause starting at 8:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: Fetch clause starting at 10:
+; EG-NEXT: VTX_READ_32 T1.X, T1.X, 0, #1
+; EG-NEXT: ALU clause starting at 12:
+; EG-NEXT: MOV * T0.X, KC0[2].W,
+; EG-NEXT: ALU clause starting at 13:
+; EG-NEXT: MOV * T1.X, KC0[3].X,
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MIN_INT T2.X, T0.X, T1.X,
+; EG-NEXT: LSHR * T3.X, KC0[2].Z, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: MAX_INT T0.X, T0.X, T1.X,
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%val0 = load volatile i32, ptr addrspace(1) %ptr0
%val1 = load volatile i32, ptr addrspace(1) %ptr1
@@ -230,16 +842,120 @@ define amdgpu_kernel void @v_min_max_i32(ptr addrspace(1) %out0, ptr addrspace(1
ret void
}
-; FUNC-LABEL: {{^}}s_min_max_v4i32:
-; GCN-DAG: s_min_i32
-; GCN-DAG: s_min_i32
-; GCN-DAG: s_min_i32
-; GCN-DAG: s_min_i32
-; GCN-DAG: s_max_i32
-; GCN-DAG: s_max_i32
-; GCN-DAG: s_max_i32
-; GCN-DAG: s_max_i32
define amdgpu_kernel void @s_min_max_v4i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %val0, <4 x i32> %val1) nounwind {
+; SI-LABEL: s_min_max_v4i32:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s15, 0xf000
+; SI-NEXT: s_mov_b32 s14, -1
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s8
+; SI-NEXT: s_max_i32 s16, s3, s7
+; SI-NEXT: s_max_i32 s17, s2, s6
+; SI-NEXT: s_max_i32 s18, s1, s5
+; SI-NEXT: s_max_i32 s19, s0, s4
+; SI-NEXT: s_mov_b32 s13, s9
+; SI-NEXT: v_mov_b32_e32 v0, s19
+; SI-NEXT: v_mov_b32_e32 v1, s18
+; SI-NEXT: v_mov_b32_e32 v2, s17
+; SI-NEXT: v_mov_b32_e32 v3, s16
+; SI-NEXT: s_min_i32 s3, s3, s7
+; SI-NEXT: s_min_i32 s2, s2, s6
+; SI-NEXT: s_min_i32 s1, s1, s5
+; SI-NEXT: s_min_i32 s0, s0, s4
+; SI-NEXT: s_mov_b32 s8, s10
+; SI-NEXT: s_mov_b32 s9, s11
+; SI-NEXT: s_mov_b32 s10, s14
+; SI-NEXT: s_mov_b32 s11, s15
+; SI-NEXT: v_mov_b32_e32 v4, s0
+; SI-NEXT: v_mov_b32_e32 v5, s1
+; SI-NEXT: v_mov_b32_e32 v6, s2
+; SI-NEXT: v_mov_b32_e32 v7, s3
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[8:11], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: s_min_max_v4i32:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x34
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v8, s8
+; VI-NEXT: v_mov_b32_e32 v9, s9
+; VI-NEXT: v_mov_b32_e32 v10, s10
+; VI-NEXT: v_mov_b32_e32 v11, s11
+; VI-NEXT: s_max_i32 s8, s3, s7
+; VI-NEXT: s_max_i32 s9, s2, s6
+; VI-NEXT: s_max_i32 s10, s1, s5
+; VI-NEXT: s_max_i32 s11, s0, s4
+; VI-NEXT: v_mov_b32_e32 v0, s11
+; VI-NEXT: v_mov_b32_e32 v1, s10
+; VI-NEXT: v_mov_b32_e32 v2, s9
+; VI-NEXT: v_mov_b32_e32 v3, s8
+; VI-NEXT: s_min_i32 s3, s3, s7
+; VI-NEXT: s_min_i32 s2, s2, s6
+; VI-NEXT: s_min_i32 s1, s1, s5
+; VI-NEXT: s_min_i32 s0, s0, s4
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v6, s2
+; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: s_min_max_v4i32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_max_i32 s4, s11, s15
+; GFX9-NEXT: s_max_i32 s5, s10, s14
+; GFX9-NEXT: s_max_i32 s6, s9, s13
+; GFX9-NEXT: s_max_i32 s7, s8, s12
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
+; GFX9-NEXT: s_min_i32 s4, s11, s15
+; GFX9-NEXT: s_min_i32 s5, s10, s14
+; GFX9-NEXT: s_min_i32 s6, s9, s13
+; GFX9-NEXT: s_min_i32 s7, s8, s12
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v6, s5
+; GFX9-NEXT: v_mov_b32_e32 v7, s4
+; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: s_min_max_v4i32:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 11, @4, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T2.XYZW, T3.X, 0
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XYZW, T1.X, 1
+; EG-NEXT: CF_END
+; EG-NEXT: ALU clause starting at 4:
+; EG-NEXT: MIN_INT * T0.W, KC0[4].X, KC0[5].X,
+; EG-NEXT: MIN_INT * T0.Z, KC0[3].W, KC0[4].W,
+; EG-NEXT: MIN_INT * T0.Y, KC0[3].Z, KC0[4].Z,
+; EG-NEXT: MIN_INT * T0.X, KC0[3].Y, KC0[4].Y,
+; EG-NEXT: LSHR * T1.X, KC0[2].Z, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: MAX_INT * T2.W, KC0[4].X, KC0[5].X,
+; EG-NEXT: MAX_INT * T2.Z, KC0[3].W, KC0[4].W,
+; EG-NEXT: MAX_INT * T2.Y, KC0[3].Z, KC0[4].Z,
+; EG-NEXT: MAX_INT * T2.X, KC0[3].Y, KC0[4].Y,
+; EG-NEXT: LSHR * T3.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%cond0 = icmp sgt <4 x i32> %val0, %val1
%sel0 = select <4 x i1> %cond0, <4 x i32> %val0, <4 x i32> %val1
%sel1 = select <4 x i1> %cond0, <4 x i32> %val1, <4 x i32> %val0
@@ -249,12 +965,131 @@ define amdgpu_kernel void @s_min_max_v4i32(ptr addrspace(1) %out0, ptr addrspace
ret void
}
-; FUNC-LABEL: {{^}}v_min_max_i32_user:
-; GCN: v_cmp_gt_i32_e32
-; GCN-DAG: v_cndmask_b32_e32
-; GCN-DAG: v_cndmask_b32_e32
-; GCN-DAG: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1, vcc
define amdgpu_kernel void @v_min_max_i32_user(ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %ptr0, ptr addrspace(1) %ptr1) nounwind {
+; SI-LABEL: v_min_max_i32_user:
+; SI: ; %bb.0:
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s14, s10
+; SI-NEXT: s_mov_b32 s15, s11
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s12, s4
+; SI-NEXT: s_mov_b32 s13, s5
+; SI-NEXT: s_mov_b32 s4, s6
+; SI-NEXT: s_mov_b32 s5, s7
+; SI-NEXT: s_mov_b32 s6, s10
+; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 glc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s4, s2
+; SI-NEXT: s_mov_b32 s5, s3
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: v_cndmask_b32_e32 v2, v1, v0, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; SI-NEXT: s_cselect_b32 s0, 1, 0
+; SI-NEXT: buffer_store_dword v2, off, s[8:11], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_byte v0, off, s[8:11], 0
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: s_endpgm
+;
+; VI-LABEL: v_min_max_i32_user:
+; VI: ; %bb.0:
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: flat_load_dword v4, v[0:1] glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_load_dword v5, v[2:3] glc
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_cmp_gt_i32_e32 vcc, v4, v5
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
+; VI-NEXT: flat_store_dword v[0:1], v6
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: flat_store_dword v[2:3], v4
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: flat_store_byte v[0:1], v0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: s_endpgm
+;
+; GFX9-LABEL: v_min_max_i32_user:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: global_load_dword v1, v0, s[12:13] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_load_dword v2, v0, s[14:15] glc
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_cmp_gt_i32_e32 vcc, v1, v2
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v2, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: global_store_dword v0, v3, s[8:9]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: global_store_dword v0, v1, s[10:11]
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_byte v[0:1], v0, off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_endpgm
+;
+; EG-LABEL: v_min_max_i32_user:
+; EG: ; %bb.0:
+; EG-NEXT: ALU 0, @14, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @10
+; EG-NEXT: ALU 0, @15, KC0[CB0:0-32], KC1[]
+; EG-NEXT: TEX 0 @12
+; EG-NEXT: ALU 13, @16, KC0[CB0:0-32], KC1[]
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.X, T1.X, 0
+; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T4.X, T5.X, 0
+; EG-NEXT: MEM_RAT MSKOR T2.XW, T3.X
+; EG-NEXT: CF_END
+; EG-NEXT: PAD
+; EG-NEXT: Fetch clause starting at 10:
+; EG-NEXT: VTX_READ_32 T0.X, T0.X, 0, #1
+; EG-NEXT: Fetch clause starting at 12:
+; EG-NEXT: VTX_READ_32 T1.X, T1.X, 0, #1
+; EG-NEXT: ALU clause starting at 14:
+; EG-NEXT: MOV * T0.X, KC0[2].W,
+; EG-NEXT: ALU clause starting at 15:
+; EG-NEXT: MOV * T1.X, KC0[3].X,
+; EG-NEXT: ALU clause starting at 16:
+; EG-NEXT: SETGT_INT * T0.W, T0.X, T1.X,
+; EG-NEXT: AND_INT T2.X, PV.W, 1,
+; EG-NEXT: MOV * T2.W, literal.x,
+; EG-NEXT: 255(3.573311e-43), 0(0.000000e+00)
+; EG-NEXT: MOV T2.Y, 0.0,
+; EG-NEXT: MOV * T2.Z, 0.0,
+; EG-NEXT: MOV T3.X, literal.x,
+; EG-NEXT: CNDE_INT * T4.X, T0.W, T0.X, T1.X,
+; EG-NEXT: 0(0.000000e+00), 0(0.000000e+00)
+; EG-NEXT: LSHR T5.X, KC0[2].Z, literal.x,
+; EG-NEXT: CNDE_INT * T0.X, T0.W, T1.X, T0.X,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
+; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
+; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
%val0 = load volatile i32, ptr addrspace(1) %ptr0
%val1 = load volatile i32, ptr addrspace(1) %ptr1
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 76f8f484fc763..29295312307cb 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -860,21 +860,22 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; GFX9-NEXT: global_load_dword v2, v0, s[14:15] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX9-NEXT: v_cmp_gt_i32_sdwa vcc, sext(v1), sext(v2) src0_sel:WORD_0 src1_sel:WORD_0
; GFX9-NEXT: v_cmp_gt_i32_sdwa s[0:1], sext(v1), sext(v2) src0_sel:WORD_1 src1_sel:WORD_1
-; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v2
; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v6, v4, v3, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v4, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 1, v2
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v4
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX9-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX9-NEXT: v_or_b32_e32 v2, s0, v2
; GFX9-NEXT: global_store_dword v0, v4, s[8:9]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[10:11]
@@ -914,13 +915,14 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; VI-NEXT: v_lshlrev_b32_e32 v8, 16, v10
; VI-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[0:1]
; VI-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; VI-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[0:1]
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], exec
; VI-NEXT: v_or_b32_sdwa v6, v6, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_lshlrev_b32_e32 v5, 1, v5
+; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; VI-NEXT: flat_store_dword v[0:1], v6
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_e32 v0, v9, v5
+; VI-NEXT: v_or_b32_e32 v0, s0, v5
; VI-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-NEXT: v_and_b32_e32 v0, 3, v0
; VI-NEXT: flat_store_dword v[2:3], v4
@@ -962,6 +964,7 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; CI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; CI-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1]
; CI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; CI-NEXT: s_and_b64 s[0:1], vcc, exec
; CI-NEXT: buffer_store_short v5, off, s[4:7], 0 offset:2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: buffer_store_short v4, off, s[4:7], 0
@@ -971,8 +974,8 @@ define amdgpu_kernel void @v_min_max_v2i16_user(ptr addrspace(1) %out0, ptr addr
; CI-NEXT: buffer_store_short v0, off, s[12:15], 0
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_lshlrev_b32_e32 v0, 1, v2
-; CI-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; CI-NEXT: v_or_b32_e32 v0, v1, v0
+; CI-NEXT: s_cselect_b32 s0, 1, 0
+; CI-NEXT: v_or_b32_e32 v0, s0, v0
; CI-NEXT: v_and_b32_e32 v0, 3, v0
; CI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; CI-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/ssubo.ll b/llvm/test/CodeGen/AMDGPU/ssubo.ll
index 382d8928a28b0..dced5788eaec3 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubo.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubo.ll
@@ -177,10 +177,12 @@ define amdgpu_kernel void @s_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_sub_i32 s4, s6, s7
; GFX9-NEXT: v_sub_i32 v1, s6, v1 clamp
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_byte v0, v1, s[2:3]
; GFX9-NEXT: s_endpgm
;
@@ -189,15 +191,17 @@ define amdgpu_kernel void @s_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_sub_nc_i32 v0, s6, s7 clamp
; GFX10-NEXT: s_sub_i32 s4, s6, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v2, s[0:1]
-; GFX10-NEXT: global_store_byte v1, v0, s[2:3]
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_and_b32 s5, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s5, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s5
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: global_store_byte v0, v2, s[2:3]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_ssubo_i32:
@@ -209,12 +213,16 @@ define amdgpu_kernel void @s_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: v_sub_nc_i32 v0, s6, s7 clamp
; GFX11-NEXT: s_sub_i32 s4, s6, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, s4, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_and_b32 s5, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b32 v1, v2, s[0:1]
-; GFX11-NEXT: global_store_b8 v1, v0, s[2:3]
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]
; GFX11-NEXT: s_endpgm
%ssub = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 %a, i32 %b) nounwind
%val = extractvalue { i32, i1 } %ssub, 0
@@ -290,8 +298,10 @@ define amdgpu_kernel void @v_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: v_sub_i32 v3, v1, v2 clamp
; GFX9-NEXT: v_sub_u32_e32 v1, v1, v2
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v1, v3
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
; GFX9-NEXT: global_store_dword v0, v1, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_byte v0, v1, s[10:11]
; GFX9-NEXT: s_endpgm
;
@@ -307,7 +317,9 @@ define amdgpu_kernel void @v_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: v_sub_nc_i32 v3, v1, v2 clamp
; GFX10-NEXT: v_sub_nc_u32_e32 v1, v1, v2
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX10-NEXT: s_and_b32 s0, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: global_store_dword v0, v1, s[8:9]
; GFX10-NEXT: global_store_byte v0, v2, s[10:11]
; GFX10-NEXT: s_endpgm
@@ -323,9 +335,11 @@ define amdgpu_kernel void @v_ssubo_i32(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_sub_nc_i32 v3, v1, v2 clamp
; GFX11-NEXT: v_sub_nc_u32_e32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v3
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: global_store_b8 v0, v2, s[2:3]
@@ -653,10 +667,14 @@ define amdgpu_kernel void @v_ssubo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; GFX9-NEXT: v_sub_i32 v1, v1, v3 clamp
; GFX9-NEXT: v_sub_u32_e32 v4, v0, v2
; GFX9-NEXT: v_sub_i32 v0, v0, v2 clamp
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v5, v1
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], v5, v1
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, v4, v0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cselect_b32 s2, 1, 0
+; GFX9-NEXT: s_and_b64 s[0:1], vcc, exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_store_dwordx2 v6, v[4:5], s[8:9]
; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[10:11]
; GFX9-NEXT: s_endpgm
@@ -675,34 +693,42 @@ define amdgpu_kernel void @v_ssubo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
; GFX10-NEXT: v_sub_nc_u32_e32 v3, v0, v2
; GFX10-NEXT: v_sub_nc_i32 v0, v0, v2 clamp
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX10-NEXT: v_cmp_ne_u32_e64 s0, v3, v0
+; GFX10-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX10-NEXT: s_cselect_b32 s1, 1, 0
+; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: global_store_dwordx2 v5, v[3:4], s[8:9]
; GFX10-NEXT: global_store_dwordx2 v5, v[0:1], s[10:11]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_ssubo_v2i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX11-NEXT: s_load_b256 s[4:11], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b64 v[0:1], v5, s[4:5]
-; GFX11-NEXT: global_load_b64 v[2:3], v5, s[6:7]
+; GFX11-NEXT: global_load_b64 v[0:1], v5, s[8:9]
+; GFX11-NEXT: global_load_b64 v[2:3], v5, s[10:11]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_sub_nc_u32_e32 v4, v1, v3
; GFX11-NEXT: v_sub_nc_i32 v1, v1, v3 clamp
; GFX11-NEXT: v_sub_nc_u32_e32 v3, v0, v2
; GFX11-NEXT: v_sub_nc_i32 v0, v0, v2 clamp
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_ne_u32_e64 s0, v3, v0
+; GFX11-NEXT: s_and_b32 s1, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v0, s0
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b64 v5, v[3:4], s[0:1]
-; GFX11-NEXT: global_store_b64 v5, v[0:1], s[2:3]
+; GFX11-NEXT: global_store_b64 v5, v[3:4], s[4:5]
+; GFX11-NEXT: global_store_b64 v5, v[0:1], s[6:7]
; GFX11-NEXT: s_endpgm
%a = load <2 x i32>, ptr addrspace(1) %aptr, align 4
%b = load <2 x i32>, ptr addrspace(1) %bptr, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
index 2d1c85e9b047c..1807f0ed3c12c 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-cmp-constant.ll
@@ -16,12 +16,14 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %o
; SI-NEXT: s_mov_b32 s8, s2
; SI-NEXT: s_mov_b32 s9, s3
; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 1, v0
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_and_b64 s[2:3], vcc, exec
+; SI-NEXT: s_cselect_b32 s2, 1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -41,7 +43,9 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_eq_0(ptr addrspace(1) %o
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, 1, v0
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%load = load i1, ptr addrspace(1) %in
@@ -417,12 +421,14 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s8, s2
; SI-NEXT: s_mov_b32 s9, s3
; SI-NEXT: buffer_load_ubyte v0, off, s[8:11], 0
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v0, 1, v0
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_and_b64 s[2:3], vcc, exec
+; SI-NEXT: s_cselect_b32 s2, 1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -442,7 +448,9 @@ define amdgpu_kernel void @sextload_i1_to_i32_trunc_cmp_ne_neg1(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, 1, v0
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%load = load i1, ptr addrspace(1) %in
diff --git a/llvm/test/CodeGen/AMDGPU/uaddo.ll b/llvm/test/CodeGen/AMDGPU/uaddo.ll
index b000fae124ede..76901a5b7b6f0 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddo.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefix=SI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefix=VI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 | FileCheck %s --check-prefix=GFX9
@@ -655,9 +655,11 @@ define amdgpu_kernel void @v_uaddo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, v1, v0
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -680,8 +682,10 @@ define amdgpu_kernel void @v_uaddo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; VI-NEXT: v_and_b32_e32 v6, 0xffff, v5
; VI-NEXT: v_cmp_lt_u32_e32 vcc, v6, v4
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_short v[0:1], v5
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_byte v[2:3], v0
; VI-NEXT: s_endpgm
;
@@ -695,7 +699,9 @@ define amdgpu_kernel void @v_uaddo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v2, v1, v2
; GFX9-NEXT: v_cmp_lt_u32_sdwa s[0:1], v2, v1 src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_short v0, v2, s[8:9]
; GFX9-NEXT: global_store_byte v0, v1, s[10:11]
; GFX9-NEXT: s_endpgm
@@ -711,7 +717,9 @@ define amdgpu_kernel void @v_uaddo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v2, v1, v2
; GFX10-NEXT: v_cmp_lt_u32_sdwa s0, v2, v1 src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: global_store_short v0, v2, s[8:9]
; GFX10-NEXT: global_store_byte v0, v1, s[10:11]
; GFX10-NEXT: s_endpgm
@@ -730,7 +738,10 @@ define amdgpu_kernel void @v_uaddo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v2
; GFX11-NEXT: v_cmp_lt_u32_e32 vcc_lo, v3, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b16 v0, v2, s[0:1]
; GFX11-NEXT: global_store_b8 v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/usubo.ll b/llvm/test/CodeGen/AMDGPU/usubo.ll
index 8a54ad301f48a..454a0e608c341 100644
--- a/llvm/test/CodeGen/AMDGPU/usubo.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubo.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tahiti | FileCheck %s --check-prefix=SI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=tonga | FileCheck %s --check-prefix=VI
; RUN: llc < %s -amdgpu-scalarize-global-loads=false -mtriple=amdgcn -mcpu=gfx900 | FileCheck %s --check-prefix=GFX9
@@ -654,9 +654,11 @@ define amdgpu_kernel void @v_usubo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; SI-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, v1, v0
+; SI-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-NEXT: s_cselect_b32 s0, 1, 0
; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -679,8 +681,10 @@ define amdgpu_kernel void @v_usubo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; VI-NEXT: v_and_b32_e32 v4, 0xffff, v4
; VI-NEXT: v_and_b32_e32 v6, 0xffff, v5
; VI-NEXT: v_cmp_gt_u32_e32 vcc, v6, v4
+; VI-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-NEXT: s_cselect_b32 s0, 1, 0
; VI-NEXT: flat_store_short v[0:1], v5
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: flat_store_byte v[2:3], v0
; VI-NEXT: s_endpgm
;
@@ -694,7 +698,9 @@ define amdgpu_kernel void @v_usubo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_sub_u32_e32 v2, v1, v2
; GFX9-NEXT: v_cmp_gt_u32_sdwa s[0:1], v2, v1 src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX9-NEXT: s_cselect_b32 s0, 1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_short v0, v2, s[8:9]
; GFX9-NEXT: global_store_byte v0, v1, s[10:11]
; GFX9-NEXT: s_endpgm
@@ -710,7 +716,9 @@ define amdgpu_kernel void @v_usubo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_sub_nc_u32_e32 v2, v1, v2
; GFX10-NEXT: v_cmp_gt_u32_sdwa s0, v2, v1 src0_sel:WORD_0 src1_sel:WORD_0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX10-NEXT: s_cselect_b32 s0, 1, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: global_store_short v0, v2, s[8:9]
; GFX10-NEXT: global_store_byte v0, v1, s[10:11]
; GFX10-NEXT: s_endpgm
@@ -729,7 +737,10 @@ define amdgpu_kernel void @v_usubo_i16(ptr addrspace(1) %out, ptr addrspace(1) %
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v2
; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, v3, v1
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11-NEXT: s_and_b32 s4, vcc_lo, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, s4
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b16 v0, v2, s[0:1]
; GFX11-NEXT: global_store_b8 v0, v1, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend.ll b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
index 0199aa6c00427..5b3fd7e67528f 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
@@ -1,16 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope --check-prefixes=GCN,SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope --check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tahiti < %s | FileCheck -enable-var-scope --check-prefix=SI %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope --check-prefix=VI %s
; RUN: llc -mtriple=r600 -mcpu=redwood < %s | FileCheck -check-prefix=R600 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GFX1250 %s
-; R600: {{^}}s_mad_zext_i32_to_i64:
-; R600: MEM_RAT_CACHELESS STORE_RAW
-; R600: MEM_RAT_CACHELESS STORE_RAW
-
-; GCN: {{^}}s_mad_zext_i32_to_i64:
-; GCN: v_mov_b32_e32 v[[V_ZERO:[0-9]]], 0{{$}}
-; GCN: buffer_store_dwordx2 v[0:[[V_ZERO]]]
define amdgpu_kernel void @s_mad_zext_i32_to_i64(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) #0 {
; SI-LABEL: s_mad_zext_i32_to_i64:
; SI: ; %bb.0: ; %entry
@@ -74,8 +67,6 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}s_cmp_zext_i1_to_i32
-; GCN: v_cndmask_b32
define amdgpu_kernel void @s_cmp_zext_i1_to_i32(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
; SI-LABEL: s_cmp_zext_i1_to_i32:
; SI: ; %bb.0: ; %entry
@@ -139,7 +130,6 @@ entry:
ret void
}
-; GCN-LABEL: {{^}}s_arg_zext_i1_to_i64:
define amdgpu_kernel void @s_arg_zext_i1_to_i64(ptr addrspace(1) %out, i1 zeroext %arg) #0 {
; SI-LABEL: s_arg_zext_i1_to_i64:
; SI: ; %bb.0:
@@ -200,10 +190,6 @@ define amdgpu_kernel void @s_arg_zext_i1_to_i64(ptr addrspace(1) %out, i1 zeroex
ret void
}
-; GCN-LABEL: {{^}}s_cmp_zext_i1_to_i64:
-; GCN-DAG: s_mov_b32 s{{[0-9]+}}, 0
-; GCN-DAG: s_cmp_eq_u32
-; GCN: v_cndmask_b32
define amdgpu_kernel void @s_cmp_zext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
; SI-LABEL: s_cmp_zext_i1_to_i64:
; SI: ; %bb.0:
@@ -271,17 +257,7 @@ define amdgpu_kernel void @s_cmp_zext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i
}
; FIXME: Why different commute?
-; GCN-LABEL: {{^}}s_cmp_zext_i1_to_i16
-; GCN: s_load_dword [[A:s[0-9]+]]
-; GCN: s_load_dword [[B:s[0-9]+]]
-; GCN-DAG: s_and_b32 [[MASK_A:s[0-9]+]], [[A]], 0xffff{{$}}
-; GCN-DAG: s_and_b32 [[MASK_B:s[0-9]+]], [[B]], 0xffff{{$}}
-; VI: s_cmp_eq_u32 [[MASK_B]], [[MASK_A]]
-; SI: s_cmp_eq_u32 [[MASK_A]], [[MASK_B]]
-; GCN: s_cselect_b64 [[CC:s\[[0-9:]+\]]], -1, 0
-; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
-; GCN: buffer_store_short [[RESULT]]
define amdgpu_kernel void @s_cmp_zext_i1_to_i16(ptr addrspace(1) %out, [8 x i32], i16 zeroext %a, [8 x i32], i16 zeroext %b) #0 {
; SI-LABEL: s_cmp_zext_i1_to_i16:
; SI: ; %bb.0:
@@ -382,13 +358,19 @@ define i32 @divergent_i32_eq_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_eq_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_eq_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_eq_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_eq_i32:
; R600: ; %bb.0: ; %entry
@@ -417,14 +399,21 @@ define i64 @divergent_i32_eq_i64(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 0
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_eq_i64:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_eq_i64:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_eq_i64:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_eq_i64:
; R600: ; %bb.0: ; %entry
@@ -453,13 +442,19 @@ define i32 @divergent_i32_ne_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_ne_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_ne_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_ne_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_ne_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_ne_i32:
; R600: ; %bb.0: ; %entry
@@ -487,13 +482,19 @@ define i32 @divergent_i32_ugt_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_ugt_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_ugt_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_ugt_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_ugt_i32:
; R600: ; %bb.0: ; %entry
@@ -521,13 +522,19 @@ define i32 @divergent_i32_uge_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_uge_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_uge_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_uge_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_ge_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_uge_i32:
; R600: ; %bb.0: ; %entry
@@ -555,13 +562,19 @@ define i32 @divergent_i32_ult_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_ult_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_ult_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_ult_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_lt_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_ult_i32:
; R600: ; %bb.0: ; %entry
@@ -589,13 +602,19 @@ define i32 @divergent_i32_ule_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_ule_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_ule_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_ule_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_le_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_ule_i32:
; R600: ; %bb.0: ; %entry
@@ -623,13 +642,19 @@ define i32 @divergent_i32_sgt_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_sgt_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_sgt_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_sgt_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_gt_i32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_sgt_i32:
; R600: ; %bb.0: ; %entry
@@ -657,13 +682,19 @@ define i32 @divergent_i32_sge_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_sge_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_sge_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_sge_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_sge_i32:
; R600: ; %bb.0: ; %entry
@@ -691,13 +722,19 @@ define i32 @divergent_i32_slt_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_slt_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_slt_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_slt_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_lt_i32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_slt_i32:
; R600: ; %bb.0: ; %entry
@@ -725,13 +762,19 @@ define i32 @divergent_i32_sle_i32(i32 %a, i32 %b) #0 {
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX950-NEXT: s_setpc_b64 s[30:31]
+; SI-LABEL: divergent_i32_sle_i32:
+; SI: ; %bb.0: ; %entry
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; SI-NEXT: s_setpc_b64 s[30:31]
;
-; GCN-LABEL: divergent_i32_sle_i32:
-; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; VI-LABEL: divergent_i32_sle_i32:
+; VI: ; %bb.0: ; %entry
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_cmp_le_i32_e32 vcc, v0, v1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; VI-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: divergent_i32_sle_i32:
; R600: ; %bb.0: ; %entry
>From aa4ab9dd48d975463142c01120aa9d8837a0d919 Mon Sep 17 00:00:00 2001
From: Zeng Wu <Zeng.Wu2 at amd.com>
Date: Thu, 11 Jun 2026 07:07:53 +0000
Subject: [PATCH 31/31] update lit test and other RC selection in InstrEmitter
---
.../lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 18 +-
.../AMDGPU/amdgpu-cs-chain-fp-nosave.ll | 140 +-
.../AMDGPU/atomic-optimizer-strict-wqm.ll | 6 +-
.../atomic_optimizations_global_pointer.ll | 195 ++-
.../atomic_optimizations_local_pointer.ll | 262 ++--
.../atomic_optimizations_pixelshader.ll | 8 +-
llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 98 +-
.../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 16 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 1128 +++++++++--------
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 15 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 8 +-
.../AMDGPU/global-atomic-fadd.f32-no-rtn.ll | 2 +-
.../AMDGPU/global-atomic-fadd.f32-rtn.ll | 2 +-
.../AMDGPU/global_atomics_scan_fadd.ll | 415 +++---
.../AMDGPU/global_atomics_scan_fmax.ll | 197 ++-
.../AMDGPU/global_atomics_scan_fmin.ll | 197 ++-
.../AMDGPU/global_atomics_scan_fsub.ll | 415 +++---
.../CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll | 3 +-
.../CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll | 6 +-
.../AMDGPU/llvm.amdgcn.readfirstlane.ll | 10 +-
llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll | 47 +-
.../AMDGPU/sched_mfma_rewrite_copies.mir | 1 +
llvm/test/CodeGen/AMDGPU/zero_extend.ll | 6 +-
.../AMDGPU/zero_extend_i1_to_i32_i64.ll | 22 +-
24 files changed, 1530 insertions(+), 1687 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 2ec959f392738..d926f343f307f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -125,10 +125,8 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
if (User->isMachineOpcode()) {
const MCInstrDesc &II = TII->get(User->getMachineOpcode());
const TargetRegisterClass *RC = nullptr;
- if (i + II.getNumDefs() < II.getNumOperands()) {
- RC = TRI->getAllocatableClass(
- TII->getRegClass(II, i + II.getNumDefs()));
- }
+ if (i + II.getNumDefs() < II.getNumOperands())
+ RC = TII->getRegClass(II, i + II.getNumDefs());
if (!UseRC)
UseRC = RC;
else if (RC) {
@@ -152,12 +150,16 @@ void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
const TargetRegisterClass *RegClassForVT =
VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, Op->isDivergent());
- if (!UseRC || !UseRC->isAllocatable()) {
+ if (!UseRC) {
UseRC = RegClassForVT;
- } else if (const TargetRegisterClass *CommonSubClass =
- TRI->getCommonSubClass(UseRC, RegClassForVT)) {
- UseRC = CommonSubClass;
+ } else if (RegClassForVT) {
+ if (const TargetRegisterClass *CommonSubClass =
+ TRI->getCommonSubClass(UseRC, RegClassForVT))
+ UseRC = CommonSubClass;
+ else if (!UseRC->isAllocatable())
+ UseRC = RegClassForVT;
}
+ UseRC = TRI->getAllocatableClass(UseRC);
const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
SrcRC = TRI->getMinimalPhysRegClass(SrcReg, VT);
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
index 4d9652b210f66..0835837a98f98 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-fp-nosave.ll
@@ -96,23 +96,28 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) #0 {
; GFX12-NEXT: v_lshl_add_u32 v2, v8, 2, 15
; GFX12-NEXT: s_mov_b32 s33, s32
; GFX12-NEXT: s_add_co_i32 s32, s32, 16
-; GFX12-NEXT: v_and_b32_e32 v1, -16, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_readlane_b32 s4, v1, s3
-; GFX12-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX12-NEXT: s_max_u32 s2, s2, s4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX12-NEXT: s_or_saveexec_b32 s0, -1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX12-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: v_mov_b32_e32 v1, s32
+; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, v2, s0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: ds_swizzle_b32 v1, v0 offset:swizzle(BROADCAST,32,15)
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_max_u32_e32 v0, v0, v1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_lshl_add_u32 v2, s2, 6, v1
-; GFX12-NEXT: scratch_store_b32 v1, v0, off
+; GFX12-NEXT: v_readlane_b32 s1, v0, 31
+; GFX12-NEXT: s_mov_b32 exec_lo, s0
+; GFX12-NEXT: s_mov_b32 s0, s32
+; GFX12-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_lshl_add_u32 v2, s1, 5, s0
+; GFX12-NEXT: scratch_store_b32 off, v3, s0
; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_endpgm
;
@@ -122,18 +127,31 @@ define amdgpu_cs_chain void @test_alloca_var(i32 %count) #0 {
; GFX942-NEXT: v_lshl_add_u32 v1, v8, 2, 15
; GFX942-NEXT: s_mov_b32 s33, s32
; GFX942-NEXT: s_add_i32 s32, s32, 16
-; GFX942-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX942-NEXT: v_readlane_b32 s4, v1, s3
-; GFX942-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX942-NEXT: s_max_u32 s2, s2, s4
-; GFX942-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX942-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX942-NEXT: ; %bb.2:
-; GFX942-NEXT: v_mov_b32_e32 v1, s32
-; GFX942-NEXT: v_lshl_add_u32 v2, s2, 6, v1
-; GFX942-NEXT: scratch_store_dword v1, v0, off
-; GFX942-NEXT: v_readfirstlane_b32 s32, v2
+; GFX942-NEXT: v_and_b32_e32 v1, -16, v1
+; GFX942-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GFX942-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_readlane_b32 s2, v0, 63
+; GFX942-NEXT: s_mov_b64 exec, s[0:1]
+; GFX942-NEXT: s_mov_b32 s0, s32
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_readfirstlane_b32 s32, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: scratch_store_dword off, v1, s0
; GFX942-NEXT: s_endpgm
%v = alloca i32, i32 %count, align 4, addrspace(5)
store i32 0, ptr addrspace(5) %v, align 4
@@ -271,11 +289,27 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) #0 {
; GFX12-NEXT: s_add_co_u32 s0, s0, foo at gotpcrel32@lo+12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
-; GFX12-NEXT: v_mov_b32_e32 v1, s32
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: ds_swizzle_b32 v1, v0 offset:swizzle(BROADCAST,32,15)
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-NEXT: s_or_saveexec_b32 s2, -1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_lshl_add_u32 v2, s2, 6, v1
-; GFX12-NEXT: scratch_store_b32 v1, v0, off
+; GFX12-NEXT: v_readlane_b32 s3, v0, 31
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 s2, s32
+; GFX12-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_lshl_add_u32 v2, s3, 5, s2
+; GFX12-NEXT: scratch_store_b32 off, v3, s2
; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
@@ -310,10 +344,13 @@ define amdgpu_cs_chain void @test_alloca_and_call_var(i32 %count) #0 {
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v1, s32
-; GFX942-NEXT: v_lshl_add_u32 v2, s2, 6, v1
-; GFX942-NEXT: scratch_store_dword v1, v0, off
-; GFX942-NEXT: v_readfirstlane_b32 s32, v2
+; GFX942-NEXT: s_mov_b32 s3, s32
+; GFX942-NEXT: v_mov_b32_e32 v1, s3
+; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_readfirstlane_b32 s32, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: scratch_store_dword off, v1, s3
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX942-NEXT: s_endpgm
@@ -453,15 +490,32 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) #0 {
; GFX12-NEXT: s_add_co_u32 s0, s0, foo at gotpcrel32@lo+12
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_add_co_ci_u32 s1, s1, foo at gotpcrel32@hi+24
-; GFX12-NEXT: v_mov_b32_e32 v41, s32
-; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshl_add_u32 v0, s2, 6, v41
-; GFX12-NEXT: v_readfirstlane_b32 s32, v0
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX12-NEXT: ds_swizzle_b32 v1, v0 offset:swizzle(BROADCAST,32,15)
+; GFX12-NEXT: s_wait_dscnt 0x0
+; GFX12-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-NEXT: s_or_saveexec_b32 s2, -1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_readlane_b32 s3, v0, 31
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_mov_b32 exec_lo, s2
+; GFX12-NEXT: s_mov_b32 s4, s32
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_lshl_add_u32 v2, s3, 5, s4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_readfirstlane_b32 s32, v2
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX12-NEXT: scratch_store_b32 v41, v40, off
+; GFX12-NEXT: v_mov_b32_e32 v2, 0
+; GFX12-NEXT: scratch_store_b32 off, v2, s4
; GFX12-NEXT: s_endpgm
;
; GFX942-LABEL: test_call_and_alloca_var:
@@ -492,13 +546,15 @@ define amdgpu_cs_chain void @test_call_and_alloca_var(i32 %count) #0 {
; GFX942-NEXT: s_add_u32 s0, s0, foo at gotpcrel32@lo+4
; GFX942-NEXT: s_addc_u32 s1, s1, foo at gotpcrel32@hi+12
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v41, s32
-; GFX942-NEXT: v_lshl_add_u32 v0, s2, 6, v41
+; GFX942-NEXT: s_mov_b32 s4, s32
+; GFX942-NEXT: v_mov_b32_e32 v1, s4
+; GFX942-NEXT: v_lshl_add_u32 v1, s2, 6, v1
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_readfirstlane_b32 s32, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX942-NEXT: scratch_store_dword v41, v40, off
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: scratch_store_dword off, v1, s4
; GFX942-NEXT: s_endpgm
%v = alloca i32, i32 %count, align 4, addrspace(5)
call amdgpu_gfx void @foo()
diff --git a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
index c80a0f91bf7cb..e20e794d4909a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic-optimizer-strict-wqm.ll
@@ -26,11 +26,9 @@ define amdgpu_ps void @main(i32 %arg) {
; GFX10-NEXT: s_and_saveexec_b32 s3, s1
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.3: ; in Loop: Header=BB0_2 Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v1, exec_lo
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v1
-; GFX10-NEXT: s_and_b32 s0, exec_lo, s0
-; GFX10-NEXT: s_mov_b32 exec_lo, s0
+; GFX10-NEXT: s_and_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB0_1
; GFX10-NEXT: ; %bb.4: ; in Loop: Header=BB0_2 Depth=1
; GFX10-NEXT: s_mov_b32 s5, s4
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 5d4ec53921181..4ee0c3aa9e2c2 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -1257,7 +1257,6 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: add_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1274,7 +1273,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -1378,9 +1377,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-LABEL: add_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1400,11 +1398,11 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132_DPP-NEXT: s_mov_b32 s4, s6
; GFX1132_DPP-NEXT: s_mov_b32 s6, -1
@@ -1507,9 +1505,8 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-LABEL: add_i32_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1529,11 +1526,11 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232_DPP-NEXT: s_mov_b32 s4, s6
; GFX1232_DPP-NEXT: s_mov_b32 s6, -1
@@ -1722,20 +1719,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB3_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
+; GFX1032-NEXT: s_mul_i32 s5, s5, 5
; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: v_mov_b32_e32 v0, s5
; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_mov_b32 s8, s2
-; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1032-NEXT: s_mov_b32 s9, s3
; GFX1032-NEXT: buffer_atomic_add_x2 v[0:1], off, s[8:11], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
@@ -1793,21 +1791,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
; GFX1132-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
-; GFX1132-NEXT: v_mov_b32_e32 v1, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-NEXT: s_mul_i32 s5, s5, 5
; GFX1132-NEXT: s_mov_b32 s10, -1
+; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_mov_b32 s8, s2
-; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1132-NEXT: s_mov_b32 s9, s3
; GFX1132-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
@@ -1866,21 +1864,21 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: add_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB3_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: v_bcnt_u32_b32 v0, v3, 0
-; GFX1232-NEXT: v_mov_b32_e32 v1, 0
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
-; GFX1232-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_add_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -3059,7 +3057,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: add_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -3107,7 +3104,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s7, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v9, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 16
; GFX1032_DPP-NEXT: v_writelane_b32 v7, s7, 16
@@ -3258,7 +3255,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: add_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -3306,7 +3302,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -3456,7 +3453,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_DPP-LABEL: add_i64_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
-; GFX1232_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -3506,7 +3502,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
+; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -5179,7 +5176,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: sub_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -5196,7 +5192,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -5327,7 +5323,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: sub_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -5350,7 +5345,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -5472,9 +5468,8 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-LABEL: sub_i32_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1232_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1232_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -5494,11 +5489,11 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1232_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1232_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232_DPP-NEXT: s_mov_b32 s4, s6
; GFX1232_DPP-NEXT: s_mov_b32 s6, -1
@@ -5767,20 +5762,20 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, v2, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v2, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mul_i32 s10, s6, 5
; GFX1032-NEXT: s_mov_b32 s6, -1
-; GFX1032-NEXT: v_mul_lo_u32 v5, v0, 5
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v0, s4
; GFX1032-NEXT: v_mov_b32_e32 v1, s5
@@ -5788,19 +5783,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032-NEXT: s_mov_b32 s5, s3
; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032-NEXT: v_mov_b32_e32 v9, v1
-; GFX1032-NEXT: v_mov_b32_e32 v8, v0
-; GFX1032-NEXT: v_sub_co_u32 v6, vcc_lo, v8, v5
-; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v9, vcc_lo
-; GFX1032-NEXT: v_mov_b32_e32 v0, v6
-; GFX1032-NEXT: v_mov_b32_e32 v2, v8
-; GFX1032-NEXT: v_mov_b32_e32 v1, v7
-; GFX1032-NEXT: v_mov_b32_e32 v3, v9
+; GFX1032-NEXT: v_mov_b32_e32 v8, v1
+; GFX1032-NEXT: v_mov_b32_e32 v7, v0
+; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, v5
+; GFX1032-NEXT: v_mov_b32_e32 v2, v7
+; GFX1032-NEXT: v_mov_b32_e32 v1, v6
+; GFX1032-NEXT: v_mov_b32_e32 v3, v8
; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
@@ -5888,22 +5883,21 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, v2, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-NEXT: s_cbranch_execz .LBB9_4
; GFX1132-NEXT: ; %bb.1:
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
-; GFX1132-NEXT: v_bcnt_u32_b32 v0, v2, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mul_i32 s10, s6, 5
; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_mul_lo_u32 v5, v0, 5
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
; GFX1132-NEXT: s_mov_b32 s4, s2
@@ -5911,19 +5905,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX1132-NEXT: v_sub_co_u32 v6, vcc_lo, v8, v5
+; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v7, null, 0, v9, vcc_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, v6
-; GFX1132-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, v5
+; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1132-NEXT: v_mov_b32_e32 v1, v7
+; GFX1132-NEXT: v_mov_b32_e32 v1, v6
; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
@@ -5990,21 +5984,21 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-LABEL: sub_i64_constant:
; GFX1232: ; %bb.0: ; %entry
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1232-NEXT: s_mov_b32 s6, exec_lo
; GFX1232-NEXT: s_mov_b32 s4, exec_lo
+; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
; GFX1232-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1232-NEXT: s_cbranch_execz .LBB9_2
; GFX1232-NEXT: ; %bb.1:
-; GFX1232-NEXT: v_bcnt_u32_b32 v0, v3, 0
-; GFX1232-NEXT: v_mov_b32_e32 v1, 0
+; GFX1232-NEXT: s_bcnt1_i32_b32 s5, s6
; GFX1232-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-NEXT: s_mul_i32 s5, s5, 5
; GFX1232-NEXT: s_mov_b32 s10, -1
+; GFX1232-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1232-NEXT: s_wait_kmcnt 0x0
; GFX1232-NEXT: s_mov_b32 s8, s2
-; GFX1232-NEXT: v_mul_lo_u32 v0, v0, 5
; GFX1232-NEXT: s_mov_b32 s9, s3
; GFX1232-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX1232-NEXT: s_wait_loadcnt 0x0
@@ -7559,7 +7553,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_DPP-LABEL: sub_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -7607,7 +7600,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v9, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 16
@@ -7799,7 +7792,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_DPP-LABEL: sub_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -7847,7 +7839,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 16
; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 16
@@ -8019,7 +8012,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_DPP-LABEL: sub_i64_varying:
; GFX1232_DPP: ; %bb.0: ; %entry
-; GFX1232_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1232_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -8069,7 +8061,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: v_readlane_b32 s5, v1, 31
; GFX1232_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1232_DPP-NEXT: s_mov_b32 exec_lo, s6
-; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v8, 0
+; GFX1232_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232_DPP-NEXT: s_or_saveexec_b32 s6, -1
; GFX1232_DPP-NEXT: v_writelane_b32 v6, s7, 16
; GFX1232_DPP-NEXT: v_writelane_b32 v7, s8, 16
@@ -8322,9 +8315,8 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s3, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8483,10 +8475,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8537,10 +8528,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8705,10 +8695,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -8761,10 +8750,9 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10194,9 +10182,8 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: s_mov_b32 s3, 0
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10355,10 +10342,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-TRUE16-NEXT: s_clause 0x1
; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10409,10 +10395,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1132-FAKE16-NEXT: s_clause 0x1
; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10577,10 +10562,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-TRUE16-NEXT: s_clause 0x1
; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
@@ -10633,10 +10617,9 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_clause 0x1
; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 91b5a72987156..9f62cd6fdfd33 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -887,7 +887,6 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: add_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -901,7 +900,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -993,7 +992,6 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: add_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -1008,12 +1006,12 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -1353,7 +1351,6 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i32_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -1363,7 +1360,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
@@ -1416,7 +1413,7 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
;
; GFX1132_DPP-LABEL: add_i32_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
@@ -1430,10 +1427,11 @@ define amdgpu_kernel void @add_i32_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -1578,16 +1576,17 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: add_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
+; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v0, s1
; GFX1032-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
@@ -1638,18 +1637,19 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: add_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
+; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s1
; GFX1132-NEXT: ds_add_rtn_u64 v[0:1], v1, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -2549,7 +2549,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: add_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -2594,7 +2593,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v9, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
@@ -2738,7 +2737,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: add_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -2779,12 +2777,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, v8, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -3238,7 +3236,6 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_DPP-LABEL: add_i64_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
@@ -3271,8 +3268,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v7, 0
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, v1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v8, v2
@@ -3352,7 +3348,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1132_DPP-LABEL: add_i64_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v5, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s0
@@ -3386,11 +3382,11 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v4, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v3, null, v1, v4, vcc_lo
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v5, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v5, v2
; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v3
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v7
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB7_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -4284,7 +4280,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: sub_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4298,7 +4293,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -4390,7 +4385,6 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: sub_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -4405,12 +4399,12 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -4750,7 +4744,6 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1032_DPP-LABEL: sub_i32_varying_nouse:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v1, v1 row_xmask:1 row_mask:0xf bank_mask:0xf bound_ctrl:1
@@ -4760,7 +4753,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1032_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1032_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, v1
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
@@ -4813,7 +4806,7 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
;
; GFX1132_DPP-LABEL: sub_i32_varying_nouse:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v3, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
@@ -4827,10 +4820,11 @@ define amdgpu_kernel void @sub_i32_varying_nouse() {
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132_DPP-NEXT: v_add_nc_u32_e32 v1, v1, v2
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v3, 0
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132_DPP-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, v1
; GFX1132_DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_2
; GFX1132_DPP-NEXT: ; %bb.1:
@@ -4979,16 +4973,17 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: sub_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1032-NEXT: s_mov_b32 s1, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: v_mul_lo_u32 v0, v0, 5
+; GFX1032-NEXT: s_mul_i32 s1, s1, 5
+; GFX1032-NEXT: v_mov_b32_e32 v0, s1
; GFX1032-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
@@ -5045,18 +5040,19 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: sub_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v3, exec_lo
+; GFX1132-NEXT: s_mov_b32 s1, exec_lo
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s1, 0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v3, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: v_bcnt_u32_b32 v0, v3, 0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s1, s1
; GFX1132-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_mul_lo_u32 v0, v0, 5
+; GFX1132-NEXT: s_mul_i32 s1, s1, 5
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s1
; GFX1132-NEXT: ds_sub_rtn_u64 v[0:1], v1, v[0:1]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
@@ -5967,7 +5963,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: sub_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -6012,7 +6007,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, v9, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v9, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
@@ -6156,7 +6151,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: sub_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
@@ -6197,12 +6191,12 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, v8, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v8, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -6679,7 +6673,6 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: and_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1
@@ -6693,7 +6686,7 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -6786,9 +6779,8 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: and_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6800,12 +6792,12 @@ define amdgpu_kernel void @and_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_and_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -7370,7 +7362,6 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: and_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, v0, s2
@@ -7395,7 +7386,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -7520,7 +7511,7 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: and_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, 0, s2
@@ -7552,7 +7543,8 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -8029,7 +8021,6 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: or_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8043,7 +8034,7 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -8135,7 +8126,6 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: or_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -8150,12 +8140,12 @@ define amdgpu_kernel void @or_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_or_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -8719,7 +8709,6 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: or_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -8744,7 +8733,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -8869,7 +8858,7 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: or_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
@@ -8901,7 +8890,8 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -9378,7 +9368,6 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: xor_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -9392,7 +9381,7 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -9484,7 +9473,6 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: xor_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -9499,12 +9487,12 @@ define amdgpu_kernel void @xor_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_xor_b32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -10068,7 +10056,6 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: xor_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, v0, s2
@@ -10093,7 +10080,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -10218,7 +10205,7 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: xor_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0, 0, s2
@@ -10250,7 +10237,8 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v6, s3, 16
@@ -10727,7 +10715,6 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: max_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v3, 1
@@ -10741,7 +10728,7 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -10834,9 +10821,8 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: max_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0x80000000, v0, s0
; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10848,12 +10834,12 @@ define amdgpu_kernel void @max_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_max_i32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -11024,8 +11010,7 @@ define amdgpu_kernel void @max_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: max_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11090,9 +11075,8 @@ define amdgpu_kernel void @max_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: max_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -11804,7 +11788,6 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: max_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s2
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, 1
@@ -11854,7 +11837,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -12021,7 +12004,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: max_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, 0, s2
@@ -12071,13 +12054,13 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -12555,7 +12538,6 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: min_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fffffff, v0, s0
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v3, -2
@@ -12569,7 +12551,7 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -12662,9 +12644,8 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: min_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, 0x7fffffff, v0, s0
; GFX1132_DPP-NEXT: v_bfrev_b32_e32 v3, -2
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12676,12 +12657,12 @@ define amdgpu_kernel void @min_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_min_i32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -12852,8 +12833,7 @@ define amdgpu_kernel void @min_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: min_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -12918,9 +12898,8 @@ define amdgpu_kernel void @min_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: min_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -13632,7 +13611,6 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: min_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s2
; GFX1032_DPP-NEXT: v_bfrev_b32_e32 v4, -2
@@ -13682,7 +13660,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -13849,7 +13827,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: min_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fffffff, 0, s2
@@ -13899,13 +13877,13 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, -1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -14383,7 +14361,6 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umax_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -14397,7 +14374,7 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -14489,7 +14466,6 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umax_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
@@ -14504,12 +14480,12 @@ define amdgpu_kernel void @umax_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_max_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
@@ -14676,8 +14652,7 @@ define amdgpu_kernel void @umax_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: umax_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -14742,9 +14717,8 @@ define amdgpu_kernel void @umax_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: umax_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -15451,7 +15425,6 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umax_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, 0
@@ -15501,7 +15474,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -15668,7 +15641,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umax_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, 0, 0, s2
@@ -15712,13 +15685,13 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_cndmask_b32 v1, v3, v1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -16196,7 +16169,6 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umin_i32_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, -1
@@ -16210,7 +16182,7 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -16303,9 +16275,8 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-LABEL: umin_i32_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v1, -1, v0, s0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -16317,12 +16288,12 @@ define amdgpu_kernel void @umin_i32_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_permlanex16_b32 v2, v1, -1, -1
; GFX1132_DPP-NEXT: v_min_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v1, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s2, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s1, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
@@ -16490,8 +16461,7 @@ define amdgpu_kernel void @umin_i64_constant(ptr addrspace(1) %out) {
;
; GFX1032-LABEL: umin_i64_constant:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -16556,9 +16526,8 @@ define amdgpu_kernel void @umin_i64_constant(ptr addrspace(1) %out) {
;
; GFX1132-LABEL: umin_i64_constant:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -17263,7 +17232,6 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_DPP-LABEL: umin_i64_varying:
; GFX1032_DPP: ; %bb.0: ; %entry
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v7, exec_lo
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s2
; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, -1
@@ -17313,7 +17281,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1032_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1032_DPP-NEXT: v_writelane_b32 v5, s3, 16
@@ -17480,7 +17448,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_DPP-LABEL: umin_i64_varying:
; GFX1132_DPP: ; %bb.0: ; %entry
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v7, exec_lo :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX1132_DPP-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_cndmask_b32_e64 v2, -1, 0, s2
@@ -17524,13 +17492,13 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1132_DPP-NEXT: v_dual_mov_b32 v4, -1 :: v_dual_cndmask_b32 v1, v3, v1
; GFX1132_DPP-NEXT: v_readlane_b32 s3, v2, 15
; GFX1132_DPP-NEXT: v_readlane_b32 s1, v2, 31
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_readlane_b32 s0, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v5, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_dpp v4, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s2
-; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v7, 0
+; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s2, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v5, s3, 16
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 031420831dbaa..0e6774853ce54 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -438,7 +438,6 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1032-NEXT: s_and_saveexec_b32 s8, s9
; GFX1032-NEXT: s_cbranch_execz .LBB1_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1032-NEXT: s_or_saveexec_b32 s9, -1
; GFX1032-NEXT: v_cndmask_b32_e64 v1, 0, v0, s9
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
@@ -452,7 +451,7 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1032-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032-NEXT: v_readlane_b32 s10, v1, 15
; GFX1032-NEXT: s_mov_b32 exec_lo, s9
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: s_or_saveexec_b32 s9, -1
; GFX1032-NEXT: v_writelane_b32 v3, s10, 16
; GFX1032-NEXT: s_mov_b32 exec_lo, s9
@@ -561,7 +560,6 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1132-NEXT: s_and_saveexec_b32 s8, s9
; GFX1132-NEXT: s_cbranch_execz .LBB1_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: v_mov_b32_e32 v4, exec_lo
; GFX1132-NEXT: s_or_saveexec_b32 s9, -1
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1132-NEXT: v_cndmask_b32_e64 v1, 0, v0, s9
@@ -579,11 +577,11 @@ define amdgpu_ps void @add_i32_varying(ptr addrspace(8) inreg %out, ptr addrspac
; GFX1132-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132-NEXT: v_readlane_b32 s10, v1, 15
; GFX1132-NEXT: s_mov_b32 exec_lo, s9
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v4, 0
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_or_saveexec_b32 s9, -1
; GFX1132-NEXT: v_writelane_b32 v3, s10, 16
; GFX1132-NEXT: s_mov_b32 exec_lo, s9
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-NEXT: ; implicit-def: $vgpr0
; GFX1132-NEXT: s_and_saveexec_b32 s9, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index 3055e6a4aa55d..15a93435c0af8 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -458,13 +458,9 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX908-NEXT: s_add_u32 s6, s6, global at rel32@lo+4
; GFX908-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX908-NEXT: s_cmp_eq_u32 s7, s5
-; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX908-NEXT: s_cselect_b32 s6, 1, 0
-; GFX908-NEXT: s_mov_b64 s[4:5], -1
-; GFX908-NEXT: s_mov_b32 s7, 1
-; GFX908-NEXT: s_cmp_lg_u32 s6, s7
; GFX908-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX908-NEXT: s_mov_b64 s[4:5], -1
+; GFX908-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_cbranch_vccnz .LBB4_3
@@ -508,13 +504,9 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX90A-NEXT: s_add_u32 s6, s6, global at rel32@lo+4
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
-; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
-; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s7, 1
-; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], -1
+; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB4_3
@@ -558,13 +550,9 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX942-NEXT: s_add_u32 s2, s2, global at rel32@lo+4
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
-; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX942-NEXT: s_cselect_b32 s2, 1, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s3, 1
-; GFX942-NEXT: s_cmp_lg_u32 s2, s3
; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT: s_mov_b64 s[0:1], -1
+; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB4_3
@@ -605,13 +593,9 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX1100-NEXT: s_add_u32 s2, s2, global at rel32@lo+4
; GFX1100-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX1100-NEXT: s_cmp_eq_u32 s3, s1
-; GFX1100-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1100-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1100-NEXT: s_cselect_b32 s1, 1, 0
-; GFX1100-NEXT: s_mov_b32 s0, -1
-; GFX1100-NEXT: s_mov_b32 s2, 1
-; GFX1100-NEXT: s_cmp_lg_u32 s1, s2
; GFX1100-NEXT: s_cselect_b32 s1, -1, 0
+; GFX1100-NEXT: s_mov_b32 s0, -1
+; GFX1100-NEXT: s_xor_b32 s1, s1, s0
; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1100-NEXT: s_cbranch_vccnz .LBB4_3
@@ -661,15 +645,10 @@ define i64 @optnone_atomicrmw_add_i64_expand(i64 %val) #1 {
; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global at rel32@hi+24
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_cmp_eq_u32 s3, s1
-; GFX1200-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1200-NEXT: s_cselect_b32 s1, 1, 0
+; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
; GFX1200-NEXT: s_mov_b32 s0, -1
-; GFX1200-NEXT: s_mov_b32 s2, 1
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
+; GFX1200-NEXT: s_xor_b32 s1, s1, s0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
@@ -727,13 +706,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX908-NEXT: s_add_u32 s6, s6, global at rel32@lo+4
; GFX908-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX908-NEXT: s_cmp_eq_u32 s7, s5
-; GFX908-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX908-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX908-NEXT: s_cselect_b32 s6, 1, 0
-; GFX908-NEXT: s_mov_b64 s[4:5], -1
-; GFX908-NEXT: s_mov_b32 s7, 1
-; GFX908-NEXT: s_cmp_lg_u32 s6, s7
; GFX908-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX908-NEXT: s_mov_b64 s[4:5], -1
+; GFX908-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX908-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX908-NEXT: s_cbranch_vccnz .LBB5_2
@@ -796,13 +771,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_add_u32 s6, s6, global at rel32@lo+4
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
-; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
-; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s7, 1
-; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], -1
+; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3
@@ -820,13 +791,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_add_u32 s6, s6, global at rel32@lo+4
; GFX90A-NEXT: s_addc_u32 s7, s7, global at rel32@hi+12
; GFX90A-NEXT: s_cmp_eq_u32 s7, s5
-; GFX90A-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX90A-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX90A-NEXT: s_cselect_b32 s6, 1, 0
-; GFX90A-NEXT: s_mov_b64 s[4:5], -1
-; GFX90A-NEXT: s_mov_b32 s7, 1
-; GFX90A-NEXT: s_cmp_lg_u32 s6, s7
; GFX90A-NEXT: s_cselect_b64 s[6:7], -1, 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], -1
+; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5
@@ -891,13 +858,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_add_u32 s2, s2, global at rel32@lo+4
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
-; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX942-NEXT: s_cselect_b32 s2, 1, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s3, 1
-; GFX942-NEXT: s_cmp_lg_u32 s2, s3
; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT: s_mov_b64 s[0:1], -1
+; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_3
@@ -915,13 +878,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_add_u32 s2, s2, global at rel32@lo+4
; GFX942-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX942-NEXT: s_cmp_eq_u32 s3, s1
-; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX942-NEXT: s_cselect_b32 s2, 1, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], -1
-; GFX942-NEXT: s_mov_b32 s3, 1
-; GFX942-NEXT: s_cmp_lg_u32 s2, s3
; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT: s_mov_b64 s[0:1], -1
+; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_5
@@ -984,13 +943,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX1100-NEXT: s_add_u32 s2, s2, global at rel32@lo+4
; GFX1100-NEXT: s_addc_u32 s3, s3, global at rel32@hi+12
; GFX1100-NEXT: s_cmp_eq_u32 s3, s1
-; GFX1100-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1100-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1100-NEXT: s_cselect_b32 s1, 1, 0
-; GFX1100-NEXT: s_mov_b32 s0, -1
-; GFX1100-NEXT: s_mov_b32 s2, 1
-; GFX1100-NEXT: s_cmp_lg_u32 s1, s2
; GFX1100-NEXT: s_cselect_b32 s1, -1, 0
+; GFX1100-NEXT: s_mov_b32 s0, -1
+; GFX1100-NEXT: s_xor_b32 s1, s1, s0
; GFX1100-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1100-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1100-NEXT: s_cbranch_vccnz .LBB5_2
@@ -1058,15 +1013,10 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX1200-NEXT: s_add_co_ci_u32 s3, s3, global at rel32@hi+24
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_cmp_eq_u32 s3, s1
-; GFX1200-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX1200-NEXT: s_cselect_b32 s1, 1, 0
+; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
; GFX1200-NEXT: s_mov_b32 s0, -1
-; GFX1200-NEXT: s_mov_b32 s2, 1
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_cmp_lg_u32 s1, s2
-; GFX1200-NEXT: s_cselect_b32 s1, -1, 0
+; GFX1200-NEXT: s_xor_b32 s1, s1, s0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_and_b32 vcc_lo, exec_lo, s1
; GFX1200-NEXT: ; implicit-def: $vgpr3_vgpr4
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
index 15476baf33109..2f635913dcb5d 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -478,7 +478,7 @@ define amdgpu_gfx void @test34(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -499,7 +499,7 @@ define amdgpu_gfx void @test35(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -520,7 +520,7 @@ define amdgpu_gfx void @test36(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -541,7 +541,7 @@ define amdgpu_gfx void @test37(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -562,7 +562,7 @@ define amdgpu_gfx void @test38(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -583,7 +583,7 @@ define amdgpu_gfx void @test39(i32 inreg %arg1, i32 inreg %arg2) {
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -604,7 +604,7 @@ define amdgpu_gfx void @test40(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
@@ -625,7 +625,7 @@ define amdgpu_gfx void @test41(i32 inreg %arg1, i32 inreg %arg2, i32 inreg %arg3
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_and_b32 s0, s0, exec_lo
; GCN-NEXT: s_cselect_b32 s0, 1, 0
-; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: global_store_b8 v[0:1], v2, off dlc
; GCN-NEXT: s_waitcnt_vscnt null, 0x0
; GCN-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 8044e3673e48f..927da53d1664c 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -250,31 +250,31 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() #0 {
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_mov_b32 s33, 0
; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB3_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB3_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7b
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_3
-; GFX9-SDAG-NEXT: ; %bb.4:
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent:
@@ -320,19 +320,25 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() #0 {
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB3_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x7b
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent:
@@ -516,31 +522,31 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s17
; GFX9-SDAG-NEXT: s_mov_b32 s33, 0
; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB5_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX9-SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB5_3
-; GFX9-SDAG-NEXT: ; %bb.4:
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent_under_aligned:
@@ -585,19 +591,25 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_under_aligne
; GFX11-SDAG-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB5_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x29a
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: test_dynamic_stackalloc_kernel_divergent_under_aligned:
@@ -646,8 +658,7 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x2000
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x2000
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_6
+; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.0
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s5, s5, 15
; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
@@ -657,39 +668,34 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_add_i32 s32, s8, s5
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: s_add_i32 s32, s9, s5
-; GFX9-SDAG-NEXT: .LBB6_2: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s5, s[6:7]
-; GFX9-SDAG-NEXT: v_readlane_b32 s10, v0, s5
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[6:7], s5
-; GFX9-SDAG-NEXT: s_max_u32 s8, s8, s10
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB6_2
-; GFX9-SDAG-NEXT: ; %bb.3:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 3
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-SDAG-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[6:7]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s5, v1, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s5, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s8
+; GFX9-SDAG-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 4
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s5, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s5
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 4
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB6_4
-; GFX9-SDAG-NEXT: ; %bb.5:
-; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: .LBB6_6: ; %bb.1
+; GFX9-SDAG-NEXT: .LBB6_2: ; %bb.1
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s4, s4, 15
; GFX9-SDAG-NEXT: s_and_b32 s4, s4, -16
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
@@ -779,22 +785,36 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_multiple_allocas(i32 %
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s2, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s5, v0, s4
-; GFX11-SDAG-NEXT: s_bitset0_b32 s3, s4
-; GFX11-SDAG-NEXT: s_max_u32 s2, s2, s5
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB6_2
-; GFX11-SDAG-NEXT: ; %bb.3:
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s32 :: v_dual_mov_b32 v3, 4
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 3
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s1 dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s2
+; GFX11-SDAG-NEXT: s_lshl2_add_u32 s1, s1, 15
+; GFX11-SDAG-NEXT: s_add_i32 s3, s32, 0x7ff
+; GFX11-SDAG-NEXT: s_and_b32 s1, s1, -16
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s4, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-SDAG-NEXT: s_and_b32 s2, s3, 0xfffff800
+; GFX11-SDAG-NEXT: s_lshl_b32 s1, s1, 5
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v3, 3 :: v_dual_mov_b32 v4, 4
+; GFX11-SDAG-NEXT: s_add_i32 s32, s2, s1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s2 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v3, off dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v4, s1 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s2, 5, v0
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s4, 5, s1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX11-SDAG-NEXT: .LBB6_4: ; %bb.1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: .LBB6_2: ; %bb.1
; GFX11-SDAG-NEXT: s_lshl2_add_u32 s0, s0, 15
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 1 :: v_dual_mov_b32 v3, 2
; GFX11-SDAG-NEXT: s_and_b32 s0, s0, -16
@@ -891,40 +911,35 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x1000
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_cmp_lg_u32 s4, 0
-; GFX9-SDAG-NEXT: s_mov_b32 s4, 0
-; GFX9-SDAG-NEXT: s_cbranch_scc0 .LBB7_8
+; GFX9-SDAG-NEXT: s_cbranch_scc0 .LBB7_4
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.1
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: .LBB7_2: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s8, s[6:7]
-; GFX9-SDAG-NEXT: v_readlane_b32 s9, v0, s8
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[6:7], s8
-; GFX9-SDAG-NEXT: s_max_u32 s4, s4, s9
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB7_2
-; GFX9-SDAG-NEXT: ; %bb.3:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s4, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: .LBB7_4: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s4, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_4
-; GFX9-SDAG-NEXT: ; %bb.5:
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[6:7]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s4, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_7
-; GFX9-SDAG-NEXT: .LBB7_6: ; %bb.0
+; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s4, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB7_3
+; GFX9-SDAG-NEXT: .LBB7_2: ; %bb.0
; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0xfff
; GFX9-SDAG-NEXT: s_lshl2_add_u32 s5, s5, 15
; GFX9-SDAG-NEXT: s_and_b32 s4, s4, 0xfffff000
@@ -935,10 +950,10 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX9-SDAG-NEXT: s_add_i32 s32, s4, s5
; GFX9-SDAG-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: .LBB7_7: ; %bb.2
+; GFX9-SDAG-NEXT: .LBB7_3: ; %bb.2
; GFX9-SDAG-NEXT: s_endpgm
-; GFX9-SDAG-NEXT: .LBB7_8:
-; GFX9-SDAG-NEXT: s_branch .LBB7_6
+; GFX9-SDAG-NEXT: .LBB7_4:
+; GFX9-SDAG-NEXT: s_branch .LBB7_2
;
; GFX9-GISEL-LABEL: test_dynamic_stackalloc_kernel_control_flow:
; GFX9-GISEL: ; %bb.0: ; %entry
@@ -1011,21 +1026,27 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_control_flow(i32 %n, i
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s4, v0, s3
-; GFX11-SDAG-NEXT: s_bitset0_b32 s2, s3
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s4
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB7_2
-; GFX11-SDAG-NEXT: ; %bb.3:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s2, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s3, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s2, 5, s3
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s3 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB7_5
-; GFX11-SDAG-NEXT: .LBB7_4: ; %bb.0
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB7_3
+; GFX11-SDAG-NEXT: .LBB7_2: ; %bb.0
; GFX11-SDAG-NEXT: s_lshl2_add_u32 s1, s1, 15
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 2
; GFX11-SDAG-NEXT: s_add_i32 s0, s32, 0x7ff
@@ -1120,32 +1141,30 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) #0 {
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB8_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB8_3
-; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x7b
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -1212,21 +1231,33 @@ define void @test_dynamic_stackalloc_device_uniform(i32 %n) #0 {
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB8_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x7b
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
-; GFX11-SDAG-NEXT: s_mov_b32 s33, s4
+; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
+; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX11-SDAG-NEXT: scratch_load_b32 v1, off, s33
+; GFX11-SDAG-NEXT: scratch_load_b32 v2, off, s33 offset:4
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s33, s2
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_dynamic_stackalloc_device_uniform:
@@ -1491,32 +1522,30 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) #0 {
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB10_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 22
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB10_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB10_3
-; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 22
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -1583,19 +1612,25 @@ define void @test_dynamic_stackalloc_device_uniform_under_aligned(i32 %n) #0 {
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB10_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 22
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 22
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
@@ -1667,32 +1702,30 @@ define void @test_dynamic_stackalloc_device_divergent() #0 {
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v31
; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB11_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v0, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v1
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x7b
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB11_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB11_3
-; GFX9-SDAG-NEXT: ; %bb.4:
-; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -1754,26 +1787,32 @@ define void @test_dynamic_stackalloc_device_divergent() #0 {
; GFX11-SDAG-NEXT: scratch_store_b32 off, v0, s33
; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s33 offset:4
; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v31
-; GFX11-SDAG-NEXT: s_add_i32 s32, s32, 16
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, v2, 2, 15
-; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x1ff0, v2
-; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB11_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x7b
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x3ff, v31
+; GFX11-SDAG-NEXT: s_add_i32 s32, s32, 16
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, v2, 2, 15
+; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x1ff0, v2
+; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v2, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v1, v0 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v0, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x7b
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v2
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
@@ -2055,32 +2094,30 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() #0 {
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v31
; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB13_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB13_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v0, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v1
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB13_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB13_3
-; GFX9-SDAG-NEXT: ; %bb.4:
-; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -2149,19 +2186,25 @@ define void @test_dynamic_stackalloc_device_divergent_under_aligned() #0 {
; GFX11-SDAG-NEXT: v_and_b32_e32 v2, 0x1ff0, v2
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB13_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v2, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v0, v0, v0 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v1, v0 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v0, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x29a
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v2
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
@@ -2239,93 +2282,90 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX9-SDAG-NEXT: s_mov_b32 s34, s32
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x3000
; GFX9-SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-SDAG-NEXT: s_cbranch_execz .LBB14_8
+; GFX9-SDAG-NEXT: s_cbranch_execz .LBB14_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.0
; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: s_mov_b32 s10, 0
-; GFX9-SDAG-NEXT: .LBB14_2: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s11, s[6:7]
-; GFX9-SDAG-NEXT: v_readlane_b32 s12, v1, s11
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[6:7], s11
-; GFX9-SDAG-NEXT: s_max_u32 s10, s10, s12
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_2
-; GFX9-SDAG-NEXT: ; %bb.3:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s9
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s10, 6, v1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v31
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
-; GFX9-SDAG-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: s_mov_b32 s10, 0
-; GFX9-SDAG-NEXT: .LBB14_4: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s11, s[6:7]
-; GFX9-SDAG-NEXT: v_readlane_b32 s12, v1, s11
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[6:7], s11
-; GFX9-SDAG-NEXT: s_max_u32 s10, s10, s12
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
-; GFX9-SDAG-NEXT: ; %bb.5:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s10, 6, v1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v2
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 3
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s9
-; GFX9-SDAG-NEXT: buffer_store_dword v2, v3, s[0:3], 0 offen
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s8, v2, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
+; GFX9-SDAG-NEXT: s_and_b32 s6, s6, 0xfffff000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, s8, 6, v1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v3
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, 0x3ff, v31
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, v3, 2, 15
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, 0x1ff0, v3
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v3, s[6:7]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s8, v2, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
+; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, s8, 6, v3
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, 3
+; GFX9-SDAG-NEXT: buffer_store_dword v3, v1, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 4
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: .LBB14_6: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s9, v1
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s9, v1
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v2, off, s[0:3], s9
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 4
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr2
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB14_6
-; GFX9-SDAG-NEXT: ; %bb.7:
-; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: .LBB14_8: ; %bb.1
+; GFX9-SDAG-NEXT: .LBB14_2: ; %bb.1
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 2
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB14_9: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s6, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s7, v0, s6
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s6
-; GFX9-SDAG-NEXT: s_max_u32 s8, s8, s7
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB14_9
-; GFX9-SDAG-NEXT: ; %bb.10:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s33
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v2, 63
+; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s33
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 2
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB14_11: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB14_11
-; GFX9-SDAG-NEXT: ; %bb.12:
-; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
; GFX9-SDAG-NEXT: s_mov_b32 s34, s10
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
@@ -2503,48 +2543,43 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) #0
; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s2, 5, s3
; GFX11-SDAG-NEXT: v_dual_mov_b32 v6, 3 :: v_dual_mov_b32 v7, 4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0x1ff0, v1
-; GFX11-SDAG-NEXT: .LBB14_4: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT: s_ctz_i32_b32 s5, s4
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s6, v1, s5
-; GFX11-SDAG-NEXT: s_bitset0_b32 s4, s5
-; GFX11-SDAG-NEXT: s_max_u32 s3, s3, s6
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB14_4
-; GFX11-SDAG-NEXT: ; %bb.5:
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, s32 :: v_dual_mov_b32 v4, 4
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 3
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s2 dlc
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v6, s3 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 v1, v4, off dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v7, s1 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v2, s3, 5, v1
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s4, 5, s1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v2
-; GFX11-SDAG-NEXT: .LBB14_6: ; %bb.1
-; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: .LBB14_2: ; %bb.1
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB14_7
-; GFX11-SDAG-NEXT: ; %bb.8:
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s32 :: v_dual_mov_b32 v3, 2
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
-; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s33 dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v3, v2 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v2, v2, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v2, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v6, 2
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s33 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v3, off dlc
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v6, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: s_mov_b32 s33, s7
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s34
; GFX11-SDAG-NEXT: s_mov_b32 s34, s6
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
@@ -2731,40 +2766,35 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX9-SDAG-NEXT: ; implicit-def: $vgpr31
; GFX9-SDAG-NEXT: .LBB15_2: ; %Flow
; GFX9-SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX9-SDAG-NEXT: s_cbranch_execz .LBB15_10
-; GFX9-SDAG-NEXT: ; %bb.5: ; %bb.0
+; GFX9-SDAG-NEXT: s_cbranch_execz .LBB15_4
+; GFX9-SDAG-NEXT: ; %bb.3: ; %bb.0
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v31
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: s_mov_b32 s8, 0
-; GFX9-SDAG-NEXT: .LBB15_6: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s9, s[6:7]
-; GFX9-SDAG-NEXT: v_readlane_b32 s10, v0, s9
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[6:7], s9
-; GFX9-SDAG-NEXT: s_max_u32 s8, s8, s10
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
-; GFX9-SDAG-NEXT: ; %bb.7:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s8, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 1
-; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], exec
-; GFX9-SDAG-NEXT: .LBB15_8: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s8, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s8, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s8
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB15_8
-; GFX9-SDAG-NEXT: ; %bb.9:
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[6:7]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s8, v2, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: .LBB15_10: ; %bb.2
+; GFX9-SDAG-NEXT: s_mov_b32 s6, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s8, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s6
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX9-SDAG-NEXT: .LBB15_4: ; %bb.2
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
; GFX9-SDAG-NEXT: s_mov_b32 s34, s10
@@ -2916,20 +2946,26 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) #0 {
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s1, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s4, v0, s3
-; GFX11-SDAG-NEXT: s_bitset0_b32 s2, s3
-; GFX11-SDAG-NEXT: s_max_u32 s1, s1, s4
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB15_6
-; GFX11-SDAG-NEXT: ; %bb.7:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s1, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s1
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v3, v2 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v2, v2, v3
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s2, v2, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-SDAG-NEXT: s_mov_b32 s1, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 1
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s2, 5, s1
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s1 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX11-SDAG-NEXT: .LBB15_8: ; %bb.2
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX11-SDAG-NEXT: .LBB15_4: ; %bb.2
; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s34
; GFX11-SDAG-NEXT: s_mov_b32 s34, s4
@@ -3058,32 +3094,30 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB16_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB16_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB16_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB16_3
-; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x7fff0, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -3153,19 +3187,25 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x7fff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB16_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x29a
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
@@ -3238,32 +3278,30 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x400
-; GFX9-SDAG-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: s_ff1_i32_b64 s7, s[4:5]
-; GFX9-SDAG-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9-SDAG-NEXT: s_bitset0_b64 s[4:5], s7
-; GFX9-SDAG-NEXT: s_max_u32 s6, s6, s8
-; GFX9-SDAG-NEXT: s_cmp_lg_u64 s[4:5], 0
-; GFX9-SDAG-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX9-SDAG-NEXT: ; %bb.2:
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s6, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0x29a
-; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], exec
-; GFX9-SDAG-NEXT: .LBB17_3: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, s6, v0
-; GFX9-SDAG-NEXT: s_and_saveexec_b64 vcc, vcc
-; GFX9-SDAG-NEXT: s_nop 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s6
-; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr0
-; GFX9-SDAG-NEXT: ; implicit-def: $vgpr1
-; GFX9-SDAG-NEXT: s_xor_b64 exec, exec, vcc
-; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB17_3
-; GFX9-SDAG-NEXT: ; %bb.4:
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
+; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: s_nop 1
+; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s32
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, s6, 6, v0
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x29a
+; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s33
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
@@ -3330,19 +3368,25 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i64(i64
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: v_readlane_b32 s3, v0, s2
-; GFX11-SDAG-NEXT: s_bitset0_b32 s1, s2
-; GFX11-SDAG-NEXT: s_max_u32 s0, s0, s3
-; GFX11-SDAG-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-SDAG-NEXT: s_cbranch_scc1 .LBB17_1
-; GFX11-SDAG-NEXT: ; %bb.2:
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s32
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0x29a
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; GFX11-SDAG-NEXT: scratch_store_b32 v0, v2, off dlc
+; GFX11-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; GFX11-SDAG-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_max_u32_e32 v1, v1, v2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_readlane_b32 s1, v1, 31
+; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
+; GFX11-SDAG-NEXT: s_mov_b32 s0, s32
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0x29a
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, s1, 5, s0
+; GFX11-SDAG-NEXT: scratch_store_b32 off, v3, s0 dlc
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v1
+; GFX11-SDAG-NEXT: v_readfirstlane_b32 s32, v0
; GFX11-SDAG-NEXT: s_mov_b32 s32, s33
; GFX11-SDAG-NEXT: s_xor_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_clause 0x1 ; 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 41bf491decf6f..7e909db790ee1 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -1764,9 +1764,12 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
; GFX1250-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f32_f64_e32 v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_3)
+; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
+; GFX1250-GISEL-TRUE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-TRUE16-NEXT: s_endpgm
;
@@ -1779,9 +1782,12 @@ define amdgpu_kernel void @fptrunc_f64_to_f16_afn(
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_3)
+; GFX1250-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_cvt_f16_f32 s2, s2
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1250-GISEL-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
@@ -4143,7 +4149,6 @@ define amdgpu_kernel void @fptrunc_v2f64_to_v2f16_afn(
; GFX1250-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v0, s[4:5]
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_f32_f64_e32 v1, s[6:7]
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s3, v1
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 3dd4adcebfa22..9a0e7bdb4c646 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -15856,7 +15856,9 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_cmp_eq_u32 s16, s17
; GFX8-GFX803-SDAG-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-GFX803-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8-GFX803-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-GFX803-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-GFX803-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX8-GFX803-SDAG-NEXT: flat_store_byte v[0:1], v2
; GFX8-GFX803-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX803-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -15879,7 +15881,9 @@ define void @freeze_i1_scc(i32 inreg %a, i32 inreg %b, ptr addrspace(1) %ptrb) {
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_cmp_eq_u32 s16, s17
; GFX8-GFX900-SDAG-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-GFX900-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5]
+; GFX8-GFX900-SDAG-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX8-GFX900-SDAG-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-GFX900-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX8-GFX900-SDAG-NEXT: global_store_byte v[0:1], v2, off
; GFX8-GFX900-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-GFX900-SDAG-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
index 1edc47fe9b5e9..c6dbe08313f94 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-no-rtn.ll
@@ -175,7 +175,7 @@ define amdgpu_ps void @global_atomic_fadd_f32_saddr_no_rtn_atomicrmw(ptr addrspa
; GFX11_GFX12-NEXT: bb.1 (%ir-block.5):
; GFX11_GFX12-NEXT: successors: %bb.2(0x40000000), %bb.3(0x40000000)
; GFX11_GFX12-NEXT: {{ $}}
- ; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $exec_lo
+ ; GFX11_GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
; GFX11_GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11_GFX12-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
; GFX11_GFX12-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
index 38c41dde0bf9c..8ade6e5586377 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
@@ -223,7 +223,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX11-NEXT: bb.1 (%ir-block.5):
; GFX11-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; GFX11-NEXT: {{ $}}
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $exec_lo
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $exec_lo
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GFX11-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]], [[S_MOV_B32_]], implicit $exec
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
index fc68d3dfa3675..a0765411c9669 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
@@ -975,7 +975,6 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -993,7 +992,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -1102,8 +1101,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -1126,10 +1124,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1275,27 +1274,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1342,12 +1341,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -1496,27 +1494,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1563,12 +1561,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -2205,7 +2202,6 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -2223,7 +2219,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -2332,8 +2328,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -2356,10 +2351,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2505,27 +2501,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2587,13 +2583,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -2756,27 +2751,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2838,13 +2833,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -3495,7 +3489,6 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -3513,7 +3506,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -3622,8 +3615,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -3646,10 +3638,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4279,7 +4272,6 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -4297,7 +4289,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -4406,8 +4398,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -4430,10 +4421,11 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4579,27 +4571,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4661,13 +4653,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -4830,27 +4821,27 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4912,13 +4903,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fadd_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -5594,7 +5584,6 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -5612,7 +5601,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -5734,8 +5723,7 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -5758,11 +5746,12 @@ define amdgpu_kernel void @global_atomic_fadd_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6912,7 +6901,6 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6940,8 +6928,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -7079,8 +7066,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -7116,9 +7102,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -7284,24 +7269,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7368,13 +7353,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -7542,24 +7526,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7626,13 +7610,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -8359,7 +8342,6 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8387,8 +8369,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -8526,8 +8507,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -8563,9 +8543,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -8731,24 +8710,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -8815,13 +8794,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -8989,24 +8967,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -9073,13 +9051,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -9806,7 +9783,6 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -9834,8 +9810,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -9973,8 +9948,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -10010,9 +9984,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -10733,7 +10706,6 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -10761,8 +10733,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -10900,8 +10871,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -10937,9 +10907,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -11105,24 +11074,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11189,13 +11158,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -11363,24 +11331,24 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11447,13 +11415,12 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fadd_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -12180,7 +12147,6 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -12208,8 +12174,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -12347,8 +12312,7 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -12384,9 +12348,8 @@ define amdgpu_kernel void @global_atomic_fadd_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 6b4bef3cfae18..b7e34871e75b2 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -96,8 +96,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
@@ -129,10 +128,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
@@ -223,8 +221,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_2
@@ -256,10 +253,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -887,7 +883,6 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -911,7 +906,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1018,8 +1013,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -1048,10 +1042,11 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1147,8 +1142,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_2
@@ -1180,10 +1174,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
@@ -1274,8 +1267,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_2
@@ -1307,10 +1299,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1939,7 +1930,6 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -1963,7 +1953,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2070,8 +2060,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -2100,10 +2089,11 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2200,8 +2190,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
@@ -2233,10 +2222,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
@@ -2327,8 +2315,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_2
@@ -2360,10 +2347,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2991,7 +2977,6 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -3015,7 +3000,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_max_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3122,8 +3107,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -3152,10 +3136,11 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_max_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -3258,8 +3243,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_2
@@ -3309,11 +3293,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
@@ -3426,8 +3409,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_2
@@ -3477,11 +3459,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4204,7 +4185,6 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4238,8 +4218,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -4378,8 +4357,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -4419,9 +4397,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -4543,8 +4520,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
@@ -4594,11 +4570,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
@@ -4711,8 +4686,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_2
@@ -4762,11 +4736,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -5489,7 +5462,6 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -5523,8 +5495,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -5663,8 +5634,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -5704,9 +5674,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -5828,8 +5797,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_2
@@ -5879,11 +5847,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
@@ -5996,8 +5963,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_2
@@ -6047,11 +6013,10 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fmax_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6774,7 +6739,6 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6808,8 +6772,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_max_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -6948,8 +6911,7 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -6989,9 +6951,8 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_max_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -7106,8 +7067,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
@@ -7139,10 +7099,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
@@ -7233,8 +7192,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_2
@@ -7266,10 +7224,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7364,8 +7321,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
@@ -7397,10 +7353,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
@@ -7491,8 +7446,7 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_2
@@ -7524,10 +7478,9 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmax_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index c7e21711a3000..7ee9b9890310b 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -96,8 +96,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB0_2
@@ -129,10 +128,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-NEXT: ; %bb.1:
@@ -223,8 +221,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB0_2
@@ -256,10 +253,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB0_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -887,7 +883,6 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -911,7 +906,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -1018,8 +1013,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -1048,10 +1042,11 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1147,8 +1142,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_2
@@ -1180,10 +1174,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-NEXT: ; %bb.1:
@@ -1274,8 +1267,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_2
@@ -1307,10 +1299,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB2_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1939,7 +1930,6 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -1963,7 +1953,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -2070,8 +2060,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -2100,10 +2089,11 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2200,8 +2190,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_2
@@ -2233,10 +2222,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-NEXT: ; %bb.1:
@@ -2327,8 +2315,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_2
@@ -2360,10 +2347,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB4_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2991,7 +2977,6 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0x7fc00000
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x7fc00000, v0, s0
@@ -3015,7 +3000,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_max_f32_e32 v4, v4, v4
; GFX1032-DPP-NEXT: v_min_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
@@ -3122,8 +3107,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x7fc00000, v0, s0
@@ -3152,10 +3136,11 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_max_f32_e32 v2, v2, v2
; GFX1132-DPP-NEXT: v_min_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -3258,8 +3243,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB6_2
@@ -3309,11 +3293,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-NEXT: ; %bb.1:
@@ -3426,8 +3409,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB6_2
@@ -3477,11 +3459,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_agent_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4204,7 +4185,6 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -4238,8 +4218,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -4378,8 +4357,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -4419,9 +4397,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -4543,8 +4520,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB8_2
@@ -4594,11 +4570,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-NEXT: ; %bb.1:
@@ -4711,8 +4686,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB8_2
@@ -4762,11 +4736,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_one_as_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -5489,7 +5462,6 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -5523,8 +5495,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -5663,8 +5634,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -5704,9 +5674,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -5828,8 +5797,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB10_2
@@ -5879,11 +5847,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-NEXT: ; %bb.1:
@@ -5996,8 +5963,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB10_2
@@ -6047,11 +6013,10 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fmin_double_uni_address_uni_value_default_scope_unsafe:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB10_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -6774,7 +6739,6 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v4, 0x7ff80000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -6808,8 +6772,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX1032-DPP-NEXT: v_min_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -6948,8 +6911,7 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_dual_mov_b32 v3, 0x7ff80000 :: v_dual_mov_b32 v2, 0
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v5, 0x7ff80000, v1, s0
@@ -6989,9 +6951,8 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-DPP-NEXT: v_min_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
@@ -7106,8 +7067,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB12_2
@@ -7139,10 +7099,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-NEXT: ; %bb.1:
@@ -7233,8 +7192,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB12_2
@@ -7266,10 +7224,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_ignore_denormal_mode__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB12_2
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7364,8 +7321,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_2
@@ -7397,10 +7353,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-NEXT: ; %bb.1:
@@ -7491,8 +7446,7 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1032-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_2
@@ -7524,10 +7478,9 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_uni_value_system_scope
;
; GFX1132-DPP-LABEL: global_atomic_fmin_uni_address_uni_value_system_scope__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v0, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB13_2
; GFX1132-DPP-NEXT: ; %bb.1:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
index 7d3de7b8ead01..7b573464b467e 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
@@ -1061,7 +1061,6 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -1079,7 +1078,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -1201,8 +1200,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -1225,11 +1223,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB1_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -1387,27 +1386,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1469,13 +1468,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -1638,27 +1636,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB2_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -1720,13 +1718,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_one_as_scope
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -2403,7 +2400,6 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -2421,7 +2417,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -2543,8 +2539,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -2567,11 +2562,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_one_as_scope
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB3_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -2729,27 +2725,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -2811,13 +2807,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -2980,27 +2975,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB4_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -3062,13 +3057,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_agent_scope_
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -3745,7 +3739,6 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -3763,7 +3756,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -3885,8 +3878,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -3909,11 +3901,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB5_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4581,7 +4574,6 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -4599,7 +4591,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -4721,8 +4713,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -4745,11 +4736,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_agent_scope_
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB6_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -4907,27 +4899,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1032-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -4989,13 +4981,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -5158,27 +5149,27 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1032-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
-; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s0, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s1, 0x43300000
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[0:1]
+; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX1032-DPP-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, s3
; GFX1032-DPP-NEXT: v_mul_f32_e32 v2, 4.0, v0
@@ -5240,13 +5231,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_uni_value_default_scop
;
; GFX1132-DPP-LABEL: global_atomic_fsub_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -5922,7 +5912,6 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1032-DPP-NEXT: v_cndmask_b32_e64 v4, 0x80000000, v0, s0
@@ -5940,7 +5929,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1032-DPP-NEXT: v_permlanex16_b32 v4, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f32_e32 v3, v3, v4
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, v1, 0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, v3
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -6062,8 +6051,7 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v4, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v1, 1
; GFX1132-DPP-NEXT: v_cndmask_b32_e64 v2, 0x80000000, v0, s0
@@ -6086,11 +6074,12 @@ define amdgpu_kernel void @global_atomic_fsub_uni_address_div_value_default_scop
; GFX1132-DPP-NEXT: v_permlanex16_b32 v2, v1, 0, 0
; GFX1132-DPP-NEXT: v_add_f32_e32 v1, v1, v2
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, v4, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v4, exec_lo, 0
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1132-DPP-NEXT: v_cmpx_eq_u32_e32 0, v4
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB8_3
; GFX1132-DPP-NEXT: ; %bb.1:
@@ -7240,7 +7229,6 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -7268,8 +7256,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -7407,8 +7394,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -7444,9 +7430,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -7612,24 +7597,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7696,13 +7681,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -7870,24 +7854,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -7954,13 +7938,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_one_a
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_one_as_scope_unsafe_structfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -8686,7 +8669,6 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -8714,8 +8696,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -8853,8 +8834,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -8890,9 +8870,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -9058,24 +9037,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -9142,13 +9121,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -9316,24 +9294,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB13_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -9400,13 +9378,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_agent
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_agent_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -10133,7 +10110,6 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -10161,8 +10137,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -10300,8 +10275,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -10337,9 +10311,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -11060,7 +11033,6 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -11088,8 +11060,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -11227,8 +11198,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -11264,9 +11234,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
@@ -11431,24 +11400,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1032-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-NEXT: s_mov_b32 s14, -1
; GFX1032-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-NEXT: s_mov_b32 s2, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11515,13 +11484,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1132-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132: ; %bb.0:
-; GFX1132-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-NEXT: s_mov_b32 s2, 0
; GFX1132-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-NEXT: scratch_store_b32 off, v1, off
@@ -11689,24 +11657,24 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1032-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1032-DPP: ; %bb.0:
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, exec_lo
+; GFX1032-DPP-NEXT: s_mov_b32 s0, exec_lo
; GFX1032-DPP-NEXT: s_mov_b32 s12, SCRATCH_RSRC_DWORD0
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX1032-DPP-NEXT: s_mov_b32 s13, SCRATCH_RSRC_DWORD1
; GFX1032-DPP-NEXT: s_mov_b32 s14, -1
; GFX1032-DPP-NEXT: s_mov_b32 s15, 0x31c16000
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v1, v0, 0
; GFX1032-DPP-NEXT: s_add_u32 s12, s12, s11
+; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-DPP-NEXT: s_addc_u32 s13, s13, 0
; GFX1032-DPP-NEXT: s_mov_b32 s2, 0
-; GFX1032-DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1032-DPP-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX1032-DPP-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX1032-DPP-NEXT: s_cbranch_execz .LBB16_3
; GFX1032-DPP-NEXT: ; %bb.1:
-; GFX1032-DPP-NEXT: v_bcnt_u32_b32 v0, v0, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, 0x43300000
+; GFX1032-DPP-NEXT: s_bcnt1_i32_b32 s6, s0
+; GFX1032-DPP-NEXT: s_mov_b32 s7, 0x43300000
; GFX1032-DPP-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, s[6:7]
; GFX1032-DPP-NEXT: v_mov_b32_e32 v6, 0
-; GFX1032-DPP-NEXT: v_add_f64 v[0:1], 0xc3300000, v[0:1]
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1032-DPP-NEXT: v_mul_f64 v[4:5], 4.0, v[0:1]
@@ -11773,13 +11741,12 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_uni_value_defau
;
; GFX1132-DPP-LABEL: global_atomic_fsub_double_uni_address_uni_value_default_scope_strictfp:
; GFX1132-DPP: ; %bb.0:
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, 0x43300000
+; GFX1132-DPP-NEXT: s_bcnt1_i32_b32 s0, exec_lo
+; GFX1132-DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-DPP-NEXT: v_dual_mov_b32 v0, 0x43300000 :: v_dual_mov_b32 v1, s0
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
; GFX1132-DPP-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1132-DPP-NEXT: v_bcnt_u32_b32 v1, v2, 0
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v2, v2, 0
; GFX1132-DPP-NEXT: s_clause 0x1
; GFX1132-DPP-NEXT: scratch_store_b32 off, v0, off offset:4
; GFX1132-DPP-NEXT: scratch_store_b32 off, v1, off
@@ -12506,7 +12473,6 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1032-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, exec_lo
; GFX1032-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1032-DPP-NEXT: v_bfrev_b32_e32 v4, 1
; GFX1032-DPP-NEXT: v_mov_b32_e32 v3, 0
@@ -12534,8 +12500,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1032-DPP-NEXT: v_permlanex16_b32 v5, v3, 0, 0
; GFX1032-DPP-NEXT: v_add_f64 v[3:4], v[3:4], v[5:6]
; GFX1032-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, v2, 0
-; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-DPP-NEXT: v_mbcnt_lo_u32_b32 v7, exec_lo, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032-DPP-NEXT: v_mov_b32_e32 v1, v4
@@ -12673,8 +12638,7 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_mov_b32 s32, 0
; GFX1132-DPP-NEXT: ; implicit-def: $sgpr15
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v6, exec_lo
+; GFX1132-DPP-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GFX1132-DPP-NEXT: s_or_saveexec_b32 s0, -1
; GFX1132-DPP-NEXT: v_bfrev_b32_e32 v3, 1
; GFX1132-DPP-NEXT: v_mov_b32_e32 v2, 0
@@ -12710,9 +12674,8 @@ define amdgpu_kernel void @global_atomic_fsub_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132-DPP-NEXT: v_add_f64 v[2:3], v[2:3], v[4:5]
; GFX1132-DPP-NEXT: s_mov_b32 exec_lo, s0
-; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
-; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, v6, 0
-; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1132-DPP-NEXT: v_mbcnt_lo_u32_b32 v6, exec_lo, 0
+; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1132-DPP-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v1, v3
; GFX1132-DPP-NEXT: v_mov_b32_e32 v0, v2
; GFX1132-DPP-NEXT: s_mov_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index 50aa005f573e9..abdcec596d131 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -21,8 +21,7 @@ define amdgpu_cs i32 @constant_false() {
define amdgpu_cs i32 @constant_true() {
; CHECK-LABEL: constant_true:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_mov_b32_e32 v0, exec_lo
-; CHECK-NEXT: v_readfirstlane_b32 s0, v0
+; CHECK-NEXT: s_mov_b32 s0, exec_lo
; CHECK-NEXT: ; return to shader part epilog
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 1)
ret i32 %ballot
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
index 535454436ace9..fd21f5cc6a357 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.quadmask.ll
@@ -193,7 +193,7 @@ define amdgpu_kernel void @test_scc_quadmask_32(i32 %val0, i32 %val1, ptr addrsp
; GFX11-SDAG-LABEL: test_scc_quadmask_32:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_and_b32 s0, s0, 1
; GFX11-SDAG-NEXT: s_quadmask_b32 s1, s1
@@ -202,7 +202,7 @@ define amdgpu_kernel void @test_scc_quadmask_32(i32 %val0, i32 %val1, ptr addrsp
; GFX11-SDAG-NEXT: s_cselect_b32 s0, -1, 0
; GFX11-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
; GFX11-SDAG-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v4, s0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, s0
; GFX11-SDAG-NEXT: global_store_b32 v2, v3, s[2:3]
; GFX11-SDAG-NEXT: global_store_b32 v[0:1], v4, off
; GFX11-SDAG-NEXT: s_endpgm
@@ -240,6 +240,7 @@ define amdgpu_kernel void @test_scc_quadmask_64(i32 %val0, i64 %val1, ptr addrsp
; GFX11-SDAG-NEXT: s_clause 0x1
; GFX11-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_and_b32 s4, s6, 1
; GFX11-SDAG-NEXT: s_quadmask_b64 s[0:1], s[0:1]
@@ -250,7 +251,6 @@ define amdgpu_kernel void @test_scc_quadmask_64(i32 %val0, i64 %val1, ptr addrsp
; GFX11-SDAG-NEXT: s_and_b32 s0, s4, exec_lo
; GFX11-SDAG-NEXT: s_cselect_b32 s0, 1, 0
; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v5, s0
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v3, 0
; GFX11-SDAG-NEXT: global_store_b64 v4, v[0:1], s[2:3]
; GFX11-SDAG-NEXT: global_store_b32 v[2:3], v5, off
; GFX11-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index af99e159057e1..df20d43ee49fd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -511,18 +511,16 @@ define amdgpu_kernel void @test_readfirstlane_m0(ptr addrspace(1) %out) #0 {
; CHECK-SDAG-LABEL: test_readfirstlane_m0:
; CHECK-SDAG: ; %bb.0:
; CHECK-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-SDAG-NEXT: s_add_i32 s12, s12, s17
+; CHECK-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CHECK-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CHECK-SDAG-NEXT: ;;#ASMSTART
; CHECK-SDAG-NEXT: s_mov_b32 m0, -1
; CHECK-SDAG-NEXT: ;;#ASMEND
-; CHECK-SDAG-NEXT: v_mov_b32_e32 v0, m0
-; CHECK-SDAG-NEXT: s_add_i32 s12, s12, s17
-; CHECK-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; CHECK-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; CHECK-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CHECK-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CHECK-SDAG-NEXT: v_mov_b32_e32 v2, m0
; CHECK-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; CHECK-SDAG-NEXT: v_mov_b32_e32 v2, s2
; CHECK-SDAG-NEXT: flat_store_dword v[0:1], v2
; CHECK-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll b/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
index 26a8911710535..618a4c294cae5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sponentry.ll
@@ -331,17 +331,27 @@ define amdgpu_gfx ptr addrspace(5) @sponentry_gfx_dyn_alloc(i32 %val) #0 {
; DAGISEL-NEXT: v_and_b32_e32 v3, -16, v3
; DAGISEL-NEXT: s_or_saveexec_b32 s0, -1
; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; DAGISEL-NEXT: s_cmp_lg_u32 s1, 0
-; DAGISEL-NEXT: s_cbranch_scc1 .LBB9_1
-; DAGISEL-NEXT: ; %bb.2:
-; DAGISEL-NEXT: v_mov_b32_e32 v1, s32
+; DAGISEL-NEXT: v_cndmask_b32_e64 v1, 0, v3, s0
+; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; DAGISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:1 row_mask:0xf bank_mask:0xf
+; DAGISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:2 row_mask:0xf bank_mask:0xf
+; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; DAGISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:4 row_mask:0xf bank_mask:0xf
+; DAGISEL-NEXT: v_max_u32_dpp v1, v1, v1 row_shr:8 row_mask:0xf bank_mask:0xf
+; DAGISEL-NEXT: ds_swizzle_b32 v2, v1 offset:swizzle(BROADCAST,32,15)
+; DAGISEL-NEXT: s_wait_dscnt 0x0
+; DAGISEL-NEXT: v_max_u32_e32 v1, v1, v2
+; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; DAGISEL-NEXT: v_readlane_b32 s1, v1, 31
+; DAGISEL-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL-NEXT: s_mov_b32 s0, s32
+; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; DAGISEL-NEXT: v_lshl_add_u32 v3, s1, 5, s0
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 v1, v0, off scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 off, v0, s0 scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
; DAGISEL-NEXT: v_mov_b32_e32 v0, s33
-; DAGISEL-NEXT: v_lshl_add_u32 v2, s0, 5, v1
-; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; DAGISEL-NEXT: v_readfirstlane_b32 s32, v2
+; DAGISEL-NEXT: v_readfirstlane_b32 s32, v3
; DAGISEL-NEXT: s_mov_b32 s32, s33
; DAGISEL-NEXT: s_xor_saveexec_b32 s0, -1
; DAGISEL-NEXT: s_clause 0x1 ; 8-byte Folded Reload
@@ -540,23 +550,18 @@ define amdgpu_cs_chain void @sponentry_cs_chain_dyn_alloc(i32 %val) #0 {
; DAGISEL-NEXT: s_wait_dscnt 0x0
; DAGISEL-NEXT: v_max_u32_e32 v0, v0, v1
; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; DAGISEL-NEXT: v_readlane_b32 s3, v0, s2
-; DAGISEL-NEXT: s_bitset0_b32 s1, s2
-; DAGISEL-NEXT: s_max_u32 s0, s0, s3
+; DAGISEL-NEXT: v_readlane_b32 s1, v0, 31
+; DAGISEL-NEXT: s_mov_b32 exec_lo, s0
+; DAGISEL-NEXT: s_mov_b32 s0, s32
+; DAGISEL-NEXT: v_mov_b32_e32 v3, s33
; DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; DAGISEL-NEXT: s_cmp_lg_u32 s1, 0
-; DAGISEL-NEXT: s_cbranch_scc1 .LBB12_1
-; DAGISEL-NEXT: ; %bb.2:
-; DAGISEL-NEXT: v_mov_b32_e32 v0, s32
-; DAGISEL-NEXT: v_mov_b32_e32 v2, s33
+; DAGISEL-NEXT: v_lshl_add_u32 v2, s1, 5, s0
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 v0, v8, off scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 off, v8, s0 scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: scratch_store_b32 v0, v2, off scope:SCOPE_SYS
+; DAGISEL-NEXT: scratch_store_b32 off, v3, s0 scope:SCOPE_SYS
; DAGISEL-NEXT: s_wait_storecnt 0x0
-; DAGISEL-NEXT: v_lshl_add_u32 v1, s0, 5, v0
-; DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; DAGISEL-NEXT: v_readfirstlane_b32 s32, v1
+; DAGISEL-NEXT: v_readfirstlane_b32 s32, v2
; DAGISEL-NEXT: s_alloc_vgpr 0
; DAGISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
index 2238914b24e7d..2bff1b97f65e6 100644
--- a/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched_mfma_rewrite_copies.mir
@@ -295,6 +295,7 @@ body: |
KILL %1, %2, %3, %4, %5, %6, %7, %8, %10, %11, %12, %13, %62, %72, %85, %86, %87, %88, %94
S_NOP 0, implicit %50, implicit %51
S_ENDPGM 0
+
...
---
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend.ll b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
index 5b3fd7e67528f..5c85b180f8c8f 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend.ll
@@ -330,15 +330,15 @@ define amdgpu_kernel void @s_cmp_zext_i1_to_i16(ptr addrspace(1) %out, [8 x i32]
; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x70 nv
; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
; GFX1250-NEXT: s_and_b32 s3, s3, 0xffff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-NEXT: s_cmp_eq_u32 s3, s2
; GFX1250-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 1, s2
-; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
+; GFX1250-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
%tmp0 = icmp eq i16 %a, %b
%tmp1 = zext i1 %tmp0 to i16
diff --git a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
index 7aef9c35cd745..21dca29e74b1f 100644
--- a/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/zero_extend_i1_to_i32_i64.ll
@@ -810,7 +810,7 @@ define amdgpu_ps void @ps_uniform_i16_eq_i32(
; GFX1250-LABEL: ps_uniform_i16_eq_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -837,7 +837,7 @@ define amdgpu_ps void @ps_uniform_i16_eq_i64(
; GFX1250-LABEL: ps_uniform_i16_eq_i64:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_eq_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: s_mov_b32 s0, 0
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: v_mov_b32_e32 v3, s0
@@ -866,7 +866,7 @@ define amdgpu_ps void @ps_uniform_i16_ne_i32(
; GFX1250-LABEL: ps_uniform_i16_ne_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_ne_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_ne_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -892,7 +892,7 @@ define amdgpu_ps void @ps_uniform_i16_ugt_i32(
; GFX1250-LABEL: ps_uniform_i16_ugt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_gt_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_gt_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -918,7 +918,7 @@ define amdgpu_ps void @ps_uniform_i16_uge_i32(
; GFX1250-LABEL: ps_uniform_i16_uge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_ge_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_ge_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -944,7 +944,7 @@ define amdgpu_ps void @ps_uniform_i16_ult_i32(
; GFX1250-LABEL: ps_uniform_i16_ult_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_lt_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_lt_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -970,7 +970,7 @@ define amdgpu_ps void @ps_uniform_i16_ule_i32(
; GFX1250-LABEL: ps_uniform_i16_ule_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_le_u16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_le_u16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -996,7 +996,7 @@ define amdgpu_ps void @ps_uniform_i16_sgt_i32(
; GFX1250-LABEL: ps_uniform_i16_sgt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_gt_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_gt_i16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -1022,7 +1022,7 @@ define amdgpu_ps void @ps_uniform_i16_sge_i32(
; GFX1250-LABEL: ps_uniform_i16_sge_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_ge_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_ge_i16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -1048,7 +1048,7 @@ define amdgpu_ps void @ps_uniform_i16_slt_i32(
; GFX1250-LABEL: ps_uniform_i16_slt_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_lt_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_lt_i16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
@@ -1074,7 +1074,7 @@ define amdgpu_ps void @ps_uniform_i16_sle_i32(
; GFX1250-LABEL: ps_uniform_i16_sle_i32:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: v_cmp_le_i16_e32 vcc_lo, v2, v3
+; GFX1250-NEXT: v_cmp_le_i16_e32 vcc_lo, v2.l, v3.l
; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
More information about the llvm-commits
mailing list