[llvm] [AMDGPU][SIPreEmitPeephole] Fold consecutive `s_cselect_b32`s (PR #183357)
Zach Goldthorpe via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 09:33:11 PDT 2026
https://github.com/zGoldthorpe updated https://github.com/llvm/llvm-project/pull/183357
>From 8b51dbe5965554f8541196a8b4deec4eaeb45388 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Wed, 25 Feb 2026 12:09:45 -0600
Subject: [PATCH] Peephole that folds consecutive `s_cselect_b32`s
---
llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp | 89 +++++++++
llvm/test/CodeGen/AMDGPU/addrspacecast.ll | 32 ++--
.../AMDGPU/amdgpu-codegenprepare-idiv.ll | 6 +-
.../atomic_optimizations_local_pointer.ll | 52 ++----
...tor-flatscratchinit-undefined-behavior2.ll | 18 +-
.../AMDGPU/av-split-dead-valno-crash.ll | 3 +-
.../test/CodeGen/AMDGPU/carryout-selection.ll | 6 +-
llvm/test/CodeGen/AMDGPU/dagcombine-select.ll | 24 +--
.../CodeGen/AMDGPU/extract_vector_dynelt.ll | 3 +-
.../CodeGen/AMDGPU/extract_vector_elt-f64.ll | 14 +-
.../CodeGen/AMDGPU/extract_vector_elt-i64.ll | 17 +-
llvm/test/CodeGen/AMDGPU/fceil64.ll | 72 +++-----
.../CodeGen/AMDGPU/fneg-modifier-casting.ll | 6 +-
.../AMDGPU/fold-consecutive-cselect32.mir | 120 ++++++++++++
llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll | 3 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 6 +-
llvm/test/CodeGen/AMDGPU/ftrunc.f64.ll | 3 +-
llvm/test/CodeGen/AMDGPU/idiv-licm.ll | 11 +-
.../AMDGPU/indirect-call-known-callees.ll | 6 +-
.../AMDGPU/integer-select-src-modifiers.ll | 45 ++---
llvm/test/CodeGen/AMDGPU/llvm.mulo.ll | 6 +-
llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll | 39 ++--
llvm/test/CodeGen/AMDGPU/min.ll | 172 +++++++++---------
llvm/test/CodeGen/AMDGPU/sdiv.ll | 6 +-
llvm/test/CodeGen/AMDGPU/select-vectors.ll | 13 +-
llvm/test/CodeGen/AMDGPU/select64.ll | 14 +-
llvm/test/CodeGen/AMDGPU/selectcc.ll | 2 +-
llvm/test/CodeGen/AMDGPU/shift-i128.ll | 15 +-
llvm/test/CodeGen/AMDGPU/udiv.ll | 15 +-
.../umin-sub-to-usubo-select-combine.ll | 6 +-
30 files changed, 457 insertions(+), 367 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/fold-consecutive-cselect32.mir
diff --git a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
index 411c14fef63b6..dc4f2215ff0bf 100644
--- a/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreEmitPeephole.cpp
@@ -81,6 +81,17 @@ class SIPreEmitPeephole {
void addOperandAndMods(MachineInstrBuilder &NewMI, unsigned SrcMods,
bool IsHiBits, const MachineOperand &SrcMO);
+ // Scan basic block for consecutive s_cselect_b32 instructions whose operands
+ // may be concatenated to form a single s_cselect_b64.
+ // Corresponding operands may be concatenated if they are adjacent SGPRs, or
+ // they are both constant i32s.
+ // E.g.
+ // s_cselect_b32 s0, s2, s4
+ // s_cselect_b32 s1, s3, s5
+ // ==>
+ // s_cselect_b64 s[0:1], s[2:3], s[4:5]
+ bool foldConsecutiveSelects(MachineBasicBlock &MBB) const;
+
public:
bool run(MachineFunction &MF, MachineLoopInfo *MLI);
};
@@ -642,6 +653,82 @@ void SIPreEmitPeephole::addOperandAndMods(MachineInstrBuilder &NewMI,
NewMI.add(UnpackedSrcMO);
}
+bool SIPreEmitPeephole::foldConsecutiveSelects(MachineBasicBlock &MBB) const {
+ bool Changed = false;
+
+ const auto GetCombinedRegister = [&](const MachineOperand &Lo,
+ const MachineOperand &Hi) -> MCRegister {
+ if (!Lo.isReg() || !Hi.isReg())
+ return MCRegister::NoRegister;
+
+ const MCRegister LoReg = Lo.getReg();
+ const MCRegister HiReg = Hi.getReg();
+
+ MCRegister SuperReg =
+ TRI->getMatchingSuperReg(LoReg, AMDGPU::sub0, &AMDGPU::SReg_64RegClass);
+ if (!SuperReg || TRI->getSubReg(SuperReg, AMDGPU::sub1) != HiReg)
+ return MCRegister::NoRegister;
+
+ return SuperReg;
+ };
+
+ MachineBasicBlock::iterator I = MBB.begin();
+ MachineBasicBlock::iterator End = MBB.end();
+ while (I != End) {
+ MachineInstr *First = &*I++;
+ if (I == End)
+ break;
+
+ if (First->getOpcode() != AMDGPU::S_CSELECT_B32)
+ continue;
+
+ MachineInstr *Second = &*I;
+ if (Second->getOpcode() != AMDGPU::S_CSELECT_B32)
+ continue;
+
+ Register Dst =
+ GetCombinedRegister(First->getOperand(0), Second->getOperand(0));
+ if (!Dst) {
+ // Check whether destination registers can be combined in the other order.
+ std::swap(First, Second);
+ Dst = GetCombinedRegister(First->getOperand(0), Second->getOperand(0));
+ if (!Dst)
+ continue;
+ }
+
+ // First operand is always a register.
+ Register Src1 =
+ GetCombinedRegister(First->getOperand(1), Second->getOperand(1));
+ if (!Src1)
+ continue;
+
+ // Second operand may be a register or an immediatea constant.
+ std::optional<MachineOperand> Src2 = std::nullopt;
+ MachineOperand &Src2Lo = First->getOperand(2);
+ MachineOperand &Src2Hi = Second->getOperand(2);
+ if (MCRegister Src2Reg = GetCombinedRegister(Src2Lo, Src2Hi)) {
+ Src2 = MachineOperand::CreateReg(Src2Reg, /* isDef */ false);
+ } else if (Src2Lo.isImm() && Src2Hi.isImm()) {
+ Src2 =
+ MachineOperand::CreateImm(Make_64(Src2Hi.getImm(), Src2Lo.getImm()));
+ } else {
+ continue;
+ }
+
+ BuildMI(MBB, *First, First->getDebugLoc(), TII->get(AMDGPU::S_CSELECT_B64),
+ Dst)
+ .addReg(Src1)
+ .add(*Src2);
+
+ ++I;
+ First->eraseFromParent();
+ Second->eraseFromParent();
+ Changed = true;
+ }
+
+ return Changed;
+}
+
void SIPreEmitPeephole::collectUnpackingCandidates(
MachineInstr &BeginMI, SetVector<MachineInstr *> &InstrsToUnpack,
uint16_t NumMFMACycles) {
@@ -794,6 +881,8 @@ bool SIPreEmitPeephole::run(MachineFunction &MF, MachineLoopInfo *LoopInfo) {
MF.RenumberBlocks();
for (MachineBasicBlock &MBB : MF) {
+ Changed |= foldConsecutiveSelects(MBB);
+
MachineBasicBlock::iterator TermI = MBB.getFirstTerminator();
// Check first terminator for branches to optimize
if (TermI != MBB.end()) {
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 4652d0ae60fd0..70c93ca9a8eb0 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -5,20 +5,18 @@ target triple = "amdgcn-amd-amdhsa"
; HSA-LABEL: {{^}}use_group_to_flat_addrspacecast:
-; CI-DAG: s_load_dword [[PTR:s[0-9]+]], s[6:7], 0x0{{$}}
-; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x10{{$}}
-; CI-DAG: s_cmp_lg_u32 [[PTR]], -1
-; CI-DAG: s_cselect_b32 s[[SHI:[0-9]+]], [[APERTURE]], 0
-; CI-DAG: s_cselect_b32 s[[SLO:[0-9]+]], [[PTR]], 0
+; CI-DAG: s_load_dword s[[PTR:[0-9]+]], s[6:7], 0x0{{$}}
+; CI-DAG: s_load_dword s[[APERTURE:[0-9]+]], s[4:5], 0x10{{$}}
+; CI-DAG: s_cmp_lg_u32 s[[PTR]], -1
+; CI-DAG: s_cselect_b64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[[[PTR]]:[[APERTURE]]], 0
; GFX9-DAG: s_mov_b64 s[{{[0-9]+}}:[[HIBASE:[0-9]+]]], src_shared_base
; HSA-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7
-; GFX9-DAG: s_load_dword [[PTR:s[0-9]+]], s[4:5], 0x0{{$}}
+; GFX9-DAG: s_load_dword s[[PTR:[0-9]+]], s[4:5], 0x0{{$}}
-; GFX9: s_cmp_lg_u32 [[PTR]], -1
-; GFX9-DAG: s_cselect_b32 s[[SHI:[0-9]+]], s[[HIBASE]], 0
-; GFX9-DAG: s_cselect_b32 s[[SLO:[0-9]+]], [[PTR]], 0
+; GFX9: s_cmp_lg_u32 s[[PTR]], -1
+; GFX9-DAG: s_cselect_b64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[[[PTR]]:[[HIBASE]]], 0
; HSA-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
; HSA-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
@@ -66,21 +64,19 @@ define void @use_group_to_flat_addrspacecast_func(ptr addrspace(3) %ptr) #0 {
; HSA-LABEL: {{^}}use_private_to_flat_addrspacecast:
-; CI-DAG: s_load_dword [[PTR:s[0-9]+]], s[6:7], 0x0{{$}}
-; CI-DAG: s_load_dword [[APERTURE:s[0-9]+]], s[4:5], 0x11{{$}}
+; CI-DAG: s_load_dword s[[PTR:[0-9]+]], s[6:7], 0x0{{$}}
+; CI-DAG: s_load_dword s[[APERTURE:[0-9]+]], s[4:5], 0x11{{$}}
; CI-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7
-; CI-DAG: s_cmp_lg_u32 [[PTR]], -1
-; CI-DAG: s_cselect_b32 s[[SHI:[0-9]+]], [[APERTURE]], 0
-; CI-DAG: s_cselect_b32 s[[SLO:[0-9]+]], [[PTR]], 0
+; CI-DAG: s_cmp_lg_u32 s[[PTR]], -1
+; CI-DAG: s_cselect_b64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[[[PTR]]:[[APERTURE]]], 0
-; GFX9-DAG: s_load_dword [[PTR:s[0-9]+]], s[4:5], 0x0{{$}}
+; GFX9-DAG: s_load_dword s[[PTR:[0-9]+]], s[4:5], 0x0{{$}}
; GFX9-DAG: s_mov_b64 s[{{[0-9]+}}:[[HIBASE:[0-9]+]]], src_private_base
; GFX9-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7
-; GFX9: s_cmp_lg_u32 [[PTR]], -1
-; GFX9: s_cselect_b32 s[[SHI:[0-9]+]], s[[HIBASE]], 0
-; GFX9: s_cselect_b32 s[[SLO:[0-9]+]], [[PTR]], 0
+; GFX9: s_cmp_lg_u32 s[[PTR]], -1
+; GFX9: s_cselect_b64 s[[[SLO:[0-9]+]]:[[SHI:[0-9]+]]], s[[[PTR]]:[[HIBASE]]], 0
; HSA-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], s[[SHI]]
; HSA-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], s[[SLO]]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index df77e7de43bf6..38f64427ac300 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -1336,8 +1336,7 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: s_cmp_ge_u32 s5, s14
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_cselect_b32 s4, s6, s4
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
; GFX9-NEXT: s_add_i32 s6, s4, 1
; GFX9-NEXT: s_cmp_ge_u32 s5, s14
; GFX9-NEXT: s_cselect_b32 s4, s6, s4
@@ -1984,8 +1983,7 @@ define amdgpu_kernel void @sdiv_v4i32(ptr addrspace(1) %out, <4 x i32> %x, <4 x
; GFX9-NEXT: s_add_i32 s7, s3, 1
; GFX9-NEXT: s_sub_i32 s6, s2, s0
; GFX9-NEXT: s_cmp_ge_u32 s2, s0
-; GFX9-NEXT: s_cselect_b32 s3, s7, s3
-; GFX9-NEXT: s_cselect_b32 s2, s6, s2
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[6:7], s[2:3]
; GFX9-NEXT: s_add_i32 s6, s3, 1
; GFX9-NEXT: s_cmp_ge_u32 s2, s0
; GFX9-NEXT: s_cselect_b32 s0, s6, s3
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index b71577385606a..1bc6b8574dfa5 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -11292,8 +11292,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1064_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
@@ -11343,8 +11342,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
@@ -11396,8 +11394,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1164_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -11451,8 +11448,7 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
@@ -13116,8 +13112,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1064_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
@@ -13167,8 +13162,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
@@ -13220,8 +13214,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1164_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -13275,8 +13268,7 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
@@ -14932,8 +14924,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1064_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
@@ -14982,8 +14973,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
@@ -15034,8 +15024,7 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1164_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -15086,11 +15075,10 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
@@ -16742,8 +16730,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1064_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
@@ -16792,8 +16779,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
@@ -16844,8 +16830,7 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1164_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -16896,11 +16881,10 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_cselect_b64 s[0:1], s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
diff --git a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll
index 554f40b5bfdfa..49d4d0356678b 100644
--- a/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll
+++ b/llvm/test/CodeGen/AMDGPU/attributor-flatscratchinit-undefined-behavior2.ll
@@ -109,8 +109,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_cmp_lg_u32 s0, -1
-; GFX8-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8-NEXT: s_cselect_b32 s0, s0, 0
+; GFX8-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -124,8 +123,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX8-ARCH-FLAT-NEXT: v_mov_b32_e32 v2, 0
; GFX8-ARCH-FLAT-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-ARCH-FLAT-NEXT: s_cmp_lg_u32 s0, -1
-; GFX8-ARCH-FLAT-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8-ARCH-FLAT-NEXT: s_cselect_b32 s0, s0, 0
+; GFX8-ARCH-FLAT-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX8-ARCH-FLAT-NEXT: v_mov_b32_e32 v0, s0
; GFX8-ARCH-FLAT-NEXT: v_mov_b32_e32 v1, s1
; GFX8-ARCH-FLAT-NEXT: flat_store_dword v[0:1], v2
@@ -139,8 +137,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lg_u32 s0, -1
-; GFX9-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: flat_store_dword v[0:1], v2
@@ -154,8 +151,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX9-ARCH-FLAT-NEXT: v_mov_b32_e32 v2, 0
; GFX9-ARCH-FLAT-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ARCH-FLAT-NEXT: s_cmp_lg_u32 s0, -1
-; GFX9-ARCH-FLAT-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9-ARCH-FLAT-NEXT: s_cselect_b32 s0, s0, 0
+; GFX9-ARCH-FLAT-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX9-ARCH-FLAT-NEXT: v_mov_b32_e32 v0, s0
; GFX9-ARCH-FLAT-NEXT: v_mov_b32_e32 v1, s1
; GFX9-ARCH-FLAT-NEXT: flat_store_dword v[0:1], v2
@@ -169,8 +165,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX942-ARCH-FLAT-NEXT: v_mov_b32_e32 v2, 0
; GFX942-ARCH-FLAT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-ARCH-FLAT-NEXT: s_cmp_lg_u32 s0, -1
-; GFX942-ARCH-FLAT-NEXT: s_cselect_b32 s1, s1, 0
-; GFX942-ARCH-FLAT-NEXT: s_cselect_b32 s0, s0, 0
+; GFX942-ARCH-FLAT-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX942-ARCH-FLAT-NEXT: v_mov_b32_e32 v0, s0
; GFX942-ARCH-FLAT-NEXT: v_mov_b32_e32 v1, s1
; GFX942-ARCH-FLAT-NEXT: flat_store_dword v[0:1], v2 sc0 sc1
@@ -184,8 +179,7 @@ define amdgpu_kernel void @with_private_to_flat_addrspacecast_cc_kernel(ptr addr
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u32 s0, -1
-; GFX10-NEXT: s_cselect_b32 s0, s0, 0
-; GFX10-NEXT: s_cselect_b32 s1, s1, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: flat_store_dword v[0:1], v2
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 42f76c4a10d2a..cd6b1f082ca5d 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -136,8 +136,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v27, v26
; CHECK-NEXT: s_and_b64 s[8:9], exec, s[16:17]
; CHECK-NEXT: global_store_dwordx2 v20, v[26:27], s[12:13]
-; CHECK-NEXT: s_cselect_b32 s23, s23, 0
-; CHECK-NEXT: s_cselect_b32 s22, s22, 0
+; CHECK-NEXT: s_cselect_b64 s[22:23], s[22:23], 0
; CHECK-NEXT: s_mov_b64 s[8:9], -1
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 87b761268f1f1..baa943d318490 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -3147,8 +3147,7 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1250-NEXT: s_cselect_b32 s16, s15, s14
; GFX1250-NEXT: s_add_nc_u64 s[14:15], s[10:11], 2
; GFX1250-NEXT: s_cmp_lg_u32 s16, 0
-; GFX1250-NEXT: s_cselect_b32 s12, s14, s12
-; GFX1250-NEXT: s_cselect_b32 s13, s15, s13
+; GFX1250-NEXT: s_cselect_b64 s[12:13], s[14:15], s[12:13]
; GFX1250-NEXT: s_cmp_lg_u32 s8, 0
; GFX1250-NEXT: s_sub_co_ci_u32 s3, s3, s9
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -3160,8 +3159,7 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GFX1250-NEXT: s_cselect_b32 s3, s4, s8
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_cmp_lg_u32 s3, 0
-; GFX1250-NEXT: s_cselect_b32 s9, s13, s11
-; GFX1250-NEXT: s_cselect_b32 s8, s12, s10
+; GFX1250-NEXT: s_cselect_b64 s[8:9], s[12:13], s[10:11]
; GFX1250-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s5
; GFX1250-NEXT: s_cbranch_vccnz .LBB16_3
; GFX1250-NEXT: .LBB16_2:
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
index 65a1443f83c6e..ab28af27d253f 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
@@ -65,10 +65,8 @@ define amdgpu_kernel void @select_and_v4(ptr addrspace(1) %p, i32 %x, <4 x i32>
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_gt_i32 s8, 10
-; GFX9-NEXT: s_cselect_b32 s3, s3, 0
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], 0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -84,10 +82,8 @@ define amdgpu_kernel void @select_and_v4(ptr addrspace(1) %p, i32 %x, <4 x i32>
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_gt_i32 s8, 10
-; GFX942-NEXT: s_cselect_b32 s3, s3, 0
-; GFX942-NEXT: s_cselect_b32 s2, s2, 0
-; GFX942-NEXT: s_cselect_b32 s1, s1, 0
-; GFX942-NEXT: s_cselect_b32 s0, s0, 0
+; GFX942-NEXT: s_cselect_b64 s[2:3], s[2:3], 0
+; GFX942-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: v_mov_b32_e32 v3, s1
; GFX942-NEXT: v_mov_b32_e32 v4, s2
@@ -164,10 +160,8 @@ define amdgpu_kernel void @select_or_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_cmp_lt_i32 s8, 11
-; GFX9-NEXT: s_cselect_b32 s3, s3, -1
-; GFX9-NEXT: s_cselect_b32 s2, s2, -1
-; GFX9-NEXT: s_cselect_b32 s1, s1, -1
-; GFX9-NEXT: s_cselect_b32 s0, s0, -1
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], -1
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[0:1], -1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -183,10 +177,8 @@ define amdgpu_kernel void @select_or_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_lt_i32 s8, 11
-; GFX942-NEXT: s_cselect_b32 s3, s3, -1
-; GFX942-NEXT: s_cselect_b32 s2, s2, -1
-; GFX942-NEXT: s_cselect_b32 s1, s1, -1
-; GFX942-NEXT: s_cselect_b32 s0, s0, -1
+; GFX942-NEXT: s_cselect_b64 s[2:3], s[2:3], -1
+; GFX942-NEXT: s_cselect_b64 s[0:1], s[0:1], -1
; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: v_mov_b32_e32 v3, s1
; GFX942-NEXT: v_mov_b32_e32 v4, s2
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index 975695b03c114..a0bb5563cd3c1 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -202,8 +202,7 @@ define amdgpu_kernel void @double5_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: s_mov_b32 s3, 0xc28f5c29
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s6, 1
-; GCN-NEXT: s_cselect_b32 s2, s2, 0x3f847ae1
-; GCN-NEXT: s_cselect_b32 s3, s3, 0x47ae147b
+; GCN-NEXT: s_cselect_b64 s[2:3], s[2:3], 0x47ae147b3f847ae1
; GCN-NEXT: s_cmp_eq_u32 s6, 2
; GCN-NEXT: s_cselect_b32 s8, 0xe147ae14, s3
; GCN-NEXT: s_cselect_b32 s7, 0x4000147a, s2
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
index 9201f60a32535..b3ed1be24a4aa 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f64.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-- < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-- < %s | FileCheck -enable-var-scope -check-prefixes=GCN,BASE %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefix=GCN,FLAT %s
; GCN-LABEL: {{^}}extract_vector_elt_v3f64_2:
; GCN: buffer_load_dwordx4
@@ -15,11 +15,13 @@ define amdgpu_kernel void @extract_vector_elt_v3f64_2(ptr addrspace(1) %out, ptr
; GCN-LABEL: {{^}}dyn_extract_vector_elt_v3f64:
; GCN-NOT: buffer_load
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 1
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; BASE-DAG: s_cselect_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 2
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; BASE-DAG: s_cselect_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN: store_dwordx2 v[{{[0-9:]+}}]
define amdgpu_kernel void @dyn_extract_vector_elt_v3f64(ptr addrspace(1) %out, <3 x double> %foo, i32 %elt) #0 {
%dynelt = extractelement <3 x double> %foo, i32 %elt
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
index eb0ed5e6ee4b9..ccea9cf80ebd0 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn-- < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-- < %s | FileCheck -check-prefix=GCN,BASE %s
+; RUN: llc -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefixes=GCN,FLAT %s
; How the replacement of i64 stores with v2i32 stores resulted in
; breaking other users of the bitcast if they already existed
@@ -32,8 +32,7 @@ define amdgpu_kernel void @extract_vector_elt_v2i64(ptr addrspace(1) %out, <2 x
; GCN-LABEL: {{^}}dyn_extract_vector_elt_v2i64:
; GCN-NOT: buffer_load
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 1
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; GCN-DAG: s_cselect_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
; GCN: store_dwordx2 v[{{[0-9:]+}}]
define amdgpu_kernel void @dyn_extract_vector_elt_v2i64(ptr addrspace(1) %out, <2 x i64> %foo, i32 %elt) #0 {
%dynelt = extractelement <2 x i64> %foo, i32 %elt
@@ -60,11 +59,13 @@ define amdgpu_kernel void @dyn_extract_vector_elt_v2i64_2(ptr addrspace(1) %out,
; GCN-LABEL: {{^}}dyn_extract_vector_elt_v3i64:
; GCN-NOT: buffer_load
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 1
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; BASE-DAG: s_cselect_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 2
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; BASE-DAG: s_cselect_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+; FLAT-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN: store_dwordx2 v[{{[0-9:]+}}]
define amdgpu_kernel void @dyn_extract_vector_elt_v3i64(ptr addrspace(1) %out, <3 x i64> %foo, i32 %elt) #0 {
%dynelt = extractelement <3 x i64> %foo, i32 %elt
diff --git a/llvm/test/CodeGen/AMDGPU/fceil64.ll b/llvm/test/CodeGen/AMDGPU/fceil64.ll
index bd1f98a39c252..ff622a6ba8504 100644
--- a/llvm/test/CodeGen/AMDGPU/fceil64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fceil64.ll
@@ -28,8 +28,7 @@ define amdgpu_kernel void @fceil_f64(ptr addrspace(1) %out, double %x) {
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s9, s5
; SI-NEXT: s_cmp_gt_i32 s8, 51
-; SI-NEXT: s_cselect_b32 s9, s3, s5
-; SI-NEXT: s_cselect_b32 s8, s2, s4
+; SI-NEXT: s_cselect_b64 s[8:9], s[2:3], s[4:5]
; SI-NEXT: v_cmp_gt_f64_e64 s[4:5], s[2:3], 0
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: v_mov_b32_e32 v1, s9
@@ -69,8 +68,7 @@ define amdgpu_kernel void @fceil_v2f64(ptr addrspace(1) %out, <2 x double> %x) {
; SI-NEXT: s_cselect_b32 s10, 0, s10
; SI-NEXT: s_cselect_b32 s11, s12, s11
; SI-NEXT: s_cmp_gt_i32 s13, 51
-; SI-NEXT: s_cselect_b32 s11, s7, s11
-; SI-NEXT: s_cselect_b32 s10, s6, s10
+; SI-NEXT: s_cselect_b64 s[10:11], s[6:7], s[10:11]
; SI-NEXT: v_cmp_gt_f64_e64 s[12:13], s[6:7], 0
; SI-NEXT: v_mov_b32_e32 v1, s10
; SI-NEXT: v_mov_b32_e32 v2, s11
@@ -87,8 +85,7 @@ define amdgpu_kernel void @fceil_v2f64(ptr addrspace(1) %out, <2 x double> %x) {
; SI-NEXT: s_cselect_b32 s6, 0, s6
; SI-NEXT: s_cselect_b32 s7, s13, s7
; SI-NEXT: s_cmp_gt_i32 s14, 51
-; SI-NEXT: s_cselect_b32 s7, s5, s7
-; SI-NEXT: s_cselect_b32 s6, s4, s6
+; SI-NEXT: s_cselect_b64 s[6:7], s[4:5], s[6:7]
; SI-NEXT: v_cmp_gt_f64_e64 s[8:9], s[4:5], 0
; SI-NEXT: v_mov_b32_e32 v1, s12
; SI-NEXT: v_mov_b32_e32 v2, s6
@@ -139,8 +136,7 @@ define amdgpu_kernel void @fceil_v4f64(ptr addrspace(1) %out, <4 x double> %x) {
; SI-NEXT: s_cselect_b32 s18, 0, s18
; SI-NEXT: s_cselect_b32 s19, s20, s19
; SI-NEXT: s_cmp_gt_i32 s21, 51
-; SI-NEXT: s_cselect_b32 s19, s3, s19
-; SI-NEXT: s_cselect_b32 s18, s2, s18
+; SI-NEXT: s_cselect_b64 s[18:19], s[2:3], s[18:19]
; SI-NEXT: v_cmp_gt_f64_e64 s[20:21], s[6:7], 0
; SI-NEXT: v_mov_b32_e32 v1, s18
; SI-NEXT: v_mov_b32_e32 v2, s19
@@ -157,8 +153,7 @@ define amdgpu_kernel void @fceil_v4f64(ptr addrspace(1) %out, <4 x double> %x) {
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s14, s3
; SI-NEXT: s_cmp_gt_i32 s15, 51
-; SI-NEXT: s_cselect_b32 s3, s1, s3
-; SI-NEXT: s_cselect_b32 s2, s0, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[0:1], s[2:3]
; SI-NEXT: v_cmp_gt_f64_e64 s[14:15], s[4:5], 0
; SI-NEXT: v_mov_b32_e32 v1, s22
; SI-NEXT: v_mov_b32_e32 v2, s2
@@ -176,8 +171,7 @@ define amdgpu_kernel void @fceil_v4f64(ptr addrspace(1) %out, <4 x double> %x) {
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s17, s1
; SI-NEXT: s_cmp_gt_i32 s22, 51
-; SI-NEXT: s_cselect_b32 s1, s7, s1
-; SI-NEXT: s_cselect_b32 s0, s6, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
; SI-NEXT: v_add_f64 v[4:5], s[18:19], v[0:1]
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v3, s1
@@ -195,8 +189,7 @@ define amdgpu_kernel void @fceil_v4f64(ptr addrspace(1) %out, <4 x double> %x) {
; SI-NEXT: s_cselect_b32 s6, 0, s6
; SI-NEXT: s_cselect_b32 s7, s17, s7
; SI-NEXT: s_cmp_gt_i32 s18, 51
-; SI-NEXT: s_cselect_b32 s7, s5, s7
-; SI-NEXT: s_cselect_b32 s6, s4, s6
+; SI-NEXT: s_cselect_b64 s[6:7], s[4:5], s[6:7]
; SI-NEXT: v_add_f64 v[2:3], s[2:3], v[0:1]
; SI-NEXT: v_mov_b32_e32 v1, s16
; SI-NEXT: v_mov_b32_e32 v6, s6
@@ -243,8 +236,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s38, 0, s38
; SI-NEXT: s_cselect_b32 s39, s40, s39
; SI-NEXT: s_cmp_gt_i32 s33, 51
-; SI-NEXT: s_cselect_b32 s41, s3, s39
-; SI-NEXT: s_cselect_b32 s40, s2, s38
+; SI-NEXT: s_cselect_b64 s[40:41], s[2:3], s[38:39]
; SI-NEXT: v_cmp_gt_f64_e64 s[38:39], s[12:13], 0
; SI-NEXT: v_mov_b32_e32 v0, s40
; SI-NEXT: v_mov_b32_e32 v1, s41
@@ -262,8 +254,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s22, s3
; SI-NEXT: s_cmp_gt_i32 s23, 51
-; SI-NEXT: s_cselect_b32 s3, s1, s3
-; SI-NEXT: s_cselect_b32 s2, s0, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[0:1], s[2:3]
; SI-NEXT: v_add_f64 v[2:3], s[40:41], v[4:5]
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
@@ -281,8 +272,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s22, s1
; SI-NEXT: s_cmp_gt_i32 s23, 51
-; SI-NEXT: s_cselect_b32 s1, s7, s1
-; SI-NEXT: s_cselect_b32 s0, s6, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[6:7], s[0:1]
; SI-NEXT: v_add_f64 v[0:1], s[2:3], v[4:5]
; SI-NEXT: v_mov_b32_e32 v6, s1
; SI-NEXT: v_mov_b32_e32 v5, s0
@@ -300,8 +290,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s6, s3
; SI-NEXT: s_cmp_gt_i32 s7, 51
-; SI-NEXT: s_cselect_b32 s3, s5, s3
-; SI-NEXT: s_cselect_b32 s2, s4, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[4:5], s[2:3]
; SI-NEXT: v_add_f64 v[7:8], s[0:1], v[4:5]
; SI-NEXT: v_mov_b32_e32 v6, s3
; SI-NEXT: v_mov_b32_e32 v5, s2
@@ -320,8 +309,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s4, s1
; SI-NEXT: s_cmp_gt_i32 s2, 51
-; SI-NEXT: s_cselect_b32 s1, s11, s1
-; SI-NEXT: s_cselect_b32 s0, s10, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[10:11], s[0:1]
; SI-NEXT: buffer_store_dwordx4 v[5:8], off, s[16:19], 0 offset:16
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v6, s1
@@ -340,8 +328,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s4, s3
; SI-NEXT: s_cmp_gt_i32 s5, 51
-; SI-NEXT: s_cselect_b32 s3, s9, s3
-; SI-NEXT: s_cselect_b32 s2, s8, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[8:9], s[2:3]
; SI-NEXT: v_add_f64 v[7:8], s[0:1], v[4:5]
; SI-NEXT: v_mov_b32_e32 v6, s3
; SI-NEXT: v_mov_b32_e32 v5, s2
@@ -359,8 +346,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s4, s1
; SI-NEXT: s_cmp_gt_i32 s5, 51
-; SI-NEXT: s_cselect_b32 s1, s15, s1
-; SI-NEXT: s_cselect_b32 s0, s14, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[14:15], s[0:1]
; SI-NEXT: v_add_f64 v[5:6], s[2:3], v[4:5]
; SI-NEXT: v_mov_b32_e32 v10, s1
; SI-NEXT: v_mov_b32_e32 v9, s0
@@ -377,8 +363,7 @@ define amdgpu_kernel void @fceil_v8f64(ptr addrspace(1) %out, <8 x double> %x) {
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s5, s3
; SI-NEXT: s_cmp_gt_i32 s6, 51
-; SI-NEXT: s_cselect_b32 s3, s13, s3
-; SI-NEXT: s_cselect_b32 s2, s12, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[12:13], s[2:3]
; SI-NEXT: buffer_store_dwordx4 v[5:8], off, s[16:19], 0 offset:32
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v5, s4
@@ -418,8 +403,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s2, s1
; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b32 s3, s11, s1
-; SI-NEXT: s_cselect_b32 s2, s10, s0
+; SI-NEXT: s_cselect_b64 s[2:3], s[10:11], s[0:1]
; SI-NEXT: v_cmp_gt_f64_e64 s[0:1], s[10:11], 0
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
@@ -436,8 +420,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s6, s1
; SI-NEXT: s_cmp_gt_i32 s7, 51
-; SI-NEXT: s_cselect_b32 s7, s9, s1
-; SI-NEXT: s_cselect_b32 s6, s8, s0
+; SI-NEXT: s_cselect_b64 s[6:7], s[8:9], s[0:1]
; SI-NEXT: v_cmp_gt_f64_e64 s[0:1], s[8:9], 0
; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: v_mov_b32_e32 v1, s7
@@ -454,8 +437,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s8, s1
; SI-NEXT: s_cmp_gt_i32 s9, 51
-; SI-NEXT: s_cselect_b32 s9, s15, s1
-; SI-NEXT: s_cselect_b32 s8, s14, s0
+; SI-NEXT: s_cselect_b64 s[8:9], s[14:15], s[0:1]
; SI-NEXT: v_cmp_gt_f64_e64 s[0:1], s[14:15], 0
; SI-NEXT: v_mov_b32_e32 v9, s10
; SI-NEXT: v_mov_b32_e32 v0, s8
@@ -474,8 +456,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s1, s10, s1
; SI-NEXT: v_cmp_gt_f64_e64 s[10:11], s[12:13], 0
; SI-NEXT: s_cmp_gt_i32 s15, 51
-; SI-NEXT: s_cselect_b32 s1, s13, s1
-; SI-NEXT: s_cselect_b32 s0, s12, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[12:13], s[0:1]
; SI-NEXT: v_add_f64 v[2:3], s[2:3], v[8:9]
; SI-NEXT: v_mov_b32_e32 v9, s27
; SI-NEXT: v_mov_b32_e32 v0, s0
@@ -552,8 +533,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s1, s2, s1
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b32 s17, s21, s1
-; SI-NEXT: s_cselect_b32 s16, s20, s0
+; SI-NEXT: s_cselect_b64 s[16:17], s[20:21], s[0:1]
; SI-NEXT: v_cmp_gt_f64_e64 s[22:23], s[20:21], 0
; SI-NEXT: v_mov_b32_e32 v9, s16
; SI-NEXT: v_mov_b32_e32 v10, s17
@@ -673,8 +653,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s18, s5
; SI-NEXT: s_cmp_gt_i32 s16, 51
-; SI-NEXT: s_cselect_b32 s5, s11, s5
-; SI-NEXT: s_cselect_b32 s4, s10, s4
+; SI-NEXT: s_cselect_b64 s[4:5], s[10:11], s[4:5]
; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[24:27], 0 offset:80
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v4, s4
@@ -693,8 +672,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s10, 0, s10
; SI-NEXT: s_cselect_b32 s11, s16, s11
; SI-NEXT: s_cmp_gt_i32 s17, 51
-; SI-NEXT: s_cselect_b32 s11, s9, s11
-; SI-NEXT: s_cselect_b32 s10, s8, s10
+; SI-NEXT: s_cselect_b64 s[10:11], s[8:9], s[10:11]
; SI-NEXT: v_add_f64 v[6:7], s[4:5], v[8:9]
; SI-NEXT: v_mov_b32_e32 v4, s10
; SI-NEXT: v_mov_b32_e32 v5, s11
@@ -712,8 +690,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s4, s3
; SI-NEXT: s_cmp_gt_i32 s5, 51
-; SI-NEXT: s_cselect_b32 s3, s15, s3
-; SI-NEXT: s_cselect_b32 s2, s14, s2
+; SI-NEXT: s_cselect_b64 s[2:3], s[14:15], s[2:3]
; SI-NEXT: v_add_f64 v[4:5], s[10:11], v[8:9]
; SI-NEXT: v_mov_b32_e32 v10, s3
; SI-NEXT: v_mov_b32_e32 v9, s2
@@ -730,8 +707,7 @@ define amdgpu_kernel void @fceil_v16f64(ptr addrspace(1) %out, <16 x double> %x)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s5, s1
; SI-NEXT: s_cmp_gt_i32 s8, 51
-; SI-NEXT: s_cselect_b32 s1, s13, s1
-; SI-NEXT: s_cselect_b32 s0, s12, s0
+; SI-NEXT: s_cselect_b64 s[0:1], s[12:13], s[0:1]
; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[24:27], 0 offset:96
; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt expcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 9b44acd5c0716..b52fbf11bdd89 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1656,11 +1656,11 @@ define amdgpu_kernel void @multiple_uses_fneg_select_f64(double %x, double %y, i
; GFX11-NEXT: v_mov_b32_e32 v0, s1
; GFX11-NEXT: s_bitcmp1_b32 s6, 0
; GFX11-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v0, s3, v0, vcc_lo
; GFX11-NEXT: s_and_b32 s6, vcc_lo, exec_lo
-; GFX11-NEXT: s_cselect_b32 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s0, s0, s2
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, s1, -v0, vcc_lo
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/fold-consecutive-cselect32.mir b/llvm/test/CodeGen/AMDGPU/fold-consecutive-cselect32.mir
new file mode 100644
index 0000000000000..7d9d41cefee50
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-consecutive-cselect32.mir
@@ -0,0 +1,120 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=si-pre-emit-peephole %s -o - | FileCheck %s
+
+---
+name: consecutive
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $scc
+ ; CHECK-LABEL: name: consecutive
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0_sgpr1 = S_CSELECT_B64 $sgpr2_sgpr3, $sgpr4_sgpr5, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ $sgpr1 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: consecutive_swapped
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $scc
+ ; CHECK-LABEL: name: consecutive_swapped
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0_sgpr1 = S_CSELECT_B64 $sgpr2_sgpr3, $sgpr4_sgpr5, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr1 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: consecutive_imm
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $scc
+ ; CHECK-LABEL: name: consecutive_imm
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0_sgpr1 = S_CSELECT_B64 $sgpr2_sgpr3, 4294967296333333, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr2, 333333, implicit $scc
+ $sgpr1 = S_CSELECT_B32 $sgpr3, 1000000, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: nonconsecutive_dst
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $scc
+ ; CHECK-LABEL: name: nonconsecutive_dst
+ ; CHECK: liveins: $sgpr0, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ ; CHECK-NEXT: $sgpr2 = S_CSELECT_B32 $sgpr4, $sgpr6, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ $sgpr2 = S_CSELECT_B32 $sgpr4, $sgpr6, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: nonconsecutive_src
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr4, $sgpr6, $sgpr7, $scc
+ ; CHECK-LABEL: name: nonconsecutive_src
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr4, $sgpr6, $sgpr7, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr6, implicit $scc
+ ; CHECK-NEXT: $sgpr1 = S_CSELECT_B32 $sgpr4, $sgpr7, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr6, implicit $scc
+ $sgpr1 = S_CSELECT_B32 $sgpr4, $sgpr7, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: mixed_src
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $scc
+ ; CHECK-LABEL: name: mixed_src
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ ; CHECK-NEXT: $sgpr1 = S_CSELECT_B32 $sgpr3, 42, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ $sgpr1 = S_CSELECT_B32 $sgpr3, 42, implicit $scc
+ S_ENDPGM 0
+...
+
+---
+name: nonconsecutive_cselects
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $scc
+ ; CHECK-LABEL: name: nonconsecutive_cselects
+ ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $scc
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ ; CHECK-NEXT: $sgpr6 = S_MOV_B32 0
+ ; CHECK-NEXT: $sgpr1 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ ; CHECK-NEXT: S_ENDPGM 0
+ $sgpr0 = S_CSELECT_B32 $sgpr2, $sgpr4, implicit $scc
+ $sgpr6 = S_MOV_B32 0
+ $sgpr1 = S_CSELECT_B32 $sgpr3, $sgpr5, implicit $scc
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll b/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
index 49204f84acb85..d42fa1cc434b8 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
@@ -252,8 +252,7 @@ define amdgpu_kernel void @fptoui_f64_to_i64_to_f64(ptr addrspace(1) %out, doubl
; GFX6-NEXT: s_cselect_b32 s4, 0, s4
; GFX6-NEXT: s_cselect_b32 s5, s8, s5
; GFX6-NEXT: s_cmp_gt_i32 s7, 51
-; GFX6-NEXT: s_cselect_b32 s3, s3, s5
-; GFX6-NEXT: s_cselect_b32 s2, s2, s4
+; GFX6-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX6-NEXT: v_ldexp_f64 v[0:1], s[2:3], v0
; GFX6-NEXT: v_mov_b32_e32 v4, -1
; GFX6-NEXT: v_fract_f64_e32 v[2:3], v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index bf153a88982e0..f67e1528d79b2 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -4452,8 +4452,7 @@ define amdgpu_kernel void @fast_frem_f64(ptr addrspace(1) %out, ptr addrspace(1)
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s9, s3
; SI-NEXT: s_cmp_gt_i32 s8, 51
-; SI-NEXT: s_cselect_b32 s1, s1, s3
-; SI-NEXT: s_cselect_b32 s0, s0, s2
+; SI-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; SI-NEXT: v_fma_f64 v[0:1], -s[0:1], v[2:3], v[0:1]
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
@@ -4730,8 +4729,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: s_cselect_b32 s2, 0, s2
; SI-NEXT: s_cselect_b32 s3, s9, s3
; SI-NEXT: s_cmp_gt_i32 s8, 51
-; SI-NEXT: s_cselect_b32 s1, s1, s3
-; SI-NEXT: s_cselect_b32 s0, s0, s2
+; SI-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; SI-NEXT: v_fma_f64 v[0:1], -s[0:1], v[2:3], v[0:1]
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ftrunc.f64.ll b/llvm/test/CodeGen/AMDGPU/ftrunc.f64.ll
index cb5d6e732c701..d893d112e47c9 100644
--- a/llvm/test/CodeGen/AMDGPU/ftrunc.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ftrunc.f64.ll
@@ -32,8 +32,7 @@ define amdgpu_kernel void @v_ftrunc_f64(ptr addrspace(1) %out, ptr addrspace(1)
; SI-DAG: s_cselect_b32
; SI-DAG: s_cselect_b32
; SI-DAG: cmp_lt_i32
-; SI-DAG: s_cselect_b32
-; SI-DAG: s_cselect_b32
+; SI-DAG: s_cselect_b64
; SI: s_endpgm
define amdgpu_kernel void @ftrunc_f64(ptr addrspace(1) %out, double %x) {
%y = call double @llvm.trunc.f64(double %x) nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index b018fd3545374..e2dee1ad6d138 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -351,8 +351,7 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_add_i32 s8, s6, 1
; GFX9-NEXT: s_sub_i32 s9, s7, s2
; GFX9-NEXT: s_cmp_ge_u32 s7, s2
-; GFX9-NEXT: s_cselect_b32 s6, s8, s6
-; GFX9-NEXT: s_cselect_b32 s7, s9, s7
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX9-NEXT: s_add_i32 s8, s6, 1
; GFX9-NEXT: s_cmp_ge_u32 s7, s2
; GFX9-NEXT: s_cselect_b32 s6, s8, s6
@@ -395,8 +394,7 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_sub_i32 s7, s4, s7
; GFX10-NEXT: s_sub_i32 s9, s7, s2
; GFX10-NEXT: s_cmp_ge_u32 s7, s2
-; GFX10-NEXT: s_cselect_b32 s6, s8, s6
-; GFX10-NEXT: s_cselect_b32 s7, s9, s7
+; GFX10-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX10-NEXT: s_add_i32 s8, s6, 1
; GFX10-NEXT: s_cmp_ge_u32 s7, s2
; GFX10-NEXT: s_cselect_b32 s6, s8, s6
@@ -443,11 +441,10 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_mul_i32 s7, s6, s2
; GFX11-NEXT: s_add_i32 s8, s6, 1
; GFX11-NEXT: s_sub_i32 s7, s4, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_sub_i32 s9, s7, s2
; GFX11-NEXT: s_cmp_ge_u32 s7, s2
-; GFX11-NEXT: s_cselect_b32 s6, s8, s6
-; GFX11-NEXT: s_cselect_b32 s7, s9, s7
+; GFX11-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX11-NEXT: s_add_i32 s8, s6, 1
; GFX11-NEXT: s_cmp_ge_u32 s7, s2
; GFX11-NEXT: s_cselect_b32 s6, s8, s6
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
index f8770642cc006..3a8dac3ac07d4 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call-known-callees.ll
@@ -32,8 +32,7 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX9-NEXT: s_and_b32 s14, 1, s17
; GFX9-NEXT: s_cmp_eq_u32 s14, 1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1
-; GFX9-NEXT: s_cselect_b32 s19, s23, s21
-; GFX9-NEXT: s_cselect_b32 s18, s22, s20
+; GFX9-NEXT: s_cselect_b64 s[18:19], s[22:23], s[20:21]
; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2
; GFX9-NEXT: s_mov_b32 s14, s16
; GFX9-NEXT: v_mov_b32_e32 v1, 0
@@ -65,8 +64,7 @@ define amdgpu_kernel void @indirect_call_known_no_special_inputs() {
; GFX12-NEXT: s_and_b32 s12, 1, s14
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_cmp_eq_u32 s12, 1
-; GFX12-NEXT: s_cselect_b32 s13, s7, s5
-; GFX12-NEXT: s_cselect_b32 s12, s6, s4
+; GFX12-NEXT: s_cselect_b64 s[12:13], s[6:7], s[4:5]
; GFX12-NEXT: s_mov_b64 s[4:5], s[0:1]
; GFX12-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX12-NEXT: s_swappc_b64 s[30:31], s[12:13]
diff --git a/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll b/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll
index 23ebfb817096f..ba6cf07210b8b 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll
@@ -591,8 +591,7 @@ define i64 @s_fneg_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s3, s3, 0x80000000
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s2, s16
-; GFX11-NEXT: s_cselect_b32 s1, s3, s17
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[16:17]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -631,8 +630,7 @@ define i64 @s_fneg_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s3, s3, 0x80000000
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s16, s2
-; GFX11-NEXT: s_cselect_b32 s1, s17, s3
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[16:17], s[2:3]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -674,8 +672,7 @@ define i64 @s_fneg_1_fabs_2_select_i64(i64 inreg %cond, i64 inreg %a, i64 inreg
; GFX11-NEXT: s_xor_b32 s3, s3, 0x80000000
; GFX11-NEXT: s_bitset0_b32 s17, 31
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s2, s16
-; GFX11-NEXT: s_cselect_b32 s1, s3, s17
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[16:17]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -693,8 +690,7 @@ define i64 @s_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
; GFX7-NEXT: s_bitset0_b32 s19, 31
; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX7-NEXT: s_cselect_b32 s4, s18, s20
-; GFX7-NEXT: s_cselect_b32 s5, s19, s21
+; GFX7-NEXT: s_cselect_b64 s[4:5], s[18:19], s[20:21]
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -704,8 +700,7 @@ define i64 @s_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_bitset0_b32 s19, 31
; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
-; GFX9-NEXT: s_cselect_b32 s4, s18, s20
-; GFX9-NEXT: s_cselect_b32 s5, s19, s21
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[18:19], s[20:21]
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -715,8 +710,7 @@ define i64 @s_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_bitset0_b32 s3, 31
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s2, s16
-; GFX11-NEXT: s_cselect_b32 s1, s3, s17
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[16:17]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -733,8 +727,7 @@ define i64 @s_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
; GFX7-NEXT: s_bitset0_b32 s19, 31
; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX7-NEXT: s_cselect_b32 s4, s20, s18
-; GFX7-NEXT: s_cselect_b32 s5, s21, s19
+; GFX7-NEXT: s_cselect_b64 s[4:5], s[20:21], s[18:19]
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -744,8 +737,7 @@ define i64 @s_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_bitset0_b32 s19, 31
; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
-; GFX9-NEXT: s_cselect_b32 s4, s20, s18
-; GFX9-NEXT: s_cselect_b32 s5, s21, s19
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[20:21], s[18:19]
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -755,8 +747,7 @@ define i64 @s_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_bitset0_b32 s3, 31
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s16, s2
-; GFX11-NEXT: s_cselect_b32 s1, s17, s3
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[16:17], s[2:3]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -773,8 +764,7 @@ define i64 @s_fneg_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
; GFX7-NEXT: s_bitset1_b32 s19, 31
; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX7-NEXT: s_cselect_b32 s4, s18, s20
-; GFX7-NEXT: s_cselect_b32 s5, s19, s21
+; GFX7-NEXT: s_cselect_b64 s[4:5], s[18:19], s[20:21]
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -784,8 +774,7 @@ define i64 @s_fneg_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s19, 31
; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
-; GFX9-NEXT: s_cselect_b32 s4, s18, s20
-; GFX9-NEXT: s_cselect_b32 s5, s19, s21
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[18:19], s[20:21]
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -795,8 +784,7 @@ define i64 @s_fneg_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_bitset1_b32 s3, 31
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s2, s16
-; GFX11-NEXT: s_cselect_b32 s1, s3, s17
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[16:17]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
@@ -813,8 +801,7 @@ define i64 @s_fneg_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX7-NEXT: v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
; GFX7-NEXT: s_bitset1_b32 s19, 31
; GFX7-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX7-NEXT: s_cselect_b32 s4, s20, s18
-; GFX7-NEXT: s_cselect_b32 s5, s21, s19
+; GFX7-NEXT: s_cselect_b64 s[4:5], s[20:21], s[18:19]
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: v_mov_b32_e32 v1, s5
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -824,8 +811,7 @@ define i64 @s_fneg_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s19, 31
; GFX9-NEXT: s_cmp_eq_u64 s[16:17], 0
-; GFX9-NEXT: s_cselect_b32 s4, s20, s18
-; GFX9-NEXT: s_cselect_b32 s5, s21, s19
+; GFX9-NEXT: s_cselect_b64 s[4:5], s[20:21], s[18:19]
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -835,8 +821,7 @@ define i64 @s_fneg_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_bitset1_b32 s3, 31
; GFX11-NEXT: s_cmp_eq_u64 s[0:1], 0
-; GFX11-NEXT: s_cselect_b32 s0, s16, s2
-; GFX11-NEXT: s_cselect_b32 s1, s17, s3
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[16:17], s[2:3]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
index 2263231b77fb2..67615dc99a0de 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
@@ -683,13 +683,11 @@ define amdgpu_kernel void @smulo_i64_s(i64 %x, i64 %y) {
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_sub_nc_u64 s[8:9], s[6:7], s[4:5]
; GFX12-NEXT: s_mov_b32 s4, s0
-; GFX12-NEXT: s_cselect_b32 s7, s9, s7
-; GFX12-NEXT: s_cselect_b32 s6, s8, s6
+; GFX12-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX12-NEXT: s_cmp_lt_i32 s3, 0
; GFX12-NEXT: s_sub_nc_u64 s[4:5], s[6:7], s[4:5]
; GFX12-NEXT: s_mul_u64 s[0:1], s[0:1], s[2:3]
-; GFX12-NEXT: s_cselect_b32 s3, s5, s7
-; GFX12-NEXT: s_cselect_b32 s2, s4, s6
+; GFX12-NEXT: s_cselect_b64 s[2:3], s[4:5], s[6:7]
; GFX12-NEXT: s_ashr_i32 s4, s1, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_mov_b32 s5, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index 76b97e843d777..46e016d0ddae9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -19,8 +19,7 @@ define amdgpu_kernel void @round_f64(ptr addrspace(1) %out, double %x) #0 {
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s8, s5
; SI-NEXT: s_cmp_gt_i32 s7, 51
-; SI-NEXT: s_cselect_b32 s8, s2, s4
-; SI-NEXT: s_cselect_b32 s9, s3, s5
+; SI-NEXT: s_cselect_b64 s[8:9], s[2:3], s[4:5]
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: v_add_f64 v[0:1], s[2:3], -v[0:1]
@@ -150,8 +149,7 @@ define amdgpu_kernel void @round_v2f64(ptr addrspace(1) %out, <2 x double> %in)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s3, s1
; SI-NEXT: s_cmp_gt_i32 s12, 51
-; SI-NEXT: s_cselect_b32 s12, s10, s0
-; SI-NEXT: s_cselect_b32 s13, s11, s1
+; SI-NEXT: s_cselect_b64 s[12:13], s[10:11], s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s12
; SI-NEXT: v_mov_b32_e32 v1, s13
; SI-NEXT: v_add_f64 v[0:1], s[10:11], -v[0:1]
@@ -170,8 +168,7 @@ define amdgpu_kernel void @round_v2f64(ptr addrspace(1) %out, <2 x double> %in)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s6, s5
; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b32 s4, s8, s4
-; SI-NEXT: s_cselect_b32 s5, s9, s5
+; SI-NEXT: s_cselect_b64 s[4:5], s[8:9], s[4:5]
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: v_add_f64 v[2:3], s[8:9], -v[2:3]
@@ -243,8 +240,7 @@ define amdgpu_kernel void @round_v4f64(ptr addrspace(1) %out, <4 x double> %in)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s3, s1
; SI-NEXT: s_cmp_gt_i32 s16, 51
-; SI-NEXT: s_cselect_b32 s16, s10, s0
-; SI-NEXT: s_cselect_b32 s17, s11, s1
+; SI-NEXT: s_cselect_b64 s[16:17], s[10:11], s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s16
; SI-NEXT: v_mov_b32_e32 v1, s17
; SI-NEXT: v_add_f64 v[0:1], s[10:11], -v[0:1]
@@ -285,8 +281,7 @@ define amdgpu_kernel void @round_v4f64(ptr addrspace(1) %out, <4 x double> %in)
; SI-NEXT: s_cselect_b32 s8, 0, s8
; SI-NEXT: s_cselect_b32 s9, s10, s9
; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b32 s8, s14, s8
-; SI-NEXT: s_cselect_b32 s9, s15, s9
+; SI-NEXT: s_cselect_b64 s[8:9], s[14:15], s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: v_add_f64 v[0:1], s[14:15], -v[0:1]
@@ -305,8 +300,7 @@ define amdgpu_kernel void @round_v4f64(ptr addrspace(1) %out, <4 x double> %in)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s6, s5
; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b32 s5, s13, s5
-; SI-NEXT: s_cselect_b32 s4, s12, s4
+; SI-NEXT: s_cselect_b64 s[4:5], s[12:13], s[4:5]
; SI-NEXT: v_mov_b32_e32 v6, s5
; SI-NEXT: v_mov_b32_e32 v5, s4
; SI-NEXT: v_add_f64 v[6:7], s[12:13], -v[5:6]
@@ -397,8 +391,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s0, 0, s0
; SI-NEXT: s_cselect_b32 s1, s3, s1
; SI-NEXT: s_cmp_gt_i32 s24, 51
-; SI-NEXT: s_cselect_b32 s24, s10, s0
-; SI-NEXT: s_cselect_b32 s25, s11, s1
+; SI-NEXT: s_cselect_b64 s[24:25], s[10:11], s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s24
; SI-NEXT: v_mov_b32_e32 v1, s25
; SI-NEXT: v_add_f64 v[0:1], s[10:11], -v[0:1]
@@ -439,8 +432,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s8, 0, s8
; SI-NEXT: s_cselect_b32 s9, s11, s9
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s8, s14, s8
-; SI-NEXT: s_cselect_b32 s9, s15, s9
+; SI-NEXT: s_cselect_b64 s[8:9], s[14:15], s[8:9]
; SI-NEXT: v_mov_b32_e32 v0, s8
; SI-NEXT: v_mov_b32_e32 v1, s9
; SI-NEXT: v_add_f64 v[0:1], s[14:15], -v[0:1]
@@ -459,8 +451,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s11, s5
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s4, s12, s4
-; SI-NEXT: s_cselect_b32 s5, s13, s5
+; SI-NEXT: s_cselect_b64 s[4:5], s[12:13], s[4:5]
; SI-NEXT: v_mov_b32_e32 v4, s4
; SI-NEXT: v_mov_b32_e32 v5, s5
; SI-NEXT: v_add_f64 v[4:5], s[12:13], -v[4:5]
@@ -480,8 +471,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s8, 0, s8
; SI-NEXT: s_cselect_b32 s9, s11, s9
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s8, s18, s8
-; SI-NEXT: s_cselect_b32 s9, s19, s9
+; SI-NEXT: s_cselect_b64 s[8:9], s[18:19], s[8:9]
; SI-NEXT: v_mov_b32_e32 v4, s8
; SI-NEXT: v_mov_b32_e32 v5, s9
; SI-NEXT: v_add_f64 v[4:5], s[18:19], -v[4:5]
@@ -501,8 +491,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s11, s5
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s5, s17, s5
-; SI-NEXT: s_cselect_b32 s4, s16, s4
+; SI-NEXT: s_cselect_b64 s[4:5], s[16:17], s[4:5]
; SI-NEXT: v_mov_b32_e32 v10, s5
; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: v_add_f64 v[10:11], s[16:17], -v[9:10]
@@ -522,8 +511,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s8, 0, s8
; SI-NEXT: s_cselect_b32 s9, s11, s9
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s9, s23, s9
-; SI-NEXT: s_cselect_b32 s8, s22, s8
+; SI-NEXT: s_cselect_b64 s[8:9], s[22:23], s[8:9]
; SI-NEXT: v_mov_b32_e32 v10, s9
; SI-NEXT: v_mov_b32_e32 v9, s8
; SI-NEXT: v_add_f64 v[10:11], s[22:23], -v[9:10]
@@ -543,8 +531,7 @@ define amdgpu_kernel void @round_v8f64(ptr addrspace(1) %out, <8 x double> %in)
; SI-NEXT: s_cselect_b32 s4, 0, s4
; SI-NEXT: s_cselect_b32 s5, s6, s5
; SI-NEXT: s_cmp_gt_i32 s10, 51
-; SI-NEXT: s_cselect_b32 s5, s21, s5
-; SI-NEXT: s_cselect_b32 s4, s20, s4
+; SI-NEXT: s_cselect_b64 s[4:5], s[20:21], s[4:5]
; SI-NEXT: v_mov_b32_e32 v15, s5
; SI-NEXT: v_mov_b32_e32 v14, s4
; SI-NEXT: v_add_f64 v[14:15], s[20:21], -v[14:15]
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 5283233a0b461..fcfef2b0a7c5e 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -131,8 +131,8 @@ define amdgpu_kernel void @v_test_imin_sle_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -228,7 +228,7 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
; GFX1250-LABEL: s_test_imin_sle_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -316,7 +316,7 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
; GFX1250-LABEL: s_test_imin_sle_v1i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -444,8 +444,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i32(ptr addrspace(1) %out, <4 x i32
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s11, s15
@@ -580,9 +580,9 @@ define amdgpu_kernel void @s_test_imin_sle_i8(ptr addrspace(1) %out, [8 x i32],
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i8 s2, s2
@@ -836,9 +836,9 @@ define amdgpu_kernel void @s_test_imin_sle_v4i8(ptr addrspace(1) %out, [8 x i32]
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i16 s5, s2
@@ -981,7 +981,7 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
; GFX1250-LABEL: s_test_imin_sle_v2i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_i16 v1, s2, s3
@@ -1179,8 +1179,8 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_i16 v1, s1, s3
@@ -1315,8 +1315,8 @@ define amdgpu_kernel void @v_test_imin_slt_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -1493,8 +1493,8 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_clause 0x1
@@ -1509,8 +1509,8 @@ define amdgpu_kernel void @v_test_imin_slt_i16(ptr addrspace(1) %out, ptr addrsp
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: s_clause 0x1
@@ -1607,7 +1607,7 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
; GFX1250-LABEL: s_test_imin_slt_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, s3
@@ -1713,8 +1713,8 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s0, s0, s2
@@ -1810,7 +1810,7 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
; GFX1250-LABEL: s_test_imin_slt_imm_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, 8
@@ -1905,7 +1905,7 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
; GFX1250-LABEL: s_test_imin_sle_imm_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_i32 s2, s2, 8
@@ -2041,8 +2041,8 @@ define amdgpu_kernel void @v_test_umin_ule_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2202,8 +2202,8 @@ define amdgpu_kernel void @v_test_umin_ule_v3i32(ptr addrspace(1) %out, ptr addr
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 4, v0
@@ -2406,8 +2406,8 @@ define amdgpu_kernel void @v_test_umin_ule_v3i16(ptr addrspace(1) %out, ptr addr
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2509,7 +2509,7 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
; GFX1250-LABEL: s_test_umin_ule_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -2645,8 +2645,8 @@ define amdgpu_kernel void @v_test_umin_ult_i32(ptr addrspace(1) %out, ptr addrsp
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -2806,8 +2806,8 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-TRUE16-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_clause 0x1
@@ -2822,8 +2822,8 @@ define amdgpu_kernel void @v_test_umin_ult_i8(ptr addrspace(1) %out, ptr addrspa
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: s_clause 0x1
@@ -2920,7 +2920,7 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
; GFX1250-LABEL: s_test_umin_ult_i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -3075,7 +3075,7 @@ define amdgpu_kernel void @v_test_umin_ult_i32_multi_use(ptr addrspace(1) %out0,
; GFX1250-LABEL: v_test_umin_ult_i32_multi_use:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s0, s[12:13], 0x0
@@ -3272,7 +3272,7 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-LABEL: v_test_umin_ult_i16_multi_use:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -3373,7 +3373,7 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
; GFX1250-LABEL: s_test_umin_ult_v1i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s2, s2, s3
@@ -3564,8 +3564,8 @@ define amdgpu_kernel void @s_test_umin_ult_v8i32(ptr addrspace(1) %out, <8 x i32
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b512 s[8:23], s[4:5], 0x20
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b512 s[8:23], s[4:5], 0x20 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v8, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_min_u32 s4, s9, s17
@@ -3881,8 +3881,8 @@ define amdgpu_kernel void @s_test_umin_ult_v8i16(ptr addrspace(1) %out, <8 x i16
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x10 nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_min_u16 v3, s11, s15
@@ -4006,9 +4006,9 @@ define amdgpu_kernel void @simplify_demanded_bits_test_umin_ult_i16(ptr addrspac
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
@@ -4136,9 +4136,9 @@ define amdgpu_kernel void @simplify_demanded_bits_test_min_slt_i16(ptr addrspace
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x2
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28
-; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x28 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x4c nv
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_sext_i32_i16 s2, s2
@@ -4268,7 +4268,7 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
; GFX1250-LABEL: s_test_imin_sle_i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_ashr_i32 s3, s2, 16
@@ -4355,8 +4355,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, s5
-; GFX9-NEXT: s_cselect_b32 s2, s2, s4
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4371,8 +4370,7 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_lt_u64_e64 s6, s[2:3], s[4:5]
; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s2, s2, s4
-; GFX10-NEXT: s_cselect_b32 s3, s3, s5
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4387,8 +4385,8 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_lt_u64_e64 s6, s[2:3], s[4:5]
; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s2, s2, s4
-; GFX11-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -4398,8 +4396,8 @@ define amdgpu_kernel void @test_umin_ult_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
@@ -4480,8 +4478,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_le_u64_e32 vcc, s[2:3], v[0:1]
; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, s5
-; GFX9-NEXT: s_cselect_b32 s2, s2, s4
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4496,8 +4493,7 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_le_u64_e64 s6, s[2:3], s[4:5]
; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s2, s2, s4
-; GFX10-NEXT: s_cselect_b32 s3, s3, s5
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4512,8 +4508,8 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_le_u64_e64 s6, s[2:3], s[4:5]
; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s2, s2, s4
-; GFX11-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -4523,8 +4519,8 @@ define amdgpu_kernel void @test_umin_ule_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_u64 v[0:1], s[2:3], s[6:7]
@@ -4605,8 +4601,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[0:1]
; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, s5
-; GFX9-NEXT: s_cselect_b32 s2, s2, s4
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4621,8 +4616,7 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_lt_i64_e64 s6, s[2:3], s[4:5]
; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s2, s2, s4
-; GFX10-NEXT: s_cselect_b32 s3, s3, s5
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4637,8 +4631,8 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_lt_i64_e64 s6, s[2:3], s[4:5]
; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s2, s2, s4
-; GFX11-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -4648,8 +4642,8 @@ define amdgpu_kernel void @test_imin_slt_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
@@ -4730,8 +4724,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_cmp_le_i64_e32 vcc, s[2:3], v[0:1]
; GFX9-NEXT: s_and_b64 s[6:7], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s3, s3, s5
-; GFX9-NEXT: s_cselect_b32 s2, s2, s4
+; GFX9-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4746,8 +4739,7 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_le_i64_e64 s6, s[2:3], s[4:5]
; GFX10-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX10-NEXT: s_cselect_b32 s2, s2, s4
-; GFX10-NEXT: s_cselect_b32 s3, s3, s5
+; GFX10-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
@@ -4762,8 +4754,8 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_le_i64_e64 s6, s[2:3], s[4:5]
; GFX11-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX11-NEXT: s_cselect_b32 s2, s2, s4
-; GFX11-NEXT: s_cselect_b32 s3, s3, s5
+; GFX11-NEXT: s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s3
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -4773,8 +4765,8 @@ define amdgpu_kernel void @test_imin_sle_i64(ptr addrspace(1) %out, i64 %a, i64
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_min_i64 v[0:1], s[2:3], s[6:7]
@@ -4936,8 +4928,8 @@ define amdgpu_kernel void @v_test_imin_sle_v2i16(ptr addrspace(1) %out, ptr addr
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
@@ -5107,8 +5099,8 @@ define amdgpu_kernel void @v_test_imin_ule_v2i16(ptr addrspace(1) %out, ptr addr
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll
index 441509ba01f64..d5379e92d4294 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll
@@ -137,8 +137,7 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, ptr addrspace(1) %in)
; GFX9-NEXT: s_add_i32 s9, s7, 1
; GFX9-NEXT: s_sub_i32 s8, s6, s5
; GFX9-NEXT: s_cmp_ge_u32 s6, s5
-; GFX9-NEXT: s_cselect_b32 s7, s9, s7
-; GFX9-NEXT: s_cselect_b32 s6, s8, s6
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX9-NEXT: s_add_i32 s8, s7, 1
; GFX9-NEXT: s_cmp_ge_u32 s6, s5
; GFX9-NEXT: s_cselect_b32 s5, s8, s7
@@ -2091,8 +2090,7 @@ define amdgpu_kernel void @v_sdiv_i25(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX9-NEXT: s_add_i32 s9, s7, 1
; GFX9-NEXT: s_sub_i32 s8, s6, s5
; GFX9-NEXT: s_cmp_ge_u32 s6, s5
-; GFX9-NEXT: s_cselect_b32 s7, s9, s7
-; GFX9-NEXT: s_cselect_b32 s6, s8, s6
+; GFX9-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX9-NEXT: s_add_i32 s8, s7, 1
; GFX9-NEXT: s_cmp_ge_u32 s6, s5
; GFX9-NEXT: s_cselect_b32 s5, s8, s7
diff --git a/llvm/test/CodeGen/AMDGPU/select-vectors.ll b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
index e754f665c5f43..ed6b191bf4d8c 100644
--- a/llvm/test/CodeGen/AMDGPU/select-vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
@@ -204,8 +204,7 @@ define amdgpu_kernel void @v_select_v32i16(ptr addrspace(1) %out, ptr addrspace(
; vector select with SGPR inputs.
; GCN-LABEL: {{^}}s_select_v2i32:
-; GCN: s_cselect_b32
-; GCN: s_cselect_b32
+; GCN: s_cselect_b64
; GCN: buffer_store_dwordx2
define amdgpu_kernel void @s_select_v2i32(ptr addrspace(1) %out, <2 x i32> %a, <2 x i32> %b, i32 %c) #0 {
%cmp = icmp eq i32 %c, 0
@@ -263,8 +262,7 @@ define amdgpu_kernel void @select_v8i32(ptr addrspace(1) %out, <8 x i32> %a, <8
; GCN-LABEL: {{^}}s_select_v2f32:
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0{{$}}
-; GCN-DAG: s_cselect_b32
-; GCN-DAG: s_cselect_b32
+; GCN-DAG: s_cselect_b64
; GCN: buffer_store_dwordx2
define amdgpu_kernel void @s_select_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b, i32 %c) #0 {
%cmp = icmp eq i32 %c, 0
@@ -329,8 +327,11 @@ bb:
; GCN: s_cselect_b32
; GCN: s_cselect_b32
; GCN: s_cselect_b32
-; GCN: s_cselect_b32
-; GCN: s_cselect_b32
+; SI: s_cselect_b32
+; SI: s_cselect_b32
+; VI: s_cselect_b64
+; GFX9: s_cselect_b32
+; GFX9: s_cselect_b32
; GCN: buffer_store_dwordx
define amdgpu_kernel void @s_select_v5f32(ptr addrspace(1) %out, <5 x float> %a, <5 x float> %b, i32 %c) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/select64.ll b/llvm/test/CodeGen/AMDGPU/select64.ll
index 2013db7bc7be3..607876c42555c 100644
--- a/llvm/test/CodeGen/AMDGPU/select64.ll
+++ b/llvm/test/CodeGen/AMDGPU/select64.ll
@@ -1,12 +1,13 @@
-; RUN: llc < %s -mtriple=amdgcn -mcpu=tahiti | FileCheck -check-prefix=GCN %s
-; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=GCN %s
+; RUN: llc < %s -mtriple=amdgcn -mcpu=tahiti | FileCheck -check-prefixes=GCN,TAHITI %s
+; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefix=GCN,TONGA %s
; GCN-LABEL: {{^}}select0:
; i64 select should be split into two i32 selects, and we shouldn't need
; to use a shfit to extract the hi dword of the input.
; GCN-NOT: s_lshr_b64
-; GCN: s_cselect_b32
-; GCN: s_cselect_b32
+; TAHITI: s_cselect_b32
+; TAHITI: s_cselect_b32
+; TONGA: s_cselect_b64
define amdgpu_kernel void @select0(ptr addrspace(1) %out, i32 %cond, i64 %in) {
entry:
%0 = icmp ugt i32 %cond, 5
@@ -51,8 +52,9 @@ define amdgpu_kernel void @v_select_trunc_i64_2(ptr addrspace(1) %out, i32 %cond
}
; GCN-LABEL: {{^}}v_select_i64_split_imm:
-; GCN-DAG: s_cselect_b32
-; GCN-DAG: s_cselect_b32
+; TAHITI-DAG: s_cselect_b32
+; TAHITI-DAG: s_cselect_b32
+; TONGA-DAG: s_cselect_b64
; GCN: s_endpgm
define amdgpu_kernel void @v_select_i64_split_imm(ptr addrspace(1) %out, i32 %cond, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) nounwind {
%cmp = icmp ugt i32 %cond, 5
diff --git a/llvm/test/CodeGen/AMDGPU/selectcc.ll b/llvm/test/CodeGen/AMDGPU/selectcc.ll
index a16ad927cb893..6e292d8cc0a04 100644
--- a/llvm/test/CodeGen/AMDGPU/selectcc.ll
+++ b/llvm/test/CodeGen/AMDGPU/selectcc.ll
@@ -10,7 +10,7 @@
; EG: CNDE_INT
; SI: v_cmp_eq_u64
; VI: s_cmp_eq_u64
-; GCN: s_cselect_b32
+; GCN: s_cselect_b64
define amdgpu_kernel void @selectcc_i64(ptr addrspace(1) %out, i64 %lhs, i64 %rhs, i64 %true, i64 %false) {
entry:
%0 = icmp eq i64 %lhs, %rhs
diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
index 3a2d056dc504a..864d131733d00 100644
--- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll
@@ -196,13 +196,11 @@ define amdgpu_kernel void @s_shl_i128_ss(i128 %lhs, i128 %rhs) {
; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], s12
; GCN-NEXT: s_or_b64 s[0:1], s[6:7], s[0:1]
; GCN-NEXT: s_cmp_lt_u32 s4, 64
-; GCN-NEXT: s_cselect_b32 s0, s0, s10
-; GCN-NEXT: s_cselect_b32 s1, s1, s11
+; GCN-NEXT: s_cselect_b64 s[0:1], s[0:1], s[10:11]
; GCN-NEXT: s_cselect_b32 s5, s9, 0
; GCN-NEXT: s_cselect_b32 s6, s8, 0
; GCN-NEXT: s_cmp_eq_u32 s4, 0
-; GCN-NEXT: s_cselect_b32 s1, s3, s1
-; GCN-NEXT: s_cselect_b32 s0, s2, s0
+; GCN-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: v_mov_b32_e32 v2, s0
@@ -232,13 +230,11 @@ define amdgpu_kernel void @s_lshr_i128_ss(i128 %lhs, i128 %rhs) {
; GCN-NEXT: s_lshl_b64 s[2:3], s[2:3], s12
; GCN-NEXT: s_or_b64 s[2:3], s[6:7], s[2:3]
; GCN-NEXT: s_cmp_lt_u32 s4, 64
-; GCN-NEXT: s_cselect_b32 s2, s2, s10
-; GCN-NEXT: s_cselect_b32 s3, s3, s11
+; GCN-NEXT: s_cselect_b64 s[2:3], s[2:3], s[10:11]
; GCN-NEXT: s_cselect_b32 s5, s9, 0
; GCN-NEXT: s_cselect_b32 s6, s8, 0
; GCN-NEXT: s_cmp_eq_u32 s4, 0
-; GCN-NEXT: s_cselect_b32 s1, s1, s3
-; GCN-NEXT: s_cselect_b32 s0, s0, s2
+; GCN-NEXT: s_cselect_b64 s[0:1], s[0:1], s[2:3]
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: v_mov_b32_e32 v2, s6
@@ -493,8 +489,7 @@ define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) {
; GCN-NEXT: s_cselect_b32 s7, s0, 0
; GCN-NEXT: s_lshl_b64 s[0:1], s[4:5], s12
; GCN-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GCN-NEXT: s_cselect_b32 s1, s1, 0
-; GCN-NEXT: s_cselect_b32 s0, s0, 0
+; GCN-NEXT: s_cselect_b64 s[0:1], s[0:1], 0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: v_mov_b32_e32 v2, s10
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 1f93bf7a68972..2223b3587286d 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -542,8 +542,7 @@ define amdgpu_kernel void @udiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_add_i32 s6, s4, 1
; GFX1030-NEXT: s_sub_i32 s7, s5, s2
; GFX1030-NEXT: s_cmp_ge_u32 s5, s2
-; GFX1030-NEXT: s_cselect_b32 s4, s6, s4
-; GFX1030-NEXT: s_cselect_b32 s5, s7, s5
+; GFX1030-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
; GFX1030-NEXT: s_add_i32 s6, s4, 1
; GFX1030-NEXT: s_cmp_ge_u32 s5, s2
; GFX1030-NEXT: v_readfirstlane_b32 s5, v1
@@ -559,8 +558,7 @@ define amdgpu_kernel void @udiv_v2i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_add_i32 s6, s4, 1
; GFX1030-NEXT: s_sub_i32 s7, s5, s3
; GFX1030-NEXT: s_cmp_ge_u32 s5, s3
-; GFX1030-NEXT: s_cselect_b32 s4, s6, s4
-; GFX1030-NEXT: s_cselect_b32 s5, s7, s5
+; GFX1030-NEXT: s_cselect_b64 s[4:5], s[6:7], s[4:5]
; GFX1030-NEXT: s_add_i32 s6, s4, 1
; GFX1030-NEXT: s_cmp_ge_u32 s5, s3
; GFX1030-NEXT: s_cselect_b32 s3, s6, s4
@@ -970,8 +968,7 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_add_i32 s8, s6, 1
; GFX1030-NEXT: s_sub_i32 s9, s7, s3
; GFX1030-NEXT: s_cmp_ge_u32 s7, s3
-; GFX1030-NEXT: s_cselect_b32 s6, s8, s6
-; GFX1030-NEXT: s_cselect_b32 s7, s9, s7
+; GFX1030-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX1030-NEXT: s_add_i32 s8, s6, 1
; GFX1030-NEXT: s_cmp_ge_u32 s7, s3
; GFX1030-NEXT: v_readfirstlane_b32 s7, v6
@@ -990,8 +987,7 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_sub_i32 s9, s7, s5
; GFX1030-NEXT: s_cmp_ge_u32 s7, s5
; GFX1030-NEXT: v_readfirstlane_b32 s10, v0
-; GFX1030-NEXT: s_cselect_b32 s6, s8, s6
-; GFX1030-NEXT: s_cselect_b32 s7, s9, s7
+; GFX1030-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX1030-NEXT: s_add_i32 s8, s6, 1
; GFX1030-NEXT: s_cmp_ge_u32 s7, s5
; GFX1030-NEXT: v_readfirstlane_b32 s7, v7
@@ -1008,8 +1004,7 @@ define amdgpu_kernel void @udiv_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %i
; GFX1030-NEXT: s_add_i32 s8, s6, 1
; GFX1030-NEXT: s_sub_i32 s9, s7, s2
; GFX1030-NEXT: s_cmp_ge_u32 s7, s2
-; GFX1030-NEXT: s_cselect_b32 s6, s8, s6
-; GFX1030-NEXT: s_cselect_b32 s7, s9, s7
+; GFX1030-NEXT: s_cselect_b64 s[6:7], s[8:9], s[6:7]
; GFX1030-NEXT: s_add_i32 s8, s6, 1
; GFX1030-NEXT: s_cmp_ge_u32 s7, s2
; GFX1030-NEXT: s_cselect_b32 s2, s8, s6
diff --git a/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll b/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
index 22e4a24435f12..a3a021bdc9a95 100644
--- a/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/umin-sub-to-usubo-select-combine.ll
@@ -214,8 +214,7 @@ define amdgpu_ps i64 @s_underflow_compare_fold_i64(i64 inreg %a, i64 inreg %b) #
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[0:1]
; GFX9-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX9-NEXT: s_cselect_b32 s1, s3, s1
-; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
; GFX9-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_underflow_compare_fold_i64:
@@ -225,8 +224,7 @@ define amdgpu_ps i64 @s_underflow_compare_fold_i64(i64 inreg %a, i64 inreg %b) #
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_cmp_lt_u64_e64 s4, s[2:3], s[0:1]
; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s0, s2, s0
-; GFX11-NEXT: s_cselect_b32 s1, s3, s1
+; GFX11-NEXT: s_cselect_b64 s[0:1], s[2:3], s[0:1]
; GFX11-NEXT: ; return to shader part epilog
%sub = sub i64 %a, %b
%cond = call i64 @llvm.umin.i64(i64 %sub, i64 %a)
More information about the llvm-commits
mailing list