[llvm-branch-commits] [llvm] [SelectionDAG] Don't assume a value from another block is frozen (PR #240477)
Mitch Briles via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Nov 9 13:47:03 PST 2026
https://github.com/MitchBriles updated https://github.com/llvm/llvm-project/pull/240477
>From 321b910ccda2ba98d9f6f67212ab8f2aa1b18826 Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 6 Nov 2026 19:30:21 -0700
Subject: [PATCH 1/2] Precommit cross-block freeze tests
Assisted-By: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/test/CodeGen/AArch64/freeze-multi-use.ll | 23 +++++
llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll | 94 +++++++++++++++++++
llvm/test/CodeGen/RISCV/freeze-multi-use.ll | 32 +++++++
llvm/test/CodeGen/X86/freeze-multi-use.ll | 20 ++++
4 files changed, 169 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
index 0fbf4c61db5bf..aa56cff3d0200 100644
--- a/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AArch64/freeze-multi-use.ll
@@ -128,5 +128,28 @@ define <4 x i32> @vec_across_call() nounwind {
ret <4 x i32> %w
}
+; Both calls must receive the same value.
+define void @other_block(<2 x i32> %v) nounwind {
+; CHECK-LABEL: other_block:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT: // freeze: $w19
+; CHECK-NEXT: mov w0, w19
+; CHECK-NEXT: bl use
+; CHECK-NEXT: mov w0, w19
+; CHECK-NEXT: bl use
+; CHECK-NEXT: ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT: ret
+entry:
+ %u = extractelement <2 x i32> %v, i32 5
+ br label %next
+
+next:
+ %f = freeze i32 %u
+ call void @use(i32 %f)
+ call void @use(i32 %f)
+ ret void
+}
+
declare void @use(i32)
declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
index a0ce9aec62f12..8a0e5b613240b 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze-multi-use.ll
@@ -109,5 +109,99 @@ define <4 x i32> @vec_across_call() nounwind {
ret <4 x i32> %w
}
+; Both calls must receive the same value.
+define void @other_block(<2 x i32> %v) nounwind {
+; CHECK-LABEL: other_block:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s0, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_store_b32 off, v41, s33 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: v_writelane_b32 v41, s0, 17
+; CHECK-NEXT: s_add_i32 s32, s32, 16
+; CHECK-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; CHECK-NEXT: v_writelane_b32 v41, s34, 0
+; CHECK-NEXT: v_writelane_b32 v41, s35, 1
+; CHECK-NEXT: v_writelane_b32 v41, s36, 2
+; CHECK-NEXT: v_writelane_b32 v41, s37, 3
+; CHECK-NEXT: v_writelane_b32 v41, s38, 4
+; CHECK-NEXT: v_writelane_b32 v41, s39, 5
+; CHECK-NEXT: v_writelane_b32 v41, s48, 6
+; CHECK-NEXT: v_writelane_b32 v41, s49, 7
+; CHECK-NEXT: v_writelane_b32 v41, s50, 8
+; CHECK-NEXT: v_writelane_b32 v41, s51, 9
+; CHECK-NEXT: v_writelane_b32 v41, s52, 10
+; CHECK-NEXT: v_writelane_b32 v41, s53, 11
+; CHECK-NEXT: v_writelane_b32 v41, s54, 12
+; CHECK-NEXT: v_writelane_b32 v41, s55, 13
+; CHECK-NEXT: v_writelane_b32 v41, s64, 14
+; CHECK-NEXT: v_writelane_b32 v41, s30, 15
+; CHECK-NEXT: v_writelane_b32 v41, s31, 16
+; CHECK-NEXT: s_getpc_b64 s[0:1]
+; CHECK-NEXT: s_add_u32 s0, s0, use at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s1, s1, use at gotpcrel32@hi+12
+; CHECK-NEXT: ; freeze: $sgpr64
+; CHECK-NEXT: v_mov_b32_e32 v0, s64
+; CHECK-NEXT: s_load_b64 s[54:55], s[0:1], 0x0
+; CHECK-NEXT: v_mov_b32_e32 v40, v31
+; CHECK-NEXT: s_mov_b32 s53, s15
+; CHECK-NEXT: s_mov_b32 s52, s14
+; CHECK-NEXT: s_mov_b32 s51, s13
+; CHECK-NEXT: s_mov_b32 s50, s12
+; CHECK-NEXT: s_mov_b64 s[48:49], s[10:11]
+; CHECK-NEXT: s_mov_b64 s[38:39], s[8:9]
+; CHECK-NEXT: s_mov_b64 s[36:37], s[6:7]
+; CHECK-NEXT: s_mov_b64 s[34:35], s[4:5]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
+; CHECK-NEXT: v_dual_mov_b32 v31, v40 :: v_dual_mov_b32 v0, s64
+; CHECK-NEXT: s_mov_b64 s[4:5], s[34:35]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[36:37]
+; CHECK-NEXT: s_mov_b64 s[8:9], s[38:39]
+; CHECK-NEXT: s_mov_b64 s[10:11], s[48:49]
+; CHECK-NEXT: s_mov_b32 s12, s50
+; CHECK-NEXT: s_mov_b32 s13, s51
+; CHECK-NEXT: s_mov_b32 s14, s52
+; CHECK-NEXT: s_mov_b32 s15, s53
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
+; CHECK-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; CHECK-NEXT: v_readlane_b32 s30, v41, 15
+; CHECK-NEXT: v_readlane_b32 s31, v41, 16
+; CHECK-NEXT: v_readlane_b32 s64, v41, 14
+; CHECK-NEXT: v_readlane_b32 s55, v41, 13
+; CHECK-NEXT: v_readlane_b32 s54, v41, 12
+; CHECK-NEXT: v_readlane_b32 s53, v41, 11
+; CHECK-NEXT: v_readlane_b32 s52, v41, 10
+; CHECK-NEXT: v_readlane_b32 s51, v41, 9
+; CHECK-NEXT: v_readlane_b32 s50, v41, 8
+; CHECK-NEXT: v_readlane_b32 s49, v41, 7
+; CHECK-NEXT: v_readlane_b32 s48, v41, 6
+; CHECK-NEXT: v_readlane_b32 s39, v41, 5
+; CHECK-NEXT: v_readlane_b32 s38, v41, 4
+; CHECK-NEXT: v_readlane_b32 s37, v41, 3
+; CHECK-NEXT: v_readlane_b32 s36, v41, 2
+; CHECK-NEXT: v_readlane_b32 s35, v41, 1
+; CHECK-NEXT: v_readlane_b32 s34, v41, 0
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s0, v41, 17
+; CHECK-NEXT: s_or_saveexec_b32 s1, -1
+; CHECK-NEXT: scratch_load_b32 v41, off, s33 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b32 exec_lo, s1
+; CHECK-NEXT: s_mov_b32 s33, s0
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+entry:
+ %u = extractelement <2 x i32> %v, i32 5
+ br label %next
+
+next:
+ %f = freeze i32 %u
+ call void @use(i32 %f)
+ call void @use(i32 %f)
+ ret void
+}
+
declare void @use(i32)
declare void @usev(<4 x i32>)
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
index 202d37fcfbb59..a5ac217b35e10 100644
--- a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -160,6 +160,38 @@ define i32 @across_call() nounwind {
ret i32 %w
}
+; Both calls must receive the same value.
+define void @other_block(<2 x i32> %v) nounwind {
+; RV32-LABEL: other_block:
+; RV32: # %bb.0: # %entry
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: call use
+; RV32-NEXT: call use
+; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: ret
+;
+; RV64-LABEL: other_block:
+; RV64: # %bb.0: # %entry
+; RV64-NEXT: addi sp, sp, -16
+; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: call use
+; RV64-NEXT: call use
+; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: addi sp, sp, 16
+; RV64-NEXT: ret
+entry:
+ %u = extractelement <2 x i32> %v, i32 5
+ br label %next
+
+next:
+ %f = freeze i32 %u
+ call void @use(i32 %f)
+ call void @use(i32 %f)
+ ret void
+}
+
declare void @use(i32)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
index 07ae3e751c95b..abff065b7bbd1 100644
--- a/llvm/test/CodeGen/X86/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -90,5 +90,25 @@ define <4 x i32> @vec_across_call() nounwind {
ret <4 x i32> %w
}
+; Both calls must receive the same value.
+define void @other_block(<2 x i32> %v) nounwind {
+; CHECK-LABEL: other_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: callq use at PLT
+; CHECK-NEXT: callq use at PLT
+; CHECK-NEXT: popq %rax
+; CHECK-NEXT: retq
+entry:
+ %u = extractelement <2 x i32> %v, i32 5
+ br label %next
+
+next:
+ %f = freeze i32 %u
+ call void @use(i32 %f)
+ call void @use(i32 %f)
+ ret void
+}
+
declare void @use(i32)
declare void @usev(<4 x i32>)
>From 4a98e18ccf523362702d4e21f75d9a290573fc5e Mon Sep 17 00:00:00 2001
From: Mitch <mitchbriles at gmail.com>
Date: Fri, 6 Nov 2026 19:41:41 -0700
Subject: [PATCH 2/2] [SelectionDAG] Don't assume a value from another block is
frozen
A virtual register from another block can hold an undefined
value. We can only treat CopyFromReg of a physical register or of a
live-in virtual register as not undef or poison so that a freeze of
a value from another block is kept and selected as FREEZE.
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 3 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 7 +-
llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll | 13 +-
llvm/test/CodeGen/AArch64/div-i256.ll | 212 ++++++++---------
.../AMDGPU/av-split-dead-valno-crash.ll | 33 +--
llvm/test/CodeGen/AMDGPU/div_i128.ll | 56 +++--
llvm/test/CodeGen/AMDGPU/extract-subvector.ll | 56 ++---
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 56 +++--
.../splitkit-getsubrangeformask-phi-extend.ll | 87 ++++---
llvm/test/CodeGen/PowerPC/p10-spill-creq.ll | 28 +--
llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll | 134 +++++------
.../CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll | 28 +--
.../CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll | 32 +--
llvm/test/CodeGen/RISCV/bitint-fp-conv-200.ll | 88 +++----
llvm/test/CodeGen/RISCV/fpclamptosat.ll | 18 +-
llvm/test/CodeGen/RISCV/freeze-multi-use.ll | 10 +
llvm/test/CodeGen/RISCV/idiv_large.ll | 220 +++++++++---------
llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll | 64 ++---
llvm/test/CodeGen/X86/bittest-big-integer.ll | 183 +++++++--------
llvm/test/CodeGen/X86/freeze-multi-use.ll | 7 +-
llvm/test/CodeGen/X86/sttni.ll | 28 +--
21 files changed, 702 insertions(+), 661 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index d894b3c54dfda..532b39fa61f2b 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -228,7 +228,8 @@ enum NodeType {
/// CopyFromReg - This node indicates that the input value is a virtual or
/// physical register that is defined outside of the scope of this
/// SelectionDAG. The register is available from the RegisterSDNode object.
- /// Note that CopyFromReg is considered as also freezing the value.
+ /// Note that CopyFromReg of a physical register or of a virtual register
+ /// holding a live-in value is considered as also freezing the value.
CopyFromReg,
/// UNDEF - An undefined node.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 8aa132f231a45..2edef29492f68 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5742,9 +5742,14 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
case ISD::VALUETYPE:
case ISD::FrameIndex:
case ISD::TargetFrameIndex:
- case ISD::CopyFromReg:
return true;
+ case ISD::CopyFromReg: {
+ // A virtual register from another block may hold an undefined value.
+ Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
+ return !Reg.isVirtual() || MF->getRegInfo().getLiveInPhysReg(Reg).isValid();
+ }
+
case ISD::POISON:
return !includesPoison(Kind);
diff --git a/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll b/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
index 01bb811051ed5..1898cab6e3591 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll
@@ -207,17 +207,18 @@ define <4 x i32> @test_bit_sink_operand(<4 x i32> %src, <4 x i32> %dst, <4 x i32
; CHECK-SD-NEXT: .LBB11_1: // %do.body
; CHECK-SD-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-SD-NEXT: bit v1.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT: add x10, sp, #16
-; CHECK-SD-NEXT: mov x11, sp
-; CHECK-SD-NEXT: bfi x10, x9, #2, #2
-; CHECK-SD-NEXT: bfi x11, x9, #2, #2
+; CHECK-SD-NEXT: mov w10, w9
+; CHECK-SD-NEXT: add x11, sp, #16
+; CHECK-SD-NEXT: bfi x11, x10, #2, #2
+; CHECK-SD-NEXT: mov x12, sp
; CHECK-SD-NEXT: add w9, w9, #1
+; CHECK-SD-NEXT: bfi x12, x10, #2, #2
; CHECK-SD-NEXT: cmp w9, #5
; CHECK-SD-NEXT: str q1, [sp, #16]
-; CHECK-SD-NEXT: str w0, [x10]
+; CHECK-SD-NEXT: str w0, [x11]
; CHECK-SD-NEXT: ldr q1, [sp, #16]
; CHECK-SD-NEXT: str q0, [sp]
-; CHECK-SD-NEXT: str w8, [x11]
+; CHECK-SD-NEXT: str w8, [x12]
; CHECK-SD-NEXT: ldr q0, [sp]
; CHECK-SD-NEXT: b.ne .LBB11_1
; CHECK-SD-NEXT: // %bb.2: // %do.end
diff --git a/llvm/test/CodeGen/AArch64/div-i256.ll b/llvm/test/CodeGen/AArch64/div-i256.ll
index abf87887166b8..e3d005e6e64af 100644
--- a/llvm/test/CodeGen/AArch64/div-i256.ll
+++ b/llvm/test/CodeGen/AArch64/div-i256.ll
@@ -205,169 +205,169 @@ define i256 @sdiv256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: eor x8, x0, x11
; CHECK-NEXT: eor x9, x1, x11
; CHECK-NEXT: eor x10, x4, x13
-; CHECK-NEXT: subs x0, x8, x11
+; CHECK-NEXT: subs x17, x8, x11
; CHECK-NEXT: eor x8, x2, x11
; CHECK-NEXT: eor x12, x5, x13
-; CHECK-NEXT: sbcs x1, x9, x11
+; CHECK-NEXT: sbcs x18, x9, x11
; CHECK-NEXT: eor x9, x3, x11
-; CHECK-NEXT: clz x14, x0
-; CHECK-NEXT: sbcs x3, x8, x11
-; CHECK-NEXT: clz x15, x1
+; CHECK-NEXT: clz x14, x17
+; CHECK-NEXT: sbcs x2, x8, x11
+; CHECK-NEXT: clz x15, x18
; CHECK-NEXT: add x14, x14, #64
-; CHECK-NEXT: sbc x5, x9, x11
+; CHECK-NEXT: sbc x4, x9, x11
; CHECK-NEXT: subs x8, x10, x13
; CHECK-NEXT: eor x10, x6, x13
; CHECK-NEXT: sbcs x9, x12, x13
; CHECK-NEXT: eor x12, x7, x13
-; CHECK-NEXT: orr x16, x0, x3
+; CHECK-NEXT: orr x16, x17, x2
; CHECK-NEXT: sbcs x10, x10, x13
-; CHECK-NEXT: orr x2, x1, x5
-; CHECK-NEXT: clz x4, x8
+; CHECK-NEXT: orr x3, x18, x4
+; CHECK-NEXT: clz x5, x8
; CHECK-NEXT: sbc x12, x12, x13
-; CHECK-NEXT: orr x17, x8, x10
-; CHECK-NEXT: orr x16, x16, x2
-; CHECK-NEXT: orr x18, x9, x12
-; CHECK-NEXT: add x4, x4, #64
+; CHECK-NEXT: orr x0, x8, x10
+; CHECK-NEXT: orr x16, x16, x3
+; CHECK-NEXT: orr x1, x9, x12
+; CHECK-NEXT: add x5, x5, #64
; CHECK-NEXT: orr x6, x10, x12
-; CHECK-NEXT: orr x17, x17, x18
-; CHECK-NEXT: clz x18, x10
+; CHECK-NEXT: orr x0, x0, x1
+; CHECK-NEXT: clz x1, x10
; CHECK-NEXT: eor x11, x13, x11
-; CHECK-NEXT: cmp x17, #0
-; CHECK-NEXT: add x18, x18, #64
-; CHECK-NEXT: clz x17, x9
+; CHECK-NEXT: cmp x0, #0
+; CHECK-NEXT: add x1, x1, #64
+; CHECK-NEXT: clz x0, x9
; CHECK-NEXT: ccmp x16, #0, #4, ne
; CHECK-NEXT: clz x16, x12
-; CHECK-NEXT: cset w2, eq
+; CHECK-NEXT: cset w3, eq
; CHECK-NEXT: cmp x12, #0
-; CHECK-NEXT: csel x16, x16, x18, ne
+; CHECK-NEXT: csel x16, x16, x1, ne
; CHECK-NEXT: cmp x9, #0
-; CHECK-NEXT: clz x18, x3
-; CHECK-NEXT: csel x17, x17, x4, ne
+; CHECK-NEXT: clz x1, x2
+; CHECK-NEXT: csel x0, x0, x5, ne
; CHECK-NEXT: cmp x6, #0
-; CHECK-NEXT: add x18, x18, #64
-; CHECK-NEXT: add x17, x17, #128
-; CHECK-NEXT: clz x4, x5
-; CHECK-NEXT: csel x16, x16, x17, ne
-; CHECK-NEXT: cmp x5, #0
-; CHECK-NEXT: csel x17, x4, x18, ne
-; CHECK-NEXT: cmp x1, #0
-; CHECK-NEXT: orr x18, x3, x5
-; CHECK-NEXT: csel x14, x15, x14, ne
+; CHECK-NEXT: add x1, x1, #64
+; CHECK-NEXT: add x0, x0, #128
+; CHECK-NEXT: clz x5, x4
+; CHECK-NEXT: csel x16, x16, x0, ne
+; CHECK-NEXT: cmp x4, #0
+; CHECK-NEXT: csel x0, x5, x1, ne
; CHECK-NEXT: cmp x18, #0
-; CHECK-NEXT: mov w18, #255 // =0xff
+; CHECK-NEXT: orr x1, x2, x4
+; CHECK-NEXT: csel x14, x15, x14, ne
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: mov w1, #255 // =0xff
; CHECK-NEXT: add x14, x14, #128
-; CHECK-NEXT: csel x14, x17, x14, ne
-; CHECK-NEXT: subs x17, x16, x14
+; CHECK-NEXT: csel x14, x0, x14, ne
+; CHECK-NEXT: subs x0, x16, x14
; CHECK-NEXT: ngcs x14, xzr
; CHECK-NEXT: ngcs x16, xzr
; CHECK-NEXT: ngc x15, xzr
-; CHECK-NEXT: cmp x18, x17
+; CHECK-NEXT: cmp x1, x0
; CHECK-NEXT: ngcs xzr, x14
; CHECK-NEXT: ngcs xzr, x16
; CHECK-NEXT: ngcs xzr, x15
-; CHECK-NEXT: csinc w4, w2, wzr, hs
-; CHECK-NEXT: cmp w4, #0
-; CHECK-NEXT: csel x13, xzr, x5, ne
-; CHECK-NEXT: csel x2, xzr, x3, ne
-; CHECK-NEXT: csel x7, xzr, x1, ne
-; CHECK-NEXT: csel x18, xzr, x0, ne
-; CHECK-NEXT: tbnz w4, #0, .LBB1_6
+; CHECK-NEXT: csinc w5, w3, wzr, hs
+; CHECK-NEXT: cmp w5, #0
+; CHECK-NEXT: csel x13, xzr, x4, ne
+; CHECK-NEXT: csel x3, xzr, x2, ne
+; CHECK-NEXT: csel x7, xzr, x18, ne
+; CHECK-NEXT: csel x1, xzr, x17, ne
+; CHECK-NEXT: tbnz w5, #0, .LBB1_6
; CHECK-NEXT: // %bb.1: // %_udiv-special-cases
-; CHECK-NEXT: eor x4, x17, #0xff
+; CHECK-NEXT: eor x5, x0, #0xff
; CHECK-NEXT: orr x6, x14, x15
-; CHECK-NEXT: orr x4, x4, x16
-; CHECK-NEXT: orr x4, x4, x6
-; CHECK-NEXT: cbz x4, .LBB1_6
+; CHECK-NEXT: orr x5, x5, x16
+; CHECK-NEXT: orr x5, x5, x6
+; CHECK-NEXT: cbz x5, .LBB1_6
; CHECK-NEXT: // %bb.2: // %udiv-bb1
; CHECK-NEXT: sub sp, sp, #208
; CHECK-NEXT: mov w13, #255 // =0xff
; CHECK-NEXT: movi v0.2d, #0000000000000000
; CHECK-NEXT: stp x28, x27, [sp, #128] // 16-byte Folded Spill
-; CHECK-NEXT: sub x18, x13, x17
+; CHECK-NEXT: sub x1, x13, x0
; CHECK-NEXT: add x13, sp, #64
; CHECK-NEXT: stp x26, x25, [sp, #144] // 16-byte Folded Spill
-; CHECK-NEXT: ubfx x2, x18, #6, #2
+; CHECK-NEXT: ubfx x3, x1, #6, #2
; CHECK-NEXT: add x13, x13, #32
; CHECK-NEXT: stp x24, x23, [sp, #160] // 16-byte Folded Spill
; CHECK-NEXT: stp x22, x21, [sp, #176] // 16-byte Folded Spill
-; CHECK-NEXT: sub x2, x13, x2, lsl #3
+; CHECK-NEXT: sub x3, x13, x3, lsl #3
; CHECK-NEXT: stp x20, x19, [sp, #192] // 16-byte Folded Spill
-; CHECK-NEXT: adds x13, x17, #1
-; CHECK-NEXT: stp x0, x1, [sp, #96]
-; CHECK-NEXT: and x19, x18, #0x3f
+; CHECK-NEXT: adds x13, x0, #1
+; CHECK-NEXT: stp x17, x18, [sp, #96]
+; CHECK-NEXT: and x19, x1, #0x3f
; CHECK-NEXT: adcs x14, x14, xzr
-; CHECK-NEXT: stp x3, x5, [sp, #112]
-; CHECK-NEXT: mvn w20, w18
+; CHECK-NEXT: stp x2, x4, [sp, #112]
+; CHECK-NEXT: mvn w20, w1
; CHECK-NEXT: eor x19, x19, #0x3f
; CHECK-NEXT: stp q0, q0, [sp, #64]
; CHECK-NEXT: adcs x16, x16, xzr
-; CHECK-NEXT: ldp x17, x6, [x2, #8]
-; CHECK-NEXT: ldr x7, [x2]
-; CHECK-NEXT: ldr x4, [x2, #24]
+; CHECK-NEXT: ldp x0, x6, [x3, #8]
+; CHECK-NEXT: ldr x7, [x3]
+; CHECK-NEXT: ldr x5, [x3, #24]
; CHECK-NEXT: lsr x22, x7, #1
-; CHECK-NEXT: lsr x2, x17, #1
+; CHECK-NEXT: lsr x3, x0, #1
; CHECK-NEXT: lsr x21, x6, #1
-; CHECK-NEXT: lsl x4, x4, x18
-; CHECK-NEXT: lsl x6, x6, x18
-; CHECK-NEXT: lsl x17, x17, x18
-; CHECK-NEXT: lsr x20, x2, x20
+; CHECK-NEXT: lsl x5, x5, x1
+; CHECK-NEXT: lsl x6, x6, x1
+; CHECK-NEXT: lsl x0, x0, x1
+; CHECK-NEXT: lsr x20, x3, x20
; CHECK-NEXT: lsr x21, x21, x19
; CHECK-NEXT: lsr x19, x22, x19
-; CHECK-NEXT: adcs x2, x15, xzr
-; CHECK-NEXT: lsl x15, x7, x18
+; CHECK-NEXT: adcs x3, x15, xzr
+; CHECK-NEXT: lsl x15, x7, x1
; CHECK-NEXT: cset w22, hs
-; CHECK-NEXT: orr x4, x4, x21
-; CHECK-NEXT: orr x18, x6, x20
-; CHECK-NEXT: orr x17, x17, x19
+; CHECK-NEXT: orr x5, x5, x21
+; CHECK-NEXT: orr x1, x6, x20
+; CHECK-NEXT: orr x0, x0, x19
; CHECK-NEXT: mov x6, xzr
; CHECK-NEXT: tbnz w22, #0, .LBB1_5
; CHECK-NEXT: // %bb.3: // %udiv-preheader
; CHECK-NEXT: ubfx x21, x13, #6, #2
-; CHECK-NEXT: stp x0, x1, [sp]
-; CHECK-NEXT: mov x0, sp
+; CHECK-NEXT: stp x17, x18, [sp]
+; CHECK-NEXT: mov x17, sp
; CHECK-NEXT: stp q0, q0, [sp, #32]
; CHECK-NEXT: mov x19, xzr
; CHECK-NEXT: mov x20, xzr
-; CHECK-NEXT: add x0, x0, x21, lsl #3
-; CHECK-NEXT: stp x3, x5, [sp, #16]
-; CHECK-NEXT: and x5, x13, #0x3f
+; CHECK-NEXT: add x17, x17, x21, lsl #3
+; CHECK-NEXT: stp x2, x4, [sp, #16]
+; CHECK-NEXT: and x4, x13, #0x3f
; CHECK-NEXT: mvn w21, w13
-; CHECK-NEXT: eor x5, x5, #0x3f
+; CHECK-NEXT: eor x4, x4, #0x3f
; CHECK-NEXT: mov x7, xzr
-; CHECK-NEXT: ldp x1, x3, [x0, #16]
-; CHECK-NEXT: ldp x25, x22, [x0]
-; CHECK-NEXT: subs x0, x8, #1
-; CHECK-NEXT: lsl x23, x3, #1
-; CHECK-NEXT: lsl x24, x1, #1
-; CHECK-NEXT: lsr x26, x1, x13
-; CHECK-NEXT: lsl x1, x22, #1
+; CHECK-NEXT: ldp x18, x2, [x17, #16]
+; CHECK-NEXT: ldp x25, x22, [x17]
+; CHECK-NEXT: subs x17, x8, #1
+; CHECK-NEXT: lsl x23, x2, #1
+; CHECK-NEXT: lsl x24, x18, #1
+; CHECK-NEXT: lsr x26, x18, x13
+; CHECK-NEXT: lsl x18, x22, #1
; CHECK-NEXT: lsr x27, x22, x13
; CHECK-NEXT: lsr x25, x25, x13
-; CHECK-NEXT: lsl x23, x23, x5
+; CHECK-NEXT: lsl x23, x23, x4
; CHECK-NEXT: lsl x24, x24, x21
-; CHECK-NEXT: lsr x22, x3, x13
-; CHECK-NEXT: lsl x5, x1, x5
-; CHECK-NEXT: sbcs x1, x9, xzr
-; CHECK-NEXT: sbcs x3, x10, xzr
+; CHECK-NEXT: lsr x22, x2, x13
+; CHECK-NEXT: lsl x4, x18, x4
+; CHECK-NEXT: sbcs x18, x9, xzr
+; CHECK-NEXT: sbcs x2, x10, xzr
; CHECK-NEXT: orr x21, x23, x26
; CHECK-NEXT: orr x24, x27, x24
-; CHECK-NEXT: orr x23, x5, x25
-; CHECK-NEXT: sbc x5, x12, xzr
+; CHECK-NEXT: orr x23, x4, x25
+; CHECK-NEXT: sbc x4, x12, xzr
; CHECK-NEXT: .LBB1_4: // %udiv-do-while
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: extr x25, x23, x4, #63
+; CHECK-NEXT: extr x25, x23, x5, #63
; CHECK-NEXT: extr x26, x24, x23, #63
; CHECK-NEXT: extr x27, x21, x24, #63
; CHECK-NEXT: extr x22, x22, x21, #63
-; CHECK-NEXT: extr x4, x4, x18, #63
-; CHECK-NEXT: extr x18, x18, x17, #63
-; CHECK-NEXT: cmp x0, x25
-; CHECK-NEXT: sbcs xzr, x1, x26
-; CHECK-NEXT: orr x18, x20, x18
-; CHECK-NEXT: orr x4, x7, x4
-; CHECK-NEXT: sbcs xzr, x3, x27
+; CHECK-NEXT: extr x5, x5, x1, #63
+; CHECK-NEXT: extr x1, x1, x0, #63
+; CHECK-NEXT: cmp x17, x25
+; CHECK-NEXT: sbcs xzr, x18, x26
+; CHECK-NEXT: orr x1, x20, x1
+; CHECK-NEXT: orr x5, x7, x5
+; CHECK-NEXT: sbcs xzr, x2, x27
; CHECK-NEXT: mov x7, xzr
-; CHECK-NEXT: sbc x21, x5, x22
+; CHECK-NEXT: sbc x21, x4, x22
; CHECK-NEXT: asr x28, x21, #63
; CHECK-NEXT: and x21, x28, x8
; CHECK-NEXT: subs x23, x25, x21
@@ -378,35 +378,35 @@ define i256 @sdiv256(i256 %a, i256 %b) nounwind {
; CHECK-NEXT: sbcs x21, x27, x21
; CHECK-NEXT: sbc x22, x22, x25
; CHECK-NEXT: subs x13, x13, #1
-; CHECK-NEXT: extr x25, x17, x15, #63
+; CHECK-NEXT: extr x25, x0, x15, #63
; CHECK-NEXT: sbcs x14, x14, xzr
; CHECK-NEXT: orr x15, x6, x15, lsl #1
; CHECK-NEXT: and x6, x28, #0x1
; CHECK-NEXT: sbcs x16, x16, xzr
-; CHECK-NEXT: orr x17, x19, x25
-; CHECK-NEXT: sbc x2, x2, xzr
+; CHECK-NEXT: orr x0, x19, x25
+; CHECK-NEXT: sbc x3, x3, xzr
; CHECK-NEXT: orr x20, x13, x16
-; CHECK-NEXT: orr x19, x14, x2
+; CHECK-NEXT: orr x19, x14, x3
; CHECK-NEXT: orr x25, x20, x19
; CHECK-NEXT: mov x19, xzr
; CHECK-NEXT: mov x20, xzr
; CHECK-NEXT: cbnz x25, .LBB1_4
; CHECK-NEXT: .LBB1_5: // %udiv-loop-exit
; CHECK-NEXT: ldp x20, x19, [sp, #192] // 16-byte Folded Reload
-; CHECK-NEXT: extr x7, x17, x15, #63
+; CHECK-NEXT: extr x7, x0, x15, #63
; CHECK-NEXT: ldp x22, x21, [sp, #176] // 16-byte Folded Reload
-; CHECK-NEXT: extr x2, x18, x17, #63
+; CHECK-NEXT: extr x3, x1, x0, #63
; CHECK-NEXT: ldp x24, x23, [sp, #160] // 16-byte Folded Reload
-; CHECK-NEXT: extr x13, x4, x18, #63
+; CHECK-NEXT: extr x13, x5, x1, #63
; CHECK-NEXT: ldp x26, x25, [sp, #144] // 16-byte Folded Reload
-; CHECK-NEXT: orr x18, x6, x15, lsl #1
+; CHECK-NEXT: orr x1, x6, x15, lsl #1
; CHECK-NEXT: ldp x28, x27, [sp, #128] // 16-byte Folded Reload
; CHECK-NEXT: add sp, sp, #208
; CHECK-NEXT: .LBB1_6: // %udiv-end
-; CHECK-NEXT: eor x8, x18, x11
+; CHECK-NEXT: eor x8, x1, x11
; CHECK-NEXT: eor x9, x7, x11
; CHECK-NEXT: subs x0, x8, x11
-; CHECK-NEXT: eor x8, x2, x11
+; CHECK-NEXT: eor x8, x3, x11
; CHECK-NEXT: sbcs x1, x9, x11
; CHECK-NEXT: eor x9, x13, x11
; CHECK-NEXT: sbcs x2, x8, x11
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index aecf913ee672e..b3d4eeed0c6c4 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -8,13 +8,13 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
; CHECK-NEXT: s_mov_b64 s[16:17], 0
-; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
+; CHECK-NEXT: v_mov_b32_e32 v28, 0x9037ab78
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
-; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
+; CHECK-NEXT: v_mov_b32_e32 v29, 0x3e21eeb6
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
; CHECK-NEXT: v_mov_b32_e32 v4, 0x19f4ec90
@@ -39,13 +39,11 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b64_e32 v[20:21], s[10:11]
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
-; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
+; CHECK-NEXT: ; implicit-def: $vgpr30_vgpr31
; CHECK-NEXT: ; implicit-def: $sgpr20_sgpr21
; CHECK-NEXT: s_branch .LBB0_3
; CHECK-NEXT: .LBB0_1: ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[12:13]
-; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], s[12:13]
; CHECK-NEXT: s_mov_b64 s[6:7], s[20:21]
; CHECK-NEXT: s_mov_b64 s[16:17], 0
; CHECK-NEXT: .LBB0_2: ; %._crit_edge2105.i.i.i2330
@@ -68,7 +66,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[24:25], v[30:31]
+; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[24:25], v[28:29]
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[2:3]
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[4:5]
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[6:7]
@@ -77,24 +75,25 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[12:13]
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[14:15]
; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[16:17]
-; CHECK-NEXT: v_fma_f64 v[28:29], 0, v[26:27], v[18:19]
+; CHECK-NEXT: v_fma_f64 v[26:27], 0, v[26:27], v[18:19]
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_3 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
+; CHECK-NEXT: v_accvgpr_mov_b32 a0, a2
+; CHECK-NEXT: v_accvgpr_mov_b32 a1, a3
; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
; CHECK-NEXT: s_cbranch_vccz .LBB0_10
; CHECK-NEXT: ; %bb.7: ; %.lr.ph2070.i.i.i3291
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a2, a0
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v30
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v31
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_6
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v26
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v27
; CHECK-NEXT: s_mov_b64 vcc, s[4:5]
; CHECK-NEXT: s_cbranch_vccz .LBB0_6
; CHECK-NEXT: ; %bb.9: ; %loop.exit.guard
@@ -106,9 +105,11 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_execz .LBB0_12
; CHECK-NEXT: .LBB0_11: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a0
; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
-; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], v[24:25]
+; CHECK-NEXT: s_nop 0
; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
; CHECK-NEXT: v_mov_b32_e32 v27, v26
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index dd0bc17083dcb..42b77b73d9fe7 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -891,17 +891,20 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v18
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
@@ -910,9 +913,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_mov_b32 s10, s5
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s10
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
@@ -1048,17 +1052,20 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v3, v2, s10
; GFX9-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
+; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[12:13]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
@@ -1066,8 +1073,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
@@ -2994,17 +3002,20 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v18
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
@@ -3013,9 +3024,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_mov_b32 s10, s5
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s10
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
@@ -3151,17 +3163,20 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v3, v2, s10
; GFX9-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
+; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[12:13]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
@@ -3169,8 +3184,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
index cd58a256b5101..acac98c8fd2f5 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector.ll
@@ -113,9 +113,9 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[2:3], s[8:11], 0 addr64 offset:16 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[2:3], s[8:11], 0 addr64 offset:32 glc
+; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[2:3], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[2:3], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[2:3], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB1_2: ; %Flow
@@ -130,9 +130,9 @@ define <2 x i64> @extract_2xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:32 glc
+; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB1_4: ; %exit
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -183,7 +183,7 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[2:3], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[2:3], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[2:3], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB2_2: ; %Flow
@@ -200,7 +200,7 @@ define <4 x i64> @extract_4xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB2_4: ; %exit
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -251,13 +251,13 @@ define <8 x i64> @extract_8xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_mov_b32 s11, 0xf000
; GCN-NEXT: s_mov_b32 s8, s10
; GCN-NEXT: s_mov_b32 s9, s10
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:112 glc
+; GCN-NEXT: buffer_load_dwordx4 v[32:35], v[2:3], s[8:11], 0 addr64 offset:112 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:96 glc
+; GCN-NEXT: buffer_load_dwordx4 v[28:31], v[2:3], s[8:11], 0 addr64 offset:96 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:80 glc
+; GCN-NEXT: buffer_load_dwordx4 v[24:27], v[2:3], s[8:11], 0 addr64 offset:80 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:64 glc
+; GCN-NEXT: buffer_load_dwordx4 v[20:23], v[2:3], s[8:11], 0 addr64 offset:64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -276,13 +276,13 @@ define <8 x i64> @extract_8xi64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i1 %
; GCN-NEXT: s_mov_b32 s11, 0xf000
; GCN-NEXT: s_mov_b32 s8, s10
; GCN-NEXT: s_mov_b32 s9, s10
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:112 glc
+; GCN-NEXT: buffer_load_dwordx4 v[32:35], v[0:1], s[8:11], 0 addr64 offset:112 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:96 glc
+; GCN-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[8:11], 0 addr64 offset:96 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:80 glc
+; GCN-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[8:11], 0 addr64 offset:80 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:64 glc
+; GCN-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[8:11], 0 addr64 offset:64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -357,9 +357,9 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[2:3], s[8:11], 0 addr64 offset:16 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[2:3], s[8:11], 0 addr64 offset:32 glc
+; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[2:3], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[2:3], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[2:3], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB4_2: ; %Flow
@@ -374,9 +374,9 @@ define <2 x double> @extract_2xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:16 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[8:11], 0 addr64 offset:32 glc
+; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB4_4: ; %exit
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -427,7 +427,7 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[2:3], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[2:3], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[2:3], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB5_2: ; %Flow
@@ -444,7 +444,7 @@ define <4 x double> @extract_4xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[12:15], v[0:1], s[8:11], 0 addr64 offset:32 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64 offset:48 glc
+; GCN-NEXT: buffer_load_dwordx4 v[16:19], v[0:1], s[8:11], 0 addr64 offset:48 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB5_4: ; %exit
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
@@ -495,13 +495,13 @@ define <8 x double> @extract_8xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_mov_b32 s11, 0xf000
; GCN-NEXT: s_mov_b32 s8, s10
; GCN-NEXT: s_mov_b32 s9, s10
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:112 glc
+; GCN-NEXT: buffer_load_dwordx4 v[32:35], v[2:3], s[8:11], 0 addr64 offset:112 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:96 glc
+; GCN-NEXT: buffer_load_dwordx4 v[28:31], v[2:3], s[8:11], 0 addr64 offset:96 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:80 glc
+; GCN-NEXT: buffer_load_dwordx4 v[24:27], v[2:3], s[8:11], 0 addr64 offset:80 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 offset:64 glc
+; GCN-NEXT: buffer_load_dwordx4 v[20:23], v[2:3], s[8:11], 0 addr64 offset:64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[2:3], s[8:11], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -520,13 +520,13 @@ define <8 x double> @extract_8xf64(ptr addrspace(1) %p0, ptr addrspace(1) %p1, i
; GCN-NEXT: s_mov_b32 s11, 0xf000
; GCN-NEXT: s_mov_b32 s8, s10
; GCN-NEXT: s_mov_b32 s9, s10
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:112 glc
+; GCN-NEXT: buffer_load_dwordx4 v[32:35], v[0:1], s[8:11], 0 addr64 offset:112 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:96 glc
+; GCN-NEXT: buffer_load_dwordx4 v[28:31], v[0:1], s[8:11], 0 addr64 offset:96 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:80 glc
+; GCN-NEXT: buffer_load_dwordx4 v[24:27], v[0:1], s[8:11], 0 addr64 offset:80 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[8:11], 0 addr64 offset:64 glc
+; GCN-NEXT: buffer_load_dwordx4 v[20:23], v[0:1], s[8:11], 0 addr64 offset:64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index c80c06ac691f1..2d5784179e5d9 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -934,17 +934,20 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v18
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
@@ -953,9 +956,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_mov_b32 s10, s5
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s10
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
@@ -1091,17 +1095,20 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v3, v2, s10
; GFX9-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
+; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[12:13]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
@@ -1109,8 +1116,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
@@ -2274,17 +2282,20 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v20
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[6:7]
+; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[6:7]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v4, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v20, v19
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v20, s[8:9]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v18
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
@@ -2293,9 +2304,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
+; GFX9-O0-NEXT: s_mov_b32 s10, s5
+; GFX9-O0-NEXT: s_mov_b64 s[8:9], s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s10
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[8:9]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
@@ -2431,17 +2443,20 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_sub_u32_e64 v3, v2, s10
; GFX9-O0-NEXT: v_lshlrev_b64 v[12:13], v3, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v13
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[10:11]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v12
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[4:5]
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: s_mov_b32 s10, 0
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[10:11], v2, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v11
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[10:11]
+; GFX9-O0-NEXT: s_mov_b64 s[12:13], s[10:11]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v3, v3, v12, s[12:13]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v10
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[10:11]
@@ -2449,8 +2464,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: v_lshlrev_b64 v[6:7], v2, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v7
+; GFX9-O0-NEXT: s_mov_b64 s[10:11], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s9
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[4:5]
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[10:11]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v3, v6, s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
index c2ee66f99b598..6241f9d075f74 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask-phi-extend.ll
@@ -106,10 +106,10 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword a17, off, s[0:3], s32 offset:300
; CHECK-NEXT: buffer_load_dword a14, off, s[0:3], s32 offset:296
; CHECK-NEXT: buffer_load_dword a15, off, s[0:3], s32 offset:292
-; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:288
-; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:284
-; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:280
-; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:276
+; CHECK-NEXT: buffer_load_dword a20, off, s[0:3], s32 offset:288
+; CHECK-NEXT: buffer_load_dword a21, off, s[0:3], s32 offset:284
+; CHECK-NEXT: buffer_load_dword a18, off, s[0:3], s32 offset:280
+; CHECK-NEXT: buffer_load_dword a19, off, s[0:3], s32 offset:276
; CHECK-NEXT: buffer_load_dword a22, off, s[0:3], s32 offset:272
; CHECK-NEXT: buffer_load_dword a23, off, s[0:3], s32 offset:268
; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:264
@@ -169,7 +169,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_and_b32_e32 v29, 1, v31
; CHECK-NEXT: v_and_b32_e32 v31, 1, v32
; CHECK-NEXT: v_cmp_eq_u32_e64 s[40:41], 1, v21
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v23
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v23
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword a30, off, s[0:3], s32 offset:232
; CHECK-NEXT: buffer_load_dword a31, off, s[0:3], s32 offset:228
@@ -186,15 +186,15 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: v_and_b32_e32 v27, 1, v38
; CHECK-NEXT: v_and_b32_e32 v29, 1, v39
; CHECK-NEXT: v_and_b32_e32 v31, 1, v48
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v21
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v21
; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: v_and_b32_e32 v21, 1, v50
; CHECK-NEXT: v_cmp_eq_u32_e64 s[44:45], 1, v23
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v25
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v25
; CHECK-NEXT: v_cmp_eq_u32_e64 s[42:43], 1, v27
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[18:19], 1, v29
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v31
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v21
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v29
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[26:27], 1, v31
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[28:29], 1, v21
; CHECK-NEXT: buffer_load_dword v48, off, s[0:3], s32 offset:376
; CHECK-NEXT: buffer_load_dword v49, off, s[0:3], s32 offset:372
; CHECK-NEXT: buffer_load_dword v47, off, s[0:3], s32 offset:368
@@ -203,7 +203,7 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v45, off, s[0:3], s32 offset:356
; CHECK-NEXT: buffer_load_dword v57, off, s[0:3], s32 offset:352
; CHECK-NEXT: buffer_load_dword v58, off, s[0:3], s32 offset:348
-; CHECK-NEXT: v_cmp_eq_u32_e64 s[24:25], 1, v19
+; CHECK-NEXT: v_cmp_eq_u32_e64 s[22:23], 1, v19
; CHECK-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:784
; CHECK-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:780
; CHECK-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:788
@@ -261,16 +261,16 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:160
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:156
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:932 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:152
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:148
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:936 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:144
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:948 ; 4-byte Folded Spill
@@ -998,10 +998,10 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: ; implicit-def: $agpr17
; CHECK-NEXT: ; implicit-def: $agpr14
; CHECK-NEXT: ; implicit-def: $agpr15
-; CHECK-NEXT: ; implicit-def: $agpr18
; CHECK-NEXT: ; implicit-def: $agpr20
-; CHECK-NEXT: ; implicit-def: $agpr19
; CHECK-NEXT: ; implicit-def: $agpr21
+; CHECK-NEXT: ; implicit-def: $agpr18
+; CHECK-NEXT: ; implicit-def: $agpr19
; CHECK-NEXT: ; implicit-def: $agpr22
; CHECK-NEXT: ; implicit-def: $agpr23
; CHECK-NEXT: ; implicit-def: $agpr25
@@ -1184,18 +1184,21 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: s_cbranch_execz .LBB0_49
; CHECK-NEXT: ; %bb.48: ; %bb1a
; CHECK-NEXT: v_accvgpr_read_b32 v0, a29
-; CHECK-NEXT: v_cndmask_b32_e64 v4, v41, v0, s[24:25]
+; CHECK-NEXT: v_cndmask_b32_e64 v4, v41, v0, s[22:23]
; CHECK-NEXT: v_accvgpr_read_b32 v0, a25
-; CHECK-NEXT: v_cndmask_b32_e64 v5, v55, v7, s[24:25]
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v55, v7, s[22:23]
; CHECK-NEXT: v_cndmask_b32_e64 v7, v53, v6, s[4:5]
; CHECK-NEXT: v_cndmask_b32_e64 v6, v54, v0, s[4:5]
; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v9, v47, v0, s[16:17]
+; CHECK-NEXT: v_cndmask_b32_e64 v9, v47, v0, s[18:19]
; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v8, v56, v0, s[16:17]
+; CHECK-NEXT: v_cndmask_b32_e64 v8, v56, v0, s[18:19]
; CHECK-NEXT: v_accvgpr_read_b32 v0, a26
; CHECK-NEXT: v_cndmask_b32_e64 v11, v48, v0, s[12:13]
; CHECK-NEXT: v_accvgpr_read_b32 v0, a27
@@ -1248,43 +1251,40 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a12
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v27, v0, v1, s[26:27]
+; CHECK-NEXT: v_cndmask_b32_e64 v27, v0, v1, s[24:25]
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a13
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v26, v0, v1, s[26:27]
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a18
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v29, v0, v1, s[28:29]
+; CHECK-NEXT: v_cndmask_b32_e64 v26, v0, v1, s[24:25]
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:932 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a20
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v28, v0, v1, s[28:29]
+; CHECK-NEXT: v_cndmask_b32_e64 v29, v0, v1, s[26:27]
+; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
+; CHECK-NEXT: v_accvgpr_read_b32 v0, a21
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: v_cndmask_b32_e64 v28, v0, v1, s[26:27]
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a14
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v31, v0, v1, s[18:19]
+; CHECK-NEXT: v_cndmask_b32_e64 v31, v0, v1, s[20:21]
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a15
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v30, v0, v1, s[18:19]
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a19
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v39, v0, v1, s[20:21]
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:936 ; 4-byte Folded Reload
-; CHECK-NEXT: v_accvgpr_read_b32 v0, a21
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v38, v0, v1, s[20:21]
+; CHECK-NEXT: v_cndmask_b32_e64 v30, v0, v1, s[20:21]
; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:948 ; 4-byte Folded Reload
; CHECK-NEXT: v_accvgpr_read_b32 v0, a22
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v37, v0, v1, s[22:23]
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:952 ; 4-byte Folded Reload
+; CHECK-NEXT: v_cndmask_b32_e64 v1, v0, v1, s[28:29]
; CHECK-NEXT: v_accvgpr_read_b32 v0, a23
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_cndmask_b32_e64 v36, v0, v1, s[22:23]
+; CHECK-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[28:29]
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a18
+; CHECK-NEXT: v_cndmask_b32_e64 v3, v2, v3, s[16:17]
+; CHECK-NEXT: v_accvgpr_read_b32 v2, a19
+; CHECK-NEXT: v_cndmask_b32_e64 v2, v2, v32, s[16:17]
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[0:3]
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[28:31] offset:16
+; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[24:27] offset:32
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 48, v34
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
@@ -1293,9 +1293,6 @@ define void @f(ptr %p, <16 x i1> %m, <16 x i63> %pt, <16 x i1> %sc,
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[8:11] offset:48
; CHECK-NEXT: v_add_co_u32_e32 v0, vcc, 0x70, v34
; CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v35, vcc
-; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[36:39]
-; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[28:31] offset:16
-; CHECK-NEXT: flat_store_dwordx4 v[34:35], v[24:27] offset:32
; CHECK-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
; CHECK-NEXT: .LBB0_49: ; %UnifiedReturnBlock
; CHECK-NEXT: s_or_b64 exec, exec, s[56:57]
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll b/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
index 5a4020022f77a..6bb2a9c793466 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-creq.ll
@@ -80,18 +80,17 @@ define dso_local double @P10_Spill_CR_EQ(ptr %arg) local_unnamed_addr #0 {
; CHECK-NEXT: # %bb.13: # %bb48
; CHECK-NEXT: ld r6, 0(r3)
; CHECK-NEXT: .LBB0_14: # %bb50
-; CHECK-NEXT: mfocrf r7, 1
; CHECK-NEXT: cmpwi cr3, r5, -1
; CHECK-NEXT: crand 4*cr6+un, 4*cr2+lt, 4*cr6+gt
; CHECK-NEXT: # implicit-def: $r5
-; CHECK-NEXT: rotlwi r7, r7, 28
-; CHECK-NEXT: stw r7, -4(r1)
; CHECK-NEXT: bc 4, 4*cr6+lt, .LBB0_16
; CHECK-NEXT: # %bb.15: # %bb52
; CHECK-NEXT: lwz r5, 0(r3)
; CHECK-NEXT: .LBB0_16: # %bb54
-; CHECK-NEXT: crmove 4*cr7+eq, 4*cr1+eq
+; CHECK-NEXT: mfocrf r7, 64
; CHECK-NEXT: crmove 4*cr1+eq, eq
+; CHECK-NEXT: rotlwi r7, r7, 4
+; CHECK-NEXT: stw r7, -4(r1)
; CHECK-NEXT: # implicit-def: $r7
; CHECK-NEXT: bc 4, 4*cr6+un, .LBB0_18
; CHECK-NEXT: # %bb.17: # %bb56
@@ -111,23 +110,24 @@ define dso_local double @P10_Spill_CR_EQ(ptr %arg) local_unnamed_addr #0 {
; CHECK-NEXT: ld r5, 0(r3)
; CHECK-NEXT: .LBB0_20: # %bb70
; CHECK-NEXT: ld r6, 0(r3)
-; CHECK-NEXT: lwz r7, -4(r1)
; CHECK-NEXT: crandc 4*cr5+gt, 4*cr5+gt, 4*cr6+eq
; CHECK-NEXT: crandc 4*cr6+gt, 4*cr6+gt, 4*cr2+eq
+; CHECK-NEXT: crandc 4*cr5+lt, 4*cr5+lt, 4*cr7+eq
; CHECK-NEXT: isel r3, r3, r5, 4*cr5+gt
; CHECK-NEXT: crnor 4*cr5+gt, 4*cr6+lt, 4*cr5+gt
-; CHECK-NEXT: crnor 4*cr6+lt, 4*cr6+un, 4*cr6+gt
-; CHECK-NEXT: lwz r12, 8(r1)
+; CHECK-NEXT: cror 4*cr6+lt, 4*cr6+un, 4*cr6+gt
+; CHECK-NEXT: cror 4*cr5+lt, 4*cr5+un, 4*cr5+lt
+; CHECK-NEXT: li r5, 0
+; CHECK-NEXT: isel r4, r4, r5, 4*cr5+lt
; CHECK-NEXT: isel r3, 0, r3, 4*cr5+gt
-; CHECK-NEXT: mtocrf 128, r7
-; CHECK-NEXT: isel r5, 0, r6, 4*cr6+lt
+; CHECK-NEXT: isel r6, r6, r5, 4*cr6+lt
+; CHECK-NEXT: lwz r5, -4(r1)
; CHECK-NEXT: isel r3, 0, r3, 4*cr1+eq
-; CHECK-NEXT: mtcrf 56, r12
-; CHECK-NEXT: crandc 4*cr5+lt, 4*cr5+lt, eq
-; CHECK-NEXT: isel r5, 0, r5, 4*cr7+eq
-; CHECK-NEXT: crnor 4*cr5+lt, 4*cr5+un, 4*cr5+lt
-; CHECK-NEXT: isel r4, 0, r4, 4*cr5+lt
; CHECK-NEXT: isel r4, 0, r4, 4*cr5+eq
+; CHECK-NEXT: lwz r12, 8(r1)
+; CHECK-NEXT: mtocrf 128, r5
+; CHECK-NEXT: mtcrf 56, r12
+; CHECK-NEXT: iseleq r5, 0, r6
; CHECK-NEXT: add r3, r5, r3
; CHECK-NEXT: add r3, r4, r3
; CHECK-NEXT: mtfprd f0, r3
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll b/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll
index e0471c92edbbc..68da52bd22e05 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/fp128.ll
@@ -2039,12 +2039,12 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: sw s1, 104(sp) # 4-byte Folded Spill
; RV32-NEXT: sw s2, 100(sp) # 4-byte Folded Spill
; RV32-NEXT: sw s3, 96(sp) # 4-byte Folded Spill
-; RV32-NEXT: lw a4, 0(a1)
-; RV32-NEXT: lw a2, 4(a1)
-; RV32-NEXT: lw a3, 8(a1)
+; RV32-NEXT: lw a2, 0(a1)
+; RV32-NEXT: lw a3, 4(a1)
+; RV32-NEXT: lw a4, 8(a1)
; RV32-NEXT: lw a5, 12(a1)
-; RV32-NEXT: or a1, a2, a5
-; RV32-NEXT: or a6, a4, a3
+; RV32-NEXT: or a1, a3, a5
+; RV32-NEXT: or a6, a2, a4
; RV32-NEXT: or a1, a6, a1
; RV32-NEXT: beqz a1, .LBB29_24
; RV32-NEXT: # %bb.1: # %itofp-if-end
@@ -2053,19 +2053,19 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: mv a1, a5
; RV32-NEXT: j .LBB29_4
; RV32-NEXT: .LBB29_3:
-; RV32-NEXT: neg a1, a3
-; RV32-NEXT: or a6, a4, a2
-; RV32-NEXT: snez a3, a3
+; RV32-NEXT: neg a1, a4
+; RV32-NEXT: or a6, a2, a3
+; RV32-NEXT: snez a4, a4
; RV32-NEXT: snez a6, a6
-; RV32-NEXT: add a3, a5, a3
+; RV32-NEXT: add a4, a5, a4
; RV32-NEXT: sltu a7, a1, a6
-; RV32-NEXT: neg t0, a3
-; RV32-NEXT: sub a3, a1, a6
+; RV32-NEXT: neg t0, a4
+; RV32-NEXT: sub a4, a1, a6
; RV32-NEXT: sub a1, t0, a7
-; RV32-NEXT: snez a6, a4
+; RV32-NEXT: snez a6, a2
+; RV32-NEXT: neg a3, a3
+; RV32-NEXT: sub a3, a3, a6
; RV32-NEXT: neg a2, a2
-; RV32-NEXT: neg a4, a4
-; RV32-NEXT: sub a2, a2, a6
; RV32-NEXT: .LBB29_4: # %itofp-if-end
; RV32-NEXT: lui a6, 349525
; RV32-NEXT: lui a7, 209715
@@ -2073,10 +2073,10 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: addi t0, a6, 1365
; RV32-NEXT: addi a7, a7, 819
; RV32-NEXT: addi a6, t1, -241
-; RV32-NEXT: bnez a2, .LBB29_6
+; RV32-NEXT: bnez a3, .LBB29_6
; RV32-NEXT: # %bb.5: # %itofp-if-end
-; RV32-NEXT: srli t1, a4, 1
-; RV32-NEXT: or t1, a4, t1
+; RV32-NEXT: srli t1, a2, 1
+; RV32-NEXT: or t1, a2, t1
; RV32-NEXT: srli t2, t1, 2
; RV32-NEXT: or t1, t1, t2
; RV32-NEXT: srli t2, t1, 4
@@ -2104,8 +2104,8 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: addi t3, t1, 32
; RV32-NEXT: j .LBB29_7
; RV32-NEXT: .LBB29_6:
-; RV32-NEXT: srli t1, a2, 1
-; RV32-NEXT: or t1, a2, t1
+; RV32-NEXT: srli t1, a3, 1
+; RV32-NEXT: or t1, a3, t1
; RV32-NEXT: srli t2, t1, 2
; RV32-NEXT: or t1, t1, t2
; RV32-NEXT: srli t2, t1, 4
@@ -2131,15 +2131,15 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: add t1, t1, t2
; RV32-NEXT: srli t3, t1, 24
; RV32-NEXT: .LBB29_7: # %itofp-if-end
-; RV32-NEXT: or t2, a3, a1
+; RV32-NEXT: or t2, a4, a1
; RV32-NEXT: addi t1, t3, 64
; RV32-NEXT: snez t4, t2
; RV32-NEXT: sltu t3, t1, t3
; RV32-NEXT: addi t4, t4, -1
; RV32-NEXT: bnez a1, .LBB29_9
; RV32-NEXT: # %bb.8: # %itofp-if-end
-; RV32-NEXT: srli t5, a3, 1
-; RV32-NEXT: or t5, a3, t5
+; RV32-NEXT: srli t5, a4, 1
+; RV32-NEXT: or t5, a4, t5
; RV32-NEXT: srli t6, t5, 2
; RV32-NEXT: or t5, t5, t6
; RV32-NEXT: srli t6, t5, 4
@@ -2246,9 +2246,9 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: sw zero, 60(sp)
; RV32-NEXT: andi t2, t2, 12
; RV32-NEXT: addi t3, sp, 32
-; RV32-NEXT: sw a4, 32(sp)
-; RV32-NEXT: sw a2, 36(sp)
-; RV32-NEXT: sw a3, 40(sp)
+; RV32-NEXT: sw a2, 32(sp)
+; RV32-NEXT: sw a3, 36(sp)
+; RV32-NEXT: sw a4, 40(sp)
; RV32-NEXT: sw a1, 44(sp)
; RV32-NEXT: add t3, t3, t2
; RV32-NEXT: lw t4, 8(t3)
@@ -2268,9 +2268,9 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: sw zero, 12(sp)
; RV32-NEXT: andi s2, s2, 12
; RV32-NEXT: addi s3, sp, 16
-; RV32-NEXT: sw a4, 16(sp)
-; RV32-NEXT: sw a2, 20(sp)
-; RV32-NEXT: sw a3, 24(sp)
+; RV32-NEXT: sw a2, 16(sp)
+; RV32-NEXT: sw a3, 20(sp)
+; RV32-NEXT: sw a4, 24(sp)
; RV32-NEXT: sw a1, 28(sp)
; RV32-NEXT: sub a1, s3, s2
; RV32-NEXT: slli t4, t4, 1
@@ -2296,35 +2296,35 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: or a3, t3, a4
; RV32-NEXT: or a1, a1, a2
; RV32-NEXT: srl t0, t2, t1
-; RV32-NEXT: snez a4, a1
-; RV32-NEXT: or a4, a3, a4
-; RV32-NEXT: mv a2, t4
-; RV32-NEXT: mv a3, s0
+; RV32-NEXT: snez a2, a1
+; RV32-NEXT: or a2, a3, a2
+; RV32-NEXT: mv a3, t4
+; RV32-NEXT: mv a4, s0
; RV32-NEXT: mv a1, t0
; RV32-NEXT: .LBB29_20: # %itofp-sw-epilog
-; RV32-NEXT: slli t0, a4, 29
+; RV32-NEXT: slli t0, a2, 29
; RV32-NEXT: srli t0, t0, 31
-; RV32-NEXT: or t2, a4, t0
-; RV32-NEXT: addi t2, t2, 1
-; RV32-NEXT: seqz t0, t2
-; RV32-NEXT: add t0, a2, t0
-; RV32-NEXT: or a2, t2, t0
-; RV32-NEXT: seqz t1, a2
-; RV32-NEXT: add t1, a3, t1
-; RV32-NEXT: sltu a4, t1, a3
+; RV32-NEXT: or a2, a2, t0
+; RV32-NEXT: addi a2, a2, 1
+; RV32-NEXT: seqz t0, a2
+; RV32-NEXT: add a3, a3, t0
+; RV32-NEXT: or t0, a2, a3
+; RV32-NEXT: seqz t0, t0
+; RV32-NEXT: add t0, a4, t0
+; RV32-NEXT: sltu a4, t0, a4
; RV32-NEXT: add a4, a1, a4
; RV32-NEXT: slli a1, a4, 12
; RV32-NEXT: bltz a1, .LBB29_26
; RV32-NEXT: # %bb.21:
-; RV32-NEXT: srli a1, t1, 2
-; RV32-NEXT: slli a2, a4, 30
-; RV32-NEXT: srli a3, t2, 2
-; RV32-NEXT: slli a7, t0, 30
-; RV32-NEXT: or a2, a1, a2
-; RV32-NEXT: or a1, a3, a7
-; RV32-NEXT: slli t1, t1, 30
-; RV32-NEXT: srli a3, t0, 2
-; RV32-NEXT: or a3, a3, t1
+; RV32-NEXT: srli a1, t0, 2
+; RV32-NEXT: slli a7, a4, 30
+; RV32-NEXT: srli t1, a2, 2
+; RV32-NEXT: slli t2, a3, 30
+; RV32-NEXT: or a2, a1, a7
+; RV32-NEXT: or a1, t1, t2
+; RV32-NEXT: slli t0, t0, 30
+; RV32-NEXT: srli a3, a3, 2
+; RV32-NEXT: or a3, a3, t0
; RV32-NEXT: srai a4, a4, 2
; RV32-NEXT: .LBB29_22: # %itofp-if-end26
; RV32-NEXT: lui a7, 524288
@@ -2361,9 +2361,9 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: sw zero, 76(sp)
; RV32-NEXT: andi a7, a7, 12
; RV32-NEXT: addi t1, sp, 80
-; RV32-NEXT: sw a4, 80(sp)
-; RV32-NEXT: sw a2, 84(sp)
-; RV32-NEXT: sw a3, 88(sp)
+; RV32-NEXT: sw a2, 80(sp)
+; RV32-NEXT: sw a3, 84(sp)
+; RV32-NEXT: sw a4, 88(sp)
; RV32-NEXT: sw a1, 92(sp)
; RV32-NEXT: sub a1, t1, a7
; RV32-NEXT: lw a2, 8(a1)
@@ -2387,12 +2387,12 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: sll a1, a1, t0
; RV32-NEXT: j .LBB29_22
; RV32-NEXT: .LBB29_26: # %itofp-if-then20
-; RV32-NEXT: srli a1, t2, 3
+; RV32-NEXT: srli a2, a2, 3
+; RV32-NEXT: slli a1, a3, 29
+; RV32-NEXT: srli a3, a3, 3
+; RV32-NEXT: or a1, a2, a1
; RV32-NEXT: slli a2, t0, 29
-; RV32-NEXT: srli a3, t0, 3
-; RV32-NEXT: or a1, a1, a2
-; RV32-NEXT: slli a2, t1, 29
-; RV32-NEXT: srli a6, t1, 3
+; RV32-NEXT: srli a6, t0, 3
; RV32-NEXT: slli t0, a4, 29
; RV32-NEXT: or a3, a3, a2
; RV32-NEXT: or a2, a6, t0
@@ -2400,18 +2400,18 @@ define fp128 @sitofp_i128(i128 %x) nounwind {
; RV32-NEXT: mv a6, a7
; RV32-NEXT: j .LBB29_22
; RV32-NEXT: .LBB29_27: # %itofp-sw-bb
-; RV32-NEXT: srli t0, a4, 31
-; RV32-NEXT: slli t1, a2, 1
-; RV32-NEXT: srli a2, a2, 31
-; RV32-NEXT: or t0, t1, t0
+; RV32-NEXT: srli t0, a2, 31
; RV32-NEXT: slli t1, a3, 1
; RV32-NEXT: srli a3, a3, 31
-; RV32-NEXT: or t1, t1, a2
+; RV32-NEXT: or t0, t1, t0
+; RV32-NEXT: slli t1, a4, 1
+; RV32-NEXT: srli a4, a4, 31
+; RV32-NEXT: or t1, t1, a3
; RV32-NEXT: slli a1, a1, 1
-; RV32-NEXT: or a1, a1, a3
-; RV32-NEXT: slli a4, a4, 1
-; RV32-NEXT: mv a2, t0
-; RV32-NEXT: mv a3, t1
+; RV32-NEXT: or a1, a1, a4
+; RV32-NEXT: slli a2, a2, 1
+; RV32-NEXT: mv a3, t0
+; RV32-NEXT: mv a4, t1
; RV32-NEXT: j .LBB29_20
;
; RV64-LABEL: sitofp_i128:
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
index b4e744808721f..7ffd839b23d38 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-cond-sub-clamp.ll
@@ -1105,20 +1105,20 @@ define i32 @atomicrmw_usub_sat_i32(ptr %ptr, i32 %val) {
; RV32IA-NEXT: .LBB6_1: # %atomicrmw.start
; RV32IA-NEXT: # =>This Loop Header: Depth=1
; RV32IA-NEXT: # Child Loop BB6_3 Depth 2
-; RV32IA-NEXT: mv a3, a2
-; RV32IA-NEXT: sub a2, a2, a1
-; RV32IA-NEXT: sltu a4, a3, a2
+; RV32IA-NEXT: sub a3, a2, a1
+; RV32IA-NEXT: sltu a4, a2, a3
; RV32IA-NEXT: addi a4, a4, -1
-; RV32IA-NEXT: and a4, a4, a2
+; RV32IA-NEXT: mv a5, a2
+; RV32IA-NEXT: and a3, a4, a3
; RV32IA-NEXT: .LBB6_3: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB6_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
; RV32IA-NEXT: lr.w.aqrl a2, (a0)
-; RV32IA-NEXT: bne a2, a3, .LBB6_1
+; RV32IA-NEXT: bne a2, a5, .LBB6_1
; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB6_3 Depth=2
-; RV32IA-NEXT: sc.w.rl a5, a4, (a0)
-; RV32IA-NEXT: bnez a5, .LBB6_3
+; RV32IA-NEXT: sc.w.rl a4, a3, (a0)
+; RV32IA-NEXT: bnez a4, .LBB6_3
; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
; RV32IA-NEXT: mv a0, a2
@@ -1363,20 +1363,20 @@ define i64 @atomicrmw_usub_sat_i64(ptr %ptr, i64 %val) {
; RV64IA-NEXT: .LBB7_1: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB7_3 Depth 2
-; RV64IA-NEXT: mv a3, a2
-; RV64IA-NEXT: sub a2, a2, a1
-; RV64IA-NEXT: sltu a4, a3, a2
+; RV64IA-NEXT: sub a3, a2, a1
+; RV64IA-NEXT: sltu a4, a2, a3
; RV64IA-NEXT: addi a4, a4, -1
-; RV64IA-NEXT: and a4, a4, a2
+; RV64IA-NEXT: mv a5, a2
+; RV64IA-NEXT: and a3, a4, a3
; RV64IA-NEXT: .LBB7_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB7_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.d.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a3, .LBB7_1
+; RV64IA-NEXT: bne a2, a5, .LBB7_1
; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB7_3 Depth=2
-; RV64IA-NEXT: sc.d.rl a5, a4, (a0)
-; RV64IA-NEXT: bnez a5, .LBB7_3
+; RV64IA-NEXT: sc.d.rl a4, a3, (a0)
+; RV64IA-NEXT: bnez a4, .LBB7_3
; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: mv a0, a2
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
index b255098688831..f897fb10a5337 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
@@ -411,20 +411,20 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
; RV32IA-NEXT: .LBB2_1: # %atomicrmw.start
; RV32IA-NEXT: # =>This Loop Header: Depth=1
; RV32IA-NEXT: # Child Loop BB2_3 Depth 2
-; RV32IA-NEXT: mv a3, a2
-; RV32IA-NEXT: sltu a2, a2, a1
-; RV32IA-NEXT: addi a4, a3, 1
-; RV32IA-NEXT: neg a2, a2
-; RV32IA-NEXT: and a4, a2, a4
+; RV32IA-NEXT: sltu a3, a2, a1
+; RV32IA-NEXT: addi a4, a2, 1
+; RV32IA-NEXT: neg a3, a3
+; RV32IA-NEXT: mv a5, a2
+; RV32IA-NEXT: and a3, a3, a4
; RV32IA-NEXT: .LBB2_3: # %atomicrmw.start
; RV32IA-NEXT: # Parent Loop BB2_1 Depth=1
; RV32IA-NEXT: # => This Inner Loop Header: Depth=2
; RV32IA-NEXT: lr.w.aqrl a2, (a0)
-; RV32IA-NEXT: bne a2, a3, .LBB2_1
+; RV32IA-NEXT: bne a2, a5, .LBB2_1
; RV32IA-NEXT: # %bb.4: # %atomicrmw.start
; RV32IA-NEXT: # in Loop: Header=BB2_3 Depth=2
-; RV32IA-NEXT: sc.w.rl a5, a4, (a0)
-; RV32IA-NEXT: bnez a5, .LBB2_3
+; RV32IA-NEXT: sc.w.rl a4, a3, (a0)
+; RV32IA-NEXT: bnez a4, .LBB2_3
; RV32IA-NEXT: # %bb.5: # %atomicrmw.start
; RV32IA-NEXT: # %bb.2: # %atomicrmw.end
; RV32IA-NEXT: mv a0, a2
@@ -669,20 +669,20 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
; RV64IA-NEXT: .LBB3_1: # %atomicrmw.start
; RV64IA-NEXT: # =>This Loop Header: Depth=1
; RV64IA-NEXT: # Child Loop BB3_3 Depth 2
-; RV64IA-NEXT: mv a3, a2
-; RV64IA-NEXT: sltu a2, a2, a1
-; RV64IA-NEXT: addi a4, a3, 1
-; RV64IA-NEXT: neg a2, a2
-; RV64IA-NEXT: and a4, a2, a4
+; RV64IA-NEXT: sltu a3, a2, a1
+; RV64IA-NEXT: addi a4, a2, 1
+; RV64IA-NEXT: neg a3, a3
+; RV64IA-NEXT: mv a5, a2
+; RV64IA-NEXT: and a3, a3, a4
; RV64IA-NEXT: .LBB3_3: # %atomicrmw.start
; RV64IA-NEXT: # Parent Loop BB3_1 Depth=1
; RV64IA-NEXT: # => This Inner Loop Header: Depth=2
; RV64IA-NEXT: lr.d.aqrl a2, (a0)
-; RV64IA-NEXT: bne a2, a3, .LBB3_1
+; RV64IA-NEXT: bne a2, a5, .LBB3_1
; RV64IA-NEXT: # %bb.4: # %atomicrmw.start
; RV64IA-NEXT: # in Loop: Header=BB3_3 Depth=2
-; RV64IA-NEXT: sc.d.rl a5, a4, (a0)
-; RV64IA-NEXT: bnez a5, .LBB3_3
+; RV64IA-NEXT: sc.d.rl a4, a3, (a0)
+; RV64IA-NEXT: bnez a4, .LBB3_3
; RV64IA-NEXT: # %bb.5: # %atomicrmw.start
; RV64IA-NEXT: # %bb.2: # %atomicrmw.end
; RV64IA-NEXT: mv a0, a2
diff --git a/llvm/test/CodeGen/RISCV/bitint-fp-conv-200.ll b/llvm/test/CodeGen/RISCV/bitint-fp-conv-200.ll
index 780aebc147895..3f3a8ce6b561c 100644
--- a/llvm/test/CodeGen/RISCV/bitint-fp-conv-200.ll
+++ b/llvm/test/CodeGen/RISCV/bitint-fp-conv-200.ll
@@ -465,15 +465,15 @@ define void @test_bitint_200_to_float(ptr %in, ptr %out) nounwind {
; RV32-NEXT: lw a4, 16(a0)
; RV32-NEXT: lw a3, 20(a0)
; RV32-NEXT: lbu a2, 24(a0)
-; RV32-NEXT: lw t0, 8(a0)
-; RV32-NEXT: lw t1, 0(a0)
+; RV32-NEXT: lw a7, 8(a0)
+; RV32-NEXT: lw t2, 0(a0)
; RV32-NEXT: lw t5, 4(a0)
-; RV32-NEXT: lw t2, 12(a0)
-; RV32-NEXT: or a0, t0, a2
-; RV32-NEXT: or a5, t1, a4
+; RV32-NEXT: lw t1, 12(a0)
+; RV32-NEXT: or a0, a7, a2
+; RV32-NEXT: or a5, t2, a4
; RV32-NEXT: or a6, t5, a3
; RV32-NEXT: or a0, a5, a0
-; RV32-NEXT: or a5, a6, t2
+; RV32-NEXT: or a5, a6, t1
; RV32-NEXT: or a0, a0, a5
; RV32-NEXT: beqz a0, .LBB1_37
; RV32-NEXT: # %bb.1: # %itofp-if-end
@@ -486,61 +486,61 @@ define void @test_bitint_200_to_float(ptr %in, ptr %out) nounwind {
; RV32-NEXT: sw s5, 200(sp) # 4-byte Folded Spill
; RV32-NEXT: sw s6, 196(sp) # 4-byte Folded Spill
; RV32-NEXT: sw s7, 192(sp) # 4-byte Folded Spill
-; RV32-NEXT: slli a2, a2, 24
-; RV32-NEXT: srai a2, a2, 24
; RV32-NEXT: slli a0, a2, 24
-; RV32-NEXT: srai s2, a0, 31
; RV32-NEXT: srai a0, a0, 24
+; RV32-NEXT: slli a2, a0, 24
+; RV32-NEXT: srai s2, a2, 31
+; RV32-NEXT: srai a2, a2, 24
; RV32-NEXT: slli a6, s2, 25
-; RV32-NEXT: srli a0, a0, 7
-; RV32-NEXT: xor a5, t0, s2
-; RV32-NEXT: or a0, a0, a6
+; RV32-NEXT: srli a2, a2, 7
+; RV32-NEXT: xor a5, a7, s2
+; RV32-NEXT: or a2, a2, a6
; RV32-NEXT: sltu a6, a5, s2
-; RV32-NEXT: xor a7, t2, s2
+; RV32-NEXT: xor t0, t1, s2
; RV32-NEXT: mv t6, a6
-; RV32-NEXT: beqz t2, .LBB1_3
+; RV32-NEXT: beqz t1, .LBB1_3
; RV32-NEXT: # %bb.2: # %itofp-if-end
-; RV32-NEXT: sltu t6, a7, s2
+; RV32-NEXT: sltu t6, t0, s2
; RV32-NEXT: .LBB1_3: # %itofp-if-end
-; RV32-NEXT: xor t4, t1, a0
-; RV32-NEXT: sltu t3, t4, a0
+; RV32-NEXT: xor t4, t2, a2
+; RV32-NEXT: sltu t3, t4, a2
; RV32-NEXT: xor s3, t5, s2
-; RV32-NEXT: mv t1, t3
+; RV32-NEXT: mv t2, t3
; RV32-NEXT: beqz t5, .LBB1_5
; RV32-NEXT: # %bb.4: # %itofp-if-end
-; RV32-NEXT: sltu t1, s3, s2
+; RV32-NEXT: sltu t2, s3, s2
; RV32-NEXT: .LBB1_5: # %itofp-if-end
-; RV32-NEXT: or t2, t0, t2
-; RV32-NEXT: mv t0, t1
-; RV32-NEXT: beqz t2, .LBB1_7
+; RV32-NEXT: or a7, a7, t1
+; RV32-NEXT: mv t1, t2
+; RV32-NEXT: beqz a7, .LBB1_7
; RV32-NEXT: # %bb.6: # %itofp-if-end
-; RV32-NEXT: mv t0, t6
+; RV32-NEXT: mv t1, t6
; RV32-NEXT: .LBB1_7: # %itofp-if-end
; RV32-NEXT: xor a4, a4, s2
; RV32-NEXT: xor s1, a3, s2
; RV32-NEXT: sltu s0, a4, s2
-; RV32-NEXT: sub t2, s1, s2
+; RV32-NEXT: sub a7, s1, s2
; RV32-NEXT: sub a4, a4, s2
-; RV32-NEXT: sub t2, t2, s0
-; RV32-NEXT: sltu t5, a4, t0
-; RV32-NEXT: seqz t6, t2
-; RV32-NEXT: and t6, t6, t5
+; RV32-NEXT: sub t5, a7, s0
+; RV32-NEXT: sltu t6, a4, t1
+; RV32-NEXT: seqz a7, t5
+; RV32-NEXT: and a7, a7, t6
; RV32-NEXT: beqz a3, .LBB1_9
; RV32-NEXT: # %bb.8: # %itofp-if-end
; RV32-NEXT: sltu s0, s1, s2
; RV32-NEXT: .LBB1_9: # %itofp-if-end
-; RV32-NEXT: xor a3, a2, s2
+; RV32-NEXT: xor a3, a0, s2
; RV32-NEXT: add s0, s2, s0
-; RV32-NEXT: sub s1, a7, s2
+; RV32-NEXT: sub t0, t0, s2
; RV32-NEXT: sub a3, a3, s0
; RV32-NEXT: sub s0, a5, s2
-; RV32-NEXT: sub a7, a3, t6
-; RV32-NEXT: sub a3, s1, a6
-; RV32-NEXT: sltu a5, s0, t1
+; RV32-NEXT: sub a7, a3, a7
+; RV32-NEXT: sub a3, t0, a6
+; RV32-NEXT: sltu a5, s0, t2
; RV32-NEXT: sub a5, a3, a5
-; RV32-NEXT: sub a3, t2, t5
-; RV32-NEXT: sub a4, a4, t0
-; RV32-NEXT: sub a6, s0, t1
+; RV32-NEXT: sub a3, t5, t6
+; RV32-NEXT: sub a4, a4, t1
+; RV32-NEXT: sub a6, s0, t2
; RV32-NEXT: srli t0, a4, 8
; RV32-NEXT: slli t1, a3, 24
; RV32-NEXT: or t1, t0, t1
@@ -678,7 +678,7 @@ define void @test_bitint_200_to_float(ptr %in, ptr %out) nounwind {
; RV32-NEXT: .LBB1_15: # %itofp-if-end
; RV32-NEXT: or s5, t1, t2
; RV32-NEXT: sub t3, s3, t3
-; RV32-NEXT: sub t4, t4, a0
+; RV32-NEXT: sub t4, t4, a2
; RV32-NEXT: bnez s5, .LBB1_17
; RV32-NEXT: # %bb.16: # %itofp-if-end
; RV32-NEXT: addi t0, s4, 64
@@ -924,22 +924,22 @@ define void @test_bitint_200_to_float(ptr %in, ptr %out) nounwind {
; RV32-NEXT: lw s6, 196(sp) # 4-byte Folded Reload
; RV32-NEXT: lw s7, 192(sp) # 4-byte Folded Reload
; RV32-NEXT: addi sp, sp, 224
-; RV32-NEXT: bgez a2, .LBB1_35
+; RV32-NEXT: bgez a0, .LBB1_35
; RV32-NEXT: # %bb.33: # %itofp-if-end26
-; RV32-NEXT: li a2, 127
-; RV32-NEXT: bgeu a2, t1, .LBB1_36
+; RV32-NEXT: li a0, 127
+; RV32-NEXT: bgeu a0, t1, .LBB1_36
; RV32-NEXT: .LBB1_34: # %itofp-return
; RV32-NEXT: sw a4, 0(a1)
; RV32-NEXT: ret
; RV32-NEXT: .LBB1_35: # %itofp-if-end26
; RV32-NEXT: lui a4, 522240
-; RV32-NEXT: li a2, 127
-; RV32-NEXT: bltu a2, t1, .LBB1_34
+; RV32-NEXT: li a0, 127
+; RV32-NEXT: bltu a0, t1, .LBB1_34
; RV32-NEXT: .LBB1_36: # %itofp-if-end26
-; RV32-NEXT: lui a2, 524288
+; RV32-NEXT: lui a0, 524288
; RV32-NEXT: slli a3, a3, 9
; RV32-NEXT: slli t2, t2, 23
-; RV32-NEXT: and a0, a0, a2
+; RV32-NEXT: and a0, a2, a0
; RV32-NEXT: lui a2, 260096
; RV32-NEXT: srli a3, a3, 9
; RV32-NEXT: add a2, t2, a2
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index cbf9b6a24b2e0..0388effd3c8a0 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -1286,15 +1286,14 @@ define i64 @stest_f64i64(double %x) {
; RV32IF-NEXT: .LBB18_8: # %fp-to-i-cleanup
; RV32IF-NEXT: lui a2, 524288
; RV32IF-NEXT: addi a3, a2, -1
+; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: beq s2, a3, .LBB18_10
; RV32IF-NEXT: # %bb.9: # %fp-to-i-cleanup
; RV32IF-NEXT: sltu a4, s2, a3
-; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: bnez a5, .LBB18_11
; RV32IF-NEXT: j .LBB18_12
; RV32IF-NEXT: .LBB18_10:
; RV32IF-NEXT: sltiu a4, s1, -1
-; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: beqz a5, .LBB18_12
; RV32IF-NEXT: .LBB18_11: # %fp-to-i-cleanup
; RV32IF-NEXT: srli a4, a1, 31
@@ -1604,15 +1603,14 @@ define i64 @stest_f64i64(double %x) {
; RV32IFD-NEXT: .LBB18_8: # %fp-to-i-cleanup
; RV32IFD-NEXT: lui a2, 524288
; RV32IFD-NEXT: addi a3, a2, -1
+; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: beq s2, a3, .LBB18_10
; RV32IFD-NEXT: # %bb.9: # %fp-to-i-cleanup
; RV32IFD-NEXT: sltu a4, s2, a3
-; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: bnez a5, .LBB18_11
; RV32IFD-NEXT: j .LBB18_12
; RV32IFD-NEXT: .LBB18_10:
; RV32IFD-NEXT: sltiu a4, s1, -1
-; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: beqz a5, .LBB18_12
; RV32IFD-NEXT: .LBB18_11: # %fp-to-i-cleanup
; RV32IFD-NEXT: srli a4, a1, 31
@@ -2666,15 +2664,14 @@ define i64 @stest_f32i64(float %x) {
; RV32-NEXT: .LBB21_5: # %fp-to-i-cleanup
; RV32-NEXT: lui a2, 524288
; RV32-NEXT: addi a3, a2, -1
+; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: beq s2, a3, .LBB21_7
; RV32-NEXT: # %bb.6: # %fp-to-i-cleanup
; RV32-NEXT: sltu a4, s2, a3
-; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: bnez a5, .LBB21_8
; RV32-NEXT: j .LBB21_9
; RV32-NEXT: .LBB21_7:
; RV32-NEXT: sltiu a4, s1, -1
-; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: beqz a5, .LBB21_9
; RV32-NEXT: .LBB21_8: # %fp-to-i-cleanup
; RV32-NEXT: srli a4, a1, 31
@@ -4599,15 +4596,14 @@ define i64 @stest_f64i64_mm(double %x) {
; RV32IF-NEXT: .LBB45_8: # %fp-to-i-cleanup
; RV32IF-NEXT: lui a2, 524288
; RV32IF-NEXT: addi a3, a2, -1
+; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: beq s2, a3, .LBB45_10
; RV32IF-NEXT: # %bb.9: # %fp-to-i-cleanup
; RV32IF-NEXT: sltu a4, s2, a3
-; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: bnez a5, .LBB45_11
; RV32IF-NEXT: j .LBB45_12
; RV32IF-NEXT: .LBB45_10:
; RV32IF-NEXT: sltiu a4, s1, -1
-; RV32IF-NEXT: or a5, a0, a1
; RV32IF-NEXT: beqz a5, .LBB45_12
; RV32IF-NEXT: .LBB45_11: # %fp-to-i-cleanup
; RV32IF-NEXT: srli a4, a1, 31
@@ -4917,15 +4913,14 @@ define i64 @stest_f64i64_mm(double %x) {
; RV32IFD-NEXT: .LBB45_8: # %fp-to-i-cleanup
; RV32IFD-NEXT: lui a2, 524288
; RV32IFD-NEXT: addi a3, a2, -1
+; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: beq s2, a3, .LBB45_10
; RV32IFD-NEXT: # %bb.9: # %fp-to-i-cleanup
; RV32IFD-NEXT: sltu a4, s2, a3
-; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: bnez a5, .LBB45_11
; RV32IFD-NEXT: j .LBB45_12
; RV32IFD-NEXT: .LBB45_10:
; RV32IFD-NEXT: sltiu a4, s1, -1
-; RV32IFD-NEXT: or a5, a0, a1
; RV32IFD-NEXT: beqz a5, .LBB45_12
; RV32IFD-NEXT: .LBB45_11: # %fp-to-i-cleanup
; RV32IFD-NEXT: srli a4, a1, 31
@@ -5937,15 +5932,14 @@ define i64 @stest_f32i64_mm(float %x) {
; RV32-NEXT: .LBB48_5: # %fp-to-i-cleanup
; RV32-NEXT: lui a2, 524288
; RV32-NEXT: addi a3, a2, -1
+; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: beq s2, a3, .LBB48_7
; RV32-NEXT: # %bb.6: # %fp-to-i-cleanup
; RV32-NEXT: sltu a4, s2, a3
-; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: bnez a5, .LBB48_8
; RV32-NEXT: j .LBB48_9
; RV32-NEXT: .LBB48_7:
; RV32-NEXT: sltiu a4, s1, -1
-; RV32-NEXT: or a5, a0, a1
; RV32-NEXT: beqz a5, .LBB48_9
; RV32-NEXT: .LBB48_8: # %fp-to-i-cleanup
; RV32-NEXT: srli a4, a1, 31
diff --git a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
index a5ac217b35e10..022b4f549a05e 100644
--- a/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/RISCV/freeze-multi-use.ll
@@ -166,9 +166,14 @@ define void @other_block(<2 x i32> %v) nounwind {
; RV32: # %bb.0: # %entry
; RV32-NEXT: addi sp, sp, -16
; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT: # freeze: $x8
+; RV32-NEXT: mv a0, s0
; RV32-NEXT: call use
+; RV32-NEXT: mv a0, s0
; RV32-NEXT: call use
; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload
; RV32-NEXT: addi sp, sp, 16
; RV32-NEXT: ret
;
@@ -176,9 +181,14 @@ define void @other_block(<2 x i32> %v) nounwind {
; RV64: # %bb.0: # %entry
; RV64-NEXT: addi sp, sp, -16
; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s0, 0(sp) # 8-byte Folded Spill
+; RV64-NEXT: # freeze: $x8
+; RV64-NEXT: mv a0, s0
; RV64-NEXT: call use
+; RV64-NEXT: mv a0, s0
; RV64-NEXT: call use
; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s0, 0(sp) # 8-byte Folded Reload
; RV64-NEXT: addi sp, sp, 16
; RV64-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/RISCV/idiv_large.ll b/llvm/test/CodeGen/RISCV/idiv_large.ll
index 82042aa44d6fc..ffd2385b203ca 100644
--- a/llvm/test/CodeGen/RISCV/idiv_large.ll
+++ b/llvm/test/CodeGen/RISCV/idiv_large.ll
@@ -30,14 +30,14 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: lui t1, 61681
; RV32-NEXT: addi t2, a4, 1365
; RV32-NEXT: srli a4, a3, 1
-; RV32-NEXT: slli t0, a6, 31
-; RV32-NEXT: addi a7, a5, 819
-; RV32-NEXT: or t0, t0, a4
+; RV32-NEXT: slli a7, a6, 31
+; RV32-NEXT: addi t0, a5, 819
+; RV32-NEXT: or a7, a7, a4
; RV32-NEXT: slli a4, a3, 31
; RV32-NEXT: srli a5, a2, 1
; RV32-NEXT: addi t1, t1, -241
; RV32-NEXT: or t4, a5, a4
-; RV32-NEXT: bnez t0, .LBB1_2
+; RV32-NEXT: bnez a7, .LBB1_2
; RV32-NEXT: # %bb.1: # %_udiv-special-cases
; RV32-NEXT: srli a4, t4, 1
; RV32-NEXT: or a4, t4, a4
@@ -54,8 +54,8 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: and a5, a5, t2
; RV32-NEXT: sub a4, a4, a5
; RV32-NEXT: srli a5, a4, 2
-; RV32-NEXT: and a4, a4, a7
-; RV32-NEXT: and a5, a5, a7
+; RV32-NEXT: and a4, a4, t0
+; RV32-NEXT: and a5, a5, t0
; RV32-NEXT: add a4, a4, a5
; RV32-NEXT: srli a5, a4, 4
; RV32-NEXT: add a4, a4, a5
@@ -68,8 +68,8 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: addi t3, a4, 32
; RV32-NEXT: j .LBB1_3
; RV32-NEXT: .LBB1_2:
-; RV32-NEXT: srli a4, t0, 1
-; RV32-NEXT: or a4, t0, a4
+; RV32-NEXT: srli a4, a7, 1
+; RV32-NEXT: or a4, a7, a4
; RV32-NEXT: srli a5, a4, 2
; RV32-NEXT: or a4, a4, a5
; RV32-NEXT: srli a5, a4, 4
@@ -83,8 +83,8 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: and a5, a5, t2
; RV32-NEXT: sub a4, a4, a5
; RV32-NEXT: srli a5, a4, 2
-; RV32-NEXT: and a4, a4, a7
-; RV32-NEXT: and a5, a5, a7
+; RV32-NEXT: and a4, a4, t0
+; RV32-NEXT: and a5, a5, t0
; RV32-NEXT: add a4, a4, a5
; RV32-NEXT: srli a5, a4, 4
; RV32-NEXT: add a4, a4, a5
@@ -125,8 +125,8 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: and a5, a5, t2
; RV32-NEXT: sub a4, a4, a5
; RV32-NEXT: srli a5, a4, 2
-; RV32-NEXT: and a4, a4, a7
-; RV32-NEXT: and a5, a5, a7
+; RV32-NEXT: and a4, a4, t0
+; RV32-NEXT: and a5, a5, t0
; RV32-NEXT: add a4, a4, a5
; RV32-NEXT: srli a5, a4, 4
; RV32-NEXT: add a4, a4, a5
@@ -140,9 +140,9 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: lw a5, 0(a1)
; RV32-NEXT: lw a4, 4(a1)
; RV32-NEXT: lw s3, 8(a1)
-; RV32-NEXT: or t0, t4, t0
+; RV32-NEXT: or a7, t4, a7
; RV32-NEXT: addi s2, s0, 64
-; RV32-NEXT: bnez t0, .LBB1_8
+; RV32-NEXT: bnez a7, .LBB1_8
; RV32-NEXT: # %bb.7: # %_udiv-special-cases
; RV32-NEXT: mv t3, s2
; RV32-NEXT: .LBB1_8: # %_udiv-special-cases
@@ -151,69 +151,69 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: or a1, t4, a1
; RV32-NEXT: slli t4, a4, 31
; RV32-NEXT: srli t6, a5, 1
-; RV32-NEXT: snez s4, t0
+; RV32-NEXT: snez s4, a7
; RV32-NEXT: or t6, t6, t4
; RV32-NEXT: bnez a1, .LBB1_10
; RV32-NEXT: # %bb.9: # %_udiv-special-cases
-; RV32-NEXT: srli t0, t6, 1
-; RV32-NEXT: or t0, t6, t0
-; RV32-NEXT: srli t4, t0, 2
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 4
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 8
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 16
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: not t0, t0
-; RV32-NEXT: srli t4, t0, 1
+; RV32-NEXT: srli a7, t6, 1
+; RV32-NEXT: or a7, t6, a7
+; RV32-NEXT: srli t4, a7, 2
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 4
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 8
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 16
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: not a7, a7
+; RV32-NEXT: srli t4, a7, 1
; RV32-NEXT: and t4, t4, t2
-; RV32-NEXT: sub t0, t0, t4
-; RV32-NEXT: srli t4, t0, 2
-; RV32-NEXT: and t0, t0, a7
-; RV32-NEXT: and t4, t4, a7
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: srli t4, t0, 4
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: and t0, t0, t1
-; RV32-NEXT: slli t4, t0, 8
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: slli t4, t0, 16
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: srli t0, t0, 24
-; RV32-NEXT: addi s1, t0, 32
+; RV32-NEXT: sub a7, a7, t4
+; RV32-NEXT: srli t4, a7, 2
+; RV32-NEXT: and a7, a7, t0
+; RV32-NEXT: and t4, t4, t0
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: srli t4, a7, 4
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: and a7, a7, t1
+; RV32-NEXT: slli t4, a7, 8
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: slli t4, a7, 16
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: srli a7, a7, 24
+; RV32-NEXT: addi s1, a7, 32
; RV32-NEXT: j .LBB1_11
; RV32-NEXT: .LBB1_10:
-; RV32-NEXT: srli t0, a1, 1
-; RV32-NEXT: or t0, a1, t0
-; RV32-NEXT: srli t4, t0, 2
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 4
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 8
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: srli t4, t0, 16
-; RV32-NEXT: or t0, t0, t4
-; RV32-NEXT: not t0, t0
-; RV32-NEXT: srli t4, t0, 1
+; RV32-NEXT: srli a7, a1, 1
+; RV32-NEXT: or a7, a1, a7
+; RV32-NEXT: srli t4, a7, 2
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 4
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 8
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: srli t4, a7, 16
+; RV32-NEXT: or a7, a7, t4
+; RV32-NEXT: not a7, a7
+; RV32-NEXT: srli t4, a7, 1
; RV32-NEXT: and t4, t4, t2
-; RV32-NEXT: sub t0, t0, t4
-; RV32-NEXT: srli t4, t0, 2
-; RV32-NEXT: and t0, t0, a7
-; RV32-NEXT: and t4, t4, a7
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: srli t4, t0, 4
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: and t0, t0, t1
-; RV32-NEXT: slli t4, t0, 8
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: slli t4, t0, 16
-; RV32-NEXT: add t0, t0, t4
-; RV32-NEXT: srli s1, t0, 24
+; RV32-NEXT: sub a7, a7, t4
+; RV32-NEXT: srli t4, a7, 2
+; RV32-NEXT: and a7, a7, t0
+; RV32-NEXT: and t4, t4, t0
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: srli t4, a7, 4
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: and a7, a7, t1
+; RV32-NEXT: slli t4, a7, 8
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: slli t4, a7, 16
+; RV32-NEXT: add a7, a7, t4
+; RV32-NEXT: srli s1, a7, 24
; RV32-NEXT: .LBB1_11: # %_udiv-special-cases
; RV32-NEXT: andi t4, s3, 1
; RV32-NEXT: andi a6, a6, 1
-; RV32-NEXT: or t0, a2, a3
+; RV32-NEXT: or a7, a2, a3
; RV32-NEXT: sltu s0, s2, s0
; RV32-NEXT: slli s3, a5, 31
; RV32-NEXT: or s2, a5, a4
@@ -235,20 +235,20 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: and t2, s3, t2
; RV32-NEXT: sub t2, t5, t2
; RV32-NEXT: srli t5, t2, 2
-; RV32-NEXT: and t2, t2, a7
-; RV32-NEXT: and a7, t5, a7
-; RV32-NEXT: add a7, t2, a7
-; RV32-NEXT: srli t2, a7, 4
-; RV32-NEXT: add a7, a7, t2
-; RV32-NEXT: and a7, a7, t1
-; RV32-NEXT: slli t1, a7, 8
-; RV32-NEXT: add a7, a7, t1
-; RV32-NEXT: slli t1, a7, 16
-; RV32-NEXT: add a7, a7, t1
-; RV32-NEXT: srli t5, a7, 24
+; RV32-NEXT: and t2, t2, t0
+; RV32-NEXT: and t0, t5, t0
+; RV32-NEXT: add t0, t2, t0
+; RV32-NEXT: srli t2, t0, 4
+; RV32-NEXT: add t0, t0, t2
+; RV32-NEXT: and t0, t0, t1
+; RV32-NEXT: slli t1, t0, 8
+; RV32-NEXT: add t0, t0, t1
+; RV32-NEXT: slli t1, t0, 16
+; RV32-NEXT: add t0, t0, t1
+; RV32-NEXT: srli t5, t0, 24
; RV32-NEXT: .LBB1_13: # %_udiv-special-cases
-; RV32-NEXT: or t1, t0, a6
-; RV32-NEXT: or a7, s2, t4
+; RV32-NEXT: or t1, a7, a6
+; RV32-NEXT: or t0, s2, t4
; RV32-NEXT: or t2, t6, a1
; RV32-NEXT: and a1, s4, s0
; RV32-NEXT: addi t6, t5, 64
@@ -262,23 +262,23 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: addi t2, t2, -1
; RV32-NEXT: and t6, t2, t5
; RV32-NEXT: sltu s0, t3, s1
-; RV32-NEXT: seqz a7, a7
+; RV32-NEXT: seqz t0, t0
; RV32-NEXT: mv t2, s0
; RV32-NEXT: beq a1, t6, .LBB1_17
; RV32-NEXT: # %bb.16: # %_udiv-special-cases
; RV32-NEXT: sltu t2, a1, t6
; RV32-NEXT: .LBB1_17: # %_udiv-special-cases
-; RV32-NEXT: or t5, t1, a7
+; RV32-NEXT: or t5, t1, t0
; RV32-NEXT: andi t2, t2, 1
; RV32-NEXT: sub a1, a1, t6
; RV32-NEXT: sub t1, a1, s0
-; RV32-NEXT: sub a7, t3, s1
+; RV32-NEXT: sub t0, t3, s1
; RV32-NEXT: beqz t2, .LBB1_19
; RV32-NEXT: # %bb.18: # %_udiv-special-cases
; RV32-NEXT: mv a1, t2
; RV32-NEXT: j .LBB1_20
; RV32-NEXT: .LBB1_19:
-; RV32-NEXT: sltiu a1, a7, 65
+; RV32-NEXT: sltiu a1, t0, 65
; RV32-NEXT: xori a1, a1, 1
; RV32-NEXT: snez t3, t1
; RV32-NEXT: or a1, a1, t3
@@ -290,15 +290,15 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: and t5, t5, a5
; RV32-NEXT: bnez t6, .LBB1_29
; RV32-NEXT: # %bb.21: # %_udiv-special-cases
-; RV32-NEXT: xori t6, a7, 64
+; RV32-NEXT: xori t6, t0, 64
; RV32-NEXT: or s0, t2, t1
; RV32-NEXT: or t6, t6, s0
; RV32-NEXT: beqz t6, .LBB1_29
; RV32-NEXT: # %bb.22: # %udiv-bb1
-; RV32-NEXT: addi a1, a7, 1
+; RV32-NEXT: addi a1, t0, 1
; RV32-NEXT: li t3, 64
; RV32-NEXT: seqz t5, a1
-; RV32-NEXT: sub t6, t3, a7
+; RV32-NEXT: sub t6, t3, t0
; RV32-NEXT: add t1, t1, t5
; RV32-NEXT: sw zero, 32(sp)
; RV32-NEXT: sw zero, 36(sp)
@@ -321,10 +321,10 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: xori s1, t2, 31
; RV32-NEXT: srli t2, s0, 1
; RV32-NEXT: srl t2, t2, s1
-; RV32-NEXT: neg s3, a7
-; RV32-NEXT: sll a7, s2, s3
+; RV32-NEXT: neg s3, t0
+; RV32-NEXT: sll t0, s2, s3
; RV32-NEXT: or t5, t5, t3
-; RV32-NEXT: or a7, a7, t2
+; RV32-NEXT: or t0, t0, t2
; RV32-NEXT: sll t2, s0, s3
; RV32-NEXT: beqz t5, .LBB1_28
; RV32-NEXT: # %bb.23: # %udiv-preheader
@@ -362,7 +362,7 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: andi s1, a5, 1
; RV32-NEXT: srl a5, s2, a1
; RV32-NEXT: or s2, a5, a4
-; RV32-NEXT: snez a4, t0
+; RV32-NEXT: snez a4, a7
; RV32-NEXT: seqz a5, a2
; RV32-NEXT: add a6, a6, a4
; RV32-NEXT: sub a4, a3, a5
@@ -372,7 +372,7 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: j .LBB1_26
; RV32-NEXT: .LBB1_24: # %udiv-do-while
; RV32-NEXT: # in Loop: Header=BB1_26 Depth=1
-; RV32-NEXT: sltu s1, a4, t0
+; RV32-NEXT: sltu s1, a4, a7
; RV32-NEXT: .LBB1_25: # %udiv-do-while
; RV32-NEXT: # in Loop: Header=BB1_26 Depth=1
; RV32-NEXT: srli t4, t4, 31
@@ -382,51 +382,51 @@ define i65 @udiv_i65(i65 %x, i65 %y) nounwind {
; RV32-NEXT: srai t4, t4, 31
; RV32-NEXT: and s3, t4, a3
; RV32-NEXT: srli s1, t2, 31
-; RV32-NEXT: slli s4, a7, 1
+; RV32-NEXT: slli s4, t0, 1
; RV32-NEXT: slli t2, t2, 1
; RV32-NEXT: or s1, s4, s1
-; RV32-NEXT: srli s4, a7, 31
+; RV32-NEXT: srli s4, t0, 31
; RV32-NEXT: or t2, t5, t2
-; RV32-NEXT: or a7, t6, s1
+; RV32-NEXT: or t0, t6, s1
; RV32-NEXT: or s1, s0, s4
; RV32-NEXT: andi s1, s1, 1
; RV32-NEXT: and t6, t4, a2
; RV32-NEXT: sltu s0, s2, t6
-; RV32-NEXT: sub t0, t0, s3
+; RV32-NEXT: sub a7, a7, s3
; RV32-NEXT: andi t5, t4, 1
-; RV32-NEXT: sub t4, t0, s0
+; RV32-NEXT: sub t4, a7, s0
; RV32-NEXT: sub s2, s2, t6
-; RV32-NEXT: or t0, a1, t1
+; RV32-NEXT: or a7, a1, t1
; RV32-NEXT: seqz t6, a1
-; RV32-NEXT: snez t0, t0
+; RV32-NEXT: snez a7, a7
; RV32-NEXT: sub t1, t1, t6
-; RV32-NEXT: add t0, t3, t0
-; RV32-NEXT: not t0, t0
+; RV32-NEXT: add a7, t3, a7
+; RV32-NEXT: not a7, a7
; RV32-NEXT: addi a1, a1, -1
-; RV32-NEXT: andi t3, t0, 1
-; RV32-NEXT: or t0, a1, t1
-; RV32-NEXT: or t0, t0, t3
+; RV32-NEXT: andi t3, a7, 1
+; RV32-NEXT: or a7, a1, t1
+; RV32-NEXT: or a7, a7, t3
; RV32-NEXT: li t6, 0
; RV32-NEXT: li s0, 0
-; RV32-NEXT: beqz t0, .LBB1_28
+; RV32-NEXT: beqz a7, .LBB1_28
; RV32-NEXT: .LBB1_26: # %udiv-do-while
; RV32-NEXT: # =>This Inner Loop Header: Depth=1
-; RV32-NEXT: srli t0, s2, 31
+; RV32-NEXT: srli a7, s2, 31
; RV32-NEXT: slli s3, t4, 1
-; RV32-NEXT: or t0, s3, t0
+; RV32-NEXT: or a7, s3, a7
; RV32-NEXT: slli s2, s2, 1
; RV32-NEXT: andi s1, s1, 1
; RV32-NEXT: or s2, s2, s1
-; RV32-NEXT: bne a4, t0, .LBB1_24
+; RV32-NEXT: bne a4, a7, .LBB1_24
; RV32-NEXT: # %bb.27: # in Loop: Header=BB1_26 Depth=1
; RV32-NEXT: sltu s1, a6, s2
; RV32-NEXT: j .LBB1_25
; RV32-NEXT: .LBB1_28: # %udiv-loop-exit
; RV32-NEXT: srli a1, t2, 31
-; RV32-NEXT: slli a2, a7, 1
+; RV32-NEXT: slli a2, t0, 1
; RV32-NEXT: slli t2, t2, 1
; RV32-NEXT: or t3, a2, a1
-; RV32-NEXT: srli a1, a7, 31
+; RV32-NEXT: srli a1, t0, 31
; RV32-NEXT: or t5, t5, t2
; RV32-NEXT: .LBB1_29: # %udiv-end
; RV32-NEXT: sw t5, 0(a0)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index f9649ecb3bf85..9759a8de96e94 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -149,7 +149,7 @@ define void @foo_sext_v2i64_v2i32(ptr %dest, ptr %mask, ptr %src) {
; CHECK-BE-NEXT: rsbs.w r3, lr, #0
; CHECK-BE-NEXT: mov.w r1, #0
; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31
-; CHECK-BE-NEXT: vmov q0[3], q0[1], r12, lr
+; CHECK-BE-NEXT: vmov q1[3], q1[1], r12, lr
; CHECK-BE-NEXT: csetm lr, lt
; CHECK-BE-NEXT: rsbs.w r3, r12, #0
; CHECK-BE-NEXT: @ implicit-def: $q2
@@ -161,33 +161,33 @@ define void @foo_sext_v2i64_v2i32(ptr %dest, ptr %mask, ptr %src) {
; CHECK-BE-NEXT: bpl .LBB5_2
; CHECK-BE-NEXT: @ %bb.1: @ %cond.load
; CHECK-BE-NEXT: ldr r3, [r2]
-; CHECK-BE-NEXT: vmov.32 q1[1], r3
-; CHECK-BE-NEXT: vrev64.32 q2, q1
+; CHECK-BE-NEXT: vmov.32 q0[1], r3
+; CHECK-BE-NEXT: vrev64.32 q2, q0
; CHECK-BE-NEXT: .LBB5_2: @ %else
-; CHECK-BE-NEXT: vrev64.32 q1, q0
+; CHECK-BE-NEXT: vrev64.32 q0, q1
; CHECK-BE-NEXT: lsls r1, r1, #31
; CHECK-BE-NEXT: beq .LBB5_4
; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1
; CHECK-BE-NEXT: ldr r1, [r2, #4]
-; CHECK-BE-NEXT: vrev64.32 q0, q2
-; CHECK-BE-NEXT: vmov.32 q0[3], r1
-; CHECK-BE-NEXT: vrev64.32 q2, q0
-; CHECK-BE-NEXT: .LBB5_4: @ %else2
-; CHECK-BE-NEXT: vrev64.32 q0, q2
+; CHECK-BE-NEXT: vrev64.32 q1, q2
+; CHECK-BE-NEXT: vmov.32 q1[3], r1
; CHECK-BE-NEXT: vrev64.32 q2, q1
-; CHECK-BE-NEXT: vmov r2, s3
+; CHECK-BE-NEXT: .LBB5_4: @ %else2
+; CHECK-BE-NEXT: vrev64.32 q1, q2
; CHECK-BE-NEXT: movs r1, #0
-; CHECK-BE-NEXT: vmov r3, s1
-; CHECK-BE-NEXT: vmov r4, s11
+; CHECK-BE-NEXT: vmov r2, s7
+; CHECK-BE-NEXT: vmov r3, s5
+; CHECK-BE-NEXT: vrev64.32 q1, q0
+; CHECK-BE-NEXT: vmov r4, s7
; CHECK-BE-NEXT: asr.w r12, r2, #31
; CHECK-BE-NEXT: asr.w lr, r3, #31
; CHECK-BE-NEXT: rsbs r5, r4, #0
-; CHECK-BE-NEXT: vmov q1[2], q1[0], lr, r12
+; CHECK-BE-NEXT: vmov q2[2], q2[0], lr, r12
; CHECK-BE-NEXT: sbcs.w r4, r1, r4, asr #31
-; CHECK-BE-NEXT: vmov q1[3], q1[1], r3, r2
-; CHECK-BE-NEXT: vmov r3, s9
+; CHECK-BE-NEXT: vmov q2[3], q2[1], r3, r2
+; CHECK-BE-NEXT: vmov r3, s5
; CHECK-BE-NEXT: csetm r2, lt
-; CHECK-BE-NEXT: vrev64.32 q0, q1
+; CHECK-BE-NEXT: vrev64.32 q0, q2
; CHECK-BE-NEXT: rsbs r5, r3, #0
; CHECK-BE-NEXT: sbcs.w r3, r1, r3, asr #31
; CHECK-BE-NEXT: bfi r1, r2, #0, #1
@@ -275,7 +275,7 @@ define void @foo_sext_v2i64_v2i32_unaligned(ptr %dest, ptr %mask, ptr %src) {
; CHECK-BE-NEXT: rsbs.w r3, lr, #0
; CHECK-BE-NEXT: mov.w r1, #0
; CHECK-BE-NEXT: sbcs.w r3, r1, lr, asr #31
-; CHECK-BE-NEXT: vmov q0[3], q0[1], r12, lr
+; CHECK-BE-NEXT: vmov q1[3], q1[1], r12, lr
; CHECK-BE-NEXT: csetm lr, lt
; CHECK-BE-NEXT: rsbs.w r3, r12, #0
; CHECK-BE-NEXT: @ implicit-def: $q2
@@ -287,33 +287,33 @@ define void @foo_sext_v2i64_v2i32_unaligned(ptr %dest, ptr %mask, ptr %src) {
; CHECK-BE-NEXT: bpl .LBB6_2
; CHECK-BE-NEXT: @ %bb.1: @ %cond.load
; CHECK-BE-NEXT: ldr r3, [r2]
-; CHECK-BE-NEXT: vmov.32 q1[1], r3
-; CHECK-BE-NEXT: vrev64.32 q2, q1
+; CHECK-BE-NEXT: vmov.32 q0[1], r3
+; CHECK-BE-NEXT: vrev64.32 q2, q0
; CHECK-BE-NEXT: .LBB6_2: @ %else
-; CHECK-BE-NEXT: vrev64.32 q1, q0
+; CHECK-BE-NEXT: vrev64.32 q0, q1
; CHECK-BE-NEXT: lsls r1, r1, #31
; CHECK-BE-NEXT: beq .LBB6_4
; CHECK-BE-NEXT: @ %bb.3: @ %cond.load1
; CHECK-BE-NEXT: ldr r1, [r2, #4]
-; CHECK-BE-NEXT: vrev64.32 q0, q2
-; CHECK-BE-NEXT: vmov.32 q0[3], r1
-; CHECK-BE-NEXT: vrev64.32 q2, q0
-; CHECK-BE-NEXT: .LBB6_4: @ %else2
-; CHECK-BE-NEXT: vrev64.32 q0, q2
+; CHECK-BE-NEXT: vrev64.32 q1, q2
+; CHECK-BE-NEXT: vmov.32 q1[3], r1
; CHECK-BE-NEXT: vrev64.32 q2, q1
-; CHECK-BE-NEXT: vmov r2, s3
+; CHECK-BE-NEXT: .LBB6_4: @ %else2
+; CHECK-BE-NEXT: vrev64.32 q1, q2
; CHECK-BE-NEXT: movs r1, #0
-; CHECK-BE-NEXT: vmov r3, s1
-; CHECK-BE-NEXT: vmov r4, s11
+; CHECK-BE-NEXT: vmov r2, s7
+; CHECK-BE-NEXT: vmov r3, s5
+; CHECK-BE-NEXT: vrev64.32 q1, q0
+; CHECK-BE-NEXT: vmov r4, s7
; CHECK-BE-NEXT: asr.w r12, r2, #31
; CHECK-BE-NEXT: asr.w lr, r3, #31
; CHECK-BE-NEXT: rsbs r5, r4, #0
-; CHECK-BE-NEXT: vmov q1[2], q1[0], lr, r12
+; CHECK-BE-NEXT: vmov q2[2], q2[0], lr, r12
; CHECK-BE-NEXT: sbcs.w r4, r1, r4, asr #31
-; CHECK-BE-NEXT: vmov q1[3], q1[1], r3, r2
-; CHECK-BE-NEXT: vmov r3, s9
+; CHECK-BE-NEXT: vmov q2[3], q2[1], r3, r2
+; CHECK-BE-NEXT: vmov r3, s5
; CHECK-BE-NEXT: csetm r2, lt
-; CHECK-BE-NEXT: vrev64.32 q0, q1
+; CHECK-BE-NEXT: vrev64.32 q0, q2
; CHECK-BE-NEXT: rsbs r5, r3, #0
; CHECK-BE-NEXT: sbcs.w r3, r1, r3, asr #31
; CHECK-BE-NEXT: bfi r1, r2, #0, #1
diff --git a/llvm/test/CodeGen/X86/bittest-big-integer.ll b/llvm/test/CodeGen/X86/bittest-big-integer.ll
index 0fd295115eca2..805d196544a53 100644
--- a/llvm/test/CodeGen/X86/bittest-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bittest-big-integer.ll
@@ -1440,73 +1440,74 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $224, %esp
-; X86-NEXT: movl 8(%ebp), %ebx
-; X86-NEXT: movl 12(%ebx), %esi
-; X86-NEXT: movl 28(%ebx), %eax
-; X86-NEXT: movl 60(%ebx), %ecx
+; X86-NEXT: movl 8(%ebp), %edi
+; X86-NEXT: movl 12(%edi), %ecx
+; X86-NEXT: movl 28(%edi), %eax
+; X86-NEXT: movl 60(%edi), %edx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %eax
+; X86-NEXT: movl 44(%edi), %edx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl 44(%ebx), %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %edx, %ecx
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl 20(%ebx), %edx
-; X86-NEXT: movl 52(%ebx), %eax
+; X86-NEXT: movl 20(%edi), %edx
+; X86-NEXT: movl 52(%edi), %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl 4(%ebx), %edi
-; X86-NEXT: movl 36(%ebx), %esi
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl 4(%edi), %ebx
+; X86-NEXT: movl 36(%edi), %esi
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %eax
; X86-NEXT: orl %edx, %eax
; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl 24(%ebx), %edx
-; X86-NEXT: movl 56(%ebx), %ecx
+; X86-NEXT: movl 24(%edi), %edx
+; X86-NEXT: movl 56(%edi), %ecx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl 8(%ebx), %ecx
-; X86-NEXT: movl 40(%ebx), %esi
+; X86-NEXT: movl 8(%edi), %ecx
+; X86-NEXT: movl 40(%edi), %esi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl 16(%ebx), %edx
-; X86-NEXT: movl 48(%ebx), %esi
+; X86-NEXT: movl 16(%edi), %edx
+; X86-NEXT: movl 48(%edi), %esi
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl (%ebx), %esi
-; X86-NEXT: movl 32(%ebx), %ebx
+; X86-NEXT: movl (%edi), %esi
+; X86-NEXT: movl 32(%edi), %edi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: orl %edx, %esi
; X86-NEXT: orl %ecx, %esi
; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB26_1
; X86-NEXT: # %bb.2: # %cond.false
-; X86-NEXT: testl %ebx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB26_3
; X86-NEXT: # %bb.4: # %cond.false
-; X86-NEXT: rep bsfl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: addl $32, %eax
+; X86-NEXT: rep bsfl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: addl $32, %ebx
; X86-NEXT: jmp .LBB26_5
; X86-NEXT: .LBB26_1:
; X86-NEXT: movl $512, %ecx # imm = 0x200
; X86-NEXT: jmp .LBB26_41
; X86-NEXT: .LBB26_3:
-; X86-NEXT: rep bsfl %ebx, %eax
+; X86-NEXT: rep bsfl %edi, %ebx
; X86-NEXT: .LBB26_5: # %cond.false
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: testl %ecx, %ecx
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: jne .LBB26_6
; X86-NEXT: # %bb.7: # %cond.false
; X86-NEXT: rep bsfl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
@@ -1520,7 +1521,7 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: jne .LBB26_10
; X86-NEXT: # %bb.9: # %cond.false
; X86-NEXT: addl $64, %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: .LBB26_10: # %cond.false
; X86-NEXT: testl %esi, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
@@ -1556,18 +1557,18 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: jne .LBB26_20
; X86-NEXT: # %bb.19: # %cond.false
; X86-NEXT: subl $-128, %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: .LBB26_20: # %cond.false
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: testl %edx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: jne .LBB26_21
; X86-NEXT: # %bb.22: # %cond.false
-; X86-NEXT: rep bsfl %edi, %ebx
-; X86-NEXT: addl $32, %ebx
+; X86-NEXT: rep bsfl %edi, %edi
+; X86-NEXT: addl $32, %edi
; X86-NEXT: jmp .LBB26_23
; X86-NEXT: .LBB26_21:
-; X86-NEXT: rep bsfl %edx, %ebx
+; X86-NEXT: rep bsfl %edx, %edi
; X86-NEXT: .LBB26_23: # %cond.false
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: testl %ecx, %ecx
@@ -1575,16 +1576,16 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: # %bb.25: # %cond.false
; X86-NEXT: rep bsfl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: addl $32, %ecx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: je .LBB26_27
; X86-NEXT: jmp .LBB26_28
; X86-NEXT: .LBB26_24:
; X86-NEXT: rep bsfl %ecx, %ecx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: jne .LBB26_28
; X86-NEXT: .LBB26_27: # %cond.false
; X86-NEXT: addl $64, %ecx
-; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl %ecx, %edi
; X86-NEXT: .LBB26_28: # %cond.false
; X86-NEXT: testl %esi, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
@@ -1614,40 +1615,39 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: .LBB26_36: # %cond.false
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: orl %edx, %esi
; X86-NEXT: jne .LBB26_38
; X86-NEXT: # %bb.37: # %cond.false
; X86-NEXT: subl $-128, %ecx
-; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl %ecx, %edi
; X86-NEXT: .LBB26_38: # %cond.false
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: addl $256, %eax # imm = 0x100
+; X86-NEXT: addl $256, %ebx # imm = 0x100
; X86-NEXT: orl %edx, %esi
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: jne .LBB26_40
; X86-NEXT: # %bb.39: # %cond.false
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: .LBB26_40: # %cond.false
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: .LBB26_41: # %cond.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %ecx, %esi
; X86-NEXT: shrl $3, %esi
; X86-NEXT: andl $60, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: leal {{[0-9]+}}(%esp), %edx
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: leal {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: subl %esi, %ebx
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
@@ -1682,50 +1682,44 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $31, %ecx
-; X86-NEXT: movl 56(%edx), %edi
-; X86-NEXT: movl 60(%edx), %esi
-; X86-NEXT: shldl %cl, %edi, %esi
-; X86-NEXT: notl %esi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 52(%edx), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %esi, %edi
-; X86-NEXT: notl %edi
-; X86-NEXT: andl %eax, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 40(%edx), %eax
-; X86-NEXT: movl 44(%edx), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %eax, %esi
+; X86-NEXT: movl 56(%ebx), %edx
+; X86-NEXT: movl 60(%ebx), %esi
+; X86-NEXT: shldl %cl, %edx, %esi
; X86-NEXT: notl %esi
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 36(%edx), %esi
-; X86-NEXT: shldl %cl, %esi, %eax
+; X86-NEXT: movl 52(%ebx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: notl %edx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 40(%ebx), %eax
+; X86-NEXT: movl 44(%ebx), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: notl %edx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 36(%ebx), %edx
+; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: notl %eax
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%edx), %eax
-; X86-NEXT: shldl %cl, %eax, %esi
-; X86-NEXT: notl %esi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 28(%edx), %esi
-; X86-NEXT: shldl %cl, %esi, %eax
+; X86-NEXT: movl 32(%ebx), %eax
+; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: notl %edx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 28(%ebx), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: notl %eax
-; X86-NEXT: andl %ebx, %eax
+; X86-NEXT: andl %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 24(%edx), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %eax, %esi
-; X86-NEXT: notl %esi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 4(%edx), %esi
+; X86-NEXT: movl 4(%ebx), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 8(%edx), %eax
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl 8(%ebx), %eax
; X86-NEXT: movl %eax, %edx
; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: notl %edx
@@ -1748,11 +1742,17 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: notl %esi
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: notl %eax
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 24(%ebx), %eax
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shldl %cl, %edx, %esi
+; X86-NEXT: notl %esi
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: notl %edx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: negl %eax
; X86-NEXT: movl 208(%esp,%eax), %edx
@@ -1776,10 +1776,12 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: shldl %cl, %edi, %ebx
; X86-NEXT: notl %ebx
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 8(%ebp), %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, 24(%ecx)
+; X86-NEXT: movl %ebx, 28(%ecx)
+; X86-NEXT: movl %esi, 24(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, 20(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, 16(%ecx)
@@ -1787,11 +1789,10 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: movl %esi, 12(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, 8(%ecx)
-; X86-NEXT: movl %edi, 4(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, 4(%ecx)
; X86-NEXT: movl %eax, (%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, 28(%ecx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, 32(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, 36(%ecx)
diff --git a/llvm/test/CodeGen/X86/freeze-multi-use.ll b/llvm/test/CodeGen/X86/freeze-multi-use.ll
index abff065b7bbd1..da9a6cb1d27df 100644
--- a/llvm/test/CodeGen/X86/freeze-multi-use.ll
+++ b/llvm/test/CodeGen/X86/freeze-multi-use.ll
@@ -94,10 +94,13 @@ define <4 x i32> @vec_across_call() nounwind {
define void @other_block(<2 x i32> %v) nounwind {
; CHECK-LABEL: other_block:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: # freeze: $ebx
+; CHECK-NEXT: movl %ebx, %edi
; CHECK-NEXT: callq use at PLT
+; CHECK-NEXT: movl %ebx, %edi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: popq %rax
+; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
entry:
%u = extractelement <2 x i32> %v, i32 5
diff --git a/llvm/test/CodeGen/X86/sttni.ll b/llvm/test/CodeGen/X86/sttni.ll
index 7cbfd46c39079..2e91e62d96ce4 100644
--- a/llvm/test/CodeGen/X86/sttni.ll
+++ b/llvm/test/CodeGen/X86/sttni.ll
@@ -315,11 +315,10 @@ define i32 @pcmpestri_reg_diff_i16(<8 x i16> %lhs, i32 %lhs_len, <8 x i16> %rhs,
; X86-NEXT: jmp .LBB8_3
; X86-NEXT: .LBB8_2: # %compare
; X86-NEXT: movdqa %xmm0, (%esp)
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: andl $14, %ecx
-; X86-NEXT: movzwl (%esp,%ecx), %eax
+; X86-NEXT: andl $7, %ecx
+; X86-NEXT: movzwl (%esp,%ecx,2), %eax
; X86-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT: subw 16(%esp,%ecx), %ax
+; X86-NEXT: subw 16(%esp,%ecx,2), %ax
; X86-NEXT: .LBB8_3: # %exit
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: movl %ebp, %esp
@@ -453,11 +452,10 @@ define i32 @pcmpestri_mem_diff_i16(ptr %lhs_ptr, i32 %lhs_len, ptr %rhs_ptr, i32
; X86-NEXT: jmp .LBB11_3
; X86-NEXT: .LBB11_2: # %compare
; X86-NEXT: movdqa %xmm1, (%esp)
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: andl $14, %ecx
-; X86-NEXT: movzwl (%esp,%ecx), %eax
+; X86-NEXT: andl $7, %ecx
+; X86-NEXT: movzwl (%esp,%ecx,2), %eax
; X86-NEXT: movdqa %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: subw 16(%esp,%ecx), %ax
+; X86-NEXT: subw 16(%esp,%ecx,2), %ax
; X86-NEXT: .LBB11_3: # %exit
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: leal -4(%ebp), %esp
@@ -774,11 +772,10 @@ define i32 @pcmpistri_reg_diff_i16(<8 x i16> %lhs, <8 x i16> %rhs) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $32, %esp
; X86-NEXT: movdqa %xmm0, (%esp)
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: andl $14, %ecx
-; X86-NEXT: movzwl (%esp,%ecx), %eax
+; X86-NEXT: andl $7, %ecx
+; X86-NEXT: movzwl (%esp,%ecx,2), %eax
; X86-NEXT: movdqa %xmm1, {{[0-9]+}}(%esp)
-; X86-NEXT: subw 16(%esp,%ecx), %ax
+; X86-NEXT: subw 16(%esp,%ecx,2), %ax
; X86-NEXT: movl %ebp, %esp
; X86-NEXT: popl %ebp
; X86-NEXT: movzwl %ax, %eax
@@ -892,11 +889,10 @@ define i32 @pcmpistri_mem_diff_i16(ptr %lhs_ptr, ptr %rhs_ptr) nounwind {
; X86-NEXT: jmp .LBB23_3
; X86-NEXT: .LBB23_2: # %compare
; X86-NEXT: movdqa %xmm1, (%esp)
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: andl $14, %ecx
-; X86-NEXT: movzwl (%esp,%ecx), %eax
+; X86-NEXT: andl $7, %ecx
+; X86-NEXT: movzwl (%esp,%ecx,2), %eax
; X86-NEXT: movdqa %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: subw 16(%esp,%ecx), %ax
+; X86-NEXT: subw 16(%esp,%ecx,2), %ax
; X86-NEXT: .LBB23_3: # %exit
; X86-NEXT: movzwl %ax, %eax
; X86-NEXT: movl %ebp, %esp
More information about the llvm-branch-commits
mailing list