[llvm] SplitKit: Fix rematerialization undoing subclass based split (PR #122110)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 8 05:49:18 PST 2025
https://github.com/arsenm created https://github.com/llvm/llvm-project/pull/122110
This fixes an allocation failure in the new test.
In cases where getLargestLegalSuperClass can inflate the register class,
rematerialization could effectively undo a split which was done to inflate
the register class, if the defining instruction can only write a subclass
and the use can read the superclass.
Some of the x86 tests changes look like improvements, but some are
likely regressions.
I'm not entirely sure this is the correct place to fix this. It also
seems more complicated than necessary, but the decision to change
the register class is far removed from the point where the decision
to split the virtual register is made. I'm also also not sure if this
should be considering the register classes of all the use indexes
in getUseSlots, rather than just checking if this use index instruction
reads the register.
>From 57f9368d44b4a3406b41a62daffe5448ad722633 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 6 Jan 2025 22:11:42 +0700
Subject: [PATCH] SplitKit: Fix rematerialization undoing subclass based split
This fixes an allocation failure in the new test.
In cases where getLargestLegalSuperClass can inflate the register class,
rematerialization could effectively undo a split which was done to inflate
the register class, if the defining instruction can only write a subclass
and the use can read the superclass.
Some of the x86 tests changes look like improvements, but some are
likely regressions.
I'm not entirely sure this is the correct place to fix this. It also
seems more complicated than necessary, but the decision to change
the register class is far removed from the point where the decision
to split the virtual register is made. I'm also also not sure if this
should be considering the register classes of all the use indexes
in getUseSlots, rather than just checking if this use index instruction
reads the register.
---
llvm/lib/CodeGen/SplitKit.cpp | 45 ++++-
llvm/lib/CodeGen/SplitKit.h | 6 +
...-do-not-undo-subclass-split-with-remat.mir | 155 ++++++++++++++++++
llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll | 3 +-
.../test/CodeGen/X86/fptosi-sat-vector-128.ll | 56 +++----
.../test/CodeGen/X86/fptoui-sat-vector-128.ll | 72 ++++----
.../X86/statepoint-invoke-ra-enter-at-end.mir | 5 +-
7 files changed, 271 insertions(+), 71 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index eb33b93c197d7c..d3ae29e2580999 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -588,6 +588,38 @@ SlotIndex SplitEditor::buildCopy(Register FromReg, Register ToReg,
return Def;
}
+bool SplitEditor::rematWillIncreaseRestriction(const MachineInstr *DefMI,
+ MachineBasicBlock &MBB,
+ SlotIndex UseIdx) const {
+ if (!DefMI)
+ return false;
+
+ const MachineInstr *UseMI = LIS.getInstructionFromIndex(UseIdx);
+ if (!UseMI)
+ return false;
+
+ Register Reg = Edit->getReg();
+ const TargetRegisterClass *RC = MRI.getRegClass(Reg);
+
+ // We want to find the register class that can be inflated to after the split
+ // occurs in recomputeRegClass
+ const TargetRegisterClass *SuperRC =
+ TRI.getLargestLegalSuperClass(RC, *MBB.getParent());
+
+ // We want to compute the static register class constraint for the instruction
+ // def. If it is a smaller subclass than getLargestLegalSuperClass at the use
+ // site, then rematerializing it will increase the constraints.
+ const TargetRegisterClass *DefConstrainRC =
+ DefMI->getRegClassConstraintEffectForVReg(Reg, SuperRC, &TII, &TRI,
+ /*ExploreBundle=*/true);
+
+ const TargetRegisterClass *UseConstrainRC =
+ UseMI->getRegClassConstraintEffectForVReg(Reg, SuperRC, &TII, &TRI,
+ /*ExploreBundle=*/true);
+
+ return UseConstrainRC->hasSubClass(DefConstrainRC);
+}
+
VNInfo *SplitEditor::defFromParent(unsigned RegIdx, const VNInfo *ParentVNI,
SlotIndex UseIdx, MachineBasicBlock &MBB,
MachineBasicBlock::iterator I) {
@@ -609,9 +641,16 @@ VNInfo *SplitEditor::defFromParent(unsigned RegIdx, const VNInfo *ParentVNI,
LiveRangeEdit::Remat RM(ParentVNI);
RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
- Def = Edit->rematerializeAt(MBB, I, Reg, RM, TRI, Late);
- ++NumRemats;
- DidRemat = true;
+ if (!rematWillIncreaseRestriction(RM.OrigMI, MBB, UseIdx)) {
+ Def = Edit->rematerializeAt(MBB, I, Reg, RM, TRI, Late);
+ ++NumRemats;
+ DidRemat = true;
+ } else {
+ LLVM_DEBUG(
+ dbgs() << "skipping rematerialize of " << printReg(Reg) << " at "
+ << UseIdx
+ << " since it will increase register class restrictions\n");
+ }
}
}
if (!DidRemat) {
diff --git a/llvm/lib/CodeGen/SplitKit.h b/llvm/lib/CodeGen/SplitKit.h
index cc277ecc0e882b..de255911268f24 100644
--- a/llvm/lib/CodeGen/SplitKit.h
+++ b/llvm/lib/CodeGen/SplitKit.h
@@ -379,6 +379,12 @@ class LLVM_LIBRARY_VISIBILITY SplitEditor {
/// predecessors in case of a phi definition.
void forceRecomputeVNI(const VNInfo &ParentVNI);
+ /// \return true if rematerializing \p DefMI at \p UseIdx will make the
+ /// register class requirements stricter at the use.
+ bool rematWillIncreaseRestriction(const MachineInstr *DefMI,
+ MachineBasicBlock &MBB,
+ SlotIndex UseIdx) const;
+
/// defFromParent - Define Reg from ParentVNI at UseIdx using either
/// rematerialization or a COPY from parent. Return the new value.
VNInfo *defFromParent(unsigned RegIdx, const VNInfo *ParentVNI,
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
new file mode 100644
index 00000000000000..9e2501a65c6e4d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-do-not-undo-subclass-split-with-remat.mir
@@ -0,0 +1,155 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-regalloc -start-before=greedy,2 -stop-before=virtregrewriter,2 -o - %s | FileCheck %s
+# FIXME: Assert if run to end
+
+# The V_MFMA_F32_32X32X1F32_vgprcd_e64 as written requires 66 VGPRs
+# to allocate, but the register budget (with a forced AGPR usage and
+# occupancy 4) permits 64 VGPRs and 64 AGPRs, so we need to force a
+# copy from VGPR to AGPR. The minimal copies required for this need to
+# copy %3 and %4 V_MOV_B32s into temporary AGPRs for use by the MFMA.
+
+# Previously we would attempt a register subclass based split, but
+# immediately rematerialize the V_MOV_B32 into the new temporary
+# register, defeating the point of introducing the split. The
+# allocation would fail since it's 2 registers short.
+
+---
+name: temp_vgpr_to_agpr_should_not_undo_split_with_remat
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+ scratchRSrcReg: '$sgpr96_sgpr97_sgpr98_sgpr99'
+ stackPtrOffsetReg: '$sgpr32'
+ argumentInfo:
+ privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+ kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+ workGroupIDX: { reg: '$sgpr6' }
+ privateSegmentWaveByteOffset: { reg: '$sgpr7' }
+ workItemIDX: { reg: '$vgpr0' }
+ occupancy: 4
+ sgprForEXECCopy: '$sgpr100_sgpr101'
+body: |
+ bb.0:
+ liveins: $vgpr0, $sgpr4_sgpr5
+
+ ; CHECK-LABEL: name: temp_vgpr_to_agpr_should_not_undo_split_with_remat
+ ; CHECK: liveins: $vgpr0, $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: S_NOP 0, implicit-def $agpr0
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 7, [[COPY]], implicit $exec
+ ; CHECK-NEXT: undef [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub28_sub29_sub30_sub31:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 112, 0, implicit $exec :: (load (s128), addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub24_sub25_sub26_sub27:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 96, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub20_sub21_sub22_sub23:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 80, 0, implicit $exec :: (load (s128), addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub16_sub17_sub18_sub19:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 64, 0, implicit $exec :: (load (s128), align 64, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 48, 0, implicit $exec :: (load (s128), addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 32, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 16, 0, implicit $exec :: (load (s128), addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, [[V_LSHLREV_B32_e32_]], 0, 0, implicit $exec :: (load (s128), align 128, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY [[V_MOV_B32_e32_1]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_32 = COPY [[V_MOV_B32_e32_]]
+ ; CHECK-NEXT: early-clobber %5:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_vgprcd_e64 [[COPY2]], [[COPY1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub2:vreg_1024_align2 = COPY %5.sub0
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub3:vreg_1024_align2 = COPY %5.sub1
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub4:vreg_1024_align2 = COPY %5.sub2
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub5:vreg_1024_align2 = COPY %5.sub3
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub6:vreg_1024_align2 = COPY %5.sub4
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub7:vreg_1024_align2 = COPY %5.sub5
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub8:vreg_1024_align2 = COPY %5.sub6
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub9:vreg_1024_align2 = COPY %5.sub7
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub10:vreg_1024_align2 = COPY %5.sub8
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub11:vreg_1024_align2 = COPY %5.sub9
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub12:vreg_1024_align2 = COPY %5.sub10
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub13:vreg_1024_align2 = COPY %5.sub11
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub14:vreg_1024_align2 = COPY %5.sub12
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub15:vreg_1024_align2 = COPY %5.sub13
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub16:vreg_1024_align2 = COPY %5.sub14
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub17:vreg_1024_align2 = COPY %5.sub15
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub18:vreg_1024_align2 = COPY %5.sub16
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub19:vreg_1024_align2 = COPY %5.sub17
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub20:vreg_1024_align2 = COPY %5.sub18
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub21:vreg_1024_align2 = COPY %5.sub19
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub22:vreg_1024_align2 = COPY %5.sub20
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub23:vreg_1024_align2 = COPY %5.sub21
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub24:vreg_1024_align2 = COPY %5.sub22
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub25:vreg_1024_align2 = COPY %5.sub23
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub26:vreg_1024_align2 = COPY %5.sub24
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub27:vreg_1024_align2 = COPY %5.sub25
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub28:vreg_1024_align2 = COPY %5.sub26
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub29:vreg_1024_align2 = COPY %5.sub27
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub30:vreg_1024_align2 = COPY %5.sub28
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]].sub31:vreg_1024_align2 = COPY %5.sub29
+ ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 [[COPY2]], [[COPY1]], [[GLOBAL_LOAD_DWORDX4_SADDR]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub24_sub25_sub26_sub27, renamable $sgpr0_sgpr1, 96, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub28_sub29_sub30_sub31, renamable $sgpr0_sgpr1, 112, 0, implicit $exec :: (store (s128), addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub16_sub17_sub18_sub19, renamable $sgpr0_sgpr1, 64, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub20_sub21_sub22_sub23, renamable $sgpr0_sgpr1, 80, 0, implicit $exec :: (store (s128), addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub8_sub9_sub10_sub11, renamable $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub12_sub13_sub14_sub15, renamable $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub0_sub1_sub2_sub3, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR [[V_MOV_B32_e32_2]], [[GLOBAL_LOAD_DWORDX4_SADDR]].sub4_sub5_sub6_sub7, killed renamable $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ S_NOP 0, implicit-def $agpr0
+ %0:vgpr_32 = COPY $vgpr0
+ renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr4_sgpr5, 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %1:vgpr_32 = V_LSHLREV_B32_e32 7, %0, implicit $exec
+ undef %2.sub28_sub29_sub30_sub31:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 112, 0, implicit $exec :: (load (s128), addrspace 1)
+ %2.sub24_sub25_sub26_sub27:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 96, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+ %2.sub20_sub21_sub22_sub23:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 80, 0, implicit $exec :: (load (s128), addrspace 1)
+ %2.sub16_sub17_sub18_sub19:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 64, 0, implicit $exec :: (load (s128), align 64, addrspace 1)
+ %2.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 48, 0, implicit $exec :: (load (s128), addrspace 1)
+ %2.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 32, 0, implicit $exec :: (load (s128), align 32, addrspace 1)
+ %2.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 16, 0, implicit $exec :: (load (s128), addrspace 1)
+ %2.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4_SADDR renamable $sgpr0_sgpr1, %1, 0, 0, implicit $exec :: (load (s128), align 128, addrspace 1)
+ %3:vgpr_32 = V_MOV_B32_e32 1065353216, implicit $exec
+ %4:vgpr_32 = V_MOV_B32_e32 1073741824, implicit $exec
+ %2:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec
+ early-clobber %5:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec, implicit $mode, implicit $exec
+ %2.sub2:vreg_1024_align2 = COPY %5.sub0
+ %2.sub3:vreg_1024_align2 = COPY %5.sub1
+ %2.sub4:vreg_1024_align2 = COPY %5.sub2
+ %2.sub5:vreg_1024_align2 = COPY %5.sub3
+ %2.sub6:vreg_1024_align2 = COPY %5.sub4
+ %2.sub7:vreg_1024_align2 = COPY %5.sub5
+ %2.sub8:vreg_1024_align2 = COPY %5.sub6
+ %2.sub9:vreg_1024_align2 = COPY %5.sub7
+ %2.sub10:vreg_1024_align2 = COPY %5.sub8
+ %2.sub11:vreg_1024_align2 = COPY %5.sub9
+ %2.sub12:vreg_1024_align2 = COPY %5.sub10
+ %2.sub13:vreg_1024_align2 = COPY %5.sub11
+ %2.sub14:vreg_1024_align2 = COPY %5.sub12
+ %2.sub15:vreg_1024_align2 = COPY %5.sub13
+ %2.sub16:vreg_1024_align2 = COPY %5.sub14
+ %2.sub17:vreg_1024_align2 = COPY %5.sub15
+ %2.sub18:vreg_1024_align2 = COPY %5.sub16
+ %2.sub19:vreg_1024_align2 = COPY %5.sub17
+ %2.sub20:vreg_1024_align2 = COPY %5.sub18
+ %2.sub21:vreg_1024_align2 = COPY %5.sub19
+ %2.sub22:vreg_1024_align2 = COPY %5.sub20
+ %2.sub23:vreg_1024_align2 = COPY %5.sub21
+ %2.sub24:vreg_1024_align2 = COPY %5.sub22
+ %2.sub25:vreg_1024_align2 = COPY %5.sub23
+ %2.sub26:vreg_1024_align2 = COPY %5.sub24
+ %2.sub27:vreg_1024_align2 = COPY %5.sub25
+ %2.sub28:vreg_1024_align2 = COPY %5.sub26
+ %2.sub29:vreg_1024_align2 = COPY %5.sub27
+ %2.sub30:vreg_1024_align2 = COPY %5.sub28
+ %2.sub31:vreg_1024_align2 = COPY %5.sub29
+ %2:vreg_1024_align2 = V_MFMA_F32_32X32X1F32_mac_vgprcd_e64 %3, %4, %2, 0, 0, 0, implicit $mode, implicit $exec
+ %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub24_sub25_sub26_sub27, renamable $sgpr0_sgpr1, 96, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub28_sub29_sub30_sub31, renamable $sgpr0_sgpr1, 112, 0, implicit $exec :: (store (s128), addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub16_sub17_sub18_sub19, renamable $sgpr0_sgpr1, 64, 0, implicit $exec :: (store (s128), align 64, addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub20_sub21_sub22_sub23, renamable $sgpr0_sgpr1, 80, 0, implicit $exec :: (store (s128), addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub8_sub9_sub10_sub11, renamable $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub12_sub13_sub14_sub15, renamable $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub0_sub1_sub2_sub3, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+ GLOBAL_STORE_DWORDX4_SADDR %6, %2.sub4_sub5_sub6_sub7, killed renamable $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+ S_ENDPGM 0
+
+...
diff --git a/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll
index 527995bc2139ec..95a7a10d50f592 100644
--- a/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll
+++ b/llvm/test/CodeGen/X86/eq-or-eq-range-of-2.ll
@@ -106,8 +106,7 @@ define <4 x i32> @eq_or_eq_ult_2_fail_multiuse(<4 x i32> %x) {
; AVX512: # %bb.0:
; AVX512-NEXT: subq $24, %rsp
; AVX512-NEXT: .cfi_def_cfa_offset 32
-; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
; AVX512-NEXT: callq use.v4.i32 at PLT
; AVX512-NEXT: vmovdqa (%rsp), %xmm0 # 16-byte Reload
diff --git a/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll b/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
index 536a1ae3b918de..d2faed51bc502a 100644
--- a/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
+++ b/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
@@ -241,21 +241,21 @@ define <4 x i128> @test_signed_v4i128_v4f32(<4 x float> %f) nounwind {
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: callq __fixsfti at PLT
; CHECK-NEXT: movq %rdx, %r15
-; CHECK-NEXT: xorl %r14d, %r14d
+; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r14, %rax
-; CHECK-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; CHECK-NEXT: cmovbq %rcx, %r15
+; CHECK-NEXT: cmovbq %rdx, %rax
+; CHECK-NEXT: movabsq $-9223372036854775808, %r14 # imm = 0x8000000000000000
+; CHECK-NEXT: cmovbq %r14, %r15
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: movabsq $9223372036854775807, %rbp # imm = 0x7FFFFFFFFFFFFFFF
; CHECK-NEXT: cmovaq %rbp, %r15
; CHECK-NEXT: movq $-1, %rcx
; CHECK-NEXT: cmovaq %rcx, %rax
; CHECK-NEXT: ucomiss %xmm0, %xmm0
-; CHECK-NEXT: cmovpq %r14, %rax
+; CHECK-NEXT: cmovpq %rdx, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovpq %r14, %r15
+; CHECK-NEXT: cmovpq %rdx, %r15
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -264,16 +264,16 @@ define <4 x i128> @test_signed_v4i128_v4f32(<4 x float> %f) nounwind {
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r14, %r12
-; CHECK-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; CHECK-NEXT: cmovbq %rax, %r13
+; CHECK-NEXT: movl $0, %eax
+; CHECK-NEXT: cmovbq %rax, %r12
+; CHECK-NEXT: cmovbq %r14, %r13
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovaq %rbp, %r13
-; CHECK-NEXT: movq $-1, %rax
-; CHECK-NEXT: cmovaq %rax, %r12
+; CHECK-NEXT: movq $-1, %rcx
+; CHECK-NEXT: cmovaq %rcx, %r12
; CHECK-NEXT: ucomiss %xmm0, %xmm0
-; CHECK-NEXT: cmovpq %r14, %r12
-; CHECK-NEXT: cmovpq %r14, %r13
+; CHECK-NEXT: cmovpq %rax, %r12
+; CHECK-NEXT: cmovpq %rax, %r13
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1187,14 +1187,14 @@ define <8 x i128> @test_signed_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovbq %r12, %rax
-; CHECK-NEXT: movabsq $-9223372036854775808, %rbp # imm = 0x8000000000000000
-; CHECK-NEXT: cmovbq %rbp, %rdx
+; CHECK-NEXT: movabsq $-9223372036854775808, %r13 # imm = 0x8000000000000000
+; CHECK-NEXT: cmovbq %r13, %rdx
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: movabsq $9223372036854775807, %r15 # imm = 0x7FFFFFFFFFFFFFFF
; CHECK-NEXT: cmovaq %r15, %rdx
; CHECK-NEXT: movq $-1, %rcx
; CHECK-NEXT: cmovaq %rcx, %rax
-; CHECK-NEXT: movq $-1, %r13
+; CHECK-NEXT: movq $-1, %rbp
; CHECK-NEXT: ucomiss %xmm0, %xmm0
; CHECK-NEXT: cmovpq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -1209,10 +1209,10 @@ define <8 x i128> @test_signed_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovbq %r12, %rax
-; CHECK-NEXT: cmovbq %rbp, %rdx
+; CHECK-NEXT: cmovbq %r13, %rdx
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovaq %r15, %rdx
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: cmovaq %rbp, %rax
; CHECK-NEXT: movq $-1, %r14
; CHECK-NEXT: ucomiss %xmm0, %xmm0
; CHECK-NEXT: cmovpq %r12, %rax
@@ -1228,7 +1228,7 @@ define <8 x i128> @test_signed_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovbq %r12, %rax
-; CHECK-NEXT: cmovbq %rbp, %rdx
+; CHECK-NEXT: cmovbq %r13, %rdx
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovaq %r15, %rdx
; CHECK-NEXT: cmovaq %r14, %rax
@@ -1247,8 +1247,7 @@ define <8 x i128> @test_signed_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovbq %r12, %rax
-; CHECK-NEXT: cmovbq %rbp, %rdx
-; CHECK-NEXT: movq %rbp, %r13
+; CHECK-NEXT: cmovbq %r13, %rdx
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovaq %r15, %rdx
; CHECK-NEXT: cmovaq %r14, %rax
@@ -1287,16 +1286,17 @@ define <8 x i128> @test_signed_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %r14
-; CHECK-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; CHECK-NEXT: cmovbq %rax, %r15
+; CHECK-NEXT: movl $0, %eax
+; CHECK-NEXT: cmovbq %rax, %r14
+; CHECK-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; CHECK-NEXT: cmovbq %rcx, %r15
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; CHECK-NEXT: cmovaq %r13, %r15
-; CHECK-NEXT: movq $-1, %rax
-; CHECK-NEXT: cmovaq %rax, %r14
+; CHECK-NEXT: movq $-1, %rcx
+; CHECK-NEXT: cmovaq %rcx, %r14
; CHECK-NEXT: ucomiss %xmm0, %xmm0
-; CHECK-NEXT: cmovpq %r12, %r14
-; CHECK-NEXT: cmovpq %r12, %r15
+; CHECK-NEXT: cmovpq %rax, %r14
+; CHECK-NEXT: cmovpq %rax, %r15
; CHECK-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: callq __extendhfsf2 at PLT
diff --git a/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll b/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
index 4305886168abed..4d28ef7884954f 100644
--- a/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
+++ b/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
@@ -263,17 +263,17 @@ define <4 x i128> @test_unsigned_v4i128_v4f32(<4 x float> %f) nounwind {
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; CHECK-NEXT: callq __fixunssfti at PLT
; CHECK-NEXT: movq %rdx, %r15
-; CHECK-NEXT: xorl %r14d, %r14d
+; CHECK-NEXT: xorl %ebp, %ebp
; CHECK-NEXT: xorps %xmm0, %xmm0
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
; CHECK-NEXT: ucomiss %xmm0, %xmm1
-; CHECK-NEXT: cmovbq %r14, %r15
-; CHECK-NEXT: cmovbq %r14, %rax
+; CHECK-NEXT: cmovbq %rbp, %r15
+; CHECK-NEXT: cmovbq %rbp, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: movq $-1, %rbp
-; CHECK-NEXT: cmovaq %rbp, %rax
+; CHECK-NEXT: movq $-1, %r14
+; CHECK-NEXT: cmovaq %r14, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %rbp, %r15
+; CHECK-NEXT: cmovaq %r14, %r15
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -282,11 +282,11 @@ define <4 x i128> @test_unsigned_v4i128_v4f32(<4 x float> %f) nounwind {
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r14, %r13
-; CHECK-NEXT: cmovbq %r14, %r12
+; CHECK-NEXT: cmovbq %rbp, %r13
+; CHECK-NEXT: cmovbq %rbp, %r12
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %rbp, %r12
-; CHECK-NEXT: cmovaq %rbp, %r13
+; CHECK-NEXT: cmovaq %r14, %r12
+; CHECK-NEXT: cmovaq %r14, %r13
; CHECK-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload
; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; CHECK-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -1149,18 +1149,18 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: callq __extendhfsf2 at PLT
; CHECK-NEXT: movd %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Folded Spill
; CHECK-NEXT: callq __fixunssfti at PLT
-; CHECK-NEXT: xorl %r12d, %r12d
+; CHECK-NEXT: xorl %r13d, %r13d
; CHECK-NEXT: pxor %xmm0, %xmm0
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 4-byte Reload
; CHECK-NEXT: # xmm1 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss %xmm0, %xmm1
-; CHECK-NEXT: cmovbq %r12, %rdx
-; CHECK-NEXT: cmovbq %r12, %rax
+; CHECK-NEXT: cmovbq %r13, %rdx
+; CHECK-NEXT: cmovbq %r13, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; CHECK-NEXT: movq $-1, %r13
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: movq $-1, %r12
+; CHECK-NEXT: cmovaq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %r13, %rdx
+; CHECK-NEXT: cmovaq %r12, %rdx
; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
@@ -1170,12 +1170,12 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %rdx
-; CHECK-NEXT: cmovbq %r12, %rax
+; CHECK-NEXT: cmovbq %r13, %rdx
+; CHECK-NEXT: cmovbq %r13, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: cmovaq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %r13, %rdx
+; CHECK-NEXT: cmovaq %r12, %rdx
; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: psrlq $48, %xmm0
@@ -1185,12 +1185,12 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %rdx
-; CHECK-NEXT: cmovbq %r12, %rax
+; CHECK-NEXT: cmovbq %r13, %rdx
+; CHECK-NEXT: cmovbq %r13, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: cmovaq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %r13, %rdx
+; CHECK-NEXT: cmovaq %r12, %rdx
; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: movhlps {{.*#+}} xmm0 = xmm0[1,1]
@@ -1200,12 +1200,12 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %rdx
-; CHECK-NEXT: cmovbq %r12, %rax
+; CHECK-NEXT: cmovbq %r13, %rdx
+; CHECK-NEXT: cmovbq %r13, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: cmovaq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %r13, %rdx
+; CHECK-NEXT: cmovaq %r12, %rdx
; CHECK-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
@@ -1216,12 +1216,12 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %rbp
-; CHECK-NEXT: cmovbq %r12, %rax
+; CHECK-NEXT: cmovbq %r13, %rbp
+; CHECK-NEXT: cmovbq %r13, %rax
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %r13, %rax
+; CHECK-NEXT: cmovaq %r12, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: cmovaq %r13, %rbp
+; CHECK-NEXT: cmovaq %r12, %rbp
; CHECK-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
; CHECK-NEXT: callq __extendhfsf2 at PLT
@@ -1232,11 +1232,11 @@ define <8 x i128> @test_unsigned_v8i128_v8f16(<8 x half> %f) nounwind {
; CHECK-NEXT: movss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Reload
; CHECK-NEXT: # xmm0 = mem[0],zero,zero,zero
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovbq %r12, %r15
-; CHECK-NEXT: cmovbq %r12, %r14
+; CHECK-NEXT: cmovbq %r13, %r15
+; CHECK-NEXT: cmovbq %r13, %r14
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; CHECK-NEXT: cmovaq %r13, %r14
-; CHECK-NEXT: cmovaq %r13, %r15
+; CHECK-NEXT: cmovaq %r12, %r14
+; CHECK-NEXT: cmovaq %r12, %r15
; CHECK-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; CHECK-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: callq __extendhfsf2 at PLT
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
index 5f05270729fdec..fc0efd275ae259 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
@@ -343,8 +343,9 @@ body: |
; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = NOT64r [[MOV64rm4]]
; CHECK-NEXT: CMP64rr [[MOV64rm4]], [[COPY7]], implicit-def $eflags
; CHECK-NEXT: undef [[MOV32ri1:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32ri 0
- ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = CMOV64rr [[MOV64rm4]], [[MOV32ri1]], 4, implicit killed $eflags
- ; CHECK-NEXT: INLINEASM &"lock btsq $0,($1)", 1 /* sideeffect attdialect */, 4784137 /* reguse:GR64 */, [[MOV32ri1]], 4784137 /* reguse:GR64 */, undef %56:gr64, 12 /* clobber */, implicit-def dead early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def dead early-clobber $eflags
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gr64 = COPY [[MOV32ri1]]
+ ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = CMOV64rr [[MOV64rm4]], [[COPY8]], 4, implicit killed $eflags
+ ; CHECK-NEXT: INLINEASM &"lock btsq $0,($1)", 1 /* sideeffect attdialect */, 4784137 /* reguse:GR64 */, [[COPY8]], 4784137 /* reguse:GR64 */, undef %56:gr64, 12 /* clobber */, implicit-def dead early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def dead early-clobber $eflags
; CHECK-NEXT: LCMPXCHG32 undef %67:gr64, 1, $noreg, 0, $noreg, [[COPY6]], implicit-def dead $eax, implicit-def dead $eflags, implicit undef $eax :: (load store acquire monotonic (s32) on `ptr addrspace(1) undef`, addrspace 1)
; CHECK-NEXT: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp
; CHECK-NEXT: $rdi = COPY [[COPY5]]
More information about the llvm-commits
mailing list